Centre foveation on the player's gaze on headsets with eye tracking

With [vr] eye_tracked_foveation (on by default on the Steam Frame, off
elsewhere) the runtime asks for XR_EXT_eye_gaze_interaction. When the
system reports an eye tracker, OpenXRInput binds the gaze pose and
locates it for each packet's display time, in the space the eye views
are located in; vr/eye_gaze.h turns it into tangents of each eye's own
view, which AuroraStereoFrame now carries (appended, after the existing
prefix).

Aurora centres the eye's fragment density map on the gaze snapped to a
cell of two map texels (about 3 degrees). Each eye keeps up to 32 maps,
one per cell looked at, so a glance back reuses its map; a new map is
bound once its upload completes, and until then the eye keeps the map
it had. Without a tracked gaze (a blink, no tracker, the setting off)
foveation centres on the forward direction exactly as before: the
forward maps are byte-identical.

Also logs every extension the OpenXR runtime offers at startup, so the
first Steam Frame session shows what SteamVR's Android runtime has.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019HBRGKTE1GnN2ah8gcZKr3

android/app/src/main/java/org/wiicompiled/quest/launcher/SettingsPage.kt

@@ -259,6 +259,14 @@ class SettingsPage(
                 read = { stringIndex(it, "vr", "foveation", FOVEATION_LEVELS, FOVEATION_DEFAULT) },
                 write = { c, index -> c.setString("vr", "foveation", FOVEATION_LEVELS[index]) },
             )
+            // runtime_config.h's kVrEyeTrackedFoveationDefault: on for the Steam Frame, whose eyes are
+            // tracked; off elsewhere, since Horizon OS asks for an eye tracking permission.
+            toggle(
+                R.string.vr_eye_tracked_foveation, R.string.vr_eye_tracked_foveation_helper,
+                read = { it.bool("vr", "eye_tracked_foveation") ?: BuildConfig.STEAM_FRAME },
+                write = { c, value -> c.setBool("vr", "eye_tracked_foveation", value) },
+                enabledIf = { stringIndex(it, "vr", "foveation", FOVEATION_LEVELS, FOVEATION_DEFAULT) != 0 },
+            )
             choice(
                 R.string.vr_interpolation, R.string.vr_interpolation_helper,
                 listOf(activity.getString(R.string.vr_interpolation_off), activity.getString(R.string.vr_interpolation_auto), "72 FPS", "90 FPS", "120 FPS"),

android/app/src/main/res/values/strings.xml

@@ -564,6 +564,8 @@
     <string name="vr_refresh_rate_headset">Headset\'s own</string>
     <string name="vr_foveation">Foveated rendering</string>
     <string name="vr_foveation_helper">Shades the edges of the race view in coarser blocks, where the lenses blur the picture anyway, to free GPU time for a higher render scale or a steadier frame rate. Higher levels start closer to the centre; High also coarsens the corners of the HUD. Menus are never foveated.</string>
+    <string name="vr_eye_tracked_foveation">Foveation follows the eyes</string>
+    <string name="vr_eye_tracked_foveation_helper">With eye tracking (the Steam Frame), the sharp centre of the foveated race view moves to where you look instead of staying straight ahead.</string>
     <string name="vr_foveation_off">Off</string>
     <string name="vr_foveation_low">Low</string>
     <string name="vr_foveation_medium">Medium</string>

aurora-main/include/aurora/aurora.h

@@ -209,6 +209,11 @@ typedef struct {
   // everywhere else, for the host's compositor to show its own background
   // (the room, on a headset with passthrough) around it.
   bool window;
+  // Eye-tracked foveation (aurora_set_stereo_foveation), while gazeValid: where the player looks, in
+  // tangents of each eye's view (x right, y up, as in the projection's frustum). The immersive eyes'
+  // full-density region then centres there instead of on each eye's forward direction.
+  float gaze[AURORA_STEREO_EYE_COUNT][2];
+  bool gazeValid;
 } AuroraStereoFrame;
 
 /**

aurora-main/lib/aurora.cpp

@@ -607,6 +607,17 @@ std::mutex g_surfaceMutex;
 std::atomic<bool> g_surfaceReconfigurePending{false};
 std::atomic<bool> g_surfaceRecreatePending{false};
 
+// One fragment density map of an eye (see StereoEyeTarget): centred on the eye's forward direction,
+// or on a gaze cell (gfx/foveation.hpp) with eye-tracked foveation.
+struct EyeDensityMap {
+  gfx::foveation::GazeCell cell;
+  bool forward = true;
+  uint64_t map = 0;
+  uint64_t lastUse = 0;
+};
+// The gaze cells' maps an eye keeps: a few glances' worth, each 2 bytes per 32x32 pixels.
+constexpr size_t kEyeDensityMapCacheSize = 32;
+
 struct StereoEyeTarget {
   webgpu::TextureWithSampler color;
   webgpu::TextureWithSampler resolvedColor;
@@ -620,24 +631,31 @@ struct StereoEyeTarget {
   // the target when ensure_stereo_eye_target replaces the textures.
   wgpu::BindGroup copyBindGroup;
   // Foveated rendering: a second view of `color` for the immersive eye passes,
-  // which the patched Dawn binds to this eye's fragment density map
-  // (webgpu/fdm.hpp), and what that map was built for.
+  // which the patched Dawn binds to one of this eye's fragment density maps
+  // (webgpu/fdm.hpp). The maps share what densityBase records (the eye's size,
+  // level and field of view); with eye tracking there is one per gaze cell
+  // looked at, the least recently used dropped beyond kEyeDensityMapCacheSize.
   wgpu::TextureView foveatedView;
-  uint64_t densityMap = 0;
-  std::array<int32_t, 7> densityKey{};
+  std::array<int32_t, 7> densityBase{};
+  std::vector<EyeDensityMap> densityMaps;
+  uint64_t boundDensityMap = 0;
+  uint64_t densityUses = 0;
 
   const webgpu::TextureWithSampler& output() const noexcept { return resolvedColor.texture ? resolvedColor : color; }
 };
 std::array<StereoEyeTarget, AURORA_STEREO_EYE_COUNT> g_stereoEyeTargets;
 stereo::MirrorState g_stereoMirrorState;
 
-// The map's binding holds the foveated view, and with it the eye texture, until it is released.
+// A map's binding holds the foveated view, and with it the eye texture, until it is released.
 void release_eye_density_map(StereoEyeTarget& target) noexcept {
-  if (target.densityMap != 0) {
-    webgpu::fdm::release_map(target.densityMap);
-    target.densityMap = 0;
+  for (const EyeDensityMap& entry : target.densityMaps) {
+    if (entry.map != 0) {
+      webgpu::fdm::release_map(entry.map);
+    }
   }
-  target.densityKey = {};
+  target.densityMaps.clear();
+  target.boundDensityMap = 0;
+  target.densityBase = {};
 }
 
 // The eye targets outlive a frame, so the mirror samples them through a bind
@@ -689,9 +707,12 @@ void ensure_stereo_eye_target(uint32_t eyeIndex, uint32_t width, uint32_t height
 }
 
 // The view an immersive eye's passes render through while foveated, or none. The eye's fragment
-// density map is rebuilt whenever its size, field of view or level changes (a map is immutable), and
-// is used once its upload has completed.
-wgpu::TextureView foveated_eye_view(uint32_t eyeIndex, const AuroraStereoEye& input) {
+// density maps are rebuilt whenever its size, field of view or level changes (a map is immutable).
+// `gaze`, the tangents the player looks at when eye tracking provides them, picks the map centred on
+// the gaze cell it falls in, built on first use; without it the map is centred on the eye's forward
+// direction. A map is bound once its upload has completed, and until then the eye keeps the map it
+// had, so a glance never leaves the eye unfoveated.
+wgpu::TextureView foveated_eye_view(uint32_t eyeIndex, const AuroraStereoEye& input, const float* gaze) {
   auto& target = g_stereoEyeTargets[eyeIndex];
   const auto level = static_cast<gfx::foveation::Level>(gfx::get_stereo_foveation());
   if (level == gfx::foveation::Level::Off || target.samples > 1 || !webgpu::fdm::available()) {
@@ -700,42 +721,84 @@ wgpu::TextureView foveated_eye_view(uint32_t eyeIndex, const AuroraStereoEye& in
   const auto fov = gfx::foveation::fov_from_projection(input.projection);
   // Hundredths of a tangent: finer than a map texel, coarse enough to ignore pose noise.
   const auto hundredths = [](float value) { return static_cast<int32_t>(std::lround(value * 100.0f)); };
-  const std::array<int32_t, 7> key{static_cast<int32_t>(target.color.size.width),
-                                   static_cast<int32_t>(target.color.size.height),
-                                   static_cast<int32_t>(level),
-                                   hundredths(fov.tanLeft),
-                                   hundredths(fov.tanRight),
-                                   hundredths(fov.tanDown),
-                                   hundredths(fov.tanUp)};
-  if (key != target.densityKey) {
+  const std::array<int32_t, 7> base{static_cast<int32_t>(target.color.size.width),
+                                    static_cast<int32_t>(target.color.size.height),
+                                    static_cast<int32_t>(level),
+                                    hundredths(fov.tanLeft),
+                                    hundredths(fov.tanRight),
+                                    hundredths(fov.tanDown),
+                                    hundredths(fov.tanUp)};
+  if (base != target.densityBase) {
     release_eye_density_map(target);
-    target.densityKey = key;
-    if (!target.foveatedView) {
-      const wgpu::TextureViewDescriptor descriptor{
-          .label = eyeIndex == 0 ? "Foveated left eye" : "Foveated right eye",
-          .usage = wgpu::TextureUsage::RenderAttachment,
-      };
-      target.foveatedView = target.color.texture.CreateView(&descriptor);
+    target.densityBase = base;
+  }
+  if (!target.foveatedView) {
+    const wgpu::TextureViewDescriptor descriptor{
+        .label = eyeIndex == 0 ? "Foveated left eye" : "Foveated right eye",
+        .usage = wgpu::TextureUsage::RenderAttachment,
+    };
+    target.foveatedView = target.color.texture.CreateView(&descriptor);
+  }
+
+  const uint32_t width = target.color.size.width;
+  const uint32_t height = target.color.size.height;
+  const uint32_t texel = webgpu::fdm::texel_size();
+  const bool forward = gaze == nullptr;
+  const gfx::foveation::GazeCell cell =
+      forward ? gfx::foveation::GazeCell{}
+              : gfx::foveation::gaze_cell(width, height, texel, fov, {.tanX = gaze[0], .tanY = gaze[1]});
+  auto& maps = target.densityMaps;
+  auto entry = std::find_if(maps.begin(), maps.end(), [&](const EyeDensityMap& candidate) {
+    return candidate.forward == forward && (forward || candidate.cell == cell);
+  });
+  if (entry == maps.end()) {
+    if (maps.size() >= kEyeDensityMapCacheSize) {
+      // The least recently used map, never the one the eye renders with.
+      auto oldest = maps.end();
+      for (auto it = maps.begin(); it != maps.end(); ++it) {
+        if (it->map != target.boundDensityMap && (oldest == maps.end() || it->lastUse < oldest->lastUse)) {
+          oldest = it;
+        }
+      }
+      if (oldest != maps.end()) {
+        if (oldest->map != 0) {
+          webgpu::fdm::release_map(oldest->map);
+        }
+        maps.erase(oldest);
+      }
     }
+    const bool firstOfKind =
+        std::none_of(maps.begin(), maps.end(), [&](const EyeDensityMap& other) { return other.forward == forward; });
     gfx::foveation::Map map;
-    gfx::foveation::build(target.color.size.width, target.color.size.height, webgpu::fdm::texel_size(), fov, level,
-                          map);
-    target.densityMap = webgpu::fdm::create_map(map.width, map.height, map.rg8.data());
-    if (target.densityMap != 0 && !webgpu::fdm::bind(target.foveatedView, target.densityMap)) {
-      webgpu::fdm::release_map(target.densityMap);
-      target.densityMap = 0;
-    }
+    gfx::foveation::build(width, height, texel, fov, level, map,
+                          forward ? gfx::foveation::Gaze{}
+                                  : gfx::foveation::cell_gaze(width, height, texel, fov, cell));
+    EyeDensityMap created{.cell = cell, .forward = forward};
+    created.map = webgpu::fdm::create_map(map.width, map.height, map.rg8.data());
     static constexpr std::array<const char*, gfx::foveation::kLevelCount> kLevelNames{"off", "low", "medium", "high"};
-    if (target.densityMap != 0) {
-      Log.info("{} eye foveation {}: {}x{} density map, {} pixels per texel", eyeIndex == 0 ? "Left" : "Right",
-               kLevelNames[static_cast<uint32_t>(level)], map.width, map.height, webgpu::fdm::texel_size());
-    } else {
+    if (created.map == 0) {
       Log.warn("{} eye foveation {}: the {}x{} density map could not be created", eyeIndex == 0 ? "Left" : "Right",
                kLevelNames[static_cast<uint32_t>(level)], map.width, map.height);
+    } else if (firstOfKind) {
+      // Gaze maps come and go with the player's glances; the first says the eye follows the gaze.
+      Log.info("{} eye foveation {}{}: {}x{} density map, {} pixels per texel", eyeIndex == 0 ? "Left" : "Right",
+               kLevelNames[static_cast<uint32_t>(level)], forward ? "" : " following the gaze", map.width, map.height,
+               texel);
+    }
+    maps.push_back(created);
+    entry = std::prev(maps.end());
+  }
+  entry->lastUse = ++target.densityUses;
+  if (entry->map != 0 && entry->map != target.boundDensityMap && webgpu::fdm::map_ready(entry->map)) {
+    if (webgpu::fdm::bind(target.foveatedView, entry->map)) {
+      target.boundDensityMap = entry->map;
+    } else {
+      Log.warn("{} eye foveation: a density map could not be bound to the eye", eyeIndex == 0 ? "Left" : "Right");
+      webgpu::fdm::release_map(entry->map);
+      entry->map = 0;
     }
   }
-  return target.densityMap != 0 && webgpu::fdm::map_ready(target.densityMap) ? target.foveatedView
-                                                                             : wgpu::TextureView{};
+  return target.boundDensityMap != 0 ? target.foveatedView : wgpu::TextureView{};
 }
 
 std::optional<AuroraStereoFrame> request_stereo_frame(uint32_t logicalFrame, uint64_t contentTag) noexcept {
@@ -864,7 +927,8 @@ gfx::StereoReplayFrame make_stereo_replay_frame(const AuroraStereoFrame& input,
     // Not the immersive window's eyes: the host may aim them through the window, whose field of
     // view then changes with every head movement and would rebuild the density map each frame.
     if (input.mode == AURORA_STEREO_FRAME_IMMERSIVE_REPLAY && !input.window) {
-      view.target.foveatedColorView = foveated_eye_view(eye, input.eyes[eye]);
+      view.target.foveatedColorView =
+          foveated_eye_view(eye, input.eyes[eye], input.gazeValid ? input.gaze[eye] : nullptr);
     }
     std::memcpy(&view.projection, input.eyes[eye].projection, sizeof(view.projection));
     std::memcpy(&view.viewFromCenter, input.eyes[eye].viewFromCenter, sizeof(view.viewFromCenter));

aurora-main/lib/gfx/foveation.hpp

@@ -5,10 +5,10 @@
 #include <cstdint>
 #include <vector>
 
-// Fixed foveated rendering for the immersive eyes: the fragment density map an eye's render pass
-// runs under (webgpu/fdm.hpp). Each texel says how finely the framebuffer area it covers is shaded:
+// Foveated rendering for the immersive eyes: the fragment density map an eye's render pass runs
+// under (webgpu/fdm.hpp). Each texel says how finely the framebuffer area it covers is shaded:
 // fully at the centre of the view, in 2x2 then 4x4 pixel blocks towards the edges, where the
-// headset's lenses blur the picture anyway.
+// headset's lenses blur the picture anyway. With eye tracking the centre is where the player looks.
 namespace aurora::gfx::foveation {
 
 enum class Level : uint32_t {
@@ -79,6 +79,65 @@ inline float eccentricity_degrees(float tanX, float tanY) noexcept {
   return std::atan(std::sqrt(tanX * tanX + tanY * tanY)) * (180.0f / 3.14159265358979f);
 }
 
+// Where the map's full density is centred, in tangents of the eye's view like EyeFov's (x right,
+// y up): the forward direction, or the point the player looks at.
+struct Gaze {
+  float tanX = 0.0f;
+  float tanY = 0.0f;
+};
+
+// The angle between the rays through tangents (x, y) and through the gaze.
+inline float angle_from_gaze_degrees(float tanX, float tanY, const Gaze& gaze) noexcept {
+  const float dot = tanX * gaze.tanX + tanY * gaze.tanY + 1.0f;
+  const float norms = std::sqrt((tanX * tanX + tanY * tanY + 1.0f) * (gaze.tanX * gaze.tanX + gaze.tanY * gaze.tanY + 1.0f));
+  return std::acos(std::clamp(dot / norms, -1.0f, 1.0f)) * (180.0f / 3.14159265358979f);
+}
+
+// Eye-tracked maps are built for the gaze snapped to cells of this many map texels square, so an
+// eye's map changes only when the gaze moves that far (about 3 degrees with 32-pixel texels), and a
+// few maps serve a whole session's glances.
+inline constexpr uint32_t kGazeCellTexels = 2;
+
+struct GazeCell {
+  int32_t x = 0;
+  int32_t y = 0;
+  bool operator==(const GazeCell&) const = default;
+};
+
+// The cell of an eye of `eyeWidth` by `eyeHeight` pixels the gaze falls in, counted from the top
+// left and clamped to the eye. A gaze that is not a number counts as the forward direction.
+inline GazeCell gaze_cell(uint32_t eyeWidth, uint32_t eyeHeight, uint32_t texel, const EyeFov& fov,
+                          Gaze gaze) noexcept {
+  const float cellPixels = static_cast<float>(std::max(texel, 1u) * kGazeCellTexels);
+  if (!std::isfinite(gaze.tanX) || !std::isfinite(gaze.tanY)) {
+    gaze = {};
+  }
+  const float spanX = fov.tanRight - fov.tanLeft;
+  const float spanY = fov.tanDown - fov.tanUp;
+  const float u = spanX != 0.0f ? (gaze.tanX - fov.tanLeft) / spanX : 0.5f;
+  const float v = spanY != 0.0f ? (gaze.tanY - fov.tanUp) / spanY : 0.5f;
+  const auto cell = [cellPixels](float fraction, uint32_t pixels) {
+    const int32_t count = std::max(1, static_cast<int32_t>(std::ceil(static_cast<float>(pixels) / cellPixels)));
+    const float position = std::clamp(fraction, 0.0f, 1.0f) * static_cast<float>(pixels) / cellPixels;
+    return std::clamp(static_cast<int32_t>(std::floor(position)), 0, count - 1);
+  };
+  return {cell(u, eyeWidth), cell(v, eyeHeight)};
+}
+
+// The gaze through the centre of a cell, clamped to the eye for an overhanging last row or column.
+inline Gaze cell_gaze(uint32_t eyeWidth, uint32_t eyeHeight, uint32_t texel, const EyeFov& fov,
+                      GazeCell cell) noexcept {
+  const float cellPixels = static_cast<float>(std::max(texel, 1u) * kGazeCellTexels);
+  const float u = eyeWidth > 0 ? std::min((static_cast<float>(cell.x) + 0.5f) * cellPixels, static_cast<float>(eyeWidth)) /
+                                     static_cast<float>(eyeWidth)
+                               : 0.5f;
+  const float v = eyeHeight > 0 ? std::min((static_cast<float>(cell.y) + 0.5f) * cellPixels, static_cast<float>(eyeHeight)) /
+                                      static_cast<float>(eyeHeight)
+                                : 0.5f;
+  return Gaze{.tanX = fov.tanLeft + (fov.tanRight - fov.tanLeft) * u,
+              .tanY = fov.tanUp + (fov.tanDown - fov.tanUp) * v};
+}
+
 inline uint8_t density(Level level, float eccentricity) noexcept {
   const Rings ring = rings(level);
   if (eccentricity < ring.full) {
@@ -95,15 +154,18 @@ struct Map {
 };
 
 // The map for an eye of `eyeWidth` by `eyeHeight` pixels whose field of view is `fov`, `texel` pixels
-// per map texel. The map covers the whole eye, its last row and column possibly overhanging it.
+// per map texel, centred on `gaze` (the forward direction by default). The map covers the whole eye,
+// its last row and column possibly overhanging it.
 inline void build(uint32_t eyeWidth, uint32_t eyeHeight, uint32_t texel, const EyeFov& fov, Level level,
-                  Map& map) {
+                  Map& map, const Gaze& gaze = {}) {
   map.width = texel > 0 ? (eyeWidth + texel - 1) / texel : 0;
   map.height = texel > 0 ? (eyeHeight + texel - 1) / texel : 0;
   map.rg8.assign(static_cast<size_t>(map.width) * map.height * 2, kFullDensity);
   if (level == Level::Off || eyeWidth == 0 || eyeHeight == 0) {
     return;
   }
+  // The forward direction keeps its own, exact formula, so the fixed maps do not change.
+  const bool forward = gaze.tanX == 0.0f && gaze.tanY == 0.0f;
   for (uint32_t y = 0; y < map.height; ++y) {
     // Texel centres, clamped to the eye for an overhanging last row or column.
     const float v = std::min((static_cast<float>(y) + 0.5f) * static_cast<float>(texel), static_cast<float>(eyeHeight)) /
@@ -113,7 +175,8 @@ inline void build(uint32_t eyeWidth, uint32_t eyeHeight, uint32_t texel, const E
       const float u = std::min((static_cast<float>(x) + 0.5f) * static_cast<float>(texel), static_cast<float>(eyeWidth)) /
                       static_cast<float>(eyeWidth);
       const float tanX = fov.tanLeft + (fov.tanRight - fov.tanLeft) * u;
-      const uint8_t value = density(level, eccentricity_degrees(tanX, tanY));
+      const uint8_t value =
+          density(level, forward ? eccentricity_degrees(tanX, tanY) : angle_from_gaze_degrees(tanX, tanY, gaze));
       uint8_t* texelBytes = &map.rg8[(static_cast<size_t>(y) * map.width + x) * 2];
       texelBytes[0] = value;
       texelBytes[1] = value;

aurora-main/tests/foveation_test.cpp

@@ -175,5 +175,112 @@ TEST(Foveation, ReadsTheFieldOfViewBackFromTheEyeProjection) {
   EXPECT_EQ(fallback.tanUp, 1.0f);
 }
 
+// Eye-tracked foveation: the full-density centre follows the gaze.
+
+// The pixel a gaze lands on, as build lays the eye out.
+std::pair<float, float> gaze_pixel(const EyeFov& fov, const Gaze& gaze, uint32_t width, uint32_t height) {
+  return {(gaze.tanX - fov.tanLeft) / (fov.tanRight - fov.tanLeft) * width,
+          (gaze.tanY - fov.tanUp) / (fov.tanDown - fov.tanUp) * height};
+}
+
+TEST(Foveation, TheForwardGazeKeepsTheFixedMap) {
+  for (Level level : {Level::Low, Level::Medium, Level::High}) {
+    Map gazed;
+    foveation::build(1344, 1408, 32, left_eye(), level, gazed, Gaze{});
+    EXPECT_TRUE(gazed.rg8 == build_map(level).rg8) << "level " << int(level);
+  }
+  // The general angle agrees with the forward one.
+  for (float tanX : {-1.2f, -0.3f, 0.0f, 0.4f, 0.9f}) {
+    for (float tanY : {-1.0f, 0.0f, 0.7f}) {
+      EXPECT_NEAR(angle_from_gaze_degrees(tanX, tanY, Gaze{}), eccentricity_degrees(tanX, tanY), 0.01f);
+    }
+  }
+}
+
+TEST(Foveation, TheFullDensityRegionFollowsTheGaze) {
+  const EyeFov fov = left_eye();
+  // Down and to the right, well off the forward direction.
+  const Gaze gaze{.tanX = std::tan(20.0f * kDegrees), .tanY = std::tan(-15.0f * kDegrees)};
+  Map map;
+  foveation::build(1344, 1408, 32, fov, Level::High, map, gaze);
+  double sumX = 0.0;
+  double sumY = 0.0;
+  uint32_t count = 0;
+  for (uint32_t y = 0; y < map.height; ++y) {
+    for (uint32_t x = 0; x < map.width; ++x) {
+      if (at(map, x, y) == kFullDensity) {
+        sumX += x + 0.5;
+        sumY += y + 0.5;
+        ++count;
+      }
+    }
+  }
+  ASSERT_GT(count, 0u);
+  const auto [pixelX, pixelY] = gaze_pixel(fov, gaze, 1344, 1408);
+  EXPECT_NEAR(sumX / count, pixelX / 32.0, 1.5);
+  EXPECT_NEAR(sumY / count, pixelY / 32.0, 1.5);
+  // Where the forward map was sharpest, the far side of the gaze is now coarse.
+  const Map fixed = build_map(Level::High, fov);
+  EXPECT_EQ(at(map, static_cast<uint32_t>(pixelX / 32.0f), static_cast<uint32_t>(pixelY / 32.0f)), kFullDensity);
+  EXPECT_EQ(at(fixed, 0, 0), kQuarterDensity);
+  EXPECT_EQ(at(map, 0, 0), kQuarterDensity);
+}
+
+TEST(Foveation, DensityNeverRisesAwayFromTheGaze) {
+  const EyeFov fov = left_eye();
+  const Gaze gaze{.tanX = -0.35f, .tanY = 0.2f};
+  for (Level level : {Level::Low, Level::Medium, Level::High}) {
+    Map map;
+    foveation::build(1344, 1408, 32, fov, level, map, gaze);
+    std::vector<std::pair<float, uint8_t>> texels;
+    for (uint32_t y = 0; y < map.height; ++y) {
+      for (uint32_t x = 0; x < map.width; ++x) {
+        const auto [tanX, tanY] = tangents(map, fov, x, y, 1344, 1408, 32);
+        texels.emplace_back(angle_from_gaze_degrees(tanX, tanY, gaze), at(map, x, y));
+      }
+    }
+    std::sort(texels.begin(), texels.end());
+    for (size_t i = 1; i < texels.size(); ++i) {
+      EXPECT_LE(texels[i].second, texels[i - 1].second);
+    }
+    EXPECT_EQ(texels.front().second, kFullDensity);
+  }
+}
+
+TEST(Foveation, GazeCellsSnapTheGazeAndStayInsideTheEye) {
+  const EyeFov fov = left_eye();
+  constexpr uint32_t kWidth = 1344, kHeight = 1408, kTexel = 32;
+  constexpr float kCellPixels = kTexel * kGazeCellTexels;
+  // A cell's own gaze lies within half a cell of every gaze that falls in it.
+  for (float tanX : {-0.9f, -0.2f, 0.0f, 0.31f, 0.8f}) {
+    for (float tanY : {-0.8f, 0.0f, 0.45f}) {
+      const Gaze gaze{.tanX = tanX, .tanY = tanY};
+      const GazeCell cell = gaze_cell(kWidth, kHeight, kTexel, fov, gaze);
+      const Gaze centre = cell_gaze(kWidth, kHeight, kTexel, fov, cell);
+      const auto [gx, gy] = gaze_pixel(fov, gaze, kWidth, kHeight);
+      const auto [cx, cy] = gaze_pixel(fov, centre, kWidth, kHeight);
+      EXPECT_LE(std::abs(gx - cx), kCellPixels / 2.0f + 0.01f);
+      EXPECT_LE(std::abs(gy - cy), kCellPixels / 2.0f + 0.01f);
+      EXPECT_TRUE(gaze_cell(kWidth, kHeight, kTexel, fov, centre) == cell);
+    }
+  }
+  // Gazes a few pixels apart share a cell; the forward direction has one of its own.
+  const Gaze forward{};
+  const GazeCell forwardCell = gaze_cell(kWidth, kHeight, kTexel, fov, forward);
+  const auto [fx, fy] = gaze_pixel(fov, forward, kWidth, kHeight);
+  EXPECT_EQ(forwardCell.x, static_cast<int32_t>(fx / kCellPixels));
+  EXPECT_EQ(forwardCell.y, static_cast<int32_t>(fy / kCellPixels));
+  // Beyond the eye, and not a number at all.
+  const int32_t lastColumn = static_cast<int32_t>(std::ceil(kWidth / kCellPixels)) - 1;
+  const int32_t lastRow = static_cast<int32_t>(std::ceil(kHeight / kCellPixels)) - 1;
+  const GazeCell far = gaze_cell(kWidth, kHeight, kTexel, fov, Gaze{.tanX = 10.0f, .tanY = -10.0f});
+  EXPECT_EQ(far.x, lastColumn);
+  EXPECT_EQ(far.y, lastRow);
+  const GazeCell farOther = gaze_cell(kWidth, kHeight, kTexel, fov, Gaze{.tanX = -10.0f, .tanY = 10.0f});
+  EXPECT_EQ(farOther.x, 0);
+  EXPECT_EQ(farOther.y, 0);
+  EXPECT_TRUE(gaze_cell(kWidth, kHeight, kTexel, fov, Gaze{.tanX = NAN, .tanY = 0.2f}) == forwardCell);
+}
+
 } // namespace
 } // namespace aurora::gfx::foveation

runtime/CMakeLists.txt

@@ -456,6 +456,11 @@ add_executable(mkw_vr_wii_remote_tests "${CMAKE_CURRENT_LIST_DIR}/tests/vr_wii_r
 target_include_directories(mkw_vr_wii_remote_tests PRIVATE "${CMAKE_CURRENT_LIST_DIR}/include")
 target_compile_features(mkw_vr_wii_remote_tests PRIVATE cxx_std_17)
 add_test(NAME mkw_vr_wii_remote_tests COMMAND mkw_vr_wii_remote_tests)
+# Eye-tracked foveation: the gaze in each eye's view (vr/eye_gaze.h).
+add_executable(mkw_vr_eye_gaze_tests "${CMAKE_CURRENT_LIST_DIR}/tests/vr_eye_gaze_tests.cpp")
+target_include_directories(mkw_vr_eye_gaze_tests PRIVATE "${CMAKE_CURRENT_LIST_DIR}/include")
+target_compile_features(mkw_vr_eye_gaze_tests PRIVATE cxx_std_17)
+add_test(NAME mkw_vr_eye_gaze_tests COMMAND mkw_vr_eye_gaze_tests)
 
 # The in-headset settings panel's controller chord, release latch, selection,
 # scrolling and canvas mapping, plus the thread bridge they publish through.

runtime/include/runtime_config.h

@@ -92,6 +92,7 @@ struct RuntimeUserConfig {
     std::optional<bool> vrWheelHaptics;
     std::optional<std::string> vrPerformanceLevel;
     std::optional<std::string> vrFoveation;
+    std::optional<bool> vrEyeTrackedFoveation;
     std::optional<std::string> vrRecenterKey;
     std::optional<float> vrLeanBackDegrees;
     // F10 > Diagnostics: OpenXR pacing and presentation logging in console.log.
@@ -320,6 +321,16 @@ inline bool IsSupportedVrFoveation(std::string_view value) {
     return std::find(kVrFoveationLevels.begin(), kVrFoveationLevels.end(), value) != kVrFoveationLevels.end();
 }
 
+// Eye-tracked foveation: with a headset that tracks the eyes (XR_EXT_eye_gaze_interaction, the Steam
+// Frame's), the foveation level's full-density region follows the gaze instead of staying on each
+// eye's forward direction. Live while the session's runtime offered the gaze at launch. On by
+// default on the Steam Frame; elsewhere off, since Horizon OS asks for an eye tracking permission.
+#if defined(MKW_HEADSET_STEAM_FRAME)
+inline constexpr bool kVrEyeTrackedFoveationDefault = true;
+#else
+inline constexpr bool kVrEyeTrackedFoveationDefault = false;
+#endif
+
 // The level aurora_set_stereo_foveation takes; anything unknown is off.
 inline uint32_t VrFoveationLevelIndex(std::string_view value) {
     const auto it = std::find(kVrFoveationLevels.begin(), kVrFoveationLevels.end(), value);
@@ -891,6 +902,7 @@ inline RuntimeUserConfig ParseConfigDocument(const toml::value& document) {
         value && IsSupportedVrFoveation(*value)) {
         config.vrFoveation = *value;
     }
+    config.vrEyeTrackedFoveation = FindConfigValue<bool>(document, "vr", "eye_tracked_foveation");
     if (auto value = FindConfigValue<std::string>(document, "vr", "mirror_view");
         value && IsSupportedVrMirrorView(*value)) {
         config.vrMirrorView = *value;
@@ -1303,6 +1315,11 @@ inline bool SetVrFoveation(std::string value) {
     return WriteSetting("vr", "foveation", FormatString(value));
 }
 
+inline bool SetVrEyeTrackedFoveation(bool value) {
+    Mutable().vrEyeTrackedFoveation = value;
+    return WriteSetting("vr", "eye_tracked_foveation", value ? "true" : "false");
+}
+
 inline bool SetVrFirstPersonSeat(std::string value) {
     if (!IsSupportedVrFirstPersonSeat(value)) {
         return false;
@@ -1828,6 +1845,10 @@ inline std::string VrFoveation(std::string fallback = kVrFoveationDefault) {
     return value && IsSupportedVrFoveation(*value) ? *value : std::move(fallback);
 }
 
+inline bool VrEyeTrackedFoveation(bool fallback = kVrEyeTrackedFoveationDefault) {
+    return Get().vrEyeTrackedFoveation.value_or(fallback);
+}
+
 inline int32_t VrFirstPersonHiddenModel(int32_t fallback = kVrFirstPersonHiddenModelDefault) {
     return std::clamp(Get().vrFirstPersonHiddenModel.value_or(fallback), -1, 31);
 }

runtime/include/vr/eye_gaze.h

@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#pragma once
+
+#include <cmath>
+
+// Eye-tracked foveation ([vr] eye_tracked_foveation): where the player looks, as each eye's image
+// measures it. Kept free of OpenXR types so it can be checked headlessly (tests/vr_eye_gaze_tests.cpp).
+//
+// Conventions are OpenXR's: right-handed, +Y up, and a pose looks down its -Z axis.
+namespace mkw::vr::eye_gaze {
+
+struct Quaternion {
+    float x = 0.0f;
+    float y = 0.0f;
+    float z = 0.0f;
+    float w = 1.0f;
+};
+
+// Tangents of an eye's view, x right and y up, as its frustum (XrFovf) measures them.
+struct Tangents {
+    float x = 0.0f;
+    float y = 0.0f;
+    bool valid = false;
+};
+
+// Beyond this angle from an eye's forward direction a gaze is no point of its image (cos 80 deg).
+inline constexpr float kMinForwardCosine = 0.17364818f;
+
+// The gaze pose's look direction in one eye's view, from both orientations in the same space. The
+// eyes' views can be canted outwards, so each eye gets its own tangents.
+inline Tangents InEye(Quaternion gaze, Quaternion eye) noexcept {
+    const auto normalized = [](Quaternion q) {
+        const float length = std::sqrt(q.x * q.x + q.y * q.y + q.z * q.z + q.w * q.w);
+        if (!(length > 1.0e-6f)) {
+            return Quaternion{};
+        }
+        return Quaternion{q.x / length, q.y / length, q.z / length, q.w / length};
+    };
+    // q * v * conjugate(q).
+    const auto rotate = [](const Quaternion& q, float vx, float vy, float vz, float out[3]) {
+        const float tx = 2.0f * (q.y * vz - q.z * vy);
+        const float ty = 2.0f * (q.z * vx - q.x * vz);
+        const float tz = 2.0f * (q.x * vy - q.y * vx);
+        out[0] = vx + q.w * tx + (q.y * tz - q.z * ty);
+        out[1] = vy + q.w * ty + (q.z * tx - q.x * tz);
+        out[2] = vz + q.w * tz + (q.x * ty - q.y * tx);
+    };
+    gaze = normalized(gaze);
+    eye = normalized(eye);
+    float look[3];
+    rotate(gaze, 0.0f, 0.0f, -1.0f, look);
+    float seen[3];
+    rotate(Quaternion{-eye.x, -eye.y, -eye.z, eye.w}, look[0], look[1], look[2], seen);
+    if (!(-seen[2] > kMinForwardCosine)) {
+        return {};
+    }
+    return {seen[0] / -seen[2], seen[1] / -seen[2], true};
+}
+
+} // namespace mkw::vr::eye_gaze

runtime/include/vr/openxr_input.h

@@ -130,6 +130,15 @@ public:
     // The tracked hands' joints in the seated frame, as the last Sync located
     // them; read on the pacing thread only.
     const hand_tracking::HandJointFrame& HandJoints() const noexcept { return m_joint_frame; }
+    // Eye-tracked foveation: the gaze pose's orientation in the app space, as the last Sync located
+    // it for the frame's display time. False while there is no tracked gaze (no eye tracker, a
+    // blink, or the runtime not offering XR_EXT_eye_gaze_interaction).
+    bool EyeGaze(XrQuaternionf* orientation) const noexcept {
+        if (m_gaze_valid && orientation != nullptr) {
+            *orientation = m_gaze_orientation;
+        }
+        return m_gaze_valid;
+    }
     // A hand's tracker while tracked hands keep one, else XR_NULL_HANDLE.
     XrHandTrackerEXT HandTracker(uint32_t hand) const noexcept {
         return hand < kHands ? m_hand_trackers[hand] : XR_NULL_HANDLE;
@@ -150,6 +159,9 @@ private:
 
     bool CreateActions();
     bool SuggestBindings();
+    // XR_EXT_eye_gaze_interaction is enabled and the system has an eye tracker.
+    bool EyeGazeOffered();
+    void LocateEyeGaze(XrTime time);
     void CreatePoseSpaces();
     void DestroyPoseSpaces();
     void LoadInputClock();
@@ -201,6 +213,12 @@ private:
     XrAction m_aim_pose = XR_NULL_HANDLE;
     XrAction m_grip_pose = XR_NULL_HANDLE;
     XrAction m_haptic = XR_NULL_HANDLE;
+    // The eyes' gaze (XR_EXT_eye_gaze_interaction), one pose for both, and where it was last located.
+    XrAction m_gaze_pose = XR_NULL_HANDLE;
+    XrSpace m_gaze_space = XR_NULL_HANDLE;
+    XrQuaternionf m_gaze_orientation{0.0f, 0.0f, 0.0f, 1.0f};
+    bool m_gaze_valid = false;
+    bool m_gaze_logged = false;
     XrPath m_hand_paths[kHands]{};
     XrSpace m_aim_spaces[kHands]{};
     XrSpace m_grip_spaces[kHands]{};

runtime/src/settings_overlay.cpp

@@ -156,6 +156,7 @@ bool g_vrHandTracking = RuntimeConfigFile::VrHandTracking();
 constexpr std::array<const char*, 4> kVrFoveationLabels{"Off", "Low", "Medium", "High"};
 static_assert(kVrFoveationLabels.size() == RuntimeConfigFile::kVrFoveationLevels.size());
 int g_vrFoveation = static_cast<int>(RuntimeConfigFile::VrFoveationLevelIndex(RuntimeConfigFile::VrFoveation()));
+bool g_vrEyeTrackedFoveation = RuntimeConfigFile::VrEyeTrackedFoveation();
 #endif
 bool g_vrFirstPerson = RuntimeConfigFile::VrFirstPerson(false);
 bool g_vrFirstPersonToggleClick = RuntimeConfigFile::VrFirstPersonToggleClick();
@@ -1523,6 +1524,15 @@ void DrawVrSettings() {
                         "lenses blur the picture anyway, to free GPU time. This session started with it "
                         "off, or without a GPU that supports it: a new level applies after a restart.");
     }
+    if (ImGui::Checkbox("Foveation follows the eyes", &g_vrEyeTrackedFoveation)) {
+        RuntimeConfigFile::SetVrEyeTrackedFoveation(g_vrEyeTrackedFoveation);
+    }
+    if (ImGui::IsItemHovered()) {
+        ImGui::SetTooltip(
+            "With a headset that tracks the eyes (the Steam Frame), the sharp centre of the foveated "
+            "race view moves to where you look instead of staying straight ahead. Turning it off "
+            "applies immediately; turning it on needs a restart if the session started without it.");
+    }
 #endif
     {
         const auto rateLabel = [](uint32_t hz) {

runtime/src/vr/openxr_input.cpp

@@ -423,6 +423,36 @@ bool OpenXRInput::CreateActions() {
             return false;
         }
     }
+    // Eye-tracked foveation: one gaze pose for both eyes, with no hand to name.
+    if (EyeGazeOffered()) {
+        XrActionCreateInfo info{XR_TYPE_ACTION_CREATE_INFO};
+        info.actionType = XR_ACTION_TYPE_POSE_INPUT;
+        std::strncpy(info.actionName, "eye_gaze", XR_MAX_ACTION_NAME_SIZE - 1);
+        std::strncpy(info.localizedActionName, "Eye Gaze", XR_MAX_LOCALIZED_ACTION_NAME_SIZE - 1);
+        if (!Check(xrCreateAction(m_action_set, &info, &m_gaze_pose), "eye_gaze")) {
+            // Not worth the controllers: carry on with the fixed foveation centre.
+            m_gaze_pose = XR_NULL_HANDLE;
+        }
+    }
+    return true;
+}
+
+bool OpenXRInput::EyeGazeOffered() {
+    const auto& extensions = m_runtime->EnabledExtensions();
+    if (std::find(extensions.begin(), extensions.end(), "XR_EXT_eye_gaze_interaction") == extensions.end()) {
+        return false;
+    }
+    XrSystemEyeGazeInteractionPropertiesEXT gaze{XR_TYPE_SYSTEM_EYE_GAZE_INTERACTION_PROPERTIES_EXT};
+    XrSystemProperties properties{XR_TYPE_SYSTEM_PROPERTIES, &gaze};
+    const XrResult result = xrGetSystemProperties(m_runtime->Instance(), m_runtime->SystemId(), &properties);
+    if (XR_FAILED(result) || gaze.supportsEyeGazeInteraction != XR_TRUE) {
+        std::ostringstream message;
+        message << "OpenXR eye gaze: the runtime reports no eye tracker (" << result
+                << "); foveation stays on each eye's forward direction";
+        Log(OpenXRLogLevel::Info, message.str());
+        return false;
+    }
+    Log(OpenXRLogLevel::Info, "OpenXR eye gaze: available; foveation follows the gaze ([vr] eye_tracked_foveation)");
     return true;
 }
 
@@ -533,6 +563,10 @@ bool OpenXRInput::SuggestBindings() {
         };
         suggest("/interaction_profiles/valve/frame_controller_valve", frame, false);
     }
+    if (m_gaze_pose != XR_NULL_HANDLE) {
+        suggest("/interaction_profiles/ext/eye_gaze_interaction",
+                {{&m_gaze_pose, "/user/eyes_ext/input/gaze_ext/pose"}}, false);
+    }
     return true;
 }
 
@@ -558,9 +592,56 @@ void OpenXRInput::CreatePoseSpaces() {
             }
         }
     }
+    if (m_gaze_pose != XR_NULL_HANDLE) {
+        XrActionSpaceCreateInfo info{XR_TYPE_ACTION_SPACE_CREATE_INFO};
+        info.action = m_gaze_pose;
+        info.poseInActionSpace.orientation.w = 1.0f;
+        const XrResult result = xrCreateActionSpace(m_runtime->Session(), &info, &m_gaze_space);
+        m_runtime->ObserveResult(result);
+        if (XR_FAILED(result)) {
+            m_gaze_space = XR_NULL_HANDLE;
+            std::ostringstream message;
+            message << "xrCreateActionSpace(eye gaze) failed (" << result
+                    << "); foveation stays on each eye's forward direction";
+            Log(OpenXRLogLevel::Warning, message.str());
+        }
+    }
+}
+
+// Eye-tracked foveation: the gaze for the display time the eyes are rendered for, the time their
+// views are located at, so the full-density region lands where the eyes look in that frame. Only a
+// tracked orientation counts; the runtime reports an untracked one through blinks.
+void OpenXRInput::LocateEyeGaze(XrTime time) {
+    m_gaze_valid = false;
+    if (m_gaze_space == XR_NULL_HANDLE) {
+        return;
+    }
+    XrActionStateGetInfo info{XR_TYPE_ACTION_STATE_GET_INFO};
+    info.action = m_gaze_pose;
+    XrActionStatePose state{XR_TYPE_ACTION_STATE_POSE};
+    if (XR_FAILED(xrGetActionStatePose(m_runtime->Session(), &info, &state)) || state.isActive != XR_TRUE) {
+        return;
+    }
+    constexpr XrSpaceLocationFlags kTracked =
+        XR_SPACE_LOCATION_ORIENTATION_VALID_BIT | XR_SPACE_LOCATION_ORIENTATION_TRACKED_BIT;
+    XrSpaceLocation location{XR_TYPE_SPACE_LOCATION};
+    if (XR_FAILED(xrLocateSpace(m_gaze_space, m_runtime->AppSpace(), time, &location)) ||
+        (location.locationFlags & kTracked) != kTracked) {
+        return;
+    }
+    m_gaze_orientation = location.pose.orientation;
+    m_gaze_valid = true;
+    if (!m_gaze_logged) {
+        m_gaze_logged = true;
+        Log(OpenXRLogLevel::Info, "OpenXR eye gaze: tracking");
+    }
 }
 
 void OpenXRInput::DestroyPoseSpaces() {
+    if (m_gaze_space != XR_NULL_HANDLE) {
+        xrDestroySpace(m_gaze_space);
+        m_gaze_space = XR_NULL_HANDLE;
+    }
     for (uint32_t hand = 0; hand < kHandCount; ++hand) {
         for (XrSpace* space : {&m_aim_spaces[hand], &m_grip_spaces[hand]}) {
             if (*space != XR_NULL_HANDLE) {
@@ -909,6 +990,9 @@ void OpenXRInput::Destroy() {
     m_thumbstick = m_thumbstick_click = m_trigger = m_squeeze = XR_NULL_HANDLE;
     m_button_primary = m_button_secondary = m_menu = m_haptic = XR_NULL_HANDLE;
     m_dpad_up = m_dpad_down = m_dpad_left = m_dpad_right = XR_NULL_HANDLE;
+    m_gaze_pose = XR_NULL_HANDLE;
+    m_gaze_valid = false;
+    m_gaze_logged = false;
     m_aim_pose = m_grip_pose = XR_NULL_HANDLE;
     m_hand_paths[0] = m_hand_paths[1] = XR_NULL_PATH;
     m_convert_now_to_xr_time = nullptr;
@@ -933,6 +1017,7 @@ void OpenXRInput::Idle() {
     }
     m_pointer.Reset();
     m_horizon = {1.0f, 0.0f};
+    m_gaze_valid = false;
     OpenXRPublishWiiRemote(Relay().JoystickId(), OpenXRWiiRemoteSample{});
     // The panel stays as it was; only what the controllers were holding is forgotten.
     m_panel_controls.Reset();
@@ -976,6 +1061,7 @@ void OpenXRInput::Sync(XrTime predicted_display_time, const OpenXRPointerScreen&
         Idle();
         return;
     }
+    LocateEyeGaze(predicted_display_time);
 
     // `active`, when given, says whether the action is bound to a source the
     // runtime has right now (a controller, or a tracked hand).

runtime/src/vr/openxr_integration.cpp

@@ -9,6 +9,7 @@
 #include "runtime_config.h"
 #include "gx_thread.h"
 #include "runtime_log.h"
+#include "vr/eye_gaze.h"
 #include "vr/mkw_vr_culling.h"
 #include "vr/mkw_vr_first_person.h"
 #include "vr/mkw_vr_policy.h"
@@ -428,6 +429,10 @@ public:
         // Horizon OS's room view; the Steam Frame build neither asks for it nor offers the setting.
         config.optional_extensions.push_back("XR_FB_passthrough");
 #endif
+        // Eye-tracked foveation: the gaze the density maps centre on (OpenXRInput::EyeGaze).
+        if (RuntimeConfigFile::VrEyeTrackedFoveation()) {
+            config.optional_extensions.push_back("XR_EXT_eye_gaze_interaction");
+        }
         AddHandMeshExtensions(config);
         config.instance_create_next = OpenXRAndroidInstanceCreateNext();
 #endif
@@ -1465,9 +1470,30 @@ private:
                          position_valid && base_position_valid_, units_per_meter,
                          lean_back_radians, destination.eyes[eye].viewFromCenter);
         }
+        BuildEyeGaze(source, destination);
         BuildCockpit(source, position_valid, units_per_meter, lean_back_radians, destination.cockpit);
     }
 
+    // Eye-tracked foveation: where the eyes look, in each eye's own view (the views may be canted),
+    // from the gaze the input located for this packet's display time. Without a tracked gaze, or
+    // with the setting off, Aurora centres foveation on each eye's forward direction.
+    void BuildEyeGaze(const OpenXRBackendFrame& source, AuroraStereoFrame& destination) const noexcept {
+        XrQuaternionf gaze{};
+        if (input_ == nullptr || !RuntimeConfigFile::VrEyeTrackedFoveation() || !input_->EyeGaze(&gaze)) {
+            return;
+        }
+        bool valid = true;
+        for (uint32_t eye = 0; eye < kOpenXREyeCount; ++eye) {
+            const XrQuaternionf& view = source.xr_frame.views[eye].pose.orientation;
+            const eye_gaze::Tangents seen =
+                eye_gaze::InEye({gaze.x, gaze.y, gaze.z, gaze.w}, {view.x, view.y, view.z, view.w});
+            valid = valid && seen.valid;
+            destination.gaze[eye][0] = seen.x;
+            destination.gaze[eye][1] = seen.y;
+        }
+        destination.gazeValid = valid;
+    }
+
     // The first-person cockpit's hands and separate wheel, in the seated frame
     // the eye transforms place at base + lean * seat (metres). Always carries
     // the packet's world scale, which Aurora rescales to the sealed frame's.

runtime/src/vr/openxr_runtime.cpp

@@ -151,6 +151,14 @@ bool OpenXRRuntime::EnumerateInstanceCapabilities() {
     for (const XrExtensionProperties& extension : extension_properties) {
         m_available_extensions.emplace_back(extension.extensionName);
     }
+    // Everything the runtime offers, once per instance: what a new headset's runtime can do (the
+    // Steam Frame's SteamVR, say) is in the first session log rather than behind a debugger.
+    std::ostringstream offered;
+    offered << "OpenXR runtime offers " << m_available_extensions.size() << " extensions:";
+    for (const std::string& name : m_available_extensions) {
+        offered << ' ' << name;
+    }
+    Log(OpenXRLogLevel::Info, offered.str());
 
     uint32_t layer_count = 0;
     if (!Check(xrEnumerateApiLayerProperties(0, &layer_count, nullptr),

runtime/tests/vr_config_tests.cpp

@@ -68,6 +68,17 @@ int main() {
     Require(std::string_view(MKW_VR_REFRESH_RATE_DEFAULT_TEXT) == "0");
 #endif
 
+    // [vr] eye_tracked_foveation: the foveation centre follows the gaze, on the Steam Frame by default.
+    Require(Parse("[vr]\neye_tracked_foveation = true\n").vrEyeTrackedFoveation == true);
+    Require(Parse("[vr]\neye_tracked_foveation = false\n").vrEyeTrackedFoveation == false);
+    Require(!Parse("[vr]\neye_tracked_foveation = 1\n").vrEyeTrackedFoveation.has_value());
+    Require(!Parse("[vr]\n").vrEyeTrackedFoveation.has_value());
+#if defined(MKW_HEADSET_STEAM_FRAME)
+    Require(RuntimeConfigFile::kVrEyeTrackedFoveationDefault);
+#else
+    Require(!RuntimeConfigFile::kVrEyeTrackedFoveationDefault);
+#endif
+
     // [vr] passthrough: Horizon OS's room view, which the Steam Frame build does not offer.
     Require(Parse("[vr]\npassthrough = false\n").vrPassthrough == false);
     Require(!Parse("[vr]\n").vrPassthrough.has_value());

runtime/tests/vr_eye_gaze_tests.cpp

@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#include "vr/eye_gaze.h"
+
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+
+using mkw::vr::eye_gaze::InEye;
+using mkw::vr::eye_gaze::Quaternion;
+using mkw::vr::eye_gaze::Tangents;
+
+namespace {
+
+void Check(bool condition, const char* what) {
+    if (!condition) {
+        std::fprintf(stderr, "vr_eye_gaze_tests: %s\n", what);
+        std::exit(1);
+    }
+}
+
+void CheckNear(float value, float expected, const char* what) {
+    Check(std::fabs(value - expected) <= 1.0e-4f, what);
+}
+
+constexpr float kDegrees = 3.14159265358979f / 180.0f;
+
+// A turn by `degrees` about the unit axis (x, y, z).
+Quaternion Turn(float degrees, float x, float y, float z) {
+    const float half = 0.5f * degrees * kDegrees;
+    return {x * std::sin(half), y * std::sin(half), z * std::sin(half), std::cos(half)};
+}
+
+} // namespace
+
+int main() {
+    // Looking straight ahead through an eye looking straight ahead: its forward direction.
+    Tangents t = InEye({}, {});
+    Check(t.valid, "straight ahead is valid");
+    CheckNear(t.x, 0.0f, "straight ahead: x");
+    CheckNear(t.y, 0.0f, "straight ahead: y");
+
+    // A turn about +Y by a positive angle looks left (-X), about +X looks up (+Y).
+    t = InEye(Turn(20.0f, 0.0f, 1.0f, 0.0f), {});
+    Check(t.valid, "20 degrees left is valid");
+    CheckNear(t.x, -std::tan(20.0f * kDegrees), "20 degrees left: x");
+    CheckNear(t.y, 0.0f, "20 degrees left: y");
+    t = InEye(Turn(15.0f, 1.0f, 0.0f, 0.0f), {});
+    CheckNear(t.x, 0.0f, "15 degrees up: x");
+    CheckNear(t.y, std::tan(15.0f * kDegrees), "15 degrees up: y");
+
+    // An eye canted outwards sees the same gaze off its own centre.
+    t = InEye({}, Turn(10.0f, 0.0f, 1.0f, 0.0f));
+    CheckNear(t.x, std::tan(10.0f * kDegrees), "a left-canted eye sees straight ahead to its right");
+    t = InEye(Turn(10.0f, 0.0f, 1.0f, 0.0f), Turn(10.0f, 0.0f, 1.0f, 0.0f));
+    CheckNear(t.x, 0.0f, "gaze along the canted eye: x");
+    CheckNear(t.y, 0.0f, "gaze along the canted eye: y");
+
+    // The head's own turn cancels out: only the gaze relative to the eye counts.
+    const Quaternion head = Turn(70.0f, 0.0f, 1.0f, 0.0f);
+    const Quaternion look = Turn(70.0f + 12.0f, 0.0f, 1.0f, 0.0f);
+    t = InEye(look, head);
+    CheckNear(t.x, -std::tan(12.0f * kDegrees), "a turned head: x");
+
+    // Sideways or behind is no point of the image; an unnormalised quaternion still works.
+    Check(!InEye(Turn(85.0f, 0.0f, 1.0f, 0.0f), {}).valid, "85 degrees off is invalid");
+    Check(!InEye(Turn(180.0f, 0.0f, 1.0f, 0.0f), {}).valid, "behind is invalid");
+    Quaternion scaled = Turn(15.0f, 1.0f, 0.0f, 0.0f);
+    scaled = {scaled.x * 3.0f, scaled.y * 3.0f, scaled.z * 3.0f, scaled.w * 3.0f};
+    t = InEye(scaled, {});
+    CheckNear(t.y, std::tan(15.0f * kDegrees), "unnormalised quaternion");
+    Check(InEye({0.0f, 0.0f, 0.0f, 0.0f}, {}).valid, "a zero quaternion reads as identity");
+
+    std::printf("vr_eye_gaze_tests: all checks passed\n");
+    return 0;
+}