Centre foveation on the player's gaze on headsets with eye tracking

With [vr] eye_tracked_foveation (on by default on the Steam Frame, off
elsewhere) the runtime asks for XR_EXT_eye_gaze_interaction. When the
system reports an eye tracker, OpenXRInput binds the gaze pose and
locates it for each packet's display time, in the space the eye views
are located in; vr/eye_gaze.h turns it into tangents of each eye's own
view, which AuroraStereoFrame now carries (appended, after the existing
prefix).

Aurora centres the eye's fragment density map on the gaze snapped to a
cell of two map texels (about 3 degrees). Each eye keeps up to 32 maps,
one per cell looked at, so a glance back reuses its map; a new map is
bound once its upload completes, and until then the eye keeps the map
it had. Without a tracked gaze (a blink, no tracker, the setting off)
foveation centres on the forward direction exactly as before: the
forward maps are byte-identical.

Also logs every extension the OpenXR runtime offers at startup, so the
first Steam Frame session shows what SteamVR's Android runtime has.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019HBRGKTE1GnN2ah8gcZKr3

aurora-main/lib/aurora.cpp

@@ -607,6 +607,17 @@ std::mutex g_surfaceMutex;
 std::atomic<bool> g_surfaceReconfigurePending{false};
 std::atomic<bool> g_surfaceRecreatePending{false};
 
+// One fragment density map of an eye (see StereoEyeTarget): centred on the eye's forward direction,
+// or on a gaze cell (gfx/foveation.hpp) with eye-tracked foveation.
+struct EyeDensityMap {
+  gfx::foveation::GazeCell cell;
+  bool forward = true;
+  uint64_t map = 0;
+  uint64_t lastUse = 0;
+};
+// The gaze cells' maps an eye keeps: a few glances' worth, each 2 bytes per 32x32 pixels.
+constexpr size_t kEyeDensityMapCacheSize = 32;
+
 struct StereoEyeTarget {
   webgpu::TextureWithSampler color;
   webgpu::TextureWithSampler resolvedColor;
@@ -620,24 +631,31 @@ struct StereoEyeTarget {
   // the target when ensure_stereo_eye_target replaces the textures.
   wgpu::BindGroup copyBindGroup;
   // Foveated rendering: a second view of `color` for the immersive eye passes,
-  // which the patched Dawn binds to this eye's fragment density map
-  // (webgpu/fdm.hpp), and what that map was built for.
+  // which the patched Dawn binds to one of this eye's fragment density maps
+  // (webgpu/fdm.hpp). The maps share what densityBase records (the eye's size,
+  // level and field of view); with eye tracking there is one per gaze cell
+  // looked at, the least recently used dropped beyond kEyeDensityMapCacheSize.
   wgpu::TextureView foveatedView;
-  uint64_t densityMap = 0;
-  std::array<int32_t, 7> densityKey{};
+  std::array<int32_t, 7> densityBase{};
+  std::vector<EyeDensityMap> densityMaps;
+  uint64_t boundDensityMap = 0;
+  uint64_t densityUses = 0;
 
   const webgpu::TextureWithSampler& output() const noexcept { return resolvedColor.texture ? resolvedColor : color; }
 };
 std::array<StereoEyeTarget, AURORA_STEREO_EYE_COUNT> g_stereoEyeTargets;
 stereo::MirrorState g_stereoMirrorState;
 
-// The map's binding holds the foveated view, and with it the eye texture, until it is released.
+// A map's binding holds the foveated view, and with it the eye texture, until it is released.
 void release_eye_density_map(StereoEyeTarget& target) noexcept {
-  if (target.densityMap != 0) {
-    webgpu::fdm::release_map(target.densityMap);
-    target.densityMap = 0;
+  for (const EyeDensityMap& entry : target.densityMaps) {
+    if (entry.map != 0) {
+      webgpu::fdm::release_map(entry.map);
+    }
   }
-  target.densityKey = {};
+  target.densityMaps.clear();
+  target.boundDensityMap = 0;
+  target.densityBase = {};
 }
 
 // The eye targets outlive a frame, so the mirror samples them through a bind
@@ -689,9 +707,12 @@ void ensure_stereo_eye_target(uint32_t eyeIndex, uint32_t width, uint32_t height
 }
 
 // The view an immersive eye's passes render through while foveated, or none. The eye's fragment
-// density map is rebuilt whenever its size, field of view or level changes (a map is immutable), and
-// is used once its upload has completed.
-wgpu::TextureView foveated_eye_view(uint32_t eyeIndex, const AuroraStereoEye& input) {
+// density maps are rebuilt whenever its size, field of view or level changes (a map is immutable).
+// `gaze`, the tangents the player looks at when eye tracking provides them, picks the map centred on
+// the gaze cell it falls in, built on first use; without it the map is centred on the eye's forward
+// direction. A map is bound once its upload has completed, and until then the eye keeps the map it
+// had, so a glance never leaves the eye unfoveated.
+wgpu::TextureView foveated_eye_view(uint32_t eyeIndex, const AuroraStereoEye& input, const float* gaze) {
   auto& target = g_stereoEyeTargets[eyeIndex];
   const auto level = static_cast<gfx::foveation::Level>(gfx::get_stereo_foveation());
   if (level == gfx::foveation::Level::Off || target.samples > 1 || !webgpu::fdm::available()) {
@@ -700,42 +721,84 @@ wgpu::TextureView foveated_eye_view(uint32_t eyeIndex, const AuroraStereoEye& in
   const auto fov = gfx::foveation::fov_from_projection(input.projection);
   // Hundredths of a tangent: finer than a map texel, coarse enough to ignore pose noise.
   const auto hundredths = [](float value) { return static_cast<int32_t>(std::lround(value * 100.0f)); };
-  const std::array<int32_t, 7> key{static_cast<int32_t>(target.color.size.width),
-                                   static_cast<int32_t>(target.color.size.height),
-                                   static_cast<int32_t>(level),
-                                   hundredths(fov.tanLeft),
-                                   hundredths(fov.tanRight),
-                                   hundredths(fov.tanDown),
-                                   hundredths(fov.tanUp)};
-  if (key != target.densityKey) {
+  const std::array<int32_t, 7> base{static_cast<int32_t>(target.color.size.width),
+                                    static_cast<int32_t>(target.color.size.height),
+                                    static_cast<int32_t>(level),
+                                    hundredths(fov.tanLeft),
+                                    hundredths(fov.tanRight),
+                                    hundredths(fov.tanDown),
+                                    hundredths(fov.tanUp)};
+  if (base != target.densityBase) {
     release_eye_density_map(target);
-    target.densityKey = key;
-    if (!target.foveatedView) {
-      const wgpu::TextureViewDescriptor descriptor{
-          .label = eyeIndex == 0 ? "Foveated left eye" : "Foveated right eye",
-          .usage = wgpu::TextureUsage::RenderAttachment,
-      };
-      target.foveatedView = target.color.texture.CreateView(&descriptor);
+    target.densityBase = base;
+  }
+  if (!target.foveatedView) {
+    const wgpu::TextureViewDescriptor descriptor{
+        .label = eyeIndex == 0 ? "Foveated left eye" : "Foveated right eye",
+        .usage = wgpu::TextureUsage::RenderAttachment,
+    };
+    target.foveatedView = target.color.texture.CreateView(&descriptor);
+  }
+
+  const uint32_t width = target.color.size.width;
+  const uint32_t height = target.color.size.height;
+  const uint32_t texel = webgpu::fdm::texel_size();
+  const bool forward = gaze == nullptr;
+  const gfx::foveation::GazeCell cell =
+      forward ? gfx::foveation::GazeCell{}
+              : gfx::foveation::gaze_cell(width, height, texel, fov, {.tanX = gaze[0], .tanY = gaze[1]});
+  auto& maps = target.densityMaps;
+  auto entry = std::find_if(maps.begin(), maps.end(), [&](const EyeDensityMap& candidate) {
+    return candidate.forward == forward && (forward || candidate.cell == cell);
+  });
+  if (entry == maps.end()) {
+    if (maps.size() >= kEyeDensityMapCacheSize) {
+      // The least recently used map, never the one the eye renders with.
+      auto oldest = maps.end();
+      for (auto it = maps.begin(); it != maps.end(); ++it) {
+        if (it->map != target.boundDensityMap && (oldest == maps.end() || it->lastUse < oldest->lastUse)) {
+          oldest = it;
+        }
+      }
+      if (oldest != maps.end()) {
+        if (oldest->map != 0) {
+          webgpu::fdm::release_map(oldest->map);
+        }
+        maps.erase(oldest);
+      }
     }
+    const bool firstOfKind =
+        std::none_of(maps.begin(), maps.end(), [&](const EyeDensityMap& other) { return other.forward == forward; });
     gfx::foveation::Map map;
-    gfx::foveation::build(target.color.size.width, target.color.size.height, webgpu::fdm::texel_size(), fov, level,
-                          map);
-    target.densityMap = webgpu::fdm::create_map(map.width, map.height, map.rg8.data());
-    if (target.densityMap != 0 && !webgpu::fdm::bind(target.foveatedView, target.densityMap)) {
-      webgpu::fdm::release_map(target.densityMap);
-      target.densityMap = 0;
-    }
+    gfx::foveation::build(width, height, texel, fov, level, map,
+                          forward ? gfx::foveation::Gaze{}
+                                  : gfx::foveation::cell_gaze(width, height, texel, fov, cell));
+    EyeDensityMap created{.cell = cell, .forward = forward};
+    created.map = webgpu::fdm::create_map(map.width, map.height, map.rg8.data());
     static constexpr std::array<const char*, gfx::foveation::kLevelCount> kLevelNames{"off", "low", "medium", "high"};
-    if (target.densityMap != 0) {
-      Log.info("{} eye foveation {}: {}x{} density map, {} pixels per texel", eyeIndex == 0 ? "Left" : "Right",
-               kLevelNames[static_cast<uint32_t>(level)], map.width, map.height, webgpu::fdm::texel_size());
-    } else {
+    if (created.map == 0) {
       Log.warn("{} eye foveation {}: the {}x{} density map could not be created", eyeIndex == 0 ? "Left" : "Right",
                kLevelNames[static_cast<uint32_t>(level)], map.width, map.height);
+    } else if (firstOfKind) {
+      // Gaze maps come and go with the player's glances; the first says the eye follows the gaze.
+      Log.info("{} eye foveation {}{}: {}x{} density map, {} pixels per texel", eyeIndex == 0 ? "Left" : "Right",
+               kLevelNames[static_cast<uint32_t>(level)], forward ? "" : " following the gaze", map.width, map.height,
+               texel);
+    }
+    maps.push_back(created);
+    entry = std::prev(maps.end());
+  }
+  entry->lastUse = ++target.densityUses;
+  if (entry->map != 0 && entry->map != target.boundDensityMap && webgpu::fdm::map_ready(entry->map)) {
+    if (webgpu::fdm::bind(target.foveatedView, entry->map)) {
+      target.boundDensityMap = entry->map;
+    } else {
+      Log.warn("{} eye foveation: a density map could not be bound to the eye", eyeIndex == 0 ? "Left" : "Right");
+      webgpu::fdm::release_map(entry->map);
+      entry->map = 0;
     }
   }
-  return target.densityMap != 0 && webgpu::fdm::map_ready(target.densityMap) ? target.foveatedView
-                                                                             : wgpu::TextureView{};
+  return target.boundDensityMap != 0 ? target.foveatedView : wgpu::TextureView{};
 }
 
 std::optional<AuroraStereoFrame> request_stereo_frame(uint32_t logicalFrame, uint64_t contentTag) noexcept {
@@ -864,7 +927,8 @@ gfx::StereoReplayFrame make_stereo_replay_frame(const AuroraStereoFrame& input,
     // Not the immersive window's eyes: the host may aim them through the window, whose field of
     // view then changes with every head movement and would rebuild the density map each frame.
     if (input.mode == AURORA_STEREO_FRAME_IMMERSIVE_REPLAY && !input.window) {
-      view.target.foveatedColorView = foveated_eye_view(eye, input.eyes[eye]);
+      view.target.foveatedColorView =
+          foveated_eye_view(eye, input.eyes[eye], input.gazeValid ? input.gaze[eye] : nullptr);
     }
     std::memcpy(&view.projection, input.eyes[eye].projection, sizeof(view.projection));
     std::memcpy(&view.viewFromCenter, input.eyes[eye].viewFromCenter, sizeof(view.viewFromCenter));

aurora-main/lib/gfx/foveation.hpp

@@ -5,10 +5,10 @@
 #include <cstdint>
 #include <vector>
 
-// Fixed foveated rendering for the immersive eyes: the fragment density map an eye's render pass
-// runs under (webgpu/fdm.hpp). Each texel says how finely the framebuffer area it covers is shaded:
+// Foveated rendering for the immersive eyes: the fragment density map an eye's render pass runs
+// under (webgpu/fdm.hpp). Each texel says how finely the framebuffer area it covers is shaded:
 // fully at the centre of the view, in 2x2 then 4x4 pixel blocks towards the edges, where the
-// headset's lenses blur the picture anyway.
+// headset's lenses blur the picture anyway. With eye tracking the centre is where the player looks.
 namespace aurora::gfx::foveation {
 
 enum class Level : uint32_t {
@@ -79,6 +79,65 @@ inline float eccentricity_degrees(float tanX, float tanY) noexcept {
   return std::atan(std::sqrt(tanX * tanX + tanY * tanY)) * (180.0f / 3.14159265358979f);
 }
 
+// Where the map's full density is centred, in tangents of the eye's view like EyeFov's (x right,
+// y up): the forward direction, or the point the player looks at.
+struct Gaze {
+  float tanX = 0.0f;
+  float tanY = 0.0f;
+};
+
+// The angle between the rays through tangents (x, y) and through the gaze.
+inline float angle_from_gaze_degrees(float tanX, float tanY, const Gaze& gaze) noexcept {
+  const float dot = tanX * gaze.tanX + tanY * gaze.tanY + 1.0f;
+  const float norms = std::sqrt((tanX * tanX + tanY * tanY + 1.0f) * (gaze.tanX * gaze.tanX + gaze.tanY * gaze.tanY + 1.0f));
+  return std::acos(std::clamp(dot / norms, -1.0f, 1.0f)) * (180.0f / 3.14159265358979f);
+}
+
+// Eye-tracked maps are built for the gaze snapped to cells of this many map texels square, so an
+// eye's map changes only when the gaze moves that far (about 3 degrees with 32-pixel texels), and a
+// few maps serve a whole session's glances.
+inline constexpr uint32_t kGazeCellTexels = 2;
+
+struct GazeCell {
+  int32_t x = 0;
+  int32_t y = 0;
+  bool operator==(const GazeCell&) const = default;
+};
+
+// The cell of an eye of `eyeWidth` by `eyeHeight` pixels the gaze falls in, counted from the top
+// left and clamped to the eye. A gaze that is not a number counts as the forward direction.
+inline GazeCell gaze_cell(uint32_t eyeWidth, uint32_t eyeHeight, uint32_t texel, const EyeFov& fov,
+                          Gaze gaze) noexcept {
+  const float cellPixels = static_cast<float>(std::max(texel, 1u) * kGazeCellTexels);
+  if (!std::isfinite(gaze.tanX) || !std::isfinite(gaze.tanY)) {
+    gaze = {};
+  }
+  const float spanX = fov.tanRight - fov.tanLeft;
+  const float spanY = fov.tanDown - fov.tanUp;
+  const float u = spanX != 0.0f ? (gaze.tanX - fov.tanLeft) / spanX : 0.5f;
+  const float v = spanY != 0.0f ? (gaze.tanY - fov.tanUp) / spanY : 0.5f;
+  const auto cell = [cellPixels](float fraction, uint32_t pixels) {
+    const int32_t count = std::max(1, static_cast<int32_t>(std::ceil(static_cast<float>(pixels) / cellPixels)));
+    const float position = std::clamp(fraction, 0.0f, 1.0f) * static_cast<float>(pixels) / cellPixels;
+    return std::clamp(static_cast<int32_t>(std::floor(position)), 0, count - 1);
+  };
+  return {cell(u, eyeWidth), cell(v, eyeHeight)};
+}
+
+// The gaze through the centre of a cell, clamped to the eye for an overhanging last row or column.
+inline Gaze cell_gaze(uint32_t eyeWidth, uint32_t eyeHeight, uint32_t texel, const EyeFov& fov,
+                      GazeCell cell) noexcept {
+  const float cellPixels = static_cast<float>(std::max(texel, 1u) * kGazeCellTexels);
+  const float u = eyeWidth > 0 ? std::min((static_cast<float>(cell.x) + 0.5f) * cellPixels, static_cast<float>(eyeWidth)) /
+                                     static_cast<float>(eyeWidth)
+                               : 0.5f;
+  const float v = eyeHeight > 0 ? std::min((static_cast<float>(cell.y) + 0.5f) * cellPixels, static_cast<float>(eyeHeight)) /
+                                      static_cast<float>(eyeHeight)
+                                : 0.5f;
+  return Gaze{.tanX = fov.tanLeft + (fov.tanRight - fov.tanLeft) * u,
+              .tanY = fov.tanUp + (fov.tanDown - fov.tanUp) * v};
+}
+
 inline uint8_t density(Level level, float eccentricity) noexcept {
   const Rings ring = rings(level);
   if (eccentricity < ring.full) {
@@ -95,15 +154,18 @@ struct Map {
 };
 
 // The map for an eye of `eyeWidth` by `eyeHeight` pixels whose field of view is `fov`, `texel` pixels
-// per map texel. The map covers the whole eye, its last row and column possibly overhanging it.
+// per map texel, centred on `gaze` (the forward direction by default). The map covers the whole eye,
+// its last row and column possibly overhanging it.
 inline void build(uint32_t eyeWidth, uint32_t eyeHeight, uint32_t texel, const EyeFov& fov, Level level,
-                  Map& map) {
+                  Map& map, const Gaze& gaze = {}) {
   map.width = texel > 0 ? (eyeWidth + texel - 1) / texel : 0;
   map.height = texel > 0 ? (eyeHeight + texel - 1) / texel : 0;
   map.rg8.assign(static_cast<size_t>(map.width) * map.height * 2, kFullDensity);
   if (level == Level::Off || eyeWidth == 0 || eyeHeight == 0) {
     return;
   }
+  // The forward direction keeps its own, exact formula, so the fixed maps do not change.
+  const bool forward = gaze.tanX == 0.0f && gaze.tanY == 0.0f;
   for (uint32_t y = 0; y < map.height; ++y) {
     // Texel centres, clamped to the eye for an overhanging last row or column.
     const float v = std::min((static_cast<float>(y) + 0.5f) * static_cast<float>(texel), static_cast<float>(eyeHeight)) /
@@ -113,7 +175,8 @@ inline void build(uint32_t eyeWidth, uint32_t eyeHeight, uint32_t texel, const E
       const float u = std::min((static_cast<float>(x) + 0.5f) * static_cast<float>(texel), static_cast<float>(eyeWidth)) /
                       static_cast<float>(eyeWidth);
       const float tanX = fov.tanLeft + (fov.tanRight - fov.tanLeft) * u;
-      const uint8_t value = density(level, eccentricity_degrees(tanX, tanY));
+      const uint8_t value =
+          density(level, forward ? eccentricity_degrees(tanX, tanY) : angle_from_gaze_degrees(tanX, tanY, gaze));
       uint8_t* texelBytes = &map.rg8[(static_cast<size_t>(y) * map.width + x) * 2];
       texelBytes[0] = value;
       texelBytes[1] = value;