Implement retired swapchain management for OpenXR backends: add logic to handle swapchain lifecycle and ensure proper resource cleanup

runtime/src/settings_overlay.cpp

@@ -2261,11 +2261,20 @@ VrSettingsPanel g_vrSettingsPanel;
 
 ImGuiContext* CreateVrSettingsPanelContext() {
     ImGuiContext* const desktop = ImGui::GetCurrentContext();
+    const ImGuiIO& desktopIo = ImGui::GetIO();
     ImGuiContext* const context = ImGui::CreateContext();
     ImGui::SetCurrentContext(context);
     ImGuiIO& io = ImGui::GetIO();
     io.IniFilename = nullptr;
     io.LogFilename = nullptr;
+    // ImGui's current context is one process-wide pointer, and Aurora's frame
+    // worker renders draw data (the desktop overlay, this panel) while this
+    // thread may have switched to this context to draw the panel. The WebGPU
+    // backend finds its device objects through the current context's IO, so
+    // this context carries the desktop's: without them, a render on the worker
+    // during the switch reads through a null backend.
+    io.BackendRendererUserData = desktopIo.BackendRendererUserData;
+    io.BackendRendererName = desktopIo.BackendRendererName;
     // No platform backend draws a cursor for it, and nothing else shows where
     // the controller is aiming.
     io.MouseDrawCursor = true;

runtime/src/vr/openxr_d3d12.cpp

@@ -851,6 +851,7 @@ private:
     // before the old ones go, so a size the runtime cannot allocate leaves the pair as it was.
     // False only on a failure that ends the session.
     bool ResizeWritablePair() {
+        ReapRetiredPairs(false);
         std::array<OpenXREyeSize, kOpenXREyeCount> current{};
         for (uint32_t eye = 0; eye < kOpenXREyeCount; ++eye) {
             if (eye_swapchains_[eye].acquired) {
@@ -874,17 +875,8 @@ private:
             eye_size_ = current;
             return true;
         }
-        std::vector<void*> retired;
-        for (const EyeSwapchain& swapchain : eye_swapchains_) {
-            for (const auto& image : swapchain.images) {
-                retired.push_back(image.texture);
-            }
-        }
-        if (!aurora_d3d12_forget_stereo_targets(retired.data(), static_cast<uint32_t>(retired.size()))) {
-            DestroySwapchainPair(replacement);
-            return Fail("Aurora could not retire its copies into the old D3D12 eye swapchains");
-        }
-        DestroySwapchainPair(eye_swapchains_);
+        // The compositor may still be reading the old pair (kOpenXRRetiredSwapchainCycles).
+        retired_pairs_.push_back({std::move(eye_swapchains_), kOpenXRRetiredSwapchainCycles});
         eye_swapchains_ = std::move(replacement);
         std::ostringstream message;
         message << "OpenXR D3D12 eyes resized: " << eye_size_[0].width << 'x' << eye_size_[0].height << " / "
@@ -893,6 +885,32 @@ private:
         return true;
     }
 
+    // Destroys the retired pairs whose time is up (every one of them at shutdown), once Aurora's
+    // copies into their images are known complete. A pair Aurora cannot vouch for is left to
+    // xrDestroySession rather than destroyed under a possibly live copy.
+    void ReapRetiredPairs(bool all) {
+        for (auto it = retired_pairs_.begin(); it != retired_pairs_.end();) {
+            if (!all && --it->cycles_left != 0) {
+                ++it;
+                continue;
+            }
+            std::vector<void*> images;
+            for (const EyeSwapchain& swapchain : it->swapchains) {
+                for (const auto& image : swapchain.images) {
+                    images.push_back(image.texture);
+                }
+            }
+            if (aurora_d3d12_forget_stereo_targets(images.data(), static_cast<uint32_t>(images.size()))) {
+                DestroySwapchainPair(it->swapchains);
+            } else {
+                Log(OpenXRLogLevel::Warning,
+                    "Aurora could not retire its copies into a replaced D3D12 eye swapchain pair; "
+                    "deferring its destruction to xrDestroySession");
+            }
+            it = retired_pairs_.erase(it);
+        }
+    }
+
     bool CreateSwapchain(EyeSwapchain& swapchain, uint32_t width, uint32_t height, const char* what) {
         swapchain.width = width;
         swapchain.height = height;
@@ -1058,6 +1076,7 @@ private:
 
     void DestroySwapchains() {
         DestroyPanelSwapchains();
+        ReapRetiredPairs(true);
         DestroySwapchainPair(eye_swapchains_);
         DestroySwapchainPair(retained_swapchains_);
         have_retained_frame_ = false;
@@ -1137,6 +1156,12 @@ private:
     // for, which differ while a size the runtime refused is being kept.
     std::array<OpenXREyeSize, kOpenXREyeCount> eye_size_{};
     std::array<OpenXREyeSize, kOpenXREyeCount> requested_eye_size_{};
+    // Pairs replaced by a new size, destroyed once the compositor has moved on from them.
+    struct RetiredPair {
+        std::array<EyeSwapchain, kOpenXREyeCount> swapchains;
+        uint32_t cycles_left;
+    };
+    std::vector<RetiredPair> retired_pairs_;
     // The settings panel's layer: written like the eyes into panel_swapchain_,
     // shown from retained_panel_swapchain_ (see FinishFrame).
     EyeSwapchain panel_swapchain_{};

runtime/src/vr/openxr_vulkan.cpp

@@ -1848,6 +1848,7 @@ private:
     // writes). Everything new is allocated before anything old goes, so a size that cannot be had
     // leaves the eyes as they were.
     bool ResizeWritablePair() {
+        ReapRetiredPairs(false);
         std::array<OpenXREyeSize, kOpenXREyeCount> current{};
         for (uint32_t eye = 0; eye < kOpenXREyeCount; ++eye) {
             if (eye_swapchains_[eye].acquired) {
@@ -1945,7 +1946,8 @@ private:
                 }
             }
         }
-        DestroySwapchainPair(eye_swapchains_);
+        // The compositor may still be reading the old pair (kOpenXRRetiredSwapchainCycles).
+        retired_pairs_.push_back({std::move(eye_swapchains_), kOpenXRRetiredSwapchainCycles});
         eye_swapchains_ = std::move(replacement);
         std::ostringstream message;
         message << "OpenXR Vulkan eyes resized: " << eye_size_[0].width << 'x' << eye_size_[0].height << " / "
@@ -1957,6 +1959,31 @@ private:
         return true;
     }
 
+    // As the D3D12 backend's. Only this device's copies write the images, so its idle is the
+    // completion the spec asks for before xrDestroySwapchain.
+    void ReapRetiredPairs(bool all) {
+        bool waited = false;
+        for (auto it = retired_pairs_.begin(); it != retired_pairs_.end();) {
+            if (!all && --it->cycles_left != 0) {
+                ++it;
+                continue;
+            }
+            if (!waited && vk_device_ != VK_NULL_HANDLE) {
+                std::lock_guard lock(vk_mutex_);
+                waited = vkDeviceWaitIdle(vk_device_) == VK_SUCCESS;
+                if (!waited) {
+                    Log(OpenXRLogLevel::Warning,
+                        "vkDeviceWaitIdle failed before destroying replaced Vulkan eye swapchains; "
+                        "deferring their destruction to xrDestroySession");
+                }
+            }
+            if (waited || vk_device_ == VK_NULL_HANDLE) {
+                DestroySwapchainPair(it->swapchains);
+            }
+            it = retired_pairs_.erase(it);
+        }
+    }
+
     bool CreateSwapchain(EyeSwapchain& swapchain, uint32_t width, uint32_t height, const char* what) {
         swapchain.width = width;
         swapchain.height = height;
@@ -2115,6 +2142,7 @@ private:
 
     void DestroySwapchains() {
         DestroyPanelSwapchains();
+        ReapRetiredPairs(true);
         DestroySwapchainPair(eye_swapchains_);
         DestroySwapchainPair(retained_swapchains_);
         have_retained_frame_ = false;
@@ -2177,6 +2205,11 @@ private:
     // As the D3D12 backend's.
     std::array<OpenXREyeSize, kOpenXREyeCount> eye_size_{};
     std::array<OpenXREyeSize, kOpenXREyeCount> requested_eye_size_{};
+    struct RetiredPair {
+        std::array<EyeSwapchain, kOpenXREyeCount> swapchains;
+        uint32_t cycles_left;
+    };
+    std::vector<RetiredPair> retired_pairs_;
     // The settings panel's layer: written like the eyes into panel_swapchain_,
     // shown from retained_panel_swapchain_ (see FinishFrame).
     EyeSwapchain panel_swapchain_{};

runtime/src/vr/openxr_vulkan_win32.cpp

@@ -876,6 +876,7 @@ private:
     // As the D3D12 backend's. Aurora's bridge wraps each swapchain VkImage for Dawn, and the
     // runtime may hand the old handles out again, so the wraps go with the swapchains.
     bool ResizeWritablePair() {
+        ReapRetiredPairs(false);
         std::array<OpenXREyeSize, kOpenXREyeCount> current{};
         for (uint32_t eye = 0; eye < kOpenXREyeCount; ++eye) {
             if (eye_swapchains_[eye].acquired) {
@@ -899,17 +900,8 @@ private:
             eye_size_ = current;
             return true;
         }
-        std::vector<void*> retired;
-        for (const EyeSwapchain& swapchain : eye_swapchains_) {
-            for (const auto& image : swapchain.images) {
-                retired.push_back(reinterpret_cast<void*>(image.image));
-            }
-        }
-        if (!aurora_vulkan_win32_forget_targets(retired.data(), static_cast<uint32_t>(retired.size()))) {
-            DestroySwapchainPair(replacement);
-            return Fail("Aurora could not retire its copies into the old Vulkan eye swapchains");
-        }
-        DestroySwapchainPair(eye_swapchains_);
+        // The compositor may still be reading the old pair (kOpenXRRetiredSwapchainCycles).
+        retired_pairs_.push_back({std::move(eye_swapchains_), kOpenXRRetiredSwapchainCycles});
         eye_swapchains_ = std::move(replacement);
         std::ostringstream message;
         message << "OpenXR Vulkan eyes resized: " << eye_size_[0].width << 'x' << eye_size_[0].height << " / "
@@ -918,6 +910,31 @@ private:
         return true;
     }
 
+    // As the D3D12 backend's: Aurora drains Dawn's queue and drops its wraps of the images (which
+    // it does under Dawn's device guard, so before the guard is taken to destroy them).
+    void ReapRetiredPairs(bool all) {
+        for (auto it = retired_pairs_.begin(); it != retired_pairs_.end();) {
+            if (!all && --it->cycles_left != 0) {
+                ++it;
+                continue;
+            }
+            std::vector<void*> images;
+            for (const EyeSwapchain& swapchain : it->swapchains) {
+                for (const auto& image : swapchain.images) {
+                    images.push_back(reinterpret_cast<void*>(image.image));
+                }
+            }
+            if (aurora_vulkan_win32_forget_targets(images.data(), static_cast<uint32_t>(images.size()))) {
+                DestroySwapchainPair(it->swapchains);
+            } else {
+                Log(OpenXRLogLevel::Warning,
+                    "Aurora could not retire its copies into a replaced Vulkan eye swapchain pair; "
+                    "deferring its destruction to xrDestroySession");
+            }
+            it = retired_pairs_.erase(it);
+        }
+    }
+
     bool CreateSwapchain(EyeSwapchain& swapchain, uint32_t width, uint32_t height, const char* what) {
         swapchain.width = width;
         swapchain.height = height;
@@ -936,7 +953,14 @@ private:
         create.faceCount = 1;
         create.arraySize = 1;
         create.mipCount = 1;
-        XrResult result = xrCreateSwapchain(runtime_->Session(), &create, &swapchain.handle);
+        XrResult result;
+        {
+            // The spec keeps the runtime off the VkQueue here, but mid-session (the panel's
+            // swapchains, a new render resolution) Dawn's worker is submitting on it, and a
+            // runtime that transitions its new images would race that; the guard costs nothing.
+            const auto queue_guard = runtime_->LockGraphicsQueue();
+            result = xrCreateSwapchain(runtime_->Session(), &create, &swapchain.handle);
+        }
         ObserveResult(result);
         if (XR_FAILED(result)) {
             std::ostringstream message;
@@ -1075,6 +1099,7 @@ private:
     void DestroyPanelSwapchains() {
         for (auto* swapchain : {&panel_swapchain_, &retained_panel_swapchain_}) {
             if (swapchain->handle != XR_NULL_HANDLE && !swapchain->acquired) {
+                const auto queue_guard = runtime_ ? runtime_->LockGraphicsQueue() : OpenXRRuntime::GraphicsQueueGuard{nullptr, nullptr};
                 xrDestroySwapchain(swapchain->handle);
             } else if (swapchain->acquired) {
                 Log(OpenXRLogLevel::Warning,
@@ -1088,6 +1113,7 @@ private:
 
     void DestroySwapchains() {
         DestroyPanelSwapchains();
+        ReapRetiredPairs(true);
         DestroySwapchainPair(eye_swapchains_);
         DestroySwapchainPair(retained_swapchains_);
         have_retained_frame_ = false;
@@ -1098,6 +1124,7 @@ private:
     void DestroySwapchainPair(std::array<EyeSwapchain, kOpenXREyeCount>& pair) {
         for (auto& swapchain : pair) {
             if (swapchain.handle != XR_NULL_HANDLE && !swapchain.acquired) {
+                const auto queue_guard = runtime_ ? runtime_->LockGraphicsQueue() : OpenXRRuntime::GraphicsQueueGuard{nullptr, nullptr};
                 xrDestroySwapchain(swapchain.handle);
             } else if (swapchain.acquired) {
                 Log(OpenXRLogLevel::Warning,
@@ -1173,6 +1200,11 @@ private:
     // As the D3D12 backend's.
     std::array<OpenXREyeSize, kOpenXREyeCount> eye_size_{};
     std::array<OpenXREyeSize, kOpenXREyeCount> requested_eye_size_{};
+    struct RetiredPair {
+        std::array<EyeSwapchain, kOpenXREyeCount> swapchains;
+        uint32_t cycles_left;
+    };
+    std::vector<RetiredPair> retired_pairs_;
     // The settings panel's layer: written like the eyes into panel_swapchain_,
     // shown from retained_panel_swapchain_ (see FinishFrame).
     EyeSwapchain panel_swapchain_{};