Stabilize raw overlay pointer rendering and trace click delivery

This commit is contained in:
baketnk committed 2026-09-24 13:12:53 -04:00
1 parent 9b4bf9f047
commit 9e608193e3
6 files changed
+49 -36

No files matched your search

+5 -3
View File
@@ -24,9 +24,11 @@ preview, status and control surfaces use independently rasterized antialiased ed
and restrained baked neon halos rather than GPU bloom. The recording indicator and and restrained baked neon halos rather than GPU bloom. The recording indicator and
selected controls remain distinguishable by their labels, not color alone. Rounded selected controls remain distinguishable by their labels, not color alone. Rounded
control hit areas exclude their clipped corners. control hit areas exclude their clipped corners.
Rendering/uploads occur only for changed content, page, settings, or pointer Rendering/uploads occur only for changed content, page or settings; laser hover
feedback; static frames are reused. The caller may call `draw(Panel)` at 10 ms and button down/up are hit-tested without a raw-texture upload, to reduce
intervals. Tracking transforms do not require repainting the canvas. compositor flicker reported during hover. Static frames are reused.
The caller may call `draw(Panel)` at 10 ms intervals. Tracking transforms do
not require repainting the canvas.
The complete transcript preview is paginated by glyph width and four-line The complete transcript preview is paginated by glyph width and four-line
height; Previous and Next navigate it without changing the source transcript. height; Previous and Next navigate it without changing the source transcript.
+5 -3
View File
@@ -89,6 +89,7 @@ int main(int argc, char** argv) {
const auto end = std::chrono::steady_clock::now() + std::chrono::seconds(30); const auto end = std::chrono::steady_clock::now() + std::chrono::seconds(30);
std::string status; std::string status;
bool held = false; bool held = false;
bool quit_check = false;
std::string last_action = "Press panel controls or tap then hold right grip. No mic or typing."; std::string last_action = "Press panel controls or tap then hold right grip. No mic or typing.";
auto action_name = [](frameyap::UiAction action) { auto action_name = [](frameyap::UiAction action) {
switch (action) { switch (action) {
@@ -103,9 +104,9 @@ int main(int argc, char** argv) {
} }
return "Unknown"; return "Unknown";
}; };
while (std::chrono::steady_clock::now() < end) { while (!quit_check && std::chrono::steady_clock::now() < end) {
for (auto action : overlay.poll()) { for (auto action : overlay.poll()) {
if (action == frameyap::UiAction::Quit) return 0; if (action == frameyap::UiAction::Quit) { quit_check = true; break; }
if (action == frameyap::UiAction::BeginRecord) held = true; if (action == frameyap::UiAction::BeginRecord) held = true;
if (action == frameyap::UiAction::EndRecord || action == frameyap::UiAction::Cancel) held = false; if (action == frameyap::UiAction::EndRecord || action == frameyap::UiAction::Cancel) held = false;
last_action = std::string(action_name(action)) + " received; diagnostic only."; last_action = std::string(action_name(action)) + " received; diagnostic only.";
@@ -113,10 +114,11 @@ int main(int argc, char** argv) {
} }
auto next = overlay.controls_status(); auto next = overlay.controls_status();
if (next != status) { status = next; std::cout << status << std::endl; } if (next != status) { status = next; std::cout << status << std::endl; }
overlay.draw({"Controls ONLY - " + std::string(held ? "HOLD recognized" : "no microphone or typing"), overlay.draw({overlay.pointer_status(),
status, last_action, true, held}); status, last_action, true, held});
std::this_thread::sleep_for(std::chrono::milliseconds(10)); std::this_thread::sleep_for(std::chrono::milliseconds(10));
} }
std::cout << "Final " << overlay.pointer_status() << std::endl;
} else { } else {
const auto end = std::chrono::steady_clock::now() + std::chrono::seconds(5); const auto end = std::chrono::steady_clock::now() + std::chrono::seconds(5);
while (std::chrono::steady_clock::now() < end) { while (std::chrono::steady_clock::now() < end) {
+19 -6
View File
@@ -66,6 +66,8 @@ struct Overlay::Impl {
bool focus = true; bool focus = true;
bool persist_mount = true; bool persist_mount = true;
vr::EVRInputError action_update_error = vr::VRInputError_None; vr::EVRInputError action_update_error = vr::VRInputError_None;
unsigned pointer_downs = 0, pointer_ups = 0, pointer_actions = 0, pointer_resets = 0;
std::string last_pointer_event = "none";
Impl(const std::string& assets, const std::string& font, std::optional<Mount> requested, bool persist) Impl(const std::string& assets, const std::string& font, std::optional<Mount> requested, bool persist)
: settings_path(default_mount_settings_path()), : settings_path(default_mount_settings_path()),
@@ -236,22 +238,28 @@ struct Overlay::Impl {
case vr::VREvent_OverlayShown: case vr::VREvent_OverlayShown:
focus = true; reset_input(result); break; focus = true; reset_input(result); break;
case vr::VREvent_OverlayGamepadFocusLost: case vr::VREvent_OverlayGamepadFocusLost:
surface.reset_pointers(); ++pointer_resets;
last_pointer_event = "gamepad focus lost"; break;
case vr::VREvent_OverlayFocusChanged: case vr::VREvent_OverlayFocusChanged:
// Dashboard laser/gamepad focus can change between the two grip // This global focus notification also fires when the dashboard
// squeezes. It is not action activity or controller tracking loss: // laser enters our overlay. Do not erase a press between down/up;
// invalidate pointer presses, but leave a physical grip gesture // a release must still hit the same enabled control.
// armed. Hidden overlays and true OpenVR input capture still reset. last_pointer_event = "overlay focus changed"; break;
surface.reset_pointers(); break;
case vr::VREvent_MouseMove: case vr::VREvent_MouseMove:
surface.pointer_move(event.data.mouse.cursorIndex, event.data.mouse.x, H - event.data.mouse.y); surface.pointer_move(event.data.mouse.cursorIndex, event.data.mouse.x, H - event.data.mouse.y);
break; break;
case vr::VREvent_MouseButtonDown: case vr::VREvent_MouseButtonDown:
++pointer_downs;
last_pointer_event = "down button=" + std::to_string(event.data.mouse.button);
if (event.data.mouse.button == vr::VRMouseButton_Left) if (event.data.mouse.button == vr::VRMouseButton_Left)
surface.pointer_down(event.data.mouse.cursorIndex, event.data.mouse.x, H - event.data.mouse.y); surface.pointer_down(event.data.mouse.cursorIndex, event.data.mouse.x, H - event.data.mouse.y);
break; break;
case vr::VREvent_MouseButtonUp: case vr::VREvent_MouseButtonUp:
++pointer_ups;
last_pointer_event = "up button=" + std::to_string(event.data.mouse.button);
if (event.data.mouse.button == vr::VRMouseButton_Left) { if (event.data.mouse.button == vr::VRMouseButton_Left) {
auto event_result = surface.pointer_up(event.data.mouse.cursorIndex, event.data.mouse.x, H - event.data.mouse.y); auto event_result = surface.pointer_up(event.data.mouse.cursorIndex, event.data.mouse.x, H - event.data.mouse.y);
if (event_result.action || event_result.mount || event_result.recenter) ++pointer_actions;
if (event_result.action) result.push_back(*event_result.action); if (event_result.action) result.push_back(*event_result.action);
if (event_result.mount) { if (event_result.mount) {
mount = *event_result.mount; mount = *event_result.mount;
@@ -328,13 +336,18 @@ std::string Overlay::controls_status() {
vr::VRControllerState_t raw{}; vr::VRControllerState_t raw{};
const bool raw_ok = device < impl_->poses.size() && impl_->system->GetControllerState(device, &raw, sizeof(raw)); const bool raw_ok = device < impl_->poses.size() && impl_->system->GetControllerState(device, &raw, sizeof(raw));
result += std::string(i == 0 ? "\nLeft:" : "\nRight:") + result += std::string(i == 0 ? "\nLeft:" : "\nRight:") +
" err=" + std::to_string(int(error)) + " bound=" + (data.bActive ? "Y" : "N") + " err=" + std::to_string(int(error)) + " active=" + (data.bActive ? "Y" : "N") +
" down=" + (data.bState ? "Y" : "N") + " pose=" + (tracked ? "Y" : "N") + " down=" + (data.bState ? "Y" : "N") + " pose=" + (tracked ? "Y" : "N") +
" origin=" + (origin_ok ? "Y" : "N") + " origin=" + (origin_ok ? "Y" : "N") +
" raw=" + (raw_ok ? ((raw.ulButtonPressed & vr::ButtonMaskFromId(vr::k_EButton_Grip)) ? "down" : "up") : "n/a"); " raw=" + (raw_ok ? ((raw.ulButtonPressed & vr::ButtonMaskFromId(vr::k_EButton_Grip)) ? "down" : "up") : "n/a");
} }
return result; return result;
} }
std::string Overlay::pointer_status() const {
return "Pointer down=" + std::to_string(impl_->pointer_downs) + " up=" + std::to_string(impl_->pointer_ups) +
" hits=" + std::to_string(impl_->pointer_actions) + " resets=" + std::to_string(impl_->pointer_resets) +
" last=" + impl_->last_pointer_event;
}
int registration(const std::string& manifest, bool remove, bool autostart) { int registration(const std::string& manifest, bool remove, bool autostart) {
try { try {
+1
View File
@@ -25,6 +25,7 @@ public:
std::vector<UiAction> poll(); std::vector<UiAction> poll();
void draw(const Panel& panel); void draw(const Panel& panel);
std::string controls_status(); // diagnostic only, no input delivery std::string controls_status(); // diagnostic only, no input delivery
std::string pointer_status() const; // diagnostic counters, no input delivery
private: private:
struct Impl; struct Impl;
std::unique_ptr<Impl> impl_; std::unique_ptr<Impl> impl_;
+13 -22
View File
@@ -103,7 +103,7 @@ struct PanelSurface::Impl {
Mount mount; Mount mount;
bool settings = false, dirty = true; bool settings = false, dirty = true;
std::string placement_note; std::string placement_note;
std::array<int, 2> pressed{{-1, -1}}, hovered{{-1, -1}}; std::array<int, 2> pressed{{-1, -1}};
std::vector<std::string> lines; std::vector<std::string> lines;
size_t page = 0; size_t page = 0;
static constexpr size_t lines_per_page = 4; static constexpr size_t lines_per_page = 4;
@@ -241,9 +241,7 @@ struct PanelSurface::Impl {
return -1; return -1;
} }
void reset() { void reset() {
if (std::any_of(pressed.begin(), pressed.end(), [](int i) { return i >= 0; }) || pressed.fill(-1);
std::any_of(hovered.begin(), hovered.end(), [](int i) { return i >= 0; })) dirty = true;
pressed.fill(-1); hovered.fill(-1);
} }
bool render(const Panel& p) { bool render(const Panel& p) {
if (p.recording != panel.recording || p.enabled != panel.enabled || if (p.recording != panel.recording || p.enabled != panel.enabled ||
@@ -296,16 +294,14 @@ struct PanelSurface::Impl {
bool on = enabled(b.id); bool on = enabled(b.id);
bool selected = (b.id == Control::Review && !settings) || (b.id == Control::Settings && settings) || bool selected = (b.id == Control::Review && !settings) || (b.id == Control::Settings && settings) ||
(mounting(b.id) && *mounting(b.id) == mount); (mounting(b.id) && *mounting(b.id) == mount);
bool hover = std::find(hovered.begin(), hovered.end(), int(i)) != hovered.end(); const Color fill = !on ? Color{17, 23, 33, 255} :
bool down = std::find(pressed.begin(), pressed.end(), int(i)) != pressed.end(); selected ? Color{26, 55, 68, 255} : card;
const Color fill = !on ? Color{17, 23, 33, 255} : down ? Color{35, 77, 88, 255} :
hover || selected ? Color{26, 55, 68, 255} : card;
const Color accent = b.id == Control::Record && panel.recording ? pink : cyan; const Color accent = b.id == Control::Record && panel.recording ? pink : cyan;
const bool highlighted = on && (hover || selected || b.id == Control::Record || const bool highlighted = on && (selected || b.id == Control::Record ||
(b.id == Control::Insert && panel.transcript.size())); (b.id == Control::Insert && panel.transcript.size()));
rounded(b.r, std::min(16, b.r.h / 3), fill, rounded(b.r, std::min(16, b.r.h / 3), fill,
!on ? Color{35, 46, 59, 255} : highlighted ? accent : Color{67, 93, 112, 255}, !on ? Color{35, 46, 59, 255} : highlighted ? accent : Color{67, 93, 112, 255},
highlighted ? (hover || down ? .40f : .23f) : 0.f, highlighted ? 2 : 1); highlighted ? .23f : 0.f, highlighted ? 2 : 1);
const auto label = b.id == Control::Record && panel.recording ? "Stop" : b.label; const auto label = b.id == Control::Record && panel.recording ? "Stop" : b.label;
text(label, b.r.x + 16, b.r.y + b.r.h / 2 + 9, 27, on ? ink : Color{81, 96, 113, 255}, b.r.x + b.r.w - 8); text(label, b.r.x + 16, b.r.y + b.r.h / 2 + 9, 27, on ? ink : Color{81, 96, 113, 255}, b.r.x + b.r.w - 8);
if (mounting(b.id) && selected) text("ON", b.r.x + b.r.w - 56, b.r.y + 38, 23, cyan, b.r.x + b.r.w - 12); if (mounting(b.id) && selected) text("ON", b.r.x + b.r.w - 56, b.r.y + 38, 23, cyan, b.r.x + b.r.w - 12);
@@ -319,32 +315,27 @@ PanelSurface::~PanelSurface() = default;
bool PanelSurface::render(const Panel& p) { return impl_->render(p); } bool PanelSurface::render(const Panel& p) { return impl_->render(p); }
const std::vector<unsigned char>& PanelSurface::pixels() const { return impl_->pixels; } const std::vector<unsigned char>& PanelSurface::pixels() const { return impl_->pixels; }
bool PanelSurface::available(UiAction a) const { return impl_->available(a); } bool PanelSurface::available(UiAction a) const { return impl_->available(a); }
void PanelSurface::pointer_move(unsigned cursor, float x, float y) { void PanelSurface::pointer_move(unsigned, float, float) {
if (cursor >= impl_->hovered.size()) return; // Hit-test on down/up only. SetOverlayRaw can flicker in SteamVR when each
const int next = impl_->hit(x, y); // laser hover frame causes another full RGBA upload.
if (next != impl_->hovered[cursor]) { impl_->hovered[cursor] = next; impl_->dirty = true; }
} }
void PanelSurface::pointer_down(unsigned cursor, float x, float y) { void PanelSurface::pointer_down(unsigned cursor, float x, float y) {
if (cursor >= impl_->pressed.size()) return; if (cursor >= impl_->pressed.size()) return;
impl_->pressed[cursor] = impl_->hit(x, y); impl_->pressed[cursor] = impl_->hit(x, y);
pointer_move(cursor, x, y);
impl_->dirty = true;
} }
SurfaceEvent PanelSurface::pointer_up(unsigned cursor, float x, float y) { SurfaceEvent PanelSurface::pointer_up(unsigned cursor, float x, float y) {
SurfaceEvent result; SurfaceEvent result;
if (cursor >= impl_->pressed.size()) return result; if (cursor >= impl_->pressed.size()) return result;
int index = std::exchange(impl_->pressed[cursor], -1); int index = std::exchange(impl_->pressed[cursor], -1);
pointer_move(cursor, x, y);
impl_->dirty = true;
if (index < 0 || impl_->hit(x, y) != index) return result; if (index < 0 || impl_->hit(x, y) != index) return result;
auto c = buttons[index].id; auto c = buttons[index].id;
if (c == Control::Record) result.action = impl_->panel.recording ? UiAction::EndRecord : UiAction::BeginRecord; if (c == Control::Record) result.action = impl_->panel.recording ? UiAction::EndRecord : UiAction::BeginRecord;
else if (auto a = action(c)) result.action = a; else if (auto a = action(c)) result.action = a;
else if (auto m = mounting(c)) { impl_->mount = *m; result.mount = *m; impl_->reset(); } else if (auto m = mounting(c)) { impl_->mount = *m; result.mount = *m; impl_->reset(); impl_->dirty = true; }
else if (c == Control::Recenter) { result.recenter = true; impl_->reset(); } else if (c == Control::Recenter) { result.recenter = true; impl_->reset(); }
else if (c == Control::Review || c == Control::Settings) { impl_->settings = c == Control::Settings; impl_->reset(); } else if (c == Control::Review || c == Control::Settings) { impl_->settings = c == Control::Settings; impl_->reset(); impl_->dirty = true; }
else if (c == Control::Prev) --impl_->page; else if (c == Control::Prev) { --impl_->page; impl_->dirty = true; }
else if (c == Control::Next) ++impl_->page; else if (c == Control::Next) { ++impl_->page; impl_->dirty = true; }
return result; return result;
} }
void PanelSurface::reset_pointers() { impl_->reset(); } void PanelSurface::reset_pointers() { impl_->reset(); }
+6 -2
View File
@@ -131,8 +131,12 @@ int main(int argc, char** argv) {
assert(surface.pixels() == replaced); assert(surface.pixels() == replaced);
p.status = std::string(4096, 's'); p.detail = std::string(4096, 'd'); p.status = std::string(4096, 's'); p.detail = std::string(4096, 'd');
assert(surface.render(p)); assert(!surface.render(p)); assert(surface.render(p)); assert(!surface.render(p));
surface.pointer_move(0, 900, 610); assert(surface.render(p)); // Laser motion and a press do not re-upload raw pixels. A completed
// action still reaches the caller; meaningful panel changes redraw.
surface.pointer_move(0, 900, 610); assert(!surface.render(p)); surface.pointer_move(0, 900, 610); assert(!surface.render(p));
assert(click(surface, 900, 610).action == UiAction::Quit); surface.pointer_move(0, 900, 610); assert(!surface.render(p));
surface.pointer_down(0, 900, 610); assert(!surface.render(p));
assert(surface.pointer_up(0, 900, 610).action == UiAction::Quit);
assert(!surface.render(p));
std::cout << "panel checks passed (no OpenVR, microphone or input injection)\n"; std::cout << "panel checks passed (no OpenVR, microphone or input injection)\n";
} }