From 2d985573c05ac4e7a14ef52b287c09433e65f438 Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Wed, 30 Sep 2026 21:11:40 -0600 Subject: [PATCH] Gaze first: abandoned; keep the controller experiment for reference The last state of the experiment, kept on this branch only: - input/gazefirst.py, input/steamui.py: the relay reading the controllers from vrserver's web socket, switching the compositor binding, and driving the Steam UI filter (whose block now expires unless renewed). - The helper's trigger and bumper holds (steered by the controller's position), the laser keeper, and the click gate that waits for the laser. - pointer/probe/focustest (overlay flag 1 << 4, as an input client too), vrsetting (SteamVR settings through vrserver), lasertest --reclaim. Every press and release Steam sees takes SteamVR out of laser mode, and taking the laser back breaks clicks and drags. See docs/gaze-controllers.md on experimental. Co-Authored-By: Claude Opus 5.5 --- docs/gaze-first.md | 2 + input/gazefirst.py | 306 +++++++++ input/input-relay.py | 41 +- input/steam-gamepad-filter.js | 5 +- input/steamui.py | 40 ++ input/vrws.py | 22 +- ...compositor_frame_controller_gazefirst.json | 646 +++++------------- pointer/helper/ft-pointer.cpp | 187 ++++- pointer/probe/build.sh | 8 +- pointer/probe/focustest.cpp | 135 ++++ pointer/probe/focustest_actions/actions.json | 15 + .../bindings_frame_controller.json | 15 + pointer/probe/lasertest.cpp | 25 +- pointer/probe/vrsetting.cpp | 48 ++ 14 files changed, 1000 insertions(+), 495 deletions(-) create mode 100644 input/gazefirst.py create mode 100644 input/steamui.py create mode 100644 pointer/probe/focustest.cpp create mode 100644 pointer/probe/focustest_actions/actions.json create mode 100644 pointer/probe/focustest_actions/bindings_frame_controller.json create mode 100644 pointer/probe/vrsetting.cpp diff --git a/docs/gaze-first.md b/docs/gaze-first.md index f6f23f1..7da2dad 100644 --- a/docs/gaze-first.md +++ b/docs/gaze-first.md @@ -1,5 +1,7 @@ # Gaze first: plan +**Abandoned on 2026-09-30.** Steam reads the Frame controllers itself, outside SteamVR's bindings, and every press and release it sees takes SteamVR out of laser mode. Gaze mode stays a mouse feature; `docs/gaze-controllers.md` on `experimental` explains what was tried and why it doesn't work. This branch keeps the plan, the probes, and the code for reference; it isn't merged. + Gaze first makes the eyes the pointer for everything flat in VR, and the controllers its buttons. With gaze on and no game running, the gaze drives Frametop's 3D pointer (the `ft_pointer` driver) everywhere the mouse can go, SteamVR's dashboard included. Either controller's trigger clicks where you look, and the controllers stop showing lasers. The work happens on branch `gaze-first` (worktree `~/frametop/.worktrees/gaze-first`, from `experimental`) and is merged into `experimental` after it's been tested in the headset. The decisions below were made with the user on 2026-09-30. Nothing is built yet: the first step is a headset session with four tests (see "Tests before building"). diff --git a/input/gazefirst.py b/input/gazefirst.py new file mode 100644 index 0000000..de63636 --- /dev/null +++ b/input/gazefirst.py @@ -0,0 +1,306 @@ +"""Gaze first, the relay's part (docs/gaze-first.md). + +While the pointer helper says gaze first is on ("gazefirst 1" every 5 s: gaze mode, no game, +the headset on; "gazefirst 0" or 12 s of silence ends it), the Frame controllers belong to the +gaze: + +- SteamVR: the Frame controller's compositor binding is + pointer/bindings/vrcompositor_frame_controller_gazefirst.json (only the Steam button left), + chosen through vrserver's /input/selectconfig.action, and the stock one again after. +- Steam's UI: input/steam-gamepad-filter.js runs in Steam's SharedJSContext (input/steamui.py) + and drops the controllers' gamepad input (the Steam button still passes). The block lasts 15 s + unless it's renewed, which happens every 5 s, so it ends by itself if the relay dies; and + Steam reloads its UI with SteamVR, which the renewal also covers. +- The trigger and bumper, read from vrserver's web socket (input/vrws.py, which takes nothing + from anyone), go to the helper as "ctrl 1|0"; the right + thumbstick scrolls ("scroll ", as the mouse's wheel). + +Steam still sees the controllers, and SteamVR leaves laser mode after each press and release: +the helper takes the laser back (see "Gaze first" in pointer/helper/ft-pointer.cpp). + +The toggle macro works with gaze first on or off: both thumbstick clicks held 1 s turn gaze mode +on or off, and POINTER_GAZE in ~/.config/frametop.conf remembers it. So does `toggle()`, which +the relay's gaze_toggle action uses. + +The binding, the filter, and finding new controllers run on a thread of their own: they're HTTP +and web socket round trips that mustn't hold up the relay's mouse and keyboard. +""" +import json +import os +import threading +import time +import urllib.request + +import steamui +from vrws import HEADERS, ORIGIN, VrSocket, getstate + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +FILTER_JS = os.path.join(ROOT, "input", "steam-gamepad-filter.js") +GAZE_BINDING = "file://" + os.path.join(ROOT, "pointer", "bindings", "vrcompositor_frame_controller_gazefirst.json") +STOCK_BINDING = "file:///opt/steamvr/drivers/frame_controller/resources/input/vrcompositor_bindings_frame_controller.json" +COMPOSITOR = "openvr.component.vrcompositor" +CONF = os.path.expanduser("~/.config/frametop.conf") + +BUTTONS = {"/input/trigger/click": "trigger", "/input/bumper/click": "bumper"} +STICK_CLICK = "/input/thumbstick/click" +MACRO_HOLD = 1.0 # seconds both thumbstick clicks are held to toggle gaze mode +SCROLL_DEADZONE = 0.3 +ON_FOR = 12.0 # "gazefirst 1" lasts this long without another +RENEW = 5.0 # the binding and filter are checked (and the filter's block renewed) this often +BLOCK_FOR = 15000 # ms the filter blocks without a renewal + + +def select_binding(url): + body = json.dumps({"app_key": COMPOSITOR, "controller_type": "frame_controller", "url": url}).encode() + req = urllib.request.Request(f"{ORIGIN}/input/selectconfig.action", data=body, + headers={**HEADERS, "Origin": ORIGIN, "Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=2) as resp: + if not json.load(resp).get("success"): + raise OSError("vrserver refused the binding") + + +def current_binding(): + req = urllib.request.Request(f"{ORIGIN}/input/getactions.json?app_key={COMPOSITOR}", headers=HEADERS) + with urllib.request.urlopen(req, timeout=2) as resp: + return (json.load(resp).get("current_binding_url") or {}).get("frame_controller", "") + + +def read_gaze(path=CONF): + try: + with open(path) as f: + for line in f: + line = line.split("#", 1)[0].strip() + if line.split("=", 1)[0].strip() == "POINTER_GAZE" and "=" in line: + return line.split("=", 1)[1].strip() not in ("", "0") + except OSError: + pass + return False + + +def write_gaze(on, path=CONF): + """POINTER_GAZE=1|0 in the config, keeping every other line as it is.""" + try: + with open(path) as f: + lines = f.read().splitlines() + except OSError: + lines = [] + value = f"POINTER_GAZE={1 if on else 0}" + for i, line in enumerate(lines): + if line.split("#", 1)[0].split("=", 1)[0].strip() == "POINTER_GAZE": + lines[i] = value + break + else: + lines.append(value) + tmp = path + ".tmp" + with open(tmp, "w") as f: + f.write("\n".join(lines) + "\n") + os.replace(tmp, path) + + +class GazeFirst: + def __init__(self, send, log): + self.send = send # a command to the pointer helper + self.log = log + self.on = False + self.on_until = 0.0 + self.ws = None + self.next_connect = 0.0 + self.sides = {} # subscribed device path -> "left" or "right" + self.values = {} # (side, component) -> last value + self.held = set() # (side, button) the helper has as pressed + self.macro_since = None + self.macro_fired = False + self.scrolling = (0.0, 0.0) + # The worker thread's: a wake-up, the devices it found, what it last applied. + self.lock = threading.Lock() + self.wake = threading.Event() + self.found = [] + self.applied = None + self.problems = {} + threading.Thread(target=self.work, daemon=True, name="gazefirst").start() + + # The relay's select loop: readable while connected to vrserver. + def fileno(self): + return self.ws.fileno() + + def message(self, words, now): + """"gazefirst 1|0" from the helper.""" + on = words[1] == "1" + self.on_until = now + ON_FOR if on else 0.0 + if on != self.on: + self.set_on(on) + + def set_on(self, on): + self.on = on + if not on: + self.release_all(cancel=True) + self.log(f"gaze first {'on' if on else 'off'}") + self.wake.set() + + def release_all(self, cancel=False): + """Let go of what the helper holds: as releases (a click), or cancelled (no click).""" + if cancel and self.held: + self.send("ctrlcancel") + else: + for side, button in sorted(self.held): + self.send(f"ctrl {side} {button} 0") + self.held.clear() + if self.scrolling != (0.0, 0.0): + self.send("scroll 0 0") + self.scrolling = (0.0, 0.0) + + def toggle(self): + """Gaze mode on or off, remembered (POINTER_GAZE).""" + on = not read_gaze() + try: + write_gaze(on) + except OSError as e: + self.log(f"gaze mode: can't write {CONF}: {e}") + self.send(f"gaze {'on' if on else 'off'}") + self.log(f"gaze mode {'on' if on else 'off'}") + + def timeout(self): + return 0.05 if self.macro_since is not None and not self.macro_fired else 0.5 + + def tick(self, now): + if self.on and now > self.on_until: + self.set_on(False) # the helper went quiet + if self.ws is None and now >= self.next_connect: + self.next_connect = now + 3.0 + try: + self.ws = VrSocket(timeout=1) + self.ws.open(f"frametop_relay_{os.getpid()}") + self.sides = {} + self.wake.set() # find the controllers now + except OSError: + self.ws = None + with self.lock: + found, self.found = self.found, [] + for path, side in found: + if self.ws and self.sides.get(path) != side: + try: + self.ws.subscribe(path) + self.sides[path] = side + except OSError: + self.drop() + if self.macro_since is not None and not self.macro_fired and now - self.macro_since >= MACRO_HOLD: + self.macro_fired = True + self.release_all(cancel=True) # a press in progress doesn't click + self.toggle() + + def drop(self): + if self.ws: + try: + self.ws.close() + except OSError: + pass + self.ws = None + self.release_all(cancel=True) + + def readable(self, now): + try: + while True: + msg = self.ws.recv(timeout=0) + if isinstance(msg, dict) and msg.get("type") == "update_component_states": + side = self.sides.get(msg.get("device")) + if side: + self.update(side, msg.get("components") or {}, now) + if not self.ws.pending(): + return # the rest, if any, wakes select again + except OSError: + self.drop() + + def update(self, side, components, now): + for name, value in components.items(): + key = (side, name) + if self.values.get(key) == value: + continue + self.values[key] = value + button = BUTTONS.get(name) + if button: + down = bool(value) + if down and self.on and (side, button) not in self.held: + self.held.add((side, button)) + self.send(f"ctrl {side} {button} 1") + elif not down and (side, button) in self.held: + self.held.discard((side, button)) + self.send(f"ctrl {side} {button} 0") + elif name == STICK_CLICK: + both = self.values.get(("left", STICK_CLICK)) and self.values.get(("right", STICK_CLICK)) + if both and self.macro_since is None: + self.macro_since, self.macro_fired = now, False + elif not both: + self.macro_since = None + elif side == "right" and name in ("/input/thumbstick/x", "/input/thumbstick/y"): + self.scroll() + + def scroll(self): + def shape(v): + v = float(v or 0) + if abs(v) < SCROLL_DEADZONE: + return 0.0 + return round((abs(v) - SCROLL_DEADZONE) / (1 - SCROLL_DEADZONE) * (1 if v > 0 else -1), 1) + want = (shape(self.values.get(("right", "/input/thumbstick/x"))), + shape(self.values.get(("right", "/input/thumbstick/y")))) if self.on else (0.0, 0.0) + if want != self.scrolling: + self.scrolling = want + self.send(f"scroll {want[0]:.1f} {want[1]:.1f}") + + def shutdown(self): + """The relay is stopping: the controllers go back to SteamVR and Steam.""" + self.on = False + self.release_all(cancel=True) + try: + select_binding(STOCK_BINDING) + except OSError: + pass + try: + steamui.evaluate('window.__frametopGaze && (window.__frametopGaze.mode = "off")', timeout=1) + except (OSError, RuntimeError): + pass + + # The worker thread. + def problem(self, what, error=None): + """Log a failure once, and when it's over (error None).""" + if error is None: + if self.problems.pop(what, None) is not None: + self.log(f"gaze first: {what} works again") + return + if self.problems.get(what) != str(error): + self.problems[what] = str(error) + self.log(f"gaze first: {what}: {error}") + + def work(self): + while True: + self.wake.wait(RENEW) + self.wake.clear() + on = self.on + try: + found = [(d["root_path"], d.get("side") or d["root_path"].rsplit("/", 1)[-1]) + for d in getstate(timeout=1) if d.get("controller_type") == "frame_controller"] + with self.lock: + self.found = [(p, s) for p, s in found if s in ("left", "right")] + self.problem("vrserver") + except (OSError, ValueError) as e: + self.problem("vrserver", e) + continue + try: + want = GAZE_BINDING if on else STOCK_BINDING + if current_binding() != want: + select_binding(want) + self.log(f"gaze first: {'gaze' if on else 'stock'} controller binding") + self.problem("binding") + except (OSError, ValueError) as e: + self.problem("binding", e) + try: + if on: + with open(FILTER_JS) as f: + steamui.evaluate(f.read()) + steamui.evaluate('(() => { const G = window.__frametopGaze; G.mode = "block"; ' + f'G.until = Date.now() + {BLOCK_FOR}; return G.mode; }})()') + elif self.applied is not False: + steamui.evaluate('window.__frametopGaze && (window.__frametopGaze.mode = "off")') + self.applied = on + self.problem("Steam's UI") + except (OSError, RuntimeError, ValueError) as e: + self.problem("Steam's UI", e) diff --git a/input/input-relay.py b/input/input-relay.py index 6a34b9b..b995624 100755 --- a/input/input-relay.py +++ b/input/input-relay.py @@ -19,8 +19,9 @@ keyboard node for its extra buttons). Roles, from ~/.config/frametop-input.json ignore not grabbed, only observed for identification in the settings app Buttons and keys of pointer devices go through a per-device map to actions (left, right, middle, back, scroll_up, scroll_down, dashboard, recenter, -pointer_toggle, follow_toggle = head follow on or off, gaze_toggle = gaze mode on or off -(the pointer goes where you look; see pointer/helper/ft-pointer.cpp), gaze_precision = while +pointer_toggle, follow_toggle = head follow on or off, gaze_toggle = gaze mode on or off, +remembered as POINTER_GAZE (the pointer goes where you look; see pointer/helper/ft-pointer.cpp +and input/gazefirst.py), gaze_precision = while held, the pointer stops where you look and the button's device (the mouse, or that controller's aim) steers it, and the release clicks there, gaze_drag = the same, but pressed at once, so it drags ("precision|gazedrag mouse|left|right|keyboard 1|0" to the helper), sens_up, sens_down, @@ -84,6 +85,7 @@ Control socket (abstract datagram @frametop_relay, JSON replies to the sender): vrcapture take every controller button for s seconds (0: stop), so the settings app can capture one; watchers see them as events with id frame_controller vrbtn, vrhello, gazeawake from the pointer helper (above) + gazefirst 1|0 from the pointer helper: gaze first on or off (input/gazefirst.py) textfield 1|0 from the desktop's input method (above) Runs on the Frame host as a user service (frametop-input-relay.service). The @@ -110,6 +112,8 @@ import subprocess import sys import time +import gazefirst + # Linux input constants (include/uapi/linux/input-event-codes.h, input.h, uinput.h). EV_SYN, EV_KEY, EV_REL, EV_MSC = 0x00, 0x01, 0x02, 0x04 SYN_REPORT = 0 @@ -465,9 +469,6 @@ class Pointer: elif name == "follow_toggle": self.send("follow toggle") # until the next restart; the setting is POINTER_FOLLOW log("head follow toggled") - elif name == "gaze_toggle": - self.send("gaze toggle") # until the next restart; the setting is POINTER_GAZE - log("gaze mode toggled") elif name == "screens_toggle": try: self.sock.sendto(b"toggle", SCREENS) @@ -658,6 +659,8 @@ def main(): reply(addr, {"t": "event", "id": VR_DEVICE, "path": "", "name": "Steam Frame controllers", "type": "vr", "code": button, "value": value}) action = state["rules"]["controller_buttons"].get(button) + if gaze.on and button.split("/")[-1] in ("trigger", "bumper"): + return # gaze first has them (input/gazefirst.py) if state["pointer"] and action in ACTIONS and action not in ("key", "none"): do_action(action, value, now, button.split("/")[0]) @@ -684,7 +687,10 @@ def main(): def do_action(action, value, now, source="mouse"): """A mapped mouse or controller button, or key combination (pointer mode only).""" - if action == "keyboard_toggle": + if action == "gaze_toggle": + if value == 1: + gaze.toggle() # remembered (POINTER_GAZE), like the toggle macro + elif action == "keyboard_toggle": if value == 1 and vr_keyboard_mode() != "never": vr_keyboard("toggle") else: @@ -882,6 +888,9 @@ def main(): if cmd == "textfield" and len(words) == 2: text_field(words[1] == "1") continue + if cmd == "gazefirst" and len(words) == 2: + gaze.message(words, now) + continue if cmd == "gazeawake" and len(words) == 2: if state["pointer"]: state["pointer"].gaze_awake_until = now + 12.0 if words[1] == "1" else 0.0 @@ -925,6 +934,17 @@ def main(): else: reply(addr, msg) + def to_helper(command): + try: + screens_sock.sendto(command.encode(), HELPER) + except OSError: + pass # helper not running + + # Gaze first (input/gazefirst.py): the controllers' trigger, bumper, thumbstick, and the + # toggle macro, from vrserver's web socket; SteamVR's and Steam's muting. + gaze = gazefirst.GazeFirst(to_helper, log) + atexit.register(gaze.shutdown) + vr_bind(time.monotonic()) # a helper that's already running keeps its buttons in step waiting = False # a keyboard's grab waits for its keys to come up while True: @@ -959,11 +979,12 @@ def main(): if added: apply_roles() - ready, _, _ = select.select(list(nodes) + [control], [], [], - volume.timeout(now, pointer.timeout() if pointer else 0.5)) + ready, _, _ = select.select(list(nodes) + [control] + ([gaze] if gaze.ws else []), [], [], + min(gaze.timeout(), volume.timeout(now, pointer.timeout() if pointer else 0.5))) now = time.monotonic() if pointer: pointer.tick(now) + gaze.tick(now) volume.tick(now) if state["vr_capture_until"] and now >= state["vr_capture_until"]: vr_bind(now) # capture over: back to the mapped buttons @@ -973,6 +994,10 @@ def main(): if fd is control: handle_control(now) continue + if fd is gaze: + if gaze.ws: + gaze.readable(now) + continue node = nodes[fd] try: data = os.read(fd, EVENT.size * 64) diff --git a/input/steam-gamepad-filter.js b/input/steam-gamepad-filter.js index f9cfa8d..d4ef719 100644 --- a/input/steam-gamepad-filter.js +++ b/input/steam-gamepad-filter.js @@ -16,6 +16,9 @@ // window.__frametopGaze: // mode "off" (pass everything), "log" (pass, and log), "block" (drop all but `allow`), // "auto" (block while the dashboard is in laser mode, pass in gamepad mode) +// until with "block" or "auto": blocking stops at this time (Date.now(), ms) unless it's +// moved on. The input relay (input/gazefirst.py) sets it 15 s ahead every 10 s or so, +// so if the relay dies, the controllers come back to Steam's UI by themselves. // allow buttons that always pass: the Steam button's guide and quick menu, and Steam's // own dummy input // log the last 256 events: [ms, "down"|"up"|"analog", button, controller, passed] @@ -44,7 +47,7 @@ if (G.log.length > 256) G.log.shift(); }; G.blocks = button => { - if (G.allow.includes(button)) return false; + if (G.allow.includes(button) || (G.until && Date.now() > G.until)) return false; return G.mode === "block" || (G.mode === "auto" && !G.gamepadMode()); }; if (proto.OnButtonDown && proto.OnButtonDown.__frametop !== VERSION) { diff --git a/input/steamui.py b/input/steamui.py new file mode 100644 index 0000000..69b63d7 --- /dev/null +++ b/input/steamui.py @@ -0,0 +1,40 @@ +"""steamui: run JavaScript in Steam's UI (its SharedJSContext) through Steam's CEF debugger on +127.0.0.1:8080, with nothing but the standard library (input/vrws.py's web socket client). + + evaluate(expression) the expression's value (JSON-able), or an OSError when Steam or its + UI isn't there, or a RuntimeError when the script threw +""" +import json +import time +import urllib.request + +from vrws import VrSocket + +DEBUGGER = "http://127.0.0.1:8080" + + +def evaluate(expression, target="SharedJSContext", timeout=2.0): + with urllib.request.urlopen(f"{DEBUGGER}/json", timeout=timeout) as resp: + pages = json.load(resp) + url = next((p.get("webSocketDebuggerUrl") for p in pages if p.get("title") == target), None) + if not url: + raise OSError(f"Steam's {target} isn't open") + ws = VrSocket(timeout=timeout, url=url) + try: + ws.send(json.dumps({"id": 1, "method": "Runtime.evaluate", + "params": {"expression": expression, "returnByValue": True}})) + deadline = time.monotonic() + timeout + while True: + left = deadline - time.monotonic() + msg = ws.recv(timeout=max(left, 0)) if left > 0 else None + if msg is None: + raise OSError("Steam's UI didn't answer") + if isinstance(msg, dict) and msg.get("id") == 1: + break + finally: + ws.close() + result = msg.get("result") or {} + if "exceptionDetails" in result: + details = result["exceptionDetails"] + raise RuntimeError((details.get("exception") or {}).get("description") or details.get("text", "error")) + return (result.get("result") or {}).get("value") diff --git a/input/vrws.py b/input/vrws.py index 00163d9..baae093 100755 --- a/input/vrws.py +++ b/input/vrws.py @@ -44,12 +44,22 @@ def getstate(timeout=3): class VrSocket: - def __init__(self, timeout=3): - self.sock = socket.create_connection((HOST, PORT), timeout=timeout) + """vrserver's web socket, or with `url` (ws://host:port/path) any other local one, such as + Steam's CEF debugger (input/steamui.py).""" + + def __init__(self, timeout=3, url=None): + if url: + hostport, path = url.removeprefix("ws://").split("/", 1) + host, port = hostport.rsplit(":", 1) + extra = "" + else: + host, port, path, hostport = HOST, PORT, "", f"{HOST}:{PORT}" + extra = f"Origin: {ORIGIN}\r\nReferer: {HEADERS['Referer']}\r\n" + self.sock = socket.create_connection((host, int(port)), timeout=timeout) key = base64.b64encode(os.urandom(16)).decode() - self.sock.sendall((f"GET / HTTP/1.1\r\nHost: {HOST}:{PORT}\r\nUpgrade: websocket\r\n" + self.sock.sendall((f"GET /{path} HTTP/1.1\r\nHost: {hostport}\r\nUpgrade: websocket\r\n" f"Connection: Upgrade\r\nSec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n" - f"Origin: {ORIGIN}\r\nReferer: {HEADERS['Referer']}\r\n\r\n").encode()) + f"{extra}\r\n").encode()) head = b"" while b"\r\n\r\n" not in head: chunk = self.sock.recv(4096) @@ -106,6 +116,10 @@ class VrSocket: out, self.buf = self.buf[:n], self.buf[n:] return out + def pending(self): + """A message (or part of one) is already read and waiting.""" + return bool(self.buf) + def recv(self, timeout=None): """The next text message as parsed JSON (None for one that isn't JSON), or None at the timeout. Raises OSError when the connection ends.""" diff --git a/pointer/bindings/vrcompositor_frame_controller_gazefirst.json b/pointer/bindings/vrcompositor_frame_controller_gazefirst.json index 49bfc26..4c021a4 100644 --- a/pointer/bindings/vrcompositor_frame_controller_gazefirst.json +++ b/pointer/bindings/vrcompositor_frame_controller_gazefirst.json @@ -1,463 +1,187 @@ { - "action_manifest_version": 0, - "alias_info": {}, - "app_key": "openvr.component.vrcompositor", - "bindings": { - "/actions/dualanalog": { - "sources": [ - { - "inputs": { - "click": { - "output": "/actions/dualanalog/in/modeswitch1" - } - }, - "mode": "button", - "path": "/user/hand/left/input/grip" - }, - { - "inputs": { - "click": { - "output": "/actions/dualanalog/in/modeswitch2" - } - }, - "mode": "button", - "path": "/user/hand/right/input/grip" - }, - { - "inputs": { - "click": { - "output": "/actions/dualanalog/in/leftclick" - }, - "position": { - "output": "/actions/dualanalog/in/leftvalue" - } - }, - "mode": "joystick", - "path": "/user/hand/left/input/thumbstick" - }, - { - "inputs": { - "click": { - "output": "/actions/dualanalog/in/rightclick" - }, - "position": { - "output": "/actions/dualanalog/in/rightvalue" - } - }, - "mode": "joystick", - "path": "/user/hand/right/input/thumbstick" - }, - { - "inputs": { - "click": { - "output": "/actions/dualanalog/in/leftclick" - } - }, - "mode": "button", - "path": "/user/hand/left/input/trigger" - }, - { - "inputs": { - "click": { - "output": "/actions/dualanalog/in/rightclick" - } - }, - "mode": "button", - "path": "/user/hand/right/input/trigger" - } - ] - }, - "/actions/lasermouse": { - "haptics": [ - { - "output": "/actions/lasermouse/out/haptic", - "path": "/user/hand/left/output/haptic" - }, - { - "output": "/actions/lasermouse/out/haptic", - "path": "/user/hand/right/output/haptic" - } - ], - "poses": [ - { - "output": "/actions/lasermouse/in/Pointer", - "path": "/user/hand/left/pose/tip" - }, - { - "output": "/actions/lasermouse/in/Pointer", - "path": "/user/hand/right/pose/tip" - } - ], - "sources": [ - { - "inputs": { - "click": { - "output": "/actions/lasermouse/in/rightclick" - } - }, - "mode": "joystick", - "path": "/user/hand/left/input/thumbstick" - }, - { - "inputs": { - "click": { - "output": "/actions/lasermouse/in/middleclick" - } - }, - "mode": "button", - "path": "/user/hand/left/input/dpad_right" - }, - { - "inputs": { - "click": { - "output": "/actions/lasermouse/in/back" - } - }, - "mode": "button", - "path": "/user/hand/left/input/dpad_left" - }, - { - "inputs": { - "click": { - "output": "/actions/lasermouse/in/home" - } - }, - "mode": "button", - "path": "/user/hand/left/input/dpad_down" - }, - { - "inputs": { - "click": { - "output": "/actions/lasermouse/in/rightclick" - } - }, - "mode": "joystick", - "path": "/user/hand/right/input/thumbstick" - }, - { - "inputs": { - "click": { - "output": "/actions/lasermouse/in/middleclick" - } - }, - "mode": "button", - "path": "/user/hand/right/input/x" - }, - { - "inputs": { - "click": { - "output": "/actions/lasermouse/in/back" - } - }, - "mode": "button", - "path": "/user/hand/right/input/b" - }, - { - "inputs": { - "click": { - "output": "/actions/lasermouse/in/home" - } - }, - "mode": "button", - "path": "/user/hand/right/input/a" - } - ] - }, - "/actions/lasermouse_secondary": { - "sources": [ - { - "inputs": { - "click": { - "output": "/actions/lasermouse_secondary/in/switchlaserhand" - } - }, - "mode": "trigger", - "path": "/user/hand/left/input/grip" - }, - { - "inputs": { - "click": { - "output": "/actions/lasermouse_secondary/in/switchlaserhand" - } - }, - "mode": "trigger", - "path": "/user/hand/right/input/grip" - } - ] - }, - "/actions/scroll_discrete": { - "sources": [ - { - "inputs": { - "scroll": { - "output": "/actions/scroll_discrete/in/scroll" - } - }, - "mode": "scroll", - "path": "/user/hand/left/input/thumbstick" - }, - { - "inputs": { - "scroll": { - "output": "/actions/scroll_discrete/in/scroll" - } - }, - "mode": "scroll", - "path": "/user/hand/right/input/thumbstick" - } - ] - }, - "/actions/scroll_smooth": { - "sources": [ - { - "inputs": { - "scroll": { - "output": "/actions/scroll_smooth/in/scroll" - } - }, - "mode": "scroll", - "parameters": { - "scroll_mode": "smooth" - }, - "path": "/user/hand/left/input/thumbstick" - }, - { - "inputs": { - "scroll": { - "output": "/actions/scroll_smooth/in/scroll" - } - }, - "mode": "scroll", - "parameters": { - "scroll_mode": "smooth" - }, - "path": "/user/hand/right/input/thumbstick" - } - ] - }, - "/actions/system": { - "chords": [ - { - "inputs": [ - [ - "/user/hand/left/input/system", - "held" - ], - [ - "/user/hand/left/input/trigger", - "click" - ] - ], - "output": "/actions/system/in/TakeScreenshot" - }, - { - "inputs": [ - [ - "/user/hand/right/input/system", - "held" - ], - [ - "/user/hand/right/input/trigger", - "click" - ] - ], - "output": "/actions/system/in/TakeScreenshot" - }, - { - "inputs": [ - [ - "/user/hand/left/input/system", - "held" - ], - [ - "/user/hand/right/input/trigger", - "click" - ] - ], - "output": "/actions/system/in/TakeScreenshot" - }, - { - "inputs": [ - [ - "/user/hand/right/input/system", - "held" - ], - [ - "/user/hand/left/input/trigger", - "click" - ] - ], - "output": "/actions/system/in/TakeScreenshot" - } - ], - "sources": [ - { - "inputs": {}, - "mode": "button", - "parameters": { - "haptic_amplitude": "0" - }, - "path": "/user/hand/left/input/trigger" - }, - { - "inputs": {}, - "mode": "button", - "parameters": { - "haptic_amplitude": "0" - }, - "path": "/user/hand/right/input/trigger" - }, - { - "inputs": { - "click": { - "output": "/actions/system/in/ToggleDashboard" - }, - "double": { - "output": "/actions/system/in/ToggleRoomView" - }, - "held": { - "output": "/actions/system/in/SystemButtonChord" - } - }, - "mode": "button", - "path": "/user/hand/left/input/system" - }, - { - "inputs": { - "click": { - "output": "/actions/system/in/ToggleDashboard" - }, - "double": { - "output": "/actions/system/in/ToggleRoomView" - }, - "held": { - "output": "/actions/system/in/SystemButtonChord" - } - }, - "mode": "button", - "path": "/user/hand/right/input/system" - } - ] - }, - "/actions/quickmouse": { - "sources": [ - { - "inputs": { - "click": { - "output": "/actions/quickmouse/in/ActivateQuickMouse" - } - }, - "mode": "trigger", - "path": "/user/hand/left/input/grip" - }, - { - "inputs": { - "click": { - "output": "/actions/quickmouse/in/ActivateQuickMouse" - } - }, - "mode": "trigger", - "path": "/user/hand/right/input/grip" - } - ] - }, - "/actions/locomotion": { - "sources": [ - { - "inputs": { - "position": { - "output": "/actions/locomotion/in/movement" - } - }, - "mode": "joystick", - "path": "/user/hand/left/input/thumbstick" - }, - { - "inputs": { - "position": { - "output": "/actions/locomotion/in/rotation" - } - }, - "mode": "joystick", - "path": "/user/hand/right/input/thumbstick" - } - ] - }, - "/actions/quickrecenter": { - "sources": [ - { - "inputs": { - "long": { - "output": "/actions/quickrecenter/in/Recenter" - } - }, - "parameters": { - "long_press_delay": 1.0, - "long_press_expiry": 4.0 - }, - "mode": "button", - "path": "/user/hand/left/input/system" - }, - { - "inputs": { - "long": { - "output": "/actions/quickrecenter/in/Recenter" - } - }, - "parameters": { - "long_press_delay": 1.0, - "long_press_expiry": 4.0 - }, - "mode": "button", - "path": "/user/hand/right/input/system" - } - ] - }, - "/actions/roomsetup": { - "sources": [ - { - "inputs": { - "position": { - "output": "/actions/roomsetup/in/FloorFineAdjust" - } - }, - "mode": "joystick", - "path": "/user/hand/left/input/thumbstick" - }, - { - "inputs": { - "position": { - "output": "/actions/roomsetup/in/FloorFineAdjust" - } - }, - "mode": "joystick", - "path": "/user/hand/right/input/thumbstick" - }, - { - "inputs": { - "click": { - "output": "/actions/roomsetup/in/Accept" - } - }, - "mode": "button", - "path": "/user/hand/right/input/a" - }, - { - "inputs": { - "click": { - "output": "/actions/roomsetup/in/Back" - } - }, - "mode": "button", - "path": "/user/hand/right/input/b" - } - ] - } - }, - "category": "steamvr_input", - "controller_type": "frame_controller", - "description": "The Frame controller stock compositor binding without its trigger and bumper laser entries (docs/gaze-first.md)", - "name": "Frametop gaze first: no trigger or bumper laser", - "options": {}, - "simulated_actions": [] + "action_manifest_version": 0, + "alias_info": {}, + "app_key": "openvr.component.vrcompositor", + "bindings": { + "/actions/system": { + "chords": [ + { + "inputs": [ + [ + "/user/hand/left/input/system", + "held" + ], + [ + "/user/hand/left/input/trigger", + "click" + ] + ], + "output": "/actions/system/in/TakeScreenshot" + }, + { + "inputs": [ + [ + "/user/hand/right/input/system", + "held" + ], + [ + "/user/hand/right/input/trigger", + "click" + ] + ], + "output": "/actions/system/in/TakeScreenshot" + }, + { + "inputs": [ + [ + "/user/hand/left/input/system", + "held" + ], + [ + "/user/hand/right/input/trigger", + "click" + ] + ], + "output": "/actions/system/in/TakeScreenshot" + }, + { + "inputs": [ + [ + "/user/hand/right/input/system", + "held" + ], + [ + "/user/hand/left/input/trigger", + "click" + ] + ], + "output": "/actions/system/in/TakeScreenshot" + } + ], + "sources": [ + { + "inputs": {}, + "mode": "button", + "parameters": { + "haptic_amplitude": "0" + }, + "path": "/user/hand/left/input/trigger" + }, + { + "inputs": {}, + "mode": "button", + "parameters": { + "haptic_amplitude": "0" + }, + "path": "/user/hand/right/input/trigger" + }, + { + "inputs": { + "click": { + "output": "/actions/system/in/ToggleDashboard" + }, + "double": { + "output": "/actions/system/in/ToggleRoomView" + }, + "held": { + "output": "/actions/system/in/SystemButtonChord" + } + }, + "mode": "button", + "path": "/user/hand/left/input/system" + }, + { + "inputs": { + "click": { + "output": "/actions/system/in/ToggleDashboard" + }, + "double": { + "output": "/actions/system/in/ToggleRoomView" + }, + "held": { + "output": "/actions/system/in/SystemButtonChord" + } + }, + "mode": "button", + "path": "/user/hand/right/input/system" + } + ] + }, + "/actions/quickrecenter": { + "sources": [ + { + "inputs": { + "long": { + "output": "/actions/quickrecenter/in/Recenter" + } + }, + "parameters": { + "long_press_delay": 1.0, + "long_press_expiry": 4.0 + }, + "mode": "button", + "path": "/user/hand/left/input/system" + }, + { + "inputs": { + "long": { + "output": "/actions/quickrecenter/in/Recenter" + } + }, + "parameters": { + "long_press_delay": 1.0, + "long_press_expiry": 4.0 + }, + "mode": "button", + "path": "/user/hand/right/input/system" + } + ] + }, + "/actions/roomsetup": { + "sources": [ + { + "inputs": { + "position": { + "output": "/actions/roomsetup/in/FloorFineAdjust" + } + }, + "mode": "joystick", + "path": "/user/hand/left/input/thumbstick" + }, + { + "inputs": { + "position": { + "output": "/actions/roomsetup/in/FloorFineAdjust" + } + }, + "mode": "joystick", + "path": "/user/hand/right/input/thumbstick" + }, + { + "inputs": { + "click": { + "output": "/actions/roomsetup/in/Accept" + } + }, + "mode": "button", + "path": "/user/hand/right/input/a" + }, + { + "inputs": { + "click": { + "output": "/actions/roomsetup/in/Back" + } + }, + "mode": "button", + "path": "/user/hand/right/input/b" + } + ] + } + }, + "category": "steamvr_input", + "controller_type": "frame_controller", + "description": "Gaze first (Frametop, docs/gaze-first.md): the stock compositor binding with only the Steam button (dashboard, room view, recenter, screenshot chord) and room setup left. The relay reads the rest from vrserver's web socket.", + "name": "Frametop gaze first: Frame controller, Steam button only", + "options": {}, + "simulated_actions": [] } diff --git a/pointer/helper/ft-pointer.cpp b/pointer/helper/ft-pointer.cpp index 849e17b..133bd78 100644 --- a/pointer/helper/ft-pointer.cpp +++ b/pointer/helper/ft-pointer.cpp @@ -150,8 +150,8 @@ // release it when the mouse is idle ("gazeawake 1|0" tells it). A controller that moves // still releases it (the mouse is gaze mode's only pointer device), and in games the mouse // wakes it and idling releases it, as without gaze. -// The dot only shows while the mouse moves it (within POINTER_GAZE_SHOW, 1 s), while a -// press is held, and briefly for each click (a pulse); +// With POINTER_GAZE_DOT=moving, the dot only shows while the mouse moves it (within +// POINTER_GAZE_SHOW, 1 s), while a press is held, and briefly for each click (a pulse); // otherwise it's transparent (still there for the laser to land on). The gaze moving it // doesn't show it: you know where you're looking. // When the mouse took the pointer and you then click, the nudge was probably onto what @@ -183,6 +183,25 @@ // the right hand as the precision tool, the pointer needs the left hand, or a role that's no // hand. Treadmill is being tested for that (docs/gaze-first.md, test 1). // +// Gaze first (docs/gaze-first.md): with gaze mode on outside games, the relay mutes the Frame +// controllers in SteamVR and in Steam's UI, and sends their trigger and bumper here ("ctrl +// 1|0", from vrserver's web socket). A press stops the pointer +// where you look, like the mouse's held-back press. Moving the controller past +// POINTER_TRIGGER_DEADZONE (1 deg: its position seen from the eye, as the press began) within +// POINTER_GAZE_HOLD steers the pointer at POINTER_TRIGGER_GAIN (0.5), and the release clicks +// there (a lesson, as with the mouse). Held still that long instead, it's a real press, and the +// controller drags at POINTER_GAZE_DRAG_GAIN (1). The trigger is the left button, the bumper the +// right. "gazefirst 1|0" tells the relay when gaze first is on (gaze mode, no game, headset on). +// Steam reads the controllers as a gamepad, and SteamVR leaves laser mode about 40 ms after +// each press and each release Steam sees, whatever the bindings say. So while gaze first is on +// and the pointer is awake, the helper takes the laser back whenever it isn't ours (on no device, +// or on a controller) with the driver's "a" (switchlaserhand), at most every 100 ms +// (POINTER_GAZE_KEEP_LASER, on): a blink of 20 to 40 ms. A controller's click or press waits for +// the laser: until it has left and come back after the last press or release (100 ms at least), +// or 250 ms. +// POINTER_GAZE_DOT: always (the default) shows the dot all the time in gaze mode, which also +// covers those blinks; moving shows it as described under gaze mode. +// // Hands (POINTER_HANDS, off by default; needs hand tracking, hands/): ft-hands publishes // pinches and grips (hands/include/fh_gestures.h), read here every frame. // A pinch works like gaze mode's mouse press: the pointer stops (where the gaze put it), and @@ -669,6 +688,10 @@ int main() { // Gaze mode (see the top); gazeConf is POINTER_GAZE as last read, like followConf. bool gazeOn = false, gazeConf = false; double gazeRetake = 5, gazeNudgeMax = 8, gazeHold = 0.5, gazeShow = 1; + // Gaze first (see the top): POINTER_GAZE_DOT, POINTER_GAZE_KEEP_LASER, POINTER_TRIGGER_GAIN, + // POINTER_TRIGGER_DEADZONE. + bool gazeDotAlways = true, keepLaser = true; + double triggerGain = 0.5, triggerDeadzone = 1.0; // Hands (see the top): POINTER_HANDS, POINTER_PINCH_GAIN, POINTER_PINCH_DEADZONE, POINTER_GRIP_GAIN. bool handsOn = false; double pinchGain = 0.5, pinchDeadzone = 1.5, gripGain = 1.0, handBelow = 0.35, typingHold = 1.0; @@ -702,6 +725,11 @@ int main() { gazeShow = std::clamp(ConfDouble(conf, "POINTER_GAZE_SHOW", 1), 0.0, 30.0); const bool wantGaze = ConfDouble(conf, "POINTER_GAZE", 0) != 0; if (wantGaze != gazeConf) gazeOn = gazeConf = wantGaze; + const auto gd = conf.find("POINTER_GAZE_DOT"); + gazeDotAlways = gd == conf.end() || gd->second != "moving"; + keepLaser = ConfDouble(conf, "POINTER_GAZE_KEEP_LASER", 1) != 0; + triggerGain = std::clamp(ConfDouble(conf, "POINTER_TRIGGER_GAIN", 0.5), 0.05, 3.0); + triggerDeadzone = std::clamp(ConfDouble(conf, "POINTER_TRIGGER_DEADZONE", 1.0), 0.0, 10.0); handsOn = ConfDouble(conf, "POINTER_HANDS", 0) != 0; pinchGain = std::clamp(ConfDouble(conf, "POINTER_PINCH_GAIN", 0.5), 0.05, 3.0); pinchDeadzone = std::clamp(ConfDouble(conf, "POINTER_PINCH_DEADZONE", 1.5), 0.0, 10.0); @@ -857,6 +885,9 @@ int main() { bool grip = false; // pressed at once and dragging (a grip, gaze drag), not a click on release bool engaged = false; // past the dead zone bool pressed = false; // a real press went out (a grip or gaze drag, or a pinch without gaze mode) + bool ctrlPos = false; // gaze first's trigger or bumper: steered by the controller's position + bool promote = false; // held still for POINTER_GAZE_HOLD, it becomes a real press + bool right = false; // the right button (the bumper) Vec3 origin; vr::TrackedDeviceIndex_t ctrl = vr::k_unTrackedDeviceIndexInvalid; double refYaw = 0, refPitch = 0, startYaw = 0, startPitch = 0, lastYaw = 0, lastPitch = 0; @@ -867,6 +898,18 @@ int main() { bool drag, down; }; std::vector devicePresses; + // "ctrl 1|0" from the relay, done in the frame (see Gaze first). + struct CtrlPress { + std::string side; + bool right, down; + }; + std::vector ctrlPresses; + bool ctrlCancel = false; // "ctrlcancel": the held trigger or bumper ends without a click + // Gaze first's laser keeper, and its controller clicks' wait (see the top): who has the + // laser, since when it's ours, whether it left after the last controller press or release. + vr::TrackedDeviceIndex_t primary = vr::k_unTrackedDeviceIndexInvalid; + Clock::time_point ctrlEventAt{}, laserOursSince{}, keepPressAt{}; + bool laserLeft = false, keepHeld = false; uint32_t seenBegins[2][2] = {}, seenEnds[2][2] = {}; // [pinch, grip][side], as last read bool handBaseline = false; int handOpens = 0; @@ -874,10 +917,13 @@ int main() { Clock::time_point handUsed{}; // a gesture began then (keeps the pointer, like gaze mode) Clock::time_point lastTyping{}; // the relay's last "typing": a key on a keyboard // Gaze mode outside games, and its dot (see the top): lastMove/lastHeld/pulseAt. - bool inGame = false, gazeAwake = false; - Clock::time_point inGameAt{}, gazeAwakeAt{}; + bool inGame = false, gazeAwake = false, gazeFirst = false; + Clock::time_point inGameAt{}, gazeAwakeAt{}, gazeFirstAt{}; Clock::time_point lastMove{}, lastHeld{}, pulseAt{}; - // The left button, as sent to the driver. + // The left button, as sent to the driver; pressRight: the next press is the right button + // instead (gaze first's bumper), heldButton: the one pressed. + bool pressRight = false; + std::string heldButton = "trigger"; auto pressLeft = [&] { // ft-screens sends the keyboard to the panel clicked last; it sees clicks on // its own screens, but only we know when one lands on another panel. @@ -907,14 +953,16 @@ int main() { tiltYaw = tiltPitch = 0; // a new drag starts untilted dropHoldUntil = {}; pulseAt = Clock::now(); - SendTo(out, "ft_pointer", "btn trigger 1"); + heldButton = pressRight ? "b" : "trigger"; + pressRight = false; + SendTo(out, "ft_pointer", "btn " + heldButton + " 1"); }; auto releaseLeft = [&] { leftHeld = false; tilting = false; // Hold the drag pose (tilt, frozen distance) while SteamVR finishes the drop. dropHoldUntil = Clock::now() + std::chrono::milliseconds(500); - SendTo(out, "ft_pointer", "btn trigger 0"); + SendTo(out, "ft_pointer", "btn " + heldButton + " 0"); // ft-screens releases a button held on its screens in KWin even when SteamVR hands // the release to some other overlay (its catcher usually gets it; this is the backstop). SendTo(out, "ft_screens", "up"); @@ -1141,6 +1189,15 @@ int main() { gazeAwakeAt = t; SendTo(out, "frametop_relay", awake ? "gazeawake 1" : "gazeawake 0"); } + // Gaze first (see the top): the relay mutes the controllers while it's on. + const bool first = gazeOn && !inGame && !headsetOff; + if (first != gazeFirst || t - gazeFirstAt > std::chrono::seconds(5)) { + if (first != gazeFirst) std::printf("gaze first %s\n", first ? "on" : "off"); + if (first != gazeFirst) std::fflush(stdout); + gazeFirst = first; + gazeFirstAt = t; + SendTo(out, "frametop_relay", first ? "gazefirst 1" : "gazefirst 0"); + } } // Commands from the relay. @@ -1164,6 +1221,23 @@ int main() { gz = {g[0], g[1], g[2], g[3], Clock::now()}; continue; } + if (std::strcmp(buf, "ctrlcancel") == 0) { // the relay's macro fired, or it lost the controllers + ctrlCancel = true; + ctrlPresses.clear(); + continue; + } + { + char side[8], button[16]; + int v; + if (std::sscanf(buf, "ctrl %7s %15s %d", side, button, &v) == 3) { + lastMouse = Clock::now(); + if (!active) wake(Clock::now()); + ctrlPresses.push_back({side, !std::strcmp(button, "bumper"), v != 0}); + ctrlEventAt = Clock::now(); + laserLeft = false; + continue; + } + } { char kind[16], source[16]; int v; @@ -1199,6 +1273,13 @@ int main() { overlays.Request(sender, senderLen); // answered from the list's thread continue; } + if (std::strcmp(buf, "gazefirst ?") == 0) { // why gaze first is on or off + char msg[160]; + std::snprintf(msg, sizeof msg, "ok first %d gaze %d game %d headset %s awake %d laser %d ours %d", gazeFirst, + gazeOn, inGame, headsetOff ? "off" : "on", active, int(primary), int(ours)); + reply(msg); + continue; + } if (std::strncmp(buf, "gaze", 4) == 0) { const char *arg = buf + 4; while (*arg == ' ') ++arg; @@ -1463,11 +1544,12 @@ int main() { if (gazeOn) gazeOwns = true, nudging = false; // no click: the pointer goes back to the gaze } else { clickPress = true; // the click is on the release, where the pointer is now + pressRight = hold.right; gazeBack = nudgeMoved < 0.2; } } if (debug) - std::printf("hold %s %s%s\n", hold.src == Src::Hand ? (hold.grip ? "grip" : "pinch") : hold.grip ? "gaze drag" : "gaze precision", + std::printf("hold %s %s%s\n", hold.src == Src::Hand ? (hold.grip ? "grip" : "pinch") : hold.ctrlPos ? (hold.right ? "bumper" : "trigger") : hold.grip ? "gaze drag" : "gaze precision", lost ? "lost" : "released", hold.pressed ? "" : lost ? " (no click)" : " (click)"); if (debug) std::fflush(stdout); hold = {}; @@ -1553,6 +1635,15 @@ int main() { cp = std::asin(std::clamp(f.y, -1.0, 1.0)) * 180 / M_PI; return true; }; + // A controller's position seen from `from`: yaw and pitch, degrees (gaze first's holds). + auto controllerPosAngles = [&](vr::TrackedDeviceIndex_t i, const Vec3 &from, double &cy, double &cp) { + if (i >= vr::k_unMaxTrackedDeviceCount || !all[i].bPoseIsValid) return false; + const auto &m = all[i].mDeviceToAbsoluteTracking.m; + const Vec3 d = Normalize(Vec3{m[0][3], m[1][3], m[2][3]} - from); + cy = std::atan2(-d.x, -d.z) * 180 / M_PI; + cp = std::asin(std::clamp(d.y, -1.0, 1.0)) * 180 / M_PI; + return true; + }; // The Frame controller in a hand (not our own device, which may hold that hand's role). auto controllerFor = [&](const std::string &hand) { const int32_t want = hand == "left" ? vr::TrackedControllerRole_LeftHand : vr::TrackedControllerRole_RightHand; @@ -1589,10 +1680,37 @@ int main() { if (debug) std::fflush(stdout); } devicePresses.clear(); + // Gaze first's trigger and bumper (see the top): a held-back press at the gaze, steered + // by the controller's position seen from the eye as it began. + if (ctrlCancel) { + ctrlCancel = false; + if (hold.src != Src::None && hold.ctrlPos) endHold(true); + } + for (const CtrlPress &p : ctrlPresses) { + const int side = p.side == "right"; + if (!p.down) { + if (hold.src != Src::None && hold.ctrlPos && hold.side == side && hold.right == p.right) endHold(false); + continue; + } + if (hold.src != Src::None || leftHeld || aimHeld || clickPress || clickRelease || !hmd.bPoseIsValid) continue; + Hold h; + h.src = Src::Controller, h.ctrlPos = h.promote = true, h.right = p.right, h.side = side, h.origin = eye; + h.ctrl = controllerFor(p.side); + if (!controllerPosAngles(h.ctrl, h.origin, h.refYaw, h.refPitch)) h.src = Src::Mouse; // no pose: the mouse steers + hold = h; + startHold(false); + if (debug) std::printf("hold %s %s began\n", p.side.c_str(), p.right ? "bumper" : "trigger"); + if (debug) std::fflush(stdout); + } + ctrlPresses.clear(); if (hold.src == Src::Controller) { double cy, cp; - if (controllerAngles(hold.ctrl, cy, cp)) + if (hold.ctrlPos) { + if (controllerPosAngles(hold.ctrl, hold.origin, cy, cp)) + steer(cy, cp, hold.pressed ? 0.3 : triggerDeadzone, hold.pressed ? dragGain : triggerGain); + } else if (controllerAngles(hold.ctrl, cy, cp)) { steer(cy, cp, precisionDeadzone, hold.grip ? dragGain : precisionGain); + } } fh_gestures_t hg; const bool handOk = handsOn && handFile.Read(hg); @@ -1896,7 +2014,7 @@ int main() { // the page, with SteamVR's hit dot hidden on it. const Vec3 near = eye + sight * SETTINGS_DOT, far = eye + sight * SETTINGS_CATCHER; double alpha = 1; - if (gazeOn) { + if (gazeOn && !gazeDotAlways) { auto secs = [&](Clock::time_point t) { return std::chrono::duration(tnow - t).count(); }; alpha = std::clamp(1 - std::min(secs(lastMove) - gazeShow, secs(lastHeld)) / 0.25, 0.0, 1.0); } @@ -1921,7 +2039,7 @@ int main() { double scale = 1, alpha = 1; if (gazeOn) { auto secs = [&](Clock::time_point t) { return std::chrono::duration(tnow - t).count(); }; - alpha = std::clamp(1 - std::min(secs(lastMove) - gazeShow, secs(lastHeld)) / 0.25, 0.0, 1.0); + alpha = gazeDotAlways ? 1.0 : std::clamp(1 - std::min(secs(lastMove) - gazeShow, secs(lastHeld)) / 0.25, 0.0, 1.0); const double pulse = secs(pulseAt); if (pulse < 0.6) { scale = 1 + 1.5 * std::max(0.0, 1 - pulse / 0.3); @@ -1993,7 +2111,52 @@ int main() { gazeBack = true; pressLeft(); } - if (clickPress) { + + // Gaze first (see the top): take the laser back whenever it isn't ours, and let a + // controller's press or click out only once it's back. + primary = overlay->GetPrimaryDashboardDevice(); + if (primary == ours && ours != vr::k_unTrackedDeviceIndexInvalid) { + if (laserOursSince == Clock::time_point{}) laserOursSince = tnow; + } else { + laserOursSince = {}; + laserLeft = true; + } + const bool keeping = keepLaser && gazeOn && !inGame && active && !headsetOff && + ours != vr::k_unTrackedDeviceIndexInvalid && sys->IsTrackedDeviceConnected(ours) && + tnow - wokeAt > std::chrono::milliseconds(600); + if (keepHeld && tnow - keepPressAt >= std::chrono::milliseconds(40)) { + SendTo(out, "ft_pointer", "btn a 0"); + keepHeld = false; + } + if (keeping && primary != ours && !keepHeld && !claimPending && !claimHeld && + tnow - keepPressAt >= std::chrono::milliseconds(100)) { + SendTo(out, "ft_pointer", "btn a 1"); + keepHeld = true; + keepPressAt = tnow; + if (debug) std::printf("laser keeper: laser on %d, taking it back\n", int(primary)); + if (debug) std::fflush(stdout); + } + auto laserReady = [&] { + if (ctrlEventAt == Clock::time_point{} || !keeping) return true; + const auto since = tnow - ctrlEventAt; + if (since > std::chrono::seconds(1)) return true; // never came back: don't hold the click forever + if (laserOursSince == Clock::time_point{} || tnow - laserOursSince < std::chrono::milliseconds(25)) return false; + return since >= std::chrono::milliseconds(250) || (laserLeft && since >= std::chrono::milliseconds(100)); + }; + // A trigger or bumper held still for POINTER_GAZE_HOLD: a real press, then the + // controller drags (from where it is now). + if (aimHeld && aimHand && hold.promote && !hold.engaged && + tnow - aimSince >= std::chrono::duration(gazeHold) && laserReady()) { + aimHeld = aimHand = false; + hold.pressed = hold.grip = true; + if (hold.src == Src::Controller) controllerPosAngles(hold.ctrl, hold.origin, hold.refYaw, hold.refPitch); + gazeBack = true; + pressRight = hold.right; + pressLeft(); + if (debug) std::printf("hold %s: pressed (held still)\n", hold.right ? "bumper" : "trigger"); + if (debug) std::fflush(stdout); + } + if (clickPress && laserReady()) { clickPress = false; pressLeft(); clickRelease = true; diff --git a/pointer/probe/build.sh b/pointer/probe/build.sh index fcfc612..0e6bcb1 100755 --- a/pointer/probe/build.sh +++ b/pointer/probe/build.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Build vrprobe in the dev container on the Frame (pointer/probe/build/vrprobe). +# Build vrprobe, lasertest, vrsetting and focustest in the dev container on the Frame (pointer/probe/build/vrprobe). set -euo pipefail root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) "$root/scripts/sync.sh" >/dev/null @@ -8,4 +8,8 @@ g++ -std=c++17 -O2 -Wall -I/opt/steamvr/tools/hellovr_vulkan_linux/src/openvr/he -o build/vrprobe vrprobe.cpp -L/opt/steamvr/bin/linuxarm64 -lopenvr_api -Wl,-rpath,/opt/steamvr/bin/linuxarm64 g++ -std=c++17 -O2 -Wall -I/opt/steamvr/tools/hellovr_vulkan_linux/src/openvr/headers \ -o build/lasertest lasertest.cpp -L/opt/steamvr/bin/linuxarm64 -lopenvr_api -Wl,-rpath,/opt/steamvr/bin/linuxarm64 -echo "built build/vrprobe build/lasertest"' +g++ -std=c++17 -O2 -Wall -I/opt/steamvr/tools/hellovr_vulkan_linux/src/openvr/headers \ + -o build/vrsetting vrsetting.cpp -L/opt/steamvr/bin/linuxarm64 -lopenvr_api -Wl,-rpath,/opt/steamvr/bin/linuxarm64 +g++ -std=c++17 -O2 -Wall -I/opt/steamvr/tools/hellovr_vulkan_linux/src/openvr/headers \ + -o build/focustest focustest.cpp -L/opt/steamvr/bin/linuxarm64 -lopenvr_api -Wl,-rpath,/opt/steamvr/bin/linuxarm64 +echo "built build/vrprobe build/lasertest build/vrsetting build/focustest"' diff --git a/pointer/probe/focustest.cpp b/pointer/probe/focustest.cpp new file mode 100644 index 0000000..493fbdc --- /dev/null +++ b/pointer/probe/focustest.cpp @@ -0,0 +1,135 @@ +// Does a dashboard overlay with overlay flag 1 << 4 take SteamVR's input focus for its own process +// (docs/gaze-first.md)? SteamVR's dashboard gives an external overlay's page a "VR client" input +// focus (its owner's PID) when that flag is set, and a Steam Input one with 1 << 30. While a VR +// client has the input focus, as a game does, Steam shouldn't get the controllers. +// Shows a dashboard overlay "Frametop focus test" (a plain colour) with that flag, and prints +// input-focus events, IsInputAvailable(), and the dashboard's visibility as they change. +// Runs as an OpenVR overlay client (in the dev container). +// With --manifest it's a SteamVR input client too (focustest_actions/: Y and the triggers), and +// prints when those actions change, which shows whether it gets them while Steam doesn't. +// Usage: focustest [--seconds N] [--no-flag] [--steam-flag] [--manifest] +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using Clock = std::chrono::steady_clock; +static volatile std::sig_atomic_t g_stop = 0; + +int main(int argc, char **argv) { + double seconds = 0; + bool flag = true, steamFlag = false, manifest = false; + for (int i = 1; i < argc; ++i) { + if (!std::strcmp(argv[i], "--seconds") && i + 1 < argc) seconds = std::atof(argv[++i]); + else if (!std::strcmp(argv[i], "--no-flag")) flag = false; + else if (!std::strcmp(argv[i], "--steam-flag")) steamFlag = true; + else if (!std::strcmp(argv[i], "--manifest")) manifest = true; + else { + std::fprintf(stderr, "usage: %s [--seconds N] [--no-flag] [--steam-flag] [--manifest]\n", argv[0]); + return 2; + } + } + std::signal(SIGINT, [](int) { g_stop = 1; }); + std::signal(SIGTERM, [](int) { g_stop = 1; }); + vr::EVRInitError err = vr::VRInitError_None; + vr::IVRSystem *sys = vr::VR_Init(&err, vr::VRApplication_Overlay); + if (err != vr::VRInitError_None) { + std::printf("VR_Init failed: %s\n", vr::VR_GetVRInitErrorAsEnglishDescription(err)); + return 1; + } + vr::IVROverlay *ov = vr::VROverlay(); + vr::VRActionSetHandle_t set = vr::k_ulInvalidActionSetHandle; + vr::VRActionHandle_t actY = vr::k_ulInvalidActionHandle, actTrigger = vr::k_ulInvalidActionHandle; + if (manifest) { + char exe[PATH_MAX] = ""; + const ssize_t n = readlink("/proc/self/exe", exe, sizeof exe - 1); + std::string dir = n > 0 ? std::string(exe, n) : std::string("."); + dir = dir.substr(0, dir.rfind('/')); // build/ + dir = dir.substr(0, dir.rfind('/')); // pointer/probe + const std::string path = dir + "/focustest_actions/actions.json"; + const auto ie = vr::VRInput()->SetActionManifestPath(path.c_str()); + vr::VRInput()->GetActionSetHandle("/actions/focus", &set); + vr::VRInput()->GetActionHandle("/actions/focus/in/y", &actY); + vr::VRInput()->GetActionHandle("/actions/focus/in/trigger", &actTrigger); + std::printf("action manifest %s: error %d\n", path.c_str(), int(ie)); + } + vr::VROverlayHandle_t main = vr::k_ulOverlayHandleInvalid, thumb = vr::k_ulOverlayHandleInvalid; + auto e = ov->CreateDashboardOverlay("frametop.focustest", "Frametop focus test", &main, &thumb); + if (e != vr::VROverlayError_None) { + std::printf("CreateDashboardOverlay: %s\n", ov->GetOverlayErrorNameFromEnum(e)); + return 1; + } + // A plain teal page, and a teal thumbnail. + const uint32_t w = 256, h = 160; + std::vector px(w * h * 4); + for (uint32_t i = 0; i < w * h; ++i) px[i * 4] = 20, px[i * 4 + 1] = 140, px[i * 4 + 2] = 140, px[i * 4 + 3] = 255; + ov->SetOverlayRaw(main, px.data(), w, h, 4); + ov->SetOverlayRaw(thumb, px.data(), w, h, 4); + ov->SetOverlayWidthInMeters(main, 1.5f); + ov->SetOverlayInputMethod(main, vr::VROverlayInputMethod_Mouse); + if (flag) { + e = ov->SetOverlayFlag(main, vr::VROverlayFlags(1 << 4), true); + std::printf("SetOverlayFlag(1 << 4): %s\n", ov->GetOverlayErrorNameFromEnum(e)); + } + if (steamFlag) { + e = ov->SetOverlayFlag(main, vr::VROverlayFlags(1 << 30), true); + std::printf("SetOverlayFlag(1 << 30): %s\n", ov->GetOverlayErrorNameFromEnum(e)); + } + uint32_t flags = 0; + ov->GetOverlayFlags(main, &flags); + std::printf("overlay flags now 0x%x; pid %d\n", flags, int(getpid())); + std::fflush(stdout); + + const auto start = Clock::now(); + auto ms = [&] { return std::chrono::duration(Clock::now() - start).count(); }; + int lastAvail = -1, lastDash = -1, lastActive = -1, lastY = -1, lastTrigger = -1; + while (!g_stop && (seconds <= 0 || ms() < seconds * 1000)) { + vr::VREvent_t ev; + while (sys->PollNextEvent(&ev, sizeof ev)) { + const char *name = sys->GetEventTypeNameFromEnum(vr::EVREventType(ev.eventType)); + if (ev.eventType == vr::VREvent_InputFocusChanged || ev.eventType == vr::VREvent_InputFocusCaptured || + ev.eventType == vr::VREvent_InputFocusReleased) + std::printf("%9.1f ms %s pid %u (old %u)\n", ms(), name, ev.data.process.pid, ev.data.process.oldPid); + else if (ev.eventType == vr::VREvent_ButtonPress || ev.eventType == vr::VREvent_ButtonUnpress || + ev.eventType == vr::VREvent_DashboardActivated || ev.eventType == vr::VREvent_DashboardDeactivated) + std::printf("%9.1f ms %s device %u button %u\n", ms(), name, ev.trackedDeviceIndex, ev.data.controller.button); + } + while (ov->PollNextOverlayEvent(main, &ev, sizeof ev)) { + const char *name = sys->GetEventTypeNameFromEnum(vr::EVREventType(ev.eventType)); + if (ev.eventType != vr::VREvent_MouseMove && std::strncmp(name, "Unknown", 7) != 0 && + std::strcmp(name, "VREvent_OverlayMouseFocusChanged") != 0 && ev.eventType != vr::VREvent_PropertyChanged) + std::printf("%9.1f ms overlay: %s\n", ms(), name); + } + if (manifest) { + vr::VRActiveActionSet_t active{}; + active.ulActionSet = set; + vr::VRInput()->UpdateActionState(&active, sizeof active, 1); + vr::InputDigitalActionData_t y{}, t{}; + vr::VRInput()->GetDigitalActionData(actY, &y, sizeof y, vr::k_ulInvalidInputValueHandle); + vr::VRInput()->GetDigitalActionData(actTrigger, &t, sizeof t, vr::k_ulInvalidInputValueHandle); + const int yv = y.bActive ? y.bState : -2, tv = t.bActive ? t.bState : -2; + if (yv != lastY || tv != lastTrigger) { + std::printf("%9.1f ms our actions: y %d, trigger %d (-2: inactive)\n", ms(), yv, tv); + lastY = yv, lastTrigger = tv; + } + } + const int avail = sys->IsInputAvailable(), dash = ov->IsDashboardVisible(), active = ov->IsActiveDashboardOverlay(main); + if (avail != lastAvail || dash != lastDash || active != lastActive) { + std::printf("%9.1f ms input available %d, dashboard %d, our page active %d\n", ms(), avail, dash, active); + lastAvail = avail, lastDash = dash, lastActive = active; + } + std::fflush(stdout); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + vr::VR_Shutdown(); + return 0; +} diff --git a/pointer/probe/focustest_actions/actions.json b/pointer/probe/focustest_actions/actions.json new file mode 100644 index 0000000..c376b58 --- /dev/null +++ b/pointer/probe/focustest_actions/actions.json @@ -0,0 +1,15 @@ +{ + "default_bindings": [ + {"controller_type": "frame_controller", "binding_url": "bindings_frame_controller.json"} + ], + "actions": [ + {"name": "/actions/focus/in/y", "type": "boolean"}, + {"name": "/actions/focus/in/trigger", "type": "boolean"} + ], + "action_sets": [ + {"name": "/actions/focus", "usage": "leftright"} + ], + "localization": [ + {"language_tag": "en_US", "/actions/focus": "Focus test", "/actions/focus/in/y": "Y", "/actions/focus/in/trigger": "Trigger"} + ] +} diff --git a/pointer/probe/focustest_actions/bindings_frame_controller.json b/pointer/probe/focustest_actions/bindings_frame_controller.json new file mode 100644 index 0000000..e9e3baa --- /dev/null +++ b/pointer/probe/focustest_actions/bindings_frame_controller.json @@ -0,0 +1,15 @@ +{ + "action_manifest_version": 0, + "controller_type": "frame_controller", + "description": "focustest (pointer/probe/focustest.cpp)", + "name": "focustest", + "bindings": { + "/actions/focus": { + "sources": [ + {"path": "/user/hand/right/input/y", "mode": "button", "inputs": {"click": {"output": "/actions/focus/in/y"}}}, + {"path": "/user/hand/right/input/trigger", "mode": "button", "inputs": {"click": {"output": "/actions/focus/in/trigger"}}}, + {"path": "/user/hand/left/input/trigger", "mode": "button", "inputs": {"click": {"output": "/actions/focus/in/trigger"}}} + ] + } + } +} diff --git a/pointer/probe/lasertest.cpp b/pointer/probe/lasertest.cpp index 435acd5..729a0be 100644 --- a/pointer/probe/lasertest.cpp +++ b/pointer/probe/lasertest.cpp @@ -2,9 +2,10 @@ // Prints the dashboard's primary device and every controller's role whenever either changes, // and the dashboard and role events. With --snapback, when the laser goes to any device but // ours (the ft_pointer driver's), it presses our /input/a (switchlaserhand) to take it back, -// at most every 100 ms, and prints how long that took. +// at most every 100 ms, and prints how long that took. With --reclaim MS it also takes the +// laser back when it has been on no device (gamepad mode) for MS milliseconds. // Runs as a background OpenVR client (in the dev container). -// Usage: lasertest [--snapback] [--seconds N] +// Usage: lasertest [--snapback] [--reclaim MS] [--seconds N] #include #include @@ -47,12 +48,13 @@ static std::string Describe(vr::IVRSystem *sys, vr::TrackedDeviceIndex_t i) { int main(int argc, char **argv) { bool snapback = false; - double seconds = 0; + double seconds = 0, reclaimMs = 0; for (int i = 1; i < argc; ++i) { if (!std::strcmp(argv[i], "--snapback")) snapback = true; else if (!std::strcmp(argv[i], "--seconds") && i + 1 < argc) seconds = std::atof(argv[++i]); + else if (!std::strcmp(argv[i], "--reclaim") && i + 1 < argc) reclaimMs = std::atof(argv[++i]); else { - std::fprintf(stderr, "usage: %s [--snapback] [--seconds N]\n", argv[0]); + std::fprintf(stderr, "usage: %s [--snapback] [--reclaim MS] [--seconds N]\n", argv[0]); return 2; } } @@ -67,7 +69,7 @@ int main(int argc, char **argv) { const auto start = Clock::now(); auto ms = [&] { return std::chrono::duration(Clock::now() - start).count(); }; std::string lastPrimary, lastRoles; - Clock::time_point lostAt{}, pressedAt{}; + Clock::time_point lostAt{}, pressedAt{}, noneSince{}; bool pressed = false; while (!g_stop && (seconds <= 0 || ms() < seconds * 1000)) { vr::VREvent_t ev; @@ -112,20 +114,29 @@ int main(int argc, char **argv) { } if (primary != ours && ours != vr::k_unTrackedDeviceIndexInvalid && primary != vr::k_unTrackedDeviceIndexInvalid) lostAt = Clock::now(); + noneSince = primary == vr::k_unTrackedDeviceIndexInvalid ? Clock::now() : Clock::time_point{}; lastPrimary = p; } - if (snapback) { + if (snapback || reclaimMs > 0) { const auto now = Clock::now(); if (pressed && now - pressedAt > std::chrono::milliseconds(40)) { Send("btn a 0"); pressed = false; } - if (!pressed && lostAt != Clock::time_point{} && now - pressedAt > std::chrono::milliseconds(100)) { + if (snapback && !pressed && lostAt != Clock::time_point{} && now - pressedAt > std::chrono::milliseconds(100)) { Send("btn a 1"); pressed = true; pressedAt = now; std::printf("%9.1f ms snapback: pressed our a\n", ms()); } + if (!pressed && reclaimMs > 0 && noneSince != Clock::time_point{} && ours != vr::k_unTrackedDeviceIndexInvalid && + sys->IsTrackedDeviceConnected(ours) && + now - noneSince > std::chrono::milliseconds(int(reclaimMs)) && now - pressedAt > std::chrono::milliseconds(100)) { + Send("btn a 1"); + pressed = true; + pressedAt = now; + std::printf("%9.1f ms reclaim: pressed our a\n", ms()); + } } std::fflush(stdout); std::this_thread::sleep_for(std::chrono::milliseconds(10)); diff --git a/pointer/probe/vrsetting.cpp b/pointer/probe/vrsetting.cpp new file mode 100644 index 0000000..74c2799 --- /dev/null +++ b/pointer/probe/vrsetting.cpp @@ -0,0 +1,48 @@ +// Read or change one SteamVR setting through vrserver (IVRSettings). Editing steamvr.vrsettings +// while SteamVR runs doesn't stick: vrserver writes its own copy back on the next change. +// Runs as a background OpenVR client (in the dev container). +// Usage: vrsetting get SECTION KEY +// vrsetting set-bool SECTION KEY true|false +// vrsetting remove SECTION KEY +#include + +#include +#include + +int main(int argc, char **argv) { + if (argc < 4 || (std::strcmp(argv[1], "set-bool") == 0 && argc < 5)) { + std::fprintf(stderr, "usage: %s get|set-bool|remove SECTION KEY [true|false]\n", argv[0]); + return 2; + } + const char *cmd = argv[1], *section = argv[2], *key = argv[3]; + vr::EVRInitError err = vr::VRInitError_None; + vr::VR_Init(&err, vr::VRApplication_Background); + if (err != vr::VRInitError_None) { + std::printf("VR_Init failed: %s\n", vr::VR_GetVRInitErrorAsEnglishDescription(err)); + return 1; + } + vr::EVRSettingsError serr = vr::VRSettingsError_None; + int rc = 0; + if (!std::strcmp(cmd, "get")) { + char s[512] = ""; + vr::VRSettings()->GetString(section, key, s, sizeof s, &serr); + if (serr == vr::VRSettingsError_None) std::printf("%s\n", s); + else { + const bool b = vr::VRSettings()->GetBool(section, key, &serr); + std::printf("%s\n", serr == vr::VRSettingsError_None ? (b ? "true" : "false") : "(unset)"); + } + } else if (!std::strcmp(cmd, "set-bool")) { + vr::VRSettings()->SetBool(section, key, !std::strcmp(argv[4], "true"), &serr); + } else if (!std::strcmp(cmd, "remove")) { + vr::VRSettings()->RemoveKeyInSection(section, key, &serr); + } else { + std::fprintf(stderr, "unknown command %s\n", cmd); + rc = 2; + } + if (serr != vr::VRSettingsError_None) { + std::printf("error: %s\n", vr::VRSettings()->GetSettingsErrorNameFromEnum(serr)); + rc = 1; + } + vr::VR_Shutdown(); + return rc; +}