Gaze first: precision and drag buttons, key combinations, pointer role

Two new actions for mouse buttons, controller buttons, and key
combinations: gaze_precision (hold: the pointer stops where you look and
the button's device steers it, a controller by where it points at
POINTER_PRECISION_GAIN, the mouse by its moves; release: click there) and
gaze_drag (the same with a real press at once, dragging until the
release). The relay sends "precision|gazedrag <source> 1|0" to the
helper, which runs them through the same holds as pinches and grips.

- Key combinations on any keyboard ("key_bindings" in the rules, e.g.
  Ctrl+Alt+G for gaze on/off): the last key isn't typed.
- POINTER_GAZE_MOUSE: in gaze mode the left button is a precision button
  (the default, as before) or clicks right away (direct).
- In gaze mode a moving controller no longer takes the pointer away.
- POINTER_ROLE (right, left, stylus): the driver takes a "role" command,
  so the pointer can stay off the hand holding the precision controller,
  which would otherwise take the role back. Needs the rebuilt driver.
- Input Settings: the actions on the Controllers and Buttons pages; on the
  Gaze page the mouse choice, the role, the precision sliders, and key
  combinations (captured from any keyboard).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

input-settings/ft_input_settings.py

@@ -60,6 +60,8 @@ ACTION_LABELS = {
     "scroll_up": "Scroll up", "scroll_down": "Scroll down", "dashboard": "Toggle SteamVR dashboard",
     "recenter": "Recenter pointer", "pointer_toggle": "Pointer on/off",
     "follow_toggle": "Head follow on/off (experimental)", "gaze_toggle": "Gaze pointer on/off (experimental)",
+    "gaze_precision": "Gaze precision: hold to steer, release to click",
+    "gaze_drag": "Gaze drag: press where you look, steer, release",
     "sens_up": "Faster pointer",
     "sens_down": "Slower pointer", "layout_reset": "Reset desktop screen layout",
     "screens_toggle": "Hide/show desktop screens", "keyboard_toggle": "Open/close keyboard",
@@ -92,7 +94,18 @@ GAZE_SETTINGS = [
     ("POINTER_GAZE_NUDGE_MAX", "Largest nudge to learn", 8, 1, 30, 0.5, "°"),
     ("POINTER_GAZE_HOLD", "Hold still to drag", 0.5, 0.1, 2.0, 0.05, "s"),
     ("POINTER_GAZE_SHOW", "Dot shows after moving", 1.0, 0.0, 5.0, 0.1, "s"),
+    ("POINTER_PRECISION_GAIN", "Precision steering", 0.5, 0.1, 2.0, 0.05, "×"),
+    ("POINTER_PRECISION_DEADZONE", "Precision dead zone", 0.3, 0.0, 3.0, 0.1, "°"),
+    ("POINTER_GAZE_DRAG_GAIN", "Drag steering", 1.0, 0.1, 2.0, 0.05, "×"),
 ]
+# In gaze mode, what the mouse's left button does (POINTER_GAZE_MOUSE).
+GAZE_MOUSE = {"precision": "Gaze precision: hold to steer with the mouse, release to click",
+              "direct": "Click right away where the pointer is"}
+# The hand role the pointer's virtual controller takes (POINTER_ROLE).
+POINTER_ROLES = {"right": "Right hand", "left": "Left hand", "stylus": "Stylus (no hand)"}
+# Key combinations ("key_bindings" in the rules): modifiers, either side folded into the left code.
+MODIFIER_CODES = {29: 29, 97: 29, 42: 42, 54: 42, 56: 56, 100: 56, 125: 125, 126: 125}
+MODIFIER_NAMES = {29: "Ctrl", 42: "Shift", 56: "Alt", 125: "Meta"}
 # The gaze service's settings (gaze/ft-gazed): whose eye tracking, and the eye bias.
 GAZE_TRACKERS = {"steam": "SteamVR's eye tracker", "own": "our own eye tracker (frame-eyes)"}
 GAZE_EYES = {"auto": "auto", "left": "left eye", "right": "right eye"}
@@ -223,6 +236,7 @@ class Backend(QObject):
     activity = Signal(str)  # device id
     captured = Signal(int, str)  # code, name
     capturedController = Signal(str, str)  # button, label
+    shortcutCaptureChanged = Signal()
     message = Signal(str, bool)  # text, is error
 
     def __init__(self):
@@ -234,6 +248,8 @@ class Backend(QObject):
         self._bluetooth = []
         self._relay_ok = False
         self._capture_vr = False
+        self._capture_combo = ""  # the action a key combination is being captured for
+        self._combo_mods = set()
         self._vr = {}  # the helper's vrstatus, {} when it doesn't answer
         self._vr_at = 0.0
         self._gaze = {}  # ft-gazed's status, {} when it isn't running
@@ -338,6 +354,17 @@ class Backend(QObject):
                     self._capture_vr = False
                     self._send("vrcapture 0")
                     self.capturedController.emit(msg["code"], CONTROLLER_BUTTONS[msg["code"]])
+            elif t == "event" and self._capture_combo and msg.get("type") == "key":
+                self.activity.emit(msg["id"])
+                code, value = int(msg["code"]), msg["value"]
+                if code in MODIFIER_CODES:
+                    (self._combo_mods.add if value else self._combo_mods.discard)(MODIFIER_CODES[code])
+                elif value == 1 and code < BTN_MISC:
+                    self._save_shortcut("+".join(str(c) for c in sorted(self._combo_mods) + [code]),
+                                        self._capture_combo)
+                    self._capture_combo = ""
+                    self._combo_mods = set()
+                    self.shortcutCaptureChanged.emit()
             elif t == "event":
                 self.activity.emit(msg["id"])
                 if (self._capture_id and msg["id"] == self._capture_id and msg["type"] == "key"
@@ -765,6 +792,85 @@ class Backend(QObject):
         self.reload_timer.start()
         self.gazeChanged.emit()
 
+    @Property(str, notify=pointerChanged)
+    def gazeMouse(self):
+        v = read_conf().get("POINTER_GAZE_MOUSE", "precision")
+        return v if v in GAZE_MOUSE else "precision"
+
+    @Property("QVariantList", constant=True)
+    def gazeMouseChoices(self):
+        return [{"value": k, "text": v} for k, v in GAZE_MOUSE.items()]
+
+    @Slot(str)
+    def setGazeMouse(self, mode):
+        if mode in GAZE_MOUSE:
+            write_conf_value("POINTER_GAZE_MOUSE", mode)
+            self.reload_timer.start()
+            self.pointerChanged.emit()
+            self.message.emit(f"Mouse in gaze mode: {GAZE_MOUSE[mode].lower()}", False)
+
+    @Property(str, notify=pointerChanged)
+    def pointerRole(self):
+        v = read_conf().get("POINTER_ROLE", "right")
+        return v if v in POINTER_ROLES else "right"
+
+    @Property("QVariantList", constant=True)
+    def pointerRoles(self):
+        return [{"value": k, "text": v} for k, v in POINTER_ROLES.items()]
+
+    @Slot(str)
+    def setPointerRole(self, role):
+        if role in POINTER_ROLES:
+            write_conf_value("POINTER_ROLE", role)
+            self.reload_timer.start()
+            self.pointerChanged.emit()
+            self.message.emit(f"Pointer role: {POINTER_ROLES[role].lower()} (from its next wake)", False)
+
+    # --- key combinations ("key_bindings") ---
+    def comboName(self, combo):
+        parts = [int(c) for c in combo.split("+") if c.isdigit()]
+        return "+".join(MODIFIER_NAMES.get(c) or self.codeName(c).removeprefix("KEY_").title() for c in parts)
+
+    @Property("QVariantList", notify=mappingsChanged)
+    def keyShortcuts(self):
+        bound = read_json(RULES_PATH).get("key_bindings", {}) or {}
+        return [{"combo": c, "label": self.comboName(c), "action": a, "actionLabel": ACTION_LABELS.get(a, a)}
+                for c, a in sorted(bound.items())]
+
+    @Property("QVariantList", constant=True)
+    def shortcutActions(self):
+        return [{"value": a, "text": ACTION_LABELS[a]} for a in CONTROLLER_ACTIONS]
+
+    @Property(bool, notify=shortcutCaptureChanged)
+    def capturingShortcut(self):
+        return bool(self._capture_combo)
+
+    @Slot(str)
+    def startShortcutCapture(self, action):
+        if action in CONTROLLER_ACTIONS:
+            self._capture_combo = action
+            self._combo_mods = set()
+            self._send("watch 60")
+            self.shortcutCaptureChanged.emit()
+
+    @Slot()
+    def cancelShortcutCapture(self):
+        self._capture_combo = ""
+        self.shortcutCaptureChanged.emit()
+
+    def _save_shortcut(self, combo, action):
+        rules = read_json(RULES_PATH)
+        rules.setdefault("key_bindings", {})[combo] = action
+        self._save_rules(rules)
+        self.message.emit(f"{self.comboName(combo)} → {ACTION_LABELS[action]}", False)
+
+    @Slot(str)
+    def removeShortcut(self, combo):
+        rules = read_json(RULES_PATH)
+        (rules.get("key_bindings") or {}).pop(combo, None)
+        self._save_rules(rules)
+        self.message.emit(f"{self.comboName(combo)} removed", False)
+
     @Property(str, notify=gazeChanged)
     def gazeTracker(self):
         """Whose eye tracking the gaze service uses: "steam" or "own" (GAZE_TRACKER)."""

input-settings/main.qml

@@ -821,6 +821,91 @@ Kirigami.ApplicationWindow {
                     opacity: 0.7
                     font: Kirigami.Theme.smallFont
                 }
+                ColumnLayout {
+                    Kirigami.FormData.label: "Mouse left button:"
+                    Repeater {
+                        model: backend.gazeMouseChoices
+                        delegate: Controls.RadioButton {
+                            required property var modelData
+                            text: modelData.text
+                            checked: backend.gazeMouse === modelData.value
+                            onToggled: if (checked) backend.setGazeMouse(modelData.value)
+                        }
+                    }
+                }
+                Controls.Label {
+                    Layout.maximumWidth: Kirigami.Units.gridUnit * 30
+                    wrapMode: Text.WordWrap
+                    text: "Controllers: map a button to Gaze precision (hold, point the controller to steer, release to "
+                          + "click) or Gaze drag (the same, pressed at once) on the Controllers page. Gaze pointer on/off "
+                          + "can go on a controller button, a mouse button, or a key combination below."
+                    opacity: 0.7
+                    font: Kirigami.Theme.smallFont
+                }
+                ColumnLayout {
+                    Kirigami.FormData.label: "Pointer takes the role:"
+                    Repeater {
+                        model: backend.pointerRoles
+                        delegate: Controls.RadioButton {
+                            required property var modelData
+                            text: modelData.text
+                            checked: backend.pointerRole === modelData.value
+                            onToggled: if (checked) backend.setPointerRole(modelData.value)
+                        }
+                    }
+                }
+                Controls.Label {
+                    Layout.maximumWidth: Kirigami.Units.gridUnit * 30
+                    wrapMode: Text.WordWrap
+                    text: "A controller in your hand takes that hand's role, and then the pointer's clicks don't land. "
+                          + "Holding the right controller as the precision tool, give the pointer the left hand, or the "
+                          + "stylus role. It applies the next time the pointer wakes."
+                    opacity: 0.7
+                    font: Kirigami.Theme.smallFont
+                }
+
+                Kirigami.Separator { Kirigami.FormData.isSection: true; Kirigami.FormData.label: "Key combinations" }
+
+                Repeater {
+                    model: backend.keyShortcuts
+                    delegate: RowLayout {
+                        required property var modelData
+                        Kirigami.FormData.label: modelData.label + ":"
+                        Controls.Label { text: modelData.actionLabel }
+                        Controls.ToolButton {
+                            icon.name: "edit-delete"
+                            display: Controls.AbstractButton.IconOnly
+                            text: "Remove"
+                            Controls.ToolTip.text: text
+                            Controls.ToolTip.visible: hovered
+                            onClicked: backend.removeShortcut(modelData.combo)
+                        }
+                    }
+                }
+                RowLayout {
+                    Kirigami.FormData.label: "New:"
+                    Controls.ComboBox {
+                        id: shortcutAction
+                        model: backend.shortcutActions
+                        textRole: "text"
+                        valueRole: "value"
+                        Component.onCompleted: currentIndex = indexOfValue("gaze_toggle")
+                        Layout.preferredWidth: Kirigami.Units.gridUnit * 16
+                    }
+                    Controls.Button {
+                        text: backend.capturingShortcut ? "Press the keys… (Cancel)" : "Set keys…"
+                        onClicked: backend.capturingShortcut ? backend.cancelShortcutCapture()
+                                                             : backend.startShortcutCapture(shortcutAction.currentValue)
+                    }
+                }
+                Controls.Label {
+                    Layout.maximumWidth: Kirigami.Units.gridUnit * 30
+                    wrapMode: Text.WordWrap
+                    text: "Hold the modifiers (Ctrl, Alt, Shift, Meta), then press the key, on any keyboard. The "
+                          + "combination's last key isn't typed; the modifiers still reach the app."
+                    opacity: 0.7
+                    font: Kirigami.Theme.smallFont
+                }
                 RowLayout {
                     Kirigami.FormData.label: "Eye tracker:"
                     Controls.RadioButton {

input/input-relay.py

@@ -20,12 +20,17 @@ keyboard node for its extra buttons). Roles, from ~/.config/frametop-input.json
 Buttons and keys of pointer devices go through a per-device map to actions
 (left, right, middle, back, scroll_up, scroll_down, dashboard, recenter,
 pointer_toggle, follow_toggle = head follow on or off, gaze_toggle = gaze mode on or off
-(the pointer goes where you look; see pointer/helper/ft-pointer.cpp), sens_up, sens_down,
+(the pointer goes where you look; see pointer/helper/ft-pointer.cpp), gaze_precision = while
+held, the pointer stops where you look and the button's device (the mouse, or that
+controller's aim) steers it, and the release clicks there, gaze_drag = the same, but pressed
+at once, so it drags ("precision|gazedrag mouse|left|right|keyboard 1|0" to the helper), sens_up, sens_down,
 layout_reset = put the desktop screens back in their saved layout, screens_toggle = hide or show the desktop screens,
 keyboard_toggle = open or close Frametop's keyboard, key = pass through as a key, none).
 
 Frame controller buttons can be mapped too ("controller_buttons": {"right/a": action} in the
-rules file; any action but key). The controllers aren't input devices here, only SteamVR sees
+rules file; any action but key). So can key combinations on any keyboard ("key_bindings":
+{"29+56+34": action}, evdev codes joined by "+", modifiers first and left-hand codes for
+either side, here Ctrl+Alt+G): the combination does the action, and its last key isn't typed. The controllers aren't input devices here, only SteamVR sees
 them, so the pointer helper reads them with SteamVR input and sends "vrbtn <button> 1|0".
 It only takes the buttons the relay tells it to ("vrbind <button>..." to @ft_pointer_helper,
 sent on start, reload, and when the helper says "vrhello"), and only while no game runs,
@@ -168,8 +173,10 @@ def eviocguniq(length):
 VIRTUAL_PREFIX = "frametop virtual"
 RULES_PATH = os.path.expanduser("~/.config/frametop-input.json")
 ACTIONS = ("left", "right", "middle", "back", "scroll_up", "scroll_down", "dashboard", "recenter",
-           "pointer_toggle", "follow_toggle", "gaze_toggle", "sens_up", "sens_down", "layout_reset", "screens_toggle",
-           "keyboard_toggle", "key", "none")
+           "pointer_toggle", "follow_toggle", "gaze_toggle", "gaze_precision", "gaze_drag", "sens_up", "sens_down",
+           "layout_reset", "screens_toggle", "keyboard_toggle", "key", "none")
+# Key combinations ("key_bindings"): modifiers, each side's code folded into the left one's.
+MODIFIERS = {29: 29, 97: 29, 42: 42, 54: 42, 56: 56, 100: 56, 125: 125, 126: 125}
 VR_KEYBOARD_MODES = ("always", "no_keyboard", "button", "never")  # when Frametop's keyboard opens
 HELPER = "\0ft_pointer_helper"
 # Frame controller buttons the pointer helper can read (pointer/helper/vrbuttons.h).
@@ -275,6 +282,7 @@ def read_config(path=os.path.expanduser("~/.config/frametop.conf")):
 def read_rules(path=RULES_PATH):
     """{"devices": {id: {"role", "name"}}, "buttons": {id: {"<code>": action}},
     "controller_buttons": {"<hand>/<button>": action}, "controller_in_games": bool,
+    "key_bindings": {"<code>+<code>...": action},
     "vr_keyboard": one of VR_KEYBOARD_MODES, "vr_keyboard_persist": bool}."""
     try:
         with open(path) as f:
@@ -284,6 +292,7 @@ def read_rules(path=RULES_PATH):
     rules.setdefault("devices", {})
     rules.setdefault("buttons", {})
     rules.setdefault("controller_buttons", {})
+    rules.setdefault("key_bindings", {})
     return rules
 
 
@@ -419,10 +428,17 @@ class Pointer:
             self.send(f"scroll 0 {1 if value > 0 else -1}")
             self.scroll_until = now + self.SCROLL_PULSE
 
-    def action(self, name, value, now):
-        """A mapped button: value 1 press, 0 release, 2 autorepeat (ignored)."""
+    def action(self, name, value, now, source="mouse"):
+        """A mapped button: value 1 press, 0 release, 2 autorepeat (ignored). source: what
+        pressed it (mouse, left, right for a controller, keyboard), for the gaze actions."""
         if value == 2:
             return
+        if name in ("gaze_precision", "gaze_drag"):
+            if value == 1:
+                self.wake(now)
+            self.flush()
+            self.send(f"{'precision' if name == 'gaze_precision' else 'gazedrag'} {source} {value}")
+            return
         driver = self.DRIVER_BUTTONS.get(name)
         if driver:
             self.wake(now)
@@ -643,7 +659,7 @@ def main():
                              "type": "vr", "code": button, "value": value})
         action = state["rules"]["controller_buttons"].get(button)
         if state["pointer"] and action in ACTIONS and action not in ("key", "none"):
-            do_action(action, value, now)
+            do_action(action, value, now, button.split("/")[0])
 
     def vr_keyboard_mode():
         mode = state["rules"].get("vr_keyboard")
@@ -666,13 +682,39 @@ def main():
                 n.candidate and n.is_keyboard and n.role == "passthrough" and not n.uinput for n in nodes.values())):
             vr_keyboard("show")
 
-    def do_action(action, value, now):
-        """A mapped mouse or controller button (pointer mode only)."""
+    def do_action(action, value, now, source="mouse"):
+        """A mapped mouse or controller button, or key combination (pointer mode only)."""
         if action == "keyboard_toggle":
             if value == 1 and vr_keyboard_mode() != "never":
                 vr_keyboard("toggle")
         else:
-            state["pointer"].action(action, value, now)
+            state["pointer"].action(action, value, now, source)
+
+    held_modifiers = set()  # on any keyboard, folded (MODIFIERS)
+    combos_down = {}  # key code -> the action its combination started (released with it)
+
+    def key_binding(code, value, now):
+        """A key from a keyboard: does it complete a key combination ("key_bindings")? True if
+        it was taken for one (then it isn't typed)."""
+        if code in MODIFIERS:
+            (held_modifiers.add if value else held_modifiers.discard)(MODIFIERS[code])
+            return False
+        if value == 0 and code in combos_down:
+            action = combos_down.pop(code)
+            if state["pointer"]:
+                do_action(action, 0, now, "keyboard")
+            return True
+        if value != 1 or not state["rules"]["key_bindings"]:
+            return value == 2 and code in combos_down
+        combo = "+".join(str(c) for c in sorted(held_modifiers) + [code])
+        action = state["rules"]["key_bindings"].get(combo)
+        if action not in ACTIONS or action in ("key", "none"):
+            return False
+        combos_down[code] = action
+        if state["pointer"]:
+            do_action(action, 1, now, "keyboard")
+        log(f"key combination {combo}: {action}")
+        return True
 
 
     def to_screens(code, value):
@@ -959,6 +1001,8 @@ def main():
                 if node.role != "pointer":
                     # Observed only, unless typing goes to the desktop. With META_DASHBOARD=1,
                     # a Meta tap on any keyboard toggles the dashboard.
+                    if node.role == "passthrough" and etype == EV_KEY and code < BTN_MISC and key_binding(code, value, now):
+                        continue
                     if node.role == "passthrough" and etype == EV_KEY:
                         if value == 1 and code < BTN_MISC and now - last_typing >= 0.25:
                             last_typing = now

pointer/driver/driver_ft_pointer.cpp

@@ -17,6 +17,11 @@
 //   btn <name> <0|1>             name: trigger, b, x, system, joystick, a (a = claim the laser, no click)
 //   scroll <x> <y>               joystick deflection -1..1
 //   show | hide                  connect (take the hand role) or disconnect (give it back)
+//   role right|left|stylus       which role to hint while connected, from now on (the helper
+//                                sends POINTER_ROLE). A Frame controller in your hand takes
+//                                its hand's role back (it counts as used while held), so a
+//                                controller used beside the pointer needs the pointer on the
+//                                other hand, or on no hand at all
 //
 // The device starts disconnected, so it never holds a hand role at boot (holding
 // the right hand while SteamVR started left the Steam UI stuck loading). It
@@ -58,6 +63,7 @@ struct State {
 
     float yaw = 0.f, pitch = 0.f;  // degrees
     bool buttons[6] = {};
+    int32_t role = 0;  // "role": the hint while connected; 0 = the configured one
     float distance = 1.5f;
     float scrollX = 0.f, scrollY = 0.f;
 };
@@ -173,6 +179,7 @@ public:
             std::memcpy(snapshot.buttons, state_->buttons, sizeof snapshot.buttons);
             snapshot.scrollX = state_->scrollX;
             snapshot.scrollY = state_->scrollY;
+            snapshot.role = state_->role;
         }
 
         DriverPose_t pose{};
@@ -202,6 +209,10 @@ public:
             pose.qRotation = QuatFromYawPitch(float(std::atan2(-d[0], -d[2]) * 180.0 / M_PI),
                                               float(std::atan2(d[1], horizontal) * 180.0 / M_PI));
         }
+        if (snapshot.role > 0 && snapshot.role != role_) {
+            role_ = snapshot.role;
+            if (hinted_) VRProperties()->SetInt32Property(container_, Prop_ControllerRoleHint_Int32, role_);
+        }
         if (snapshot.visible != hinted_) {
             // Claim the hand before connecting; give it up when disconnecting.
             VRProperties()->SetInt32Property(container_, Prop_ControllerRoleHint_Int32,
@@ -328,6 +339,11 @@ private:
             state_.visible = false;
         } else if (std::strncmp(cmd, "show", 4) == 0) {
             state_.visible = true;
+        } else if (std::sscanf(cmd, "role %31s", name) == 1) {
+            state_.role = !std::strcmp(name, "left")     ? TrackedControllerRole_LeftHand
+                          : !std::strcmp(name, "right")  ? TrackedControllerRole_RightHand
+                          : !std::strcmp(name, "stylus") ? TrackedControllerRole_Stylus
+                                                         : state_.role;
         } else if (std::sscanf(cmd, "btn %31s %d", name, &v) == 2) {
             for (int i = 0; i < kButtons; ++i)
                 if (std::strcmp(name, kButtonNames[i]) == 0) state_.buttons[i] = v != 0;

pointer/helper/ft-pointer.cpp

@@ -163,6 +163,25 @@
 // press dragged onto the target is the same: from the raw gaze at the press to the release. With no
 // fresh gaze (a blink, the service stopped, the headset off), the pointer stays put.
 //
+// Gaze precision (the relay's gaze_precision and gaze_drag actions, bound to a controller
+// button, a mouse button, or a key combination; "precision|gazedrag <source> 1|0" here): gaze
+// mode aims, the button makes it exact. Pressing gaze precision stops the pointer where you
+// look, as gaze mode's mouse press does; while it's held, the button's device steers the
+// pointer: a controller by where it points, at POINTER_PRECISION_GAIN (0.5: half its turn, for
+// precision) past POINTER_PRECISION_DEADZONE (0.3 deg, the press's own jolt), and the mouse by
+// its moves. Releasing clicks where the pointer is. A correction is a lesson for the gaze
+// tracker, as with the mouse. Gaze drag is the same with a real press at once, dragging until the
+// release (at POINTER_GAZE_DRAG_GAIN, 1), for title bars, grab bars, and selections. Without
+// gaze mode both still work from wherever the pointer is.
+//   In gaze mode the mouse's left button is a gaze precision button (POINTER_GAZE_MOUSE =
+// precision, the default), or clicks at once where the pointer is (direct). And in gaze mode a
+// moving controller doesn't take the pointer away (last used wins is off): the gaze points,
+// and the controllers are its tools.
+//   POINTER_ROLE (right, left, or stylus): the hand role our device takes while connected. A
+// Frame controller in your hand counts as used through its touch sensors and takes its hand's
+// role back, and then no click lands (see "no hand role" in the main loop): with a controller in
+// the right hand as the precision tool, the pointer needs the left hand, or the stylus role.
+//
 // Hands (POINTER_HANDS, off by default; needs hand tracking, hands/): ft-hands publishes
 // pinches and grips (hands/include/fh_gestures.h), read here every frame.
 //   A pinch works like gaze mode's mouse press: the pointer stops (where the gaze put it), and
@@ -240,6 +259,8 @@
 // move to take the laser back, above. POINTER_IGNORE (empty): ignored panels, above.
 // POINTER_HANDS (0), POINTER_PINCH_GAIN (0.5), POINTER_PINCH_DEADZONE (1.5 deg),
 // POINTER_GRIP_GAIN (1), POINTER_GRIP_BELOW (0.35 m), POINTER_PINCH_TYPING (1 s): hands, above.
+// POINTER_PRECISION_GAIN (0.5), POINTER_PRECISION_DEADZONE (0.3 deg), POINTER_GAZE_DRAG_GAIN (1),
+// POINTER_GAZE_MOUSE (precision), POINTER_ROLE (right): gaze precision, above.
 #include <openvr.h>
 
 #include "vrbuttons.h"
@@ -650,6 +671,12 @@ int main() {
     // Hands (see the top): POINTER_HANDS, POINTER_PINCH_GAIN, POINTER_PINCH_DEADZONE, POINTER_GRIP_GAIN.
     bool handsOn = false;
     double pinchGain = 0.5, pinchDeadzone = 1.5, gripGain = 1.0, handBelow = 0.35, typingHold = 1.0;
+    // Gaze precision (see the top): POINTER_PRECISION_GAIN, POINTER_PRECISION_DEADZONE,
+    // POINTER_GAZE_DRAG_GAIN, POINTER_GAZE_MOUSE (precision: the mouse's left button holds
+    // back its press in gaze mode; direct: it clicks at once), POINTER_ROLE.
+    double precisionGain = 0.5, precisionDeadzone = 0.3, dragGain = 1.0;
+    bool gazeMousePrecision = true;
+    std::string role = "right";
     double pickupScale = 1;  // POINTER_CONTROLLER_PICKUP: scales the controller-moved limits
     std::vector<std::string> ignore;  // POINTER_IGNORE (see ParseIgnore)
     auto loadConfig = [&] {
@@ -680,6 +707,13 @@ int main() {
         gripGain = std::clamp(ConfDouble(conf, "POINTER_GRIP_GAIN", 1.0), 0.05, 3.0);
         handBelow = std::clamp(ConfDouble(conf, "POINTER_GRIP_BELOW", 0.35), 0.05, 1.0);
         typingHold = std::clamp(ConfDouble(conf, "POINTER_PINCH_TYPING", 1.0), 0.0, 5.0);
+        precisionGain = std::clamp(ConfDouble(conf, "POINTER_PRECISION_GAIN", 0.5), 0.05, 3.0);
+        precisionDeadzone = std::clamp(ConfDouble(conf, "POINTER_PRECISION_DEADZONE", 0.3), 0.0, 10.0);
+        dragGain = std::clamp(ConfDouble(conf, "POINTER_GAZE_DRAG_GAIN", 1.0), 0.05, 3.0);
+        const auto gm = conf.find("POINTER_GAZE_MOUSE");
+        gazeMousePrecision = gm == conf.end() || gm->second != "direct";
+        const auto ro = conf.find("POINTER_ROLE");
+        role = ro != conf.end() && (ro->second == "left" || ro->second == "stylus") ? ro->second : "right";
         pickupScale = std::clamp(ConfDouble(conf, "POINTER_CONTROLLER_PICKUP", 1), 0.5, 5.0);
         const auto ig = conf.find("POINTER_IGNORE");
         ignore = ParseIgnore(ig == conf.end() ? "" : ig->second);
@@ -779,6 +813,7 @@ int main() {
         wokeAt = t;
         active = true;
         recenter = true;
+        SendTo(out, "ft_pointer", "role " + role);  // POINTER_ROLE, before it takes it
         SendTo(out, "ft_pointer", "show");
         claimPending = true;  // take the laser without clicking, once SteamVR has bound the device
         claimAt = t + std::chrono::milliseconds(300);
@@ -806,19 +841,31 @@ int main() {
     // the gaze the pointer again when the press or click is over.
     vr::TrackedDeviceIndex_t ours = vr::k_unTrackedDeviceIndexInvalid;
     bool aimHeld = false, clickPress = false, clickRelease = false, gazeBack = false;
-    bool aimHand = false;  // the held-back press is a pinch's: it never turns into a real press
+    bool aimHand = false;  // the held-back press is a hold's (below): it never turns into a real press
     Clock::time_point aimSince{}, clickReleaseAt{};
-    // Hands (see the top): the gesture being held (side -1: none), where its hand pointed
-    // from (origin: the eye when it began, in the room) and the pointer then.
+    // Holds (see "Gaze precision" and "Hands" at the top): a pinch or grip, or a gaze
+    // precision or gaze drag button, held now. What steers the pointer meanwhile (a hand, a
+    // controller's aim, or the mouse through its own moves), from where it pointed when the
+    // hold began (for a hand: seen from the eye then, in the room), and the pointer then.
     HandGestures handFile;
     PoseHistory poses;
-    struct HandHold {
-        int side = -1;
-        bool grip = false, engaged = false;
+    enum class Src { None, Hand, Controller, Mouse };
+    struct Hold {
+        Src src = Src::None;
+        int side = -1;          // a hand's side (0 left, 1 right)
+        bool grip = false;      // pressed at once and dragging (a grip, gaze drag), not a click on release
+        bool engaged = false;   // past the dead zone
+        bool pressed = false;   // a real press went out (a grip or gaze drag, or a pinch without gaze mode)
         Vec3 origin;
-        bool pressed = false;  // a real press went out (a grip, or a pinch without gaze mode)
+        vr::TrackedDeviceIndex_t ctrl = vr::k_unTrackedDeviceIndexInvalid;
         double refYaw = 0, refPitch = 0, startYaw = 0, startPitch = 0, lastYaw = 0, lastPitch = 0;
     } hold;
+    // "precision|gazedrag <source> 1|0" from the relay, done in the frame (see Holds).
+    struct DevicePress {
+        std::string source;
+        bool drag, down;
+    };
+    std::vector<DevicePress> devicePresses;
     uint32_t seenBegins[2][2] = {}, seenEnds[2][2] = {};  // [pinch, grip][side], as last read
     bool handBaseline = false;
     int handOpens = 0;
@@ -876,7 +923,8 @@ int main() {
     // the top).
     auto leftButton = [&](bool down) {
         if (down) {
-            if (gazeOn && gazeOwns && !aimHeld && !clickPress && !clickRelease) {
+            if (gazeOn && gazeMousePrecision && gazeOwns && !aimHeld && !clickPress && !clickRelease &&
+                hold.src == Src::None) {
                 // Hold the press back: the pointer stops where the gaze put it.
                 gazeOwns = false;
                 nudging = haveHead && Clock::now() - gz.at < std::chrono::milliseconds(200);
@@ -888,7 +936,7 @@ int main() {
             pressLeft();
             return;
         }
-        if (aimHeld) {
+        if (aimHeld && !aimHand) {
             aimHeld = false;
             clickPress = true;  // after this frame's pose, so it lands where the pointer was moved to
             gazeBack = nudgeMoved < 0.2;
@@ -1115,6 +1163,17 @@ int main() {
                 gz = {g[0], g[1], g[2], g[3], Clock::now()};
                 continue;
             }
+            {
+                char kind[16], source[16];
+                int v;
+                if (std::sscanf(buf, "%15s %15s %d", kind, source, &v) == 3 &&
+                    (!std::strcmp(kind, "precision") || !std::strcmp(kind, "gazedrag"))) {
+                    lastMouse = Clock::now();
+                    if (!active) wake(Clock::now());
+                    devicePresses.push_back({source, !std::strcmp(kind, "gazedrag"), v != 0});
+                    continue;
+                }
+            }
             if (std::strcmp(buf, "typing") == 0) {  // not mouse input: it never wakes the pointer
                 lastTyping = Clock::now();
                 continue;
@@ -1308,7 +1367,9 @@ int main() {
         }
 
         // Last used wins: a real controller being moved releases the pointer (see the top).
-        if (active && tnow - lastMouse > std::chrono::milliseconds(500)) {
+        // Not in gaze mode: there the gaze points, and a controller is a tool for it (gaze
+        // precision), so moving one doesn't take the pointer away.
+        if (active && !gazeOn && hold.src == Src::None && tnow - lastMouse > std::chrono::milliseconds(500)) {
             for (vr::TrackedDeviceIndex_t i = 1; i < vr::k_unMaxTrackedDeviceCount; ++i) {
                 if (i == ours || !all[i].bPoseIsValid || all[i].eTrackingResult != vr::TrackingResult_Running_OK ||
                     sys->GetTrackedDeviceClass(i) != vr::TrackedDeviceClass_Controller) {
@@ -1403,19 +1464,54 @@ int main() {
                     gazeBack = nudgeMoved < 0.2;
                 }
             }
-            if (debug) std::printf("hand %s %s%s\n", hold.grip ? "grip" : "pinch", lost ? "lost" : "released",
-                                   hold.grip ? "" : lost ? " (no click)" : " (click)");
+            if (debug)
+                std::printf("hold %s %s%s\n", hold.src == Src::Hand ? (hold.grip ? "grip" : "pinch") : hold.grip ? "gaze drag" : "gaze precision",
+                            lost ? "lost" : "released", hold.pressed ? "" : lost ? " (no click)" : " (click)");
             if (debug) std::fflush(stdout);
-            hold.side = -1;
+            hold = {};
+        };
+        // The press, once a hold's source is set: a real press (dragging until the release), or
+        // held back like gaze mode's mouse press (see the top): the click comes on the release.
+        auto startHold = [&](bool press) {
+            hold.startYaw = hold.lastYaw = yaw, hold.startPitch = hold.lastPitch = pitch;
+            hold.pressed = press;
+            if (press) {
+                gazeBack = gazeOn && gazeOwns;
+                pressLeft();
+            } else {
+                gazeOwns = false;
+                nudging = gazeOn && haveHead && tnow - gz.at < std::chrono::milliseconds(200);
+                nudgeRawHy = gz.rhy, nudgeRawHp = gz.rhp, nudgeHead = lastHead;
+                nudgeAt = aimSince = tnow, nudgeMoved = 0;
+                aimHeld = aimHand = true;
+                pulseAt = tnow;
+            }
+        };
+        // The hold's source moved to (hy, hp): past the dead zone, the pointer follows at the gain,
+        // from where it was when it got past.
+        auto steer = [&](double hy, double hp, double deadzone, double gain) {
+            const double dy = std::remainder(hy - hold.refYaw, 360.0), dp = hp - hold.refPitch;
+            if (!hold.engaged) {
+                if (std::hypot(dy, dp) <= deadzone) return;
+                hold.engaged = true;
+                hold.refYaw = hy, hold.refPitch = hp;
+                hold.startYaw = hold.lastYaw = yaw, hold.startPitch = hold.lastPitch = pitch;
+                return;
+            }
+            yaw = std::remainder(hold.startYaw + gain * dy, 360.0);
+            pitch = std::clamp(hold.startPitch + gain * dp, -85.0, 85.0);
+            if (!hold.pressed) nudgeMoved += std::hypot(std::remainder(yaw - hold.lastYaw, 360.0), pitch - hold.lastPitch);
+            hold.lastYaw = yaw, hold.lastPitch = pitch;
+            lastMove = tnow;  // the dot shows while it moves (gaze mode)
         };
         auto beginHold = [&](int side, bool grip, const fh_pinch_t &g) {
             handUsed = tnow;
-            if (hold.side >= 0) {
+            if (hold.src != Src::None) {
                 // A grip takes over a pinch on the way to it (closing the hand passes through
-                // one); otherwise one gesture at a time.
-                if (!grip || hold.grip) return;
+                // one); otherwise one hold at a time.
+                if (hold.src != Src::Hand || !grip || hold.grip) return;
                 aimHeld = aimHand = false;
-                hold.side = -1;
+                hold = {};
             }
             if (leftHeld || aimHeld || clickPress || clickRelease) return;  // the mouse's button is busy
             if (!active) {
@@ -1433,30 +1529,69 @@ int main() {
                 return;
             }
             if (grip && Position(head).y - at.y > handBelow) {
-                if (debug) std::printf("hand %s ignored: %.2f m below the eyes\n", grip ? "grip" : "pinch", Position(head).y - at.y);
+                if (debug) std::printf("hand grip ignored: %.2f m below the eyes\n", Position(head).y - at.y);
                 return;
             }
-            hold = {side, grip, false, Position(head)};
-            if (!handAngles(g.begin_point, g.begin_ns, hold.origin, hold.refYaw, hold.refPitch)) return void(hold.side = -1);
-            hold.startYaw = yaw, hold.startPitch = pitch;
+            Hold h;
+            h.src = Src::Hand, h.side = side, h.grip = grip, h.origin = Position(head);
+            if (!handAngles(g.begin_point, g.begin_ns, h.origin, h.refYaw, h.refPitch)) return;
+            hold = h;
             // A grip, or a pinch without gaze mode: a real press where the pointer is (where
             // you look, in gaze mode), dragging with the hand until it opens.
-            hold.pressed = grip || !gazeOn;
-            if (hold.pressed) {
-                gazeBack = gazeOn && gazeOwns;
-                pressLeft();
-            } else {
-                // Held back like gaze mode's mouse press (see the top): the click comes on the release.
-                gazeOwns = false;
-                nudging = gazeOn && haveHead && tnow - gz.at < std::chrono::milliseconds(200);
-                nudgeRawHy = gz.rhy, nudgeRawHp = gz.rhp, nudgeHead = lastHead;
-                nudgeAt = aimSince = tnow, nudgeMoved = 0;
-                aimHeld = aimHand = true;
-                pulseAt = tnow;
-            }
+            startHold(grip || !gazeOn);
             if (debug) std::printf("hand %s %s began\n", side ? "right" : "left", grip ? "grip" : "pinch");
             if (debug) std::fflush(stdout);
         };
+        // A controller's aim: the yaw and pitch of its pointing direction, in the room.
+        auto controllerAngles = [&](vr::TrackedDeviceIndex_t i, double &cy, double &cp) {
+            if (i >= vr::k_unMaxTrackedDeviceCount || !all[i].bPoseIsValid) return false;
+            const auto &m = all[i].mDeviceToAbsoluteTracking.m;
+            const Vec3 f = Normalize({-m[0][2], -m[1][2], -m[2][2]});
+            cy = std::atan2(-f.x, -f.z) * 180 / M_PI;
+            cp = std::asin(std::clamp(f.y, -1.0, 1.0)) * 180 / M_PI;
+            return true;
+        };
+        // The Frame controller in a hand (not our own device, which may hold that hand's role).
+        auto controllerFor = [&](const std::string &hand) {
+            const int32_t want = hand == "left" ? vr::TrackedControllerRole_LeftHand : vr::TrackedControllerRole_RightHand;
+            for (vr::TrackedDeviceIndex_t i = 1; i < vr::k_unMaxTrackedDeviceCount; ++i) {
+                if (i == ours || sys->GetTrackedDeviceClass(i) != vr::TrackedDeviceClass_Controller) continue;
+                if (sys->GetInt32TrackedDeviceProperty(i, vr::Prop_ControllerRoleHint_Int32) == want ||
+                    sys->GetControllerRoleForTrackedDeviceIndex(i) == want)
+                    return i;
+            }
+            return vr::k_unTrackedDeviceIndexInvalid;
+        };
+        // Gaze precision and gaze drag buttons (see the top).
+        for (const DevicePress &p : devicePresses) {
+            const bool fromController = p.source == "left" || p.source == "right";
+            if (!p.down) {
+                if (hold.src == Src::Controller || hold.src == Src::Mouse) endHold(false);
+                continue;
+            }
+            if (hold.src != Src::None || leftHeld || aimHeld || clickPress || clickRelease) continue;
+            Hold h;
+            h.src = fromController ? Src::Controller : Src::Mouse, h.grip = p.drag;
+            if (fromController) {
+                h.ctrl = controllerFor(p.source);
+                if (!controllerAngles(h.ctrl, h.refYaw, h.refPitch)) {
+                    std::printf("gaze %s: no %s controller pose; the mouse steers instead\n", p.drag ? "drag" : "precision",
+                                p.source.c_str());
+                    std::fflush(stdout);
+                    h.src = Src::Mouse;
+                }
+            }
+            hold = h;
+            startHold(p.drag);
+            if (debug) std::printf("hold gaze %s began (%s)\n", p.drag ? "drag" : "precision", p.source.c_str());
+            if (debug) std::fflush(stdout);
+        }
+        devicePresses.clear();
+        if (hold.src == Src::Controller) {
+            double cy, cp;
+            if (controllerAngles(hold.ctrl, cy, cp))
+                steer(cy, cp, precisionDeadzone, hold.grip ? dragGain : precisionGain);
+        }
         fh_gestures_t hg;
         const bool handOk = handsOn && handFile.Read(hg);
         if (handOk && (hg.seq != handSeq || handFile.opens != handOpens)) {
@@ -1469,7 +1604,7 @@ int main() {
                 for (int s = 0; s < 2; ++s)
                     rebase = rebase || slots[k][s].begins < seenBegins[k][s] || slots[k][s].ends < seenEnds[k][s];
             if (rebase) {
-                if (hold.side >= 0) endHold(true);
+                if (hold.src == Src::Hand) endHold(true);
                 for (int k = 0; k < 2; ++k)
                     for (int s = 0; s < 2; ++s) seenBegins[k][s] = slots[k][s].begins, seenEnds[k][s] = slots[k][s].ends;
                 handBaseline = true, handOpens = handFile.opens;
@@ -1482,7 +1617,7 @@ int main() {
                     const bool began = g.begins != seenBegins[k][s], ended = g.ends != seenEnds[k][s];
                     const bool lost = g.flags & FH_PINCH_LOST;
                     seenBegins[k][s] = g.begins, seenEnds[k][s] = g.ends;
-                    auto holding = [&] { return hold.side == s && hold.grip == (k == 1); };
+                    auto holding = [&] { return hold.src == Src::Hand && hold.side == s && hold.grip == (k == 1); };
                     if (holding() && ended) endHold(lost);  // the one held ended (another may have begun)
                     if (began && allowed) {
                         beginHold(s, k == 1, g);
@@ -1492,30 +1627,18 @@ int main() {
                 }
             // The held gesture's hand moves the pointer: past the dead zone (a tap's jitter,
             // the pinch point shifting as the fingers close), then at the gain, from there.
-            if (hold.side >= 0) {
+            if (hold.src == Src::Hand) {
                 const fh_pinch_t &g = hold.grip ? hg.grip[hold.side] : hg.pinch[hold.side];
                 double hy, hp;
-                if ((g.flags & FH_PINCH_TRACKED) && handAngles(g.point, hg.capture_ns, hold.origin, hy, hp)) {
-                    const double dy = std::remainder(hy - hold.refYaw, 360.0), dp = hp - hold.refPitch;
-                    if (!hold.engaged && std::hypot(dy, dp) > (hold.grip ? 0.5 : pinchDeadzone)) {
-                        hold.engaged = true;
-                        hold.refYaw = hy, hold.refPitch = hp;
-                        hold.startYaw = hold.lastYaw = yaw, hold.startPitch = hold.lastPitch = pitch;
-                    } else if (hold.engaged) {
-                        const double gain = hold.grip ? gripGain : pinchGain;
-                        yaw = std::remainder(hold.startYaw + gain * dy, 360.0);
-                        pitch = std::clamp(hold.startPitch + gain * dp, -85.0, 85.0);
-                        if (!hold.grip) nudgeMoved += std::hypot(std::remainder(yaw - hold.lastYaw, 360.0), pitch - hold.lastPitch);
-                        hold.lastYaw = yaw, hold.lastPitch = pitch;
-                        lastMove = tnow;  // the dot shows while it moves (gaze mode)
-                    }
-                }
+                if ((g.flags & FH_PINCH_TRACKED) && handAngles(g.point, hg.capture_ns, hold.origin, hy, hp))
+                    steer(hy, hp, hold.grip ? 0.5 : pinchDeadzone, hold.grip ? gripGain : pinchGain);
             }
         }
         // ft-hands stopped (or hung) with a gesture held: it's over, as lost.
         const uint64_t nowNs =
             uint64_t(std::chrono::duration_cast<std::chrono::nanoseconds>(tnow.time_since_epoch()).count());
-        if (hold.side >= 0 && (!handOk || !active || nowNs - handPublished > 1'500'000'000ull)) endHold(true);
+        if (hold.src == Src::Hand && (!handOk || !active || nowNs - handPublished > 1'500'000'000ull)) endHold(true);
+        if (hold.src != Src::None && hold.src != Src::Hand && !active) endHold(true);
 
         if (aimHeld || leftHeld || clickPress || clickRelease) lastHeld = tnow;
 

session/frametop.conf.example

@@ -32,6 +32,11 @@ POINTER_GAZE_RETAKE=5      # gaze mode: how far (degrees) you look away from the
 POINTER_GAZE_NUDGE_MAX=8   # gaze mode: a mouse nudge up to this far (degrees) before a click is learned as the eye tracker's error
 POINTER_GAZE_HOLD=0.5      # gaze mode: a press held this long (s) without moving becomes a real press (to drag); moved or released sooner, it clicks where you let go
 POINTER_GAZE_SHOW=1        # gaze mode: the dot shows this long (s) after the mouse moves it; also while a press is held, and a pulse per click
+POINTER_GAZE_MOUSE=precision # gaze mode: the left button holds back its press, the mouse steers, the release clicks (precision) | clicks right away (direct)
+POINTER_PRECISION_GAIN=0.5  # gaze precision (a button mapped to gaze_precision): the pointer turns this times the controller's turn
+POINTER_PRECISION_DEADZONE=0.3 # gaze precision: degrees the controller turns before the pointer does (the press's own jolt)
+POINTER_GAZE_DRAG_GAIN=1   # gaze drag (gaze_drag): the pointer turns this times the controller's turn
+POINTER_ROLE=right         # the hand role the pointer takes: right | left | stylus. A controller held in that hand takes it back (clicks stop landing)
 GAZE_TRACKER=steam         # gaze service: steam = SteamVR's eye tracker | own = our own (frame-eyes' fe-trackd, calibrated in the gaze probe with Own tracker)
 GAZE_EYE=auto              # gaze service: eye bias. auto = each eye weighted by how far off it was at your recent nudges | left | right = that eye counts twice
 HANDS_SWAP_SIDES=0         # hand tracking (hands/run.sh install): 1 = the side cameras' names are swapped, which some SteamVR restarts cause (hands/tools/check_sides.py --ring tells)