diff --git a/gaze/README.md b/gaze/README.md index 2f47612..2ee6937 100644 --- a/gaze/README.md +++ b/gaze/README.md @@ -2,7 +2,7 @@ The Steam Frame's eye tracking as pointer input: a gaze mode for the 3D mouse (the pointer goes where you look, and the mouse does the last bit), and the tools to calibrate and measure it. -- `ft-gaze` (C++, OpenVR, runs in the dev container) reads the eye tracker and prints one JSON line per sample (90 Hz). For each source, it gives the gaze direction relative to the head and the Frametop screen pixel it lands on. +- `ft-gaze` (C++, OpenVR, runs in the dev container) reads the eye tracker and prints one JSON line per sample (90 Hz). For each source, it gives the gaze direction relative to the head and the Frametop screen pixel it lands on. The gaze service asks it for only the sources it uses (`--sources`, and `sources LIST` on its stdin): our tracker and mmap set 1 with Own tracker, about 700 bytes a line instead of 1.3 KB, and every source while a check or the calibration runs. So SteamVR's gaze action, the only source that calls into vrserver (twice a sample), is read only then. The probe gets them all. - `gazecal.py` has what the probe and the gaze service share: the correction models, filters, and the reader for SteamVR's eye tracking log. - `tracker/` is our own eye tracker, an alternative to SteamVR's: `ft-eyes` finds the pupils and glints in the eye-camera frames that `ft-eyegrab` (a small root service) copies out of SteamVR's tracker. See "Our own eye tracker" below. - `probe/ft-gazeprobe` (GTK 4, host Python) is a fullscreen playground, for developing the gaze tracking: day to day, the calibration and the checks run in the headset panel (Quick check, Calibrate, and Check headset fit on the Gaze page). It runs ft-gaze, draws where you're looking, measures accuracy, and tries out hold-to-adjust clicking with a calibration that learns from your adjustments. diff --git a/gaze/ft-gaze.cpp b/gaze/ft-gaze.cpp index f11dd4d..2e23b0a 100644 --- a/gaze/ft-gaze.cpp +++ b/gaze/ft-gaze.cpp @@ -4,7 +4,14 @@ // Every eye tracker sample (90 Hz) becomes one JSON line on stdout with each gaze source // hit-tested against the Frametop screens: // -// Options: -v (log action errors), --watch-stdin (quit when stdin closes). +// Options: -v (log action errors), --watch-stdin (quit when stdin closes; until then, a line +// "sources LIST" on stdin switches the sources as --sources does), --sources LIST (comma- +// separated: action, mmap1, mmap2, left, right, own, and eye for EYE; or all, the default). +// Sources left out are read not at all and print as {"ok":0} ("eye" as null), so every line +// keeps the same keys. The gaze service asks for the ones it uses (own and mmap1 with our +// tracker), and all of them while a check or the calibration runs. Only the action costs +// SteamVR anything (two calls into vrserver per sample), and a line with every source is +// about 1.5 KB, 130 KB a second through podman's relay. // // {"t":,"age":,"n":, // "head":{"yaw":..,"pitch":..,"hit":HIT}, head forward ray (for head nudging) @@ -391,6 +398,30 @@ std::string SrcJson(const std::vector &screens, const vr::HmdMatrix34_t return buf + extra + "\"hit\":" + HitJson(screens, head, yaw, pitch) + "}"; } +// Sources (--sources, "sources LIST" on stdin): a bit each. +enum : unsigned { kAction = 1, kMmap1 = 2, kMmap2 = 4, kLeft = 8, kRight = 16, kOwn = 32, kEye = 64, kAll = 127 }; + +bool ParseSources(const std::string &list, unsigned &mask) { + static const struct { + const char *name; + unsigned bit; + } names[] = {{"action", kAction}, {"mmap1", kMmap1}, {"mmap2", kMmap2}, {"left", kLeft}, + {"right", kRight}, {"own", kOwn}, {"eye", kEye}, {"all", kAll}}; + unsigned m = 0; + size_t at = 0; + while (at <= list.size()) { + const size_t comma = std::min(list.find(',', at), list.size()); + const std::string name = list.substr(at, comma - at); + bool known = false; + for (const auto &n : names) + if (name == n.name) m |= n.bit, known = true; + if (!known) return false; + at = comma + 1; + } + mask = m; + return true; +} + // Head poses of the last half second, so a sample can use the pose at its own time. class PoseHistory { public: @@ -428,9 +459,17 @@ std::string ExeDir() { int main(int argc, char **argv) { bool verbose = false, watchStdin = false; + std::atomic sources{kAll}; for (int i = 1; i < argc; ++i) { if (std::strcmp(argv[i], "-v") == 0) verbose = true; if (std::strcmp(argv[i], "--watch-stdin") == 0) watchStdin = true; + if (std::strcmp(argv[i], "--sources") == 0 && i + 1 < argc) { + unsigned m; + if (ParseSources(argv[++i], m)) + sources = m; + else + std::fprintf(stderr, "ft-gaze: --sources %s: unknown source (all used)\n", argv[i]); + } } // Nice 5, before any thread starts (they inherit it): we run in the dev container's podman // scope, beside vrcompositor and vrserver at nice 0, and the gaze service's unit doesn't @@ -442,11 +481,20 @@ int main(int argc, char **argv) { std::fprintf(stderr, "ft-gaze: nice: %s\n", std::strerror(errno)); // --watch-stdin: quit when stdin closes. The probe runs us through distrobox, which // passes neither its signals nor a closed stdout on to us, but does pass stdin's end. + // Lines on stdin until then: "sources LIST". std::atomic stdinClosed{false}; if (watchStdin) - std::thread([&stdinClosed] { + std::thread([&stdinClosed, &sources] { char c[256]; - while (read(0, c, sizeof c) > 0) { + std::string line; + ssize_t n; + while ((n = read(0, c, sizeof c)) > 0) { + line.append(c, size_t(n)); + for (size_t nl; (nl = line.find('\n')) != std::string::npos; line.erase(0, nl + 1)) { + unsigned m; + if (line.compare(0, 8, "sources ") == 0 && ParseSources(line.substr(8, nl - 8), m)) sources = m; + } + if (line.size() > 4096) line.clear(); // no newline in sight: not ours } stdinClosed = true; }).detach(); @@ -519,6 +567,7 @@ int main(int argc, char **argv) { if (fresh && hp.bPoseIsValid) { lastEmit = now; + const unsigned want = sources; const auto list = screens.Get(); const vr::HmdMatrix34_t &headNow = hp.mDeviceToAbsoluteTracking; vr::HmdMatrix34_t headThen = headNow; @@ -527,7 +576,7 @@ int main(int argc, char **argv) { // SteamVR's action: a room-space origin and fixation point, turned into the head // frame so every source reports the same kind of angles. std::string action = "{\"ok\":0}"; - if (!inGame) { + if (!inGame && (want & kAction)) { vr::VRActiveActionSet_t active{}; active.ulActionSet = set; active.nPriority = vr::k_nActionSetOverlayGlobalPriorityMin; @@ -570,26 +619,33 @@ int main(int argc, char **argv) { return std::string(b); }; char extra[256]; - std::snprintf(extra, sizeof extra, "\"dist\":%.3f,\"open\":[%.3f,%.3f],\"lr\":%.3f,", Length(s.fix1), - s.open[0], s.open[1], lr(s.left1, s.right1)); - m1 = SrcJson(list, headThen, s.left1 + s.right1, extra + eyes(s.left1, s.right1) + unc(s.var1)); - std::snprintf(extra, sizeof extra, "\"lr\":%.3f,", lr(s.left2, s.right2)); - m2 = SrcJson(list, headThen, s.left2 + s.right2, extra + eyes(s.left2, s.right2) + unc(s.var2)); - left = SrcJson(list, headThen, s.left2); - right = SrcJson(list, headThen, s.right2); + if (want & kMmap1) { + std::snprintf(extra, sizeof extra, "\"dist\":%.3f,\"open\":[%.3f,%.3f],\"lr\":%.3f,", Length(s.fix1), + s.open[0], s.open[1], lr(s.left1, s.right1)); + m1 = SrcJson(list, headThen, s.left1 + s.right1, extra + eyes(s.left1, s.right1) + unc(s.var1)); + } + if (want & kMmap2) { + std::snprintf(extra, sizeof extra, "\"lr\":%.3f,", lr(s.left2, s.right2)); + m2 = SrcJson(list, headThen, s.left2 + s.right2, extra + eyes(s.left2, s.right2) + unc(s.var2)); + } + if (want & kLeft) left = SrcJson(list, headThen, s.left2); + if (want & kRight) right = SrcJson(list, headThen, s.right2); + // "new" compares with the last sample, so the last measurement is kept either way. const float *m = s.meas; const bool newL = m[0] != lastMeas[0] || m[1] != lastMeas[1]; const bool newR = m[2] != lastMeas[2] || m[3] != lastMeas[3]; std::memcpy(lastMeas, m, sizeof lastMeas); - std::snprintf(extra, sizeof extra, "{\"q\":[%.3g,%.3g],\"m\":[[%.4f,%.4f],[%.4f,%.4f]],\"new\":[%d,%d]}", - (m[4] + m[5]) / 2, (m[6] + m[7]) / 2, m[0], m[1], m[2], m[3], int(newL), int(newR)); - eye = extra; + if (want & kEye) { + std::snprintf(extra, sizeof extra, "{\"q\":[%.3g,%.3g],\"m\":[[%.4f,%.4f],[%.4f,%.4f]],\"new\":[%d,%d]}", + (m[4] + m[5]) / 2, (m[6] + m[7]) / 2, m[0], m[1], m[2], m[3], int(newL), int(newR)); + eye = extra; + } } // Our tracker: its own sample time picks the head pose, like the mmap's. std::string own = "{\"ok\":0}"; OwnSample o; - if (ownFile.Read(o) && now - o.t < 0.1) { + if ((want & kOwn) && ownFile.Read(o) && now - o.t < 0.1) { vr::HmdMatrix34_t headOwn = headNow; history.At(o.t, headOwn); auto pair = [](bool ok, float a, float b) { diff --git a/gaze/ft-gazed b/gaze/ft-gazed index f751ba6..18a768b 100755 --- a/gaze/ft-gazed +++ b/gaze/ft-gazed @@ -76,6 +76,12 @@ off. A check asked for while idle waits for the tracker to start (at most WAKE_S tracker's next click after it starts again re-seats it, as after the headset was off, so turning gaze mode on after a while opens the quick check. +ft-gaze prints only the sources this uses ("sources LIST" on its stdin, see wanted_sources): +own and mmap1 with our tracker, left, right and mmap1 with SteamVR's eyes, and every source +while a check or the calibration runs or is asked for, since those record them all. So +SteamVR's gaze action, the one source that calls into vrserver, is read only then (or with +--source action). + Checks and calibration (gaze/gazecheck.py): a one-dot quick check when the headset goes on or our tracker asks for a click, and the full calibration when gaze mode comes on without one, both in a panel fixed to the headset (gaze/panel/ft-gazepanel, which this service runs too). @@ -260,6 +266,7 @@ class Service: self.sel.register(self.eyes_sock, selectors.EVENT_READ, "own") self.checks = Checks(self, self.sel) self.proc = None + self.proc_sources = None # what ft-gaze was last told to print self.buf = b"" self.restart_at = 0.0 self.running = True @@ -459,6 +466,31 @@ class Service: # --- ft-gaze --- + def wanted_sources(self): + """The sources ft-gaze should print (its --sources): what on_sample and the checks read. + mmap1 always (blinks, lost eyes, and the headset going on, from its openness and + variances); everything while a check runs or waits, since they record every source.""" + c = self.checks + if c.active or c.pending: + return "all" + kind = self.kind + if kind == "own": + return "own,mmap1" + if kind == "eyes": + return "left,right,mmap1" + return ",".join(dict.fromkeys((self.source, "mmap1", "mmap2"))) # mmap2: each eye, for the fallback + + def sync_sources(self): + want = self.wanted_sources() + if not self.proc or want == self.proc_sources: + return + try: + self.proc.stdin.write(f"sources {want}\n".encode()) + self.proc.stdin.flush() + except (OSError, ValueError): + return # it's stopping: read_stdout notices + self.proc_sources = want + def start_helper(self): if not HELPER.exists(): log(f"ft-gaze isn't built: run {REPO}/gaze/build.sh") @@ -469,9 +501,11 @@ class Service: subprocess.run([str(REPO / "scripts" / "container-up.sh")], env=env, check=False) distrobox = Path.home() / ".local" / "bin" / "distrobox" # ft-gaze quits when its stdin closes: the one thing distrobox passes on. - self.proc = subprocess.Popen([str(distrobox), "enter", "dev", "--", str(HELPER), "--watch-stdin"], env=env, - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - start_new_session=True) + sources = self.wanted_sources() + self.proc = subprocess.Popen([str(distrobox), "enter", "dev", "--", str(HELPER), "--watch-stdin", + "--sources", sources], env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, start_new_session=True) + self.proc_sources = sources os.set_blocking(self.proc.stdout.fileno(), False) os.set_blocking(self.proc.stderr.fileno(), False) self.sel.register(self.proc.stdout, selectors.EVENT_READ, "stdout") @@ -876,6 +910,7 @@ class Service: elif key.data == "stderr" and self.proc: self.read_stderr() self.checks.tick() + self.sync_sources() if now >= next_periodic: self.periodic() next_periodic = now + 1.0 diff --git a/gaze/test/idle-test.py b/gaze/test/idle-test.py index a0cd240..145becb 100755 --- a/gaze/test/idle-test.py +++ b/gaze/test/idle-test.py @@ -37,14 +37,26 @@ gazed.read_settings = lambda: ("steam", "steam", "auto", 55.0) logs = [] gazed.log = gazecheck.log = lambda msg: logs.append(msg) -# The fake ft-gaze: 90 samples a second, both eyes seen, until its stdin closes. +# The fake ft-gaze: 90 samples a second, both eyes seen, until its stdin closes. It logs the +# sources it was asked for: "argv LIST" (--sources), then "line LIST" for each "sources LIST". tmp = tempfile.mkdtemp(prefix="ft-gaze-idle-test-") FAKE = os.path.join(tmp, "ft-gaze") +SOURCES_LOG = os.path.join(tmp, "sources.log") with open(FAKE, "w") as f: - f.write('''import json, select, sys, time + f.write('''import json, os, select, sys, time +log = open(sys.argv[1], "a", buffering=1) +log.write("argv " + (sys.argv[sys.argv.index("--sources") + 1] if "--sources" in sys.argv else "-") + "\\n") +buf = b"" while True: - if select.select([sys.stdin], [], [], 1 / 90)[0] and not sys.stdin.read(1): - break + if select.select([sys.stdin], [], [], 1 / 90)[0]: + data = os.read(0, 4096) + if not data: + break + buf += data + while b"\\n" in buf: + line, buf = buf.split(b"\\n", 1) + if line.startswith(b"sources "): + log.write("line " + line[8:].decode() + "\\n") eye = {"hy": 1.0, "hp": 2.0} print(json.dumps({"t": time.monotonic(), "src": {"mmap1": {"hy": 1.0, "hp": 2.0, "unc": [0.001, 0.001], "open": [0.8, 0.8]}, "left": eye, "right": eye}}), flush=True) @@ -55,8 +67,9 @@ started = [] def start_helper(self): """ft-gaze, straight from here instead of the dev container.""" import selectors - self.proc = subprocess.Popen([sys.executable, FAKE], stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + self.proc = subprocess.Popen([sys.executable, FAKE, SOURCES_LOG, "--sources", self.wanted_sources()], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.proc_sources = self.wanted_sources() os.set_blocking(self.proc.stdout.fileno(), False) os.set_blocking(self.proc.stderr.fileno(), False) self.sel.register(self.proc.stdout, selectors.EVENT_READ, "stdout") @@ -142,17 +155,24 @@ check("lease over (2 s + IDLE_AFTER): idle", wait(lambda: not svc.awake, 4.5), T time.sleep(0.5) logs.clear() +open(SOURCES_LOG, "w").close() count = len(started) check("quick check while idle: queued, waking", ask("quickcal"), "ok waking the eye tracker first") check("it woke", wait(lambda: svc.awake and len(started) > count, 1.5), True) check("once the tracker sends, it ran (and says why it couldn't open)", wait(lambda: any("quickcal, asked for while idle: the panel isn't running" in m for m in logs), 3), True) check("nothing left pending", svc.checks.pending, None) +sources = open(SOURCES_LOG).read().split("\n") +check("ft-gaze started with every source for the check", sources[0], "argv all") +in_use = svc.wanted_sources() +check("then only those in use (SteamVR's tracker: not the action, not own)", + (f"line {in_use}" in sources, in_use != "all", "action" in in_use, "own" in in_use), (True, True, False, False)) check("idle again after", wait(lambda: not svc.awake, 3), True) print("FAILED: " + ", ".join(failures) if failures else "all passed", flush=True) svc.running = False time.sleep(0.7) os.remove(FAKE) +os.remove(SOURCES_LOG) os.rmdir(tmp) os._exit(1 if failures else 0)