Add a live replay harness for the CPU-placement test

fh-ringplay plays a recording into a frame ring in real time, and fh-tracker
--ring reads it (cameras without a device node are mapped by name). With the
same frames in every run, probes/core_ab.py compares the tracker on CPUs 2-4,
on 5-7, and not running, measuring the tracker's step time and latency, the
compositor's late frames and CPU/GPU time, XRService's timing warnings, CPU
temperature and clocks.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.gitignore

@@ -14,6 +14,7 @@ camd/fh-camd
 camd/fh-camprobe
 trackd/fh-tracker
 trackd/fh-replay
+trackd/fh-ringplay
 trackd/nettest
 probes/fh-frametime
 probes/mgrvt

trackd/Makefile

@@ -8,7 +8,7 @@ LDLIBS    = $(NCNN)/lib/libncnn.a -ljsoncpp -fopenmp -lpthread
 SRC = main.cpp calib.cpp nets.cpp tracker.cpp io.cpp record.cpp
 HDR = calib.h geom.h nets.h tracker.h io.h record.h ../camd/fhring.h ../include/fh_hands.h
 
-all: fh-tracker fh-replay
+all: fh-tracker fh-replay fh-ringplay
 
 fh-tracker: $(SRC) $(HDR)
 	$(CXX) $(CXXFLAGS) -o $@ $(SRC) $(LDLIBS)
@@ -16,8 +16,11 @@ fh-tracker: $(SRC) $(HDR)
 fh-replay: replay.cpp calib.cpp nets.cpp tracker.cpp record.cpp $(HDR)
 	$(CXX) $(CXXFLAGS) -o $@ replay.cpp calib.cpp nets.cpp tracker.cpp record.cpp $(LDLIBS)
 
+fh-ringplay: ringplay.cpp record.h ../camd/fhring.h
+	$(CXX) $(CXXFLAGS) -o $@ ringplay.cpp
+
 clean:
-	rm -f fh-tracker fh-replay nettest
+	rm -f fh-tracker fh-replay fh-ringplay nettest
 
 .PHONY: all clean
 

trackd/README.md

@@ -21,6 +21,8 @@ Options:
 - `--no-publish`: don't write the hands file.
 - `--record DIR`, `--record-for S`: save every frame set for S seconds (default 120) to `DIR/sets.bin`. That's about 80 MB/s. Sending the tracker SIGUSR1 (`pkill -USR1 -x fh-tracker`) starts a recording in `captures/rec-<time>` without a restart.
 - `--record-only`: record without tracking or publishing, so it can run beside the live tracker. Give it `--record DIR`; SIGUSR1 would reach both trackers. With `fh-camd --with-dark`, recordings also hold each camera's newest dark frame as `<name>_dk`, which doubles the rate.
+- `--keep-presence P`: the landmark presence a tracked view needs to stay tracked. New views always need 0.5. Default 0.5. Lowering it to 0.2 barely helped in the bright recording, because lost hands drop to near-zero presence.
+- `--ring PATH`: read frames from another ring, such as `fh-ringplay`'s.
 
 The status line also says how often a hand was on each side (by where the wrist is), and why views and hands came and went: views lost (the landmark model stopped seeing the hand), handoff misses (a crop projected from the hand's 3D position found nothing), duplicates, splits (two views disagreed in 3D), and hands created, merged and forgotten.
 
@@ -45,6 +47,10 @@ trackd/fh-replay captures/rec-20260929-120000 --oracle 10 --timeline /tmp/tl.txt
 - `--slow F`: live, the tracker skips sets that arrive while it's busy. Replay counts each step's time times F as busy (default 1; the headset is busier live).
 - `--timeline FILE`: a line per processed set and hand.
 
+## Playing a recording live
+
+`fh-ringplay DIR --ring PATH [--from S] [--to S] [--loop]` publishes a recording into a ring file in real time, as fh-camd would, so `fh-tracker --ring PATH --no-publish` runs the same frames run after run. It needs no root, and it skips the dark frames. `probes/core_ab.py` uses it to compare CPU placements (A: CPUs 2-4, B: 5-7, C: no tracker), with the headset on so SteamVR's compositor is running.
+
 ## Build
 
 ncnn is built from source into `vendor/ncnn/build/install`. The build also provides the `ncnn2table` and `ncnn2int8` quantization tools.

trackd/main.cpp

@@ -52,7 +52,7 @@ int main(int argc, char **argv) {
     int threads = 3, niceness = 5;
     bool int8 = false, publish = true, track = true, swap_sides = false;
     std::string models = std::string(argv[0]).substr(0, std::string(argv[0]).rfind('/') + 1) + "../models/ncnn";
-    std::string record;
+    std::string record, ring_path = FH_RING_PATH;
     // SteamOS starts user processes on CPUs 0-4 (2-4 are the big A720s) and keeps 5-7 (two
     // A720s and the X4) for SteamVR's compositor, whose threads there run at real-time
     // priority. XRService pins its head tracking to 2-3.
@@ -74,6 +74,7 @@ int main(int argc, char **argv) {
         else if (a == "--no-publish") publish = false;
         else if (a == "--swap-sides") swap_sides = true;
         else if (a == "--record-only") track = publish = false;
+        else if (a == "--ring" && more) ring_path = argv[++i];
         else if (a == "--record" && more) record = argv[++i];
         else if (a == "--record-for" && more) record_for = std::atof(argv[++i]);
         else if (a == "--keep-presence" && more) keep_presence = std::atof(argv[++i]);
@@ -92,7 +93,7 @@ int main(int argc, char **argv) {
         else {
             std::printf("usage: %s [--seconds N] [--threads N] [--int8] [--status S] [--models DIR] [--nice N] [--no-publish]\n"
                         "          [--record DIR] [--record-for S] [--record-only] [--cpus 2,3,4] [--swap-sides]\n"
-                        "          [--keep-presence P] (0.5)\n"
+                        "          [--keep-presence P] (0.5) [--ring PATH] (fh-camd's, or fh-ringplay's)\n"
                         "          [--contrast MODE|PALM/HAND] (clahe[:CLIP], none, stretch; default clahe:2/none)\n"
                         "Recording saves every frame set for S seconds (120) to DIR/sets.bin, for fh-replay; SIGUSR1\n"
                         "starts one in captures/rec-<time> next to trackd. --record-only records without tracking, so it\n"
@@ -122,7 +123,7 @@ int main(int argc, char **argv) {
         std::fflush(stdout);
         return true;
     };
-    if (!load_calibration(calib, err) || !ring.open(FH_RING_PATH, err) || !nets.load(models, int8, err) ||
+    if (!load_calibration(calib, err) || !ring.open(ring_path.c_str(), err) || !nets.load(models, int8, err) ||
         (publish && !pub.open(err)) || (!record.empty() && !start_recording(record, err))) {
         std::fprintf(stderr, "%s\n", err.c_str());
         return 1;
@@ -135,7 +136,9 @@ int main(int argc, char **argv) {
     std::map<std::string, int> dark;
     std::map<std::string, Camera> used;
     for (int i = 0; i < ring.cameras(); ++i) {
+        // fh-camd's cameras by capture pipe; fh-ringplay's (no device) by the name it gives
         const char *name = camera_for_pipe(ring.camera(i).node);
+        if (!name && ring.camera(i).node < 0) name = ring.camera(i).name;
         if (!name || !calib.count(name)) continue;
         if (ring.camera(i).flags & FH_CAM_DARK) dark[std::string(name) + "_dk"] = i;
         else index[name] = i, used[name] = calib[name];

trackd/ringplay.cpp

@@ -0,0 +1,233 @@
+// fh-ringplay: play a recording (fh-tracker --record) into a frame ring in real time, the
+// way fh-camd publishes live cameras, so fh-tracker --ring PATH processes the same frames
+// run after run. For A/B tests of how the tracker runs (probes/core-ab.sh).
+//
+//   fh-ringplay DIR --ring PATH [--from S] [--to S] [--loop] [--cpus 0,1]
+//
+// Frames are stamped as they're published, so the tracker's latency figures stay
+// meaningful. Cameras carry their calibration name and no device node (fh-tracker maps
+// them by name), so a recording made with the right names needs no --swap-sides.
+// Dark frames (<name>_dk) are skipped. Needs no root: the ring is an ordinary file.
+#include "record.h"
+
+extern "C" {
+#include "../camd/fhring.h"
+}
+
+#include <fcntl.h>
+#include <sched.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <cerrno>
+#include <csignal>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <ctime>
+#include <string>
+#include <vector>
+
+namespace {
+
+volatile std::sig_atomic_t g_stop = 0;
+
+uint64_t clock_ns(clockid_t id) {
+    timespec ts;
+    clock_gettime(id, &ts);
+    return uint64_t(ts.tv_sec) * 1'000'000'000ull + uint64_t(ts.tv_nsec);
+}
+
+// Sets from a recording, reading only the pixels of the cameras that get published.
+class Reader {
+public:
+    bool open(const std::string &path) {
+        f_ = std::fopen(path.c_str(), "rb");
+        if (f_) posix_fadvise(fileno(f_), 0, 0, POSIX_FADV_SEQUENTIAL);
+        return f_ != nullptr;
+    }
+    void rewind() { std::fseek(f_, 0, SEEK_SET); }
+    // False at the end. cams: every camera in the set; px[k]: pixels of camera k when
+    // want(name), else left empty.
+    template <class Want>
+    bool next(std::vector<fh_set_cam_t> &cams, std::vector<std::vector<uint8_t>> &px, Want want) {
+        fh_set_hdr_t h;
+        if (std::fread(&h, sizeof h, 1, f_) != 1 || std::memcmp(h.magic, FH_SET_MAGIC, 8) || h.ncams == 0 || h.ncams > 16)
+            return false;
+        cams.resize(h.ncams);
+        if (std::fread(cams.data(), sizeof(fh_set_cam_t), h.ncams, f_) != h.ncams) return false;
+        px.resize(h.ncams);
+        for (uint32_t k = 0; k < h.ncams; ++k) {
+            cams[k].name[sizeof cams[k].name - 1] = 0;
+            const size_t n = size_t(cams[k].width) * cams[k].height;
+            if (want(cams[k].name)) {
+                px[k].resize(n);
+                if (std::fread(px[k].data(), 1, n, f_) != n) return false;
+            } else {
+                px[k].clear();
+                if (std::fseek(f_, long(n), SEEK_CUR)) return false;
+            }
+        }
+        if (++sets_ % 64 == 0) posix_fadvise(fileno(f_), 0, std::ftell(f_), POSIX_FADV_DONTNEED);   // RAM is tight
+        return true;
+    }
+
+private:
+    FILE *f_ = nullptr;
+    uint64_t sets_ = 0;
+};
+
+bool is_dark(const char *name) {
+    const size_t n = std::strlen(name);
+    return n > 3 && !std::strcmp(name + n - 3, "_dk");
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+    if (argc < 2 || argv[1][0] == '-') {
+        std::fprintf(stderr, "usage: %s DIR --ring PATH [--from S] [--to S] [--loop] [--cpus 0,1]\n", argv[0]);
+        return 1;
+    }
+    const std::string dir = argv[1];
+    std::string ring_path;
+    double from = 0, to = 1e9;
+    bool loop = false;
+    std::vector<int> cpus = {0, 1};
+    for (int i = 2; i < argc; ++i) {
+        const std::string a = argv[i];
+        const bool more = i + 1 < argc;
+        if (a == "--ring" && more) ring_path = argv[++i];
+        else if (a == "--from" && more) from = std::atof(argv[++i]);
+        else if (a == "--to" && more) to = std::atof(argv[++i]);
+        else if (a == "--loop") loop = true;
+        else if (a == "--cpus" && more) {
+            cpus.clear();
+            for (char *p = argv[++i]; *p;) {
+                cpus.push_back(int(std::strtol(p, &p, 10)));
+                if (*p == ',') ++p;
+                else if (*p) break;
+            }
+        } else return std::fprintf(stderr, "unknown option %s\n", a.c_str()), 1;
+    }
+    if (ring_path.empty()) return std::fprintf(stderr, "--ring PATH is required\n"), 1;
+    if (!cpus.empty()) {
+        cpu_set_t set;
+        CPU_ZERO(&set);
+        for (int c : cpus) CPU_SET(c, &set);
+        if (sched_setaffinity(0, sizeof set, &set) < 0) std::perror("sched_setaffinity");
+    }
+    std::signal(SIGINT, [](int) { g_stop = 1; });
+    std::signal(SIGTERM, [](int) { g_stop = 1; });
+
+    Reader in;
+    if (!in.open(dir + "/sets.bin")) return std::fprintf(stderr, "%s/sets.bin: %s\n", dir.c_str(), std::strerror(errno)), 1;
+    std::vector<fh_set_cam_t> cams;
+    std::vector<std::vector<uint8_t>> px;
+    auto want = [](const char *name) { return !is_dark(name); };
+    if (!in.next(cams, px, want)) return std::fprintf(stderr, "%s: no sets\n", dir.c_str()), 1;
+    auto set_time = [&](const std::vector<fh_set_cam_t> &cs) {   // earliest bright capture, s
+        uint64_t t = UINT64_MAX;
+        for (const auto &c : cs)
+            if (!is_dark(c.name)) t = std::min(t, c.capture_ns);
+        return double(t) * 1e-9;
+    };
+    const double rec0 = set_time(cams);
+    // skip to --from before the ring exists, so a reader never finds it without a heartbeat
+    bool have = true;
+    while (have && set_time(cams) - rec0 < from && !g_stop) have = in.next(cams, px, want);
+    if (!have) return std::fprintf(stderr, "%s: nothing after %.1f s\n", dir.c_str(), from), 1;
+
+    // the ring: the recording's bright cameras, as fh-camd lays them out
+    std::vector<int> pub;   // set camera index of each ring camera
+    for (size_t k = 0; k < cams.size() && pub.size() < FH_RING_MAX_CAMS; ++k)
+        if (!is_dark(cams[k].name)) pub.push_back(int(k));
+    size_t len = sizeof(fh_ring_hdr_t);
+    std::vector<uint64_t> offset(pub.size()), slot_bytes(pub.size());
+    for (size_t r = 0; r < pub.size(); ++r) {
+        const fh_set_cam_t &c = cams[pub[r]];
+        slot_bytes[r] = (sizeof(fh_ring_slot_t) + size_t(c.width) * c.height + 63) & ~size_t(63);
+        offset[r] = len;
+        len += FH_RING_SLOTS * slot_bytes[r];
+    }
+    const int fd = ::open(ring_path.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_CLOEXEC, 0600);
+    if (fd < 0 || ftruncate(fd, off_t(len)) < 0)
+        return std::fprintf(stderr, "%s: %s\n", ring_path.c_str(), std::strerror(errno)), 1;
+    void *m = mmap(nullptr, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+    close(fd);
+    if (m == MAP_FAILED) return std::fprintf(stderr, "mmap %s: %s\n", ring_path.c_str(), std::strerror(errno)), 1;
+    auto *base = static_cast<uint8_t *>(m);
+    auto *hdr = reinterpret_cast<fh_ring_hdr_t *>(base);
+    for (size_t r = 0; r < pub.size(); ++r) {
+        const fh_set_cam_t &c = cams[pub[r]];
+        fh_ring_cam_t &rc = hdr->cams[r];
+        std::snprintf(rc.sensor, sizeof rc.sensor, "fh-ringplay");
+        std::snprintf(rc.name, sizeof rc.name, "%s", c.name);
+        rc.node = -1;
+        rc.format = FH_FMT_GREY8;
+        rc.width = rc.stride = c.width;
+        rc.height = c.height;
+        rc.nslots = FH_RING_SLOTS;
+        rc.slot_offset = offset[r];
+        rc.slot_bytes = slot_bytes[r];
+    }
+    hdr->version = FH_RING_VERSION;
+    hdr->header_bytes = sizeof(fh_ring_hdr_t);
+    hdr->ncams = uint32_t(pub.size());
+    hdr->file_bytes = len;
+    hdr->writer_pid = getpid();
+    std::memcpy(hdr->magic, FH_RING_MAGIC, 8);
+    __atomic_store_n(&hdr->heartbeat_ns, clock_ns(CLOCK_MONOTONIC), __ATOMIC_RELEASE);
+    std::printf("playing %s into %s:", dir.c_str(), ring_path.c_str());
+    for (int k : pub) std::printf(" %s", cams[k].name);
+    std::printf("\n");
+    std::fflush(stdout);
+
+    uint64_t published = 0, rounds = 0;
+    for (;;) {
+        // one pass over [from, to]: each set goes out at its recorded offset from the first
+        while (have && set_time(cams) - rec0 < from && !g_stop) {
+            __atomic_store_n(&hdr->heartbeat_ns, clock_ns(CLOCK_MONOTONIC), __ATOMIC_RELEASE);
+            have = in.next(cams, px, want);
+        }
+        const double first = set_time(cams);
+        const uint64_t start = clock_ns(CLOCK_MONOTONIC);
+        while (have && !g_stop && set_time(cams) - rec0 <= to) {
+            const uint64_t due = start + uint64_t((set_time(cams) - first) * 1e9);
+            for (uint64_t now = clock_ns(CLOCK_MONOTONIC); now < due && !g_stop; now = clock_ns(CLOCK_MONOTONIC)) {
+                __atomic_store_n(&hdr->heartbeat_ns, now, __ATOMIC_RELEASE);
+                const uint64_t wait = std::min<uint64_t>(due - now, 100'000'000);
+                const timespec ts{time_t(wait / 1'000'000'000), long(wait % 1'000'000'000)};
+                nanosleep(&ts, nullptr);
+            }
+            const uint64_t raw = clock_ns(CLOCK_MONOTONIC_RAW), mono = clock_ns(CLOCK_MONOTONIC);
+            for (size_t r = 0; r < pub.size(); ++r) {
+                fh_ring_cam_t &rc = hdr->cams[r];
+                if (px[pub[r]].size() != size_t(rc.width) * rc.height) continue;
+                const uint64_t n = rc.latest + 1;
+                auto *s = reinterpret_cast<fh_ring_slot_t *>(base + rc.slot_offset + (n % rc.nslots) * rc.slot_bytes);
+                __atomic_store_n(&s->seq, 2 * n + 1, __ATOMIC_RELAXED);
+                __atomic_thread_fence(__ATOMIC_RELEASE);
+                std::memcpy(reinterpret_cast<uint8_t *>(s + 1), px[pub[r]].data(), px[pub[r]].size());
+                s->frame = n;
+                s->capture_ns = raw;   // taken now, as a live camera's frame would be
+                s->dqbuf_ns = mono;
+                s->publish_ns = mono;
+                __atomic_store_n(&s->seq, 2 * n + 2, __ATOMIC_RELEASE);
+                __atomic_store_n(&rc.latest, n, __ATOMIC_RELEASE);
+                ++rc.published;
+            }
+            __atomic_store_n(&hdr->heartbeat_ns, mono, __ATOMIC_RELEASE);
+            ++published;
+            have = in.next(cams, px, want);
+        }
+        ++rounds;
+        if (g_stop || !loop) break;
+        in.rewind();
+        have = in.next(cams, px, want);
+    }
+    std::printf("published %llu sets in %llu pass(es)\n", (unsigned long long)published, (unsigned long long)rounds);
+    __atomic_store_n(&hdr->heartbeat_ns, 0, __ATOMIC_RELEASE);   // readers see the writer gone
+    return 0;
+}