DeeJanuz--frametop/hands/tools/check_sides.py

"""Check that the side cameras' images carry the right names (slam_left vs slam_right).

usage: python tools/check_sides.py REC [--sets N] [--pair side|upper] [--calib DIR] [--json]
       python tools/check_sides.py --ring [--sets N] [--pair side|upper]   (live, from ft-camd's ring)

REC is a recording folder (its sets.bin) or a sets file (a take's sets-N.bin). It checks the
names as they are in the files: a recording's sides.json or session.json "sides" (the hand
recorder's) isn't applied. Exits 0 when the names are right, 3 when they're swapped, and 2 when
it can't tell (too little texture in view, or the headset isn't worn). --pair upper checks the
upper pair (upper_left vs upper_right) the same way. --calib DIR uses DIR/calibration.json and
DIR/device.json (a hand recorder session's, or cut.py's) instead of /persist. --json prints one
line: {"pair", "verdict": "named"|"swapped"|"unknown", "named", "swapped", "matches", "sets"}.

ft-hands decides the side cameras' naming by itself (HANDS_SWAP_SIDES=auto, track/sides.h);
this is the independent check, from the scene rather than hands.

ft-camd tells the two side cameras' buffers apart by the order XRService allocated them,
and after some XRService restarts that order puts each camera's images under the other's
name. The tracker then sees every hand in one camera only, at the wrong depth. This
matches features between the two images and measures how close each pair's rays pass
with the factory calibration, once as named and once swapped: true matches meet in
front of both cameras only under the right naming.
"""
import argparse
import json
import os
import sys

import cv2
import numpy as np

sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
from tools.show_set import index, read_set  # noqa: E402
from tools import calib  # noqa: E402


PIPES = {'msm_vfe3_video0': 'slam_left', 'msm_vfe4_video0': 'slam_right',    # as ft-hands maps them
         'msm_vfe2_video0': 'upper_left', 'msm_vfe2_video1': 'upper_right'}
PAIRS = {'side': ('slam_left', 'slam_right'), 'upper': ('upper_left', 'upper_right')}


def load_cams(calib_dir=None):
    if calib_dir:
        return calib.load(os.path.join(calib_dir, 'calibration.json'), os.path.join(calib_dir, 'device.json'))
    return calib.load()


def matches(a, b):
    """Pixel pairs (N,2), (N,2) of ORB matches between two grey images."""
    clahe = cv2.createCLAHE(2.0, (8, 8))
    orb = cv2.ORB_create(3000)
    ka, da = orb.detectAndCompute(clahe.apply(a), None)
    kb, db = orb.detectAndCompute(clahe.apply(b), None)
    if da is None or db is None:
        return np.zeros((0, 2)), np.zeros((0, 2))
    pairs = cv2.BFMatcher(cv2.NORM_HAMMING).knnMatch(da, db, k=2)
    good = [p[0] for p in pairs if len(p) == 2 and p[0].distance < 0.75 * p[1].distance]
    return (np.array([ka[m.queryIdx].pt for m in good]).reshape(-1, 2),
            np.array([kb[m.trainIdx].pt for m in good]).reshape(-1, 2))


def meet(cam_a, cam_b, ua, ub):
    """Per match: closest distance between the two rays (m), and whether they meet in front of both."""
    ra, rb = cam_a.rays(ua), cam_b.rays(ub)
    w = cam_b.origin - cam_a.origin
    n = np.cross(ra, rb)
    nn = np.linalg.norm(n, axis=1)
    dist = np.abs(w @ n.T) / np.maximum(nn, 1e-12)
    # ray parameters at the closest points
    ta = np.einsum('ij,ij->i', np.cross(np.broadcast_to(w, rb.shape), rb), n) / np.maximum(nn ** 2, 1e-12)
    tb = np.einsum('ij,ij->i', np.cross(np.broadcast_to(w, ra.shape), ra), n) / np.maximum(nn ** 2, 1e-12)
    return dist, (ta > 0.05) & (tb > 0.05)


def score(cam_a, cam_b, ua, ub):
    """Share of matches whose rays meet within 1 cm, in front of both cameras."""
    if len(ua) == 0:
        return 0.0
    d, front = meet(cam_a, cam_b, ua, ub)
    return float(np.mean((d < 0.01) & front))


def recorded_pairs(rec, count, names=PAIRS['side']):
    """(label, left image, right image) from sets spread across a recording (or sets file)."""
    path = rec if os.path.isfile(rec) else os.path.join(rec, 'sets.bin')
    offs = index(path)
    if not offs:
        return
    for n in sorted(set(np.linspace(0, len(offs) - 1, count).astype(int))):
        images = read_set(path, offs[n])
        if names[0] in images and names[1] in images:
            yield 'set %5d' % n, images[names[0]][0], images[names[1]][0]


def live_pairs(count, names=PAIRS['side']):
    """(label, left image, right image) from ft-camd's ring, half a second apart."""
    import time
    from tools.ring import Ring
    ring = Ring()
    if not ring.alive():
        sys.exit('ft-camd isn\'t running (no heartbeat)')
    cams = {}
    for c in ring.cams:
        name = PIPES.get(open('/sys/class/video4linux/video%d/name' % c.node).read().strip())
        if name and not c.name.endswith('-dark'):
            cams[name] = c
    for k in range(count):
        a, b = ring.read(cams[names[0]]), ring.read(cams[names[1]])
        if a is not None and b is not None:
            yield 'frame %2d' % k, a.image, b.image
        time.sleep(0.5)


def verdict(named, swapped, n, total_matches):
    """'named', 'swapped' or 'unknown', from the summed per-set scores: the winner must meet
    clearly more often (by 0.08 a set) and at least 4 times as often. The upper cameras' small
    images meet within 1 cm less often (0.1-0.4 a set as named, 0 swapped) than the side ones (0.4-0.9)."""
    hi, lo = max(named, swapped), min(named, swapped)
    if n == 0 or total_matches < 100 or (hi - lo) / n < 0.08 or lo > 0.25 * hi:
        return 'unknown'
    return 'named' if named > swapped else 'swapped'


def check(pairs, left, right, out=print):
    """Score (label, left image, right image) pairs: (verdict, named, swapped, matches, sets)."""
    named = swapped = 0.0
    n = total_matches = 0
    for label, img_l, img_r in pairs:
        ua, ub = matches(img_l, img_r)
        s_named = score(left, right, ua, ub)     # the left-named image seen by the left camera
        s_swapped = score(right, left, ua, ub)   # ... by the right camera
        named, swapped, n, total_matches = named + s_named, swapped + s_swapped, n + 1, total_matches + len(ua)
        out('%s: %4d matches, meeting as named %3.0f%%, swapped %3.0f%%' %
            (label, len(ua), 100 * s_named, 100 * s_swapped))
    return verdict(named, swapped, n, total_matches), named / max(n, 1), swapped / max(n, 1), total_matches, n


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('rec', nargs='?')
    ap.add_argument('--ring', action='store_true', help='check the live cameras instead of a recording')
    ap.add_argument('--sets', type=int, default=8, help='how many sets or live frames to check')
    ap.add_argument('--pair', choices=sorted(PAIRS), default='side')
    ap.add_argument('--calib', help='folder with calibration.json and device.json (default: /persist)')
    ap.add_argument('--json', action='store_true', help='one JSON line instead of the report')
    a = ap.parse_args()
    if not a.ring and not a.rec:
        ap.error('give a recording or --ring')
    cams = load_cams(a.calib)
    names = PAIRS[a.pair]
    left, right = cams[names[0]], cams[names[1]]
    pairs = live_pairs(a.sets, names) if a.ring else recorded_pairs(a.rec, a.sets, names)
    v, named, swapped, total, n = check(pairs, left, right, out=(lambda s: None) if a.json else print)
    what = 'side cameras' if a.pair == 'side' else 'upper cameras'
    if a.json:
        print(json.dumps({'pair': a.pair, 'verdict': v, 'named': round(named, 3), 'swapped': round(swapped, 3),
                          'matches': total, 'sets': n}))
    elif v == 'unknown':
        print('%s: can\'t tell (%d matches)' % (what, total))
    else:
        print('%s: %s (named %.2f, swapped %.2f)' % (what, 'as named' if v == 'named' else 'SWAPPED', named, swapped))
    sys.exit({'named': 0, 'swapped': 3, 'unknown': 2}[v])


if __name__ == '__main__':
    main()