mirror of
https://github.com/DeeJanuz/frametop.git
synced 2026-10-06 05:00:08 +02:00
Bring in frame-hands' hand tracking under hands/
The history of frame-hands (~/Desktop/Projects/frame-hands on the Frame), filtered to what moves: the camera broker (camd/), the tracker and its offline tools (trackd/), the shared file layouts (include/), the ncnn models, the analysis tools, and the calibration and model helpers they import from the Python prototype. The reverse-engineering notes, probes, camprobe, and the rest of the prototype stay in frame-hands. Unchanged here: the renames to ft- names and Frametop paths follow. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
1a76d1560b
49 files changed
+7650
No files matched your search
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
* fh_gestures - pinch state frame-hands' tracker publishes for input: look at something
|
||||
* and pinch to click it, pinch and move to drag (the Vision Pro model, with the eye
|
||||
* tracker doing the looking). $XDG_RUNTIME_DIR/frame-hands/gestures, next to the hands
|
||||
* file, with the same sequence lock (read seq, copy, read seq again; use the copy only if
|
||||
* both reads are the same even number) and the same frame: metres in the head frame at
|
||||
* capture time, OpenVR's HMD frame (+x right, +y up, -z forward).
|
||||
*
|
||||
* One slot per side: pinch[0] is the left hand, pinch[1] the right. A pinch follows the
|
||||
* hand it began on until it ends. It begins when the thumb and index tips close within
|
||||
* begin_m and ends when they open past end_m (the gap between keeps it from flickering),
|
||||
* or when the hand stays lost too long (FH_PINCH_LOST).
|
||||
*
|
||||
* Don't miss short pinches: a reader that polls slower than a quick tap still sees it,
|
||||
* because begins and ends count every pinch. When begins changed, a pinch began at
|
||||
* begin_ns; when ends changed, one ended at end_ns. begins - ends is 1 while pinching.
|
||||
*
|
||||
* Drags: point is where the pinch is now, begin_point where it began. Turn each into the
|
||||
* room with the HMD pose at its capture time (capture_ns, begin_ns) before subtracting,
|
||||
* so turning your head doesn't drag.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <assert.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#define FH_GESTURES_MAGIC "FHGEST01"
|
||||
#define FH_GESTURES_VERSION 1
|
||||
|
||||
enum {
|
||||
FH_PINCH_TRACKED = 1u << 0, /* the hand was tracked in this frame */
|
||||
FH_PINCH_DOWN = 1u << 1, /* pinching now */
|
||||
FH_PINCH_LOST = 1u << 2, /* the last pinch ended because the hand was lost */
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
uint32_t flags; /* FH_PINCH_* */
|
||||
uint32_t hand_id; /* fh_hand_t.id of the hand, 0 if none */
|
||||
uint32_t begins; /* pinches begun so far */
|
||||
uint32_t ends; /* pinches ended so far */
|
||||
uint64_t begin_ns; /* capture time (CLOCK_MONOTONIC) the current or */
|
||||
/* last pinch began */
|
||||
uint64_t end_ns; /* ... the last pinch ended */
|
||||
float distance; /* thumb tip to index tip, m, at this user's hand */
|
||||
/* size */
|
||||
float strength; /* 0 open (end_m or more) .. 1 closed (begin_m) */
|
||||
float point[3]; /* between the thumb and index tips */
|
||||
float begin_point[3]; /* point when the current or last pinch began */
|
||||
} fh_pinch_t; /* 64 bytes */
|
||||
|
||||
typedef struct {
|
||||
char magic[8];
|
||||
uint32_t version;
|
||||
uint32_t size;
|
||||
volatile uint64_t seq;
|
||||
uint64_t capture_ns; /* CLOCK_MONOTONIC when the cameras took the frames */
|
||||
uint64_t publish_ns; /* CLOCK_MONOTONIC when this was written */
|
||||
float begin_m; /* the thresholds in use */
|
||||
float end_m;
|
||||
uint8_t reserved[16];
|
||||
fh_pinch_t pinch[2]; /* [0] left hand, [1] right hand */
|
||||
} fh_gestures_t;
|
||||
|
||||
static_assert(sizeof(fh_pinch_t) == 64, "fh_pinch_t layout");
|
||||
static_assert(sizeof(fh_gestures_t) == 64 + 2 * 64, "fh_gestures_t layout");
|
||||
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
* fh_hands - the tracked-hands file frame-hands' tracker publishes
|
||||
* ($XDG_RUNTIME_DIR/frame-hands/hands, directory mode 0700), rewritten in place
|
||||
* under a sequence lock: read seq, copy, read seq again; use the copy only if
|
||||
* both reads are the same even number.
|
||||
*
|
||||
* Positions are metres in the head frame at capture time, which is OpenVR's HMD
|
||||
* frame (+x right, +y up, -z forward). Turn them into the room with the HMD pose
|
||||
* at capture_ns (CLOCK_MONOTONIC). Writers: trackd (fh-tracker), tracker/publish.py.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <assert.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#define FH_HANDS_MAGIC "FHHANDS1"
|
||||
#define FH_HANDS_VERSION 1
|
||||
#define FH_HANDS_MAX_HANDS 2
|
||||
#define FH_HANDS_MAX_CAPSULES 64
|
||||
|
||||
enum {
|
||||
FH_HAND_RIGHT = 1u << 0, /* else the left hand */
|
||||
FH_HAND_STEREO = 1u << 1, /* triangulated from two or more cameras */
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
uint32_t id; /* stays the same while the hand is tracked */
|
||||
uint32_t flags; /* FH_HAND_* */
|
||||
float confidence;
|
||||
float reserved;
|
||||
float pts[21][3]; /* MediaPipe hand landmarks */
|
||||
uint32_t ncapsules; /* this hand's capsules, which follow the */
|
||||
/* previous hands' in capsules[] */
|
||||
} fh_hand_t; /* 272 bytes */
|
||||
|
||||
typedef struct {
|
||||
float a[3], b[3]; /* segment ends */
|
||||
float ra, rb; /* radius at each end */
|
||||
} fh_capsule_t; /* 32 bytes: the hand's shape, to cut out */
|
||||
|
||||
typedef struct {
|
||||
char magic[8];
|
||||
uint32_t version;
|
||||
uint32_t size;
|
||||
volatile uint64_t seq;
|
||||
uint64_t capture_ns; /* CLOCK_MONOTONIC when the cameras took the frames */
|
||||
uint64_t publish_ns; /* CLOCK_MONOTONIC when this was written */
|
||||
uint32_t nhands;
|
||||
uint32_t ncapsules;
|
||||
uint8_t reserved[16];
|
||||
fh_hand_t hands[FH_HANDS_MAX_HANDS];
|
||||
fh_capsule_t capsules[FH_HANDS_MAX_CAPSULES];
|
||||
} fh_hands_t;
|
||||
|
||||
static_assert(sizeof(fh_hand_t) == 272, "fh_hand_t layout");
|
||||
static_assert(sizeof(fh_capsule_t) == 32, "fh_capsule_t layout");
|
||||
static_assert(sizeof(fh_hands_t) == 64 + 2 * 272 + 64 * 32, "fh_hands_t layout");
|
||||
Reference in new issue
Block a user