mirror of
https://github.com/DeeJanuz/frametop.git
synced 2026-10-06 04:04:09 +02:00
DeeJanuzandClaude Opus 5.5
499035216c
- Programs: ft-camd (the camera broker), ft-hands (the tracker), and ft-handreplay and ft-ringplay for recordings, built by hands/build.sh into hands/build/ with one Makefile. The first build fetches ncnn at frame-hands' pinned tag and builds it with the same options. - ft-camd gets its privileges from file capabilities (CAP_SYS_PTRACE, CAP_PERFMON, CAP_DAC_READ_SEARCH) that hands/run.sh install sets with sudo, and drops them once set up. It still works under sudo. It runs on the host, linked statically, as frametop-camd.service. ft-hands runs in the dev container as frametop-hands.service. Both start and stop with SteamVR. - Files move to /run/user/UID/frametop/ (cam-ring, hands, gestures), not $XDG_RUNTIME_DIR, which a terminal in the Frametop desktop has its own of. SIGUSR1 recordings go to ~/.local/share/frametop/hands. - The calibration is read through /run/host in the container. - Settings: HANDS_SWAP_SIDES and HANDS_CPUS in frametop.conf. - install.sh offers hand tracking as an optional last step. - The container gets jsoncpp-devel, glibc-static, and NumPy and OpenCV for the Python tools. - tools/ring.py reads the ring, and models/NOTICE credits the Apache-2.0 models. Checked: ft-handreplay gives identical summaries and byte-identical depth dumps to frame-hands' fh-replay on both 2026-09-29 recordings. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
66 lines
2.2 KiB
C++
66 lines
2.2 KiB
C++
// MediaPipe's palm detector and hand landmark model on ncnn. A crop is a square region of a camera image: centre and size in
|
|
// pixels, and a rotation that turns the crop's "up" toward the image direction
|
|
// (sin r, -cos r). Crops are contrast-equalized (CLAHE) before the models see them.
|
|
// Everything here may run on several threads at once.
|
|
#pragma once
|
|
|
|
#include "geom.h"
|
|
|
|
#include <net.h>
|
|
|
|
#include <cstdint>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
struct Image {
|
|
const uint8_t *data = nullptr;
|
|
int width = 0, height = 0, stride = 0;
|
|
};
|
|
|
|
struct Roi {
|
|
V2 center{};
|
|
double size = 0, rotation = 0;
|
|
};
|
|
|
|
struct Palm {
|
|
V2 center{}, size{};
|
|
V2 kp[7]{};
|
|
double score = 0;
|
|
Roi roi() const; // MediaPipe's hand crop for this palm
|
|
};
|
|
|
|
struct Landmarks {
|
|
V2 pts[21]{}; // image pixels
|
|
double world[21][3]{}; // MediaPipe's metric landmarks, hand-centred
|
|
double presence = 0, right = 0;
|
|
Roi next_roi() const; // MediaPipe's crop to track the hand in the next frame
|
|
};
|
|
|
|
Roi roi_from_points(const V2 *pts21);
|
|
|
|
// How crops are contrast-equalized before the models see them.
|
|
struct Contrast {
|
|
enum Mode { Clahe, None, Stretch } mode = Clahe;
|
|
double clip = 2.0; // Clahe: OpenCV's clip limit (4x4 tiles)
|
|
// "clahe:2", "none", "stretch" (1st..99th percentile to 0..255)
|
|
static bool parse(const std::string &s, Contrast &out);
|
|
// "PALM/HAND" (each as above), or one for both
|
|
static bool parse_pair(const std::string &s, Contrast &palm, Contrast &hand);
|
|
};
|
|
|
|
class Nets {
|
|
public:
|
|
// Loads <dir>/palm.ncnn.* and <dir>/hand.ncnn.*, or the -int8 variants.
|
|
bool load(const std::string &dir, bool int8, std::string &err);
|
|
std::vector<Palm> palms(const Image &img, V2 center, double size, double rotation) const;
|
|
Landmarks landmarks(const Image &img, const Roi &roi) const;
|
|
// Before any palms()/landmarks(): how the palm search's and the landmark model's crops
|
|
// are equalized.
|
|
void set_contrast(const Contrast &palm, const Contrast &hand) { palm_contrast_ = palm, hand_contrast_ = hand; }
|
|
|
|
private:
|
|
Contrast palm_contrast_, hand_contrast_;
|
|
ncnn::Net palm_, hand_;
|
|
std::vector<V2> anchors_;
|
|
};
|