// MediaPipe's palm detector and hand landmark model on ncnn. A crop is a square region of a camera image: centre and size in // pixels, and a rotation that turns the crop's "up" toward the image direction // (sin r, -cos r). Crops are contrast-equalized (CLAHE) before the models see them. // Everything here may run on several threads at once. #pragma once #include "geom.h" #include #include #include #include struct Image { const uint8_t *data = nullptr; int width = 0, height = 0, stride = 0; }; struct Roi { V2 center{}; double size = 0, rotation = 0; }; struct Palm { V2 center{}, size{}; V2 kp[7]{}; double score = 0; Roi roi() const; // MediaPipe's hand crop for this palm }; struct Landmarks { V2 pts[21]{}; // image pixels double world[21][3]{}; // MediaPipe's metric landmarks, hand-centred double presence = 0, right = 0; Roi next_roi() const; // MediaPipe's crop to track the hand in the next frame }; Roi roi_from_points(const V2 *pts21); // How crops are contrast-equalized before the models see them. struct Contrast { enum Mode { Clahe, None, Stretch } mode = Clahe; double clip = 2.0; // Clahe: OpenCV's clip limit (4x4 tiles) // "clahe:2", "none", "stretch" (1st..99th percentile to 0..255) static bool parse(const std::string &s, Contrast &out); // "PALM/HAND" (each as above), or one for both static bool parse_pair(const std::string &s, Contrast &palm, Contrast &hand); }; class Nets { public: // Loads /palm.ncnn.* and /hand.ncnn.*, or the -int8 variants. bool load(const std::string &dir, bool int8, std::string &err); std::vector palms(const Image &img, V2 center, double size, double rotation) const; Landmarks landmarks(const Image &img, const Roi &roi) const; // Before any palms()/landmarks(): how the palm search's and the landmark model's crops // are equalized. void set_contrast(const Contrast &palm, const Contrast &hand) { palm_contrast_ = palm, hand_contrast_ = hand; } private: Contrast palm_contrast_, hand_contrast_; ncnn::Net palm_, hand_; std::vector anchors_; };