mirror of
https://github.com/DeeJanuz/frametop.git
synced 2026-10-06 01:00:06 +02:00
Cut tracked hands out of screens so you see them through it (work in progress)
Where frame-hands tracks a hand between an eye and a screen, that eye sees the room through the screen. handcut.cpp draws the screen's buffer side by side (one half per eye) with the hands cut out, only while a hand is in front of it. The cutouts command turns it on or off. ft-handtest tries it on a test panel. Not yet tested in the headset. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
DeeJanuzandClaude Opus 5.5 committed
1 parent
ed9542d3b8
commit
b7cbd9fe16
5 files changed
+950
-9
No files matched your search
+11
-6
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build ft-screens in the dev container on the Frame (screens/build/ft-screens).
|
||||
# Build ft-screens in the dev container on the Frame (screens/build/ft-screens), and
|
||||
# ft-handtest, which tries the hand cutouts (handcut.cpp) on a test panel of its own.
|
||||
# compositor.c is the wlroots side (C; wlroots headers aren't C++), vr.cpp the OpenVR side.
|
||||
# vr.cpp needs OpenVR's IVRIPCResourceManagerClient (ImportDmabuf), which the header
|
||||
# shipped with SteamVR on the Frame predates, so the build uses the public header from
|
||||
@@ -12,8 +13,12 @@ openvr=v2.15.6
|
||||
[ -f build/include/openvr-$openvr ] || { curl -fsSL "https://raw.githubusercontent.com/ValveSoftware/openvr/$openvr/headers/openvr.h" -o build/include/openvr.h && touch build/include/openvr-$openvr; }
|
||||
gcc -std=c11 -O2 -Wall -Wno-unused-parameter -c -o build/compositor.o compositor.c \
|
||||
$(pkg-config --cflags wlroots-0.20 wayland-server xkbcommon libdrm pixman-1)
|
||||
g++ -std=c++17 -O2 -Wall -Wno-missing-field-initializers -Ibuild/include -c -o build/vr.o vr.cpp
|
||||
g++ -o build/ft-screens build/compositor.o build/vr.o \
|
||||
$(pkg-config --libs wlroots-0.20 wayland-server xkbcommon) \
|
||||
-L/opt/steamvr/bin/linuxarm64 -lopenvr_api -Wl,-rpath,/opt/steamvr/bin/linuxarm64
|
||||
echo "built build/ft-screens"'
|
||||
cxx="g++ -std=c++17 -O2 -Wall -Wno-missing-field-initializers -Ibuild/include $(pkg-config --cflags egl glesv2 gbm libdrm)"
|
||||
$cxx -c -o build/vr.o vr.cpp
|
||||
$cxx -c -o build/handcut.o handcut.cpp
|
||||
$cxx -c -o build/handtest.o handtest.cpp
|
||||
vrlibs="$(pkg-config --libs egl glesv2 gbm) -L/opt/steamvr/bin/linuxarm64 -lopenvr_api -Wl,-rpath,/opt/steamvr/bin/linuxarm64"
|
||||
g++ -o build/ft-screens build/compositor.o build/vr.o build/handcut.o \
|
||||
$(pkg-config --libs wlroots-0.20 wayland-server xkbcommon) $vrlibs
|
||||
g++ -o build/ft-handtest build/handtest.o build/handcut.o $vrlibs
|
||||
echo "built build/ft-screens build/ft-handtest"'
|
||||
@@ -0,0 +1,491 @@
|
||||
// Hand cutouts (see handcut.h).
|
||||
#include "handcut.h"
|
||||
|
||||
#include <EGL/egl.h>
|
||||
#include <EGL/eglext.h>
|
||||
#include <GLES2/gl2.h>
|
||||
#include <GLES2/gl2ext.h>
|
||||
#include <drm_fourcc.h>
|
||||
#include <fcntl.h>
|
||||
#include <gbm.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
|
||||
namespace handcut {
|
||||
namespace {
|
||||
|
||||
// The hands file (frame-hands/include/fh_hands.h).
|
||||
constexpr char kMagic[8] = {'F', 'H', 'H', 'A', 'N', 'D', 'S', '1'};
|
||||
constexpr size_t kHeader = 64, kHand = 272, kCapsule = 32, kMaxHands = 2, kMaxCapsules = 64;
|
||||
constexpr size_t kFileSize = kHeader + kMaxHands * kHand + kMaxCapsules * kCapsule;
|
||||
constexpr int64_t kStaleNs = 300'000'000; // hands older than this are gone
|
||||
constexpr int64_t kHistoryNs = 1'000'000'000;
|
||||
|
||||
int64_t MonoNs() {
|
||||
timespec ts;
|
||||
clock_gettime(CLOCK_MONOTONIC, &ts);
|
||||
return int64_t(ts.tv_sec) * 1'000'000'000 + ts.tv_nsec;
|
||||
}
|
||||
|
||||
void Apply(const Mat &m, const float p[3], float out[3]) {
|
||||
for (int i = 0; i < 3; ++i) out[i] = m.m[i][0] * p[0] + m.m[i][1] * p[1] + m.m[i][2] * p[2] + m.m[i][3];
|
||||
}
|
||||
|
||||
// Room -> panel-local: R^T (p - t).
|
||||
void ToLocal(const Mat &m, const double p[3], double out[3]) {
|
||||
const double d[3] = {p[0] - m.m[0][3], p[1] - m.m[1][3], p[2] - m.m[2][3]};
|
||||
for (int i = 0; i < 3; ++i) out[i] = m.m[0][i] * d[0] + m.m[1][i] * d[1] + m.m[2][i] * d[2];
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// ------------------------------------------------------------------------------- hands
|
||||
|
||||
bool Hands::Read() {
|
||||
if (!map_) {
|
||||
const int64_t now = MonoNs();
|
||||
if (now - lastOpenTry_ < 1'000'000'000) return false;
|
||||
lastOpenTry_ = now;
|
||||
const char *run = std::getenv("XDG_RUNTIME_DIR");
|
||||
const std::string path = std::string(run ? run : "/run/user/" + std::to_string(getuid())) + "/frame-hands/hands";
|
||||
fd_ = open(path.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW);
|
||||
if (fd_ < 0) return false;
|
||||
struct stat st;
|
||||
if (fstat(fd_, &st) < 0 || st.st_uid != getuid() || size_t(st.st_size) < kFileSize) {
|
||||
close(fd_), fd_ = -1;
|
||||
return false;
|
||||
}
|
||||
void *m = mmap(nullptr, kFileSize, PROT_READ, MAP_SHARED, fd_, 0);
|
||||
if (m == MAP_FAILED) {
|
||||
close(fd_), fd_ = -1;
|
||||
return false;
|
||||
}
|
||||
map_ = m;
|
||||
}
|
||||
const auto *p = static_cast<const volatile uint8_t *>(map_);
|
||||
auto u64 = [&](size_t off) { uint64_t v; std::memcpy(&v, const_cast<const uint8_t *>(p) + off, 8); return v; };
|
||||
const uint64_t s1 = __atomic_load_n(reinterpret_cast<const uint64_t *>(const_cast<const uint8_t *>(p) + 16), __ATOMIC_ACQUIRE);
|
||||
if ((s1 & 1) || s1 == seq_) return false;
|
||||
uint8_t copy[kFileSize];
|
||||
std::memcpy(copy, const_cast<const uint8_t *>(p), kFileSize);
|
||||
__atomic_thread_fence(__ATOMIC_ACQUIRE);
|
||||
if (u64(16) != s1 || std::memcmp(copy, kMagic, 8) != 0) return false;
|
||||
seq_ = s1;
|
||||
uint64_t capture, publish;
|
||||
uint32_t ncaps;
|
||||
std::memcpy(&capture, copy + 24, 8);
|
||||
std::memcpy(&publish, copy + 32, 8);
|
||||
std::memcpy(&ncaps, copy + 44, 4);
|
||||
captureNs_ = int64_t(capture), publishNs_ = int64_t(publish);
|
||||
const Mat head = HeadAt(captureNs_);
|
||||
world_.clear();
|
||||
for (uint32_t k = 0; k < std::min<uint32_t>(ncaps, kMaxCapsules); ++k) {
|
||||
float f[8];
|
||||
std::memcpy(f, copy + kHeader + kMaxHands * kHand + k * kCapsule, sizeof f);
|
||||
bool ok = true;
|
||||
for (float v : f) ok = ok && std::isfinite(v) && std::fabs(v) < 10;
|
||||
if (!ok || f[6] <= 0 || f[7] <= 0) continue;
|
||||
Capsule c;
|
||||
Apply(head, f, c.a);
|
||||
Apply(head, f + 3, c.b);
|
||||
c.ra = f[6], c.rb = f[7];
|
||||
world_.push_back(c);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
Mat Hands::HeadAt(int64_t ns) const {
|
||||
const Past *best = nullptr;
|
||||
for (const Past &p : history_)
|
||||
if (!best || std::llabs(p.ns - ns) < std::llabs(best->ns - ns)) best = &p;
|
||||
return best ? best->head : Mat{};
|
||||
}
|
||||
|
||||
bool Hands::Update(const Mat &head, int64_t nowNs) {
|
||||
history_.push_back({nowNs, head});
|
||||
while (!history_.empty() && nowNs - history_.front().ns > kHistoryNs) history_.erase(history_.begin());
|
||||
Read();
|
||||
if (nowNs - publishNs_ > kStaleNs) world_.clear();
|
||||
return !world_.empty();
|
||||
}
|
||||
|
||||
void EyePositions(const Mat &head, double out[2][3]) {
|
||||
const vr::EVREye eyes[2] = {vr::Eye_Left, vr::Eye_Right};
|
||||
for (int e = 0; e < 2; ++e) {
|
||||
const Mat t = vr::VRSystem()->GetEyeToHeadTransform(eyes[e]);
|
||||
for (int i = 0; i < 3; ++i)
|
||||
out[e][i] = head.m[i][0] * t.m[0][3] + head.m[i][1] * t.m[1][3] + head.m[i][2] * t.m[2][3] + head.m[i][3];
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------- project
|
||||
|
||||
namespace {
|
||||
|
||||
// Where the line from eye e through point q (both panel-local) meets the panel, as texture
|
||||
// pixels, and how much a size at q grows there. False if q isn't between the eye and it.
|
||||
bool OnPanel(const Panel &p, const double e[3], const double q[3], double *x, double *y, double *grow) {
|
||||
const double d[3] = {q[0] - e[0], q[1] - e[1], q[2] - e[2]};
|
||||
double s, u, v;
|
||||
if (p.curve <= 0) {
|
||||
if (e[2] <= q[2] || q[2] <= 0) return false;
|
||||
s = e[2] / (e[2] - q[2]);
|
||||
u = e[0] + s * d[0];
|
||||
v = e[1] + s * d[1];
|
||||
} else {
|
||||
// OpenVR bends a curved panel into a cylinder around (0, *, r), toward its front.
|
||||
const double r = p.curve, ez = e[2] - r;
|
||||
const double A = d[0] * d[0] + d[2] * d[2], B = 2 * (e[0] * d[0] + ez * d[2]), C = e[0] * e[0] + ez * ez - r * r;
|
||||
const double disc = B * B - 4 * A * C;
|
||||
if (A < 1e-12 || disc < 0) return false;
|
||||
s = (-B + std::sqrt(disc)) / (2 * A); // the far side: the panel, seen from inside
|
||||
const double px = e[0] + s * d[0], pz = e[2] + s * d[2];
|
||||
if (pz > r) return false;
|
||||
u = r * std::atan2(px, r - pz);
|
||||
v = e[1] + s * d[1];
|
||||
}
|
||||
if (s <= 1) return false; // the hand is behind the panel
|
||||
*x = (u / p.width + 0.5) * p.pxWidth;
|
||||
*y = (0.5 - v / p.height) * p.pxHeight;
|
||||
*grow = s;
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool Project(const Panel &p, const std::vector<Capsule> &caps, const double eyes[2][3], std::vector<Capsule2D> out[2]) {
|
||||
const double pxPerM = p.pxWidth / p.width;
|
||||
bool any = false;
|
||||
for (int e = 0; e < 2; ++e) {
|
||||
out[e].clear();
|
||||
double eye[3];
|
||||
ToLocal(p.pose, eyes[e], eye);
|
||||
if (eye[2] <= 0.01) continue; // behind the panel
|
||||
for (const Capsule &c : caps) {
|
||||
double a[3], b[3];
|
||||
const double wa[3] = {c.a[0], c.a[1], c.a[2]}, wb[3] = {c.b[0], c.b[1], c.b[2]};
|
||||
ToLocal(p.pose, wa, a);
|
||||
ToLocal(p.pose, wb, b);
|
||||
// a hand pushed through the panel: keep the part in front
|
||||
const double eps = 0.002;
|
||||
if (a[2] < eps && b[2] < eps) continue;
|
||||
if (a[2] < eps || b[2] < eps) {
|
||||
double *in = a[2] < eps ? b : a, *out3 = a[2] < eps ? a : b;
|
||||
const double t = (in[2] - eps) / (in[2] - out3[2]);
|
||||
for (int i = 0; i < 3; ++i) out3[i] = in[i] + t * (out3[i] - in[i]);
|
||||
}
|
||||
double ax, ay, ga, bx, by, gb;
|
||||
if (!OnPanel(p, eye, a, &ax, &ay, &ga) || !OnPanel(p, eye, b, &bx, &by, &gb)) continue;
|
||||
const float ra = float(c.ra * ga * pxPerM), rb = float(c.rb * gb * pxPerM);
|
||||
const float r = std::max(ra, rb);
|
||||
if (std::max(ax, bx) + r < 0 || std::min(ax, bx) - r > p.pxWidth || std::max(ay, by) + r < 0 ||
|
||||
std::min(ay, by) - r > p.pxHeight)
|
||||
continue;
|
||||
out[e].push_back({float(ax), float(ay), float(bx), float(by), ra, rb});
|
||||
any = true;
|
||||
}
|
||||
}
|
||||
return any;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------- renderer
|
||||
|
||||
namespace {
|
||||
|
||||
PFNEGLGETPLATFORMDISPLAYEXTPROC pGetPlatformDisplay;
|
||||
PFNEGLCREATEIMAGEKHRPROC pCreateImage;
|
||||
PFNEGLDESTROYIMAGEKHRPROC pDestroyImage;
|
||||
PFNGLEGLIMAGETARGETTEXTURE2DOESPROC pImageTargetTexture;
|
||||
PFNGLEGLIMAGETARGETRENDERBUFFERSTORAGEOESPROC pImageTargetRenderbuffer;
|
||||
|
||||
const char *kVertex = R"(
|
||||
attribute vec2 pos; // the unit square
|
||||
uniform vec4 rect; // where it goes, in pixels of the eye's half: x0 y0 x1 y1
|
||||
uniform vec2 size; // the half's size in pixels
|
||||
varying vec2 px;
|
||||
varying vec2 uv;
|
||||
void main() {
|
||||
px = mix(rect.xy, rect.zw, pos);
|
||||
uv = px / size;
|
||||
gl_Position = vec4(uv * 2.0 - 1.0, 0.0, 1.0);
|
||||
})";
|
||||
|
||||
// The client's pixels, opaque (its alpha is ignored, as IgnoreTextureAlpha did).
|
||||
const char *kCopy = R"(
|
||||
#extension GL_OES_EGL_image_external : require
|
||||
precision mediump float;
|
||||
uniform samplerExternalOES tex;
|
||||
varying vec2 uv;
|
||||
void main() { gl_FragColor = vec4(texture2D(tex, uv).rgb, 1.0); })";
|
||||
|
||||
// Coverage of one tapered capsule; blended to take that much alpha away.
|
||||
const char *kCut = R"(
|
||||
precision highp float;
|
||||
uniform vec2 a, b, r;
|
||||
uniform float feather;
|
||||
varying vec2 px;
|
||||
void main() {
|
||||
vec2 ab = b - a;
|
||||
float t = clamp(dot(px - a, ab) / max(dot(ab, ab), 1e-6), 0.0, 1.0);
|
||||
float d = length(px - (a + t * ab));
|
||||
float rad = mix(r.x, r.y, t);
|
||||
gl_FragColor = vec4(0.0, 0.0, 0.0, 1.0 - smoothstep(rad - feather, rad + feather, d));
|
||||
})";
|
||||
|
||||
unsigned Shader(GLenum type, const char *src) {
|
||||
const GLuint s = glCreateShader(type);
|
||||
glShaderSource(s, 1, &src, nullptr);
|
||||
glCompileShader(s);
|
||||
GLint ok = 0;
|
||||
glGetShaderiv(s, GL_COMPILE_STATUS, &ok);
|
||||
if (!ok) {
|
||||
char log[1024] = "";
|
||||
glGetShaderInfoLog(s, sizeof log, nullptr, log);
|
||||
std::fprintf(stderr, "handcut: shader: %s\n", log);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
unsigned Program(const char *fs) {
|
||||
const GLuint p = glCreateProgram();
|
||||
glAttachShader(p, Shader(GL_VERTEX_SHADER, kVertex));
|
||||
glAttachShader(p, Shader(GL_FRAGMENT_SHADER, fs));
|
||||
glBindAttribLocation(p, 0, "pos");
|
||||
glLinkProgram(p);
|
||||
GLint ok = 0;
|
||||
glGetProgramiv(p, GL_LINK_STATUS, &ok);
|
||||
if (!ok) {
|
||||
char log[1024] = "";
|
||||
glGetProgramInfoLog(p, sizeof log, nullptr, log);
|
||||
std::fprintf(stderr, "handcut: program: %s\n", log);
|
||||
return 0;
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
EGLImageKHR ImageFor(EGLDisplay dpy, const ft_dmabuf &b) {
|
||||
static const EGLint fd[4] = {EGL_DMA_BUF_PLANE0_FD_EXT, EGL_DMA_BUF_PLANE1_FD_EXT, EGL_DMA_BUF_PLANE2_FD_EXT,
|
||||
EGL_DMA_BUF_PLANE3_FD_EXT};
|
||||
static const EGLint off[4] = {EGL_DMA_BUF_PLANE0_OFFSET_EXT, EGL_DMA_BUF_PLANE1_OFFSET_EXT,
|
||||
EGL_DMA_BUF_PLANE2_OFFSET_EXT, EGL_DMA_BUF_PLANE3_OFFSET_EXT};
|
||||
static const EGLint pitch[4] = {EGL_DMA_BUF_PLANE0_PITCH_EXT, EGL_DMA_BUF_PLANE1_PITCH_EXT,
|
||||
EGL_DMA_BUF_PLANE2_PITCH_EXT, EGL_DMA_BUF_PLANE3_PITCH_EXT};
|
||||
static const EGLint lo[4] = {EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT, EGL_DMA_BUF_PLANE1_MODIFIER_LO_EXT,
|
||||
EGL_DMA_BUF_PLANE2_MODIFIER_LO_EXT, EGL_DMA_BUF_PLANE3_MODIFIER_LO_EXT};
|
||||
static const EGLint hi[4] = {EGL_DMA_BUF_PLANE0_MODIFIER_HI_EXT, EGL_DMA_BUF_PLANE1_MODIFIER_HI_EXT,
|
||||
EGL_DMA_BUF_PLANE2_MODIFIER_HI_EXT, EGL_DMA_BUF_PLANE3_MODIFIER_HI_EXT};
|
||||
EGLint a[64];
|
||||
int n = 0;
|
||||
a[n++] = EGL_WIDTH, a[n++] = b.width, a[n++] = EGL_HEIGHT, a[n++] = b.height;
|
||||
a[n++] = EGL_LINUX_DRM_FOURCC_EXT, a[n++] = EGLint(b.format);
|
||||
for (int i = 0; i < b.n_planes && i < 4; ++i) {
|
||||
a[n++] = fd[i], a[n++] = b.fd[i], a[n++] = off[i], a[n++] = EGLint(b.offset[i]);
|
||||
a[n++] = pitch[i], a[n++] = EGLint(b.stride[i]);
|
||||
if (b.modifier != DRM_FORMAT_MOD_INVALID) {
|
||||
a[n++] = lo[i], a[n++] = EGLint(b.modifier & 0xffffffff);
|
||||
a[n++] = hi[i], a[n++] = EGLint(b.modifier >> 32);
|
||||
}
|
||||
}
|
||||
a[n++] = EGL_NONE;
|
||||
return pCreateImage(dpy, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, a);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Renderer::~Renderer() {
|
||||
for (auto &[k, r] : rings_)
|
||||
for (Output &o : r.out) FreeOutput(o);
|
||||
for (auto &[k, im] : imported_) {
|
||||
glDeleteTextures(1, &im.tex);
|
||||
pDestroyImage(EGLDisplay(dpy_), EGLImageKHR(im.image));
|
||||
}
|
||||
if (ctx_) eglDestroyContext(EGLDisplay(dpy_), EGLContext(ctx_));
|
||||
if (dpy_) eglTerminate(EGLDisplay(dpy_));
|
||||
if (gbm_) gbm_device_destroy(static_cast<gbm_device *>(gbm_));
|
||||
if (drm_ >= 0) close(drm_);
|
||||
}
|
||||
|
||||
bool Renderer::Init(const std::vector<uint64_t> &modifiers, std::function<void(const Output *)> released) {
|
||||
if (ready_) return true;
|
||||
modifiers_ = modifiers;
|
||||
released_ = std::move(released);
|
||||
drm_ = open("/dev/dri/renderD128", O_RDWR | O_CLOEXEC);
|
||||
if (drm_ < 0) return std::perror("handcut: /dev/dri/renderD128"), false;
|
||||
gbm_ = gbm_create_device(drm_);
|
||||
pGetPlatformDisplay = reinterpret_cast<PFNEGLGETPLATFORMDISPLAYEXTPROC>(eglGetProcAddress("eglGetPlatformDisplayEXT"));
|
||||
pCreateImage = reinterpret_cast<PFNEGLCREATEIMAGEKHRPROC>(eglGetProcAddress("eglCreateImageKHR"));
|
||||
pDestroyImage = reinterpret_cast<PFNEGLDESTROYIMAGEKHRPROC>(eglGetProcAddress("eglDestroyImageKHR"));
|
||||
pImageTargetTexture = reinterpret_cast<PFNGLEGLIMAGETARGETTEXTURE2DOESPROC>(eglGetProcAddress("glEGLImageTargetTexture2DOES"));
|
||||
pImageTargetRenderbuffer = reinterpret_cast<PFNGLEGLIMAGETARGETRENDERBUFFERSTORAGEOESPROC>(
|
||||
eglGetProcAddress("glEGLImageTargetRenderbufferStorageOES"));
|
||||
if (!gbm_ || !pGetPlatformDisplay || !pCreateImage || !pImageTargetTexture || !pImageTargetRenderbuffer) {
|
||||
std::fprintf(stderr, "handcut: GBM or EGL extensions missing\n");
|
||||
return false;
|
||||
}
|
||||
EGLDisplay dpy = pGetPlatformDisplay(EGL_PLATFORM_GBM_KHR, gbm_, nullptr);
|
||||
if (dpy == EGL_NO_DISPLAY || !eglInitialize(dpy, nullptr, nullptr)) return std::fprintf(stderr, "handcut: no EGL display\n"), false;
|
||||
dpy_ = dpy;
|
||||
eglBindAPI(EGL_OPENGL_ES_API);
|
||||
const EGLint attrs[] = {EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE};
|
||||
EGLContext ctx = eglCreateContext(dpy, EGL_NO_CONFIG_KHR, EGL_NO_CONTEXT, attrs);
|
||||
if (ctx == EGL_NO_CONTEXT || !eglMakeCurrent(dpy, EGL_NO_SURFACE, EGL_NO_SURFACE, ctx))
|
||||
return std::fprintf(stderr, "handcut: no surfaceless GLES context\n"), false;
|
||||
ctx_ = ctx;
|
||||
copyProg_ = Program(kCopy);
|
||||
cutProg_ = Program(kCut);
|
||||
if (!copyProg_ || !cutProg_) return false;
|
||||
const float quad[] = {0, 0, 1, 0, 0, 1, 1, 1};
|
||||
glGenBuffers(1, &vbo_);
|
||||
glBindBuffer(GL_ARRAY_BUFFER, vbo_);
|
||||
glBufferData(GL_ARRAY_BUFFER, sizeof quad, quad, GL_STATIC_DRAW);
|
||||
ready_ = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
unsigned Renderer::Texture(const void *key, const ft_dmabuf &src) {
|
||||
auto it = imported_.find(key);
|
||||
if (it != imported_.end()) return it->second.tex;
|
||||
EGLImageKHR image = ImageFor(EGLDisplay(dpy_), src);
|
||||
if (image == EGL_NO_IMAGE_KHR) {
|
||||
std::fprintf(stderr, "handcut: can't import a %dx%d client buffer (format 0x%x modifier 0x%llx)\n", src.width,
|
||||
src.height, src.format, (unsigned long long)src.modifier);
|
||||
return 0;
|
||||
}
|
||||
GLuint tex;
|
||||
glGenTextures(1, &tex);
|
||||
glBindTexture(GL_TEXTURE_EXTERNAL_OES, tex);
|
||||
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
|
||||
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
|
||||
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
|
||||
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
|
||||
pImageTargetTexture(GL_TEXTURE_EXTERNAL_OES, image);
|
||||
imported_[key] = {image, tex};
|
||||
return tex;
|
||||
}
|
||||
|
||||
void Renderer::Forget(const void *key) {
|
||||
auto it = imported_.find(key);
|
||||
if (it == imported_.end()) return;
|
||||
glDeleteTextures(1, &it->second.tex);
|
||||
pDestroyImage(EGLDisplay(dpy_), EGLImageKHR(it->second.image));
|
||||
imported_.erase(it);
|
||||
}
|
||||
|
||||
bool Renderer::MakeOutput(Output &o, int w, int h) {
|
||||
auto *gbm = static_cast<gbm_device *>(gbm_);
|
||||
std::vector<uint64_t> mods;
|
||||
for (uint64_t m : modifiers_)
|
||||
if (m != DRM_FORMAT_MOD_INVALID) mods.push_back(m);
|
||||
gbm_bo *bo = mods.empty() ? gbm_bo_create(gbm, w, h, GBM_FORMAT_ABGR8888, GBM_BO_USE_RENDERING | GBM_BO_USE_LINEAR)
|
||||
: gbm_bo_create_with_modifiers2(gbm, w, h, GBM_FORMAT_ABGR8888, mods.data(),
|
||||
unsigned(mods.size()), GBM_BO_USE_RENDERING);
|
||||
if (!bo) return std::fprintf(stderr, "handcut: can't allocate a %dx%d output\n", w, h), false;
|
||||
o.bo = bo;
|
||||
o.buf = {};
|
||||
o.buf.width = w, o.buf.height = h;
|
||||
o.buf.format = DRM_FORMAT_ABGR8888;
|
||||
o.buf.modifier = mods.empty() ? DRM_FORMAT_MOD_LINEAR : gbm_bo_get_modifier(bo);
|
||||
o.buf.n_planes = gbm_bo_get_plane_count(bo);
|
||||
for (int i = 0; i < o.buf.n_planes && i < 4; ++i) {
|
||||
o.buf.fd[i] = gbm_bo_get_fd_for_plane(bo, i);
|
||||
o.buf.offset[i] = gbm_bo_get_offset(bo, i);
|
||||
o.buf.stride[i] = gbm_bo_get_stride_for_plane(bo, i);
|
||||
}
|
||||
EGLImageKHR image = ImageFor(EGLDisplay(dpy_), o.buf);
|
||||
if (image == EGL_NO_IMAGE_KHR) return std::fprintf(stderr, "handcut: can't render to the output\n"), FreeOutput(o), false;
|
||||
o.image = image;
|
||||
glGenRenderbuffers(1, &o.rb);
|
||||
glBindRenderbuffer(GL_RENDERBUFFER, o.rb);
|
||||
pImageTargetRenderbuffer(GL_RENDERBUFFER, image);
|
||||
glGenFramebuffers(1, &o.fbo);
|
||||
glBindFramebuffer(GL_FRAMEBUFFER, o.fbo);
|
||||
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, o.rb);
|
||||
if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE)
|
||||
return std::fprintf(stderr, "handcut: output framebuffer incomplete\n"), FreeOutput(o), false;
|
||||
return true;
|
||||
}
|
||||
|
||||
void Renderer::FreeOutput(Output &o) {
|
||||
if (o.bo && released_) released_(&o);
|
||||
if (o.fbo) glDeleteFramebuffers(1, &o.fbo);
|
||||
if (o.rb) glDeleteRenderbuffers(1, &o.rb);
|
||||
if (o.image) pDestroyImage(EGLDisplay(dpy_), EGLImageKHR(o.image));
|
||||
for (int i = 0; i < o.buf.n_planes && i < 4; ++i)
|
||||
if (o.buf.fd[i] >= 0) close(o.buf.fd[i]);
|
||||
if (o.bo) gbm_bo_destroy(static_cast<gbm_bo *>(o.bo));
|
||||
o = Output{};
|
||||
}
|
||||
|
||||
void Renderer::DropPanel(int panel) {
|
||||
auto it = rings_.find(panel);
|
||||
if (it == rings_.end()) return;
|
||||
for (Output &o : it->second.out) FreeOutput(o);
|
||||
rings_.erase(it);
|
||||
}
|
||||
|
||||
const Output *Renderer::Composite(int panel, const void *key, const ft_dmabuf &src, const std::vector<Capsule2D> eyes[2]) {
|
||||
if (!ready_) return nullptr;
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
const int w = src.width, h = src.height;
|
||||
Ring &ring = rings_[panel];
|
||||
if (ring.w != w || ring.h != h) {
|
||||
for (Output &old : ring.out) FreeOutput(old);
|
||||
ring.w = w, ring.h = h, ring.next = 0;
|
||||
}
|
||||
Output &o = ring.out[ring.next];
|
||||
if (!o.bo && !MakeOutput(o, 2 * w, h)) return nullptr;
|
||||
const GLuint tex = Texture(key, src);
|
||||
if (!tex) return nullptr;
|
||||
ring.next = (ring.next + 1) % 3;
|
||||
|
||||
glBindFramebuffer(GL_FRAMEBUFFER, o.fbo);
|
||||
glBindBuffer(GL_ARRAY_BUFFER, vbo_);
|
||||
glEnableVertexAttribArray(0);
|
||||
glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 0, nullptr);
|
||||
for (int e = 0; e < 2; ++e) {
|
||||
glViewport(e * w, 0, w, h);
|
||||
glDisable(GL_BLEND);
|
||||
glUseProgram(copyProg_);
|
||||
glActiveTexture(GL_TEXTURE0);
|
||||
glBindTexture(GL_TEXTURE_EXTERNAL_OES, tex);
|
||||
glUniform1i(glGetUniformLocation(copyProg_, "tex"), 0);
|
||||
glUniform4f(glGetUniformLocation(copyProg_, "rect"), 0, 0, float(w), float(h));
|
||||
glUniform2f(glGetUniformLocation(copyProg_, "size"), float(w), float(h));
|
||||
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
|
||||
// take alpha away where the hand is; the colour stays (straight alpha)
|
||||
glEnable(GL_BLEND);
|
||||
glBlendFuncSeparate(GL_ZERO, GL_ONE, GL_ZERO, GL_ONE_MINUS_SRC_ALPHA);
|
||||
glUseProgram(cutProg_);
|
||||
glUniform2f(glGetUniformLocation(cutProg_, "size"), float(w), float(h));
|
||||
const GLint uRect = glGetUniformLocation(cutProg_, "rect"), uA = glGetUniformLocation(cutProg_, "a"),
|
||||
uB = glGetUniformLocation(cutProg_, "b"), uR = glGetUniformLocation(cutProg_, "r"),
|
||||
uF = glGetUniformLocation(cutProg_, "feather");
|
||||
for (const Capsule2D &c : eyes[e]) {
|
||||
const float feather = std::max(1.5f, 0.15f * std::min(c.ra, c.rb));
|
||||
const float r = std::max(c.ra, c.rb) + feather;
|
||||
glUniform4f(uRect, std::min(c.ax, c.bx) - r, std::min(c.ay, c.by) - r, std::max(c.ax, c.bx) + r,
|
||||
std::max(c.ay, c.by) + r);
|
||||
glUniform2f(uA, c.ax, c.ay);
|
||||
glUniform2f(uB, c.bx, c.by);
|
||||
glUniform2f(uR, c.ra, c.rb);
|
||||
glUniform1f(uF, feather);
|
||||
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
|
||||
}
|
||||
}
|
||||
glDisable(GL_BLEND);
|
||||
// SteamVR reads the buffer from another process and GPU queue; make sure it's done.
|
||||
glFinish();
|
||||
lastMs_ = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
|
||||
return &o;
|
||||
}
|
||||
|
||||
} // namespace handcut
|
||||
@@ -0,0 +1,112 @@
|
||||
// Hand cutouts: where a tracked hand is between an eye and a screen, that eye sees the
|
||||
// room (Room View) through the screen instead of the screen drawn over the hand.
|
||||
//
|
||||
// frame-hands' tracker (a separate project, ~/Desktop/Projects/frame-hands) publishes
|
||||
// the hands it sees with the headset's cameras to $XDG_RUNTIME_DIR/frame-hands/hands:
|
||||
// capsules (finger bones, palm, forearm) in the head frame at capture time. Hands turns
|
||||
// them into the room with the head pose at that time. Project() finds where each eye
|
||||
// sees them on a panel, and Renderer draws the panel's client buffer into a side-by-side
|
||||
// buffer (left eye | right eye) with those spots transparent. A panel shows that buffer,
|
||||
// with the overlay's SideBySide_Parallel flag, only while a hand is in front of it.
|
||||
#pragma once
|
||||
|
||||
#include "vr.h"
|
||||
|
||||
#include <openvr.h>
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <vector>
|
||||
|
||||
namespace handcut {
|
||||
|
||||
using Mat = vr::HmdMatrix34_t;
|
||||
|
||||
struct Capsule { // in the room (standing universe), metres
|
||||
float a[3], b[3];
|
||||
float ra, rb;
|
||||
};
|
||||
|
||||
struct Capsule2D { // on a panel's texture, pixels from the top left
|
||||
float ax, ay, bx, by;
|
||||
float ra, rb;
|
||||
};
|
||||
|
||||
// A panel in the room: pose of its centre (+x right, +y up, +z out of its front),
|
||||
// size in metres, cylinder radius (0: flat), and its texture size in pixels.
|
||||
struct Panel {
|
||||
Mat pose;
|
||||
double width, height, curve;
|
||||
int pxWidth, pxHeight;
|
||||
};
|
||||
|
||||
class Hands {
|
||||
public:
|
||||
// Once per tick: the head pose now, CLOCK_MONOTONIC ns. Re-reads the hands file when
|
||||
// it changed. Returns true while fresh hands are known.
|
||||
bool Update(const Mat &head, int64_t nowNs);
|
||||
const std::vector<Capsule> &capsules() const { return world_; }
|
||||
|
||||
private:
|
||||
bool Read();
|
||||
Mat HeadAt(int64_t ns) const;
|
||||
struct Past { int64_t ns; Mat head; };
|
||||
std::vector<Past> history_; // the last second of head poses
|
||||
std::vector<Capsule> world_;
|
||||
int fd_ = -1;
|
||||
const void *map_ = nullptr;
|
||||
uint64_t seq_ = 0;
|
||||
int64_t captureNs_ = 0, publishNs_ = 0, lastOpenTry_ = 0;
|
||||
};
|
||||
|
||||
// Where each eye sees the capsules on the panel, for those in front of it. Eyes are
|
||||
// positions in the room. False if no capsule reaches the panel for either eye.
|
||||
bool Project(const Panel &p, const std::vector<Capsule> &caps, const double eyes[2][3],
|
||||
std::vector<Capsule2D> out[2]);
|
||||
|
||||
// The eye positions in the room for a head pose.
|
||||
void EyePositions(const Mat &head, double out[2][3]);
|
||||
|
||||
// An output buffer: the dmabuf SteamVR imports (see vr.cpp), stable while it exists.
|
||||
struct Output {
|
||||
ft_dmabuf buf{};
|
||||
void *bo = nullptr;
|
||||
unsigned fbo = 0, rb = 0;
|
||||
void *image = nullptr;
|
||||
};
|
||||
|
||||
class Renderer {
|
||||
public:
|
||||
~Renderer();
|
||||
// modifiers: what SteamVR takes for DRM_FORMAT_ABGR8888, the outputs' format.
|
||||
// released: an output is about to be freed (drop its SteamVR import).
|
||||
bool Init(const std::vector<uint64_t> &modifiers, std::function<void(const Output *)> released);
|
||||
// Draw client buffer `src` (identified by `key`) into the next output buffer of
|
||||
// panel `panel`, both eyes, cutting out `eyes`. Returns that buffer, or null.
|
||||
const Output *Composite(int panel, const void *key, const ft_dmabuf &src, const std::vector<Capsule2D> eyes[2]);
|
||||
// A client buffer is going away.
|
||||
void Forget(const void *key);
|
||||
// A panel is gone: drop its outputs.
|
||||
void DropPanel(int panel);
|
||||
// How long the last Composite took, ms (it waits for the GPU).
|
||||
double lastMs() const { return lastMs_; }
|
||||
|
||||
private:
|
||||
unsigned Texture(const void *key, const ft_dmabuf &src);
|
||||
bool MakeOutput(Output &o, int w, int h);
|
||||
void FreeOutput(Output &o);
|
||||
bool ready_ = false;
|
||||
int drm_ = -1;
|
||||
void *gbm_ = nullptr, *dpy_ = nullptr, *ctx_ = nullptr;
|
||||
unsigned copyProg_ = 0, cutProg_ = 0, vbo_ = 0;
|
||||
std::vector<uint64_t> modifiers_;
|
||||
std::function<void(const Output *)> released_;
|
||||
struct Imported { void *image; unsigned tex; };
|
||||
std::map<const void *, Imported> imported_;
|
||||
struct Ring { Output out[3]; int next = 0; int w = 0, h = 0; };
|
||||
std::map<int, Ring> rings_;
|
||||
double lastMs_ = 0;
|
||||
};
|
||||
|
||||
} // namespace handcut
|
||||
@@ -0,0 +1,211 @@
|
||||
// ft-handtest: try the hand cutouts without restarting the desktop. Shows a test panel
|
||||
// (a light grid) in front of you as its own overlay; where frame-hands tracks your hands
|
||||
// in front of it, each eye sees through it, like ft-screens' screens with cutouts.
|
||||
//
|
||||
// ft-handtest [--distance m] [--width m] [--seconds s]
|
||||
//
|
||||
// Needs frame-hands' tracker running (it publishes $XDG_RUNTIME_DIR/frame-hands/hands).
|
||||
// Build: screens/build.sh (build/ft-handtest), run in the dev container.
|
||||
#include "handcut.h"
|
||||
|
||||
#include <drm_fourcc.h>
|
||||
#include <fcntl.h>
|
||||
#include <gbm.h>
|
||||
#include <signal.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <map>
|
||||
#include <thread>
|
||||
|
||||
namespace {
|
||||
|
||||
volatile sig_atomic_t g_stop = 0;
|
||||
void Stop(int) { g_stop = 1; }
|
||||
|
||||
int64_t MonoNs() {
|
||||
timespec ts;
|
||||
clock_gettime(CLOCK_MONOTONIC, &ts);
|
||||
return int64_t(ts.tv_sec) * 1'000'000'000 + ts.tv_nsec;
|
||||
}
|
||||
|
||||
vr::SharedTextureHandle_t Import(const ft_dmabuf &b) {
|
||||
vr::DmabufAttributes_t a{};
|
||||
a.unWidth = uint32_t(b.width);
|
||||
a.unHeight = uint32_t(b.height);
|
||||
a.unDepth = a.unMipLevels = a.unArrayLayers = a.unSampleCount = 1;
|
||||
a.unFormat = b.format;
|
||||
a.ulModifier = b.modifier;
|
||||
a.unPlaneCount = uint32_t(b.n_planes);
|
||||
for (int i = 0; i < b.n_planes && i < int(vr::MaxDmabufPlaneCount); ++i) {
|
||||
a.plane[i].unOffset = b.offset[i];
|
||||
a.plane[i].unStride = b.stride[i];
|
||||
a.plane[i].nFd = b.fd[i];
|
||||
}
|
||||
vr::SharedTextureHandle_t h = 0;
|
||||
if (!vr::VRIPCResourceManager()->ImportDmabuf(vr::VRApplication_Overlay, &a, &h)) return 0;
|
||||
return h;
|
||||
}
|
||||
|
||||
// The stand-in "client" buffer: a light grey grid, linear so the CPU can draw it.
|
||||
bool TestPattern(int drm, int w, int h, gbm_bo **out, ft_dmabuf *b) {
|
||||
gbm_device *gbm = gbm_create_device(drm);
|
||||
gbm_bo *bo = gbm_bo_create(gbm, w, h, GBM_FORMAT_ABGR8888, GBM_BO_USE_LINEAR | GBM_BO_USE_RENDERING);
|
||||
if (!bo) return false;
|
||||
uint32_t stride = 0;
|
||||
void *data = nullptr;
|
||||
auto *px = static_cast<uint8_t *>(gbm_bo_map(bo, 0, 0, w, h, GBM_BO_TRANSFER_WRITE, &stride, &data));
|
||||
if (!px) return false;
|
||||
for (int y = 0; y < h; ++y)
|
||||
for (int x = 0; x < w; ++x) {
|
||||
uint8_t *p = px + size_t(y) * stride + size_t(x) * 4;
|
||||
const bool line = x % 80 < 3 || y % 80 < 3;
|
||||
const uint8_t g = line ? 90 : uint8_t(200 + 30 * x / w);
|
||||
p[0] = g, p[1] = g, p[2] = uint8_t(line ? 160 : g), p[3] = 255;
|
||||
}
|
||||
gbm_bo_unmap(bo, data);
|
||||
*b = {};
|
||||
b->width = w, b->height = h, b->format = DRM_FORMAT_ABGR8888, b->modifier = DRM_FORMAT_MOD_LINEAR;
|
||||
b->n_planes = 1;
|
||||
b->fd[0] = gbm_bo_get_fd(bo);
|
||||
b->stride[0] = gbm_bo_get_stride(bo);
|
||||
*out = bo;
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
double distance = 0.8, width = 1.0, seconds = 0;
|
||||
for (int i = 1; i + 1 < argc; i += 2) {
|
||||
if (!std::strcmp(argv[i], "--distance")) distance = std::atof(argv[i + 1]);
|
||||
else if (!std::strcmp(argv[i], "--width")) width = std::atof(argv[i + 1]);
|
||||
else if (!std::strcmp(argv[i], "--seconds")) seconds = std::atof(argv[i + 1]);
|
||||
}
|
||||
signal(SIGINT, Stop);
|
||||
signal(SIGTERM, Stop);
|
||||
|
||||
vr::EVRInitError err = vr::VRInitError_None;
|
||||
vr::VR_Init(&err, vr::VRApplication_Overlay);
|
||||
if (err != vr::VRInitError_None) {
|
||||
std::fprintf(stderr, "openvr: %s\n", vr::VR_GetVRInitErrorAsEnglishDescription(err));
|
||||
return 1;
|
||||
}
|
||||
uint64_t mods[64];
|
||||
uint32_t nmods = 64;
|
||||
if (!vr::VRIPCResourceManager()->GetDmabufModifiers(vr::VRApplication_Overlay, DRM_FORMAT_ABGR8888, &nmods, mods))
|
||||
nmods = 0;
|
||||
std::printf("SteamVR takes %u modifiers for ABGR8888\n", nmods);
|
||||
|
||||
std::map<const void *, vr::SharedTextureHandle_t> imports;
|
||||
handcut::Renderer renderer;
|
||||
if (!renderer.Init(std::vector<uint64_t>(mods, mods + nmods), [&](const handcut::Output *o) {
|
||||
auto it = imports.find(o);
|
||||
if (it != imports.end()) vr::VRIPCResourceManager()->UnrefResource(it->second), imports.erase(it);
|
||||
}))
|
||||
return 1;
|
||||
|
||||
const int W = 1600, H = 900;
|
||||
const int drm = open("/dev/dri/renderD128", O_RDWR | O_CLOEXEC);
|
||||
gbm_bo *bo = nullptr;
|
||||
ft_dmabuf client;
|
||||
if (drm < 0 || !TestPattern(drm, W, H, &bo, &client)) return std::fprintf(stderr, "can't make the test pattern\n"), 1;
|
||||
const vr::SharedTextureHandle_t plain = Import(client);
|
||||
if (!plain) return std::fprintf(stderr, "SteamVR can't import the test pattern\n"), 1;
|
||||
|
||||
vr::VROverlayHandle_t ov;
|
||||
if (vr::VROverlay()->CreateOverlay("frametop.handtest", "Hand cutout test", &ov) != vr::VROverlayError_None)
|
||||
return std::fprintf(stderr, "can't create the overlay (already running?)\n"), 1;
|
||||
vr::VROverlay()->SetOverlayWidthInMeters(ov, float(width));
|
||||
|
||||
// In front of the head as it is now, level, facing it.
|
||||
vr::TrackedDevicePose_t poses[vr::k_unMaxTrackedDeviceCount];
|
||||
vr::VRSystem()->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, poses, vr::k_unMaxTrackedDeviceCount);
|
||||
const auto &hm = poses[vr::k_unTrackedDeviceIndex_Hmd].mDeviceToAbsoluteTracking;
|
||||
const double yaw = std::atan2(hm.m[0][2], hm.m[2][2]);
|
||||
handcut::Panel panel{};
|
||||
auto &P = panel.pose;
|
||||
P.m[0][0] = float(std::cos(yaw)), P.m[0][2] = float(std::sin(yaw));
|
||||
P.m[1][1] = 1;
|
||||
P.m[2][0] = float(-std::sin(yaw)), P.m[2][2] = float(std::cos(yaw));
|
||||
P.m[0][3] = float(hm.m[0][3] - std::sin(yaw) * distance);
|
||||
P.m[1][3] = float(hm.m[1][3] - 0.15);
|
||||
P.m[2][3] = float(hm.m[2][3] - std::cos(yaw) * distance);
|
||||
panel.width = width, panel.height = width * H / W, panel.curve = 0, panel.pxWidth = W, panel.pxHeight = H;
|
||||
vr::VROverlay()->SetOverlayTransformAbsolute(ov, vr::TrackingUniverseStanding, &P);
|
||||
vr::VROverlay()->SetOverlayFlag(ov, vr::VROverlayFlags_IgnoreTextureAlpha, true);
|
||||
vr::SharedTextureHandle_t shown = plain;
|
||||
vr::Texture_t tex = {&shown, vr::TextureType_SharedTextureHandle, vr::ColorSpace_Gamma};
|
||||
vr::VROverlay()->SetOverlayTexture(ov, &tex);
|
||||
vr::VROverlay()->ShowOverlay(ov);
|
||||
std::printf("test panel %.2f m wide, %.2f m ahead; Ctrl+C to stop\n", width, distance);
|
||||
|
||||
handcut::Hands hands;
|
||||
bool cutting = false;
|
||||
const int64_t start = MonoNs();
|
||||
int64_t lastReport = start;
|
||||
int frames = 0, cutFrames = 0;
|
||||
double ms = 0, worst = 0;
|
||||
size_t caps2d = 0;
|
||||
while (!g_stop && (seconds <= 0 || (MonoNs() - start) / 1e9 < seconds)) {
|
||||
const auto tick = std::chrono::steady_clock::now();
|
||||
vr::VRSystem()->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, poses, vr::k_unMaxTrackedDeviceCount);
|
||||
const auto &head = poses[vr::k_unTrackedDeviceIndex_Hmd].mDeviceToAbsoluteTracking;
|
||||
const int64_t now = MonoNs();
|
||||
std::vector<handcut::Capsule2D> eyes2d[2];
|
||||
bool cut = false;
|
||||
if (hands.Update(head, now)) {
|
||||
double eyes[2][3];
|
||||
handcut::EyePositions(head, eyes);
|
||||
cut = handcut::Project(panel, hands.capsules(), eyes, eyes2d);
|
||||
}
|
||||
const handcut::Output *out = cut ? renderer.Composite(0, bo, client, eyes2d) : nullptr;
|
||||
if (out) {
|
||||
auto it = imports.find(out);
|
||||
if (it == imports.end()) {
|
||||
const vr::SharedTextureHandle_t h = Import(out->buf);
|
||||
if (!h) std::fprintf(stderr, "SteamVR can't import the output buffer\n");
|
||||
it = imports.emplace(out, h).first;
|
||||
}
|
||||
if (it->second) {
|
||||
if (!cutting) {
|
||||
vr::VROverlay()->SetOverlayFlag(ov, vr::VROverlayFlags_IgnoreTextureAlpha, false);
|
||||
vr::VROverlay()->SetOverlayFlag(ov, vr::VROverlayFlags_SideBySide_Parallel, true);
|
||||
cutting = true;
|
||||
}
|
||||
shown = it->second;
|
||||
vr::VROverlay()->SetOverlayTexture(ov, &tex);
|
||||
ms += renderer.lastMs(), worst = std::max(worst, renderer.lastMs());
|
||||
++cutFrames;
|
||||
caps2d += eyes2d[0].size() + eyes2d[1].size();
|
||||
}
|
||||
} else if (cutting) {
|
||||
vr::VROverlay()->SetOverlayFlag(ov, vr::VROverlayFlags_SideBySide_Parallel, false);
|
||||
vr::VROverlay()->SetOverlayFlag(ov, vr::VROverlayFlags_IgnoreTextureAlpha, true);
|
||||
shown = plain;
|
||||
vr::VROverlay()->SetOverlayTexture(ov, &tex);
|
||||
cutting = false;
|
||||
}
|
||||
++frames;
|
||||
if (now - lastReport > 2'000'000'000) {
|
||||
std::printf("%.0f s: %d ticks, %d with a cutout (%.1f capsules per eye), composite %.2f ms avg %.2f ms worst, "
|
||||
"%zu hand capsules known\n",
|
||||
(now - start) / 1e9, frames, cutFrames, cutFrames ? caps2d / 2.0 / cutFrames : 0.0,
|
||||
cutFrames ? ms / cutFrames : 0.0, worst, hands.capsules().size());
|
||||
std::fflush(stdout);
|
||||
lastReport = now, frames = cutFrames = 0, ms = worst = 0, caps2d = 0;
|
||||
}
|
||||
std::this_thread::sleep_until(tick + std::chrono::microseconds(11111));
|
||||
}
|
||||
vr::VROverlay()->DestroyOverlay(ov);
|
||||
for (auto &[k, h] : imports)
|
||||
if (h) vr::VRIPCResourceManager()->UnrefResource(h);
|
||||
imports.clear();
|
||||
vr::VRIPCResourceManager()->UnrefResource(plain);
|
||||
vr::VR_Shutdown();
|
||||
return 0;
|
||||
}
|
||||
+125
-3
@@ -40,10 +40,17 @@
|
||||
// own; also for flatscreen games, which aren't scene apps).
|
||||
// - during a VR game the screens hide unless the dashboard is open (g_inGames, default),
|
||||
// or stay visible over it; the hotkey still shows them.
|
||||
// - hand cutouts (handcut.cpp): where frame-hands tracks a hand between an eye and a
|
||||
// screen, that eye sees through the screen (to Room View). Only then is the screen
|
||||
// drawn by us, into a side-by-side buffer (one half per eye); otherwise its client
|
||||
// buffer is shown as is.
|
||||
// OpenVR has no overlay-relative transforms here (openvr v2.15.6), so the bar, button,
|
||||
// and handle are placed whenever their screen moves.
|
||||
#include "vr.h"
|
||||
|
||||
#include "handcut.h"
|
||||
|
||||
#include <drm_fourcc.h>
|
||||
#include <openvr.h>
|
||||
|
||||
#include <fcntl.h>
|
||||
@@ -233,6 +240,10 @@ struct Screen {
|
||||
vr::TrackedDeviceIndex_t pinTarget = kNone; // moving: rides on this controller when let go
|
||||
vr::TrackedDeviceIndex_t onWrist = kNone; // moving: the laser is in this controller's ring
|
||||
bool barLit = false;
|
||||
const void *key = nullptr; // the client buffer on it now, and its dmabuf (for cutouts)
|
||||
ft_dmabuf buf{};
|
||||
vr::SharedTextureHandle_t plain = 0; // that buffer's SteamVR import
|
||||
bool cutting = false; // showing a cutout buffer (side by side) instead
|
||||
double chrome = 0.3; // the bar's width; the other controls follow it (ChromeSize)
|
||||
double grip = 0.04; // the corner tab's and the round buttons' size
|
||||
double heightMetres() const { return width > 0 ? metres * height / width : metres * 9 / 16; }
|
||||
@@ -242,6 +253,13 @@ struct Screen {
|
||||
std::map<int, Screen> g_screens;
|
||||
std::map<const void *, vr::SharedTextureHandle_t> g_imports;
|
||||
|
||||
// Hand cutouts (see the top and handcut.h).
|
||||
bool g_cutouts = true; // the cutouts command turns them off
|
||||
handcut::Hands g_hands;
|
||||
handcut::Renderer g_cutter;
|
||||
int g_cutterState = 0; // 0 not tried, 1 ready, -1 unavailable
|
||||
std::map<const void *, vr::SharedTextureHandle_t> g_cutImports;
|
||||
|
||||
// Visibility (see the top). g_manual is the hide/show switch: in the always mode it hides
|
||||
// the screens, in the others it shows them anyway.
|
||||
Mode g_mode = Mode::Always;
|
||||
@@ -1038,6 +1056,92 @@ const char *ModeName() {
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- hand cutouts
|
||||
|
||||
void SetScreenTexture(const Screen &s, vr::SharedTextureHandle_t handle) {
|
||||
vr::Texture_t tex = {&handle, vr::TextureType_SharedTextureHandle, vr::ColorSpace_Gamma};
|
||||
vr::VROverlay()->SetOverlayTexture(s.overlay, &tex);
|
||||
}
|
||||
|
||||
// The cutout buffers' renderer, set up the first time a hand is in front of a screen.
|
||||
bool CutterReady() {
|
||||
if (g_cutterState) return g_cutterState > 0;
|
||||
uint64_t mods[64];
|
||||
const int n = ft_vr_modifiers(DRM_FORMAT_ABGR8888, mods, 64);
|
||||
const bool ok = g_cutter.Init(std::vector<uint64_t>(mods, mods + n), [](const handcut::Output *o) {
|
||||
auto it = g_cutImports.find(o);
|
||||
if (it == g_cutImports.end()) return;
|
||||
vr::VRIPCResourceManager()->UnrefResource(it->second);
|
||||
g_cutImports.erase(it);
|
||||
});
|
||||
g_cutterState = ok ? 1 : -1;
|
||||
std::printf(ok ? "hand cutouts ready\n" : "hand cutouts unavailable (see above)\n");
|
||||
return ok;
|
||||
}
|
||||
|
||||
vr::SharedTextureHandle_t ImportCutout(const handcut::Output *o) {
|
||||
auto it = g_cutImports.find(o);
|
||||
if (it != g_cutImports.end()) return it->second;
|
||||
vr::DmabufAttributes_t a{};
|
||||
a.unWidth = uint32_t(o->buf.width);
|
||||
a.unHeight = uint32_t(o->buf.height);
|
||||
a.unDepth = a.unMipLevels = a.unArrayLayers = a.unSampleCount = 1;
|
||||
a.unFormat = o->buf.format;
|
||||
a.ulModifier = o->buf.modifier;
|
||||
a.unPlaneCount = uint32_t(o->buf.n_planes);
|
||||
for (int i = 0; i < o->buf.n_planes && i < int(vr::MaxDmabufPlaneCount); ++i) {
|
||||
a.plane[i].unOffset = o->buf.offset[i];
|
||||
a.plane[i].unStride = o->buf.stride[i];
|
||||
a.plane[i].nFd = o->buf.fd[i];
|
||||
}
|
||||
vr::SharedTextureHandle_t h = 0;
|
||||
if (!vr::VRIPCResourceManager()->ImportDmabuf(vr::VRApplication_Overlay, &a, &h)) {
|
||||
std::fprintf(stderr, "openvr: ImportDmabuf failed for a cutout buffer\n");
|
||||
h = 0;
|
||||
}
|
||||
g_cutImports.emplace(o, h);
|
||||
return h;
|
||||
}
|
||||
|
||||
void StopCutting(Screen &s) {
|
||||
if (!s.cutting) return;
|
||||
vr::VROverlay()->SetOverlayFlag(s.overlay, vr::VROverlayFlags_SideBySide_Parallel, false);
|
||||
vr::VROverlay()->SetOverlayFlag(s.overlay, vr::VROverlayFlags_IgnoreTextureAlpha, true);
|
||||
if (s.plain) SetScreenTexture(s, s.plain);
|
||||
s.cutting = false;
|
||||
}
|
||||
|
||||
// Each tick: for each visible screen with a hand in front of it (for either eye), draw its
|
||||
// client buffer with the hands cut out and show that; else show the client buffer.
|
||||
void UpdateCutouts() {
|
||||
Mat head;
|
||||
const bool haveHead = DevicePose(vr::k_unTrackedDeviceIndex_Hmd, &head);
|
||||
const bool hands = g_cutouts && haveHead &&
|
||||
g_hands.Update(head, std::chrono::duration_cast<std::chrono::nanoseconds>(
|
||||
Clock::now().time_since_epoch()).count());
|
||||
double eyes[2][3];
|
||||
if (hands) handcut::EyePositions(head, eyes);
|
||||
for (auto &[i, s] : g_screens) {
|
||||
std::vector<handcut::Capsule2D> spots[2];
|
||||
Mat p;
|
||||
bool cut = hands && s.visible && s.key && s.width > 0 && ScreenPose(s, &p) &&
|
||||
handcut::Project({p, s.metres, s.heightMetres(), s.curve, s.width, s.height}, g_hands.capsules(),
|
||||
eyes, spots);
|
||||
const handcut::Output *out = cut && CutterReady() ? g_cutter.Composite(i, s.key, s.buf, spots) : nullptr;
|
||||
const vr::SharedTextureHandle_t h = out ? ImportCutout(out) : 0;
|
||||
if (!h) {
|
||||
StopCutting(s);
|
||||
continue;
|
||||
}
|
||||
if (!s.cutting) {
|
||||
vr::VROverlay()->SetOverlayFlag(s.overlay, vr::VROverlayFlags_IgnoreTextureAlpha, false);
|
||||
vr::VROverlay()->SetOverlayFlag(s.overlay, vr::VROverlayFlags_SideBySide_Parallel, true);
|
||||
s.cutting = true;
|
||||
}
|
||||
SetScreenTexture(s, h);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
extern "C" {
|
||||
@@ -1058,6 +1162,8 @@ bool ft_vr_init(void) {
|
||||
}
|
||||
|
||||
void ft_vr_shutdown(void) {
|
||||
if (g_cutterState == 1)
|
||||
for (auto &[i, s] : g_screens) g_cutter.DropPanel(i); // drops their imports while SteamVR is up
|
||||
for (auto &[i, s] : g_screens)
|
||||
for (auto o : s.All()) vr::VROverlay()->DestroyOverlay(o);
|
||||
for (auto &[dev, g] : g_guides)
|
||||
@@ -1123,6 +1229,7 @@ void ft_vr_screen_create(int index, double metres, int count) {
|
||||
void ft_vr_screen_destroy(int index) {
|
||||
auto it = g_screens.find(index);
|
||||
if (it == g_screens.end()) return;
|
||||
if (g_cutterState == 1) g_cutter.DropPanel(index);
|
||||
for (auto o : it->second.All()) vr::VROverlay()->DestroyOverlay(o);
|
||||
g_screens.erase(it);
|
||||
}
|
||||
@@ -1160,14 +1267,20 @@ bool ft_vr_screen_present(int index, const void *key, const struct ft_dmabuf *b)
|
||||
PlaceChrome(s); // the height changed
|
||||
std::printf("screen %d: %dx%d\n", index + 1, s.width, s.height);
|
||||
}
|
||||
vr::SharedTextureHandle_t handle = it->second;
|
||||
vr::Texture_t tex = {&handle, vr::TextureType_SharedTextureHandle, vr::ColorSpace_Gamma};
|
||||
vr::VROverlay()->SetOverlayTexture(s.overlay, &tex);
|
||||
s.key = key, s.buf = *b, s.plain = it->second;
|
||||
if (!s.cutting) { // otherwise the next tick draws the new buffer with the cutouts
|
||||
vr::SharedTextureHandle_t handle = it->second;
|
||||
vr::Texture_t tex = {&handle, vr::TextureType_SharedTextureHandle, vr::ColorSpace_Gamma};
|
||||
vr::VROverlay()->SetOverlayTexture(s.overlay, &tex);
|
||||
}
|
||||
s.shown = key; // UpdateVisibility shows it on the next tick
|
||||
return true;
|
||||
}
|
||||
|
||||
void ft_vr_forget(const void *key) {
|
||||
if (g_cutterState == 1) g_cutter.Forget(key);
|
||||
for (auto &[i, s] : g_screens)
|
||||
if (s.key == key) s.key = nullptr;
|
||||
auto it = g_imports.find(key);
|
||||
if (it == g_imports.end()) return;
|
||||
vr::VRIPCResourceManager()->UnrefResource(it->second);
|
||||
@@ -1276,6 +1389,7 @@ void ft_vr_poll(void (*handle)(const struct ft_event *, void *), void *data) {
|
||||
UpdateLasers();
|
||||
UpdateControls();
|
||||
UpdateGuides();
|
||||
UpdateCutouts();
|
||||
}
|
||||
|
||||
// Control commands (datagrams on @ft_screens, replies to the sender):
|
||||
@@ -1300,6 +1414,8 @@ void ft_vr_poll(void (*handle)(const struct ft_event *, void *), void *data) {
|
||||
// (hide), or stays as it is (visible)
|
||||
// state -> "ok <mode> <manual 0|1> <wrist deg> <gesture hand> <gesture deg>
|
||||
// <controllers> <game running 0|1> <ingames>"
|
||||
// cutouts on|off|state hand cutouts (see handcut.h) -> "ok <on|off> <ready|idle|unavailable>
|
||||
// <last composite ms>"
|
||||
// (size <screen> <w> <h> and key <code> <value> are handled in compositor.c.) Screens are
|
||||
// numbered from 1 here, like everywhere the user sees them.
|
||||
void ft_vr_command(const char *cmd, char *reply, int size) {
|
||||
@@ -1429,6 +1545,12 @@ void ft_vr_command(const char *cmd, char *reply, int size) {
|
||||
else return (void)std::snprintf(reply, size, "error modes: always outside_games dashboard");
|
||||
UpdateLasers();
|
||||
std::snprintf(reply, size, "ok %s", LasersName());
|
||||
} else if (std::sscanf(cmd, "cutouts %15s", word) == 1) {
|
||||
if (!std::strcmp(word, "on")) g_cutouts = true;
|
||||
else if (!std::strcmp(word, "off")) g_cutouts = false;
|
||||
else if (std::strcmp(word, "state") != 0) return (void)std::snprintf(reply, size, "error cutouts on|off|state");
|
||||
std::snprintf(reply, size, "ok %s %s %.2f ms", g_cutouts ? "on" : "off",
|
||||
g_cutterState > 0 ? "ready" : g_cutterState < 0 ? "unavailable" : "idle", g_cutter.lastMs());
|
||||
} else if (std::strncmp(cmd, "state", 5) == 0) {
|
||||
std::snprintf(reply, size, "ok %s %d %.0f %s %.0f %s %d %s", ModeName(), g_manual ? 1 : 0, g_wristAngle,
|
||||
g_gestureHand.c_str(), g_gestureAngle, LasersName(), g_gameRunning ? 1 : 0,
|
||||
|
||||
Reference in new issue
Block a user