From b7cbd9fe167222331079ab6b297fe04e8511d7ac Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:28:27 -0600 Subject: [PATCH] Cut tracked hands out of screens so you see them through it (work in progress) Where frame-hands tracks a hand between an eye and a screen, that eye sees the room through the screen. handcut.cpp draws the screen's buffer side by side (one half per eye) with the hands cut out, only while a hand is in front of it. The cutouts command turns it on or off. ft-handtest tries it on a test panel. Not yet tested in the headset. Co-Authored-By: Claude Opus 5.5 (1M context) --- screens/build.sh | 17 +- screens/handcut.cpp | 491 +++++++++++++++++++++++++++++++++++++++++++ screens/handcut.h | 112 ++++++++++ screens/handtest.cpp | 211 +++++++++++++++++++ screens/vr.cpp | 128 ++++++++++- 5 files changed, 950 insertions(+), 9 deletions(-) create mode 100644 screens/handcut.cpp create mode 100644 screens/handcut.h create mode 100644 screens/handtest.cpp diff --git a/screens/build.sh b/screens/build.sh index 26a6614..bd0f33e 100755 --- a/screens/build.sh +++ b/screens/build.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Build ft-screens in the dev container on the Frame (screens/build/ft-screens). +# Build ft-screens in the dev container on the Frame (screens/build/ft-screens), and +# ft-handtest, which tries the hand cutouts (handcut.cpp) on a test panel of its own. # compositor.c is the wlroots side (C; wlroots headers aren't C++), vr.cpp the OpenVR side. # vr.cpp needs OpenVR's IVRIPCResourceManagerClient (ImportDmabuf), which the header # shipped with SteamVR on the Frame predates, so the build uses the public header from @@ -12,8 +13,12 @@ openvr=v2.15.6 [ -f build/include/openvr-$openvr ] || { curl -fsSL "https://raw.githubusercontent.com/ValveSoftware/openvr/$openvr/headers/openvr.h" -o build/include/openvr.h && touch build/include/openvr-$openvr; } gcc -std=c11 -O2 -Wall -Wno-unused-parameter -c -o build/compositor.o compositor.c \ $(pkg-config --cflags wlroots-0.20 wayland-server xkbcommon libdrm pixman-1) -g++ -std=c++17 -O2 -Wall -Wno-missing-field-initializers -Ibuild/include -c -o build/vr.o vr.cpp -g++ -o build/ft-screens build/compositor.o build/vr.o \ - $(pkg-config --libs wlroots-0.20 wayland-server xkbcommon) \ - -L/opt/steamvr/bin/linuxarm64 -lopenvr_api -Wl,-rpath,/opt/steamvr/bin/linuxarm64 -echo "built build/ft-screens"' +cxx="g++ -std=c++17 -O2 -Wall -Wno-missing-field-initializers -Ibuild/include $(pkg-config --cflags egl glesv2 gbm libdrm)" +$cxx -c -o build/vr.o vr.cpp +$cxx -c -o build/handcut.o handcut.cpp +$cxx -c -o build/handtest.o handtest.cpp +vrlibs="$(pkg-config --libs egl glesv2 gbm) -L/opt/steamvr/bin/linuxarm64 -lopenvr_api -Wl,-rpath,/opt/steamvr/bin/linuxarm64" +g++ -o build/ft-screens build/compositor.o build/vr.o build/handcut.o \ + $(pkg-config --libs wlroots-0.20 wayland-server xkbcommon) $vrlibs +g++ -o build/ft-handtest build/handtest.o build/handcut.o $vrlibs +echo "built build/ft-screens build/ft-handtest"' diff --git a/screens/handcut.cpp b/screens/handcut.cpp new file mode 100644 index 0000000..20a61d0 --- /dev/null +++ b/screens/handcut.cpp @@ -0,0 +1,491 @@ +// Hand cutouts (see handcut.h). +#include "handcut.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace handcut { +namespace { + +// The hands file (frame-hands/include/fh_hands.h). +constexpr char kMagic[8] = {'F', 'H', 'H', 'A', 'N', 'D', 'S', '1'}; +constexpr size_t kHeader = 64, kHand = 272, kCapsule = 32, kMaxHands = 2, kMaxCapsules = 64; +constexpr size_t kFileSize = kHeader + kMaxHands * kHand + kMaxCapsules * kCapsule; +constexpr int64_t kStaleNs = 300'000'000; // hands older than this are gone +constexpr int64_t kHistoryNs = 1'000'000'000; + +int64_t MonoNs() { + timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return int64_t(ts.tv_sec) * 1'000'000'000 + ts.tv_nsec; +} + +void Apply(const Mat &m, const float p[3], float out[3]) { + for (int i = 0; i < 3; ++i) out[i] = m.m[i][0] * p[0] + m.m[i][1] * p[1] + m.m[i][2] * p[2] + m.m[i][3]; +} + +// Room -> panel-local: R^T (p - t). +void ToLocal(const Mat &m, const double p[3], double out[3]) { + const double d[3] = {p[0] - m.m[0][3], p[1] - m.m[1][3], p[2] - m.m[2][3]}; + for (int i = 0; i < 3; ++i) out[i] = m.m[0][i] * d[0] + m.m[1][i] * d[1] + m.m[2][i] * d[2]; +} + +} // namespace + +// ------------------------------------------------------------------------------- hands + +bool Hands::Read() { + if (!map_) { + const int64_t now = MonoNs(); + if (now - lastOpenTry_ < 1'000'000'000) return false; + lastOpenTry_ = now; + const char *run = std::getenv("XDG_RUNTIME_DIR"); + const std::string path = std::string(run ? run : "/run/user/" + std::to_string(getuid())) + "/frame-hands/hands"; + fd_ = open(path.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW); + if (fd_ < 0) return false; + struct stat st; + if (fstat(fd_, &st) < 0 || st.st_uid != getuid() || size_t(st.st_size) < kFileSize) { + close(fd_), fd_ = -1; + return false; + } + void *m = mmap(nullptr, kFileSize, PROT_READ, MAP_SHARED, fd_, 0); + if (m == MAP_FAILED) { + close(fd_), fd_ = -1; + return false; + } + map_ = m; + } + const auto *p = static_cast(map_); + auto u64 = [&](size_t off) { uint64_t v; std::memcpy(&v, const_cast(p) + off, 8); return v; }; + const uint64_t s1 = __atomic_load_n(reinterpret_cast(const_cast(p) + 16), __ATOMIC_ACQUIRE); + if ((s1 & 1) || s1 == seq_) return false; + uint8_t copy[kFileSize]; + std::memcpy(copy, const_cast(p), kFileSize); + __atomic_thread_fence(__ATOMIC_ACQUIRE); + if (u64(16) != s1 || std::memcmp(copy, kMagic, 8) != 0) return false; + seq_ = s1; + uint64_t capture, publish; + uint32_t ncaps; + std::memcpy(&capture, copy + 24, 8); + std::memcpy(&publish, copy + 32, 8); + std::memcpy(&ncaps, copy + 44, 4); + captureNs_ = int64_t(capture), publishNs_ = int64_t(publish); + const Mat head = HeadAt(captureNs_); + world_.clear(); + for (uint32_t k = 0; k < std::min(ncaps, kMaxCapsules); ++k) { + float f[8]; + std::memcpy(f, copy + kHeader + kMaxHands * kHand + k * kCapsule, sizeof f); + bool ok = true; + for (float v : f) ok = ok && std::isfinite(v) && std::fabs(v) < 10; + if (!ok || f[6] <= 0 || f[7] <= 0) continue; + Capsule c; + Apply(head, f, c.a); + Apply(head, f + 3, c.b); + c.ra = f[6], c.rb = f[7]; + world_.push_back(c); + } + return true; +} + +Mat Hands::HeadAt(int64_t ns) const { + const Past *best = nullptr; + for (const Past &p : history_) + if (!best || std::llabs(p.ns - ns) < std::llabs(best->ns - ns)) best = &p; + return best ? best->head : Mat{}; +} + +bool Hands::Update(const Mat &head, int64_t nowNs) { + history_.push_back({nowNs, head}); + while (!history_.empty() && nowNs - history_.front().ns > kHistoryNs) history_.erase(history_.begin()); + Read(); + if (nowNs - publishNs_ > kStaleNs) world_.clear(); + return !world_.empty(); +} + +void EyePositions(const Mat &head, double out[2][3]) { + const vr::EVREye eyes[2] = {vr::Eye_Left, vr::Eye_Right}; + for (int e = 0; e < 2; ++e) { + const Mat t = vr::VRSystem()->GetEyeToHeadTransform(eyes[e]); + for (int i = 0; i < 3; ++i) + out[e][i] = head.m[i][0] * t.m[0][3] + head.m[i][1] * t.m[1][3] + head.m[i][2] * t.m[2][3] + head.m[i][3]; + } +} + +// ----------------------------------------------------------------------------- project + +namespace { + +// Where the line from eye e through point q (both panel-local) meets the panel, as texture +// pixels, and how much a size at q grows there. False if q isn't between the eye and it. +bool OnPanel(const Panel &p, const double e[3], const double q[3], double *x, double *y, double *grow) { + const double d[3] = {q[0] - e[0], q[1] - e[1], q[2] - e[2]}; + double s, u, v; + if (p.curve <= 0) { + if (e[2] <= q[2] || q[2] <= 0) return false; + s = e[2] / (e[2] - q[2]); + u = e[0] + s * d[0]; + v = e[1] + s * d[1]; + } else { + // OpenVR bends a curved panel into a cylinder around (0, *, r), toward its front. + const double r = p.curve, ez = e[2] - r; + const double A = d[0] * d[0] + d[2] * d[2], B = 2 * (e[0] * d[0] + ez * d[2]), C = e[0] * e[0] + ez * ez - r * r; + const double disc = B * B - 4 * A * C; + if (A < 1e-12 || disc < 0) return false; + s = (-B + std::sqrt(disc)) / (2 * A); // the far side: the panel, seen from inside + const double px = e[0] + s * d[0], pz = e[2] + s * d[2]; + if (pz > r) return false; + u = r * std::atan2(px, r - pz); + v = e[1] + s * d[1]; + } + if (s <= 1) return false; // the hand is behind the panel + *x = (u / p.width + 0.5) * p.pxWidth; + *y = (0.5 - v / p.height) * p.pxHeight; + *grow = s; + return true; +} + +} // namespace + +bool Project(const Panel &p, const std::vector &caps, const double eyes[2][3], std::vector out[2]) { + const double pxPerM = p.pxWidth / p.width; + bool any = false; + for (int e = 0; e < 2; ++e) { + out[e].clear(); + double eye[3]; + ToLocal(p.pose, eyes[e], eye); + if (eye[2] <= 0.01) continue; // behind the panel + for (const Capsule &c : caps) { + double a[3], b[3]; + const double wa[3] = {c.a[0], c.a[1], c.a[2]}, wb[3] = {c.b[0], c.b[1], c.b[2]}; + ToLocal(p.pose, wa, a); + ToLocal(p.pose, wb, b); + // a hand pushed through the panel: keep the part in front + const double eps = 0.002; + if (a[2] < eps && b[2] < eps) continue; + if (a[2] < eps || b[2] < eps) { + double *in = a[2] < eps ? b : a, *out3 = a[2] < eps ? a : b; + const double t = (in[2] - eps) / (in[2] - out3[2]); + for (int i = 0; i < 3; ++i) out3[i] = in[i] + t * (out3[i] - in[i]); + } + double ax, ay, ga, bx, by, gb; + if (!OnPanel(p, eye, a, &ax, &ay, &ga) || !OnPanel(p, eye, b, &bx, &by, &gb)) continue; + const float ra = float(c.ra * ga * pxPerM), rb = float(c.rb * gb * pxPerM); + const float r = std::max(ra, rb); + if (std::max(ax, bx) + r < 0 || std::min(ax, bx) - r > p.pxWidth || std::max(ay, by) + r < 0 || + std::min(ay, by) - r > p.pxHeight) + continue; + out[e].push_back({float(ax), float(ay), float(bx), float(by), ra, rb}); + any = true; + } + } + return any; +} + +// ---------------------------------------------------------------------------- renderer + +namespace { + +PFNEGLGETPLATFORMDISPLAYEXTPROC pGetPlatformDisplay; +PFNEGLCREATEIMAGEKHRPROC pCreateImage; +PFNEGLDESTROYIMAGEKHRPROC pDestroyImage; +PFNGLEGLIMAGETARGETTEXTURE2DOESPROC pImageTargetTexture; +PFNGLEGLIMAGETARGETRENDERBUFFERSTORAGEOESPROC pImageTargetRenderbuffer; + +const char *kVertex = R"( +attribute vec2 pos; // the unit square +uniform vec4 rect; // where it goes, in pixels of the eye's half: x0 y0 x1 y1 +uniform vec2 size; // the half's size in pixels +varying vec2 px; +varying vec2 uv; +void main() { + px = mix(rect.xy, rect.zw, pos); + uv = px / size; + gl_Position = vec4(uv * 2.0 - 1.0, 0.0, 1.0); +})"; + +// The client's pixels, opaque (its alpha is ignored, as IgnoreTextureAlpha did). +const char *kCopy = R"( +#extension GL_OES_EGL_image_external : require +precision mediump float; +uniform samplerExternalOES tex; +varying vec2 uv; +void main() { gl_FragColor = vec4(texture2D(tex, uv).rgb, 1.0); })"; + +// Coverage of one tapered capsule; blended to take that much alpha away. +const char *kCut = R"( +precision highp float; +uniform vec2 a, b, r; +uniform float feather; +varying vec2 px; +void main() { + vec2 ab = b - a; + float t = clamp(dot(px - a, ab) / max(dot(ab, ab), 1e-6), 0.0, 1.0); + float d = length(px - (a + t * ab)); + float rad = mix(r.x, r.y, t); + gl_FragColor = vec4(0.0, 0.0, 0.0, 1.0 - smoothstep(rad - feather, rad + feather, d)); +})"; + +unsigned Shader(GLenum type, const char *src) { + const GLuint s = glCreateShader(type); + glShaderSource(s, 1, &src, nullptr); + glCompileShader(s); + GLint ok = 0; + glGetShaderiv(s, GL_COMPILE_STATUS, &ok); + if (!ok) { + char log[1024] = ""; + glGetShaderInfoLog(s, sizeof log, nullptr, log); + std::fprintf(stderr, "handcut: shader: %s\n", log); + } + return s; +} + +unsigned Program(const char *fs) { + const GLuint p = glCreateProgram(); + glAttachShader(p, Shader(GL_VERTEX_SHADER, kVertex)); + glAttachShader(p, Shader(GL_FRAGMENT_SHADER, fs)); + glBindAttribLocation(p, 0, "pos"); + glLinkProgram(p); + GLint ok = 0; + glGetProgramiv(p, GL_LINK_STATUS, &ok); + if (!ok) { + char log[1024] = ""; + glGetProgramInfoLog(p, sizeof log, nullptr, log); + std::fprintf(stderr, "handcut: program: %s\n", log); + return 0; + } + return p; +} + +EGLImageKHR ImageFor(EGLDisplay dpy, const ft_dmabuf &b) { + static const EGLint fd[4] = {EGL_DMA_BUF_PLANE0_FD_EXT, EGL_DMA_BUF_PLANE1_FD_EXT, EGL_DMA_BUF_PLANE2_FD_EXT, + EGL_DMA_BUF_PLANE3_FD_EXT}; + static const EGLint off[4] = {EGL_DMA_BUF_PLANE0_OFFSET_EXT, EGL_DMA_BUF_PLANE1_OFFSET_EXT, + EGL_DMA_BUF_PLANE2_OFFSET_EXT, EGL_DMA_BUF_PLANE3_OFFSET_EXT}; + static const EGLint pitch[4] = {EGL_DMA_BUF_PLANE0_PITCH_EXT, EGL_DMA_BUF_PLANE1_PITCH_EXT, + EGL_DMA_BUF_PLANE2_PITCH_EXT, EGL_DMA_BUF_PLANE3_PITCH_EXT}; + static const EGLint lo[4] = {EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT, EGL_DMA_BUF_PLANE1_MODIFIER_LO_EXT, + EGL_DMA_BUF_PLANE2_MODIFIER_LO_EXT, EGL_DMA_BUF_PLANE3_MODIFIER_LO_EXT}; + static const EGLint hi[4] = {EGL_DMA_BUF_PLANE0_MODIFIER_HI_EXT, EGL_DMA_BUF_PLANE1_MODIFIER_HI_EXT, + EGL_DMA_BUF_PLANE2_MODIFIER_HI_EXT, EGL_DMA_BUF_PLANE3_MODIFIER_HI_EXT}; + EGLint a[64]; + int n = 0; + a[n++] = EGL_WIDTH, a[n++] = b.width, a[n++] = EGL_HEIGHT, a[n++] = b.height; + a[n++] = EGL_LINUX_DRM_FOURCC_EXT, a[n++] = EGLint(b.format); + for (int i = 0; i < b.n_planes && i < 4; ++i) { + a[n++] = fd[i], a[n++] = b.fd[i], a[n++] = off[i], a[n++] = EGLint(b.offset[i]); + a[n++] = pitch[i], a[n++] = EGLint(b.stride[i]); + if (b.modifier != DRM_FORMAT_MOD_INVALID) { + a[n++] = lo[i], a[n++] = EGLint(b.modifier & 0xffffffff); + a[n++] = hi[i], a[n++] = EGLint(b.modifier >> 32); + } + } + a[n++] = EGL_NONE; + return pCreateImage(dpy, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, a); +} + +} // namespace + +Renderer::~Renderer() { + for (auto &[k, r] : rings_) + for (Output &o : r.out) FreeOutput(o); + for (auto &[k, im] : imported_) { + glDeleteTextures(1, &im.tex); + pDestroyImage(EGLDisplay(dpy_), EGLImageKHR(im.image)); + } + if (ctx_) eglDestroyContext(EGLDisplay(dpy_), EGLContext(ctx_)); + if (dpy_) eglTerminate(EGLDisplay(dpy_)); + if (gbm_) gbm_device_destroy(static_cast(gbm_)); + if (drm_ >= 0) close(drm_); +} + +bool Renderer::Init(const std::vector &modifiers, std::function released) { + if (ready_) return true; + modifiers_ = modifiers; + released_ = std::move(released); + drm_ = open("/dev/dri/renderD128", O_RDWR | O_CLOEXEC); + if (drm_ < 0) return std::perror("handcut: /dev/dri/renderD128"), false; + gbm_ = gbm_create_device(drm_); + pGetPlatformDisplay = reinterpret_cast(eglGetProcAddress("eglGetPlatformDisplayEXT")); + pCreateImage = reinterpret_cast(eglGetProcAddress("eglCreateImageKHR")); + pDestroyImage = reinterpret_cast(eglGetProcAddress("eglDestroyImageKHR")); + pImageTargetTexture = reinterpret_cast(eglGetProcAddress("glEGLImageTargetTexture2DOES")); + pImageTargetRenderbuffer = reinterpret_cast( + eglGetProcAddress("glEGLImageTargetRenderbufferStorageOES")); + if (!gbm_ || !pGetPlatformDisplay || !pCreateImage || !pImageTargetTexture || !pImageTargetRenderbuffer) { + std::fprintf(stderr, "handcut: GBM or EGL extensions missing\n"); + return false; + } + EGLDisplay dpy = pGetPlatformDisplay(EGL_PLATFORM_GBM_KHR, gbm_, nullptr); + if (dpy == EGL_NO_DISPLAY || !eglInitialize(dpy, nullptr, nullptr)) return std::fprintf(stderr, "handcut: no EGL display\n"), false; + dpy_ = dpy; + eglBindAPI(EGL_OPENGL_ES_API); + const EGLint attrs[] = {EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE}; + EGLContext ctx = eglCreateContext(dpy, EGL_NO_CONFIG_KHR, EGL_NO_CONTEXT, attrs); + if (ctx == EGL_NO_CONTEXT || !eglMakeCurrent(dpy, EGL_NO_SURFACE, EGL_NO_SURFACE, ctx)) + return std::fprintf(stderr, "handcut: no surfaceless GLES context\n"), false; + ctx_ = ctx; + copyProg_ = Program(kCopy); + cutProg_ = Program(kCut); + if (!copyProg_ || !cutProg_) return false; + const float quad[] = {0, 0, 1, 0, 0, 1, 1, 1}; + glGenBuffers(1, &vbo_); + glBindBuffer(GL_ARRAY_BUFFER, vbo_); + glBufferData(GL_ARRAY_BUFFER, sizeof quad, quad, GL_STATIC_DRAW); + ready_ = true; + return true; +} + +unsigned Renderer::Texture(const void *key, const ft_dmabuf &src) { + auto it = imported_.find(key); + if (it != imported_.end()) return it->second.tex; + EGLImageKHR image = ImageFor(EGLDisplay(dpy_), src); + if (image == EGL_NO_IMAGE_KHR) { + std::fprintf(stderr, "handcut: can't import a %dx%d client buffer (format 0x%x modifier 0x%llx)\n", src.width, + src.height, src.format, (unsigned long long)src.modifier); + return 0; + } + GLuint tex; + glGenTextures(1, &tex); + glBindTexture(GL_TEXTURE_EXTERNAL_OES, tex); + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + pImageTargetTexture(GL_TEXTURE_EXTERNAL_OES, image); + imported_[key] = {image, tex}; + return tex; +} + +void Renderer::Forget(const void *key) { + auto it = imported_.find(key); + if (it == imported_.end()) return; + glDeleteTextures(1, &it->second.tex); + pDestroyImage(EGLDisplay(dpy_), EGLImageKHR(it->second.image)); + imported_.erase(it); +} + +bool Renderer::MakeOutput(Output &o, int w, int h) { + auto *gbm = static_cast(gbm_); + std::vector mods; + for (uint64_t m : modifiers_) + if (m != DRM_FORMAT_MOD_INVALID) mods.push_back(m); + gbm_bo *bo = mods.empty() ? gbm_bo_create(gbm, w, h, GBM_FORMAT_ABGR8888, GBM_BO_USE_RENDERING | GBM_BO_USE_LINEAR) + : gbm_bo_create_with_modifiers2(gbm, w, h, GBM_FORMAT_ABGR8888, mods.data(), + unsigned(mods.size()), GBM_BO_USE_RENDERING); + if (!bo) return std::fprintf(stderr, "handcut: can't allocate a %dx%d output\n", w, h), false; + o.bo = bo; + o.buf = {}; + o.buf.width = w, o.buf.height = h; + o.buf.format = DRM_FORMAT_ABGR8888; + o.buf.modifier = mods.empty() ? DRM_FORMAT_MOD_LINEAR : gbm_bo_get_modifier(bo); + o.buf.n_planes = gbm_bo_get_plane_count(bo); + for (int i = 0; i < o.buf.n_planes && i < 4; ++i) { + o.buf.fd[i] = gbm_bo_get_fd_for_plane(bo, i); + o.buf.offset[i] = gbm_bo_get_offset(bo, i); + o.buf.stride[i] = gbm_bo_get_stride_for_plane(bo, i); + } + EGLImageKHR image = ImageFor(EGLDisplay(dpy_), o.buf); + if (image == EGL_NO_IMAGE_KHR) return std::fprintf(stderr, "handcut: can't render to the output\n"), FreeOutput(o), false; + o.image = image; + glGenRenderbuffers(1, &o.rb); + glBindRenderbuffer(GL_RENDERBUFFER, o.rb); + pImageTargetRenderbuffer(GL_RENDERBUFFER, image); + glGenFramebuffers(1, &o.fbo); + glBindFramebuffer(GL_FRAMEBUFFER, o.fbo); + glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, o.rb); + if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) + return std::fprintf(stderr, "handcut: output framebuffer incomplete\n"), FreeOutput(o), false; + return true; +} + +void Renderer::FreeOutput(Output &o) { + if (o.bo && released_) released_(&o); + if (o.fbo) glDeleteFramebuffers(1, &o.fbo); + if (o.rb) glDeleteRenderbuffers(1, &o.rb); + if (o.image) pDestroyImage(EGLDisplay(dpy_), EGLImageKHR(o.image)); + for (int i = 0; i < o.buf.n_planes && i < 4; ++i) + if (o.buf.fd[i] >= 0) close(o.buf.fd[i]); + if (o.bo) gbm_bo_destroy(static_cast(o.bo)); + o = Output{}; +} + +void Renderer::DropPanel(int panel) { + auto it = rings_.find(panel); + if (it == rings_.end()) return; + for (Output &o : it->second.out) FreeOutput(o); + rings_.erase(it); +} + +const Output *Renderer::Composite(int panel, const void *key, const ft_dmabuf &src, const std::vector eyes[2]) { + if (!ready_) return nullptr; + const auto t0 = std::chrono::steady_clock::now(); + const int w = src.width, h = src.height; + Ring &ring = rings_[panel]; + if (ring.w != w || ring.h != h) { + for (Output &old : ring.out) FreeOutput(old); + ring.w = w, ring.h = h, ring.next = 0; + } + Output &o = ring.out[ring.next]; + if (!o.bo && !MakeOutput(o, 2 * w, h)) return nullptr; + const GLuint tex = Texture(key, src); + if (!tex) return nullptr; + ring.next = (ring.next + 1) % 3; + + glBindFramebuffer(GL_FRAMEBUFFER, o.fbo); + glBindBuffer(GL_ARRAY_BUFFER, vbo_); + glEnableVertexAttribArray(0); + glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 0, nullptr); + for (int e = 0; e < 2; ++e) { + glViewport(e * w, 0, w, h); + glDisable(GL_BLEND); + glUseProgram(copyProg_); + glActiveTexture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_EXTERNAL_OES, tex); + glUniform1i(glGetUniformLocation(copyProg_, "tex"), 0); + glUniform4f(glGetUniformLocation(copyProg_, "rect"), 0, 0, float(w), float(h)); + glUniform2f(glGetUniformLocation(copyProg_, "size"), float(w), float(h)); + glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + // take alpha away where the hand is; the colour stays (straight alpha) + glEnable(GL_BLEND); + glBlendFuncSeparate(GL_ZERO, GL_ONE, GL_ZERO, GL_ONE_MINUS_SRC_ALPHA); + glUseProgram(cutProg_); + glUniform2f(glGetUniformLocation(cutProg_, "size"), float(w), float(h)); + const GLint uRect = glGetUniformLocation(cutProg_, "rect"), uA = glGetUniformLocation(cutProg_, "a"), + uB = glGetUniformLocation(cutProg_, "b"), uR = glGetUniformLocation(cutProg_, "r"), + uF = glGetUniformLocation(cutProg_, "feather"); + for (const Capsule2D &c : eyes[e]) { + const float feather = std::max(1.5f, 0.15f * std::min(c.ra, c.rb)); + const float r = std::max(c.ra, c.rb) + feather; + glUniform4f(uRect, std::min(c.ax, c.bx) - r, std::min(c.ay, c.by) - r, std::max(c.ax, c.bx) + r, + std::max(c.ay, c.by) + r); + glUniform2f(uA, c.ax, c.ay); + glUniform2f(uB, c.bx, c.by); + glUniform2f(uR, c.ra, c.rb); + glUniform1f(uF, feather); + glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + } + } + glDisable(GL_BLEND); + // SteamVR reads the buffer from another process and GPU queue; make sure it's done. + glFinish(); + lastMs_ = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + return &o; +} + +} // namespace handcut diff --git a/screens/handcut.h b/screens/handcut.h new file mode 100644 index 0000000..3defcf2 --- /dev/null +++ b/screens/handcut.h @@ -0,0 +1,112 @@ +// Hand cutouts: where a tracked hand is between an eye and a screen, that eye sees the +// room (Room View) through the screen instead of the screen drawn over the hand. +// +// frame-hands' tracker (a separate project, ~/Desktop/Projects/frame-hands) publishes +// the hands it sees with the headset's cameras to $XDG_RUNTIME_DIR/frame-hands/hands: +// capsules (finger bones, palm, forearm) in the head frame at capture time. Hands turns +// them into the room with the head pose at that time. Project() finds where each eye +// sees them on a panel, and Renderer draws the panel's client buffer into a side-by-side +// buffer (left eye | right eye) with those spots transparent. A panel shows that buffer, +// with the overlay's SideBySide_Parallel flag, only while a hand is in front of it. +#pragma once + +#include "vr.h" + +#include + +#include +#include +#include +#include + +namespace handcut { + +using Mat = vr::HmdMatrix34_t; + +struct Capsule { // in the room (standing universe), metres + float a[3], b[3]; + float ra, rb; +}; + +struct Capsule2D { // on a panel's texture, pixels from the top left + float ax, ay, bx, by; + float ra, rb; +}; + +// A panel in the room: pose of its centre (+x right, +y up, +z out of its front), +// size in metres, cylinder radius (0: flat), and its texture size in pixels. +struct Panel { + Mat pose; + double width, height, curve; + int pxWidth, pxHeight; +}; + +class Hands { +public: + // Once per tick: the head pose now, CLOCK_MONOTONIC ns. Re-reads the hands file when + // it changed. Returns true while fresh hands are known. + bool Update(const Mat &head, int64_t nowNs); + const std::vector &capsules() const { return world_; } + +private: + bool Read(); + Mat HeadAt(int64_t ns) const; + struct Past { int64_t ns; Mat head; }; + std::vector history_; // the last second of head poses + std::vector world_; + int fd_ = -1; + const void *map_ = nullptr; + uint64_t seq_ = 0; + int64_t captureNs_ = 0, publishNs_ = 0, lastOpenTry_ = 0; +}; + +// Where each eye sees the capsules on the panel, for those in front of it. Eyes are +// positions in the room. False if no capsule reaches the panel for either eye. +bool Project(const Panel &p, const std::vector &caps, const double eyes[2][3], + std::vector out[2]); + +// The eye positions in the room for a head pose. +void EyePositions(const Mat &head, double out[2][3]); + +// An output buffer: the dmabuf SteamVR imports (see vr.cpp), stable while it exists. +struct Output { + ft_dmabuf buf{}; + void *bo = nullptr; + unsigned fbo = 0, rb = 0; + void *image = nullptr; +}; + +class Renderer { +public: + ~Renderer(); + // modifiers: what SteamVR takes for DRM_FORMAT_ABGR8888, the outputs' format. + // released: an output is about to be freed (drop its SteamVR import). + bool Init(const std::vector &modifiers, std::function released); + // Draw client buffer `src` (identified by `key`) into the next output buffer of + // panel `panel`, both eyes, cutting out `eyes`. Returns that buffer, or null. + const Output *Composite(int panel, const void *key, const ft_dmabuf &src, const std::vector eyes[2]); + // A client buffer is going away. + void Forget(const void *key); + // A panel is gone: drop its outputs. + void DropPanel(int panel); + // How long the last Composite took, ms (it waits for the GPU). + double lastMs() const { return lastMs_; } + +private: + unsigned Texture(const void *key, const ft_dmabuf &src); + bool MakeOutput(Output &o, int w, int h); + void FreeOutput(Output &o); + bool ready_ = false; + int drm_ = -1; + void *gbm_ = nullptr, *dpy_ = nullptr, *ctx_ = nullptr; + unsigned copyProg_ = 0, cutProg_ = 0, vbo_ = 0; + std::vector modifiers_; + std::function released_; + struct Imported { void *image; unsigned tex; }; + std::map imported_; + struct Ring { Output out[3]; int next = 0; int w = 0, h = 0; }; + std::map rings_; + double lastMs_ = 0; +}; + +} // namespace handcut diff --git a/screens/handtest.cpp b/screens/handtest.cpp new file mode 100644 index 0000000..0b4692d --- /dev/null +++ b/screens/handtest.cpp @@ -0,0 +1,211 @@ +// ft-handtest: try the hand cutouts without restarting the desktop. Shows a test panel +// (a light grid) in front of you as its own overlay; where frame-hands tracks your hands +// in front of it, each eye sees through it, like ft-screens' screens with cutouts. +// +// ft-handtest [--distance m] [--width m] [--seconds s] +// +// Needs frame-hands' tracker running (it publishes $XDG_RUNTIME_DIR/frame-hands/hands). +// Build: screens/build.sh (build/ft-handtest), run in the dev container. +#include "handcut.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +volatile sig_atomic_t g_stop = 0; +void Stop(int) { g_stop = 1; } + +int64_t MonoNs() { + timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return int64_t(ts.tv_sec) * 1'000'000'000 + ts.tv_nsec; +} + +vr::SharedTextureHandle_t Import(const ft_dmabuf &b) { + vr::DmabufAttributes_t a{}; + a.unWidth = uint32_t(b.width); + a.unHeight = uint32_t(b.height); + a.unDepth = a.unMipLevels = a.unArrayLayers = a.unSampleCount = 1; + a.unFormat = b.format; + a.ulModifier = b.modifier; + a.unPlaneCount = uint32_t(b.n_planes); + for (int i = 0; i < b.n_planes && i < int(vr::MaxDmabufPlaneCount); ++i) { + a.plane[i].unOffset = b.offset[i]; + a.plane[i].unStride = b.stride[i]; + a.plane[i].nFd = b.fd[i]; + } + vr::SharedTextureHandle_t h = 0; + if (!vr::VRIPCResourceManager()->ImportDmabuf(vr::VRApplication_Overlay, &a, &h)) return 0; + return h; +} + +// The stand-in "client" buffer: a light grey grid, linear so the CPU can draw it. +bool TestPattern(int drm, int w, int h, gbm_bo **out, ft_dmabuf *b) { + gbm_device *gbm = gbm_create_device(drm); + gbm_bo *bo = gbm_bo_create(gbm, w, h, GBM_FORMAT_ABGR8888, GBM_BO_USE_LINEAR | GBM_BO_USE_RENDERING); + if (!bo) return false; + uint32_t stride = 0; + void *data = nullptr; + auto *px = static_cast(gbm_bo_map(bo, 0, 0, w, h, GBM_BO_TRANSFER_WRITE, &stride, &data)); + if (!px) return false; + for (int y = 0; y < h; ++y) + for (int x = 0; x < w; ++x) { + uint8_t *p = px + size_t(y) * stride + size_t(x) * 4; + const bool line = x % 80 < 3 || y % 80 < 3; + const uint8_t g = line ? 90 : uint8_t(200 + 30 * x / w); + p[0] = g, p[1] = g, p[2] = uint8_t(line ? 160 : g), p[3] = 255; + } + gbm_bo_unmap(bo, data); + *b = {}; + b->width = w, b->height = h, b->format = DRM_FORMAT_ABGR8888, b->modifier = DRM_FORMAT_MOD_LINEAR; + b->n_planes = 1; + b->fd[0] = gbm_bo_get_fd(bo); + b->stride[0] = gbm_bo_get_stride(bo); + *out = bo; + return true; +} + +} // namespace + +int main(int argc, char **argv) { + double distance = 0.8, width = 1.0, seconds = 0; + for (int i = 1; i + 1 < argc; i += 2) { + if (!std::strcmp(argv[i], "--distance")) distance = std::atof(argv[i + 1]); + else if (!std::strcmp(argv[i], "--width")) width = std::atof(argv[i + 1]); + else if (!std::strcmp(argv[i], "--seconds")) seconds = std::atof(argv[i + 1]); + } + signal(SIGINT, Stop); + signal(SIGTERM, Stop); + + vr::EVRInitError err = vr::VRInitError_None; + vr::VR_Init(&err, vr::VRApplication_Overlay); + if (err != vr::VRInitError_None) { + std::fprintf(stderr, "openvr: %s\n", vr::VR_GetVRInitErrorAsEnglishDescription(err)); + return 1; + } + uint64_t mods[64]; + uint32_t nmods = 64; + if (!vr::VRIPCResourceManager()->GetDmabufModifiers(vr::VRApplication_Overlay, DRM_FORMAT_ABGR8888, &nmods, mods)) + nmods = 0; + std::printf("SteamVR takes %u modifiers for ABGR8888\n", nmods); + + std::map imports; + handcut::Renderer renderer; + if (!renderer.Init(std::vector(mods, mods + nmods), [&](const handcut::Output *o) { + auto it = imports.find(o); + if (it != imports.end()) vr::VRIPCResourceManager()->UnrefResource(it->second), imports.erase(it); + })) + return 1; + + const int W = 1600, H = 900; + const int drm = open("/dev/dri/renderD128", O_RDWR | O_CLOEXEC); + gbm_bo *bo = nullptr; + ft_dmabuf client; + if (drm < 0 || !TestPattern(drm, W, H, &bo, &client)) return std::fprintf(stderr, "can't make the test pattern\n"), 1; + const vr::SharedTextureHandle_t plain = Import(client); + if (!plain) return std::fprintf(stderr, "SteamVR can't import the test pattern\n"), 1; + + vr::VROverlayHandle_t ov; + if (vr::VROverlay()->CreateOverlay("frametop.handtest", "Hand cutout test", &ov) != vr::VROverlayError_None) + return std::fprintf(stderr, "can't create the overlay (already running?)\n"), 1; + vr::VROverlay()->SetOverlayWidthInMeters(ov, float(width)); + + // In front of the head as it is now, level, facing it. + vr::TrackedDevicePose_t poses[vr::k_unMaxTrackedDeviceCount]; + vr::VRSystem()->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, poses, vr::k_unMaxTrackedDeviceCount); + const auto &hm = poses[vr::k_unTrackedDeviceIndex_Hmd].mDeviceToAbsoluteTracking; + const double yaw = std::atan2(hm.m[0][2], hm.m[2][2]); + handcut::Panel panel{}; + auto &P = panel.pose; + P.m[0][0] = float(std::cos(yaw)), P.m[0][2] = float(std::sin(yaw)); + P.m[1][1] = 1; + P.m[2][0] = float(-std::sin(yaw)), P.m[2][2] = float(std::cos(yaw)); + P.m[0][3] = float(hm.m[0][3] - std::sin(yaw) * distance); + P.m[1][3] = float(hm.m[1][3] - 0.15); + P.m[2][3] = float(hm.m[2][3] - std::cos(yaw) * distance); + panel.width = width, panel.height = width * H / W, panel.curve = 0, panel.pxWidth = W, panel.pxHeight = H; + vr::VROverlay()->SetOverlayTransformAbsolute(ov, vr::TrackingUniverseStanding, &P); + vr::VROverlay()->SetOverlayFlag(ov, vr::VROverlayFlags_IgnoreTextureAlpha, true); + vr::SharedTextureHandle_t shown = plain; + vr::Texture_t tex = {&shown, vr::TextureType_SharedTextureHandle, vr::ColorSpace_Gamma}; + vr::VROverlay()->SetOverlayTexture(ov, &tex); + vr::VROverlay()->ShowOverlay(ov); + std::printf("test panel %.2f m wide, %.2f m ahead; Ctrl+C to stop\n", width, distance); + + handcut::Hands hands; + bool cutting = false; + const int64_t start = MonoNs(); + int64_t lastReport = start; + int frames = 0, cutFrames = 0; + double ms = 0, worst = 0; + size_t caps2d = 0; + while (!g_stop && (seconds <= 0 || (MonoNs() - start) / 1e9 < seconds)) { + const auto tick = std::chrono::steady_clock::now(); + vr::VRSystem()->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, poses, vr::k_unMaxTrackedDeviceCount); + const auto &head = poses[vr::k_unTrackedDeviceIndex_Hmd].mDeviceToAbsoluteTracking; + const int64_t now = MonoNs(); + std::vector eyes2d[2]; + bool cut = false; + if (hands.Update(head, now)) { + double eyes[2][3]; + handcut::EyePositions(head, eyes); + cut = handcut::Project(panel, hands.capsules(), eyes, eyes2d); + } + const handcut::Output *out = cut ? renderer.Composite(0, bo, client, eyes2d) : nullptr; + if (out) { + auto it = imports.find(out); + if (it == imports.end()) { + const vr::SharedTextureHandle_t h = Import(out->buf); + if (!h) std::fprintf(stderr, "SteamVR can't import the output buffer\n"); + it = imports.emplace(out, h).first; + } + if (it->second) { + if (!cutting) { + vr::VROverlay()->SetOverlayFlag(ov, vr::VROverlayFlags_IgnoreTextureAlpha, false); + vr::VROverlay()->SetOverlayFlag(ov, vr::VROverlayFlags_SideBySide_Parallel, true); + cutting = true; + } + shown = it->second; + vr::VROverlay()->SetOverlayTexture(ov, &tex); + ms += renderer.lastMs(), worst = std::max(worst, renderer.lastMs()); + ++cutFrames; + caps2d += eyes2d[0].size() + eyes2d[1].size(); + } + } else if (cutting) { + vr::VROverlay()->SetOverlayFlag(ov, vr::VROverlayFlags_SideBySide_Parallel, false); + vr::VROverlay()->SetOverlayFlag(ov, vr::VROverlayFlags_IgnoreTextureAlpha, true); + shown = plain; + vr::VROverlay()->SetOverlayTexture(ov, &tex); + cutting = false; + } + ++frames; + if (now - lastReport > 2'000'000'000) { + std::printf("%.0f s: %d ticks, %d with a cutout (%.1f capsules per eye), composite %.2f ms avg %.2f ms worst, " + "%zu hand capsules known\n", + (now - start) / 1e9, frames, cutFrames, cutFrames ? caps2d / 2.0 / cutFrames : 0.0, + cutFrames ? ms / cutFrames : 0.0, worst, hands.capsules().size()); + std::fflush(stdout); + lastReport = now, frames = cutFrames = 0, ms = worst = 0, caps2d = 0; + } + std::this_thread::sleep_until(tick + std::chrono::microseconds(11111)); + } + vr::VROverlay()->DestroyOverlay(ov); + for (auto &[k, h] : imports) + if (h) vr::VRIPCResourceManager()->UnrefResource(h); + imports.clear(); + vr::VRIPCResourceManager()->UnrefResource(plain); + vr::VR_Shutdown(); + return 0; +} diff --git a/screens/vr.cpp b/screens/vr.cpp index 264bea0..7592ba3 100644 --- a/screens/vr.cpp +++ b/screens/vr.cpp @@ -40,10 +40,17 @@ // own; also for flatscreen games, which aren't scene apps). // - during a VR game the screens hide unless the dashboard is open (g_inGames, default), // or stay visible over it; the hotkey still shows them. +// - hand cutouts (handcut.cpp): where frame-hands tracks a hand between an eye and a +// screen, that eye sees through the screen (to Room View). Only then is the screen +// drawn by us, into a side-by-side buffer (one half per eye); otherwise its client +// buffer is shown as is. // OpenVR has no overlay-relative transforms here (openvr v2.15.6), so the bar, button, // and handle are placed whenever their screen moves. #include "vr.h" +#include "handcut.h" + +#include #include #include @@ -233,6 +240,10 @@ struct Screen { vr::TrackedDeviceIndex_t pinTarget = kNone; // moving: rides on this controller when let go vr::TrackedDeviceIndex_t onWrist = kNone; // moving: the laser is in this controller's ring bool barLit = false; + const void *key = nullptr; // the client buffer on it now, and its dmabuf (for cutouts) + ft_dmabuf buf{}; + vr::SharedTextureHandle_t plain = 0; // that buffer's SteamVR import + bool cutting = false; // showing a cutout buffer (side by side) instead double chrome = 0.3; // the bar's width; the other controls follow it (ChromeSize) double grip = 0.04; // the corner tab's and the round buttons' size double heightMetres() const { return width > 0 ? metres * height / width : metres * 9 / 16; } @@ -242,6 +253,13 @@ struct Screen { std::map g_screens; std::map g_imports; +// Hand cutouts (see the top and handcut.h). +bool g_cutouts = true; // the cutouts command turns them off +handcut::Hands g_hands; +handcut::Renderer g_cutter; +int g_cutterState = 0; // 0 not tried, 1 ready, -1 unavailable +std::map g_cutImports; + // Visibility (see the top). g_manual is the hide/show switch: in the always mode it hides // the screens, in the others it shows them anyway. Mode g_mode = Mode::Always; @@ -1038,6 +1056,92 @@ const char *ModeName() { } } +// ---------------------------------------------------------------- hand cutouts + +void SetScreenTexture(const Screen &s, vr::SharedTextureHandle_t handle) { + vr::Texture_t tex = {&handle, vr::TextureType_SharedTextureHandle, vr::ColorSpace_Gamma}; + vr::VROverlay()->SetOverlayTexture(s.overlay, &tex); +} + +// The cutout buffers' renderer, set up the first time a hand is in front of a screen. +bool CutterReady() { + if (g_cutterState) return g_cutterState > 0; + uint64_t mods[64]; + const int n = ft_vr_modifiers(DRM_FORMAT_ABGR8888, mods, 64); + const bool ok = g_cutter.Init(std::vector(mods, mods + n), [](const handcut::Output *o) { + auto it = g_cutImports.find(o); + if (it == g_cutImports.end()) return; + vr::VRIPCResourceManager()->UnrefResource(it->second); + g_cutImports.erase(it); + }); + g_cutterState = ok ? 1 : -1; + std::printf(ok ? "hand cutouts ready\n" : "hand cutouts unavailable (see above)\n"); + return ok; +} + +vr::SharedTextureHandle_t ImportCutout(const handcut::Output *o) { + auto it = g_cutImports.find(o); + if (it != g_cutImports.end()) return it->second; + vr::DmabufAttributes_t a{}; + a.unWidth = uint32_t(o->buf.width); + a.unHeight = uint32_t(o->buf.height); + a.unDepth = a.unMipLevels = a.unArrayLayers = a.unSampleCount = 1; + a.unFormat = o->buf.format; + a.ulModifier = o->buf.modifier; + a.unPlaneCount = uint32_t(o->buf.n_planes); + for (int i = 0; i < o->buf.n_planes && i < int(vr::MaxDmabufPlaneCount); ++i) { + a.plane[i].unOffset = o->buf.offset[i]; + a.plane[i].unStride = o->buf.stride[i]; + a.plane[i].nFd = o->buf.fd[i]; + } + vr::SharedTextureHandle_t h = 0; + if (!vr::VRIPCResourceManager()->ImportDmabuf(vr::VRApplication_Overlay, &a, &h)) { + std::fprintf(stderr, "openvr: ImportDmabuf failed for a cutout buffer\n"); + h = 0; + } + g_cutImports.emplace(o, h); + return h; +} + +void StopCutting(Screen &s) { + if (!s.cutting) return; + vr::VROverlay()->SetOverlayFlag(s.overlay, vr::VROverlayFlags_SideBySide_Parallel, false); + vr::VROverlay()->SetOverlayFlag(s.overlay, vr::VROverlayFlags_IgnoreTextureAlpha, true); + if (s.plain) SetScreenTexture(s, s.plain); + s.cutting = false; +} + +// Each tick: for each visible screen with a hand in front of it (for either eye), draw its +// client buffer with the hands cut out and show that; else show the client buffer. +void UpdateCutouts() { + Mat head; + const bool haveHead = DevicePose(vr::k_unTrackedDeviceIndex_Hmd, &head); + const bool hands = g_cutouts && haveHead && + g_hands.Update(head, std::chrono::duration_cast( + Clock::now().time_since_epoch()).count()); + double eyes[2][3]; + if (hands) handcut::EyePositions(head, eyes); + for (auto &[i, s] : g_screens) { + std::vector spots[2]; + Mat p; + bool cut = hands && s.visible && s.key && s.width > 0 && ScreenPose(s, &p) && + handcut::Project({p, s.metres, s.heightMetres(), s.curve, s.width, s.height}, g_hands.capsules(), + eyes, spots); + const handcut::Output *out = cut && CutterReady() ? g_cutter.Composite(i, s.key, s.buf, spots) : nullptr; + const vr::SharedTextureHandle_t h = out ? ImportCutout(out) : 0; + if (!h) { + StopCutting(s); + continue; + } + if (!s.cutting) { + vr::VROverlay()->SetOverlayFlag(s.overlay, vr::VROverlayFlags_IgnoreTextureAlpha, false); + vr::VROverlay()->SetOverlayFlag(s.overlay, vr::VROverlayFlags_SideBySide_Parallel, true); + s.cutting = true; + } + SetScreenTexture(s, h); + } +} + } // namespace extern "C" { @@ -1058,6 +1162,8 @@ bool ft_vr_init(void) { } void ft_vr_shutdown(void) { + if (g_cutterState == 1) + for (auto &[i, s] : g_screens) g_cutter.DropPanel(i); // drops their imports while SteamVR is up for (auto &[i, s] : g_screens) for (auto o : s.All()) vr::VROverlay()->DestroyOverlay(o); for (auto &[dev, g] : g_guides) @@ -1123,6 +1229,7 @@ void ft_vr_screen_create(int index, double metres, int count) { void ft_vr_screen_destroy(int index) { auto it = g_screens.find(index); if (it == g_screens.end()) return; + if (g_cutterState == 1) g_cutter.DropPanel(index); for (auto o : it->second.All()) vr::VROverlay()->DestroyOverlay(o); g_screens.erase(it); } @@ -1160,14 +1267,20 @@ bool ft_vr_screen_present(int index, const void *key, const struct ft_dmabuf *b) PlaceChrome(s); // the height changed std::printf("screen %d: %dx%d\n", index + 1, s.width, s.height); } - vr::SharedTextureHandle_t handle = it->second; - vr::Texture_t tex = {&handle, vr::TextureType_SharedTextureHandle, vr::ColorSpace_Gamma}; - vr::VROverlay()->SetOverlayTexture(s.overlay, &tex); + s.key = key, s.buf = *b, s.plain = it->second; + if (!s.cutting) { // otherwise the next tick draws the new buffer with the cutouts + vr::SharedTextureHandle_t handle = it->second; + vr::Texture_t tex = {&handle, vr::TextureType_SharedTextureHandle, vr::ColorSpace_Gamma}; + vr::VROverlay()->SetOverlayTexture(s.overlay, &tex); + } s.shown = key; // UpdateVisibility shows it on the next tick return true; } void ft_vr_forget(const void *key) { + if (g_cutterState == 1) g_cutter.Forget(key); + for (auto &[i, s] : g_screens) + if (s.key == key) s.key = nullptr; auto it = g_imports.find(key); if (it == g_imports.end()) return; vr::VRIPCResourceManager()->UnrefResource(it->second); @@ -1276,6 +1389,7 @@ void ft_vr_poll(void (*handle)(const struct ft_event *, void *), void *data) { UpdateLasers(); UpdateControls(); UpdateGuides(); + UpdateCutouts(); } // Control commands (datagrams on @ft_screens, replies to the sender): @@ -1300,6 +1414,8 @@ void ft_vr_poll(void (*handle)(const struct ft_event *, void *), void *data) { // (hide), or stays as it is (visible) // state -> "ok // " +// cutouts on|off|state hand cutouts (see handcut.h) -> "ok +// " // (size and key are handled in compositor.c.) Screens are // numbered from 1 here, like everywhere the user sees them. void ft_vr_command(const char *cmd, char *reply, int size) { @@ -1429,6 +1545,12 @@ void ft_vr_command(const char *cmd, char *reply, int size) { else return (void)std::snprintf(reply, size, "error modes: always outside_games dashboard"); UpdateLasers(); std::snprintf(reply, size, "ok %s", LasersName()); + } else if (std::sscanf(cmd, "cutouts %15s", word) == 1) { + if (!std::strcmp(word, "on")) g_cutouts = true; + else if (!std::strcmp(word, "off")) g_cutouts = false; + else if (std::strcmp(word, "state") != 0) return (void)std::snprintf(reply, size, "error cutouts on|off|state"); + std::snprintf(reply, size, "ok %s %s %.2f ms", g_cutouts ? "on" : "off", + g_cutterState > 0 ? "ready" : g_cutterState < 0 ? "unavailable" : "idle", g_cutter.lastMs()); } else if (std::strncmp(cmd, "state", 5) == 0) { std::snprintf(reply, size, "ok %s %d %.0f %s %.0f %s %d %s", ModeName(), g_manual ? 1 : 0, g_wristAngle, g_gestureHand.c_str(), g_gestureAngle, LasersName(), g_gameRunning ? 1 : 0,