From ce9e476ec7c4dade4bd999ed875235c6b6e78405 Mon Sep 17 00:00:00 2001 From: saphid <4596216+saphid@users.noreply.github.com> Date: Mon, 28 Sep 2026 22:33:02 +1000 Subject: [PATCH] feat(tracking): add local OpenXR OSC and BlueZ heart-rate tools --- README.md | 1 + docs/how-the-frame-works.md | 9 + docs/img/heart-rate-panel.png | Bin 0 -> 19410 bytes docs/testing.md | 7 + docs/tracking.md | 188 +++++++++++++++++ frame/tracking/gaze.c | 135 ++++++++++++ frame/tracking/tracking.py | 387 ++++++++++++++++++++++++++++++++++ scripts/tracking-on-frame.py | 58 +++++ tests/test_tracking.py | 188 +++++++++++++++++ 9 files changed, 973 insertions(+) create mode 100644 docs/img/heart-rate-panel.png create mode 100644 docs/tracking.md create mode 100644 frame/tracking/gaze.c create mode 100644 frame/tracking/tracking.py create mode 100644 scripts/tracking-on-frame.py create mode 100644 tests/test_tracking.py diff --git a/README.md b/README.md index 6dd32be..69edd2c 100644 --- a/README.md +++ b/README.md @@ -204,6 +204,7 @@ Frame's software fits together, all checked against a real headset and labelled | [SSH](docs/ssh.md) · [Streaming](docs/streaming.md) · [Files](docs/file-transfer.md) · [Panels](docs/panels.md) · [Tailscale](docs/tailscale.md) | Topic notes | | [Frame Control for iPhone](docs/iphone.md) | The iPhone and iPad app, how it runs the server on the Frame, pairing | | [Recovery and OS images](docs/recovery-and-images.md) | Where to download the Frame's OS, what's inside, testing without the headset | +| [Eye tracking and heart rate](docs/tracking.md) | Our OpenXR → OSC bridge, BlueZ heart-rate panel and optional local session log; SlimeVR feasibility notes | | [Testing](docs/testing.md) | Unit tests, end-to-end tests against a fake Frame in Docker, and the headset smoke test | | [Open questions](docs/open-questions.md) | What's still unchecked | diff --git a/docs/how-the-frame-works.md b/docs/how-the-frame-works.md index 5c392d1..385c00f 100644 --- a/docs/how-the-frame-works.md +++ b/docs/how-the-frame-works.md @@ -20,6 +20,15 @@ SteamVR (vrserver, vrcompositor, vrdashboard) ← renders the room + pane Lepton (Android 11, podman container "lepton-dev") ← its own panel, app 3056000 ``` +## Tracking additions (verified 2026-09-28) + +On SteamOS 0.4.1, build `20260925.6191901`, SteamVR exposes combined gaze +through `XR_EXT_eye_gaze_interaction` in a headless OpenXR 1.0 session. Our +reader obtained valid tracked samples and sent OSC to a configured loopback +receiver. BlueZ LE discovery works; GTK4/GI can render our heart-rate panel. +No BLE strap or SlimeVR trackers were attached. See [tracking](tracking.md) +for the evidence, privacy defaults and untested integration boundaries. + ## Facts worth knowing | Fact | Where it matters | diff --git a/docs/img/heart-rate-panel.png b/docs/img/heart-rate-panel.png new file mode 100644 index 0000000000000000000000000000000000000000..eab48f722265151e1a0a9f8725e6c14f1ea51fa2 GIT binary patch literal 19410 zcmeHvXIPV2yKbXT5Kn`R3c-kMrkT*WTAY;A0>1VZq_`LkCLh|M7g#DKjo`;4#NFE_mX;UA;Bm0m_! z#QLpQ+a`ba!6u_k8@Gfj3NrWhbfJtbu42;+eCeM0SgwX#ibz{veqm4F%0T|opq3it zkpi?l=v7VE9!TN6MIaV;hVb#;pXRUm@!?M!uJYc6>|A$(_wE!w-#*^EgQ19RymvcK zBcT0(7ynO)^}KhXYtvojSz^t0Yk{(s6}((PtRc!8e6H0BUNWrJ*)>wJ#-Mn)utpcx zsPM};^H2IVaVXNMJ1@;<+Z)=uwO4C zQWVC^;iB}15Y!C^G*9q-6{=}zPE;waTAkN9Y~?#pY>iEXetAg+VsbAV`D-YJ?!&oy zjOkW&`owa7&K3kBq+&}Am|OE4LIV5IETK#i{4oW$#^0Y4Z34ShLK=UPkCV)Uh6cc! z)oc)|p^(CC$$x19G5d6ch#L@yQ<@Mkgh+HwBG8km@o|kvOwQ`8n`IM+jg223es%lX z(4BeC3aP2nK+CvQi@VyHLMD@KB?4zk$6`&D<6KCans7mZP3>>Azt9G_(r(p@@$av< zEI54lb7Q*r8#T_TD*jt7Mwx>dZ^q5ntY^gIxomtB(H40uY`gfa=IJ;~*Q3+Ke3uG? zH<=AL9k^1z##b!8$y3r>N?3*<|NY!c7j321f zl2veQ6_dwa|8Sd_m#o)#(qqJovZY6VWluIlNt5zE3)CU zHbvkjUPpXsYg0KKm5np5^qZ6W(xO6Ey!pWD-N=Vsp1C&a>~?LWatdmo;93xif+Q;1 zxLMWLU$-|hXt8nskYm^&YH*TWhvdBRXw5Ahw2%n)GZq!=@AzIeODa|3aFVCC^qMST z4tZueYe`n5spTI9+b~Ro^p`gKKJgIKyj8tU$~XZ}dqyd76&L-9m}?pMMZWNQ%P~i6 z&{Br8|NLjE1Wu2&?|}Cgk2k*)?Hi-w^b(bo4~?DKp=qRi%gofZYECLWsS=80LiIwc zx?~YI=7o;+iu|Bbx4#~U_vd{-0+D8V3eR4$)fO|bKAKi;Ig&Hot51O-E`QCVS4GY}EHA<~Uv zgPyBT#pAhLJw%?TRzqc4MIP)GEYB-h^!gG7M80Kd$t2UbMvS1LwLDR`%if4wrpbCf zRarQ@y+%M-`o{8sY*VdYU$Lo!1B{kmWmRO}-{oOYbhK`6ln`-pP$9KD^ zI|PFaAukKdaelX#&82bAQm;^9lUe^WBZ|}!rpm6X3r@BI{kn@mq?pF9MNX8m#v9s* zJR|Zu@oAOpbT>;;=hvQ@o?4Rc)71WL%YAIYp)buZ6q+jOI0f2OzuWFrtntJzbCf#6 zLkL?8d(pntVX`AtCQ_-rysI23WNs)<+yk&J(*TkWC**-2Vnr3MKhs89TW?2lvc}BJ zs@0gI@>v+H>=hGo4Vy)vy96Tgq-}9CMb4QiIj#;#pPN(j*v}^trg(AW!gJi@$Kx*z zOoH5edf%_?&p$C(y+XM?(e|O?UkB#cb77RO95)GBhYr zZ1{tEr7>97M=z!t4j^l|h=9mmlS|jM;r~Wm{ib`XD)kwFkoQnc5;D-R0kCD@Fx?~L z07~Rorvc;H`prx6Za7v2sM%Kc@8gbc-XU>hWocFh8T9>{KQ+1Z%YDJ=FEIwF%d~PT z?Ka>Pag&|tzCn5*xW7xlNUJSlR`3$NGV77x_wB>xJ{&jRmgDB|`ROkX?B!wyYO8o; zfhBc+mRXfU)A;*<#lvucaOzC+W~YFneszjRwT$+L7$lwSS^TQDxA-{177(q-bg%3o z(xg%lhsBM9K;e2zLib|e4hS-IV92<<&fO7^0V~s73-hI!-hoAL<0uKu4nOPf{pCb2 zwgli$SEqKtOzxTKsZtdL0gJKrpG*~LYHNFvm+z>UsDg(ppEX_tDsgJO688(DR0?4m zwAekiNaSWuJKr!~+{5%FXj(2WFINN`0NT^`qeAQsnHH=mbR@{i;a9Jy2BXWS7qAbk zD}Cbp7sh?qIPmVju*YtvtIuLugcNmMFtCvscFC zMB?9QM|%b3&0Yx-DI+nCfH{+S)agimh-=>RefnP0W>If)D=1|U3$(^MN@*-lOy!Z& z!BDR_xOTnSFf=~sM_Ihg$XQmV4GqQR5#Ghb$=QPE?uDNF!=Q-%{+gN^Cx*+)l8ZIg zUdq_uEFfii)_7ZB%86~f(AZLQxSGpZl4s;DqufiWy-wKRiu9zQrCw{iBN%o=i#N=e zkNNSTy>)bi;pYKgvgSCB&1}??1?I8FI;U$g`stOy1jy{cJbd3BVo>Z$Pj_f|Bq}pI zkm!bMUu608K@`^^-nwUTMHnn|X*bXB2y|AML_o$^wPn$b%d>cHJ|@Uq*)2Ff#};3r zG5@I!1gfo4Fg0C^?ZoOO(4=+ZWo5EWa-|6x3zhRdmYAcqn4kJarF+{QcFwkD-zBn~4AXil|&5#97m^}$?()+xxCs3&5DGVhT3$c^tL6^7 zctZ=IdioQj5|#cFZS!vErCzB>DQ`!RNIgNBDTg?V`KF3EA}J410T_nO^HSY4a=Bo1eG0v&*NlvcEJU={GGm;6}n# zt?`?|&u{F2%o7roRqLGW;mKKL38g^2Nw-(C9KPQ+PNp({?&i4btl?tvXmy~Liw+`h z9lDebmX7E<13&nUE1q4_cY1tisOLh<*q7t^8V*&I#;~RYB|`iHK?^^lnw^yghe~nha5>tW$OfvbH zB_vlpl+fSvWt0t82Zpg%7;(F06BG8I;d0n91h2fv{ocl4a7rLe|KA2fZE2Cas@q`y z=)&7JunG6?6T=;?lxIg{VhB(h%Yx(bSXN1i@+~s17>q?Tzl|!SzV~!I<3ydqtajrd<73Iy4BkO29$ZI zNMb<@hRnlFsZp~QXD+(q&#}H@=hJ$6dh;)?SQOuMaH4vCe9*db#03mY`Z3VdHptwf zAnBxs-Ms%5QkB1<6BVF(%4aGt+etC?NQ z(eV3rYN9jU0IV-j(dmj4AG6mHJLciE>n|dEf5*GqNAFw9VNE4I*`G0 z;*XiLFl=XwU9L?JVk1N_Pthqi-5bSA<9!(U3@*UlQ9U?26+v|qv@E$LD`R5gJ3x*h z7zyH5T+N-RUQ=CRJTr{J0sm|~uaYJ$W0YrAeu$;feRcHHaUJ#axz-sSxoSn0E@X)) z*b+`A`8LKDIaQ$Pd`d7B6qdER)Te6wel%iq0Svli4qHF9Z211PjzFwsDQAC$_jDXO zt?OL0gms^4q-*7D4==objAV{P5H1SDcBX43fISp=O%hX+J6%X6d+0axHK72H{RKwY zYjyVpj}OXOp3CURW85YNDt&>4Dk)JUo$3|FTq9Bk4)9pGe70Tlq~feCGSIBC_@{&z zi)}+74CH2$XnnaFOtdh!&PTZbIuVJPup&=sgH@yr+BOI1^M4imNW_ z02u59@2rS4QjdKmeB6-`jACgV^#U9gz0g8TZ6t$gDv1>Y4)B7l)i*ZZBqui;R1Ea1 zZR@#md-kuap#*?$> z*rk~OT0tdSav%Nls)IrYm=F(dJ?k!PC(%rhi|W40&xB#9+P3d88U@yx z6$Vf4Kr(6N=H&xT&ls*XV--U~_M@cZkr1^fi%Sa5u~ghKg`p4yj{g3^e*r37)c$nPvv7 z3%@fD=p`zm*!1sv?Y1e;F(W^0n^MSBX7MWToFHR16{c)ZEYn>TAKyq$F-UKnW1`OUxhBD|gj5a`%$ z4Cs$0-x0vnNlLt-t`$yJPVx&icY=EBnC zp3`p&-KMq|PzEnjFr4!6dNLhnDGaAFJkkvwDRVBeBCF`7DDsJh*Tmydh1gesNDl)Q zG{bC_J>Nt)!=Mes5P;xa!-D0Wm-VYGj3XUHnL1_yCSL9b!S_r}qMZ_WR}{r_S2;H(@$k!YX~~+cX2d|L2(V#60UdpjGP#gb`3Hy7Mnu;}Fsi zO&C3o00nd6EUjuGs^j%FLqq!sV!rKves2OK2n)p|6*o{t)J`LWq2}}U&(I8ByNR|K z>&$j{=C@%GV*Gq&(2Sp+KoQk;0nK5hr*5qKMF)2tWuU#K1~y3)&$(B>7F-*iXs-j` zzq49%5y$%XNqoCI*NoYRk6`~9K z4to3I?+cp0!fuB?`{Qr1?$h6Q_8xCpn-G}he~Ar^%6A>UAI9Y{&ijGl{1gdzE8@}s z9B#*?5Y(}$?ELKjw6=JTGc`d3AAwIZE{$c$BWyu2d*w#PeU7G}6}_j<)YHqa)v4h{ z!33!=dSqECZ5ndJTU&@3PEk%GyNSfG%@WUd=T+Q%O9p`%(gA0)u|A@~4a9%V3s|4Y zNzg2*VfKmB4VzpI$p*F`{0nqd5GxiitLBL|xH-K2e(R)z6j@nUGze)qLd~2}k z=nP=r=*8}*?(9Wka)C7#6;xZ~6dekn=6rCG}e3{uVOeBz3l@&vMy`%p;gzJZptRkJuIA{w=P0;+V~C|bpI ztKG^`LrJWA4x}->qd^j3F`XjLO)RF0feLe}n&&XzTQwwrdw%q4--vx;Z*Eu#b9+tp zMLs#vK#V+~5(;2P2AsegxJD|G`Mn?+w3gW}fCZI9jI_Y6e#da#4lVu#D(2G1mNITx z4yf1lKJ4X1L8wP*n)`@}w$pGOB!bc}zFk7&l9;nbD<~E_1Vm){4=CpdC|uX@ZYXT9 z=Z+`l*MaUv6!3@3(h--xhab-T};hc#J9 zaYE2YjPH2!K*CQT?z&+WIOBF>!w1keluFZkaxHgFjm$I1nEY->JL&#XH-lZ50Ok+A z*9ZgS>M z+`&9>-8XJJ7@=5!y>O=mQPP^9^V5rujDY75R8hmV<78vk6 zOvN&8r$@Y=Yi%n-;Rq%Ey;4}Tm2@S0i4<|rFGniHugXjXD3>w8j#Ueu6K${Rsgx2o zJo|d~VtV*P!*o>F`~-!$Z`jhcob{I8<3i5?hww*wqQ%vhavU`We2W-mBkIp770kiG z5>#a%qjf~K&v16yt8ov#CX$YJ<DAu_2tu=G{l&DQZOw`!LO+$q&E@-bqrr-i?RIX5c+z0dpGEG1e>lBiR zMIVciBHL<|sGcKj5t7{F%e4XECqo_>MD))*)sfw@L*qp1Sn>yXMW4xzv7$oUQeToX zQH2~CF!Lsyr0%TGcQS=UDb-9w^bdOSgW9pk!@B;=K|Zm1Mw0~K@P2zd7T!~_m^@}P zC0fhp9C;3CoA-cB5t@*YVA2H66XxUG183&PpZjG!53BZv)uSs6>^((qwuHQYvgjff#%{L zb1dcy`_g2frPs3Jq2HC#AUK@tJAUt{GqT6y#E~Iy z0Mv&qPGg|9Q{cGCYTNww5hni#GkMOl#W%LWPn2P6X&DIYO^>)7Pz&Q3(#nMZ*Y1Fl``|0)r)EE*(Uex-`ctAK zN-{R2E4JVu?4<(4jSfD+ua3#p2MP_a1y0Gr-;OBBE_#D=Dz)Q~>32JAugT<^?Laud zlgYrJeO5&@s@JsEC$}Vz(I#7em3q7x`keQ?C-z7_K}0!3;t%qM0}t_<3MYtYJ$``L zb_(tz&O=o)1X2>=e&mz|yp=a^s1LWE&=7_-XRJOL1 z7V0H@qXL$|P90izq9`{nPgdGy^#YTQi4ILxKTr$q7wsSPceL4c7Xws!J64syTj{bc zKA2rwb7y@FQvD!kfll50@yIhI=&)gzKRIB!<&v!}rgr(ayPNKTbSbr18t`ojC%i@Z zz>evH2+sMli2EhE<_q>WZmR8@nhs2wbKGz>Ljs!X9}9Ztfq`21iS{#y`wMVW_4oc= zEX!~E@A6F751J?<*?4zAt8!%FSLBX9WmmwQr(1+1)-ul8<&k$sWj{Z3lOp3i!+M zoMoN47od>~1zK6jt-x;M!Y@KdZ*hRHw7(871} zobJow6^l5V2EC zi_z!oX%S}&tP@9%^c8G;*#8Un9CPd8t5j2Za-N*X_>6*jf(wv~4 z-e$|!GvM?@6dY%Ky-6DPMU)m}%?qy!PrFu3rYYm?SC(>zKRr3~3Wvj)^@xRl!&Lpm zm5Au5sOhk6qEA=FepC8K#P13jZeFAfRQgGMD{-+5XYN0I%f-c|Y1we&(C-n#!gor6 z+~AUjRiV$06w0qQa9y|8G!E`X=ZwEZQzBI)wd5{E{?jv8Z{VSeg}zqZ@^@S5_bua&flLWkbW zb?#_eH!BhPAy;R1c0TS(4k*&E>M>mUuC{z&Xy}qe4t_kr`6*T)>|v+hJvGHXFg zV^X}P5hYyFdpy6&DhC|(jBJ@6Spru!4Zi}S+T@tH{@dq2=Ru9y3(kf1LB(m5x-RR# zgfxWk?sbAh>`u>0?`dJ^QpQLS4kF}0%9NQFma%Z}V9eWi)N`SXE&$@8N&CrwWHthoVqfCe7mV7w7mhoOb9vB+E zalmZ^fY#>LyYv+yR<9mRu}`OHQ7~L4WvVUT7C%beniUJxlz*k}0YE1FMxTV$fuk#% z{z=l;GA%{LTLQpUF3$lOh`}{6?rcumLqeV`kOP>d@km0mq8o4neBJ{*bVvlr#~TKq zE~)@Kj?nO>SzhbOd2tOC-+K8Rvv(TJDQxrvxG^yAxuIJZ0?4AjAZMye3@}D~X7t8Z zSZDbgYcheVo@z}pZd#I3=PU)2KPzq?QuVaS{lqyQv^qPm{fp$kLZ?G4#QrT`RwFno zDA_R7)wqrd>oPp--o>{s3kZth&>_sw}-J1|#im?#bZcdm_S^BfcUh5Y$RPnFwN$p9@@y z;A4;7;h2Nm0eUd;wp78+Z8a=Oo5}?GY9vu{4xA|eRnWBOwYV(8@D2>~%v{(4bMP$U zB>5w+Dkl8PS!Sze{5|QI@=I<1 z6nd@6POur)ls4E5YicEIhX1QT8|wTUAnpQ2(Ek?zAYm36M*VWU2*TFAc z{;RwH_D&7d$@3k`4ue=VI5Yp8)h{C9xkA(^e5qxU8d R&!A+y@Voxmywg|z{6BGgKkxtm literal 0 HcmV?d00001 diff --git a/docs/testing.md b/docs/testing.md index b06577d..6dc3bdf 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -148,3 +148,10 @@ For example, on 2026-09-27 the smoke test found that Steam's `create-shortcut` refuses ids with a hyphen (`missing/invalid arguments`), which the fake had accepted. The fake now refuses them the same way, and Frame Control makes ids Steam accepts. + +## Tracking protocols and fake BlueZ + +`tests/test_tracking.py` exercises our gaze conversion, OSC sender, HRS parser +and BlueZ lifecycle with an in-memory fake object tree. It runs in the normal +unit suite without Bluetooth, GTK or OpenXR. Real Frame results and the absent +strap/tracker boundaries are recorded in [tracking](tracking.md). diff --git a/docs/tracking.md b/docs/tracking.md new file mode 100644 index 0000000..9347949 --- /dev/null +++ b/docs/tracking.md @@ -0,0 +1,188 @@ +# Eye tracking and heart rate + +Frame Control's own tools run on the Frame, using OpenXR and BlueZ. No +VRCFaceTracking, LunaHR, Pulsoid or other tracking app is required. This is a +command-line first version; it does not add a desktop app tab. + +## What was checked + +**Verified 2026-09-28**, on a real aarch64 Frame running SteamOS 0.4.1, +BUILD_ID `20260925.6191901`: + +| Check | Result | +|---|---| +| OpenXR gaze | SteamVR advertises `XR_EXT_eye_gaze_interaction`, `XR_MND_headless` and `XR_KHR_convert_timespec_time`. `supportsEyeGazeInteraction=1`. A headless session reached FOCUSED and produced 269 valid, tracked orientations in the first ten-second probe. | +| Our gaze → OSC bridge | A separate ten-second run produced 280 valid samples and 280 correctly padded 44-byte `/tracking/eye/CenterPitchYaw` messages at an explicitly configured loopback receiver. Only counters and packet-layout checks were retained. | +| Bluetooth stack | BlueZ active, adapter powered, central/peripheral roles available. LE discovery started and stopped successfully. No pairing or adapter power settings changed. | +| Our heart-rate panel | GTK4/GI runs on the stock image. A **synthetic 72 BPM** notification displayed in our X11 window, tagged `STEAM_GAME=2000000027`. Window capture checked; no real heart-rate measurement was taken. | +| SlimeVR, separate feasibility check | Native aarch64 server v21.1.0 ran with an isolated Temurin 21 JRE, created its driver sockets and accepted a local TCP connection on port 21110. Driver v6.0.0 loaded with all shared libraries resolved; `HmdDriverFactory("IServerTrackedDeviceProvider_004")` returned a non-null provider and error 0. | + +![Our heart-rate panel on the Frame, showing synthetic 72 BPM](img/heart-rate-panel.png) + +The image is a capture of our own Frame window using a fake notification, +not a real sensor reading. + +**Untested:** a real BLE strap's notifications, physical fit/contact behaviour, +end-to-end heart-rate display/OSC/log with a strap, avatar response in VRChat, +gaze accuracy/calibration, coexistence with every immersive app, in-headset +panel placement, SlimeVR tracker/calibration data and the SlimeVR driver running +inside SteamVR. No trackers or strap are attached. The driver was loaded in a +separate process; it was **not registered or activated in SteamVR**. Steam and +SteamVR were not stopped or restarted. + +**Verified blocker resolved:** importing `tkinter` fails because `libtk8.6.so` +is absent. The panel uses the installed GTK4/GI bindings instead. The Frame's +OpenXR headers advertise a newer API version than the runtime accepts; our +reader requests OpenXR 1.0 explicitly. + +## Install our tools + +From this checkout on your computer, while the Frame is awake: + +```sh +python3 scripts/tracking-on-frame.py install +``` + +This copies our Python code and compiles our small C OpenXR reader into +`~/.local/share/frame-control/tracking/` on the Frame. It uses the Frame's +existing compiler, OpenXR headers/loader, Python, dbus-python, GI and GTK4. +Nothing is downloaded, and no sudo, driver registration, system setting, +service or autostart is added. `FRAME_ALIAS` can select another SSH alias. +The desktop app/server keeps its existing stdlib-only dependency set. + +## Eye tracking → OSC + +Start with a ten-second capability/data-availability check: + +```sh +python3 scripts/tracking-on-frame.py gaze --seconds 10 +``` + +This prints support, session-state numbers and sample counters. It opens no +OSC socket and prints no gaze coordinates. Exit 0 means at least one valid +sample, 3 means no valid sample was observed, and 1 means an API/runtime error. +If there are no valid samples, wake/wear the headset and check its tracking +setup; a successful capability check alone does not prove usable gaze. + +To send to VRChat running **on the Frame**, explicitly enable OSC in VRChat +and choose its local UDP endpoint: + +```sh +python3 scripts/tracking-on-frame.py gaze --seconds 3600 --osc 127.0.0.1 9000 +``` + +For a receiver on another computer, replace `127.0.0.1` with that computer's +IP address and choose its listening port. Addresses are IP literals (IPv4 or +IPv6); there is no discovery or default destination. Loopback here always +means **the Frame**, not the computer running the SSH command. OSC uses +unencrypted UDP: configure only a receiver you intend to receive this data. + +**Documented:** [VRChat's eye OSC interface](https://docs.vrchat.com/docs/osc-eye-tracking) +accepts `/tracking/eye/CenterPitchYaw` with two floats in degrees, positive down +and right. We locate OpenXR's combined gaze pose relative to VIEW (the head), +rotate its -Z forward vector and convert that direction to these angles. +Only active, orientation-valid **and tracked** samples are sent, at up to +30 Hz. No eyelid/blink, individual-eye or face values are invented. We do not +send neutral gaze on tracking loss; VRChat's documented timeout restores its +automatic eye behaviour after input stops. + +**Privacy:** gaze is personal data. It stays in process memory and a private +pipe between our reader and bridge. There is no gaze log option, telemetry, +OSC receiver or raw gaze on stdout/stderr. Only an explicit `--osc IP PORT` +opens an output socket. Runtime diagnostics and validity counters are not +measurements. Stop with Ctrl-C or let `--seconds` expire (maximum 24 hours). +A lost headless session ends the run; it does not silently reconnect. + +## BLE heart rate → our panel, OSC and optional log + +First discover/pair your strap in SteamOS's Bluetooth settings. Select that +strap's Bluetooth address explicitly; our tool does not scan for or connect +to arbitrary nearby devices. + +```sh +python3 scripts/tracking-on-frame.py heart \ + --device AA:BB:CC:DD:EE:FF --panel --seconds 3600 +``` + +This uses BlueZ's standard Heart Rate Service (`180d`) and Heart Rate +Measurement (`2a37`) notifications. It finds the characteristic only beneath +the selected device's HRS service. The reader handles 8- and 16-bit BPM, +contact flags and optional energy/RR fields; energy and RR intervals are +validated for length but discarded. Zero BPM, reported loss of skin contact, +malformed packets and readings older than five seconds are not shown as a +current measurement. A disconnect stops the run; reconnect and start again. +This is a social/fitness readout, not a medical monitor. + +The panel is our GTK4 window on gamescope's X display. Use SteamVR's panel +controls to float/dock it (see [panels](panels.md)). **Stop**, closing the panel, +Ctrl-C, SSH hangup or the duration limit ends our subscription. A connection +that was already open when we started is preserved; a connection we opened +is disconnected on exit. No Bluetooth power or pairing state is changed. + +Add either output explicitly: + +```sh +python3 scripts/tracking-on-frame.py heart \ + --device AA:BB:CC:DD:EE:FF --panel --seconds 3600 \ + --osc 127.0.0.1 9000 --address /avatar/parameters/HeartRate \ + --log /home/steamos/heart-session.csv +``` + +The OSC value is integer BPM. `HeartRate` is a chosen avatar parameter, **not a +built-in VRChat heart-rate feature**; your avatar/receiver must define the +matching parameter. `--address` can select another literal OSC path. The local +panel works without OSC, a log or an avatar integration. + +The optional CSV contains only `unix_seconds,bpm`. It is created privately +(mode 0600), refuses existing files/symlinks, and lives **on the Frame** at the +path you specify. Nothing is logged by default, and heart-rate values are not +printed to the terminal. Delete your session file when you no longer need it. + +## SlimeVR: feasibility only + +SlimeVR is an independent application stack. Neither of our features installs, +launches or depends on it. Users who want it can follow +[SlimeVR's setup documentation](https://docs.slimevr.dev/server/index.html). +The consented upstream releases tested were +[server v21.1.0](https://github.com/SlimeVR/SlimeVR-Server/releases/tag/v21.1.0) +and [driver v6.0.0](https://github.com/SlimeVR/SlimeVR-OpenVR-Driver/releases/tag/v6.0.0), +under SlimeVR's MIT/Apache-2.0 licensing. + +**Verified layout, read-only:** the Frame's registered runtime is `/opt/steamvr`; +its native driver is `drivers/cv/bin/linuxarm64/driver_cv.so`, with a +`drivers/cv/driver.vrdrivermanifest`. Frame controller manifests/resources are +under `drivers/frame_controller/`. Configuration is under +`~/.config/openvr/config/`, not the Steam client's config directory. The +SlimeVR release also uses `slimevr/bin/linuxarm64/driver_slimevr.so` plus its +manifest. Nothing in those installed SteamVR directories was changed. + +**Inferred:** the matching ABI/layout and standalone factory success make +SteamVR integration plausible. They do not prove successful driver `Init`, +server/driver IPC, tracking, or calibration. That needs a separate integration +check with hardware and an agreed SteamVR restart. No Java executable was on +PATH for this check, so an isolated JRE was used. SlimeVR's server opens LAN +listeners; our temporary server was stopped and the temporary downloads, +configuration and logs were removed. It is not left installed or running. + +## Tests and remaining checks + +```sh +python3 -m unittest discover -s tests +``` + +`tests/test_tracking.py` covers HRS packet parsing, contact/staleness, OSC +padding/types and a real loopback socket, quaternion signs, opt-in networking, +private/exclusive logging and a fake BlueZ object tree. The fake checks service +ownership, notification routing, delayed GATT discovery and connection cleanup. +It does not pretend to be a physical strap or a real OpenXR runtime. + +Before calling hardware support complete, attach a strap and check BPM against +its own display/reference, loss of contact, disconnect/reconnect, Stop, OSC and +CSV together. Check avatar eyes while looking up/down/left/right in a supported +VRChat session. No third-party tracking app is needed for either test. + +Independent review attempt: `devin -p --model swe-2-max` with the frozen diff, +contribution standards and read-only instructions returned no output for ten +minutes. It was terminated with exit 143. No completed review or actual model +identity was returned; hardware checks and independent review remain follow-up +work before making the draft ready. diff --git a/frame/tracking/gaze.c b/frame/tracking/gaze.c new file mode 100644 index 0000000..95a5789 --- /dev/null +++ b/frame/tracking/gaze.c @@ -0,0 +1,135 @@ +/* Frame Control's OpenXR gaze source. No values on stdout/stderr or disk. + * The Python bridge supplies a private pipe with --fd; standalone probes only + * report counters. Uses a headless session, never submits frames or takes focus. */ +#define XR_USE_TIMESPEC +#include +#include +#include +#include +#include +#include +#include +#include + +static volatile sig_atomic_t stopped; +static void stop(int sig) { (void)sig; stopped = 1; } +#define CHECK(call) do { result = (call); if (XR_FAILED(result)) { \ + fprintf(stderr, "%s failed (%d)\n", #call, result); goto cleanup; } } while (0) + +int main(int argc, char **argv) { + int seconds = 10, fd = -1, running = 0, rc = 1; + unsigned active = 0, valid = 0, samples = 0; + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--seconds") && i+1 < argc) seconds = atoi(argv[++i]); + else if (!strcmp(argv[i], "--fd") && i+1 < argc) fd = atoi(argv[++i]); + else { fprintf(stderr, "usage: gaze [--seconds 1..86400] [--fd private-pipe]\n"); return 2; } + } + if (seconds < 1 || seconds > 86400 || (fd != -1 && fd < 3)) return 2; + FILE *out = fd == -1 ? NULL : fdopen(fd, "w"); + if (fd != -1 && !out) return 2; + signal(SIGINT, stop); signal(SIGTERM, stop); signal(SIGHUP, stop); signal(SIGPIPE, SIG_IGN); + XrResult result; + XrInstance instance = XR_NULL_HANDLE; + XrSession session = XR_NULL_HANDLE; + XrActionSet set = XR_NULL_HANDLE; + XrSpace gaze = XR_NULL_HANDLE, view = XR_NULL_HANDLE; + const char *extensions[] = {"XR_EXT_eye_gaze_interaction", "XR_MND_headless", "XR_KHR_convert_timespec_time"}; + XrInstanceCreateInfo create = {.type = XR_TYPE_INSTANCE_CREATE_INFO}; + strcpy(create.applicationInfo.applicationName, "Frame Control gaze"); + create.applicationInfo.apiVersion = XR_MAKE_VERSION(1, 0, 0); + create.enabledExtensionCount = 3; create.enabledExtensionNames = extensions; + CHECK(xrCreateInstance(&create, &instance)); + XrSystemGetInfo get = {.type = XR_TYPE_SYSTEM_GET_INFO, .formFactor = XR_FORM_FACTOR_HEAD_MOUNTED_DISPLAY}; + XrSystemId system; + CHECK(xrGetSystem(instance, &get, &system)); + XrSystemEyeGazeInteractionPropertiesEXT eye = {.type = XR_TYPE_SYSTEM_EYE_GAZE_INTERACTION_PROPERTIES_EXT}; + XrSystemProperties props = {.type = XR_TYPE_SYSTEM_PROPERTIES, .next = &eye}; + CHECK(xrGetSystemProperties(instance, system, &props)); + printf("supportsEyeGazeInteraction=%u\n", eye.supportsEyeGazeInteraction); + if (!eye.supportsEyeGazeInteraction) goto cleanup; + XrSessionCreateInfo sc = {.type = XR_TYPE_SESSION_CREATE_INFO, .systemId = system}; + CHECK(xrCreateSession(instance, &sc, &session)); + XrActionSetCreateInfo asc = {.type = XR_TYPE_ACTION_SET_CREATE_INFO}; + strcpy(asc.actionSetName, "gaze"); strcpy(asc.localizedActionSetName, "Gaze"); + CHECK(xrCreateActionSet(instance, &asc, &set)); + XrActionCreateInfo ac = {.type = XR_TYPE_ACTION_CREATE_INFO, .actionType = XR_ACTION_TYPE_POSE_INPUT}; + strcpy(ac.actionName, "gaze_pose"); strcpy(ac.localizedActionName, "Gaze pose"); + XrAction action; + CHECK(xrCreateAction(set, &ac, &action)); + XrPath profile, input; + CHECK(xrStringToPath(instance, "/interaction_profiles/ext/eye_gaze_interaction", &profile)); + CHECK(xrStringToPath(instance, "/user/eyes_ext/input/gaze_ext/pose", &input)); + XrActionSuggestedBinding binding = {action, input}; + XrInteractionProfileSuggestedBinding suggested = {.type = XR_TYPE_INTERACTION_PROFILE_SUGGESTED_BINDING, + .interactionProfile = profile, .countSuggestedBindings = 1, .suggestedBindings = &binding}; + CHECK(xrSuggestInteractionProfileBindings(instance, &suggested)); + XrSessionActionSetsAttachInfo attach = {.type = XR_TYPE_SESSION_ACTION_SETS_ATTACH_INFO, .countActionSets = 1, .actionSets = &set}; + CHECK(xrAttachSessionActionSets(session, &attach)); + XrActionSpaceCreateInfo space = {.type = XR_TYPE_ACTION_SPACE_CREATE_INFO, .action = action, .poseInActionSpace.orientation.w = 1}; + CHECK(xrCreateActionSpace(session, &space, &gaze)); + XrReferenceSpaceCreateInfo ref = {.type = XR_TYPE_REFERENCE_SPACE_CREATE_INFO, .referenceSpaceType = XR_REFERENCE_SPACE_TYPE_VIEW, + .poseInReferenceSpace.orientation.w = 1}; + CHECK(xrCreateReferenceSpace(session, &ref, &view)); + PFN_xrConvertTimespecTimeToTimeKHR convert; + CHECK(xrGetInstanceProcAddr(instance, "xrConvertTimespecTimeToTimeKHR", (PFN_xrVoidFunction *)&convert)); + struct timespec start, now; + clock_gettime(CLOCK_MONOTONIC, &start); + while (!stopped) { + clock_gettime(CLOCK_MONOTONIC, &now); + if (now.tv_sec - start.tv_sec >= seconds) break; + XrEventDataBuffer event = {.type = XR_TYPE_EVENT_DATA_BUFFER}; + while ((result = xrPollEvent(instance, &event)) == XR_SUCCESS) { + if (event.type == XR_TYPE_EVENT_DATA_SESSION_STATE_CHANGED) { + XrSessionState state = ((XrEventDataSessionStateChanged *)&event)->state; + printf("sessionState=%d\n", state); fflush(stdout); + if (state == XR_SESSION_STATE_READY && !running) { + XrSessionBeginInfo begin = {.type = XR_TYPE_SESSION_BEGIN_INFO, .primaryViewConfigurationType = XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO}; + CHECK(xrBeginSession(session, &begin)); running = 1; + } else if (state == XR_SESSION_STATE_STOPPING) { + CHECK(xrEndSession(session)); running = 0; stopped = 1; + } else if (state == XR_SESSION_STATE_EXITING || state == XR_SESSION_STATE_LOSS_PENDING) stopped = 1; + } else if (event.type == XR_TYPE_EVENT_DATA_INSTANCE_LOSS_PENDING) stopped = 1; + event.type = XR_TYPE_EVENT_DATA_BUFFER; + } + if (XR_FAILED(result)) goto cleanup; + if (running && !stopped) { + XrActiveActionSet activeSet = {set, XR_NULL_PATH}; + XrActionsSyncInfo sync = {.type = XR_TYPE_ACTIONS_SYNC_INFO, .countActiveActionSets = 1, .activeActionSets = &activeSet}; + CHECK(xrSyncActions(session, &sync)); + if (result != XR_SUCCESS) { + struct timespec delay = {.tv_nsec = 33333333}; + nanosleep(&delay, NULL); + continue; /* No stale gaze when the runtime denies focus. */ + } + XrActionStateGetInfo ag = {.type = XR_TYPE_ACTION_STATE_GET_INFO, .action = action}; + XrActionStatePose pose = {.type = XR_TYPE_ACTION_STATE_POSE}; + CHECK(xrGetActionStatePose(session, &ag, &pose)); + samples++; + if (pose.isActive) { + active++; + XrTime time; CHECK(convert(instance, &now, &time)); + XrSpaceLocation location = {.type = XR_TYPE_SPACE_LOCATION}; + CHECK(xrLocateSpace(gaze, view, time, &location)); + XrSpaceLocationFlags needed = XR_SPACE_LOCATION_ORIENTATION_VALID_BIT | XR_SPACE_LOCATION_ORIENTATION_TRACKED_BIT; + if ((location.locationFlags & needed) == needed) { + valid++; + if (out) { + XrQuaternionf q = location.pose.orientation; + if (fprintf(out, "%g %g %g %g\n", q.x, q.y, q.z, q.w) < 0 || fflush(out)) goto cleanup; + } + } + } + } + struct timespec delay = {.tv_nsec = 33333333}; nanosleep(&delay, NULL); + } + printf("samples=%u active=%u valid=%u\n", samples, active, valid); + rc = valid ? 0 : 3; /* Distinguish a working session from observed gaze. */ +cleanup: + if (view) xrDestroySpace(view); + if (gaze) xrDestroySpace(gaze); + if (session) xrDestroySession(session); + if (set) xrDestroyActionSet(set); + if (instance) xrDestroyInstance(instance); + if (out) fclose(out); + return rc; +} diff --git a/frame/tracking/tracking.py b/frame/tracking/tracking.py new file mode 100644 index 0000000..10b8612 --- /dev/null +++ b/frame/tracking/tracking.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +"""Frame-local tracking tools. No network destination or data log by default.""" +import argparse +import math +import os +from pathlib import Path +import signal +import socket +import struct +import subprocess +import time + + +class TrackingError(RuntimeError): + """A safe, actionable status message containing no sensor data.""" + + +HRS = "0000180d-0000-1000-8000-00805f9b34fb" +MEASUREMENT = "00002a37-0000-1000-8000-00805f9b34fb" +DEVICE = "org.bluez.Device1" +SERVICE = "org.bluez.GattService1" +CHARACTERISTIC = "org.bluez.GattCharacteristic1" + + +def heart_rate(data): + """Validate the Bluetooth HRS measurement, returning BPM/contact only. + + Energy and RR intervals are checked for length but never retained. + None means contact is supported and the strap reports no skin contact. + """ + data = bytes(data) + if len(data) < 2 or data[0] & 0xe0: + raise ValueError("invalid HRS measurement") + flags = data[0] + size = 2 if flags & 1 else 1 + end = 1 + size + (2 if flags & 8 else 0) + if len(data) < end: + raise ValueError("truncated HRS measurement") + extra = len(data) - end + if (flags & 16 and (extra < 2 or extra % 2)) or (not flags & 16 and extra): + raise ValueError("invalid HRS optional fields") + if flags & 4 and not flags & 2: + return None + bpm = int.from_bytes(data[1:1 + size], "little") + return bpm if bpm else None + + +def gaze_angles(quaternion): + """OpenXR head-relative -Z forward → VRChat degrees, down/right positive.""" + if len(quaternion) != 4 or not all(math.isfinite(v) for v in quaternion): + raise ValueError("invalid gaze orientation") + norm = math.sqrt(sum(v * v for v in quaternion)) + if not 0.9 < norm < 1.1: + raise ValueError("invalid gaze orientation") + x, y, z, w = (v / norm for v in quaternion) + # Rotate OpenXR's forward vector (0, 0, -1) into VIEW space. + dx, dy, dz = -2 * (x*z + w*y), 2 * (w*x - y*z), 2 * (x*x + y*y) - 1 + return math.degrees(math.atan2(-dy, math.hypot(dx, dz))), math.degrees(math.atan2(dx, -dz)) + + +def osc_message(address, values): + if not address.startswith("/") or any(c.isspace() or c in '\0#*,?[]{}' for c in address): + raise ValueError("OSC address must be a literal path") + def string(value): + encoded = value.encode("utf-8") + b"\0" + return encoded + b"\0" * (-len(encoded) % 4) + tags, payload = ",", b"" + for value in values: + if type(value) is int: + tags += "i" + payload += struct.pack(">i", value) + else: + if not math.isfinite(value): + raise ValueError("OSC value must be finite") + tags += "f" + payload += struct.pack(">f", value) + return string(address) + string(tags) + payload + + +class Osc: + def __init__(self, endpoint=None): + self.sock = None + self.target = None + if endpoint: + import ipaddress + address = ipaddress.ip_address(endpoint[0]) + port = int(endpoint[1]) + if address.is_unspecified or address.is_multicast or not 1 <= port <= 65535: + raise ValueError("OSC needs a unicast IP address and port 1..65535") + self.sock = socket.socket(socket.AF_INET6 if address.version == 6 else socket.AF_INET, socket.SOCK_DGRAM) + self.target = (str(address), port) + + def send(self, address, values): + if self.sock: + self.sock.sendto(osc_message(address, values), self.target) + + def close(self): + if self.sock: + self.sock.close() + + +class HeartSession: + def __init__(self, osc, address, log=None, clock=time.monotonic): + self.osc, self.address, self.clock = osc, address, clock + self.bpm, self.updated = None, None + self.log = None + if log: + # Exclusive creation refuses existing files and symlinks; mode is + # private even with a permissive process umask. + fd = os.open(log, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + self.log = os.fdopen(fd, "w") + self.log.write("unix_seconds,bpm\n") + + def notification(self, data): + self.bpm = heart_rate(data) + self.updated = self.clock() + if self.bpm is not None: + self.osc.send(self.address, [self.bpm]) + if self.log: + self.log.write(f"{time.time():.3f},{self.bpm}\n") + self.log.flush() + + def current(self): + if self.updated is None or self.clock() - self.updated > 5: + return None + return self.bpm + + def close(self): + if self.log: + self.log.close() + + +class BluezHeart: + """One explicitly selected, already discovered strap; no ambient scan.""" + def __init__(self, bus, interface, address, on_value): + self.bus, self.interface, self.on_value = bus, interface, on_value + self.device = self.characteristic = None + self.connected_here = False + self.notifying = False + self.match = None + objects = self.objects() + matches = [path for path, interfaces in objects.items() + if str(interfaces.get(DEVICE, {}).get("Address", "")).upper() == address.upper()] + if len(matches) != 1: + raise TrackingError("Strap not found uniquely in BlueZ; pair/discover it in SteamOS Bluetooth settings first") + self.device = matches[0] + self.match = bus.add_signal_receiver(self.changed, signal_name="PropertiesChanged", + dbus_interface="org.freedesktop.DBus.Properties", + bus_name="org.bluez", path_keyword="path") + try: + if not objects[self.device][DEVICE].get("Connected"): + self.call(self.device, DEVICE).Connect(timeout=20) + self.connected_here = True + except Exception: + self.close() + raise + + def objects(self): + return self.call("/", "org.freedesktop.DBus.ObjectManager").GetManagedObjects() + + def call(self, path, kind): + return self.interface(self.bus.get_object("org.bluez", path), kind) + + def subscribe(self): + objects = self.objects() + if not objects.get(self.device, {}).get(DEVICE, {}).get("ServicesResolved"): + return False + services = {p for p, obj in objects.items() if str(obj.get(SERVICE, {}).get("UUID", "")).lower() == HRS + and obj[SERVICE].get("Device") == self.device} + for path, obj in objects.items(): + props = obj.get(CHARACTERISTIC, {}) + if props.get("Service") in services and str(props.get("UUID", "")).lower() == MEASUREMENT: + if "notify" not in props.get("Flags", []): + raise TrackingError("Heart-rate characteristic does not support notifications") + self.characteristic = path + self.call(path, CHARACTERISTIC).StartNotify() + self.notifying = True + return True + raise TrackingError("Selected device has no standard Heart Rate Service measurement") + + def changed(self, kind, changes, invalidated, path=None): + if kind == CHARACTERISTIC and path == self.characteristic and "Value" in changes: + self.on_value(changes["Value"]) + elif kind == DEVICE and path == self.device and "Connected" in changes and not changes["Connected"]: + self.on_value(None) + + def close(self): + try: + if self.notifying: + self.call(self.characteristic, CHARACTERISTIC).StopNotify() + finally: + if self.match: + self.match.remove() + if self.connected_here: + self.call(self.device, DEVICE).Disconnect() + + +def run_gaze(args, osc): + binary = Path(__file__).with_name("gaze") + command = [str(binary), "--seconds", str(args.seconds)] + if not args.osc: + return subprocess.call(command) + read_fd, write_fd = os.pipe() + process = None + try: + process = subprocess.Popen(command + ["--fd", str(write_fd)], pass_fds=(write_fd,)) + os.close(write_fd) + write_fd = None + with os.fdopen(read_fd) as source: + read_fd = None + for line in source: + try: + angles = gaze_angles([float(v) for v in line.split()]) + except ValueError: + continue + osc.send("/tracking/eye/CenterPitchYaw", angles) + return process.wait() + finally: + if read_fd is not None: + os.close(read_fd) + if write_fd is not None: + os.close(write_fd) + if process and process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +class HeartPanel: + """Our GTK panel, using the Frame's existing GTK4/GI platform libraries.""" + def __init__(self): + os.environ["GDK_BACKEND"] = "x11" + import gi + gi.require_version("Gtk", "4.0") + gi.require_version("GdkX11", "4.0") + from gi.repository import Gtk, Gdk, GdkX11, GLib + Gtk.init() + self.running = True + self.window = Gtk.Window(title="Frame Control · Heart rate") + self.window.set_default_size(480, 320) + self.window.connect("close-request", self.stop) + Gtk.Settings.get_default().set_property("gtk-application-prefer-dark-theme", True) + box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=16) + box.set_valign(Gtk.Align.CENTER) + box.set_halign(Gtk.Align.CENTER) + box.set_size_request(440, -1) + for side in ("top", "bottom", "start", "end"): + getattr(box, "set_margin_" + side)(24) + self.window.set_child(box) + title = Gtk.Label(label="Heart rate") + title.add_css_class("title-2") + box.append(title) + self.reading = Gtk.Label(label="—") + self.reading.add_css_class("reading") + box.append(self.reading) + self.status = Gtk.Label(label="Waiting for strap") + box.append(self.status) + button = Gtk.Button(label="Stop") + button.connect("clicked", self.stop) + box.append(button) + css = Gtk.CssProvider() + css.load_from_data(b".reading { font-size: 144px; font-weight: 700; }") + Gtk.StyleContext.add_provider_for_display(Gdk.Display.get_default(), css, Gtk.STYLE_PROVIDER_PRIORITY_APPLICATION) + self.window.present() + context = GLib.MainContext.default() + while context.pending(): + context.iteration(False) + try: + xid = GdkX11.X11Surface.get_xid(self.window.get_surface()) + subprocess.run(["xprop", "-id", str(xid), "-f", "STEAM_GAME", "32c", "-set", "STEAM_GAME", "2000000027"], + check=True, stdout=subprocess.DEVNULL) + except Exception: + self.window.destroy() + raise + + def stop(self, *args): + self.running = False + return True + + def update(self, bpm): + self.reading.set_label(str(bpm) if bpm is not None else "—") + self.status.set_label("beats per minute" if bpm is not None else "Waiting for strap") + + def close(self): + self.window.destroy() + + +def run_heart(args, osc): + import dbus + from dbus.mainloop.glib import DBusGMainLoop + from gi.repository import GLib + DBusGMainLoop(set_as_default=True) + session = HeartSession(osc, args.address, args.log) + reader, root = None, None + failure = [] + def value(data): + if data is None: + session.bpm = None + failure.append("Strap disconnected; reconnect and start again") + return + try: + session.notification(data) + except ValueError: + session.bpm = None + except OSError: + failure.append("OSC or session log write failed") + try: + reader = BluezHeart(dbus.SystemBus(), dbus.Interface, args.device, value) + context = GLib.MainContext.default() + deadline = time.monotonic() + 20 + while not reader.subscribe(): + if time.monotonic() > deadline: + raise TrackingError("Timed out waiting for the strap's GATT services") + while context.pending(): + context.iteration(False) + time.sleep(0.1) + end = time.monotonic() + args.seconds + print("Heart-rate notifications started; readings stay local unless OSC or a log was selected.") + if args.panel: + root = HeartPanel() + running = lambda: root.running if root else True + while running() and time.monotonic() < end and not failure: + while context.pending(): + context.iteration(False) + if root: + root.update(session.current()) + time.sleep(0.05) + if failure: + raise TrackingError(failure[0]) + return 0 + finally: + if root: + root.close() + try: + if reader: + reader.close() + finally: + session.close() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + gaze = commands.add_parser("gaze", help="headless OpenXR; prints counters only without --osc") + heart = commands.add_parser("heart", help="standard BLE HRS from an explicitly selected strap") + for command in (gaze, heart): + command.add_argument("--osc", nargs=2, metavar=("IP", "PORT"), help="explicit UDP destination; no default") + command.add_argument("--seconds", type=int, default=10, help="bounded run, 1..86400 seconds (default: 10)") + heart.add_argument("--device", required=True, help="strap Bluetooth address already discovered by BlueZ") + heart.add_argument("--panel", action="store_true", help="show our panel on gamescope DISPLAY=:0") + heart.add_argument("--address", default="/avatar/parameters/HeartRate", help="integer BPM OSC parameter") + heart.add_argument("--log", help="new private CSV file; disabled by default") + args = parser.parse_args() + if not 1 <= args.seconds <= 86400: + parser.error("--seconds must be 1..86400") + def interrupted(signum, frame): + raise KeyboardInterrupt + signal.signal(signal.SIGTERM, interrupted) + if hasattr(signal, "SIGHUP"): + signal.signal(signal.SIGHUP, interrupted) + osc = None + try: + if args.command == "heart": + osc_message(args.address, [0]) + if args.panel: + os.environ["DISPLAY"] = ":0" + osc = Osc(args.osc) + return run_gaze(args, osc) if args.command == "gaze" else run_heart(args, osc) + except TrackingError as error: + print(str(error)) + return 1 + except KeyboardInterrupt: + return 130 + except Exception as error: + # Never dump notifications, gaze, BLE addresses or exception payloads. + print(f"Tracking stopped ({type(error).__name__}). Check the device, runtime and selected output.") + return 1 + finally: + if osc: + osc.close() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tracking-on-frame.py b/scripts/tracking-on-frame.py new file mode 100644 index 0000000..2aa39e0 --- /dev/null +++ b/scripts/tracking-on-frame.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Install or run our local-only tracking tools on the Frame. + + python3 scripts/tracking-on-frame.py install + python3 scripts/tracking-on-frame.py gaze --seconds 10 + python3 scripts/tracking-on-frame.py gaze --seconds 3600 --osc 127.0.0.1 9000 + python3 scripts/tracking-on-frame.py heart --device AA:BB:CC:DD:EE:FF --panel + +FRAME_ALIAS overrides the SSH alias (default: frame). Runs in the foreground; +Ctrl-C stops the reader. No service, autostart, sudo or SteamVR settings changes. +""" +import os +from pathlib import Path +import shlex +import subprocess +import sys +import tarfile + +REMOTE = '"$HOME/.local/share/frame-control/tracking"' +INSTALL = '''set -eu +base="$HOME/.local/share/frame-control/tracking" +mkdir -p "$base" +stage=$(mktemp -d "$base/.install.XXXXXX") +trap 'rm -rf "$stage"' EXIT + tar -xf - -C "$stage" +cc -O2 -Wall -Wextra -Werror "$stage/gaze.c" \\ + -L/opt/steamvr/bin/linuxarm64 -Wl,-rpath,/opt/steamvr/bin/linuxarm64 \\ + -lopenxr_loader -o "$stage/gaze" +chmod 700 "$stage/gaze" "$stage/tracking.py" +mv "$stage/gaze" "$stage/tracking.py" "$base/" +echo 'Installed Frame Control tracking tools (no service started).' +''' + + +def main(): + if len(sys.argv) < 2 or sys.argv[1] not in ("install", "gaze", "heart"): + print(__doc__) + return 2 + host = os.environ.get("FRAME_ALIAS", "frame") + if not host or host.startswith("-"): + raise ValueError("invalid SSH alias") + ssh = ["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", host] + if sys.argv[1] == "install": + import tempfile + source = Path(__file__).resolve().parents[1] / "frame" / "tracking" + with tempfile.TemporaryFile() as archive: + with tarfile.open(fileobj=archive, mode="w") as tar: + for name in ("gaze.c", "tracking.py"): + tar.add(source / name, arcname=name) + archive.seek(0) + return subprocess.call(ssh + ["bash -c " + shlex.quote(INSTALL)], stdin=archive) + # Allocate a tty so SSH forwards Ctrl-C and hangup to the foreground process. + command = 'exec python3 ' + REMOTE + '/tracking.py ' + shlex.join(sys.argv[1:]) + return subprocess.call(ssh[:-1] + ["-tt", host, command]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_tracking.py b/tests/test_tracking.py new file mode 100644 index 0000000..1cef2e6 --- /dev/null +++ b/tests/test_tracking.py @@ -0,0 +1,188 @@ +"""Tracking protocols and fake-Frame BlueZ lifecycle; no headset or strap needed.""" +import importlib.util +import math +import os +from pathlib import Path +import socket +import stat +import struct +import tempfile +import unittest +from unittest.mock import Mock, patch + +SPEC = importlib.util.spec_from_file_location("tracking", Path(__file__).resolve().parents[1] / "frame/tracking/tracking.py") +t = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(t) + + +class Protocols(unittest.TestCase): + def test_hrs_formats(self): + self.assertEqual(t.heart_rate(b"\x00\x48"), 72) + self.assertEqual(t.heart_rate(b"\x01\x2c\x01"), 300) + self.assertEqual(t.heart_rate(b"\x1e\x48\x01\x00\x00\x04\x00\x04"), 72) + self.assertEqual(t.heart_rate(b"\x02\x48"), 72) # contact not supported + self.assertIsNone(t.heart_rate(b"\x04\x48")) # no contact + self.assertIsNone(t.heart_rate(b"\x00\x00")) + + def test_hrs_malformed(self): + for packet in (b"", b"\x00", b"\x01\x48", b"\x08\x48\x00", b"\x10\x48", + b"\x10\x48\x00", b"\x00\x48\x01", b"\xe0\x48"): + with self.subTest(packet=packet), self.assertRaises(ValueError): + t.heart_rate(packet) + + def test_gaze_coordinates(self): + self.assertEqual(t.gaze_angles([0, 0, 0, 1]), (0, 0)) + angle = math.radians(15) + pitch, yaw = t.gaze_angles([math.sin(angle), 0, 0, math.cos(angle)]) + self.assertAlmostEqual(pitch, -30) # OpenXR +X rotation looks up + self.assertAlmostEqual(yaw, 0) + pitch, yaw = t.gaze_angles([0, -math.sin(angle), 0, math.cos(angle)]) + self.assertAlmostEqual(pitch, 0) + self.assertAlmostEqual(yaw, 30) # right + + def test_invalid_gaze(self): + for pose in ([0, 0, 0, 0], [math.nan, 0, 0, 1], [0, 0, 0], [0, 0, 0, math.inf]): + with self.assertRaises(ValueError): + t.gaze_angles(pose) + + def test_osc_wire(self): + self.assertEqual(t.osc_message('/x', [72]), b'/x\0\0,i\0\0' + struct.pack('>i', 72)) + self.assertEqual(t.osc_message('/x', [1.0, -2.0]), b'/x\0\0,ff\0' + struct.pack('>ff', 1, -2)) + for address in ('x', '/x\0y', '/x y', '/x*'): + with self.assertRaises(ValueError): + t.osc_message(address, [1]) + + def test_default_never_opens_socket(self): + with patch.object(t.socket, 'socket') as create: + osc = t.Osc() + osc.send('/x', [72]) + osc.close() + create.assert_not_called() + + def test_only_configured_endpoint(self): + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as receiver: + receiver.bind(('127.0.0.1', 0)) + receiver.settimeout(1) + osc = t.Osc(receiver.getsockname()) + try: + osc.send('/tracking/eye/CenterPitchYaw', [0.0, 30.0]) + self.assertEqual(receiver.recv(1024), t.osc_message('/tracking/eye/CenterPitchYaw', [0.0, 30.0])) + finally: + osc.close() + + def test_endpoint_validation(self): + for endpoint in [('example.org', 9000), ('0.0.0.0', 9000), ('224.0.0.1', 9000), ('127.0.0.1', 0), ('::1', 65536)]: + with self.assertRaises(ValueError): + t.Osc(endpoint) + + def test_heart_staleness_contact_and_log(self): + now = [0] + osc = Mock() + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'session.csv' + session = t.HeartSession(osc, '/hr', path, lambda: now[0]) + self.assertIsNone(session.current()) + session.notification(b'\x00\x48') + self.assertEqual(session.current(), 72) + osc.send.assert_called_once_with('/hr', [72]) + now[0] = 6 + self.assertIsNone(session.current()) + session.notification(b'\x04\x48') + self.assertIsNone(session.current()) + self.assertEqual(osc.send.call_count, 1) + session.close() + self.assertEqual(path.read_text().splitlines()[0], 'unix_seconds,bpm') + self.assertEqual(len(path.read_text().splitlines()), 2) + if os.name != 'nt': + self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o600) + with self.assertRaises(FileExistsError): + t.HeartSession(osc, '/hr', path) + + def test_no_log_by_default(self): + with patch.object(t.os, 'open') as create: + session = t.HeartSession(Mock(), '/hr') + session.notification(b'\x00\x48') + session.close() + create.assert_not_called() + + +class FakeBluez(unittest.TestCase): + def setUp(self): + self.device = '/org/bluez/hci0/dev_TEST' + self.service = self.device + '/service1' + self.char = self.service + '/char1' + self.objects = { + self.device: {t.DEVICE: {'Address': 'AA:BB:CC:DD:EE:FF', 'Connected': False, 'ServicesResolved': True}}, + self.service: {t.SERVICE: {'UUID': t.HRS, 'Device': self.device}}, + self.char: {t.CHARACTERISTIC: {'UUID': t.MEASUREMENT, 'Service': self.service, 'Flags': ['notify']}}, + } + self.api = Mock() + self.api.GetManagedObjects.side_effect = lambda: self.objects + self.bus = Mock() + self.interface = Mock(return_value=self.api) + self.values = Mock() + + def reader(self): + return t.BluezHeart(self.bus, self.interface, 'AA:BB:CC:DD:EE:FF', self.values) + + def test_subscribe_receive_and_cleanup(self): + reader = self.reader() + self.api.Connect.assert_called_once() + self.assertTrue(reader.subscribe()) + self.api.StartNotify.assert_called_once() + reader.changed(t.CHARACTERISTIC, {'Value': [0, 72]}, [], self.char) + self.values.assert_called_once_with([0, 72]) + reader.changed(t.CHARACTERISTIC, {'Value': [0, 73]}, [], '/other/strap') + self.assertEqual(self.values.call_count, 1) + reader.close() + self.api.StopNotify.assert_called_once() + self.api.Disconnect.assert_called_once() + self.bus.add_signal_receiver.return_value.remove.assert_called_once() + + def test_preserve_existing_connection(self): + self.objects[self.device][t.DEVICE]['Connected'] = True + reader = self.reader() + reader.subscribe() + reader.close() + self.api.Connect.assert_not_called() + self.api.Disconnect.assert_not_called() + + def test_only_selected_device_service(self): + self.objects[self.service][t.SERVICE]['Device'] = '/other/device' + reader = self.reader() + try: + with self.assertRaises(RuntimeError): + reader.subscribe() + finally: + reader.close() + self.api.StartNotify.assert_not_called() + + def test_wait_for_services(self): + self.objects[self.device][t.DEVICE]['ServicesResolved'] = False + reader = self.reader() + self.assertFalse(reader.subscribe()) + reader.close() + self.api.StartNotify.assert_not_called() + self.api.StopNotify.assert_not_called() + + def test_disconnect_notification(self): + reader = self.reader() + reader.changed(t.DEVICE, {'Connected': 0}, [], self.device) # dbus.Boolean behaves as int + self.values.assert_called_once_with(None) + reader.close() + + def test_failed_notify_cleans_connection(self): + reader = self.reader() + self.api.StartNotify.side_effect = RuntimeError('failure') + with self.assertRaises(RuntimeError): + reader.subscribe() + reader.close() + self.api.StopNotify.assert_not_called() + self.api.Disconnect.assert_called_once() + + def test_unknown_device_does_not_connect_or_scan(self): + self.objects.clear() + with self.assertRaises(RuntimeError): + self.reader() + self.api.Connect.assert_not_called() + self.api.StartDiscovery.assert_not_called()