#!/usr/bin/env python3 """ft-camwatch: watches SteamVR's XRService log for the camera failure that turns off the headset's upper cameras and IR light (a VCINT FPGA load that fails, usually after a wake; hands/camcheck.py), tells the Frametop desktop, and can restart SteamVR by itself. Off by default: hands/frametop-camwatch.service is a template the installer doesn't enable. Settings in ~/.config/frametop.conf: CAMWATCH_AUTO_RESTART=1 restart SteamVR on a failure (default 0: only a notification). Only while the headset isn't worn (frame-job's check: vrcompositor runs and a backlight is on), after it has been off for CAMWATCH_IDLE_S, with no Steam-launched app running and nobody on the remote desktop (VNC). At most once per failure, and not again within CAMWATCH_COOLDOWN_MIN. CAMWATCH_IDLE_S=60 how long the headset must be off first CAMWATCH_COOLDOWN_MIN=30 the least time between two automatic restarts CAMWATCH_IGNORE_APPIDS= Steam app ids that don't count as a running VR app (comma-separated) CAMWATCH_NOTIFY=1 post a notification in the Frametop desktop (0: log only) A SteamVR restart also closes the Frametop desktop and every window in it (ft-screens quits with SteamVR, and the desktop's unit doesn't restart: see hands/README.md, "Camera check"). It follows the log with a stat every 2 s (no inotify, nothing else while all is well), and logs every decision to stdout (the journal). --dry-run never notifies or restarts; --once prints the state and what it would do now, and exits. """ import argparse import json import os import subprocess import sys import time HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) import camcheck # noqa: E402 CONF = os.path.expanduser("~/.config/frametop.conf") STATE = os.path.expanduser("~/.local/state/frametop/camwatch.json") BACKLIGHTS = "/sys/class/backlight" POLL_S = 2.0 MAX_READ = 16 << 20 # a poll reads at most this much of a log that grew NOTIFY_TITLE = "Hand tracking: cameras off" NOTIFY_TEXT = ("The headset's upper cameras and IR light are off: SteamVR couldn't start the colour camera " "module (it happens sometimes after the headset sleeps). Restart SteamVR, or the headset if " "that doesn't fix it. Restarting SteamVR closes this desktop and its windows.") DEFAULTS = {"CAMWATCH_AUTO_RESTART": "0", "CAMWATCH_IDLE_S": "60", "CAMWATCH_COOLDOWN_MIN": "30", "CAMWATCH_IGNORE_APPIDS": "", "CAMWATCH_NOTIFY": "1", "VNC_PORT": "5900"} def log(text): print("%s %s" % (time.strftime("%H:%M:%S"), text), flush=True) def read_conf(path=CONF): """KEY=VALUE lines of a shell-style file (comments and quotes stripped), over DEFAULTS.""" conf = dict(DEFAULTS) try: with open(path) as f: for line in f: line = line.split("#", 1)[0].strip() if "=" not in line: continue k, v = line.split("=", 1) k, v = k.strip(), v.strip().strip("'\"") if k.replace("_", "").isalnum(): conf[k] = v except OSError: pass return conf def conf_int(conf, key): try: return int(float(conf.get(key, DEFAULTS.get(key, "0")))) except ValueError: return int(DEFAULTS.get(key, "0") or 0) # ------------------------------------------------------------------------------------------ # What's going on around (all read-only, from /proc and /sys) def process_running(name): for pid in os.listdir("/proc"): if pid.isdigit(): try: with open("/proc/%s/comm" % pid) as f: if f.read().strip() == name: return True except OSError: pass return False def headset_worn(backlights=BACKLIGHTS): """frame-job's check: vrcompositor runs and any panel's backlight is on (SteamVR turns the panels off 5 s after the headset comes off).""" lit = False try: for name in os.listdir(backlights): try: with open(os.path.join(backlights, name, "brightness")) as f: lit = lit or int(f.read().strip()) > 0 except (OSError, ValueError): pass except OSError: return False return lit and process_running("vrcompositor") def vr_apps(ignore=()): """Apps Steam launched ("SteamLaunch AppId=N" in a process's arguments, as Steam's reaper runs them): ["AppId=N ..."]. Steam's own processes, SteamVR's and Frametop's services aren't launched this way. A guess: no VR game has been run on the Frame to confirm the form.""" out = set() for pid in os.listdir("/proc"): if not pid.isdigit(): continue argv = camcheck.proc_argv(pid) if "SteamLaunch" not in argv: continue ids = [a.split("=", 1)[1] for a in argv if a.startswith("AppId=")] if ids and ids[0] not in ignore: out.add("AppId=" + ids[0]) return sorted(out) def remote_viewers(port=5900, tables=("/proc/net/tcp", "/proc/net/tcp6")): """Established connections to the remote desktop's VNC port, from /proc/net/tcp(6).""" out = [] for path in tables: try: with open(path) as f: next(f) for line in f: p = line.split() if len(p) > 3 and p[3] == "01" and int(p[1].rsplit(":", 1)[1], 16) == port: out.append(p[2]) except (OSError, StopIteration, ValueError): pass return out def frametop_bus(): """The Frametop desktop's D-Bus (from its plasmashell, as decoration/apply.sh finds it), or None.""" for pid in os.listdir("/proc"): if not pid.isdigit(): continue try: with open("/proc/%s/comm" % pid) as f: if f.read().strip() != "plasmashell": continue with open("/proc/%s/environ" % pid, "rb") as f: env = dict(kv.split(b"=", 1) for kv in f.read().split(b"\0") if b"=" in kv) except OSError: continue if env.get(b"XDG_RUNTIME_DIR", b"").endswith(b"/frametop") and b"DBUS_SESSION_BUS_ADDRESS" in env: return env[b"DBUS_SESSION_BUS_ADDRESS"].decode() return None def notify(title, text): bus = frametop_bus() if not bus: log("no Frametop desktop running: no notification") return False r = subprocess.run(["notify-send", "-a", "Frametop", "-u", "critical", "-i", "dialog-warning", title, text], env=dict(os.environ, DBUS_SESSION_BUS_ADDRESS=bus), capture_output=True, text=True, timeout=10) if r.returncode: log("notify-send failed (%d): %s" % (r.returncode, r.stderr.strip())) return r.returncode == 0 def restart_steamvr(): # --no-block: the job runs in systemd; the Frametop desktop closing doesn't cut it short. r = subprocess.run(["systemctl", "--user", "restart", "--no-block", "steamvr.service"], capture_output=True, text=True, timeout=30) log("systemctl --user restart steamvr.service: exit %d %s" % (r.returncode, (r.stderr or "").strip())) return r.returncode == 0 def environment(conf): ignore = tuple(a.strip() for a in conf.get("CAMWATCH_IGNORE_APPIDS", "").replace(",", " ").split() if a.strip()) return {"steamvr": camcheck.xrservice_pid() is not None, "worn": headset_worn(), "vr_apps": vr_apps(ignore), "remote": remote_viewers(conf_int(conf, "VNC_PORT"))} # ------------------------------------------------------------------------------------------ # The decision (pure: tests feed it made-up states) def new_memory(): return {"pending": "", "notified": [], "restarted": {}, "last_restart": 0.0, "idle_since": None, "why": ""} def decide(now, failure, env, mem, conf): """What to do now. failure: the current VCINT failure's id ("" if none: the cameras run, or they're closed); env: {"steamvr", "worn", "vr_apps", "remote"} (only looked at while a failure is current); mem: new_memory(), updated in place; now: wall-clock seconds. Returns [("log", text) | ("notify", failure) | ("restart", failure)].""" acts = [] if not failure: if mem["pending"]: acts.append(("log", "failure %s is no longer current" % mem["pending"])) mem.update(pending="", idle_since=None, why="") return acts if mem["pending"] != failure: mem.update(pending=failure, why="") acts.append(("log", "VCINT failure: %s (upper cameras and IR light off)" % failure)) if failure not in mem["notified"]: mem["notified"] = (mem["notified"] + [failure])[-20:] if conf.get("CAMWATCH_NOTIFY", "1") != "0": acts.append(("notify", failure)) if env.get("worn"): mem["idle_since"] = None elif mem["idle_since"] is None: mem["idle_since"] = now idle_s = conf_int(conf, "CAMWATCH_IDLE_S") cooldown = conf_int(conf, "CAMWATCH_COOLDOWN_MIN") * 60 if conf.get("CAMWATCH_AUTO_RESTART", "0") != "1": why = "no automatic restart (CAMWATCH_AUTO_RESTART=1 in ~/.config/frametop.conf turns it on)" elif failure in mem["restarted"]: why = "SteamVR was restarted once for this failure already: restart the headset" elif mem["last_restart"] and now - mem["last_restart"] < cooldown: why = "no restart: the last automatic one was %d min ago (cooldown %d min)" % ( (now - mem["last_restart"]) // 60, cooldown // 60) elif not env.get("steamvr"): why = "no restart: SteamVR isn't running" elif env.get("worn"): why = "no restart while the headset is worn" elif env.get("vr_apps"): why = "no restart: a VR app is running (%s)" % ", ".join(env["vr_apps"]) elif env.get("remote"): why = "no restart: someone is on the remote desktop (%s)" % ", ".join(env["remote"]) elif now - mem["idle_since"] < idle_s: why = "waiting for the headset to stay off for %d s" % idle_s else: mem["restarted"][failure] = now mem["last_restart"] = now why = "restarting SteamVR (the headset is off, nothing else in VR)" acts.append(("log", why)) acts.append(("restart", failure)) mem["why"] = why return acts if why != mem["why"]: mem["why"] = why acts.append(("log", why)) return acts def load_memory(path=STATE): mem = new_memory() try: with open(path) as f: saved = json.load(f) mem["notified"] = list(saved.get("notified", []))[-20:] mem["restarted"] = dict(saved.get("restarted", {})) mem["last_restart"] = float(saved.get("last_restart", 0.0)) except (OSError, ValueError, TypeError, AttributeError): pass return mem def save_memory(mem, path=STATE): os.makedirs(os.path.dirname(path), exist_ok=True) tmp = path + ".tmp" with open(tmp, "w") as f: json.dump({"notified": mem["notified"], "restarted": mem["restarted"], "last_restart": mem["last_restart"]}, f) os.replace(tmp, path) # ------------------------------------------------------------------------------------------ # Following the log class Follower: """The running XRService's log, read as it grows. A new log (SteamVR restarted) starts afresh.""" def __init__(self, path=None): self.fixed = path self.path, self.ino, self.offset, self.rest = "", None, 0, "" self.state = None def poll(self): path = self.fixed or camcheck.newest_log() if not path: return None try: st = os.stat(path) except OSError: return self.state if path != self.path or st.st_ino != self.ino or st.st_size < self.offset: self.path, self.ino, self.offset, self.rest = path, st.st_ino, 0, "" self.state = camcheck.LogState(path) log("following %s" % path) if st.st_size > self.offset: with open(path, "rb") as f: f.seek(self.offset) data = f.read(MAX_READ) self.offset += len(data) text = self.rest + data.decode(errors="replace") lines = text.split("\n") self.rest = lines.pop() for line in lines: self.state.feed(line) return self.state def current_failure(state, steamvr_running=True): """The failure id if the log's current camera start is a VCINT failure, else "".""" if state is None or not steamvr_running: return "" snap = state.snapshot() return snap["failure"] if snap["status"] == "degraded" and snap["reason"] == camcheck.VCINT_REASON else "" def act(acts, dry): changed = False for kind, arg in acts: if kind == "log": log(arg) elif kind == "notify": changed = True if dry: log("(dry run) would notify: %s" % NOTIFY_TITLE) else: notify(NOTIFY_TITLE, NOTIFY_TEXT) elif kind == "restart": changed = True if dry: log("(dry run) would restart SteamVR") else: restart_steamvr() return changed def main(argv=None): ap = argparse.ArgumentParser(description="Watch for the camera failure that turns off the upper cameras.") ap.add_argument("--dry-run", action="store_true", help="log what it would do; never notify or restart") ap.add_argument("--once", action="store_true", help="look once, print the decision (a dry run), and exit") ap.add_argument("--log", help="follow this file instead of the running XRService's log (tests)") ap.add_argument("--state", default=STATE, help="where it remembers past failures (default %(default)s)") a = ap.parse_args(argv) dry = a.dry_run or a.once mem = load_memory(a.state) follower = Follower(a.log) conf = read_conf() conf_mtime = 0.0 log("watching; automatic restart %s" % ("on" if conf.get("CAMWATCH_AUTO_RESTART") == "1" else "off")) while True: try: m = os.stat(CONF).st_mtime except OSError: m = 0.0 if m != conf_mtime: conf, conf_mtime = read_conf(), m state = follower.poll() failure = current_failure(state) env = {} if failure or mem["pending"]: env = environment(conf) if not env["steamvr"] and not a.log: failure = "" # the log's last word, but XRService is gone acts = decide(time.time(), failure, env, mem, conf) if a.once: snap = state.snapshot() if state else {"status": "unknown", "reason": "no log"} print("log: %s\nstate: %s %s\nenvironment: %s" % (follower.path, snap["status"], snap["reason"], json.dumps(env))) act(acts, True) return 0 if act(acts, dry) and not dry: save_memory(mem, a.state) time.sleep(POLL_S) if __name__ == "__main__": try: sys.exit(main()) except KeyboardInterrupt: sys.exit(0)