Jellyfin: hardware video decoding (jellyfin.hardwareDecoding)

Jellyfin Desktop hard-sets mpv's hwdec=auto-copy, whose probe list leaves
out V4L2 M2M, and the Flatpak can't see the Frame's V4L2 decoder. An
LD_PRELOAD shim (preloaded straight from the store, only that path exposed
read-only) rewrites hwdec to v4l2m2m-copy,auto-copy; devices=all makes the
decoder visible. Overrides via nix-flatpak, or a home-manager-owned override
file without it.

modules/jellyfin.nix

@@ -0,0 +1,74 @@
+# Hardware video decoding for the Jellyfin Desktop Flatpak
+# (org.jellyfin.JellyfinDesktop; installing it is up to the user).
+# The Frame's decoder is a V4L2 M2M device (qcom-iris, /dev/video*): the
+# Flatpak's devices=dri doesn't expose it (nothing finer than devices=all
+# exists), and Jellyfin hard-sets mpv's hwdec=auto-copy, whose probe list
+# leaves out V4L2 M2M, without a way to pass mpv options. So:
+# - an LD_PRELOAD shim (jellyfin/mpv-hwdec-shim.c) rewrites an "auto*" hwdec
+#   to $SFN_MPV_HWDEC. Preloaded from the store: the override exposes just
+#   its store path (read-only) to the sandbox.
+# - override: devices=all, that filesystem, LD_PRELOAD, SFN_MPV_HWDEC.
+#   Through nix-flatpak's services.flatpak.overrides if it is imported and
+#   enabled (merged with the user's own overrides there); else home-manager
+#   owns the app's override file.
+{ config, options, lib, pkgs, ... }:
+let
+  cfg = config.steamFrame.jellyfin.hardwareDecoding;
+  app = "org.jellyfin.JellyfinDesktop";
+
+  shim = pkgs.callPackage ./jellyfin/shim.nix { };
+
+  override = {
+    Context = { devices = [ "all" ]; filesystems = [ "${shim}:ro" ]; };
+    Environment = { LD_PRELOAD = "${shim}/lib/mpv-hwdec-shim.so"; SFN_MPV_HWDEC = cfg.hwdec; };
+  };
+
+  useNixFlatpak = options ? services.flatpak.overrides && config.services.flatpak.enable;
+
+  # Flatpak's keyfile format (lists as "a;b;").
+  overrideFile = pkgs.writeText "${app}-override" (lib.concatStringsSep "\n" (lib.mapAttrsToList
+    (section: entries: "[${section}]\n" + lib.concatStrings (lib.mapAttrsToList (key: value:
+      "${key}=${if lib.isList value then lib.concatMapStrings (v: "${v};") value else value}\n") entries))
+    override));
+in {
+  options.steamFrame.jellyfin.hardwareDecoding = {
+    enable = lib.mkEnableOption ''
+      hardware video decoding (the Frame's V4L2 decoder) in the Jellyfin
+      Desktop Flatpak: device access for the sandbox (devices=all) and an
+      LD_PRELOAD shim that makes mpv try hwdec's value below. Without
+      nix-flatpak, home-manager owns the app's Flatpak override file. Takes
+      effect at the next start of Jellyfin'';
+    hwdec = lib.mkOption {
+      type = lib.types.strMatching "[^[:space:]]+";
+      default = "v4l2m2m-copy,auto-copy";
+      example = "v4l2m2m-copy";
+      description = ''
+        mpv hwdec value used instead of Jellyfin's automatic one ("auto*");
+        tried in order, software decoding if none handles the stream.
+        Set as SFN_MPV_HWDEC in the Flatpak's environment.
+      '';
+    };
+  };
+
+  config = lib.mkMerge [
+    {
+      # Remove the shim copy of earlier versions (copied into the app's data
+      # dir, marked in $XDG_STATE_HOME/steam-frame-nix).
+      home.activation.steamFrameJellyfinHwdec = lib.hm.dag.entryAfter [ "writeBoundary" ] ''
+        marker="''${XDG_STATE_HOME:-$HOME/.local/state}/steam-frame-nix/jellyfin-hwdec-shim"
+        if [ -e "$marker" ]; then
+          run rm -f "$HOME/.var/app/${app}/mpv-hwdec-shim.so" "$marker"
+          run rmdir --ignore-fail-on-non-empty "''${marker%/*}"
+        fi
+      '';
+    }
+    (lib.mkIf (cfg.enable && !useNixFlatpak) {
+      # force: `flatpak override --user` replaces the link with a file.
+      xdg.dataFile."flatpak/overrides/${app}" = { source = overrideFile; force = true; };
+    })
+    # Only if nix-flatpak is imported: the option doesn't exist otherwise.
+    (lib.optionalAttrs (options ? services.flatpak.overrides) {
+      services.flatpak.overrides = lib.mkIf (cfg.enable && useNixFlatpak) { ${app} = override; };
+    })
+  ];
+}

modules/jellyfin/check.nix

@@ -0,0 +1,70 @@
+# Checks of the Jellyfin hardware decoding pieces: the mpv hwdec shim's ELF
+# (libc only, no rpath, GLIBC <= 2.34) and rewriting (against a fake libmpv
+# that prints what it receives).
+{ pkgs }:
+let
+  shim = pkgs.callPackage ./shim.nix { };
+in
+pkgs.runCommandCC "jellyfin-check" { nativeBuildInputs = [ pkgs.binutils ]; } ''
+  so=${shim}/lib/mpv-hwdec-shim.so
+  fail() { echo "FAIL: $*" >&2; exit 1; }
+
+  needed=$(readelf -dW $so | sed -n 's/.*(NEEDED).*\[\(.*\)\]/\1/p')
+  for lib in $needed; do
+    case $lib in libc.so.6|ld-linux*) ;; *) fail "needs $lib" ;; esac
+  done
+  ! readelf -dW $so | grep -Eq '\((RPATH|RUNPATH)\)' || fail "has an rpath"
+  glibc=$(readelf -VW $so | grep -o 'GLIBC_[0-9.]*' | sort -uV | tail -1)
+  [ "$(printf '%s\n' "$glibc" GLIBC_2.34 | sort -V | tail -1)" = GLIBC_2.34 ] ||
+    fail "needs $glibc"
+  echo "ELF ok: NEEDED" $needed", newest $glibc"
+
+  cat > mpv.c <<'C'
+  #include <stdio.h>
+  #include <stdint.h>
+  typedef struct { union { char *string; } u; int format; } mpv_node;
+  static void show(const char *n, int f, void *d) {
+    printf("%s=%s\n", n, f == 1 ? *(char **)d : ((mpv_node *)d)->u.string);
+  }
+  int mpv_set_property(void *c, const char *n, int f, void *d) { show(n, f, d); return 0; }
+  int mpv_set_option(void *c, const char *n, int f, void *d) { show(n, f, d); return 0; }
+  int mpv_set_property_async(void *c, uint64_t u, const char *n, int f, void *d) { show(n, f, d); return 0; }
+  int mpv_set_property_string(void *c, const char *n, const char *d) { printf("%s=%s\n", n, d); return 0; }
+  int mpv_set_option_string(void *c, const char *n, const char *d) { printf("%s=%s\n", n, d); return 0; }
+  C
+  cat > app.c <<'C'
+  #include <stdint.h>
+  typedef struct { union { char *string; } u; int format; } mpv_node;
+  int mpv_set_property(void *, const char *, int, void *);
+  int mpv_set_option(void *, const char *, int, void *);
+  int mpv_set_property_async(void *, uint64_t, const char *, int, void *);
+  int mpv_set_property_string(void *, const char *, const char *);
+  int mpv_set_option_string(void *, const char *, const char *);
+  int main(void) {
+    char *s = "auto-copy", *no = "no";
+    mpv_node node = { { "auto" }, 1 };
+    mpv_set_property_string(0, "hwdec", "auto-copy");
+    mpv_set_option_string(0, "hwdec", "auto");
+    mpv_set_property(0, "hwdec", 1, &s);
+    mpv_set_option(0, "hwdec", 6, &node);
+    mpv_set_property_async(0, 0, "hwdec", 1, &s);
+    mpv_set_property(0, "hwdec", 1, &no);
+    mpv_set_property_string(0, "vo", "auto");
+    return 0;
+  }
+  C
+  $CC -shared -fPIC -o libmpv.so mpv.c
+  $CC -o app app.c -L. -lmpv -Wl,-rpath,$PWD
+
+  h=v4l2m2m-copy,auto-copy
+  expect="hwdec=$h hwdec=$h hwdec=$h hwdec=$h hwdec=$h hwdec=no vo=auto"
+  got=$(LD_PRELOAD=$so ./app 2> err)
+  [ "$(echo $got)" = "$expect" ] || fail "got: $(echo $got)"
+  [ "$(grep -c mpv-hwdec-shim: err)" = 1 ] || fail "logged: $(cat err)"
+  got=$(SFN_MPV_HWDEC=vaapi LD_PRELOAD=$so ./app 2> /dev/null)
+  [ "$(echo $got)" = "$(echo "$expect" | sed "s/$h/vaapi/g")" ] || fail "SFN_MPV_HWDEC: $(echo $got)"
+  # Processes without libmpv (preloaded all over the sandbox) are unaffected.
+  [ "$(LD_PRELOAD=$so ${pkgs.coreutils}/bin/echo ok)" = ok ] || fail "without libmpv"
+  echo "rewrite ok"
+  touch $out
+''

modules/jellyfin/mpv-hwdec-shim.c

@@ -0,0 +1,96 @@
+// LD_PRELOAD shim for libmpv apps that set hwdec to an "auto*" value, such as
+// Jellyfin Desktop (hwdec=auto-copy). mpv's auto probe list leaves out V4L2
+// M2M decoders, so the Steam Frame's hardware decoder (qcom-iris) is never
+// tried. An "auto*" hwdec is rewritten to $SFN_MPV_HWDEC (default below); an
+// explicit value such as "no" is left alone. mpv falls back to software
+// decoding per stream if the decoder can't handle it.
+//
+// The preload applies to every process in the Flatpak sandbox; the wrappers
+// only run when a process calls libmpv, and need nothing but libc.
+#define _GNU_SOURCE
+#include <dlfcn.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define DEFAULT_HWDEC "v4l2m2m-copy,auto-copy"
+
+// From mpv/client.h (stable ABI).
+typedef struct mpv_handle mpv_handle;
+enum { MPV_FORMAT_STRING = 1, MPV_FORMAT_NODE = 6 };
+enum { MPV_ERROR_GENERIC = -20 };
+typedef struct mpv_node {
+  union { char *string; int flag; int64_t int64; double double_; void *list; void *ba; } u;
+  int format;
+} mpv_node;
+
+// The replacement for an automatic hwdec value, or NULL to keep `value`.
+static const char *hwdec_for(const char *name, const char *value) {
+  if (!name || !value || strcmp(name, "hwdec") != 0 || strncmp(value, "auto", 4) != 0) return NULL;
+  const char *v = getenv("SFN_MPV_HWDEC");
+  if (!v || !*v) v = DEFAULT_HWDEC;
+  static int logged;
+  if (!__atomic_exchange_n(&logged, 1, __ATOMIC_RELAXED))
+    fprintf(stderr, "mpv-hwdec-shim: hwdec \"%s\" -> \"%s\"\n", value, v);
+  return v;
+}
+
+// `data` of an mpv_set_* call in `format`, with a rewritten hwdec in *node or
+// *str if applicable.
+static void *rewrite(const char *name, int format, void *data, mpv_node *node, const char **str) {
+  const char *v;
+  if (!data) return data;
+  if (format == MPV_FORMAT_STRING && (v = hwdec_for(name, *(char **)data))) {
+    *str = v;
+    return str;
+  }
+  if (format == MPV_FORMAT_NODE && ((mpv_node *)data)->format == MPV_FORMAT_STRING &&
+      (v = hwdec_for(name, ((mpv_node *)data)->u.string))) {
+    node->format = MPV_FORMAT_STRING;
+    node->u.string = (char *)v;
+    return node;
+  }
+  return data;
+}
+
+// The real function (libmpv's), looked up once; returns MPV_ERROR_GENERIC if
+// it can't be found (no libmpv in the process: then nothing calls us anyway).
+#define NEXT(fn)                                                     \
+  static __typeof__(fn) *next;                                       \
+  __typeof__(fn) *real = __atomic_load_n(&next, __ATOMIC_RELAXED);   \
+  if (!real) {                                                       \
+    real = (__typeof__(fn) *)dlsym(RTLD_NEXT, #fn);                  \
+    if (!real) return MPV_ERROR_GENERIC;                             \
+    __atomic_store_n(&next, real, __ATOMIC_RELAXED);                 \
+  }
+
+int mpv_set_property(mpv_handle *ctx, const char *name, int format, void *data) {
+  NEXT(mpv_set_property);
+  mpv_node n; const char *s;
+  return real(ctx, name, format, rewrite(name, format, data, &n, &s));
+}
+
+int mpv_set_option(mpv_handle *ctx, const char *name, int format, void *data) {
+  NEXT(mpv_set_option);
+  mpv_node n; const char *s;
+  return real(ctx, name, format, rewrite(name, format, data, &n, &s));
+}
+
+int mpv_set_property_async(mpv_handle *ctx, uint64_t ud, const char *name, int format, void *data) {
+  NEXT(mpv_set_property_async);
+  mpv_node n; const char *s;
+  return real(ctx, ud, name, format, rewrite(name, format, data, &n, &s));
+}
+
+int mpv_set_property_string(mpv_handle *ctx, const char *name, const char *data) {
+  NEXT(mpv_set_property_string);
+  const char *v = hwdec_for(name, data);
+  return real(ctx, name, v ? v : data);
+}
+
+int mpv_set_option_string(mpv_handle *ctx, const char *name, const char *data) {
+  NEXT(mpv_set_option_string);
+  const char *v = hwdec_for(name, data);
+  return real(ctx, name, v ? v : data);
+}

modules/jellyfin/shim.nix

@@ -0,0 +1,9 @@
+# The mpv hwdec shim (mpv-hwdec-shim.c) as lib/mpv-hwdec-shim.so. It runs
+# against the Flatpak runtime's glibc, so it links only libc (no rpath) and
+# must not need symbols newer than GLIBC_2.34 (dlsym's version; see check.nix).
+{ runCommandCC, patchelf }:
+runCommandCC "mpv-hwdec-shim" { nativeBuildInputs = [ patchelf ]; } ''
+  mkdir -p $out/lib
+  $CC -shared -fPIC -O2 -Wall -Wextra -Werror -o $out/lib/mpv-hwdec-shim.so ${./mpv-hwdec-shim.c}
+  patchelf --remove-rpath $out/lib/mpv-hwdec-shim.so
+''