Jellyfin: hardware video decoding (jellyfin.hardwareDecoding)

Jellyfin Desktop hard-sets mpv's hwdec=auto-copy, whose probe list leaves
out V4L2 M2M, and the Flatpak can't see the Frame's V4L2 decoder. An
LD_PRELOAD shim (preloaded straight from the store, only that path exposed
read-only) rewrites hwdec to v4l2m2m-copy,auto-copy; devices=all makes the
decoder visible. Overrides via nix-flatpak, or a home-manager-owned override
file without it.

modules/jellyfin/check.nix

@@ -0,0 +1,70 @@
+# Checks of the Jellyfin hardware decoding pieces: the mpv hwdec shim's ELF
+# (libc only, no rpath, GLIBC <= 2.34) and rewriting (against a fake libmpv
+# that prints what it receives).
+{ pkgs }:
+let
+  shim = pkgs.callPackage ./shim.nix { };
+in
+pkgs.runCommandCC "jellyfin-check" { nativeBuildInputs = [ pkgs.binutils ]; } ''
+  so=${shim}/lib/mpv-hwdec-shim.so
+  fail() { echo "FAIL: $*" >&2; exit 1; }
+
+  needed=$(readelf -dW $so | sed -n 's/.*(NEEDED).*\[\(.*\)\]/\1/p')
+  for lib in $needed; do
+    case $lib in libc.so.6|ld-linux*) ;; *) fail "needs $lib" ;; esac
+  done
+  ! readelf -dW $so | grep -Eq '\((RPATH|RUNPATH)\)' || fail "has an rpath"
+  glibc=$(readelf -VW $so | grep -o 'GLIBC_[0-9.]*' | sort -uV | tail -1)
+  [ "$(printf '%s\n' "$glibc" GLIBC_2.34 | sort -V | tail -1)" = GLIBC_2.34 ] ||
+    fail "needs $glibc"
+  echo "ELF ok: NEEDED" $needed", newest $glibc"
+
+  cat > mpv.c <<'C'
+  #include <stdio.h>
+  #include <stdint.h>
+  typedef struct { union { char *string; } u; int format; } mpv_node;
+  static void show(const char *n, int f, void *d) {
+    printf("%s=%s\n", n, f == 1 ? *(char **)d : ((mpv_node *)d)->u.string);
+  }
+  int mpv_set_property(void *c, const char *n, int f, void *d) { show(n, f, d); return 0; }
+  int mpv_set_option(void *c, const char *n, int f, void *d) { show(n, f, d); return 0; }
+  int mpv_set_property_async(void *c, uint64_t u, const char *n, int f, void *d) { show(n, f, d); return 0; }
+  int mpv_set_property_string(void *c, const char *n, const char *d) { printf("%s=%s\n", n, d); return 0; }
+  int mpv_set_option_string(void *c, const char *n, const char *d) { printf("%s=%s\n", n, d); return 0; }
+  C
+  cat > app.c <<'C'
+  #include <stdint.h>
+  typedef struct { union { char *string; } u; int format; } mpv_node;
+  int mpv_set_property(void *, const char *, int, void *);
+  int mpv_set_option(void *, const char *, int, void *);
+  int mpv_set_property_async(void *, uint64_t, const char *, int, void *);
+  int mpv_set_property_string(void *, const char *, const char *);
+  int mpv_set_option_string(void *, const char *, const char *);
+  int main(void) {
+    char *s = "auto-copy", *no = "no";
+    mpv_node node = { { "auto" }, 1 };
+    mpv_set_property_string(0, "hwdec", "auto-copy");
+    mpv_set_option_string(0, "hwdec", "auto");
+    mpv_set_property(0, "hwdec", 1, &s);
+    mpv_set_option(0, "hwdec", 6, &node);
+    mpv_set_property_async(0, 0, "hwdec", 1, &s);
+    mpv_set_property(0, "hwdec", 1, &no);
+    mpv_set_property_string(0, "vo", "auto");
+    return 0;
+  }
+  C
+  $CC -shared -fPIC -o libmpv.so mpv.c
+  $CC -o app app.c -L. -lmpv -Wl,-rpath,$PWD
+
+  h=v4l2m2m-copy,auto-copy
+  expect="hwdec=$h hwdec=$h hwdec=$h hwdec=$h hwdec=$h hwdec=no vo=auto"
+  got=$(LD_PRELOAD=$so ./app 2> err)
+  [ "$(echo $got)" = "$expect" ] || fail "got: $(echo $got)"
+  [ "$(grep -c mpv-hwdec-shim: err)" = 1 ] || fail "logged: $(cat err)"
+  got=$(SFN_MPV_HWDEC=vaapi LD_PRELOAD=$so ./app 2> /dev/null)
+  [ "$(echo $got)" = "$(echo "$expect" | sed "s/$h/vaapi/g")" ] || fail "SFN_MPV_HWDEC: $(echo $got)"
+  # Processes without libmpv (preloaded all over the sandbox) are unaffected.
+  [ "$(LD_PRELOAD=$so ${pkgs.coreutils}/bin/echo ok)" = ok ] || fail "without libmpv"
+  echo "rewrite ok"
+  touch $out
+''

modules/jellyfin/mpv-hwdec-shim.c

@@ -0,0 +1,96 @@
+// LD_PRELOAD shim for libmpv apps that set hwdec to an "auto*" value, such as
+// Jellyfin Desktop (hwdec=auto-copy). mpv's auto probe list leaves out V4L2
+// M2M decoders, so the Steam Frame's hardware decoder (qcom-iris) is never
+// tried. An "auto*" hwdec is rewritten to $SFN_MPV_HWDEC (default below); an
+// explicit value such as "no" is left alone. mpv falls back to software
+// decoding per stream if the decoder can't handle it.
+//
+// The preload applies to every process in the Flatpak sandbox; the wrappers
+// only run when a process calls libmpv, and need nothing but libc.
+#define _GNU_SOURCE
+#include <dlfcn.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define DEFAULT_HWDEC "v4l2m2m-copy,auto-copy"
+
+// From mpv/client.h (stable ABI).
+typedef struct mpv_handle mpv_handle;
+enum { MPV_FORMAT_STRING = 1, MPV_FORMAT_NODE = 6 };
+enum { MPV_ERROR_GENERIC = -20 };
+typedef struct mpv_node {
+  union { char *string; int flag; int64_t int64; double double_; void *list; void *ba; } u;
+  int format;
+} mpv_node;
+
+// The replacement for an automatic hwdec value, or NULL to keep `value`.
+static const char *hwdec_for(const char *name, const char *value) {
+  if (!name || !value || strcmp(name, "hwdec") != 0 || strncmp(value, "auto", 4) != 0) return NULL;
+  const char *v = getenv("SFN_MPV_HWDEC");
+  if (!v || !*v) v = DEFAULT_HWDEC;
+  static int logged;
+  if (!__atomic_exchange_n(&logged, 1, __ATOMIC_RELAXED))
+    fprintf(stderr, "mpv-hwdec-shim: hwdec \"%s\" -> \"%s\"\n", value, v);
+  return v;
+}
+
+// `data` of an mpv_set_* call in `format`, with a rewritten hwdec in *node or
+// *str if applicable.
+static void *rewrite(const char *name, int format, void *data, mpv_node *node, const char **str) {
+  const char *v;
+  if (!data) return data;
+  if (format == MPV_FORMAT_STRING && (v = hwdec_for(name, *(char **)data))) {
+    *str = v;
+    return str;
+  }
+  if (format == MPV_FORMAT_NODE && ((mpv_node *)data)->format == MPV_FORMAT_STRING &&
+      (v = hwdec_for(name, ((mpv_node *)data)->u.string))) {
+    node->format = MPV_FORMAT_STRING;
+    node->u.string = (char *)v;
+    return node;
+  }
+  return data;
+}
+
+// The real function (libmpv's), looked up once; returns MPV_ERROR_GENERIC if
+// it can't be found (no libmpv in the process: then nothing calls us anyway).
+#define NEXT(fn)                                                     \
+  static __typeof__(fn) *next;                                       \
+  __typeof__(fn) *real = __atomic_load_n(&next, __ATOMIC_RELAXED);   \
+  if (!real) {                                                       \
+    real = (__typeof__(fn) *)dlsym(RTLD_NEXT, #fn);                  \
+    if (!real) return MPV_ERROR_GENERIC;                             \
+    __atomic_store_n(&next, real, __ATOMIC_RELAXED);                 \
+  }
+
+int mpv_set_property(mpv_handle *ctx, const char *name, int format, void *data) {
+  NEXT(mpv_set_property);
+  mpv_node n; const char *s;
+  return real(ctx, name, format, rewrite(name, format, data, &n, &s));
+}
+
+int mpv_set_option(mpv_handle *ctx, const char *name, int format, void *data) {
+  NEXT(mpv_set_option);
+  mpv_node n; const char *s;
+  return real(ctx, name, format, rewrite(name, format, data, &n, &s));
+}
+
+int mpv_set_property_async(mpv_handle *ctx, uint64_t ud, const char *name, int format, void *data) {
+  NEXT(mpv_set_property_async);
+  mpv_node n; const char *s;
+  return real(ctx, ud, name, format, rewrite(name, format, data, &n, &s));
+}
+
+int mpv_set_property_string(mpv_handle *ctx, const char *name, const char *data) {
+  NEXT(mpv_set_property_string);
+  const char *v = hwdec_for(name, data);
+  return real(ctx, name, v ? v : data);
+}
+
+int mpv_set_option_string(mpv_handle *ctx, const char *name, const char *data) {
+  NEXT(mpv_set_option_string);
+  const char *v = hwdec_for(name, data);
+  return real(ctx, name, v ? v : data);
+}

modules/jellyfin/shim.nix

@@ -0,0 +1,9 @@
+# The mpv hwdec shim (mpv-hwdec-shim.c) as lib/mpv-hwdec-shim.so. It runs
+# against the Flatpak runtime's glibc, so it links only libc (no rpath) and
+# must not need symbols newer than GLIBC_2.34 (dlsym's version; see check.nix).
+{ runCommandCC, patchelf }:
+runCommandCC "mpv-hwdec-shim" { nativeBuildInputs = [ patchelf ]; } ''
+  mkdir -p $out/lib
+  $CC -shared -fPIC -O2 -Wall -Wextra -Werror -o $out/lib/mpv-hwdec-shim.so ${./mpv-hwdec-shim.c}
+  patchelf --remove-rpath $out/lib/mpv-hwdec-shim.so
+''