Uninstall: wait until nothing uses /nix before removing Nix

nix-installer uninstall failed at `systemctl stop nix.mount` ("Job
failed") while programs started from the Nix store still ran: a terminal
session's tools, or apps that mapped Home Manager's mime.cache.

- Before nix-installer runs, after Home Manager is gone, uninstall scans
  /proc for the user's processes whose exe, cwd, root, an fd or a mapped
  file is in /nix, and lists them by name and PID (the shells it was
  started from included; itself, its subshells and its curl | bash
  pipeline skipped). Close them and press Enter to re-check, or abort;
  with --yes or without a terminal it stops: "close these or reboot, then
  run uninstall again". Nothing is killed (replaces the old kill prompt).
- nix-installer runs from a copy outside /nix.
- cleanup check H: a fake /proc (STEAM_FRAME_NIX_PROC) with programs from
  the store stops before Nix; without them Nix is removed.

README.md

@@ -501,8 +501,10 @@ stops Home Manager's user services (reverting the UI patches), runs
 `cleanup --all`, runs `home-manager uninstall`, then removes Nix and the
 per-user Nix state (see [Set up by install.sh](#set-up-by-installsh)). If
 SteamVR is running, its key is restored when SteamVR stops (the closing
-message says so). If Nix's own uninstaller fails (e.g. something still uses
+message says so). Programs started from the Nix store keep `/nix` busy:
-`/nix`), the rest still runs (the `~/.config/home-manager` link goes) except
+before removing Nix, `uninstall` lists them and waits until you close them
+(with `--yes`, it stops instead); close them or run it right after a reboot.
+If Nix's own uninstaller fails anyway, the rest still runs (the `~/.config/home-manager` link goes) except
 the per-user Nix state, and the closing message says to reboot and run
 `uninstall` again. Your configuration, `*.hm-backup-*` files, app data and
 Flatpaks stay.

docs/cleanup.md

@@ -44,3 +44,18 @@ What `install.sh install` sets up is listed in the README under
 `systemctl --user` against the outer session's user manager (the nested
 desktop can't reach it with its own environment) and uses the installed
 `home-manager` if there is one, else Home Manager's `master`.
+
+`uninstall` removes Nix with nix-installer, which fails when it can't
+unmount `/nix` (`systemctl stop nix.mount`). Before that it scans `/proc`
+for processes using `/nix`: their `exe`, `cwd`, `root` or an `fd` links into
+`/nix`, or `maps` names a file there (e.g. an app started before the
+uninstall that mapped Home Manager's `mime.cache`). Only the user's own
+processes are readable; the Nix daemon is nix-installer's to stop. Skipped:
+the script itself, its subshells (descendants) and its process group (the
+`curl | bash` pipeline). Its ancestors are listed with a hint to run
+`uninstall` from another terminal. Nothing is killed: it asks to close them
+and re-checks on Enter, or (`--yes`, no terminal) stops before Nix. A bash
+from the Nix store re-executes the script with `/usr/bin/bash` first, and
+nix-installer runs from a copy outside `/nix`. The check uses
+`STEAM_FRAME_NIX_PROC` as a fake `/proc` (section H of
+`modules/cleanup/check.nix`).

docs/development.md

@@ -49,7 +49,7 @@ modules/
   cleanup.nix                  switch: `cleanup --orphans`; steam-frame-nix-cleanup on PATH
   cleanup/package.nix          build: install.sh as a command (cleanup, steamvr-debugger-arm, restart-check)
   cleanup/check.nix            test: install.sh cleanup and restart-check on fake home/runtime dirs,
-                               install --clone against local bare repos
+                               install --clone against local bare repos, uninstall on a fake /proc
   portal.nix                   Steam session portal config (session.portalFix)
   portal/check.nix             test: KDE FileChooser on by default, absent when disabled
   applications-menu.nix        applications.menu link for KDE apps (session.applicationsMenu)

install.sh

@@ -27,6 +27,7 @@ USER_UNIT_DIR="$CONFIG_HOME/systemd/user"
 USER_NAME="$(id -un)"
 USER_ID="$(id -u)"
 OUTER_RUNTIME_DIR="${STEAM_FRAME_NIX_RUNTIME_DIR:-/run/user/$USER_ID}"
+SFN_PROC="${STEAM_FRAME_NIX_PROC:-/proc}"   # tests: a fake /proc
 
 ASSUME_YES=0
 RO_RELOCK=0
@@ -69,8 +70,9 @@ Commands:
   uninstall [--yes] [--keep-nix]
       Stop Home Manager's user services, run 'cleanup --all', uninstall Home
       Manager, uninstall Nix (unless --keep-nix) and remove per-user Nix
-      leftovers. Your configuration directory (also a --clone) is never
+      leftovers. Before removing Nix it lists the programs started from the
-      deleted.
+      Nix store that still use /nix and waits until they are closed. Your
+      configuration directory (also a --clone) is never deleted.
 
   cleanup [--dry-run] [--quiet] (--all | --orphans [--keep <artifact>]...)
       Remove what steam-frame-nix (any version) wrote outside the Nix store,
@@ -232,11 +234,17 @@ install_nix() {
 }
 
 uninstall_nix() {
-  local rc=0
+  local rc=0 bin
   need_sudo
   ro_unlock
   step "Uninstalling Nix"
-  sudo "$NIX_INSTALLER_BIN" uninstall --no-confirm || rc=$?
+  # From a copy: the uninstaller running from /nix would keep /nix busy.
+  bin="$(mktemp)"
+  if ! { cp "$NIX_INSTALLER_BIN" "$bin" && chmod +x "$bin" && "$bin" --version >/dev/null 2>&1; }; then
+    rm -f "$bin"; bin=$NIX_INSTALLER_BIN
+  fi
+  sudo "$bin" uninstall --no-confirm || rc=$?
+  [[ $bin == "$NIX_INSTALLER_BIN" ]] || rm -f "$bin"
   ro_relock
   return "$rc"
 }
@@ -1018,7 +1026,6 @@ cmd_cleanup() {
 # runtime (gamescope sends its own keymap to Xwayland again, e.g. whenever
 # the VR keyboard types, and only a SteamVR start opens the port).
 
-SFN_PROC="${STEAM_FRAME_NIX_PROC:-/proc}"
 SFN_CGROUP="${STEAM_FRAME_NIX_CGROUP:-/sys/fs/cgroup}"
 LAYOUT_DROPIN="$USER_UNIT_DIR/gamescope-session.service.d/keyboard.conf"
 DEBUGGER_HM_DROPIN="$USER_UNIT_DIR/steamvr.service.d/webhelper-debugger.conf"
@@ -1139,39 +1146,70 @@ remove_hm() {
   fi
 }
 
-# Processes still running from /nix/store would keep /nix busy.
+# --- /nix in use ---
-stop_nix_processes() {
+#
-  local skip=" " p=$$ pid exe pids=()
+# nix-installer can't unmount /nix while a process uses it: its program, a
-  while [[ -n $p && $p -gt 1 ]]; do
+# library, an open file or its working directory in /nix. Only this user's
-    skip+="$p "
+# processes are readable (the uninstaller stops the Nix daemon itself). This
-    p="$(awk '/^PPid:/ {print $2}' "/proc/$p/status" 2>/dev/null || true)"
+# script, its subshells and its curl | bash pipeline are skipped; the
-  done
+# shells and apps it was started from are not.
-  for pid in /proc/[0-9]*; do
-    pid=${pid#/proc/}
-    [[ -O /proc/$pid && $skip != *" $pid "* ]] || continue
-    exe="$(readlink "/proc/$pid/exe" 2>/dev/null || true)"
-    [[ $exe == /nix/store/* ]] && pids+=("$pid")
-  done
-  (( ${#pids[@]} )) || return 0
 
-  step "Processes still running from /nix"
+proc_ppid() { # pid
-  ps -o pid=,args= -p "$(IFS=,; echo "${pids[*]}")" | cut -c1-120 | sed 's/^/    /' || true
+  local k v
-  if confirm "Stop them (Nix can't be unmounted while they run)?"; then
+  { while read -r k v; do [[ $k == PPid: ]] && { echo "$v"; return 0; }; done <"$SFN_PROC/$1/status"; } 2>/dev/null
-    kill -TERM "${pids[@]}" 2>/dev/null || true
-    sleep 2
-    for pid in "${pids[@]}"; do    # only if it's still the same /nix process
-      [[ $(readlink "/proc/$pid/exe" 2>/dev/null || true) == /nix/store/* ]] && kill -KILL "$pid" 2>/dev/null || true
-    done
-  else
-    warn "left running; the Nix uninstall may fail to unmount /nix"
-  fi
-  for p in $skip; do
-    [[ $(readlink "/proc/$p/exe" 2>/dev/null || true) == /nix/store/* ]] \
-      && warn "process $p (a parent shell) runs from /nix; if the uninstall can't unmount /nix, re-run from a terminal whose shell is /usr/bin/bash"
-  done
   return 0
 }
 
+proc_pgid() { # pid
+  local s
+  { read -r s <"$SFN_PROC/$1/stat"; } 2>/dev/null || return 0
+  s=${s##*) }               # after "pid (comm) "; comm may contain spaces
+  read -r _ _ s _ <<<"$s"   # state ppid pgrp
+  echo "$s"
+}
+
+# "<name> (PID <pid>)" per process using /nix.
+nix_users() {
+  local p pid name argv0 self=$$ pgid ancestors=" "
+  p=$self
+  while [[ $p =~ ^[0-9]+$ ]] && (( p > 1 )); do ancestors+="$p "; p="$(proc_ppid "$p")"; done
+  pgid="$(proc_pgid "$self")"
+  { find "$SFN_PROC"/[0-9]*/{exe,cwd,root} "$SFN_PROC"/[0-9]*/fd -maxdepth 1 -lname '/nix/*' 2>/dev/null || true
+    grep -ls '[[:space:]]/nix/' "$SFN_PROC"/[0-9]*/maps || true
+  } | while IFS= read -r p; do p=${p#"$SFN_PROC"/}; echo "${p%%/*}"; done | sort -un \
+    | while read -r pid; do
+      [[ $pid != "$self" ]] && { read -r name <"$SFN_PROC/$pid/comm"; } 2>/dev/null || continue
+      # argv[0]'s name says more than comm (often a thread name)
+      argv0=''; { IFS= read -r -d '' argv0 <"$SFN_PROC/$pid/cmdline"; } 2>/dev/null || true
+      [[ ${argv0##*/} == '' || ${argv0##*/} == exe ]] || name=${argv0##*/}
+      if [[ $ancestors == *" $pid "* ]]; then
+        echo "$name (PID $pid, started this uninstall: close it and run uninstall from another terminal)"
+        continue
+      fi
+      [[ -n $pgid && $(proc_pgid "$pid") == "$pgid" ]] && continue      # our pipeline
+      p=$pid
+      while [[ $p =~ ^[0-9]+$ && $p != "$self" ]] && (( p > 1 )); do p="$(proc_ppid "$p")"; done
+      [[ $p == "$self" ]] && continue                                    # our subshells
+      echo "$name (PID $pid)"
+    done
+}
+
+# Waits until no process uses /nix; fails if the user gives up (or can't be
+# asked: --yes, no terminal). Nothing is stopped automatically.
+nix_idle() {
+  local users=() reply
+  while :; do
+    mapfile -t users < <(nix_users)
+    (( ${#users[@]} )) || return 0
+    warn "programs started from the Nix store still use /nix, so Nix can't be removed:"
+    printf '  - %s\n' "${users[@]}" >&2
+    (( ! ASSUME_YES )) && have_tty || return 1
+    printf 'Close them, then press Enter to check again (a: abort) ' >/dev/tty
+    read -r reply </dev/tty || return 1
+    [[ $reply != [aA]* ]] || return 1
+  done
+}
+
 remove_path() {
   if [[ -e $1 || -L $1 ]]; then rm -rf -- "$1"; info "removed $1"; fi
 }
@@ -1217,7 +1255,7 @@ cmd_uninstall() {
   need_not_root
 
   # A bash from Nix (e.g. via /usr/bin/env) would keep /nix busy.
-  if (( ! keep_nix )) && [[ $(readlink /proc/$$/exe) == /nix/store/* ]]; then
+  if (( ! keep_nix )) && [[ $(readlink "$SFN_PROC/$$/exe") == /nix/store/* ]]; then
     if [[ -f ${BASH_SOURCE[0]:-} && -x /usr/bin/bash ]]; then
       exec /usr/bin/bash "${BASH_SOURCE[0]}" uninstall "${orig_args[@]}"
     fi
@@ -1239,13 +1277,14 @@ cmd_uninstall() {
   # A failed Nix uninstall (e.g. /nix still busy) must not skip the rest:
   # Home Manager is gone by now, so its config link goes regardless; the
   # per-user Nix files stay for a Nix that may still be there.
-  local nix_failed=0
+  local nix_failed=''
   if (( ! keep_nix )); then
     if [[ -x $NIX_INSTALLER_BIN ]]; then
-      stop_nix_processes
+      if ! nix_idle; then
-      if ! uninstall_nix; then
+        nix_failed=busy keep_nix=1
+      elif ! uninstall_nix; then
         warn "the Nix uninstall failed (see above)"
-        nix_failed=1 keep_nix=1
+        nix_failed=failed keep_nix=1
       fi
     elif command -v nix >/dev/null 2>&1; then
       warn "Nix wasn't installed by nix-installer ($NIX_INSTALLER_BIN missing); not removing it"
@@ -1266,14 +1305,17 @@ Intentionally left in place:
   - app data, e.g. ~/.local/share/docker, Firefox profiles, and Flatpak apps
 Log out or reboot so running sessions drop the removed tweaks.
 EOF
-  if (( ${#CLEAN_DEFERRED[@]} || nix_failed )); then
+  if (( ${#CLEAN_DEFERRED[@]} )) || [[ -n $nix_failed ]]; then
     printf '\nNot done yet:\n'
     (( ${#CLEAN_DEFERRED[@]} )) && printf '  - %s\n' "${CLEAN_DEFERRED[@]}"
-    (( nix_failed )) && printf '%s\n' \
+    case $nix_failed in
-      "  - Nix: its uninstaller failed. Reboot (so nothing uses /nix), then run" \
+      busy) printf '%s\n' "  - Nix: still in use by the programs listed above. Close these or reboot," ;;
-      "    uninstall again; it also removes ~/.nix-profile and ~/.local/state/nix."
+      failed) printf '%s\n' "  - Nix: its uninstaller failed. Reboot (so nothing uses /nix)," ;;
+    esac
+    [[ -n $nix_failed ]] && printf '%s\n' \
+      "    then run uninstall again; it also removes ~/.nix-profile and ~/.local/state/nix."
   fi
-  (( ! nix_failed ))
+  [[ -z $nix_failed ]]
 }
 
 # --- status -----------------------------------------------------------------

modules/cleanup/check.nix

@@ -5,6 +5,8 @@
 # Also `install.sh restart-check` (what waits for a session/SteamVR restart).
 # Also `install.sh install --clone` (argument parsing and the clone step),
 # against local bare repositories through a logging git, without network.
+# Also `install.sh uninstall` stopping before Nix while programs from the
+# Nix store run (a fake /proc).
 { pkgs }:
 let
   cleanup = pkgs.callPackage ./package.nix { };
@@ -385,5 +387,50 @@ pkgs.runCommand "cleanup-check" { nativeBuildInputs = [ cleanup pkgs.jq pkgs.git
   res=$(inst --clone https://example.org/missing.git --dir $HOME/m 2>&1) && fail "missing repo"
   has "$res" "gh auth login"
   echo "G ok"
+
+  # --- H: uninstall: /nix in use ---
+  fresh h
+  export -f fail has hasnt gone
+  export root INSTALL
+  # In a process of its own: in the sandbox $$ is PID 1.
+  bash <<'SH'
+  set -euo pipefail
+  P=$root/proc
+  # pid name ppid pgid exe: a fake /proc entry
+  fake() {
+    mkdir -p $P/$1/fd; echo $2 > $P/$1/comm; printf '%s\0-x\0' $5 > $P/$1/cmdline
+    ln -sfn $5 $P/$1/exe; ln -sfn /home $P/$1/cwd
+    printf 'Name:\t%s\nPPid:\t%s\n' $2 $3 > $P/$1/status
+    echo "$1 ($2 x) S $3 $4 0" > $P/$1/stat
+    echo "00400000-00401000 r-xp 00000000 00:01 1 $5" > $P/$1/maps
+  }
+  fake $$ bash 30 $$ /usr/bin/bash                          # this uninstall
+  fake 30 zsh 1 30 /nix/store/a-zsh/bin/zsh                 # its terminal's shell
+  fake 31 curl 30 $$ /nix/store/b-curl/bin/curl             # its pipeline: skipped
+  fake 32 sort $$ 32 /nix/store/c-coreutils/bin/sort        # its subshell: skipped
+  fake 40 app 1 40 /usr/bin/app                             # a library from /nix
+  echo "7f00-7f01 r-xp 0 00:01 2 /nix/store/d-lib/lib/libx.so" >> $P/40/maps
+  fake 41 cwd 1 41 /usr/bin/cwd; ln -sfn /nix/store/e $P/41/cwd
+  fake 42 fd 1 42 /usr/bin/fd; ln -s /nix/store/f/file $P/42/fd/3
+  fake 43 clean 1 43 /usr/bin/clean
+  printf '#!/bin/sh\n' > $root/nix-installer; chmod +x $root/nix-installer
+  mkdir -p $HOME/.config; ln -s $HOME/cfg $HOME/.config/home-manager
+  uninst() {
+    (STEAM_FRAME_NIX_PROC=$P; . $INSTALL; ASSUME_YES=1 NIX_INSTALLER_BIN=$root/nix-installer
+     uninstall_nix() { echo "nix-installer uninstall"; }; cmd_uninstall) 2>&1
+  }
+  res=$(uninst) && fail "uninstall went on with /nix in use: $res"
+  echo "$res"
+  hasnt "$res" "nix-installer uninstall"
+  has "$res" "zsh (PID 30, started this uninstall"
+  for p in "app (PID 40)" "cwd (PID 41)" "fd (PID 42)" "Close these or reboot,"; do has "$res" "$p"; done
+  for p in "PID 31" "PID 32" "PID 43" "PID $$"; do hasnt "$res" "$p"; done
+  gone $HOME/.config/home-manager
+  # closed: Nix is removed
+  rm -r $P/30 $P/40 $P/41 $P/42; fake $$ bash 1 $$ /usr/bin/bash
+  res=$(uninst) || fail "uninstall failed: $res"
+  has "$res" "nix-installer uninstall"
+  SH
+  echo "H ok"
   touch $out
 ''