Library backfill only fills art Frame Control couldn't apply; remove serialised with refresh

- Automatic backfill touches only entries marked art_pending at install (a
  devkit title Steam registered later) and fills only slots Steam has no art
  for: no name, exe, VR flag or icon changes, no clearing. Older installs
  without the flag are left alone and refreshed only when the user asks.
- Android remove takes the install lock that install and refresh hold, so a
  refresh in progress can't recreate a removed app; a refresh after removal
  finds it not installed.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

frame/android/steam_shortcuts.py

@@ -9,7 +9,7 @@ Python stdlib only; the Mac runs it with `ssh frame python3 - <args> < this`.
   steam_shortcuts.py stop APPID
   steam_shortcuts.py remove APPID
 """
-import base64, json, os, re, socket, struct, sys, urllib.request
+import base64, glob, json, os, re, socket, struct, sys, urllib.request
 
 DEVTOOLS = 'http://127.0.0.1:8080/json'
 
@@ -198,9 +198,26 @@ def _render(plan, renderer, label, images):
     return {'paths': paths, 'warnings': list(result.get('warnings', []))}
 
 
+# Steam's own file for each custom-art type in userdata/*/config/grid/.
+GRID_FILES = {0: 'p', 1: '_hero', 2: '_logo', 3: ''}
+
+
+def custom_art(appid):
+    """The custom-art types this shortcut already has in Steam, for any local user."""
+    found = set()
+    for kind, suffix in GRID_FILES.items():
+        pattern = os.path.expanduser(f'~/.local/share/Steam/userdata/*/config/grid/{int(appid)}{suffix}.*')
+        if any(os.path.splitext(p)[1].lower() in ('.png', '.jpg', '.jpeg') for p in glob.glob(pattern)):
+            found.add(kind)
+    return found
+
+
 def configure(appid, name, exe, start_dir, icon, vr, artwork, options=None):
+    """options: category, details, fill_only (only empty slots and a missing icon; name and flags untouched)."""
     options = options or {}
     category = options.get('category', 'Android')
+    fill = bool(options.get('fill_only'))
+    existing = custom_art(appid) if fill else set()
     if set(artwork) != set(ASSETS):
         raise ValueError('all five Steam artwork slots are required')
     images = []
@@ -214,18 +231,22 @@ def configure(appid, name, exe, start_dir, icon, vr, artwork, options=None):
             data = f.read(MAX_ART + 1)
         if len(data) > MAX_ART:
             raise ValueError('artwork is too large')
-        if slot != 'icon':  # Frame's custom-art API maps type 4 to Header; use SetShortcutIcon.
+        # Frame's custom-art API maps type 4 to Header; use SetShortcutIcon.
+        if slot != 'icon' and ASSETS[slot] not in existing:
             images.append([ASSETS[slot], ext, base64.b64encode(data).decode()])
     return evaluate(f'''(async () => {{
-      const id = {int(appid)}, warnings = [];
-      SteamClient.Apps.SetShortcutName(id, {json.dumps(name)});
-      if ({json.dumps(exe)}) SteamClient.Apps.SetShortcutExe(id, {json.dumps(exe)});
-      if ({json.dumps(start_dir)}) SteamClient.Apps.SetShortcutStartDir(id, {json.dumps(start_dir)});
-      if (typeof SteamClient.Apps.SetShortcutSortAs === "function")
-        SteamClient.Apps.SetShortcutSortAs(id, {json.dumps(name)});
-      SteamClient.Apps.SetShortcutIcon(id, {json.dumps(icon)});
-      // null (devkit titles): leave the VR flag as Steam registered it.
-      if ({json.dumps(vr)} !== null) {{
+      const id = {int(appid)}, warnings = [], fill = {json.dumps(fill)};
+      const overview = appStore.GetAppOverviewByAppID(id);
+      if (!fill) {{
+        SteamClient.Apps.SetShortcutName(id, {json.dumps(name)});
+        if ({json.dumps(exe)}) SteamClient.Apps.SetShortcutExe(id, {json.dumps(exe)});
+        if ({json.dumps(start_dir)}) SteamClient.Apps.SetShortcutStartDir(id, {json.dumps(start_dir)});
+        if (typeof SteamClient.Apps.SetShortcutSortAs === "function")
+          SteamClient.Apps.SetShortcutSortAs(id, {json.dumps(name)});
+      }}
+      if (!fill || !(overview && overview.icon_data)) SteamClient.Apps.SetShortcutIcon(id, {json.dumps(icon)});
+      // null (devkit titles) or filling gaps: leave the VR flag as Steam has it.
+      if ({json.dumps(vr)} !== null && !fill) {{
         if (typeof SteamClient.Apps.SetShortcutIsVR === "function")
           SteamClient.Apps.SetShortcutIsVR(id, {json.dumps(vr)});
         else warnings.push("Steam VR shortcut flag API unavailable");
@@ -233,7 +254,7 @@ def configure(appid, name, exe, start_dir, icon, vr, artwork, options=None):
       if (typeof SteamClient.Apps.SetCustomArtworkForApp === "function") {{
         for (const [type, ext, data] of {json.dumps(images)}) {{
           // Steam keeps a slot's PNG and JPEG side by side; clear it so a stale one can't win.
-          if (typeof SteamClient.Apps.ClearCustomArtworkForApp === "function")
+          if (!fill && typeof SteamClient.Apps.ClearCustomArtworkForApp === "function")
             try {{ await SteamClient.Apps.ClearCustomArtworkForApp(id, type); }} catch (e) {{}}
           await SteamClient.Apps.SetCustomArtworkForApp(id, data, ext, type);
         }}

tests/fakeframe/rootfs/usr/local/lib/fakeframe/cef_shim.js

@@ -38,6 +38,7 @@ function build(steam) {
   });
   const shortcutOverview = s => ({
     appid: s.appid, display_name: s.name, sort_as: s.name, app_type: SHORTCUT_TYPE, devkit_gameid: s.devkit_gameid,
+    icon_data: s.icon ? 'fake-icon' : undefined,
     local_per_client_data: { installed: true, display_status: 1, status_percentage: 0 },
   });
   const allApps = () => [...steam.apps.map(gameOverview), ...steam.shortcuts.map(shortcutOverview)];

tests/test_frame_android_library.py

@@ -369,6 +369,44 @@ class InstallTests(unittest.TestCase):
                                         '.local/share/Steam/steamapps/shadercache/2800000001')
             self.assertIn('Steam client running', result['library_warnings'][0])
 
+    def test_remove_waits_for_refresh_and_is_never_undone(self):
+        import threading
+        state = {'meta': dict(self.existing, flatscreen=False), 'shortcuts': {3346865537}}
+        in_refresh, release, added = threading.Event(), threading.Event(), []
+        def meta(pkg):
+            if not state['meta']:
+                raise android.FrameError(pkg + ' is not installed')
+            return dict(state['meta'])
+        def ssh(cmd, input=None, **kw):
+            if cmd.startswith('rm -rf Applications/Android/org.test.vr'):
+                state['meta'] = None
+            return json.dumps({'icon_png': ''}) if cmd == 'python3 -' else '/home/steamos'
+        def tool(*args, **kw):
+            if args[0] == 'list': return json.dumps([{'appid': a} for a in state['shortcuts']])
+            if args[0] == 'remove': state['shortcuts'].discard(int(args[1])); return '{"warnings": []}'
+            if args[0] == 'add': added.append(args); return '99'
+            if args[0] == 'render': return json.dumps({'paths': {s: '/tmp/' + s + '.png' for s in art.SLOTS}})
+            return '{"warnings": []}'
+        def prepare(*args, **kw):
+            in_refresh.set(); release.wait(5)
+            return self.images, []
+        with patch.object(android, '_meta_or_fail', side_effect=meta), patch.object(android, 'ssh', side_effect=ssh), \
+                patch.object(android, 'shortcut_tool', side_effect=tool), patch.object(android, 'stop'), \
+                patch.object(android, '_write_meta', side_effect=lambda d, m: state.__setitem__('meta', m)), \
+                patch.object(android.frame_artwork, 'prepare', side_effect=prepare):
+            refresh = threading.Thread(target=android.refresh_art, args=('org.test.vr',), kwargs={'fill_only': True})
+            refresh.start()
+            self.assertTrue(in_refresh.wait(5))
+            remove = threading.Thread(target=android.remove, args=('org.test.vr',))
+            remove.start()
+            remove.join(0.3)
+            self.assertTrue(remove.is_alive(), 'remove ran while a refresh was writing')
+            release.set(); refresh.join(5); remove.join(5)
+            self.assertIsNone(state['meta']); self.assertEqual(state['shortcuts'], set())
+            with self.assertRaisesRegex(android.FrameError, 'not installed'):
+                android.refresh_art('org.test.vr', fill_only=True)  # a queued backfill after removal
+        self.assertEqual(added, [])
+
     def test_stop_requests_steam_and_has_container_fallback(self):
         with patch.object(android, '_meta_or_fail', return_value=self.existing), \
                 patch.object(android, 'shortcut_tool', side_effect=android.FrameError('offline')) as api, \
@@ -405,7 +443,7 @@ class SteamAPITests(unittest.TestCase):
         with patch.object(shortcuts, 'evaluate', return_value={'warnings': []}) as evaluate:
             shortcuts.configure(42, 'Game', '', '', '/icon', None, slots, {'category': 'Sideloaded'})
         js = evaluate.call_args.args[0]
-        self.assertIn('if (null !== null)', js)
+        self.assertIn('if (null !== null && !fill)', js)
         self.assertIn('const wanted = ["Sideloaded"]', js)
         with patch.object(sys, 'argv', ['steam_shortcuts.py', 'configure', '42', 'Game', '', '', '/icon', '',
                                         json.dumps(slots), '{}']), \
@@ -484,6 +522,36 @@ class SteamContextTests(unittest.TestCase):
             self.assertEqual(steam['shortcuts'], [])
             self.assertEqual(steam['collections'][0]['apps'], [999])
 
+    @unittest.skipUnless(__import__('shutil').which('node'), 'optional V8 fixture check requires node')
+    def test_fill_only_keeps_customised_name_icon_and_art(self):
+        custom = {'0': {'data': 'mine-grid', 'ext': 'png'}, '1': {'data': 'mine-hero', 'ext': 'jpg'}}
+        steam = {'apps': [], 'compat_tools': {}, 'collections': [],
+                 'shortcuts': [{'appid': 42, 'name': 'My Name', 'icon': '/mine.png', 'vr': True, 'artwork': dict(custom)}]}
+        def evaluate(expression, timeout=20):
+            nonlocal steam
+            proc = subprocess.run(['node', str(ROOT / 'tests/fakeframe/rootfs/usr/local/lib/fakeframe/cef_shim.js')],
+                                  input=json.dumps({'id': 1, 'expression': expression, 'awaitPromise': True,
+                                                    'steam': steam}) + '\n',
+                                  text=True, capture_output=True, timeout=10, check=True)
+            reply = json.loads(proc.stdout)
+            self.assertNotIn('exceptionDetails', reply['result'])
+            steam = reply['steam']
+            return reply['result']['result'].get('value')
+        with tempfile.TemporaryDirectory() as home:
+            grid = Path(home, '.local/share/Steam/userdata/1/config/grid')
+            grid.mkdir(parents=True)
+            (grid / '42p.png').write_bytes(b'mine')
+            (grid / '42_hero.jpg').write_bytes(b'mine')
+            with patch.dict(os.environ, {'HOME': home, 'USERPROFILE': home}), patch.object(shortcuts, 'evaluate', side_effect=evaluate):
+                self.assertEqual(shortcuts.custom_art(42), {0, 1})
+                shortcuts.configure(42, 'Generated', '/exe', '/dir', '/generated.png', False,
+                                    {slot: str(FIXTURES / 'icon.png') for slot in art.SLOTS},
+                                    {'category': 'Sideloaded', 'fill_only': True})
+        s = steam['shortcuts'][0]
+        self.assertEqual((s['name'], s['icon'], s['vr']), ('My Name', '/mine.png', True))
+        self.assertEqual({k: s['artwork'][k] for k in ('0', '1')}, custom)
+        self.assertEqual(set(s['artwork']), {'0', '1', '2', '3'})
+
     @unittest.skipUnless(__import__('shutil').which('node'), 'optional V8 fixture check requires node')
     def test_remove_without_collections_or_artwork_api_still_removes(self):
         steam = {'apps': [], 'shortcuts': [{'appid': 42, 'name': 'Game', 'exe': '"/home/steamos/devkit-game/G/g"',

tests/test_library_entrypoints.py

@@ -92,21 +92,34 @@ class EntryPoints(unittest.TestCase):
             server.titles({'action':'refresh-art', 'id':'G'})
             title_refresh.assert_called_with('G')
 
-    def test_backfill_applies_missing_art_once_steam_answers(self):
+    def test_backfill_fills_only_entries_pending_since_install(self):
         import threading
         ran = threading.Event()
         with patch.dict(server._backfill, {'running': False, 'last': 0.0}), \
                 patch.object(android, 'refresh_art', side_effect=[RuntimeError('odd'), None]) as refresh, \
-                patch.object(titles, 'refresh_art', side_effect=lambda gid: ran.set()) as title_refresh:
-            apps = [{'package':'org.a.x','art_missing':True}, {'package':'org.b.x','art_missing':True},
-                    {'package':'org.c.x','art_missing':False}]
+                patch.object(titles, 'refresh_art', side_effect=lambda gid, **kw: ran.set()) as title_refresh:
+            apps = [{'package':'org.a.x','art_pending':True}, {'package':'org.b.x','art_pending':True},
+                    {'package':'org.c.x','art_missing':True}]  # legacy: no record of art, but not pending
             with contextlib.redirect_stderr(io.StringIO()):
-                self.assertTrue(server.backfill_art(apps=apps, titles=[{'id':'G','art_missing':True}]))
+                self.assertTrue(server.backfill_art(apps=apps, titles=[{'id':'G','art_pending':True}]))
                 self.assertTrue(ran.wait(5))
             self.assertFalse(server.backfill_art(apps=apps))  # throttled
         self.assertEqual([c.args for c in refresh.call_args_list], [('org.a.x',), ('org.b.x',)])
-        title_refresh.assert_called_once_with('G')
-        self.assertFalse(server.backfill_art(apps=[{'package':'org.c.x','art_missing':False}]))
+        self.assertTrue(all(c.kwargs == {'fill_only': True} for c in refresh.call_args_list))
+        title_refresh.assert_called_once_with('G', fill_only=True)
+
+    def test_upgrade_leaves_legacy_customised_titles_alone(self):
+        # A title installed before art_pending existed, whose art the user has customised in Steam.
+        legacy = [{'id':'Game','settings':{'compat_tool':'proton-experimental'},'argv':['game.exe'],
+                   'meta':{'name':'Game','target':'game.exe','source':'game.zip'}}]
+        with patch.object(titles, 'ssh', return_value=json.dumps(legacy)):
+            listed = titles.list_titles()
+        self.assertEqual((listed[0]['art_pending'], listed[0]['art_missing']), (False, False))
+        with patch.dict(server._backfill, {'running': False, 'last': 0.0}), \
+                patch.object(titles, 'refresh_art') as refresh, patch.object(android, 'refresh_art') as app_refresh:
+            self.assertFalse(server.backfill_art(apps=[{'package':'org.old.app','library_version':1}], titles=listed))
+        refresh.assert_not_called(); app_refresh.assert_not_called()
+        self.api.assert_not_called()
 
     def test_art_missing_flags(self):
         self.assertTrue(android.art_missing({'artwork': {}}))

ui/frame_android.py

@@ -221,10 +221,11 @@ def _install(apk_path, info, pkg, flatscreen, name, source, artwork=None):
 
 
 def apply_library(shortcut, label, directory, images, vr=None, home=None, exe='', start_dir='', details=None,
-                  category='Android'):
+                  category='Android', fill_only=False):
     """Mandatory for every sideload: render all five slots before reporting success.
 
-    vr None leaves Steam's VR flag as it is (devkit titles declare their own)."""
+    vr None leaves Steam's VR flag as it is (devkit titles declare their own). fill_only (automatic
+    backfill) sets only slots Steam has no art for, and never the name, exe or flags."""
     home = home or ssh('echo $HOME').strip()
     d = directory
     ssh(f'mkdir -p {shlex.quote(d)}/artwork')
@@ -244,7 +245,8 @@ def apply_library(shortcut, label, directory, images, vr=None, home=None, exe=''
         raise FrameError('Steam artwork renderer did not produce every slot')
     result = json.loads(shortcut_tool('configure', str(shortcut), label, exe, start_dir, art['icon'],
                                      '' if vr is None else '1' if vr else '0', json.dumps(art),
-                                     json.dumps({'category': category, 'details': details or {}}), timeout=120))
+                                     json.dumps({'category': category, 'details': details or {},
+                                                 'fill_only': fill_only}), timeout=120))
     result['warnings'] = rendered.get('warnings', []) + result.get('warnings', [])
     result['artwork'] = art
     if category == 'Android':
@@ -277,8 +279,10 @@ def art_missing(m):
     return set((m or {}).get('artwork') or {}) != set(frame_artwork.SLOTS)
 
 
-def refresh_art(pkg=None, artwork=None):
-    """Refresh existing APK library entries without reinstalling or stopping them."""
+def refresh_art(pkg=None, artwork=None, fill_only=False):
+    """Refresh existing APK library entries without reinstalling or stopping them.
+
+    fill_only: the automatic backfill; it only fills empty Steam slots (see apply_library)."""
     if pkg is None:
         results = []
         for app in list_apps():
@@ -287,7 +291,7 @@ def refresh_art(pkg=None, artwork=None):
             except Exception as e:  # one app's failure must not stop the others
                 results.append({'package': app['package'], 'label': app.get('label'), 'error': str(e) or type(e).__name__})
         return results
-    with _install_lock:
+    with _install_lock:  # shared with install and remove: a removed app is never recreated
         m = _meta_or_fail(pkg)
         d = f'{APPS_DIR}/{pkg}'
         # Parse the APK on the Frame, transferring only its icon/label, not the APK.
@@ -315,7 +319,8 @@ def refresh_art(pkg=None, artwork=None):
             created = True
         try:
             result = apply_library(m['shortcut'], m['label'], d, images, vr=not m.get('flatscreen', True),
-                                   home=home, exe=f'{home}/{d}/launch.sh', start_dir=f'{home}/{d}', details=m)
+                                   home=home, exe=f'{home}/{d}/launch.sh', start_dir=f'{home}/{d}', details=m,
+                                   fill_only=fill_only and not created)
         except Exception:
             if created:
                 try:
@@ -323,7 +328,7 @@ def refresh_art(pkg=None, artwork=None):
                 except FrameError:
                     pass  # keep the render error, not the cleanup's
             raise
-        m.update(artwork=result.get('artwork', {}), library_version=2)
+        m.update(artwork=result.get('artwork', {}), library_version=2, art_pending=False)
         m['library_warnings'] = warnings + result.get('warnings', [])
         m['artwork_refreshed'] = time.strftime('%Y-%m-%dT%H:%M:%S')
         _write_meta(d, m)
@@ -411,6 +416,11 @@ def stop(pkg):
 
 
 def remove(pkg, keep_data=False):
+    with _install_lock:  # never while an install or artwork refresh of any app is writing
+        return _remove(pkg, keep_data)
+
+
+def _remove(pkg, keep_data):
     m = _meta_or_fail(pkg)
     stop(pkg)
     if m['shortcut']:

ui/frame_titles.py

@@ -696,7 +696,9 @@ def _install(plan, step, artwork=None):
         reply = _json_out(ssh(f'{PY}steam-client-create-shortcut --parms {shlex.quote(json.dumps(parms))}',
                               timeout=90), 'steam-client-create-shortcut')
         meta = {'id': gid, 'name': plan['name'], 'target': plan['target'], 'runtime': plan['runtime'],
-                'source': plan['source'], 'size': plan['size'], 'installed': time.strftime('%Y-%m-%dT%H:%M:%S')}
+                'source': plan['source'], 'size': plan['size'], 'installed': time.strftime('%Y-%m-%dT%H:%M:%S'),
+                # Until art is applied: the only entries automatic backfill may touch.
+                'art_pending': True}
         ssh(f'cat > {GAMES}/{gid}-framecontrol.json', input=json.dumps(meta, indent=1), timeout=30)
         registered = True                    # the files stay: Steam registers them once it's running
         if 'error' in reply:
@@ -709,7 +711,7 @@ def _install(plan, step, artwork=None):
             raise FrameError('Steam registered the title but its shortcut is not available for mandatory artwork; retry install')
         result = frame_android.apply_library(shortcut, plan['name'], directory + '/.frame-artwork', images,
                                              category='Sideloaded', details={'source': plan['source']})
-        meta.update(shortcut=shortcut, artwork=result.get('artwork', {}),
+        meta.update(shortcut=shortcut, artwork=result.get('artwork', {}), art_pending=False,
                     library_warnings=warnings + result.get('warnings', []))
         ssh(f'cat > {GAMES}/{gid}-framecontrol.json', input=json.dumps(meta, indent=1), timeout=30)
         library_ready = True
@@ -768,8 +770,10 @@ def _home():
     return ssh('echo $HOME', timeout=30).strip()
 
 
-def refresh_art(gid=None, artwork=None):
-    """Render and apply Steam artwork for Frame Control's titles (all of them when gid is None)."""
+def refresh_art(gid=None, artwork=None, fill_only=False):
+    """Render and apply Steam artwork for Frame Control's titles (all of them when gid is None).
+
+    fill_only: the automatic backfill; it only fills empty Steam slots (see apply_library)."""
     if gid is None:
         results = []
         for t in list_titles():
@@ -809,8 +813,9 @@ print(json.dumps({{'artwork': cached, 'icon': icon}}))
         if not shortcut:
             raise FrameError(f"Steam hasn't registered {gid} yet; with Steam running on the Frame, refresh again")
         result = frame_android.apply_library(shortcut, name, directory + '/.frame-artwork', images,
-                                             category='Sideloaded', details={'source': meta.get('source')})
-        meta.update(shortcut=shortcut, artwork=result.get('artwork', {}),
+                                             category='Sideloaded', details={'source': meta.get('source')},
+                                             fill_only=fill_only)
+        meta.update(shortcut=shortcut, artwork=result.get('artwork', {}), art_pending=False,
                     library_warnings=warnings + result.get('warnings', []),
                     artwork_refreshed=time.strftime('%Y-%m-%dT%H:%M:%S'))
         ssh(f'cat > {GAMES}/{gid}-framecontrol.json', input=json.dumps(meta, indent=1), timeout=30)
@@ -852,7 +857,9 @@ def list_titles():
                        'runtime': alias, 'runtime_label': RUNTIMES.get(alias, {}).get('label', alias or 'not set'),
                        'source': str(meta.get('source') or ''), 'size': meta.get('size'),
                        'installed': meta.get('installed'), 'registered': t.get('settings') is not None,
-                       'frame_control': bool(meta), 'art_missing': bool(meta) and frame_android.art_missing(meta)})
+                       'frame_control': bool(meta),
+                       # Only a title whose install couldn't apply art; older installs have no flag and keep theirs.
+                       'art_pending': bool(meta.get('art_pending')), 'art_missing': bool(meta.get('art_pending'))})
     return titles
 
 

ui/server.py

@@ -153,11 +153,12 @@ _backfill_lock = threading.Lock()
 
 
 def backfill_art(apps=(), titles=()):
-    """Give installs that lack Steam artwork (Steam wasn't running, say) their art once Steam is up.
+    """Apply art Frame Control couldn't at install time (Steam wasn't running), once Steam is up.
 
-    Runs in the background, at most once every five minutes; refresh-art reports failures on demand."""
-    pkgs = [a["package"] for a in apps if a.get("art_missing")]
-    gids = [t["id"] for t in titles if t.get("art_missing")]
+    Only entries marked art_pending at install; it fills empty Steam slots and never replaces art or
+    names the user may have customised. Runs in the background, at most once every five minutes."""
+    pkgs = [a["package"] for a in apps if a.get("art_pending")]
+    gids = [t["id"] for t in titles if t.get("art_pending")]
     with _backfill_lock:
         if not (pkgs or gids) or _backfill["running"] or time.time() - _backfill["last"] < 300:
             return False
@@ -169,7 +170,7 @@ def backfill_art(apps=(), titles=()):
             for refresh, key in [(frame_android.refresh_art, p) for p in pkgs] + \
                                 [(frame_titles.refresh_art, g) for g in gids]:
                 try:
-                    refresh(key)
+                    refresh(key, fill_only=True)
                 except Exception as e:
                     print(f"artwork backfill for {key}: {e}", file=sys.stderr)
         except Exception: