Merge branch 'library-fixes' into apk-store-fixes

ui/apk_sources/_images.py

@@ -65,53 +65,94 @@ def image_type(data):
 
 def fetch(url, redirects=3, deadline=None, limit=MAX_IMAGE):
     """deadline: time.monotonic() value by which the whole fetch, redirects included, must finish."""
+    data = get(url, {'Accept': 'image/png,image/jpeg,image/webp,image/gif'}, redirects, deadline, limit)
+    return data, image_type(data)
+
+
+def _resolve(host, port, timeout):
+    # getaddrinfo has no timeout of its own; a thread keeps a slow resolver inside the budget.
+    found = {}
+
+    def run():
+        try:
+            found['addresses'] = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
+        except OSError as e:
+            found['error'] = e
+    worker = threading.Thread(target=run, daemon=True)
+    worker.start()
+    worker.join(timeout)
+    if 'error' in found:
+        raise found['error']
+    if 'addresses' not in found:
+        raise SourceError('Artwork download took too long')
+    return found['addresses']
+
+
+def get(url, headers=None, redirects=3, deadline=None, limit=MAX_IMAGE):
+    """GET a public HTTP(S) URL within an overall deadline (default 60 s), redirects included.
+
+    A watchdog shuts the socket at the deadline, so a server trickling bytes can't outlast it."""
+    deadline = time.monotonic() + 60 if deadline is None else deadline
+
+    def left():
+        remaining = deadline - time.monotonic()
+        if remaining <= 0:
+            raise SourceError('Artwork download took too long')
+        return remaining
     if not valid_url(url):
         raise SourceError('Artwork URL is not allowed')
-
-    def remaining():
-        if deadline is None:
-            return 10
-        left = deadline - time.monotonic()
-        if left <= 0:
-            raise SourceError('Artwork download took too long')
-        return min(10, left)
     p = urlsplit(url)
     port = p.port or (443 if p.scheme == 'https' else 80)
-    addresses = socket.getaddrinfo(p.hostname, port, type=socket.SOCK_STREAM)
+    addresses = _resolve(p.hostname, port, left())
     if not addresses or any(not ipaddress.ip_address(a[4][0]).is_global for a in addresses):
         raise SourceError('Private network artwork is not allowed')
     # Connect to the checked IP, never resolve again between validation and use.
-    sock = socket.create_connection((addresses[0][4][0], port), timeout=remaining())
+    live = [socket.create_connection((addresses[0][4][0], port), timeout=min(10, left()))]
+
+    def expire():
+        try:
+            live[0].shutdown(socket.SHUT_RDWR)
+        except OSError:
+            pass
+    watchdog = threading.Timer(left(), expire)
+    watchdog.daemon = True
+    watchdog.start()
     conn = http.client.HTTPConnection(p.hostname, port, timeout=10)
     try:
         if p.scheme == 'https':
-            sock = ssl.create_default_context().wrap_socket(sock, server_hostname=p.hostname)
-        conn.sock = sock
+            live[0] = ssl.create_default_context().wrap_socket(live[0], server_hostname=p.hostname)
+        conn.sock = live[0]
         path = p.path or '/'
         if p.query:
             path += '?' + p.query
-        conn.request('GET', path, headers={'User-Agent': 'FrameControl/0.3.1', 'Accept': 'image/png,image/jpeg,image/webp,image/gif'})
-        sock.settimeout(remaining())
+        conn.request('GET', path, headers={'User-Agent': 'FrameControl/0.3.1', **(headers or {})})
+        live[0].settimeout(min(10, left()))
         response = conn.getresponse()
         if response.status in (301, 302, 303, 307, 308) and redirects:
             target = urljoin(url, response.getheader('Location', ''))
             conn.close()
-            return fetch(target, redirects - 1, deadline, limit)
+            return get(target, headers, redirects - 1, deadline, limit)
         if response.status != 200:
             raise SourceError('Artwork is unavailable')
         data = b''
         while len(data) <= limit:
-            sock.settimeout(remaining())
-            chunk = response.read(min(65536, limit + 1 - len(data)))
+            live[0].settimeout(min(10, left()))
+            chunk = response.read1(min(16384, limit + 1 - len(data)))  # one receive at most
             if not chunk:
                 break
             data += chunk
         if len(data) > limit:
             raise SourceError('Artwork is too large')
-        return data, image_type(data)
+        left()
+        return data
+    except (OSError, http.client.HTTPException) as e:
+        if time.monotonic() >= deadline:
+            raise SourceError('Artwork download took too long') from e
+        raise
     finally:
+        watchdog.cancel()
         conn.close()
-        sock.close()
+        live[0].close()
 
 
 def remember(url, data):

ui/frame_android.py

@@ -221,10 +221,11 @@ def _install(apk_path, info, pkg, flatscreen, name, source, artwork=None):
 
 
 def apply_library(shortcut, label, directory, images, vr=None, home=None, exe='', start_dir='', details=None,
-                  category='Android'):
+                  category='Android', fill_only=False):
     """Mandatory for every sideload: render all five slots before reporting success.
 
-    vr None leaves Steam's VR flag as it is (devkit titles declare their own)."""
+    vr None leaves Steam's VR flag as it is (devkit titles declare their own). fill_only (automatic
+    backfill) sets only slots Steam has no art for, and never the name, exe or flags."""
     home = home or ssh('echo $HOME').strip()
     d = directory
     ssh(f'mkdir -p {shlex.quote(d)}/artwork')
@@ -244,7 +245,8 @@ def apply_library(shortcut, label, directory, images, vr=None, home=None, exe=''
         raise FrameError('Steam artwork renderer did not produce every slot')
     result = json.loads(shortcut_tool('configure', str(shortcut), label, exe, start_dir, art['icon'],
                                      '' if vr is None else '1' if vr else '0', json.dumps(art),
-                                     json.dumps({'category': category, 'details': details or {}}), timeout=120))
+                                     json.dumps({'category': category, 'details': details or {},
+                                                 'fill_only': fill_only}), timeout=120))
     result['warnings'] = rendered.get('warnings', []) + result.get('warnings', [])
     result['artwork'] = art
     if category == 'Android':
@@ -277,8 +279,10 @@ def art_missing(m):
     return set((m or {}).get('artwork') or {}) != set(frame_artwork.SLOTS)
 
 
-def refresh_art(pkg=None, artwork=None):
-    """Refresh existing APK library entries without reinstalling or stopping them."""
+def refresh_art(pkg=None, artwork=None, fill_only=False):
+    """Refresh existing APK library entries without reinstalling or stopping them.
+
+    fill_only: the automatic backfill; it only fills empty Steam slots (see apply_library)."""
     if pkg is None:
         results = []
         for app in list_apps():
@@ -287,7 +291,7 @@ def refresh_art(pkg=None, artwork=None):
             except Exception as e:  # one app's failure must not stop the others
                 results.append({'package': app['package'], 'label': app.get('label'), 'error': str(e) or type(e).__name__})
         return results
-    with _install_lock:
+    with _install_lock:  # shared with install and remove: a removed app is never recreated
         m = _meta_or_fail(pkg)
         d = f'{APPS_DIR}/{pkg}'
         # Parse the APK on the Frame, transferring only its icon/label, not the APK.
@@ -315,7 +319,8 @@ def refresh_art(pkg=None, artwork=None):
             created = True
         try:
             result = apply_library(m['shortcut'], m['label'], d, images, vr=not m.get('flatscreen', True),
-                                   home=home, exe=f'{home}/{d}/launch.sh', start_dir=f'{home}/{d}', details=m)
+                                   home=home, exe=f'{home}/{d}/launch.sh', start_dir=f'{home}/{d}', details=m,
+                                   fill_only=fill_only and not created)
         except Exception:
             if created:
                 try:
@@ -323,7 +328,7 @@ def refresh_art(pkg=None, artwork=None):
                 except FrameError:
                     pass  # keep the render error, not the cleanup's
             raise
-        m.update(artwork=result.get('artwork', {}), library_version=2)
+        m.update(artwork=result.get('artwork', {}), library_version=2, art_pending=False)
         m['library_warnings'] = warnings + result.get('warnings', [])
         m['artwork_refreshed'] = time.strftime('%Y-%m-%dT%H:%M:%S')
         _write_meta(d, m)
@@ -411,6 +416,11 @@ def stop(pkg):
 
 
 def remove(pkg, keep_data=False):
+    with _install_lock:  # never while an install or artwork refresh of any app is writing
+        return _remove(pkg, keep_data)
+
+
+def _remove(pkg, keep_data):
     m = _meta_or_fail(pkg)
     stop(pkg)
     if m['shortcut']:

ui/frame_steamgriddb.py

@@ -6,7 +6,6 @@ import tempfile
 import time
 import unicodedata
 import urllib.parse
-import urllib.request
 
 import frame_host
 
@@ -51,19 +50,11 @@ def save_settings(body):
 
 
 def _get(path, key, deadline=None):
-    timeout = 12 if deadline is None else min(12, deadline - time.monotonic())
-    if timeout <= 0:
-        raise ValueError('SteamGridDB lookup took too long')
-    request = urllib.request.Request(API + path, headers={'Authorization': 'Bearer ' + key,
-                                                         'User-Agent': 'FrameControl/1.0'})
-    # Do not carry the credential to redirects or include it in error messages.
-    class NoRedirect(urllib.request.HTTPRedirectHandler):
-        def redirect_request(self, *args, **kwargs):
-            return None
-    with urllib.request.build_opener(NoRedirect()).open(request, timeout=timeout) as response:
-        data = response.read(MAX_JSON + 1)
-    if len(data) > MAX_JSON:
-        raise ValueError('SteamGridDB response too large')
+    from apk_sources import _images
+    # No redirects: the credential never goes anywhere but the API. Errors never contain it.
+    data = _images.get(API + path, {'Authorization': 'Bearer ' + key, 'Accept': 'application/json'}, redirects=0,
+                       deadline=time.monotonic() + 12 if deadline is None else min(deadline, time.monotonic() + 12),
+                       limit=MAX_JSON)
     result = json.loads(data)
     if not isinstance(result, dict) or not result.get('success') or not isinstance(result.get('data'), list):
         raise ValueError('SteamGridDB lookup failed')

ui/frame_titles.py

@@ -696,7 +696,9 @@ def _install(plan, step, artwork=None):
         reply = _json_out(ssh(f'{PY}steam-client-create-shortcut --parms {shlex.quote(json.dumps(parms))}',
                               timeout=90), 'steam-client-create-shortcut')
         meta = {'id': gid, 'name': plan['name'], 'target': plan['target'], 'runtime': plan['runtime'],
-                'source': plan['source'], 'size': plan['size'], 'installed': time.strftime('%Y-%m-%dT%H:%M:%S')}
+                'source': plan['source'], 'size': plan['size'], 'installed': time.strftime('%Y-%m-%dT%H:%M:%S'),
+                # Until art is applied: the only entries automatic backfill may touch.
+                'art_pending': True}
         ssh(f'cat > {GAMES}/{gid}-framecontrol.json', input=json.dumps(meta, indent=1), timeout=30)
         registered = True                    # the files stay: Steam registers them once it's running
         if 'error' in reply:
@@ -709,7 +711,7 @@ def _install(plan, step, artwork=None):
             raise FrameError('Steam registered the title but its shortcut is not available for mandatory artwork; retry install')
         result = frame_android.apply_library(shortcut, plan['name'], directory + '/.frame-artwork', images,
                                              category='Sideloaded', details={'source': plan['source']})
-        meta.update(shortcut=shortcut, artwork=result.get('artwork', {}),
+        meta.update(shortcut=shortcut, artwork=result.get('artwork', {}), art_pending=False,
                     library_warnings=warnings + result.get('warnings', []))
         ssh(f'cat > {GAMES}/{gid}-framecontrol.json', input=json.dumps(meta, indent=1), timeout=30)
         library_ready = True
@@ -768,8 +770,10 @@ def _home():
     return ssh('echo $HOME', timeout=30).strip()
 
 
-def refresh_art(gid=None, artwork=None):
-    """Render and apply Steam artwork for Frame Control's titles (all of them when gid is None)."""
+def refresh_art(gid=None, artwork=None, fill_only=False):
+    """Render and apply Steam artwork for Frame Control's titles (all of them when gid is None).
+
+    fill_only: the automatic backfill; it only fills empty Steam slots (see apply_library)."""
     if gid is None:
         results = []
         for t in list_titles():
@@ -809,8 +813,9 @@ print(json.dumps({{'artwork': cached, 'icon': icon}}))
         if not shortcut:
             raise FrameError(f"Steam hasn't registered {gid} yet; with Steam running on the Frame, refresh again")
         result = frame_android.apply_library(shortcut, name, directory + '/.frame-artwork', images,
-                                             category='Sideloaded', details={'source': meta.get('source')})
-        meta.update(shortcut=shortcut, artwork=result.get('artwork', {}),
+                                             category='Sideloaded', details={'source': meta.get('source')},
+                                             fill_only=fill_only)
+        meta.update(shortcut=shortcut, artwork=result.get('artwork', {}), art_pending=False,
                     library_warnings=warnings + result.get('warnings', []),
                     artwork_refreshed=time.strftime('%Y-%m-%dT%H:%M:%S'))
         ssh(f'cat > {GAMES}/{gid}-framecontrol.json', input=json.dumps(meta, indent=1), timeout=30)
@@ -852,7 +857,9 @@ def list_titles():
                        'runtime': alias, 'runtime_label': RUNTIMES.get(alias, {}).get('label', alias or 'not set'),
                        'source': str(meta.get('source') or ''), 'size': meta.get('size'),
                        'installed': meta.get('installed'), 'registered': t.get('settings') is not None,
-                       'frame_control': bool(meta), 'art_missing': bool(meta) and frame_android.art_missing(meta)})
+                       'frame_control': bool(meta),
+                       # Only a title whose install couldn't apply art; older installs have no flag and keep theirs.
+                       'art_pending': bool(meta.get('art_pending')), 'art_missing': bool(meta.get('art_pending'))})
     return titles
 
 

ui/server.py

@@ -153,11 +153,12 @@ _backfill_lock = threading.Lock()
 
 
 def backfill_art(apps=(), titles=()):
-    """Give installs that lack Steam artwork (Steam wasn't running, say) their art once Steam is up.
+    """Apply art Frame Control couldn't at install time (Steam wasn't running), once Steam is up.
 
-    Runs in the background, at most once every five minutes; refresh-art reports failures on demand."""
-    pkgs = [a["package"] for a in apps if a.get("art_missing")]
-    gids = [t["id"] for t in titles if t.get("art_missing")]
+    Only entries marked art_pending at install; it fills empty Steam slots and never replaces art or
+    names the user may have customised. Runs in the background, at most once every five minutes."""
+    pkgs = [a["package"] for a in apps if a.get("art_pending")]
+    gids = [t["id"] for t in titles if t.get("art_pending")]
     with _backfill_lock:
         if not (pkgs or gids) or _backfill["running"] or time.time() - _backfill["last"] < 300:
             return False
@@ -169,7 +170,7 @@ def backfill_art(apps=(), titles=()):
             for refresh, key in [(frame_android.refresh_art, p) for p in pkgs] + \
                                 [(frame_titles.refresh_art, g) for g in gids]:
                 try:
-                    refresh(key)
+                    refresh(key, fill_only=True)
                 except Exception as e:
                     print(f"artwork backfill for {key}: {e}", file=sys.stderr)
         except Exception: