App data: skip symlinks into the backup manifest; keep one pre-restore copy

Backups no longer abort on a symlink: it is left out and listed (path and
target) in manifest.json, now written last. A hard link is stored as a copy of
its file. Restore removes older pre-restore copies of the same package.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

frame/android/app-data.py

@@ -10,6 +10,7 @@ import time
 
 MAX_BYTES = 20 * 1024 ** 3
 MAX_FILES = 100000
+MAX_MANIFEST = 1024 * 1024
 
 
 def inspect_archive(path, package, instance):
@@ -31,7 +32,7 @@ def inspect_archive(path, package, instance):
             total += member.size
             if total > MAX_BYTES:
                 raise ValueError('archive exceeds 20 GiB')
-            if name == 'manifest.json' and member.isfile() and member.size <= 4096:
+            if name == 'manifest.json' and member.isfile() and member.size <= MAX_MANIFEST:
                 manifest = json.load(archive.extractfile(member))
             elif parts[0] != 'data':
                 raise ValueError('unexpected archive member')
@@ -39,7 +40,8 @@ def inspect_archive(path, package, instance):
                 manifest.get('package') != package or manifest.get('instance') != instance or
                 'data' not in names):
             raise ValueError('backup does not match this package and instance')
-    return {'files': len(names) - 1, 'bytes': total, 'package': package, 'instance': instance}
+    return {'files': len(names) - 1, 'bytes': total, 'package': package, 'instance': instance,
+            'skipped_links': manifest.get('skipped_link_count', 0)}
 
 
 def backup(root, package, instance, output):
@@ -47,22 +49,32 @@ def backup(root, package, instance, output):
     source = root / package
     if source.is_symlink() or not source.is_dir():
         raise ValueError('private app data does not exist or is a symlink')
-    count, total = 0, 0
+    count, total, links, skipped = 0, 0, [], 0
 
     def checked(member):
-        nonlocal count, total
+        nonlocal count, total, skipped
+        if member.issym():  # never followed or restored; listed in the manifest instead
+            skipped += 1
+            if len(links) < 1000:
+                links.append({'path': member.name[:512], 'target': member.linkname[:256]})
+            return None
+        if member.islnk():  # a second name for a file already archived: store its content again
+            member.type, member.linkname = tarfile.REGTYPE, ''
+            member.size = os.lstat(str(source / member.name[len('data/'):])).st_size
         count += 1
         total += member.size
         if not (member.isdir() or member.isfile()) or count > MAX_FILES or total > MAX_BYTES:
-            raise ValueError('private data contains links/special files or exceeds backup limits')
+            raise ValueError('private data contains special files or exceeds backup limits')
         return member
 
-    manifest = json.dumps({'format': 1, 'package': package, 'instance': instance}).encode()
     with tarfile.open(fileobj=output, mode='w|gz', dereference=False) as archive:
+        archive.add(str(source), arcname='data', filter=checked)
+        # Written last so that it can list what was skipped.
+        manifest = json.dumps({'format': 1, 'package': package, 'instance': instance,
+                               'skipped_links': links, 'skipped_link_count': skipped}).encode()
         member = tarfile.TarInfo('manifest.json')
         member.size, member.mode = len(manifest), 0o600
         archive.addfile(member, io.BytesIO(manifest))
-        archive.add(str(source), arcname='data', filter=checked)
 
 
 def restore(root, package, instance, input_stream):
@@ -108,6 +120,10 @@ def restore(root, package, instance, input_stream):
         except BaseException:
             previous.rename(source)
             raise
+        # Keep only the newest pre-restore copy of this package's data.
+        for old in root.glob('.' + package + '.before-restore-*'):
+            if old != previous and not old.is_symlink():
+                shutil.rmtree(str(old), ignore_errors=True)
         result['previous'] = str(previous)
         return result
 

tests/test_frame_android_data.py

@@ -116,6 +116,46 @@ class BackupTests(unittest.TestCase):
             self.assertEqual((source / 'files/save').read_bytes(), b'original save')
             self.assertEqual((Path(result['previous']) / 'files/save').read_bytes(), b'new save')
 
+    def test_symlinks_skipped_and_recorded_hardlinks_copied(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            source = root / PKG
+            (source / 'files').mkdir(parents=True)
+            (source / 'files/save').write_bytes(b'save')
+            os.link(str(source / 'files/save'), str(source / 'files/save-link'))
+            os.symlink('/data/app/lib', str(source / 'lib'))
+            os.symlink('save', str(source / 'files/alias'))
+            archive = root / 'backup.tar.gz'
+            with archive.open('wb') as output:
+                REMOTE['backup'](root, PKG, META['instance'], output)
+            result = REMOTE['inspect_archive'](archive, PKG, META['instance'])
+            self.assertEqual(result['skipped_links'], 2)
+            with tarfile.open(archive) as tar:
+                manifest = json.load(tar.extractfile('manifest.json'))
+                self.assertEqual(tar.extractfile('data/files/save-link').read(), b'save')
+            self.assertEqual(sorted((l['path'], l['target']) for l in manifest['skipped_links']),
+                             [('data/files/alias', 'save'), ('data/lib', '/data/app/lib')])
+            with archive.open('rb') as src:
+                REMOTE['restore'](root, PKG, META['instance'], src)
+            self.assertFalse((source / 'lib').exists() or (source / 'lib').is_symlink())
+            self.assertEqual((source / 'files/save-link').read_bytes(), b'save')
+
+    def test_restore_keeps_only_latest_previous_copy(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            (root / PKG).mkdir()
+            (root / PKG / 'save').write_bytes(b'one')
+            other = root / '.org.example.gameplus.before-restore-1'  # another package's copy is left alone
+            other.mkdir()
+            archive = io.BytesIO()
+            REMOTE['backup'](root, PKG, META['instance'], archive)
+            previous = []
+            for _ in range(3):
+                archive.seek(0)
+                previous.append(REMOTE['restore'](root, PKG, META['instance'], archive)['previous'])
+            self.assertEqual(sorted(root.glob('.' + PKG + '.before-restore-*')), [Path(previous[-1])])
+            self.assertTrue(other.exists())
+
     def make_archive(self, path, members, package=PKG):
         with tarfile.open(path, 'w:gz') as archive:
             payload = json.dumps({'format': 1, 'package': package, 'instance': META['instance']}).encode()