diff --git a/frame/android/app-data.py b/frame/android/app-data.py index 1ecc141..4b216be 100644 --- a/frame/android/app-data.py +++ b/frame/android/app-data.py @@ -10,6 +10,7 @@ import time MAX_BYTES = 20 * 1024 ** 3 MAX_FILES = 100000 +MAX_MANIFEST = 1024 * 1024 def inspect_archive(path, package, instance): @@ -31,7 +32,7 @@ def inspect_archive(path, package, instance): total += member.size if total > MAX_BYTES: raise ValueError('archive exceeds 20 GiB') - if name == 'manifest.json' and member.isfile() and member.size <= 4096: + if name == 'manifest.json' and member.isfile() and member.size <= MAX_MANIFEST: manifest = json.load(archive.extractfile(member)) elif parts[0] != 'data': raise ValueError('unexpected archive member') @@ -39,7 +40,8 @@ def inspect_archive(path, package, instance): manifest.get('package') != package or manifest.get('instance') != instance or 'data' not in names): raise ValueError('backup does not match this package and instance') - return {'files': len(names) - 1, 'bytes': total, 'package': package, 'instance': instance} + return {'files': len(names) - 1, 'bytes': total, 'package': package, 'instance': instance, + 'skipped_links': manifest.get('skipped_link_count', 0)} def backup(root, package, instance, output): @@ -47,22 +49,32 @@ def backup(root, package, instance, output): source = root / package if source.is_symlink() or not source.is_dir(): raise ValueError('private app data does not exist or is a symlink') - count, total = 0, 0 + count, total, links, skipped = 0, 0, [], 0 def checked(member): - nonlocal count, total + nonlocal count, total, skipped + if member.issym(): # never followed or restored; listed in the manifest instead + skipped += 1 + if len(links) < 1000: + links.append({'path': member.name[:512], 'target': member.linkname[:256]}) + return None + if member.islnk(): # a second name for a file already archived: store its content again + member.type, member.linkname = tarfile.REGTYPE, '' + member.size = os.lstat(str(source / member.name[len('data/'):])).st_size count += 1 total += member.size if not (member.isdir() or member.isfile()) or count > MAX_FILES or total > MAX_BYTES: - raise ValueError('private data contains links/special files or exceeds backup limits') + raise ValueError('private data contains special files or exceeds backup limits') return member - manifest = json.dumps({'format': 1, 'package': package, 'instance': instance}).encode() with tarfile.open(fileobj=output, mode='w|gz', dereference=False) as archive: + archive.add(str(source), arcname='data', filter=checked) + # Written last so that it can list what was skipped. + manifest = json.dumps({'format': 1, 'package': package, 'instance': instance, + 'skipped_links': links, 'skipped_link_count': skipped}).encode() member = tarfile.TarInfo('manifest.json') member.size, member.mode = len(manifest), 0o600 archive.addfile(member, io.BytesIO(manifest)) - archive.add(str(source), arcname='data', filter=checked) def restore(root, package, instance, input_stream): @@ -108,6 +120,10 @@ def restore(root, package, instance, input_stream): except BaseException: previous.rename(source) raise + # Keep only the newest pre-restore copy of this package's data. + for old in root.glob('.' + package + '.before-restore-*'): + if old != previous and not old.is_symlink(): + shutil.rmtree(str(old), ignore_errors=True) result['previous'] = str(previous) return result diff --git a/tests/test_frame_android_data.py b/tests/test_frame_android_data.py index 3f4c3fe..5ecf603 100644 --- a/tests/test_frame_android_data.py +++ b/tests/test_frame_android_data.py @@ -116,6 +116,46 @@ class BackupTests(unittest.TestCase): self.assertEqual((source / 'files/save').read_bytes(), b'original save') self.assertEqual((Path(result['previous']) / 'files/save').read_bytes(), b'new save') + def test_symlinks_skipped_and_recorded_hardlinks_copied(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = root / PKG + (source / 'files').mkdir(parents=True) + (source / 'files/save').write_bytes(b'save') + os.link(str(source / 'files/save'), str(source / 'files/save-link')) + os.symlink('/data/app/lib', str(source / 'lib')) + os.symlink('save', str(source / 'files/alias')) + archive = root / 'backup.tar.gz' + with archive.open('wb') as output: + REMOTE['backup'](root, PKG, META['instance'], output) + result = REMOTE['inspect_archive'](archive, PKG, META['instance']) + self.assertEqual(result['skipped_links'], 2) + with tarfile.open(archive) as tar: + manifest = json.load(tar.extractfile('manifest.json')) + self.assertEqual(tar.extractfile('data/files/save-link').read(), b'save') + self.assertEqual(sorted((l['path'], l['target']) for l in manifest['skipped_links']), + [('data/files/alias', 'save'), ('data/lib', '/data/app/lib')]) + with archive.open('rb') as src: + REMOTE['restore'](root, PKG, META['instance'], src) + self.assertFalse((source / 'lib').exists() or (source / 'lib').is_symlink()) + self.assertEqual((source / 'files/save-link').read_bytes(), b'save') + + def test_restore_keeps_only_latest_previous_copy(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / PKG).mkdir() + (root / PKG / 'save').write_bytes(b'one') + other = root / '.org.example.gameplus.before-restore-1' # another package's copy is left alone + other.mkdir() + archive = io.BytesIO() + REMOTE['backup'](root, PKG, META['instance'], archive) + previous = [] + for _ in range(3): + archive.seek(0) + previous.append(REMOTE['restore'](root, PKG, META['instance'], archive)['previous']) + self.assertEqual(sorted(root.glob('.' + PKG + '.before-restore-*')), [Path(previous[-1])]) + self.assertTrue(other.exists()) + def make_archive(self, path, members, package=PKG): with tarfile.open(path, 'w:gz') as archive: payload = json.dumps({'format': 1, 'package': package, 'instance': META['instance']}).encode()