Merge pull request #70 from DavidutzDev/feat/prefix-location

Add darling-lol --set-prefix to keep the prefix on another drive

darling-lol-launcher.sh

@@ -9,7 +9,17 @@ UNSHARE_BIN="@UNSHARE_BIN@"
 export DARLING_PACKAGE_ID="$SHARE_DIR"
 export DISPLAY="${DISPLAY:-:0}"
 export EGL_PLATFORM="${EGL_PLATFORM:-x11}"
+# The guest prefix: DPREFIX if set, else the one saved by --set-prefix, else ~/.darling.
+if [ -z "${DPREFIX:-}" ] && [ -f "$WORKSPACE/build/config.json" ]; then
+  DPREFIX=$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1])).get("prefix", ""))' \
+    "$WORKSPACE/build/config.json" 2>/dev/null) || DPREFIX=
+fi
 export DPREFIX="${DPREFIX:-$HOME/.darling}"
+if [ "${1:-}" = --set-prefix ]; then
+  [ $# -eq 2 ] || { echo 'Usage: darling-lol --set-prefix PATH' >&2; exit 2; }
+  # Resolve here: the namespace starts in the workspace, not in this directory.
+  set -- --set-prefix "$(realpath -m -- "$2")"
+fi
 export DARLING_HOST_UID="$(id -u)"
 export DARLING_HOST_GID="$(id -g)"
 export DARLING_WORKSPACE="$WORKSPACE"
@@ -73,6 +83,8 @@ session() {
 case "${1:-}" in
   --shell) shift; "${fhs_command[@]}" bash "$@";;
   --update) shift; "${fhs_command[@]}" ./scripts/update.sh "$@";;
+  # In the namespace, whose root owns the prefix files, so a copy keeps owners.
+  --set-prefix) "${fhs_command[@]}" python3 ./scripts/nix-prefix.py "$2";;
   --installer) shift; session --installer "$@";;
   *)
     if [ $# -gt 0 ]; then

docs/NIX.md

@@ -210,8 +210,18 @@ Darling namespace; `darling-lol --stop` can still stop it.
 ## Files and operational limits
 
 - Workspace: `${XDG_DATA_HOME:-$HOME/.local/share}/darling-lol`.
-- Default guest prefix: `$HOME/.darling`; set `DPREFIX` to use another prefix.
-  Explicit `DPREFIX` and `RIOT_GUEST_HOME` overrides take precedence over saved
+- Default guest prefix: `$HOME/.darling`. The prefix holds the installed
+  clients and game, about 40 GB. To keep it on another drive, close the clients
+  and run `darling-lol --set-prefix /mnt/games/league`. This moves an existing
+  prefix there and saves the path for later launches. Across filesystems it
+  copies first and removes the old prefix only after the copy completes. If the
+  path already holds a Darling prefix, it switches to it and moves nothing. The
+  path must not contain commas or colons (overlayfs uses them as separators).
+  Across filesystems the target must support `user.*` extended attributes (ext4,
+  btrfs, xfs; not NTFS or exFAT), which is checked before copying and again on
+  the finished copy. The new path is saved before the old prefix is deleted.
+  The next launch sets up the runtime in the new prefix.
+- Explicit `DPREFIX` and `RIOT_GUEST_HOME` overrides take precedence over saved
   configuration, so clear stale overrides before switching installations.
 - Package upgrades refresh managed tools and remove previously tracked obsolete
   files. Builds, the installed runtime, guest prefixes, and unmanaged files are

scripts/nix-prefix.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""darling-lol --set-prefix PATH: choose where the guest prefix, and with it the
+installed games, is stored.
+
+An existing prefix is moved to PATH. Across filesystems it is copied first and
+the old copy is removed only once the new one is complete. If PATH already holds
+a Darling prefix, the launcher switches to it and leaves both prefixes alone.
+"""
+import errno
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import tempfile
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def has_files(path):
+    return path.is_dir() and any(path.iterdir())
+
+
+def is_prefix(path):
+    # Darling creates this mount point in every prefix it starts.
+    return (path / 'Volumes/SystemRoot').is_dir()
+
+
+XATTR = 'user.darling-lol-prefix-check'
+
+
+def user_xattrs(path):
+    try:
+        return {name: os.getxattr(path, name, follow_symlinks=False)
+                for name in os.listxattr(path, follow_symlinks=False) if name.startswith('user.')}
+    except OSError as error:
+        if error.errno in (errno.ENOTSUP, errno.ENOENT):
+            return {}
+        raise
+
+
+def check_xattrs(folder):
+    """Darling keeps state in user.* xattrs; cp -a ignores a target that drops them."""
+    probe = folder / '.xattr-check'
+    probe.touch()
+    try:
+        os.setxattr(probe, XATTR, b'1')
+        if os.getxattr(probe, XATTR) != b'1':
+            raise OSError(errno.ENOTSUP, 'xattr changed')
+    except OSError as error:
+        raise SystemExit(f'{folder.parent} does not support user extended attributes ({error.strerror}), '
+                         'which the prefix needs. Choose a drive with a Linux filesystem '
+                         '(ext4, btrfs, xfs); nothing was moved.')
+    finally:
+        probe.unlink()
+
+
+def verify_copy(old, new):
+    """The copy must match the original: names, types and user.* xattrs."""
+    for folder, folders, files in os.walk(old):
+        target = new / Path(folder).relative_to(old)
+        for name in [*folders, *files]:
+            source = Path(folder) / name
+            copy = target / name
+            if not copy.is_symlink() and not copy.exists() or user_xattrs(source) != user_xattrs(copy):
+                raise SystemExit(f'The copy of {source} differs from the original; the old prefix was kept.')
+        if user_xattrs(folder) != user_xattrs(target):
+            raise SystemExit(f'The copy of {folder} differs from the original; the old prefix was kept.')
+
+
+def move(old, new):
+    """Put the prefix at new. Returns old if it still has to be removed, else None."""
+    new.parent.mkdir(parents=True, exist_ok=True)
+    if new.is_dir():
+        new.rmdir()
+    try:
+        os.rename(old, new)
+        return None
+    except OSError as error:
+        if error.errno != errno.EXDEV:
+            raise
+    # A directory made for this move: never touch anything that was already there.
+    staging = Path(tempfile.mkdtemp(prefix=new.name + '.partial-', dir=new.parent))
+    try:
+        check_xattrs(staging)
+        print(f'Copying {old} to {new}; this can take several minutes.', flush=True)
+        subprocess.run(['cp', '-a', '--', str(old), str(staging / 'prefix')], check=True)
+        verify_copy(old, staging / 'prefix')
+        os.rename(staging / 'prefix', new)
+    finally:
+        shutil.rmtree(staging, ignore_errors=True)
+    return old
+
+
+def remove(path):
+    try:
+        shutil.rmtree(path)
+    except OSError as error:
+        print(f'Could not remove {path} ({error}); delete it by hand to free the space.', file=sys.stderr)
+
+
+def main():
+    old = Path(os.environ['DPREFIX']).resolve()
+    new = Path(sys.argv[1]).expanduser()
+    if not new.is_absolute():
+        raise SystemExit('The prefix must be an absolute path.')
+    new = new.resolve()
+    # Darling passes the prefix to overlayfs, which splits its options on these.
+    if any(character in str(new) for character in ',:\n'):
+        raise SystemExit('The prefix path must not contain commas, colons or newlines.')
+    # Darling's overlay work directory; it creates a new one next to the prefix.
+    workdir = old.with_name(old.name + '.workdir')
+    leftovers = []
+    if new != old:
+        if new.is_relative_to(old) or old.is_relative_to(new):
+            raise SystemExit(f'{new} and {old} must not contain each other.')
+        if new.is_relative_to(workdir) or workdir.is_relative_to(new):
+            raise SystemExit(f'{new} must not be inside {workdir}, which is removed after the move.')
+        if has_files(new):
+            if not is_prefix(new):
+                raise SystemExit(f'{new} is not empty and is not a Darling prefix; '
+                                 'choose a new or empty directory.')
+            print(f'Using the existing prefix in {new}; {old} is left as it is.')
+        elif has_files(old):
+            print(f'Moving the prefix {old} to {new}.', flush=True)
+            if move(old, new):
+                leftovers.append(old)
+            leftovers.append(workdir)
+    # Save the new path before deleting anything, so a failed delete leaves a working install.
+    subprocess.run([sys.executable, str(ROOT / 'scripts/configure.py'), '--prefix', str(new)],
+                   cwd=ROOT, stdout=subprocess.DEVNULL, check=True)
+    for path in leftovers:
+        remove(path)
+    print(f'The prefix is now {new}. The next darling-lol launch sets up the runtime there if needed.')
+    return 0
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

tests/test_nix_launcher.py

@@ -119,6 +119,25 @@ class LauncherTests(unittest.TestCase):
         self.assertEqual(session.wait(timeout=5), 0)
         self.assertFalse(self.log.exists())
 
+    def test_saved_prefix_is_used_unless_dprefix_is_set(self):
+        (self.bin / 'unshare').write_text('#!/bin/sh\nprintf "%s\\n" "$DPREFIX" "$@" > "$TEST_LOG"\n')
+        (self.workspace / 'build').mkdir(parents=True)
+        (self.workspace / 'build/config.json').write_text('{"prefix": "/mnt/games/League Of Legends"}')
+        self.env.pop('DPREFIX', None)
+        self.assertEqual(self.launch('true').returncode, 0)
+        self.assertEqual(self.log.read_text().splitlines()[0], '/mnt/games/League Of Legends')
+        self.env['DPREFIX'] = '/explicit'
+        self.assertEqual(self.launch('true').returncode, 0)
+        self.assertEqual(self.log.read_text().splitlines()[0], '/explicit')
+
+    def test_set_prefix_resolves_the_path_from_the_callers_directory(self):
+        result = subprocess.run(['bash', str(self.launcher), '--set-prefix', 'Games/League'],
+                                env=self.env, cwd=self.folder, text=True, capture_output=True, timeout=10)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(self.log.read_text().splitlines()[-3:],
+                         ['python3', './scripts/nix-prefix.py', str(self.folder.resolve() / 'Games/League')])
+        self.assertEqual(self.launch('--set-prefix').returncode, 2)
+
     def test_setup_never_makes_a_user_writable_runtime_setuid(self):
         result = self.launch('--installer')
         self.assertEqual(result.returncode, 0, result.stderr)

tests/test_nix_prefix.py

@@ -0,0 +1,191 @@
+"""darling-lol --set-prefix on temporary directories; no Darling or namespace."""
+import errno
+import importlib.util
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+class PrefixTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.folder = Path(self.tmp.name)
+        self.workspace = self.folder / 'workspace'
+        (self.workspace / 'scripts').mkdir(parents=True)
+        (self.workspace / 'tools').mkdir()
+        for name in ('nix-prefix.py', 'configure.py', 'common.py'):
+            shutil.copy(ROOT / 'scripts' / name, self.workspace / 'scripts' / name)
+        self.old = self.folder / 'home/.darling'
+        (self.old / 'Volumes/SystemRoot').mkdir(parents=True)
+        (self.old / 'Applications/League of Legends.app').mkdir(parents=True)
+        (self.old / 'Applications/League of Legends.app/game').write_text('game data')
+        (self.folder / 'home/.darling.workdir/work').mkdir(parents=True)
+
+    def set_prefix(self, new):
+        return subprocess.run([sys.executable, str(self.workspace / 'scripts/nix-prefix.py'), str(new)],
+                              env=dict(os.environ, DPREFIX=str(self.old), HOME=str(self.folder / 'home'),
+                                       PYTHONDONTWRITEBYTECODE='1'),
+                              text=True, capture_output=True, timeout=30)
+
+    def saved_prefix(self):
+        return json.loads((self.workspace / 'build/config.json').read_text())['prefix']
+
+    def test_moves_the_prefix_and_saves_it(self):
+        new = self.folder / 'bulk/Games/League Of Legends'
+        result = self.set_prefix(new)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual((new / 'Applications/League of Legends.app/game').read_text(), 'game data')
+        self.assertFalse(self.old.exists())
+        self.assertFalse((self.folder / 'home/.darling.workdir').exists())
+        self.assertEqual(self.saved_prefix(), str(new))
+
+    def test_copies_across_filesystems_and_removes_the_old_copy_last(self):
+        spec = importlib.util.spec_from_file_location('nix_prefix', self.workspace / 'scripts/nix-prefix.py')
+        prefix = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(prefix)
+        rename = os.rename
+        calls = []
+
+        def cross_device(source, target):
+            calls.append(Path(target).name)
+            if len(calls) == 1:
+                raise OSError(errno.EXDEV, 'cross-device link')
+            self.assertTrue(self.old.exists(), 'old prefix removed before the copy finished')
+            rename(source, target)
+
+        new = self.folder / 'bulk/prefix'
+        with patch.object(prefix.os, 'rename', cross_device):
+            leftover = prefix.move(self.old, new)
+        # A failed rename of the prefix, then the rename of the finished copy.
+        self.assertEqual(calls, ['prefix', 'prefix'])
+        self.assertEqual((new / 'Applications/League of Legends.app/game').read_text(), 'game data')
+        # The caller removes the old prefix once the new path is saved.
+        self.assertEqual(leftover, self.old)
+        self.assertTrue(self.old.exists())
+        self.assertEqual(list((self.folder / 'bulk').iterdir()), [new])
+
+    def load(self):
+        spec = importlib.util.spec_from_file_location('nix_prefix_unsafe', self.workspace / 'scripts/nix-prefix.py')
+        prefix = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(prefix)
+        return prefix
+
+    def cross_device(self, prefix):
+        rename = os.rename
+        first = []
+
+        def rename_across(source, target):
+            if not first:
+                first.append(1)
+                raise OSError(errno.EXDEV, 'cross-device link')
+            rename(source, target)
+        return patch.object(prefix.os, 'rename', rename_across)
+
+    def test_copy_leaves_an_existing_partial_directory_alone(self):
+        prefix = self.load()
+        new = self.folder / 'bulk/prefix'
+        unrelated = self.folder / 'bulk/prefix.partial'
+        unrelated.mkdir(parents=True)
+        (unrelated / 'keep').write_text('mine')
+        with self.cross_device(prefix):
+            prefix.move(self.old, new)
+        self.assertEqual((unrelated / 'keep').read_text(), 'mine')
+        self.assertEqual(sorted(path.name for path in (self.folder / 'bulk').iterdir()),
+                         ['prefix', 'prefix.partial'])
+
+    def test_failed_copy_keeps_the_old_prefix_and_removes_only_its_staging(self):
+        prefix = self.load()
+        new = self.folder / 'bulk/prefix'
+        new.parent.mkdir()
+        error = subprocess.CalledProcessError(1, 'cp')
+        with self.cross_device(prefix), patch.object(prefix.subprocess, 'run', side_effect=error):
+            with self.assertRaises(subprocess.CalledProcessError):
+                prefix.move(self.old, new)
+        self.assertEqual((self.old / 'Applications/League of Legends.app/game').read_text(), 'game data')
+        self.assertEqual(list(new.parent.iterdir()), [])
+
+    def test_copy_refused_when_the_target_drops_user_xattrs(self):
+        prefix = self.load()
+        new = self.folder / 'bulk/prefix'
+        with self.cross_device(prefix), patch.object(prefix.os, 'setxattr',
+                                                     side_effect=OSError(errno.ENOTSUP, 'not supported')):
+            with self.assertRaises(SystemExit) as raised:
+                prefix.move(self.old, new)
+        self.assertIn('extended attributes', str(raised.exception))
+        self.assertTrue(self.old.exists())
+        self.assertEqual(list(new.parent.iterdir()), [])
+
+    def test_copy_that_lost_xattrs_is_refused(self):
+        prefix = self.load()
+        game = self.old / 'Applications/League of Legends.app/game'
+        try:
+            os.setxattr(game, 'user.test', b'1')
+        except OSError:
+            self.skipTest('the temporary directory has no user xattrs')
+        new = self.folder / 'bulk/prefix'
+        real = prefix.user_xattrs
+        stripped = lambda path: {} if Path(path).name == 'game' and Path(path).is_relative_to(new.parent) else real(path)
+        with self.cross_device(prefix), patch.object(prefix, 'user_xattrs', stripped):
+            with self.assertRaises(SystemExit) as raised:
+                prefix.move(self.old, new)
+        self.assertIn('old prefix was kept', str(raised.exception))
+        self.assertTrue(game.exists())
+        self.assertEqual(list(new.parent.iterdir()), [])
+
+    def test_failed_delete_of_the_old_prefix_keeps_the_new_one_saved(self):
+        prefix = self.load()
+        new = self.folder / 'bulk/prefix'
+        with patch.dict(os.environ, {'DPREFIX': str(self.old)}), \
+                patch.object(sys, 'argv', ['nix-prefix.py', str(new)]), \
+                patch.object(prefix, 'ROOT', self.workspace), \
+                patch.object(prefix, 'move', lambda old, target: (shutil.copytree(old, target), old)[1]), \
+                patch.object(prefix.shutil, 'rmtree', side_effect=OSError('busy')):
+            self.assertEqual(prefix.main(), 0)
+        self.assertEqual(self.saved_prefix(), str(new))
+        self.assertTrue((new / 'Applications/League of Legends.app/game').exists())
+        self.assertTrue(self.old.exists())
+
+    def test_refuses_a_destination_inside_the_overlay_work_directory(self):
+        for new in (self.folder / 'home/.darling.workdir/games', self.folder / 'home/.darling.workdir'):
+            result = self.set_prefix(new)
+            self.assertNotEqual(result.returncode, 0, new)
+            self.assertIn('work', result.stderr)
+        self.assertTrue((self.old / 'Applications/League of Legends.app/game').exists())
+        self.assertFalse((self.workspace / 'build/config.json').exists())
+
+    def test_switches_to_an_existing_prefix_without_moving(self):
+        other = self.folder / 'other'
+        (other / 'Volumes/SystemRoot').mkdir(parents=True)
+        result = self.set_prefix(other)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertTrue((self.old / 'Applications/League of Legends.app/game').exists())
+        self.assertEqual(self.saved_prefix(), str(other))
+
+    def test_refuses_a_directory_with_other_files(self):
+        games = self.folder / 'Games'
+        games.mkdir()
+        (games / 'other-game').mkdir()
+        result = self.set_prefix(games)
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn('not a Darling prefix', result.stderr)
+        self.assertTrue(self.old.exists())
+        self.assertFalse((self.workspace / 'build/config.json').exists())
+
+    def test_refuses_overlay_separators_and_nested_paths(self):
+        for new in (self.folder / 'a,b', self.folder / 'a:b', self.old / 'inner'):
+            result = self.set_prefix(new)
+            self.assertNotEqual(result.returncode, 0, new)
+        self.assertTrue(self.old.exists())
+
+
+if __name__ == '__main__':
+    unittest.main()