diff --git a/.gitignore b/.gitignore index fcb772d..c6bbd4b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,5 @@ __pycache__/ .claude .codex/ - +/docs test/ diff --git a/commands/db.py b/commands/db.py index 182b11c..0d0c8d0 100644 --- a/commands/db.py +++ b/commands/db.py @@ -96,7 +96,10 @@ def run( bool, typer.Option("--password-stdin", help="Read password from stdin") ] = False, path: Annotated[ - str | None, typer.Option("--path", help="SQLite file path (existing)") + str | None, + typer.Option( + "--path", help="SQLite file path (existing) or files directory" + ), ] = None, db_name: Annotated[ str | None, typer.Option("--name", help="SQLite file name (new)") @@ -243,7 +246,7 @@ def run(self, name: NameArg) -> None: ) user = ( "N/A" - if database.engine in ("sqlite", "docker-volume") + if database.engine in ("sqlite", "docker-volume", "files") else (database.username or "") ) rows.append( diff --git a/engines/__init__.py b/engines/__init__.py index 4422409..f909427 100644 --- a/engines/__init__.py +++ b/engines/__init__.py @@ -5,6 +5,7 @@ from core.errors import ValidationError from engines.base import DbEngine from engines.docker_volume import DockerVolumeEngine +from engines.files import FilesEngine from engines.firebird import FirebirdEngine from engines.mariadb import MariaDbEngine from engines.mongodb import MongoEngine @@ -58,6 +59,7 @@ def __contains__(self, key: str) -> bool: ValkeyEngine(), MssqlEngine(), DockerVolumeEngine(), + FilesEngine(), ) registry = EngineRegistry(ALL) diff --git a/engines/base.py b/engines/base.py index 444a8d7..e086d5a 100644 --- a/engines/base.py +++ b/engines/base.py @@ -126,6 +126,11 @@ def agent_database(self, spec: DatabaseSpec) -> str: def describe(self, spec: DatabaseSpec) -> str: return f"{spec.host}:{spec.port}" + @staticmethod + def mount_for(spec: DatabaseSpec) -> tuple[str, str] | None: + """(host, container) bind mount the agent needs for this database.""" + return None + def non_default_options(self, spec: DatabaseSpec) -> dict[str, Any]: defaults = {field.name: field.default for field in self.option_fields()} return { diff --git a/engines/files.py b/engines/files.py new file mode 100644 index 0000000..d6d971e --- /dev/null +++ b/engines/files.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import posixpath +from typing import Any + +from core.errors import ValidationError +from core.fields import Field +from core.specs import DatabaseSpec +from engines.base import DbEngine +from services.ports import PortAllocator + +DATA_DIR = "/data" + + +def host_dir(value: str) -> str: + path = posixpath.normpath(str(value).strip()) + if not path.startswith("/") or not path.strip("/"): + raise ValidationError( + f"Path must be an absolute directory other than /, got {value!r}", + hint="e.g. /srv/uploads", + ) + if any(char in path for char in ":#\n"): + raise ValidationError( + f"Path cannot contain ':', '#' or a newline, got {value!r}", + hint="They break the Docker Compose bind mount.", + ) + return path + + +def no_braces(patterns: list[str]) -> list[str]: + if any("{" in pattern or "}" in pattern for pattern in patterns): + raise ValidationError( + "Brace alternation ({a,b}) is not supported in exclude patterns.", + hint="List each pattern separately: *.log,*.tmp", + ) + return patterns + + +class FilesEngine(DbEngine): + key, display = "files", "Files" + template = None + has_modes = False + label_default = "Files" + warning = ( + f"The directory is mounted read-write under {DATA_DIR}: a restore replaces " + "its content (excluded paths are kept). Requires Agent 1.24.0+." + ) + + def fields_existing(self) -> list[Field]: + return [ + Field( + "path", + "Directory on the host (absolute, e.g. /srv/uploads)", + "text", + validator=host_dir, + ) + ] + + def fields_new(self) -> list[Field]: + return self.fields_existing() + + def option_fields(self) -> list[Field]: + return [ + Field( + "method", + "Backup method", + "choice", + default="archive", + choices=("archive", "snapshot", "sync"), + help=( + "archive: a tar.gz archive per run. snapshot: incremental, " + "deduplicated restic snapshots. sync: mirrors the directory onto " + "each storage; deletions propagate and it cannot be restored from " + "Portabase. Cannot be changed once the source exists." + ), + ), + Field( + "exclude", + "Exclude patterns", + "list", + default=[], + validator=no_braces, + help=( + "Comma-separated patterns, e.g. node_modules,*.tmp,/cache. Without " + "a leading / a pattern matches at any depth; with one it is " + "relative to the directory. Excluded paths are kept on restore." + ), + ), + Field( + "one_file_system", + "Stay on one file system?", + "bool", + default=False, + help="Skip mount points found inside the directory.", + ), + ] + + def generate( + self, *, auth: bool, ports: PortAllocator, answers: dict[str, Any] + ) -> DatabaseSpec: + return self.from_existing(answers) + + def from_existing(self, answers: dict[str, Any]) -> DatabaseSpec: + return DatabaseSpec( + id=self.new_id(), + engine=self.key, + name=answers.get("label") or self.label_default, + managed=False, + path=DATA_DIR + answers["path"], + ) + + @staticmethod + def mount_for(spec: DatabaseSpec) -> tuple[str, str] | None: + if spec.path and spec.path.startswith(f"{DATA_DIR}/"): + return (spec.path[len(DATA_DIR) :], spec.path) + return None + + def env_vars(self, spec: DatabaseSpec) -> dict[str, str]: + return {} + + def agent_entry(self, spec: DatabaseSpec) -> dict[str, Any]: + entry: dict[str, Any] = { + "name": spec.name, + "type": self.key, + "path": spec.path, + "generated_id": spec.id, + } + options = self.non_default_options(spec) + if options: + entry["options"] = options + return entry + + def describe(self, spec: DatabaseSpec) -> str: + mount = self.mount_for(spec) + return f"dir: {mount[0] if mount else spec.path}" diff --git a/services/project.py b/services/project.py index c59989d..78f7fb1 100644 --- a/services/project.py +++ b/services/project.py @@ -7,8 +7,8 @@ from core.errors import ConfigError, ValidationError from core.specs import DatabaseSpec +from engines import registry from engines.base import DbEngine -from engines.sqlite import SqliteEngine from services import auth_providers as ap from services import settings as cfg from services.compose_facts import CA_BUNDLE_IN_CONTAINER, ComposeFacts @@ -61,7 +61,7 @@ def spec_from_entry(entry: dict[str, Any], env: EnvFile) -> DatabaseSpec: username=str(entry["username"]) if entry.get("username") is not None else None, password=_opt_str(entry, "password"), root_password=root_password, - path=_opt_str(entry, "database") if engine == "sqlite" else None, + path=_opt_str(entry, "database" if engine == "sqlite" else "path"), volume=_opt_str(entry, "volume_name"), container=_opt_str(entry, "container_name"), options=dict(entry.get("options") or {}), @@ -117,13 +117,12 @@ def needs_docker_socket(self) -> bool: return any(database.engine == "docker-volume" for database in self.databases) @property - def sqlite_mounts(self) -> list[tuple[str, str]]: + def mounts(self) -> list[tuple[str, str]]: mounts: list[tuple[str, str]] = [] for database in self.databases: - if database.engine == "sqlite": - mount = SqliteEngine.mount_for(database) - if mount and mount not in mounts: - mounts.append(mount) + mount = registry.get(database.engine).mount_for(database) + if mount and mount not in mounts: + mounts.append(mount) return mounts def validate(self) -> None: diff --git a/services/renderer.py b/services/renderer.py index 8c69ae3..3677ecb 100644 --- a/services/renderer.py +++ b/services/renderer.py @@ -126,7 +126,7 @@ def render_agent( "docker_socket": project.needs_docker_socket, "mounts": [ {"host": host, "container": container} - for host, container in project.sqlite_mounts + for host, container in project.mounts ], "services": [self._service(spec, inline) for spec in project.managed], "tz_var": _var(env, "TZ", inline), diff --git a/tests/engines/files.py b/tests/engines/files.py new file mode 100644 index 0000000..3a0bd1c --- /dev/null +++ b/tests/engines/files.py @@ -0,0 +1,114 @@ +import pytest + +from core.errors import ValidationError +from core.specs import DatabaseSpec +from engines import registry +from engines.files import FilesEngine +from tests.support import agent_service, field_specs + +FILES = registry.get("files") +MOUNT = "/srv/uploads:/data/srv/uploads" + + +def attributes(): + assert type(FILES) is FilesEngine + assert (FILES.key, FILES.display, FILES.default_port) == ("files", "Files", None) + assert FILES.template is None + assert (FILES.auth_variants, FILES.has_modes) == (False, False) + assert "restore" in (FILES.warning or "") + + +def generate(ports): + answers = {"path": "/srv/uploads", "label": "Uploads"} + spec = FILES.generate(auth=False, ports=ports, answers=answers) + assert spec == DatabaseSpec( + id=spec.id, engine="files", name="Uploads", path="/data/srv/uploads" + ) + assert FILES.describe(spec) == "dir: /srv/uploads" + + +def env_vars(): + assert FILES.env_vars(FILES.from_existing({"path": "/srv"})) == {} + + +def agent_entry(): + bare = FILES.from_existing({"path": "/srv/uploads"}) + assert FILES.agent_entry(bare) == { + "name": "Files", + "type": "files", + "path": "/data/srv/uploads", + "generated_id": bare.id, + } + tuned = bare.with_options( + {"method": "snapshot", "exclude": ["*.tmp"], "one_file_system": False} + ) + assert FILES.agent_entry(tuned)["options"] == { + "method": "snapshot", + "exclude": ["*.tmp"], + } + + +def fields(): + assert field_specs(FILES.fields_existing()) == [("path", "text", None)] + assert FILES.fields_new() == FILES.fields_existing() + assert field_specs(FILES.option_fields()) == [ + ("method", "choice", "archive"), + ("exclude", "list", []), + ("one_file_system", "bool", False), + ] + assert FILES.option_fields()[0].choices == ("archive", "snapshot", "sync") + + +def from_existing(): + spec = FILES.from_existing({"path": "/srv/uploads"}) + assert spec == DatabaseSpec( + id=spec.id, engine="files", name="Files", path="/data/srv/uploads" + ) + + +def compose_service(render_engine): + rendered = render_engine("files", answers={"path": "/srv/uploads"}) + assert rendered.doc["services"] == {"agent": agent_service(MOUNT)} + assert "volumes" not in rendered.doc + assert rendered.databases == [FILES.agent_entry(rendered.spec)] + + +def compose_service_inline(render_engine): + rendered = render_engine("files", inline=True, answers={"path": "/srv/uploads"}) + assert rendered.doc["services"] == {"agent": agent_service(MOUNT, inline=True)} + + +@pytest.mark.parametrize( + ("raw", "normalized"), + [("/srv/uploads", "/srv/uploads"), (" /srv/uploads/ ", "/srv/uploads")], +) +def path_is_normalized(raw, normalized): + assert FILES.fields_existing()[0].validator(raw) == normalized + + +@pytest.mark.parametrize( + "raw", ["uploads", "./uploads", "/", "//", "C:\\data", "/srv/a:b", "/srv/#1"] +) +def path_rejects_unmountable_values(raw): + with pytest.raises(ValidationError): + FILES.fields_existing()[0].validator(raw) + + +def exclude_rejects_brace_alternation(): + validator = FILES.option_fields()[1].validator + assert validator(["node_modules", "/cache"]) == ["node_modules", "/cache"] + with pytest.raises(ValidationError, match="Brace"): + validator(["*.{log,tmp}"]) + + +@pytest.mark.parametrize( + ("path", "mount"), + [ + ("/data/srv/uploads", ("/srv/uploads", "/data/srv/uploads")), + ("/mnt/nas", None), + (None, None), + ], +) +def mount_for(path, mount): + spec = DatabaseSpec(id="1", engine="files", name="n", path=path) + assert FilesEngine.mount_for(spec) == mount diff --git a/tests/engines/registry.py b/tests/engines/registry.py index 0a27e17..ae67a40 100644 --- a/tests/engines/registry.py +++ b/tests/engines/registry.py @@ -16,6 +16,7 @@ "valkey", "mssql", "docker-volume", + "files", ] diff --git a/tests/services/project.py b/tests/services/project.py index 5a7c063..ea0e6ed 100644 --- a/tests/services/project.py +++ b/tests/services/project.py @@ -20,6 +20,7 @@ FIREBIRD = registry.get("firebird") SQLITE = registry.get("sqlite") VOLUME = registry.get("docker-volume") +FILES = registry.get("files") KEYCLOAK = AuthProvider( kind="oidc", id="my-kc", @@ -89,14 +90,18 @@ def spec_from_entry_external(tmp_path): assert spec.id -def spec_from_entry_sqlite_and_volume(tmp_path): +def spec_from_entry_sqlite_volume_and_files(tmp_path): env = EnvFile(tmp_path / ".env") sqlite = spec_from_entry({"type": "sqlite", "database": "/config/app.sqlite"}, env) volume = spec_from_entry( {"type": "docker-volume", "volume_name": "v", "container_name": ""}, env ) + files = spec_from_entry( + {"type": "files", "path": "/data/srv", "options": {"method": "sync"}}, env + ) assert (sqlite.path, sqlite.host) == ("/config/app.sqlite", None) assert (volume.volume, volume.container) == ("v", None) + assert (files.path, files.options) == ("/data/srv", {"method": "sync"}) def agent_create_does_not_write_env(agent): @@ -199,7 +204,7 @@ def agent_missing_ca_bundle(agent): assert not agent.env.exists -def agent_docker_socket_and_sqlite_mounts(agent, ports): +def agent_docker_socket_and_mounts(agent, ports): assert not agent.needs_docker_socket agent.add(VOLUME.from_existing({"volume": "v"}), VOLUME) for _ in range(2): @@ -207,8 +212,13 @@ def agent_docker_socket_and_sqlite_mounts(agent, ports): SQLITE.generate(auth=False, ports=ports, answers={"name": "app"}), SQLITE ) agent.add(SQLITE.from_existing({"path": "/abs/app.db"}), SQLITE) + for _ in range(2): + agent.add(FILES.from_existing({"path": "/srv"}), FILES) assert agent.needs_docker_socket - assert agent.sqlite_mounts == [("./app.sqlite", "/config/app.sqlite")] + assert agent.mounts == [ + ("./app.sqlite", "/config/app.sqlite"), + ("/srv", "/data/srv"), + ] def agent_load_round_trip(agent, ports, renderer):