feat: files-source-provider

.gitignore

@@ -8,5 +8,5 @@ __pycache__/
 
 .claude
 .codex/
-
+/docs
 test/

commands/db.py

@@ -96,7 +96,10 @@ def run(
             bool, typer.Option("--password-stdin", help="Read password from stdin")
         ] = False,
         path: Annotated[
-            str | None, typer.Option("--path", help="SQLite file path (existing)")
+            str | None,
+            typer.Option(
+                "--path", help="SQLite file path (existing) or files directory"
+            ),
         ] = None,
         db_name: Annotated[
             str | None, typer.Option("--name", help="SQLite file name (new)")
@@ -243,7 +246,7 @@ def run(self, name: NameArg) -> None:
             )
             user = (
                 "N/A"
-                if database.engine in ("sqlite", "docker-volume")
+                if database.engine in ("sqlite", "docker-volume", "files")
                 else (database.username or "")
             )
             rows.append(

engines/__init__.py

@@ -5,6 +5,7 @@
 from core.errors import ValidationError
 from engines.base import DbEngine
 from engines.docker_volume import DockerVolumeEngine
+from engines.files import FilesEngine
 from engines.firebird import FirebirdEngine
 from engines.mariadb import MariaDbEngine
 from engines.mongodb import MongoEngine
@@ -58,6 +59,7 @@ def __contains__(self, key: str) -> bool:
     ValkeyEngine(),
     MssqlEngine(),
     DockerVolumeEngine(),
+    FilesEngine(),
 )
 
 registry = EngineRegistry(ALL)

engines/base.py

@@ -126,6 +126,11 @@ def agent_database(self, spec: DatabaseSpec) -> str:
     def describe(self, spec: DatabaseSpec) -> str:
         return f"{spec.host}:{spec.port}"
 
+    @staticmethod
+    def mount_for(spec: DatabaseSpec) -> tuple[str, str] | None:
+        """(host, container) bind mount the agent needs for this database."""
+        return None
+
     def non_default_options(self, spec: DatabaseSpec) -> dict[str, Any]:
         defaults = {field.name: field.default for field in self.option_fields()}
         return {

engines/files.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import posixpath
+from typing import Any
+
+from core.errors import ValidationError
+from core.fields import Field
+from core.specs import DatabaseSpec
+from engines.base import DbEngine
+from services.ports import PortAllocator
+
+DATA_DIR = "/data"
+
+
+def host_dir(value: str) -> str:
+    path = posixpath.normpath(str(value).strip())
+    if not path.startswith("/") or not path.strip("/"):
+        raise ValidationError(
+            f"Path must be an absolute directory other than /, got {value!r}",
+            hint="e.g. /srv/uploads",
+        )
+    if any(char in path for char in ":#\n"):
+        raise ValidationError(
+            f"Path cannot contain ':', '#' or a newline, got {value!r}",
+            hint="They break the Docker Compose bind mount.",
+        )
+    return path
+
+
+def no_braces(patterns: list[str]) -> list[str]:
+    if any("{" in pattern or "}" in pattern for pattern in patterns):
+        raise ValidationError(
+            "Brace alternation ({a,b}) is not supported in exclude patterns.",
+            hint="List each pattern separately: *.log,*.tmp",
+        )
+    return patterns
+
+
+class FilesEngine(DbEngine):
+    key, display = "files", "Files"
+    template = None
+    has_modes = False
+    label_default = "Files"
+    warning = (
+        f"The directory is mounted read-write under {DATA_DIR}: a restore replaces "
+        "its content (excluded paths are kept). Requires Agent 1.24.0+."
+    )
+
+    def fields_existing(self) -> list[Field]:
+        return [
+            Field(
+                "path",
+                "Directory on the host (absolute, e.g. /srv/uploads)",
+                "text",
+                validator=host_dir,
+            )
+        ]
+
+    def fields_new(self) -> list[Field]:
+        return self.fields_existing()
+
+    def option_fields(self) -> list[Field]:
+        return [
+            Field(
+                "method",
+                "Backup method",
+                "choice",
+                default="archive",
+                choices=("archive", "snapshot", "sync"),
+                help=(
+                    "archive: a tar.gz archive per run. snapshot: incremental, "
+                    "deduplicated restic snapshots. sync: mirrors the directory onto "
+                    "each storage; deletions propagate and it cannot be restored from "
+                    "Portabase. Cannot be changed once the source exists."
+                ),
+            ),
+            Field(
+                "exclude",
+                "Exclude patterns",
+                "list",
+                default=[],
+                validator=no_braces,
+                help=(
+                    "Comma-separated patterns, e.g. node_modules,*.tmp,/cache. Without "
+                    "a leading / a pattern matches at any depth; with one it is "
+                    "relative to the directory. Excluded paths are kept on restore."
+                ),
+            ),
+            Field(
+                "one_file_system",
+                "Stay on one file system?",
+                "bool",
+                default=False,
+                help="Skip mount points found inside the directory.",
+            ),
+        ]
+
+    def generate(
+        self, *, auth: bool, ports: PortAllocator, answers: dict[str, Any]
+    ) -> DatabaseSpec:
+        return self.from_existing(answers)
+
+    def from_existing(self, answers: dict[str, Any]) -> DatabaseSpec:
+        return DatabaseSpec(
+            id=self.new_id(),
+            engine=self.key,
+            name=answers.get("label") or self.label_default,
+            managed=False,
+            path=DATA_DIR + answers["path"],
+        )
+
+    @staticmethod
+    def mount_for(spec: DatabaseSpec) -> tuple[str, str] | None:
+        if spec.path and spec.path.startswith(f"{DATA_DIR}/"):
+            return (spec.path[len(DATA_DIR) :], spec.path)
+        return None
+
+    def env_vars(self, spec: DatabaseSpec) -> dict[str, str]:
+        return {}
+
+    def agent_entry(self, spec: DatabaseSpec) -> dict[str, Any]:
+        entry: dict[str, Any] = {
+            "name": spec.name,
+            "type": self.key,
+            "path": spec.path,
+            "generated_id": spec.id,
+        }
+        options = self.non_default_options(spec)
+        if options:
+            entry["options"] = options
+        return entry
+
+    def describe(self, spec: DatabaseSpec) -> str:
+        mount = self.mount_for(spec)
+        return f"dir: {mount[0] if mount else spec.path}"

services/project.py

@@ -7,8 +7,8 @@
 
 from core.errors import ConfigError, ValidationError
 from core.specs import DatabaseSpec
+from engines import registry
 from engines.base import DbEngine
-from engines.sqlite import SqliteEngine
 from services import auth_providers as ap
 from services import settings as cfg
 from services.compose_facts import CA_BUNDLE_IN_CONTAINER, ComposeFacts
@@ -61,7 +61,7 @@ def spec_from_entry(entry: dict[str, Any], env: EnvFile) -> DatabaseSpec:
         username=str(entry["username"]) if entry.get("username") is not None else None,
         password=_opt_str(entry, "password"),
         root_password=root_password,
-        path=_opt_str(entry, "database") if engine == "sqlite" else None,
+        path=_opt_str(entry, "database" if engine == "sqlite" else "path"),
         volume=_opt_str(entry, "volume_name"),
         container=_opt_str(entry, "container_name"),
         options=dict(entry.get("options") or {}),
@@ -117,13 +117,12 @@ def needs_docker_socket(self) -> bool:
         return any(database.engine == "docker-volume" for database in self.databases)
 
     @property
-    def sqlite_mounts(self) -> list[tuple[str, str]]:
+    def mounts(self) -> list[tuple[str, str]]:
         mounts: list[tuple[str, str]] = []
         for database in self.databases:
-            if database.engine == "sqlite":
-                mount = SqliteEngine.mount_for(database)
-                if mount and mount not in mounts:
-                    mounts.append(mount)
+            mount = registry.get(database.engine).mount_for(database)
+            if mount and mount not in mounts:
+                mounts.append(mount)
         return mounts
 
     def validate(self) -> None:

services/renderer.py

@@ -126,7 +126,7 @@ def render_agent(
             "docker_socket": project.needs_docker_socket,
             "mounts": [
                 {"host": host, "container": container}
-                for host, container in project.sqlite_mounts
+                for host, container in project.mounts
             ],
             "services": [self._service(spec, inline) for spec in project.managed],
             "tz_var": _var(env, "TZ", inline),

tests/engines/files.py

@@ -0,0 +1,114 @@
+import pytest
+
+from core.errors import ValidationError
+from core.specs import DatabaseSpec
+from engines import registry
+from engines.files import FilesEngine
+from tests.support import agent_service, field_specs
+
+FILES = registry.get("files")
+MOUNT = "/srv/uploads:/data/srv/uploads"
+
+
+def attributes():
+    assert type(FILES) is FilesEngine
+    assert (FILES.key, FILES.display, FILES.default_port) == ("files", "Files", None)
+    assert FILES.template is None
+    assert (FILES.auth_variants, FILES.has_modes) == (False, False)
+    assert "restore" in (FILES.warning or "")
+
+
+def generate(ports):
+    answers = {"path": "/srv/uploads", "label": "Uploads"}
+    spec = FILES.generate(auth=False, ports=ports, answers=answers)
+    assert spec == DatabaseSpec(
+        id=spec.id, engine="files", name="Uploads", path="/data/srv/uploads"
+    )
+    assert FILES.describe(spec) == "dir: /srv/uploads"
+
+
+def env_vars():
+    assert FILES.env_vars(FILES.from_existing({"path": "/srv"})) == {}
+
+
+def agent_entry():
+    bare = FILES.from_existing({"path": "/srv/uploads"})
+    assert FILES.agent_entry(bare) == {
+        "name": "Files",
+        "type": "files",
+        "path": "/data/srv/uploads",
+        "generated_id": bare.id,
+    }
+    tuned = bare.with_options(
+        {"method": "snapshot", "exclude": ["*.tmp"], "one_file_system": False}
+    )
+    assert FILES.agent_entry(tuned)["options"] == {
+        "method": "snapshot",
+        "exclude": ["*.tmp"],
+    }
+
+
+def fields():
+    assert field_specs(FILES.fields_existing()) == [("path", "text", None)]
+    assert FILES.fields_new() == FILES.fields_existing()
+    assert field_specs(FILES.option_fields()) == [
+        ("method", "choice", "archive"),
+        ("exclude", "list", []),
+        ("one_file_system", "bool", False),
+    ]
+    assert FILES.option_fields()[0].choices == ("archive", "snapshot", "sync")
+
+
+def from_existing():
+    spec = FILES.from_existing({"path": "/srv/uploads"})
+    assert spec == DatabaseSpec(
+        id=spec.id, engine="files", name="Files", path="/data/srv/uploads"
+    )
+
+
+def compose_service(render_engine):
+    rendered = render_engine("files", answers={"path": "/srv/uploads"})
+    assert rendered.doc["services"] == {"agent": agent_service(MOUNT)}
+    assert "volumes" not in rendered.doc
+    assert rendered.databases == [FILES.agent_entry(rendered.spec)]
+
+
+def compose_service_inline(render_engine):
+    rendered = render_engine("files", inline=True, answers={"path": "/srv/uploads"})
+    assert rendered.doc["services"] == {"agent": agent_service(MOUNT, inline=True)}
+
+
+@pytest.mark.parametrize(
+    ("raw", "normalized"),
+    [("/srv/uploads", "/srv/uploads"), (" /srv/uploads/ ", "/srv/uploads")],
+)
+def path_is_normalized(raw, normalized):
+    assert FILES.fields_existing()[0].validator(raw) == normalized
+
+
+@pytest.mark.parametrize(
+    "raw", ["uploads", "./uploads", "/", "//", "C:\\data", "/srv/a:b", "/srv/#1"]
+)
+def path_rejects_unmountable_values(raw):
+    with pytest.raises(ValidationError):
+        FILES.fields_existing()[0].validator(raw)
+
+
+def exclude_rejects_brace_alternation():
+    validator = FILES.option_fields()[1].validator
+    assert validator(["node_modules", "/cache"]) == ["node_modules", "/cache"]
+    with pytest.raises(ValidationError, match="Brace"):
+        validator(["*.{log,tmp}"])
+
+
+@pytest.mark.parametrize(
+    ("path", "mount"),
+    [
+        ("/data/srv/uploads", ("/srv/uploads", "/data/srv/uploads")),
+        ("/mnt/nas", None),
+        (None, None),
+    ],
+)
+def mount_for(path, mount):
+    spec = DatabaseSpec(id="1", engine="files", name="n", path=path)
+    assert FilesEngine.mount_for(spec) == mount

tests/engines/registry.py

@@ -16,6 +16,7 @@
     "valkey",
     "mssql",
     "docker-volume",
+    "files",
 ]