diff --git a/.env.test b/.env.test index b087a71..0e37215 100644 --- a/.env.test +++ b/.env.test @@ -1,7 +1,8 @@ API_URL=http://pyro_api:5050 CAM_USER=my_dummy_login CAM_PWD=my_dummy_pwd -MEDIAMTX_SERVER_IP=localhost +# MediaMTX on the host, seen from the compose network (streams are not set up here yet) +MEDIAMTX_SERVER_IP=host.docker.internal LOKI_URL=http://my-loki-service.com PROMTAIL_DEVICE_SCOPE=tower_scope diff --git a/.github/workflows/docker-compose.yaml b/.github/workflows/docker-compose.yaml index 068bfeb..51f280a 100644 --- a/.github/workflows/docker-compose.yaml +++ b/.github/workflows/docker-compose.yaml @@ -19,30 +19,29 @@ jobs: - name: Checkout repository uses: actions/checkout@v2 - - name: Wait for containers to start + - name: Create .env from .env.test + run: make init + + - name: Build local images run: make build - name: Run docker-compose up -d run: make run + - name: Start the engine, camera API and fake cameras + run: make run-engine + # Wait for containers to start up (adjust the sleep time as needed) - name: Wait for containers to start run: sleep 120 - name: Check container health - run: docker compose ps --filter "status=unhealthy" --quiet - continue-on-error: true - id: check_health - - - name: Print container health status + env: + COMPOSE_PROFILES: front,tools,engine run: |- - if [ -z "${{ steps.check_health.outputs.stdout }}" ]; then - echo "All containers are healthy" - exit 0 - else - echo "Some containers are unhealthy" - exit 1 - fi + docker compose ps -a + test -z "$(docker compose ps --filter status=unhealthy --quiet)" || { echo "Some containers are unhealthy"; exit 1; } + test -n "$(docker compose ps --status running --quiet engine)" || { echo "The engine is not running"; exit 1; } - name: load tests requirements run: pip install -r tests/requirements.txt @@ -51,9 +50,11 @@ jobs: run: pytest tests/* - name: Capture logs - run: docker compose logs --tail=40 > logs.txt + if: always() + run: COMPOSE_PROFILES=front,tools,engine docker compose logs --tail=40 > logs.txt - name: Show logs + if: always() run: cat logs.txt - name: Filter logs diff --git a/.github/workflows/fake-camera-image.yml b/.github/workflows/fake-camera-image.yml new file mode 100644 index 0000000..50f5433 --- /dev/null +++ b/.github/workflows/fake-camera-image.yml @@ -0,0 +1,63 @@ +name: Build and Push fake-camera image + +# The image embeds the default scenario, fetched from the `scenarios` release. +# latest on main, X.Y.Z on a `fake-camera-vX.Y.Z` tag, build only on pull requests. +on: + push: + branches: [main] + tags: ['fake-camera-v*'] + paths: + - containers/fake_camera/** + - data/scenarios/*/scenario.json + - .github/workflows/fake-camera-image.yml + pull_request: + paths: + - containers/fake_camera/** + - data/scenarios/*/scenario.json + - .github/workflows/fake-camera-image.yml + +jobs: + build-and-push: + name: Build and push pyronear/fake-camera + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Fetch the default scenario images + run: make fetch-scenario + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_LOGIN }} + password: ${{ secrets.DOCKERHUB_PW }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: pyronear/fake-camera + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=match,pattern=fake-camera-v(.*),group=1 + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: containers/fake_camera + file: containers/fake_camera/Dockerfile + build-contexts: scenarios=data/scenarios + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore index 50c0312..7cae47b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,10 @@ *.onnx *.jpg data/images/ +# Model cache written by the engine container (./data is mounted) +data/models/ +# Written by init_script at the end of a successful run +data/.init_done *.pem # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/Makefile b/Makefile index 1d6526e..e45d68a 100644 --- a/Makefile +++ b/Makefile @@ -7,15 +7,17 @@ help: @echo " build Build local images in this repo" @echo " build-external Build images in sibling repos" @echo " build-all Build local and external images" + @echo " fetch-scenario Download scenario images (SCENARIO=smoke-then-clear)" @echo " run-backend Start base services only" + @echo " run-camera Start only the fake cameras, to work on pyro-engine" @echo " run-engine Start base services plus engine profile" @echo " run-tools-and-engine Start base services plus tools and engine profiles" @echo " run-tools Start base services plus tools profile" @echo " run Start base services plus front and tools profiles" @echo " run-temporal Same as run (front + tools) plus temporal model API" @echo " stop-temporal Stop only the temporal model API" - @echo " stop-engine Stop only engine services (engine + pyro_camera_api)" - @echo " restart-engine Restart engine services without re-running init_script" + @echo " stop-engine Stop only engine services (engine, pyro_camera_api, fake cameras)" + @echo " restart-engine Restart engine services and replay the scenario from the start" @echo " stop Stop and remove all services and volumes" @echo " ps Show compose status" @echo " logs Follow logs" @@ -32,9 +34,11 @@ init: # Build images in this repo # ------------------------------------------------------------------- -build: +# The fake camera image embeds the default scenario, fetched first. +build: fetch-scenario docker build -f containers/init_script/Dockerfile -t pyronear/pyro-api-init:latest containers/init_script/ docker build -f containers/notebooks/Dockerfile -t pyronear/notebooks:latest containers/notebooks/ + docker build --build-context scenarios=data/scenarios -f containers/fake_camera/Dockerfile -t pyronear/fake-camera:latest containers/fake_camera/ # ------------------------------------------------------------------- # Build images from sibling repositories @@ -55,8 +59,23 @@ build-all: build build-external run-backend: docker compose up -d -# Engine profile adds engine + pyro_camera_api -run-engine: +# Fake camera scenarios: definitions are in data/scenarios//scenario.json, +# images come from the "scenarios" GitHub release. +SCENARIO ?= smoke-then-clear +SCENARIOS_URL = https://github.com/pyronear/pyro-envdev/releases/download/scenarios +# Downloads unless every camera of scenario.json already has frames. +fetch-scenario: + @python3 -c "import glob, json, sys; d = 'data/scenarios/$(SCENARIO)'; \ + sys.exit(any(not glob.glob(f'{d}/{c}/*.jpg') for c in json.load(open(f'{d}/scenario.json'))['cameras']))" || \ + (curl -fsSL $(SCENARIOS_URL)/$(SCENARIO).zip -o /tmp/$(SCENARIO).zip && \ + unzip -oq /tmp/$(SCENARIO).zip -x '*/scenario.json' -d data/scenarios && rm /tmp/$(SCENARIO).zip) + +# Fake cameras alone, for a pyro-engine running outside compose (see README) +run-camera: fetch-scenario + docker compose up -d fakecam-1 fakecam-2 + +# Engine profile adds engine + pyro_camera_api + fake cameras +run-engine: fetch-scenario docker compose --profile engine up -d # Tools + engine (tools must be up first so the API is ready when engine starts) @@ -84,13 +103,13 @@ stop-temporal: docker compose --profile temporal stop temporal_model_api stop-engine: - docker compose --profile engine stop engine pyro_camera_api + docker compose --profile engine stop engine pyro_camera_api fakecam-1 fakecam-2 restart-engine: stop-engine - docker compose --profile engine up -d engine pyro_camera_api --no-deps + docker compose --profile engine up -d fakecam-1 fakecam-2 pyro_camera_api engine --no-deps stop: - docker compose --profile front --profile engine --profile tools --profile temporal down -v + docker compose --profile front --profile engine --profile camera --profile tools --profile temporal down -v ps: docker compose ps diff --git a/README.md b/README.md index 70fab60..76c8b98 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,8 @@ make run * **db**: PostgreSQL database * **rustfs**: S3-compatible storage (via RustFS) * **frontend**: Web app (React, [pyro-platform](https://github.com/pyronear/pyro-platform)) -* **pyro-engine**: Engine service (requires cameras, optional) -* **reolinkdev1 / reolinkdev2**: Fake Reolink cameras sending test images +* **pyro_camera_api / engine**: the real camera API and engine from [pyro-engine](https://github.com/pyronear/pyro-engine) (optional) +* **fakecam-1 / fakecam-2**: fake Reolink cameras playing a scenario (see below) * **notebooks**: Jupyter server to run helper notebooks * **db-ui**: pgAdmin to browse/manage the database @@ -60,12 +60,12 @@ make build make run-all ``` -This launches everything including the engine and simulated alerts. -You can check health with: +This launches everything, including the engine and the fake cameras playing the +`smoke-then-clear` scenario. You can check health with: ```bash docker logs init -docker logs engine +docker compose logs -f engine ``` ### Partial runs @@ -111,6 +111,88 @@ docker logs engine --- +## 📷 Fake cameras and scenarios + +`fakecam-1` (`croix-augas-01`) and `fakecam-2` (`nemours-02`) answer the Reolink HTTP +API (`/cgi-bin/api.cgi?cmd=Snap|PtzCtrl|GetPtzPreset|...`) over HTTPS with a self-signed +certificate, so the real `pyro_camera_api` and `engine` run against them unchanged +(`data/credentials.json`, adapter `reolink`). Each one plays a scenario: a list of +images shown one after the other, the last image being held forever. Each preset moves to +its next image on its next visit by the patrol (the first `Snap` after a `ToPos`), so the +engine sees every image once per preset, in order, whatever its timing; snaps taken in place, +like the autofocus ones, see the same image. Every preset of a camera serves the same images +for now. + +```bash +make run-engine # engine + camera API + fake cameras +SCENARIO=never-smoke make run-engine # pick another scenario +make restart-engine # replay the scenario from the start +``` + +Scenarios: + +* `smoke-then-clear` (default): real wildfire seen by both cameras, 40 frames of smoke then a clear view +* `never-smoke`: both cameras hold a clear view + +`GET https://localhost:8443/scenario` (8444 for `fakecam-2`) reports the camera state: +current preset and azimuth, its frame index and `finished` once every visited preset shows the +last frame. + +The image embeds `smoke-then-clear`, so it also runs alone, without the `data/scenarios` +volume (used by pi-manager-template on a test station). + +The image `pyronear/fake-camera` is published by the `fake-camera-image` workflow: `latest` +on every change to `containers/fake_camera/` on `main`, and `X.Y.Z` on a `fake-camera-vX.Y.Z` +tag (used by pi-manager-template). It needs the `DOCKERHUB_LOGIN` / `DOCKERHUB_PW` secrets. + +### Scenario format + +`data/scenarios//scenario.json` describes the scenario (committed), the images +live next to it and are downloaded by `make fetch-scenario SCENARIO=` from the +[`scenarios` release](https://github.com/pyronear/pyro-envdev/releases/tag/scenarios): + +``` +data/scenarios/smoke-then-clear/ + scenario.json # {"cameras": {"": {"poses": {"": }}}} + croix-augas-01/ # frames of fakecam-1, in order: 000.jpg, 001.jpg, ... + nemours-02/ # frames of fakecam-2 +``` + +Camera names and presets match `data/csv/API_DATA_DEV - cameras.csv` / `poses.csv`, so +the init script gives each fake camera its token and the engine finds its poses. + +To build the frames of a camera from a replayed alert zip (see `make fetch-alerts`) and +a clear image for the end of the scenario: + +```bash +python scripts/make_scenario.py data/scenarios//croix-augas-01 \ + --alert data/replay_alerts/alert_49767.zip --frames 40 --clear clear.jpg +``` + +Then zip the images (`cd data/scenarios && zip -r .zip -x '*/scenario.json'`) +and attach the zip to the `scenarios` release. `scenario.json` itself is committed. + +### Working on pyro-engine without a camera + +Start only the fake cameras: + +```bash +make run-camera +``` + +They are published on `https://localhost:8443` and `https://localhost:8444`. In your +pyro-engine checkout, point `credentials.json` at them with the host and port as key: + +```json +{ + "localhost:8443": {"name": "croix-augas-01", "adapter": "reolink", "type": "ptz", "poses": [1, 2, 3], ...} +} +``` + +and use the same `CAM_USER` / `CAM_PWD` as in this repo's `.env`. + +--- + ## 📂 Data Usage ### Add more images to Reolink Dev diff --git a/containers/fake_camera/Dockerfile b/containers/fake_camera/Dockerfile new file mode 100644 index 0000000..8912f57 --- /dev/null +++ b/containers/fake_camera/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.12-slim + +WORKDIR /app + +# Self-signed certificate: pyro_camera_api talks HTTPS to Reolink cameras without verifying it. +RUN openssl req -x509 -newkey rsa:2048 -nodes -days 3650 -subj "/CN=fakecam" \ + -keyout key.pem -out cert.pem + +COPY server.py . + +# Default scenario, so the image runs alone (e.g. on a test station); a volume on +# /scenarios overrides it. Built with --build-context scenarios=data/scenarios (Makefile). +# hadolint ignore=DL3022 +COPY --from=scenarios smoke-then-clear /scenarios/smoke-then-clear + +EXPOSE 443 + +CMD ["python", "server.py"] diff --git a/containers/fake_camera/server.py b/containers/fake_camera/server.py new file mode 100644 index 0000000..c9ad6d2 --- /dev/null +++ b/containers/fake_camera/server.py @@ -0,0 +1,233 @@ +"""Fake Reolink camera. + +Answers the Reolink HTTP API (``/cgi-bin/api.cgi?cmd=...``) the way the real +``pyro_camera_api`` Reolink adapter expects, serving the frames of a scenario. +One process is one camera: ``SCENARIO`` and ``CAMERA`` select +``///*.jpg`` (sorted, last frame held). +Each preset moves to its next frame on its next visit: the first Snap after a +ToPos, so snaps taken in place (autofocus) see the same frame. A camera never +positioned (static) moves on every Snap. + +``GET /scenario`` is not part of the Reolink API: it reports the camera state +(preset, azimuth, frame, finished) so tests know where the scenario is. +""" + +import json +import logging +import os +import ssl +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlparse + +# A Reolink PTZ camera exposes 64 preset slots, ids 0..63. +PRESET_SLOTS = 64 +# PTZ operations that leave the current preset; Stop and zoom/focus do not pan. +NO_PAN_OPS = {"ToPos", "Stop", "ZoomInc", "ZoomDec", "FocusInc", "FocusDec"} + +CAM_USER = os.environ.get("CAM_USER", "") +CAM_PWD = os.environ.get("CAM_PWD", "") + + +def frame_index(visits, n_frames): + """Frame served on the `visits`-th visit of a preset (1 = first), last one held.""" + return min(n_frames - 1, max(0, visits - 1)) + + +def reply(cmd, value=None, error=None): + if error: + return [{"cmd": cmd, "code": 1, "error": {"detail": error, "rspCode": -9}}] + return [ + {"cmd": cmd, "code": 0, "value": {"rspCode": 200} if value is None else value} + ] + + +class Camera: + def __init__(self, scenario_dir, name): + meta = json.loads((scenario_dir / "scenario.json").read_text()) + self.name = name + # preset id -> real-world azimuth, informational only (see /scenario) + self.azimuths = { + int(k): float(v) for k, v in meta["cameras"][name]["poses"].items() + } + # ponytail: every preset serves the same frames; per-preset folders if needed + self.frames = sorted((scenario_dir / name).glob("*.jpg")) + if not self.frames: + raise SystemExit( + f"no frames in {scenario_dir / name}, run `make fetch-scenario` first" + ) + self.lock = threading.Lock() + self.preset = None # unknown until a ToPos; stays None on a static camera + self.visits = {} # preset -> visits that took a Snap + self.arrived = True # the next Snap is the first at the current position + self.enabled = set(self.azimuths) + self.zoom, self.focus, self.autofocus_disabled = 0, 720, 0 + + def _frame(self, preset): + return frame_index(self.visits.get(preset, 0), len(self.frames)) + + def snap(self): + with self.lock: + if self.arrived or self.preset is None: + self.visits[self.preset] = self.visits.get(self.preset, 0) + 1 + self.arrived = False + return self.frames[self._frame(self.preset)].read_bytes() + + def status(self): + with self.lock: + return { + "camera": self.name, + "preset": self.preset, + "azimuth": self.azimuths.get(self.preset), + "frame": self._frame(self.preset), + "frames": len(self.frames), + "finished": bool(self.visits) + and all(self._frame(p) == len(self.frames) - 1 for p in self.visits), + } + + def command(self, cmd, param): + with self.lock: + if cmd == "PtzCtrl": + op = param.get("op") + if op == "ToPos": + idx = int(param.get("id", 0)) + if idx not in self.enabled: + return reply(cmd, error=f"preset {idx} is not set") + self.preset = idx + self.arrived = True + elif op not in NO_PAN_OPS: + self.preset = None + return reply(cmd) + if cmd == "GetPtzPreset": + presets = [ + { + "channel": 0, + "enable": int(i in self.enabled), + "id": i, + "name": f"pos{i}", + } + for i in range(PRESET_SLOTS) + ] + return reply(cmd, {"PtzPreset": presets}) + if cmd == "SetPtzPreset": + preset = param.get("PtzPreset", {}) + idx = int(preset.get("id", 0)) + if preset.get("enable", 1): + self.enabled.add(idx) + else: + self.enabled.discard(idx) + return reply(cmd) + if cmd == "GetZoomFocus": + zoom_focus = { + "channel": 0, + "focus": {"pos": self.focus}, + "zoom": {"pos": self.zoom}, + } + return reply(cmd, {"ZoomFocus": zoom_focus}) + if cmd == "StartZoomFocus": + zoom_focus = param.get("ZoomFocus", {}) + if zoom_focus.get("op") == "FocusPos": + self.focus = int(zoom_focus.get("pos", self.focus)) + else: + self.zoom = int(zoom_focus.get("pos", self.zoom)) + return reply(cmd) + if cmd == "GetAutoFocus": + return reply( + cmd, + {"AutoFocus": {"channel": 0, "disable": self.autofocus_disabled}}, + ) + if cmd == "SetAutoFocus": + self.autofocus_disabled = int( + param.get("AutoFocus", {}).get("disable", 0) + ) + return reply(cmd) + if cmd == "Reboot": + return reply(cmd) + return reply(cmd, error=f"unsupported command {cmd}") + + +camera = None + + +class Handler(BaseHTTPRequestHandler): + def log_message(self, fmt, *args): + # The default logs the full URL, which carries the camera password. + logging.info("%s %s", self.command, urlparse(self.path).path) + + def _send(self, status, body, content_type="application/json"): + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _json(self, data): + self._send(200, json.dumps(data).encode()) + + def do_GET(self): + self._dispatch() + + def do_POST(self): + self._dispatch() + + def _dispatch(self): + url = urlparse(self.path) + query = parse_qs(url.query) + if url.path == "/scenario": + return self._json(camera.status()) + if url.path != "/cgi-bin/api.cgi": + return self._send(404, b"not found", "text/plain") + cmd = query.get("cmd", [""])[0] + if (query.get("user", [""])[0], query.get("password", [""])[0]) != ( + CAM_USER, + CAM_PWD, + ): + return self._json( + [ + { + "cmd": cmd, + "code": 1, + "error": {"detail": "please login first", "rspCode": -6}, + } + ] + ) + if cmd == "Snap": + return self._send(200, camera.snap(), "image/jpeg") + length = int(self.headers.get("Content-Length") or 0) + body = json.loads(self.rfile.read(length) or b"[]") + param = body[0].get("param", {}) if body else {} + return self._json(camera.command(cmd, param)) + + +def main(): + global camera + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") + scenario = os.environ["SCENARIO"] + camera = Camera( + Path(os.environ.get("SCENARIOS_DIR", "/scenarios")) / scenario, + os.environ["CAMERA"], + ) + port = int(os.environ.get("PORT", "443")) + here = Path(__file__).parent + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain( + os.environ.get("CERT_FILE", here / "cert.pem"), + os.environ.get("KEY_FILE", here / "key.pem"), + ) + httpd = ThreadingHTTPServer(("", port), Handler) + httpd.socket = context.wrap_socket(httpd.socket, server_side=True) + logging.info( + "fake camera %s plays scenario %s: " + "%d frames, one per preset visit, presets %s, port %d", + camera.name, + scenario, + len(camera.frames), + sorted(camera.enabled), + port, + ) + httpd.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/containers/init_script/init_script.py b/containers/init_script/init_script.py index 94f0594..d382fa6 100644 --- a/containers/init_script/init_script.py +++ b/containers/init_script/init_script.py @@ -49,6 +49,24 @@ cameras = cameras.fillna("") poses = pd.read_csv(f"data/csv/API_DATA{sub_path} - poses.csv") +# `docker compose up` re-runs this one-shot container: skip when a previous +# run completed on this database, so services waiting on it +# (service_completed_successfully) still start. The marker alone is not +# enough: `make stop` drops the database but keeps data/ on the host. +done_marker = "data/.init_done" +existing_orgs = api_request("get", f"{api_url}/organizations/", superuser_auth) +if set(organizations.name) <= {org["name"] for org in existing_orgs}: + tokens_present = all( + cam["token"] for cam in read_json_file(credentials_path).values() + ) + if os.path.exists(done_marker) and tokens_present: + logging.info("Database already initialized, nothing to do") + sys.exit(0) + logging.warning("Database already initialized but the last init did not complete") + logging.warning("Run `make stop` to start from a fresh database") +if os.path.exists(done_marker): + os.remove(done_marker) + # ============================================================================ # ORGAS CREATION # ============================================================================ @@ -264,6 +282,8 @@ logging.info("All camera and pose updates completed") logging.info("Initialization script completed successfully") +with open(done_marker, "w") as marker: + marker.write("ok\n") # Load environment variables from .env file # load_dotenv() diff --git a/data/credentials.json b/data/credentials.json index 66a096e..8a17371 100644 --- a/data/credentials.json +++ b/data/credentials.json @@ -1,13 +1,21 @@ { - "mock_camera_1": { - "name": "brison-01", - "adapter": "mock", - "type": "static", - "pose_ids": [ - 29 - ], + "fakecam-1": { + "name": "croix-augas-01", + "adapter": "reolink", + "type": "ptz", + "poses": [1, 2, 3], + "pose_ids": [8, 9, 10], + "id": "", + "bbox_mask_url": "", + "token": "" + }, + "fakecam-2": { + "name": "nemours-02", + "adapter": "reolink", + "type": "ptz", + "poses": [0, 1, 2, 3], + "pose_ids": [25, 26, 27, 28], "id": "", - "poses": [], "bbox_mask_url": "", "token": "" } diff --git a/data/csv/API_DATA_DEV - poses.csv b/data/csv/API_DATA_DEV - poses.csv index 9b0c6d6..3cd721f 100644 --- a/data/csv/API_DATA_DEV - poses.csv +++ b/data/csv/API_DATA_DEV - poses.csv @@ -6,9 +6,9 @@ id,camera_id,azimuth,patrol_id 5,2,150,2 6,2,240,3 7,2,330,4 -8,3,0,1 -9,3,120,2 -10,3,240,3 +8,3,326.5,1 +9,3,13.2,2 +10,3,59.8,3 11,4,60,1 12,4,150,2 13,4,240,3 @@ -23,10 +23,10 @@ id,camera_id,azimuth,patrol_id 22,7,0,1 23,7,120,2 24,7,240,3 -25,8,60,1 -26,8,150,2 -27,8,240,3 -28,8,330,4 +25,8,245.9,0 +26,8,292.4,1 +27,8,339.4,2 +28,8,25.4,3 29,9,0,1 30,10,90,2 31,11,180,3 diff --git a/data/scenarios/never-smoke/scenario.json b/data/scenarios/never-smoke/scenario.json new file mode 100644 index 0000000..156f4f5 --- /dev/null +++ b/data/scenarios/never-smoke/scenario.json @@ -0,0 +1,7 @@ +{ + "description": "Both cameras hold a clear view. Expected: no alert.", + "cameras": { + "croix-augas-01": {"poses": {"1": 326.5, "2": 13.2, "3": 59.8}}, + "nemours-02": {"poses": {"0": 245.9, "1": 292.4, "2": 339.4, "3": 25.4}} + } +} diff --git a/data/scenarios/smoke-then-clear/scenario.json b/data/scenarios/smoke-then-clear/scenario.json new file mode 100644 index 0000000..ae84725 --- /dev/null +++ b/data/scenarios/smoke-then-clear/scenario.json @@ -0,0 +1,7 @@ +{ + "description": "Real wildfire (prod alert 49767) seen by two cameras: 40 frames of smoke, then a clear view held forever. All presets serve the same frames for now, so expect one sequence per pose of both cameras, then no more detections.", + "cameras": { + "croix-augas-01": {"poses": {"1": 326.5, "2": 13.2, "3": 59.8}}, + "nemours-02": {"poses": {"0": 245.9, "1": 292.4, "2": 339.4, "3": 25.4}} + } +} diff --git a/docker-compose.yml b/docker-compose.yml index b184cac..774ab8d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,17 +65,77 @@ services: interval: 10s timeout: 10s retries: 5 + fakecam-1: + image: pyronear/fake-camera:latest + profiles: + - engine + - camera + environment: + - CAMERA=croix-augas-01 + - SCENARIO=${SCENARIO:-smoke-then-clear} + - CAM_USER=${CAM_USER} + - CAM_PWD=${CAM_PWD} + volumes: + - ./data/scenarios:/scenarios:ro + ports: + - "127.0.0.1:8443:443" + healthcheck: + test: ["CMD", "python", "-c", "import ssl, urllib.request; urllib.request.urlopen('https://localhost/scenario', context=ssl._create_unverified_context())"] + interval: 10s + timeout: 5s + retries: 5 + fakecam-2: + image: pyronear/fake-camera:latest + profiles: + - engine + - camera + environment: + - CAMERA=nemours-02 + - SCENARIO=${SCENARIO:-smoke-then-clear} + - CAM_USER=${CAM_USER} + - CAM_PWD=${CAM_PWD} + volumes: + - ./data/scenarios:/scenarios:ro + ports: + - "127.0.0.1:8444:443" + healthcheck: + test: ["CMD", "python", "-c", "import ssl, urllib.request; urllib.request.urlopen('https://localhost/scenario', context=ssl._create_unverified_context())"] + interval: 10s + timeout: 5s + retries: 5 pyro_camera_api: image: pyronear/pyro-camera-api:latest profiles: - engine environment: + # Pose azimuths are fetched from the platform API. + - API_URL=http://pyro_api:5050 - CAM_USER=${CAM_USER} - CAM_PWD=${CAM_PWD} - MEDIAMTX_SERVER_IP=${MEDIAMTX_SERVER_IP} + # Fake cameras serve the same frames on every preset, which the detector reads as a stuck camera. + - ENABLE_STUCK_DETECTOR=false + # Lets MEDIAMTX_SERVER_IP=host.docker.internal reach a MediaMTX running on the host. + extra_hosts: + - "host.docker.internal:host-gateway" volumes: - ./data:/usr/src/app/data - network_mode: host + ports: + - "127.0.0.1:8081:8081" + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:8081/health || exit 1"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 20s + depends_on: + # Camera tokens are written to data/credentials.json by init_script and read once at startup. + init_script: + condition: service_completed_successfully + fakecam-1: + condition: service_healthy + fakecam-2: + condition: service_healthy logging: driver: json-file options: @@ -86,13 +146,15 @@ services: profiles: - engine environment: - - API_URL=http://localhost:5050 + - API_URL=http://pyro_api:5050 - CAM_USER=${CAM_USER} - CAM_PWD=${CAM_PWD} - command: python run.py + command: python run.py --pyro_camera_api_url http://pyro_camera_api:8081 volumes: - ./data:/usr/src/app/data - network_mode: host + depends_on: + pyro_camera_api: + condition: service_healthy logging: driver: json-file options: diff --git a/scripts/make_scenario.py b/scripts/make_scenario.py new file mode 100644 index 0000000..b9f9d58 --- /dev/null +++ b/scripts/make_scenario.py @@ -0,0 +1,60 @@ +"""Write the frames of one fake camera from a replayed alert zip and clear images. + + python scripts/make_scenario.py data/scenarios/smoke-then-clear/croix-augas-01 \ + --alert data/replay_alerts/alert_49767.zip --frames 40 --clear clean/40-xxx.jpg + +Frames are written as 000.jpg, 001.jpg, ... in time order: the alert frames of +the camera named like the output folder, then the clear images. Without +--alert only the clear images are written (a camera that never sees smoke). +The scenario.json next to the camera folders is written by hand. +""" + +import argparse +import json +import zipfile +from pathlib import Path + + +def alert_frames(alert_zip, camera_name, count): + zf = zipfile.ZipFile(alert_zip) + alert = json.loads(zf.read("alert.json")) + camera_id = next( + k for k, cam in alert["cameras"].items() if cam["name"] == camera_name + ) + sequence = next(s for s in alert["sequences"] if str(s["camera_id"]) == camera_id) + detections = sorted(sequence["detections"], key=lambda d: d["recorded_at"])[:count] + return [zf.read(d["image"]) for d in detections] + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "out", type=Path, help="data/scenarios//" + ) + parser.add_argument("--alert", type=Path, help="alert zip from `make fetch-alerts`") + parser.add_argument( + "--frames", type=int, default=40, help="alert frames to keep (default 40)" + ) + parser.add_argument( + "--clear", + type=Path, + nargs="*", + default=[], + help="clear images appended at the end", + ) + args = parser.parse_args() + + frames = alert_frames(args.alert, args.out.name, args.frames) if args.alert else [] + frames += [path.read_bytes() for path in args.clear] + args.out.mkdir(parents=True, exist_ok=True) + for stale in args.out.glob("*.jpg"): + stale.unlink() + for i, data in enumerate(frames): + (args.out / f"{i:03d}.jpg").write_bytes(data) + print(f"{len(frames)} frames written to {args.out}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_fake_camera.py b/tests/test_fake_camera.py new file mode 100644 index 0000000..6638aab --- /dev/null +++ b/tests/test_fake_camera.py @@ -0,0 +1,107 @@ +import json +import os +import ssl +import sys +import urllib.request +from pathlib import Path + +from dotenv import load_dotenv + +sys.path.insert( + 0, str(Path(__file__).resolve().parents[1] / "containers" / "fake_camera") +) +from server import Camera, frame_index # noqa: E402 + +load_dotenv() + +# fakecam-1 (croix-augas-01), published on the host by docker-compose.yml +BASE_URL = "https://127.0.0.1:8443" +AUTH = f"user={os.getenv('CAM_USER')}&password={os.getenv('CAM_PWD')}&channel=0" +# The fake camera uses a self-signed certificate, like pyro_camera_api (verify=False). +CTX = ssl._create_unverified_context() + + +def call(cmd, param=None, auth=AUTH): + data = ( + None + if param is None + else json.dumps([{"cmd": cmd, "action": 0, "param": param}]).encode() + ) + url = f"{BASE_URL}/cgi-bin/api.cgi?cmd={cmd}&{auth}" + with urllib.request.urlopen(url, data=data, context=CTX, timeout=5) as resp: + body = resp.read() + return ( + body if resp.headers["Content-Type"] == "image/jpeg" else json.loads(body) + ) + + +def test_frame_index(): + assert frame_index(0, 3) == 0 # never visited + assert frame_index(1, 3) == 0 # first visit + assert frame_index(2, 3) == 1 + assert frame_index(20, 3) == 2 # last frame held + + +def test_one_frame_per_preset_visit(tmp_path): + (tmp_path / "scenario.json").write_text( + json.dumps({"cameras": {"cam": {"poses": {"1": 0.0, "2": 90.0}}}}) + ) + (tmp_path / "cam").mkdir() + for i in range(3): + (tmp_path / "cam" / f"{i:03d}.jpg").write_bytes(b"frame%d" % i) + cam = Camera(tmp_path, "cam") + + def visit(preset, snaps=1): + cam.command("PtzCtrl", {"op": "ToPos", "id": preset}) + return [cam.snap() for _ in range(snaps)] + + assert visit(1, snaps=3) == [b"frame0"] * 3 # autofocus snaps in place: same frame + assert visit(2) == [b"frame0"] # each preset starts at the first frame + assert visit(1) == [b"frame1"] + assert visit(1) == [b"frame2"] + assert visit(1) == [b"frame2"] # last frame held + assert not cam.status()["finished"] # preset 2 is still on its first frame + + +def test_static_camera_moves_on_every_snap(tmp_path): + (tmp_path / "scenario.json").write_text( + json.dumps({"cameras": {"cam": {"poses": {}}}}) + ) + (tmp_path / "cam").mkdir() + for i in range(2): + (tmp_path / "cam" / f"{i:03d}.jpg").write_bytes(b"frame%d" % i) + cam = Camera(tmp_path, "cam") + assert [cam.snap() for _ in range(3)] == [b"frame0", b"frame1", b"frame1"] + assert cam.status()["finished"] + + +def test_snap_returns_a_jpeg(): + assert call("Snap")[:2] == b"\xff\xd8" + + +def test_wrong_password_is_rejected(): + assert call("Snap", auth="user=nobody&password=wrong&channel=0")[0]["code"] == 1 + + +def test_ptz_follows_presets(): + assert ( + call("PtzCtrl", {"channel": 0, "op": "ToPos", "id": 3, "speed": 50})[0]["code"] + == 0 + ) + with urllib.request.urlopen(f"{BASE_URL}/scenario", context=CTX, timeout=5) as resp: + status = json.load(resp) + assert status["preset"] == 3 + assert status["azimuth"] == 59.8 + # Preset 42 is not set on this camera + assert ( + call("PtzCtrl", {"channel": 0, "op": "ToPos", "id": 42, "speed": 50})[0]["code"] + == 1 + ) + presets = call("GetPtzPreset", {"channel": 0})[0]["value"]["PtzPreset"] + assert [p["id"] for p in presets if p["enable"]] == [1, 2, 3] + + +def test_zoom_focus_values(): + zoom_focus = call("GetZoomFocus", {"channel": 0})[0]["value"]["ZoomFocus"] + assert zoom_focus["focus"]["pos"] == 720 + assert zoom_focus["zoom"]["pos"] == 0