diff --git a/README.md b/README.md index 1400a9b0..625fb2ee 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,6 @@ This will install HAP-python in your Python packages, so that you can import it `pyhap`. The following optional features can be installed via pip extras: * `QRCode` — print a pairing QR code in the terminal (`base36`, `pyqrcode`). -* `Talkback` — two-way audio for the camera accessory (`pylibsrtp`). * `bench` — dependencies for running the benchmarks (`psutil`, `py-spy`). ## Demo diff --git a/pyhap/camera.py b/pyhap/camera.py index ae9fa8b0..72710107 100644 --- a/pyhap/camera.py +++ b/pyhap/camera.py @@ -39,7 +39,7 @@ tlv, ) from pyhap.accessory import Accessory -from pyhap.camera_talkback import TalkbackReceiver, talkback_available +from pyhap.camera_talkback import TalkbackReceiver from pyhap.characteristic import Characteristic from pyhap.const import CATEGORY_CAMERA from pyhap.service import Service @@ -49,6 +49,11 @@ from pyhap.camera_multi_tier import MultiTierController from pyhap.camera_webrtc import WebRTCController +# Seconds before arming a recording delegate is tried again, doubling up to +# the maximum while it keeps failing. +RECORDING_ARM_RETRY = 1.0 +MAX_RECORDING_ARM_RETRY = 60.0 + SETUP_TYPES = { "SESSION_ID": b"\x01", "STATUS": b"\x02", @@ -596,8 +601,7 @@ def __init__(self, options: dict[str, Any], *args: Any, **kwargs: Any) -> None: - srtp - boolean, defaults to False. Whether the camera supports SRTP. - talkback - boolean, defaults to False. Whether the camera supports receiving two-way ("talkback") audio from the controller during a - live view session. Requires the optional ``pylibsrtp`` dependency - (``pip install HAP-python[Talkback]``). When enabled, a "Speaker" + live view session. When enabled, a "Speaker" service is added so that HAP clients know the accessory supports it, and :meth:`talkback_audio_received` is called with the received audio for each session that negotiates it. @@ -612,6 +616,16 @@ def __init__(self, options: dict[str, Any], *args: Any, **kwargs: Any) -> None: not allowed, the camera refuses to set up live streams and ends running ones. Unknown names raise ``ValueError``, a value that is not a list raises ``TypeError``. + - recording - ``True``, or a dict whose ``delegate`` is a + :class:`pyhap.hds_recording.RecordingProducer` such as + :class:`pyhap.media.recording_delegate.RecordingMediaDelegate` + that makes the recordings: the camera arms it while the hub has + recording on and the camera is on, advertises its + ``prebuffer_ms`` and records with its ``record``. Without a + delegate, override :meth:`handle_recording_stream`. Observed + with iOS and tvOS 27: hubs do not upload the HDS recordings of + a camera that also has ``multi_tier``, they never appear in the + Home app. - start_stream_cmd - string specifying the command to be executed to start the stream. The string can contain the keywords, corresponding to the video and audio configuration that was negotiated between the camera @@ -634,6 +648,10 @@ def __init__(self, options: dict[str, Any], *args: Any, **kwargs: Any) -> None: management service as not responding. Its services come before the RTP stream management services, which then get a new ``unique_id``; for a paired camera this changes instance IDs. + Observed with iOS and tvOS 27: the hub does not upload the + recordings it gets over HDS from a camera with this service, so + they do not appear in the Home app; the camera is expected to + upload its recordings itself. - webrtc - :class:`pyhap.camera_webrtc.WebRTCConfig`, or a dict of its fields. Developer Preview of HomeKit Secure Video 17.99, experimental. Adds a Camera WebRTC Stream Management service per @@ -646,11 +664,6 @@ def __init__(self, options: dict[str, Any], *args: Any, **kwargs: Any) -> None: """ self.has_srtp = options.get("srtp", False) self.support_talkback = options.get("talkback", False) - if self.support_talkback and not talkback_available(): - raise ValueError( - "The 'talkback' option requires the optional 'pylibsrtp' " - "dependency. Install it with: pip install HAP-python[Talkback]" - ) self.start_stream_cmd = options.get("start_stream_cmd", FFMPEG_CMD) self.stream_address = options["address"] @@ -684,6 +697,16 @@ def __init__(self, options: dict[str, Any], *args: Any, **kwargs: Any) -> None: self._hds_listener = None self._hds_connections = set() self._recording_stream = None + recording = options.get("recording") + self._recording_media: hds_recording.RecordingProducer | None = ( + recording.get("delegate") if isinstance(recording, dict) else None + ) + self._recording_media_running = False + # What the delegate was last armed to; one task reconciles it. + self._recording_media_armed = False + self._recording_media_task: asyncio.Task[None] | None = None + # Set to cut a retry delay short when the wanted state changes. + self._recording_media_wake = asyncio.Event() self._homekit_camera_active_char: Characteristic | None = None # The HomeKit Camera Active of the Camera Operating Mode service, kept # in step with the global one when both exist. @@ -719,6 +742,13 @@ def __init__(self, options: dict[str, Any], *args: Any, **kwargs: Any) -> None: if transport is not None: for controller in self._stream_controllers(): controller.service.add_linked_service(transport) + if self._multi_tier and self._records_over_hds(): + logger.warning( + "%s: hubs with iOS or tvOS 27 do not upload HDS recordings of " + "a camera with multi-tier streams, they will not appear in " + "the Home app", + self.display_name, + ) if self._sensors: self._setup_camera_capabilities() # Last, so the instance IDs of the other services stay the same. @@ -726,6 +756,12 @@ def __init__(self, options: dict[str, Any], *args: Any, **kwargs: Any) -> None: self._setup_global_operating_mode(options["global_operating_mode"]) self._restore_hksv3_state() + def _records_over_hds(self) -> bool: + """Whether the camera makes HDS recordings, by delegate or override.""" + return self._recording_media is not None or ( + type(self).handle_recording_stream is not Camera.handle_recording_stream + ) + @property def streaming_status(self) -> bytes | None: """For backwards compatibility.""" @@ -957,6 +993,8 @@ def _default_recording_configs( ] audio = [hksv_recording.AudioCodecConfiguration()] general = hksv_recording.SupportedRecordingConfiguration() + if self._recording_media is not None: + general.prebuffer_length_ms = int(self._recording_media.prebuffer_ms) return general, video, audio def _setup_recording_management(self, options: dict[str, Any]) -> None: @@ -1233,11 +1271,20 @@ async def handle_recording_stream( """Produce the fragmented-MP4 recording for ``stream_id``. Override to yield :class:`pyhap.hds_recording.RecordingPacket` objects, - the first being the MP4 initialization segment. The default yields - nothing (no recording). + the first being the MP4 initialization segment. The default records + with the ``delegate`` of the ``recording`` option, without one it + yields nothing (no recording). """ - return - yield # pragma: no cover - makes this an async generator + delegate = self._recording_media + selected = self._selected_recording_config + if delegate is None or selected is None: + return + packets = delegate.record(selected, self.recording_audio_active) + try: + async for packet in packets: + yield packet + finally: + await packets.aclose() @property def selected_recording_configuration( @@ -1457,6 +1504,7 @@ def _save_state(self) -> None: recording_changed, self._recording_changed = self._recording_changed, False self.driver.async_persist() if recording_changed: + self._arm_recording_media() self.recording_state_changed() def _persist_hksv3_char( @@ -1792,6 +1840,7 @@ def _handle_homekit_camera_active(self, value: Any) -> None: self._motion_status_active_char.notify() if not active and self._recording_stream is not None: self._recording_stream.close(hds_recording.RecordingReason.NOT_ALLOWED) + self._arm_recording_media() self._end_streams_if_not_allowed() self._schedule_stream_status() self.homekit_camera_active_changed(active) @@ -2074,6 +2123,61 @@ async def run(self) -> None: await super().run() await self.start_recording_transport() + def _recording_media_wanted(self) -> bool: + return ( + self._recording_media_running + and self.recording_active + and self.homekit_camera_active + ) + + def _arm_recording_media(self) -> None: + """Keep the recording delegate armed exactly while recording may run.""" + if self._recording_media is None: + return + if self._recording_media_task is not None: + # A task reconciles already; it rechecks now if it waits to retry. + self._recording_media_wake.set() + return + if self._recording_media_wanted() == self._recording_media_armed: + return + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return + self._recording_media_task = loop.create_task(self._reconcile_recording()) + + async def _reconcile_recording(self) -> None: + """Arm or disarm until the delegate matches what the camera wants.""" + delegate = self._recording_media + assert delegate is not None # noqa: S101 - only started with a delegate + failures = 0 + try: + while (wanted := self._recording_media_wanted()) != ( + self._recording_media_armed + ): + self._recording_media_wake.clear() + try: + await delegate.set_armed(wanted) + except Exception as err: # noqa: BLE001 - retried with a backoff + # Meanwhile a recording still opens the channel on demand. + self._recording_media_armed = False + delay = min( + RECORDING_ARM_RETRY * 2**failures, MAX_RECORDING_ARM_RETRY + ) + failures += 1 + logger.warning( + "Could not arm the recording, retrying in %.0f s: %s", + delay, + err, + ) + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(self._recording_media_wake.wait(), delay) + continue + failures = 0 + self._recording_media_armed = wanted + finally: + self._recording_media_task = None + async def start_recording_transport(self) -> None: """Start the HDS listener on the HAP server's listen address(es). @@ -2082,6 +2186,8 @@ async def start_recording_transport(self) -> None: """ if self._recording_service is not None: await self._ensure_hds_listener() + self._recording_media_running = True + self._arm_recording_media() @property def recording_transport_port(self) -> int | None: @@ -2099,6 +2205,13 @@ async def stop_recording_transport(self) -> None: listener, self._hds_listener = self._hds_listener, None if listener is not None: await listener.stop() + self._recording_media_running = False + if self._recording_media is not None: + if (task := self._recording_media_task) is not None: + self._recording_media_wake.set() + await asyncio.gather(task, return_exceptions=True) + self._recording_media_armed = False + await self._recording_media.close() async def stop(self) -> None: """Stop all streaming sessions and the HDS listener. diff --git a/pyhap/camera_multi_tier.py b/pyhap/camera_multi_tier.py index 2cc21dde..b54cde9d 100644 --- a/pyhap/camera_multi_tier.py +++ b/pyhap/camera_multi_tier.py @@ -642,6 +642,10 @@ def _answer(session: MultiTierSession) -> hksv3.SetupEndpointsResponse: class FFmpegMultiTierDelegate: """Streams each session with its own ffmpeg process. + A simple reference: each session opens its own camera connection. + :class:`pyhap.media.MultiTierMediaDelegate` shares one pull per camera + channel among all local and remote sessions. + Pass it as the ``delegate`` of the ``multi_tier`` option. The command is ``ffmpeg``, :data:`FFMPEG_GLOBAL_OPTIONS`, the ``ffmpeg_input`` arguments, :data:`FFMPEG_VIDEO_OUTPUT` diff --git a/pyhap/camera_talkback.py b/pyhap/camera_talkback.py index fcfec43c..9ba3e4f7 100644 --- a/pyhap/camera_talkback.py +++ b/pyhap/camera_talkback.py @@ -31,10 +31,7 @@ import threading from typing import cast -try: - import pylibsrtp -except ImportError: # pragma: no cover - exercised via requires_talkback() - pylibsrtp = None +import pylibsrtp logger = logging.getLogger(__name__) @@ -42,8 +39,12 @@ def talkback_available() -> bool: - """Whether the optional dependency for talkback support is installed.""" - return pylibsrtp is not None + """Whether talkback is supported, always since pylibsrtp is a dependency. + + Deprecated: kept for callers of the optional dependency check, it will be + removed in a future release. + """ + return True @dataclass @@ -148,12 +149,6 @@ def __init__( is_ipv6: bool = False, controller_address: str | None = None, ) -> None: - if pylibsrtp is None: - raise RuntimeError( - "Talkback support requires the 'pylibsrtp' package. " - "Install it with: pip install HAP-python[Talkback]" - ) - policy = pylibsrtp.Policy( key=srtp_key_and_salt, ssrc_type=pylibsrtp.Policy.SSRC_ANY_INBOUND, diff --git a/pyhap/hds_recording.py b/pyhap/hds_recording.py index 0054093c..57d7e702 100644 --- a/pyhap/hds_recording.py +++ b/pyhap/hds_recording.py @@ -16,15 +16,18 @@ from __future__ import annotations import asyncio -from collections.abc import AsyncIterator, Callable +from collections.abc import AsyncGenerator, AsyncIterator, Callable from dataclasses import dataclass from enum import IntEnum import functools import logging -from typing import Any, cast +from typing import TYPE_CHECKING, Any, Protocol, cast from pyhap.hds_protocol import HDSStatus, Message +if TYPE_CHECKING: + from pyhap.hksv_recording import SelectedRecordingConfiguration + logger = logging.getLogger(__name__) _DATA_SEND = "dataSend" @@ -62,6 +65,37 @@ class RecordingPacket: # A delegate is called with the stream id and yields RecordingPackets. The first # yielded packet is the MP4 initialization segment. RecordingDelegate = Callable[[int], AsyncIterator[RecordingPacket]] + + +class ByteReader(Protocol): + """A byte stream, e.g. an :class:`asyncio.StreamReader`.""" + + async def readexactly(self, n: int) -> bytes: + """Read ``n`` bytes or raise :class:`asyncio.IncompleteReadError`.""" + + +class RecordingProducer(Protocol): + """Makes a camera's recordings, see the camera's ``recording`` option. + + :class:`pyhap.media.recording_delegate.RecordingMediaDelegate` is one. + """ + + @property + def prebuffer_ms(self) -> int: + """The prebuffer the camera advertises.""" + + async def set_armed(self, armed: bool) -> None: + """Armed while the hub has recording on and the camera is on.""" + + def record( + self, selected: SelectedRecordingConfiguration, audio: bool + ) -> AsyncGenerator[RecordingPacket, None]: + """Yield the selected recording, the init segment first.""" + + async def close(self) -> None: + """Disarm and release everything.""" + + # An open gate returns a reason to reject a dataSend/open, or None to accept it. OpenGate = Callable[["RecordingStreamManager"], RecordingReason | None] @@ -276,7 +310,7 @@ async def _send_fragment( _MAX_FRAGMENT_SIZE = 64 * 1024 * 1024 -async def _read_mp4_box(reader: asyncio.StreamReader, max_size: int) -> bytes | None: +async def _read_mp4_box(reader: ByteReader, max_size: int) -> bytes | None: """Read one top-level MP4 box (header + body) from ``reader``. Returns ``None`` at end of stream (also mid-box) and for a size smaller @@ -299,7 +333,7 @@ async def _read_mp4_box(reader: asyncio.StreamReader, max_size: int) -> bytes | async def fmp4_recording_packets( - reader: asyncio.StreamReader, + reader: ByteReader, ) -> AsyncIterator[RecordingPacket]: """Turn a fragmented-MP4 byte stream into HKSV recording packets. diff --git a/pyhap/media/__init__.py b/pyhap/media/__init__.py new file mode 100644 index 00000000..72105658 --- /dev/null +++ b/pyhap/media/__init__.py @@ -0,0 +1,114 @@ +"""Media of HomeKit camera live view: WebRTC (remote) and multi-tier RTP (local). + +:class:`WebRTCSender`, :class:`MultiTierSender` and :class:`MediaSource` +import their transport (``pyOpenSSL``, ``pylibsrtp``) on first use; +:class:`MediaWorker` runs them in a worker process. +:class:`WebRTCMediaDelegate` and :class:`MultiTierMediaDelegate` run them for +the WebRTC and multi-tier sessions of :class:`pyhap.camera.Camera`, from one +:class:`SourcePool`; :class:`RecordingMediaDelegate` records classic HomeKit +Secure Video clips with a prebuffer from the same pool. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from .sdp import AudioParams, IceCandidateLike, VideoParams + + +class MediaError(Exception): + """A media session cannot do what was asked.""" + + +if TYPE_CHECKING: + from .ffmpeg import FFmpegInput, FFmpegSourceProvider + from .multi_tier import MultiTierSender + from .multi_tier_delegate import MultiTierMediaDelegate + from .pool import ( + ChannelLease, + ChannelPull, + MediaSourceLike, + MultiTierMediaSender, + RecorderLike, + SourceLease, + SourcePool, + WebRTCMediaSender, + ) + from .recording import Recorder + from .recording_delegate import RecordingError, RecordingMediaDelegate + from .sender import SFrameKeyLike, WebRTCSender + from .source import MediaSource + from .webrtc_delegate import WebRTCMediaDelegate + from .worker import ( + MediaWorker, + RemoteMediaSource, + RemoteMultiTierSender, + RemoteRecorder, + RemoteWebRTCSender, + ) + +__all__ = [ + "AudioParams", + "ChannelLease", + "ChannelPull", + "FFmpegInput", + "FFmpegSourceProvider", + "IceCandidateLike", + "WebRTCMediaSender", + "MediaSource", + "MediaSourceLike", + "MultiTierMediaDelegate", + "MultiTierMediaSender", + "MultiTierSender", + "Recorder", + "RecorderLike", + "RecordingError", + "RecordingMediaDelegate", + "RemoteMediaSource", + "RemoteMultiTierSender", + "RemoteRecorder", + "RemoteWebRTCSender", + "SFrameKeyLike", + "SourceLease", + "SourcePool", + "VideoParams", + "MediaError", + "WebRTCMediaDelegate", + "MediaWorker", + "WebRTCSender", +] + +_LAZY = { + "ChannelLease": "pool", + "ChannelPull": "pool", + "FFmpegInput": "ffmpeg", + "FFmpegSourceProvider": "ffmpeg", + "WebRTCMediaSender": "pool", + "MediaSourceLike": "pool", + "MultiTierMediaDelegate": "multi_tier_delegate", + "MultiTierMediaSender": "pool", + "MultiTierSender": "multi_tier", + "Recorder": "recording", + "RecorderLike": "pool", + "RecordingError": "recording_delegate", + "RecordingMediaDelegate": "recording_delegate", + "SourceLease": "pool", + "SourcePool": "pool", + "WebRTCMediaDelegate": "webrtc_delegate", + "MediaSource": "source", + "SFrameKeyLike": "sender", + "WebRTCSender": "sender", + "RemoteMediaSource": "worker", + "RemoteMultiTierSender": "worker", + "RemoteRecorder": "worker", + "RemoteWebRTCSender": "worker", + "MediaWorker": "worker", +} + + +def __getattr__(name: str) -> Any: + if name in _LAZY: + from importlib import import_module # noqa: PLC0415 + + return getattr(import_module(f".{_LAZY[name]}", __name__), name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/pyhap/media/_worker.py b/pyhap/media/_worker.py new file mode 100644 index 00000000..04fa66ab --- /dev/null +++ b/pyhap/media/_worker.py @@ -0,0 +1,394 @@ +"""The media worker process, see :mod:`pyhap.media.worker`. + +Started as ``python -m pyhap.media._worker ``, one level per +logger in ``FORWARDED_LOGGERS``. Requests arrive framed on stdin; responses +and events leave framed on the original stdout, which is moved aside first. +The process ends when stdin closes. In thread mode :func:`serve_socket` runs +the same server over a socket pair instead. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import logging +import os +import socket +import sys +from typing import TYPE_CHECKING, Any, NamedTuple + +from . import MediaError +from .multi_tier import MultiTierSender +from .recording import Recorder +from .sdp import AudioParams, VideoParams +from .source import MediaSource +from .worker import FORWARDED_LOGGERS, encode_message, read_message + +if TYPE_CHECKING: + from .sender import WebRTCSender + +logger = logging.getLogger(__name__) + +# Bytes waiting for the parent above which log records are dropped, and +# above which the worker gives up on a parent that stopped reading. +LOG_DROP_BUFFER = 1024 * 1024 +MAX_OUTPUT_BUFFER = 64 * 1024 * 1024 + + +class _Key(NamedTuple): + key: bytes + kid: int + + +class _Candidate(NamedTuple): + candidate: str + sdp_mid: str | None + sdp_mline_index: int | None + + +class _LogForwarder(logging.Handler): + def __init__(self, server: _Server) -> None: + super().__init__() + self._server = server + + def emit(self, record: logging.LogRecord) -> None: + try: + message = self.format(record) + self._server.send( + { + "event": "log", + "level": record.levelno, + "name": record.name, + "message": message, + } + ) + except Exception: # noqa: BLE001 - logging must never raise + self.handleError(record) + + +class _Server: + def __init__(self, output: asyncio.WriteTransport) -> None: + self._output = output + self._sessions: dict[int, WebRTCSender | MultiTierSender | Recorder] = {} + self._sources: dict[int, MediaSource] = {} + self._tasks: set[asyncio.Task[None]] = set() + self.stopping = asyncio.Event() + + def send(self, message: dict[str, Any], blob: bytes = b"") -> None: + output = self._output + if output.is_closing(): + return + buffered = output.get_write_buffer_size() + if message.get("event") == "log" and buffered > LOG_DROP_BUFFER: + # A parent that reads slowly loses log records, not media events. + return + if buffered > MAX_OUTPUT_BUFFER: + # The parent stopped reading; it sees the worker as gone. + output.close() + self.stopping.set() + return + output.write(encode_message(message, blob)) + + def handle(self, message: dict[str, Any]) -> None: + task = asyncio.get_running_loop().create_task(self._respond(message)) + self._tasks.add(task) + task.add_done_callback(self._tasks.discard) + + async def _respond(self, message: dict[str, Any]) -> None: + request_id = message.get("id") + try: + handler = getattr(self, f"_op_{message['op']}") + result = await handler(message) + except ( + MediaError, + ValueError, + KeyError, + TypeError, + RuntimeError, + OSError, + ) as err: + self.send({"id": request_id, "error": str(err) or type(err).__name__}) + return + except Exception as err: + logger.exception("Media worker request failed") + self.send({"id": request_id, "error": repr(err)}) + return + try: + self.send({"id": request_id, "result": result}) + except (ValueError, TypeError) as err: + # Too large or not serializable: the caller still gets an answer. + self.send({"id": request_id, "error": f"Answer not sent: {err}"}) + + async def _op_open_source(self, message: dict[str, Any]) -> dict[str, Any]: + source_id = int(message["source"]) + source = MediaSource(str(message["host"]), audio=bool(message["audio"])) + source.on_closed = lambda reason: self._source_closed(source_id, reason) + self._sources[source_id] = source + try: + rtp_input = await source.start() + except BaseException: + self._sources.pop(source_id, None) + source.on_closed = None + source.close() + raise + return {"rtp_input": list(rtp_input)} + + def _source_closed(self, source_id: int, reason: str) -> None: + if self._sources.pop(source_id, None) is not None: + self.send({"event": "source_closed", "source": source_id, "reason": reason}) + + async def _op_close_source(self, message: dict[str, Any]) -> None: + source = self._sources.pop(int(message["source"]), None) + if source is not None: + source.close(str(message.get("reason", "closed"))) + + async def _op_offer(self, message: dict[str, Any]) -> dict[str, Any]: + session_id = int(message["session"]) + config = message["config"] + source = None + if message.get("source") is not None: + source = self._sources.get(int(message["source"])) + if source is None: + raise MediaError("Unknown media source") + sframe = config["sframe"] + audio = config["audio"] + from .sender import WebRTCSender # noqa: PLC0415 - only for WebRTC sessions + + sender = WebRTCSender( + video=VideoParams(**config["video"]), + audio=AudioParams(**audio) if audio is not None else None, + sframe=None + if sframe is None + else _Key(bytes.fromhex(sframe[0]), int(sframe[1])), + source=source, + host_addresses=config["host_addresses"], + use_ipv6=bool(config["use_ipv6"]), + rtp_input_host=str(config["rtp_input_host"]), + **config["options"], + ) + self._wire(session_id, sender) + self._sessions[session_id] = sender + try: + sdp = await sender.create_offer() + except BaseException: + self._sessions.pop(session_id, None) + raise + return {"sdp": sdp, "rtp_input": list(sender.rtp_input)} + + def _wire(self, session_id: int, sender: WebRTCSender) -> None: + def closed(reason: str) -> None: + self._sessions.pop(session_id, None) + self.send({"event": "closed", "session": session_id, "reason": reason}) + + sender.on_closed = closed + sender.on_connected = lambda: self.send( + {"event": "connected", "session": session_id} + ) + sender.on_keyframe_request = lambda age, gop: self.send( + {"event": "keyframe_request", "session": session_id, "age": age, "gop": gop} + ) + sender.on_audio = lambda opus, timestamp: self.send( + {"event": "audio", "session": session_id, "timestamp": timestamp}, opus + ) + + def _session(self, message: dict[str, Any]) -> WebRTCSender: + sender = self._sessions.get(int(message["session"])) + if sender is None or isinstance(sender, (MultiTierSender, Recorder)): + raise MediaError("Unknown WebRTC session") + return sender + + def _multi_tier(self, message: dict[str, Any]) -> MultiTierSender: + sender = self._sessions.get(int(message["session"])) + if not isinstance(sender, MultiTierSender): + raise MediaError("Unknown multi-tier session") + return sender + + def _source(self, message: dict[str, Any]) -> MediaSource: + source = self._sources.get(int(message["source"])) + if source is None: + raise MediaError("Unknown media source") + return source + + async def _op_mt_bind(self, message: dict[str, Any]) -> dict[str, Any]: + session_id = int(message["session"]) + config = dict(message["config"]) + config["video_key"] = bytes.fromhex(config["video_key"]) + config["audio_key"] = bytes.fromhex(config["audio_key"]) + sender = MultiTierSender(**config) + + def closed(reason: str) -> None: + self._sessions.pop(session_id, None) + self.send({"event": "closed", "session": session_id, "reason": reason}) + + sender.on_closed = closed + sender.on_rtcp = lambda: self.send({"event": "rtcp", "session": session_id}) + sender.on_keyframe_request = lambda age, gop: self.send( + {"event": "keyframe_request", "session": session_id, "age": age, "gop": gop} + ) + self._sessions[session_id] = sender + try: + ports = await sender.bind() + except BaseException: + self._sessions.pop(session_id, None) + await sender.close() + raise + return {"ports": list(ports)} + + async def _op_mt_start(self, message: dict[str, Any]) -> None: + await self._multi_tier(message).start( + self._source(message), audio=bool(message["audio"]) + ) + + async def _op_mt_switch(self, message: dict[str, Any]) -> None: + await self._multi_tier(message).switch(self._source(message)) + + async def _op_rec_open(self, message: dict[str, Any]) -> None: + session_id = int(message["session"]) + recorder = Recorder( + self._source(message), + prebuffer=float(message["prebuffer"]), + audio=bool(message["audio"]), + ) + recorder.on_data = lambda chunk: self.send( + {"event": "rec_data", "session": session_id}, chunk + ) + recorder.on_ended = lambda error: self.send( + {"event": "rec_ended", "session": session_id, "error": error} + ) + + def closed(reason: str) -> None: + self._sessions.pop(session_id, None) + self.send({"event": "closed", "session": session_id, "reason": reason}) + + recorder.on_closed = closed + self._sessions[session_id] = recorder + + def _recorder(self, message: dict[str, Any]) -> Recorder: + recorder = self._sessions.get(int(message["session"])) + if not isinstance(recorder, Recorder): + raise MediaError("Unknown recorder") + return recorder + + async def _op_rec_start(self, message: dict[str, Any]) -> None: + await self._recorder(message).start( + [str(arg) for arg in message["command"]], float(message["prebuffer"]) + ) + + async def _op_rec_stop(self, message: dict[str, Any]) -> None: + await self._recorder(message).stop() + + async def _op_answer(self, message: dict[str, Any]) -> None: + candidates = [ + _Candidate( + str(c), None if m is None else str(m), None if i is None else int(i) + ) + for c, m, i in message["candidates"] + ] + await self._session(message).apply_answer(str(message["sdp"]), candidates) + + async def _op_reoffer(self, message: dict[str, Any]) -> dict[str, Any]: + return {"sdp": await self._session(message).reoffer(str(message["sdp"]))} + + async def _op_update_keys(self, message: dict[str, Any]) -> None: + add = [_Key(bytes.fromhex(key), int(kid)) for key, kid in message["add"]] + remove = [int(kid) for kid in message["remove"]] + await self._session(message).update_receive_keys(add, remove) + + async def _op_close(self, message: dict[str, Any]) -> None: + sender = self._sessions.pop(int(message["session"]), None) + if sender is not None: + await sender.close() + + async def _op_log_levels(self, message: dict[str, Any]) -> None: + for name, level in zip(FORWARDED_LOGGERS, message["levels"], strict=False): + logging.getLogger(name).setLevel(int(level)) + + async def _op_shutdown(self, message: dict[str, Any]) -> None: + await self.close_all() + self.stopping.set() + + async def close_all(self) -> None: + sessions, self._sessions = self._sessions, {} + await asyncio.gather( + *(sender.close() for sender in sessions.values()), return_exceptions=True + ) + sources, self._sources = self._sources, {} + for source in sources.values(): + source.on_closed = None + source.close("media worker stopped") + + +async def _serve( + reader: asyncio.StreamReader, + output: asyncio.WriteTransport, + *, + forward_logs: bool, +) -> None: + loop = asyncio.get_running_loop() + server = _Server(output) + forwarder = _LogForwarder(server) if forward_logs else None + if forwarder is not None: + for name in FORWARDED_LOGGERS: + logging.getLogger(name).addHandler(forwarder) + server.send({"event": "ready"}) + stopping = loop.create_task(server.stopping.wait()) + try: + while not server.stopping.is_set(): + reading = loop.create_task(read_message(reader)) + await asyncio.wait({reading, stopping}, return_when=asyncio.FIRST_COMPLETED) + if not reading.done(): + reading.cancel() + break + try: + message, _blob = reading.result() + except asyncio.IncompleteReadError: + break + server.handle(message) + finally: + stopping.cancel() + await server.close_all() + output.close() + if forwarder is not None: + for name in FORWARDED_LOGGERS: + logging.getLogger(name).removeHandler(forwarder) + + +async def _serve_stdio(output_fd: int) -> None: + loop = asyncio.get_running_loop() + reader = asyncio.StreamReader() + await loop.connect_read_pipe( + lambda: asyncio.StreamReaderProtocol(reader), os.fdopen(0, "rb", buffering=0) + ) + output, _ = await loop.connect_write_pipe( + asyncio.Protocol, os.fdopen(output_fd, "wb", buffering=0) + ) + await _serve(reader, output, forward_logs=True) + + +async def serve_socket(sock: socket.socket) -> None: + """Serve one parent over a connected socket, in the parent's process. + + The worker thread of :class:`~pyhap.media.worker.MediaWorker` runs this in + its own event loop. Loggers are shared with the parent, so nothing is + forwarded and their levels stay as the parent sets them. + """ + reader, writer = await asyncio.open_connection(sock=sock) + await _serve(reader, writer.transport, forward_logs=False) + + +def main() -> None: + """Run the worker until its stdin closes or it is told to stop.""" + # Keep the protocol stream for frames; anything printed goes to stderr. + output_fd = os.dup(1) + os.dup2(2, 1) + levels = [int(arg) for arg in sys.argv[1:]] + levels += [logging.WARNING] * (len(FORWARDED_LOGGERS) - len(levels)) + for name, level in zip(FORWARDED_LOGGERS, levels, strict=False): + logging.getLogger(name).setLevel(level) + logging.getLogger(name).propagate = False + with contextlib.suppress(KeyboardInterrupt): + asyncio.run(_serve_stdio(output_fd)) + + +if __name__ == "__main__": + main() diff --git a/pyhap/media/ffmpeg.py b/pyhap/media/ffmpeg.py new file mode 100644 index 00000000..c122957f --- /dev/null +++ b/pyhap/media/ffmpeg.py @@ -0,0 +1,553 @@ +"""A reference source provider that pulls each camera channel with ffmpeg. + +:class:`FFmpegSourceProvider` is a ``source_provider`` for +:class:`~pyhap.media.webrtc_delegate.WebRTCMediaDelegate`, +:class:`~pyhap.media.multi_tier_delegate.MultiTierMediaDelegate` and +:class:`~pyhap.media.recording_delegate.RecordingMediaDelegate`: it +resolves the tier of a session to ffmpeg input arguments and leases that +channel's source from its :class:`~pyhap.media.pool.SourcePool`, with an +:class:`FFmpegPull` as the channel's pull. One provider for both delegates +pulls each channel once for local and remote sessions and recordings. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable, Mapping, Sequence +import contextlib +from dataclasses import dataclass, replace +from functools import partial +import logging +import re +import time +from typing import TYPE_CHECKING, Any + +from pyhap import hksv3 +from pyhap.camera_stream import StreamError + +from .pool import ChannelLease, SourcePool + +if TYPE_CHECKING: + from pyhap.camera_stream import StreamSession + + from .worker import MediaWorker + +logger = logging.getLogger(__name__) + +_OFFER_ERROR = hksv3.WebRTCSolicitOfferStatus.ERROR +# Seconds ffmpeg gets to exit after SIGTERM before it is killed. +FFMPEG_STOP_TIMEOUT = 3.0 +# Userinfo may hold spaces and quotes, so it is matched up to its "@" first. +_URL = re.compile( + r"(?P[A-Za-z][A-Za-z0-9+.-]*://)(?:[^/?#@\n]*@)?(?P[^\s'\"]+)" +) +# Bytes of one stderr line kept before it is logged anyway. +MAX_LOG_LINE = 4096 +# Restarts after failures back off up to this many seconds. +MAX_RESTART_DELAY = 30.0 +# An ffmpeg that ran this long counts as healthy again. +HEALTHY_RUN = 30.0 + + +def redact_urls(text: str) -> str: + """Keep only scheme, host and port of the URLs in ``text``. + + Credentials, path and query of a camera URL may carry secrets, such as + an access token in the path. + """ + return _URL.sub(_redact_url, text) + + +def _redact_url(match: re.Match[str]) -> str: + rest = match["rest"] + authority = re.split(r"[/?#]", rest, maxsplit=1)[0] + host = authority.rpartition("@")[2] + hidden = "/" if len(authority) < len(rest) else "" + return f"{match['scheme']}{host}{hidden}" + + +@dataclass(frozen=True) +class FFmpegInput: + """The ffmpeg input of a camera channel. + + ``args`` are ffmpeg's input arguments, such as ``["-rtsp_transport", + "tcp", "-i", url]``. The other fields override the provider's defaults + when set: ``video_map`` and ``audio_map`` select the streams, ``audio`` + turns audio off for inputs without it, and ``gop`` is the expected + interval between IRAP pictures until the source has measured it. + """ + + args: tuple[str, ...] + video_map: str | None = None + audio_map: str | None = None + audio: bool | None = None + gop: float | None = None + + +InputResolver = Callable[ + [hksv3.VideoStreamTier], Awaitable["Sequence[str] | FFmpegInput | None"] +] + + +class FFmpegSourceProvider: + """A ``source_provider`` that pulls each channel with ffmpeg. + + ``inputs`` maps a tier identifier, or a :class:`pyhap.hksv3.VideoQuality` + (also by name, e.g. ``"medium"``), to ffmpeg input arguments or an + :class:`FFmpegInput`; an identifier wins over a quality. It can instead + be an async callable that resolves a tier to the same, or None when + nothing fits. Tiers with the same input share one ffmpeg. + + Video ``video_map`` is copied with ``dump_extra`` so every IRAP picture + carries its parameter sets (an Annex B input such as RTSP; MP4 needs + ``hevc_mp4toannexb``); ``audio_map``, unless ``audio`` is false, is + encoded to mono Opus. ffmpeg fails when a mapped stream is missing, so + set ``audio=False`` for inputs without audio. The Opus bit rate and frame + duration follow the audio tier of the session that opens the channel; a + camera has one audio tier for its multi-tier and WebRTC sessions, so all + sessions of a channel get the same audio. The command is an argument + list that no shell sees; the URLs in ffmpeg's log are reduced to scheme, + host and port. + + ffmpeg is restarted when it exits while sessions need it (after + ``restart_delay``), and for a keyframe request only when no IRAP picture + came for twice the GOP and ffmpeg runs at least that long: a copied + stream gets its next IRAP picture with the next GOP anyway. The GOP is + the one the source measures, until then ``gop``. + """ + + def __init__( + self, + inputs: Mapping[int | hksv3.VideoQuality | str, Sequence[str] | FFmpegInput] + | InputResolver, + *, + worker: MediaWorker | None = None, + ffmpeg: str = "ffmpeg", + video_map: str = "0:v:0", + audio_map: str = "0:a:0", + audio: bool = True, + gop: float = 2.0, + restart_delay: float = 1.0, + stats_period: float | None = None, + host: str = "127.0.0.1", + on_error: Callable[[str, str], None] | None = None, + ) -> None: + """Configure the inputs; nothing runs before the first answer. + + ``on_error(channel, message)`` hears each unexpected ffmpeg exit or + failed start, e.g. to mark the camera unavailable. + """ + self._resolver: InputResolver | None = None + self._by_id: dict[int, FFmpegInput] = {} + self._by_quality: dict[int, FFmpegInput] = {} + if not isinstance(inputs, Mapping): + self._resolver = inputs + else: + for key, value in inputs.items(): + if isinstance(key, str): + self._by_quality[hksv3.VideoQuality[key.upper()]] = _input(value) + elif isinstance(key, hksv3.VideoQuality): + self._by_quality[key] = _input(value) + else: + self._by_id[key] = _input(value) + self.pool = SourcePool(worker, host) + self.ffmpeg = ffmpeg + self.video_map = video_map + self.audio_map = audio_map + self.audio = audio + self.gop = gop + self.restart_delay = restart_delay + self.stats_period = stats_period + self.on_error = on_error + self._pulls: dict[str, FFmpegPull] = {} + + async def input_for(self, tier: hksv3.VideoStreamTier) -> FFmpegInput: + """The input of a tier, with the provider's defaults filled in. + + :raises StreamError: If no input fits the tier. + """ + if self._resolver is not None: + value = await self._resolver(tier) + found = None if value is None else _input(value) + else: + found = self._by_id.get(tier.identifier) or self._by_quality.get( + tier.quality + ) + if found is None: + raise StreamError(_OFFER_ERROR, f"No ffmpeg input for tier {tier}") + return FFmpegInput( + found.args, + found.video_map or self.video_map, + found.audio_map or self.audio_map, + self.audio if found.audio is None else found.audio, + found.gop or self.gop, + ) + + def channel_name(self, tier: hksv3.VideoStreamTier) -> str: + """How the log names the channel of a tier, without its input.""" + if tier.identifier in self._by_id: + return f"tier {tier.identifier}" + with contextlib.suppress(ValueError): + return hksv3.VideoQuality(tier.quality).name.lower() + return f"tier {tier.identifier}" + + def command( + self, + ffmpeg_input: FFmpegInput, + rtp_input: tuple[str, int, int], + audio_bitrate: int, + packet_time: int = 20, + ) -> list[str]: + """The ffmpeg arguments sending one input to a source's ports.""" + host, video_port, audio_port = rtp_input + host = f"[{host}]" if ":" in host else host + stats = ( + ["-stats", "-stats_period", str(self.stats_period)] + if self.stats_period + else ["-nostats"] + ) + video_map = ffmpeg_input.video_map or self.video_map + command = [ + *(self.ffmpeg, "-hide_banner", "-nostdin", "-loglevel", "warning"), + *stats, + *ffmpeg_input.args, + *("-map", video_map, "-c:v", "copy", "-bsf:v", "dump_extra"), + *("-payload_type", str(hksv3.VIDEO_PAYLOAD_TYPE)), + # The source drops RTCP, and port + 1 may be the audio input. + *("-rtpflags", "skip_rtcp", "-f", "rtp", f"rtp://{host}:{video_port}"), + ] + if audio_port: + audio_map = ffmpeg_input.audio_map or self.audio_map + command += [ + *("-map", audio_map, "-c:a", "libopus", "-ac", "1", "-ar", "48000"), + *("-b:a", str(audio_bitrate), "-application", "lowdelay"), + *("-frame_duration", str(packet_time)), + *("-payload_type", str(hksv3.AUDIO_PAYLOAD_TYPE)), + *("-rtpflags", "skip_rtcp", "-f", "rtp", f"rtp://{host}:{audio_port}"), + ] + return command + + async def __call__( + self, + session: StreamSession, + video_tier: hksv3.VideoStreamTier, + audio_tier: hksv3.AudioStreamTier | None, + ) -> ChannelLease: + """Lease the source of the session's channel. + + ``audio_tier`` is None for a multi-tier session without audio; the + channel's audio then follows :data:`pyhap.hksv3.DEFAULT_AUDIO_TIER`. + """ + return await self.lease_channel(video_tier, audio_tier) + + async def lease_channel( + self, + video_tier: hksv3.VideoStreamTier, + audio_tier: hksv3.AudioStreamTier | None = None, + ) -> ChannelLease: + """Lease the source of a tier's channel without a session. + + For example to keep a recording channel buffered; see + :class:`~pyhap.media.recording_delegate.RecordingMediaDelegate`. + """ + found = await self.input_for(video_tier) + name = self.channel_name(video_tier) + audio = audio_tier or hksv3.DEFAULT_AUDIO_TIER + command = partial( + self.command, + found, + audio_bitrate=audio.target_bitrate, + packet_time=audio.packet_time, + ) + gop = found.gop or self.gop + + def make_pull() -> FFmpegPull: + pull = FFmpegPull( + command, name, gop, self.restart_delay, on_error=self._pull_error + ) + self._pulls[name] = pull + return pull + + return await self.pool.lease( + replace(found, gop=None), + make_pull, + audio=bool(found.audio), + name=name, + ) + + def _pull_error(self, channel: str, message: str) -> None: + if self.on_error is not None: + try: + self.on_error(channel, message) + except Exception: + logger.exception("Error in the ffmpeg error callback") + + def diagnostics(self) -> dict[str, Any]: + """Leases and the ffmpeg of each channel, for an integration's diagnostics.""" + counts = self.pool.counts() + worker = self.pool.worker + return { + "worker_pid": worker.pid if worker is not None else None, + "channels": { + name: { + "leases": leases, + "active": active, + **(self._pulls[name].diagnostics() if name in self._pulls else {}), + } + for name, (leases, active) in counts.items() + }, + } + + async def close(self) -> None: + """Stop every ffmpeg and close every source.""" + await self.pool.close() + + +def _input(value: Sequence[str] | FFmpegInput) -> FFmpegInput: + if isinstance(value, FFmpegInput): + return replace(value, args=tuple(value.args)) + return FFmpegInput(tuple(value)) + + +class FFmpegPull: + """One ffmpeg process feeding a channel's source, see ``ChannelPull``.""" + + def __init__( + self, + command: Callable[[tuple[str, int, int]], list[str]], + name: str, + gop: float, + restart_delay: float, + on_error: Callable[[str, str], None] | None = None, + ) -> None: + """Configure the pull; :meth:`start` runs ffmpeg. + + ``gop`` is the expected IRAP interval until the source measured one. + ``on_error(name, message)`` hears each unexpected exit or failed start. + """ + self.name = name + self.gop = gop + self.restart_delay = restart_delay + self.on_error = on_error + self.restarts = 0 + self.last_error: str | None = None + self._failures = 0 + self._command = command + self._argv: list[str] = [] + self._process: asyncio.subprocess.Process | None = None + self._watcher: asyncio.Task[None] | None = None + self._started = 0.0 + self._running = False + self._restart_now = asyncio.Event() + + @property + def pid(self) -> int | None: + """The process ID of the running ffmpeg.""" + process = self._process + return process.pid if process and process.returncode is None else None + + async def start(self, rtp_input: tuple[str, int, int]) -> None: + """Run ffmpeg and keep it running until :meth:`stop`.""" + self._argv = self._command(rtp_input) + self._running = True + try: + await self._spawn() + except BaseException as err: + self._running = False + if isinstance(err, OSError): + message = f"could not start: {err}" + logger.warning("ffmpeg for %s %s", self.name, message) + self._report(message) + raise + self._watcher = asyncio.ensure_future(self._watch()) + + async def _spawn(self) -> None: + self._restart_now.clear() + self._process = await asyncio.create_subprocess_exec( + *self._argv, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE, + ) + self._started = time.monotonic() + logger.info("ffmpeg for %s started, pid %d", self.name, self._process.pid) + + def diagnostics(self) -> dict[str, Any]: + """The pid, restarts and last error, for diagnostics.""" + return { + "pid": self.pid, + "restarts": self.restarts, + "last_error": self.last_error, + } + + async def _watch(self) -> None: + while self._running and self._process is not None: + process = self._process + last_line = await _log_output(self.name, process) + code = await process.wait() + if not self._running: + return + if not self._restart_now.is_set(): + if time.monotonic() - self._started >= HEALTHY_RUN: + self._failures = 0 + await self._back_off( + f"exited with {code}" + (f": {last_line}" if last_line else "") + ) + while self._running: + try: + await self._spawn() + except OSError as err: + # No new process: back off further before the next try. + await self._back_off(f"could not start: {err}") + continue + self.restarts += 1 + break + + async def _back_off(self, message: str) -> None: + """Report ``message`` and wait the next backoff delay or a restart.""" + delay = min(self.restart_delay * 2**self._failures, MAX_RESTART_DELAY) + self._failures += 1 + self._error(message, delay) + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(self._restart_now.wait(), delay) + + def _error(self, message: str, delay: float) -> None: + logger.warning( + "ffmpeg for %s %s, restarting in %.1f s", self.name, message, delay + ) + self._report(message) + + def _report(self, message: str) -> None: + self.last_error = message + if self.on_error is not None: + try: + self.on_error(self.name, message) + except Exception: + logger.exception("Error in the ffmpeg error callback") + + def keyframe_requested(self, age: float | None, gop: float | None) -> None: + """Restart ffmpeg if its stream has gone without IRAP pictures. + + ``gop`` is the IRAP interval the source measured, if it did. + """ + limit = 2 * (gop or self.gop) + if not self._running or self._restart_now.is_set(): + return + if age is not None and age <= limit: + return + running = time.monotonic() - self._started + if running < limit: + # Started recently, its first IRAP picture is on the way. + return + logger.info( + "Restarting ffmpeg for %s for an IRAP picture (last %s s ago)", + self.name, + "-" if age is None else f"{age:.1f}", + ) + self._restart_now.set() + if self._process is not None and self._process.returncode is None: + self._process.terminate() + + async def stop(self) -> None: + """Stop ffmpeg, killing it if it does not exit in time.""" + self._running = False + self._restart_now.set() + await self._terminate() + if self._watcher is not None: + await self._watcher + # The watcher may have started one more meanwhile. + await self._terminate() + process = self._process + if process is not None: + logger.info( + "ffmpeg for %s stopped (exit %s, %d restarts)", + self.name, + process.returncode, + self.restarts, + ) + + async def _terminate(self) -> None: + process = self._process + if process is None or process.returncode is not None: + return + with contextlib.suppress(ProcessLookupError): + process.terminate() + try: + await asyncio.wait_for(process.wait(), FFMPEG_STOP_TIMEOUT) + except TimeoutError: + logger.warning("Killing ffmpeg for %s", self.name) + with contextlib.suppress(ProcessLookupError): + process.kill() + await process.wait() + + +async def _log_output(name: str, process: asyncio.subprocess.Process) -> str | None: + """Log ffmpeg's output line by line, URLs redacted; progress ends with CR. + + Returns the last line, redacted, to explain an exit. + """ + stream = process.stderr + if stream is None: + return None + lines = StderrLines() + last: bytes | None = None + while chunk := await stream.read(4096): + for line in lines.feed(chunk): + _log_line(name, line) + last = line + for line in lines.end(): + _log_line(name, line) + last = line + if last is None: + return None + return redact_urls(last.decode(errors="replace")).strip() or None + + +class StderrLines: + """Splits ffmpeg's stderr into lines of at most about MAX_LOG_LINE bytes. + + An overlong line is returned once, cut, and its rest skipped up to the + next line end. Logging it in parts could split a URL so that its + credentials escape redaction. + """ + + def __init__(self) -> None: + self.pending = b"" + self._skipping = False + + def feed(self, chunk: bytes) -> list[bytes]: + """Add output, return the lines it completes.""" + lines = (self.pending + chunk).replace(b"\r", b"\n").split(b"\n") + self.pending = lines.pop() + if self._skipping and lines: + lines.pop(0) + self._skipping = False + done = [_cut(line) for line in lines if line] + if self._skipping: + self.pending = b"" + elif len(self.pending) > MAX_LOG_LINE: + done.append(_cut(self.pending)) + self.pending = b"" + self._skipping = True + return done + + def end(self) -> list[bytes]: + """Return the last line, which had no line end.""" + return [self.pending] if self.pending else [] + + +def _cut(line: bytes) -> bytes: + """Cut an overlong line before its last URL and its last partial word. + + A URL may hold spaces in its userinfo, so a cut URL is dropped whole. + """ + if len(line) <= MAX_LOG_LINE: + return line + head = line[:MAX_LOG_LINE] + if (url := head.rfind(b"://")) != -1: + head = head[:url] + return head.rpartition(b" ")[0] + b" [truncated]" + + +def _log_line(name: str, line: bytes) -> None: + logger.debug("ffmpeg %s: %s", name, redact_urls(line.decode(errors="replace"))) diff --git a/pyhap/media/hevc.py b/pyhap/media/hevc.py new file mode 100644 index 00000000..8bc029be --- /dev/null +++ b/pyhap/media/hevc.py @@ -0,0 +1,339 @@ +"""H.265 RTP payload handling (RFC 7798) for the camera media senders. + +The integrator's ffmpeg sends HEVC as RTP (``-c:v copy -bsf:v dump_extra -f +rtp``). That suits Annex B inputs such as RTSP; an MP4 or other hvcC input +needs ``-bsf:v hevc_mp4toannexb`` instead, which also puts the parameter sets +in front of every IRAP picture. SFrame protects whole frames, so the sender +turns that RTP back into access units, writes each one as 4-byte +length-prefixed NAL units (the hvcC sample layout) and encrypts the result. + +Only the RFC 7798 packet types without DONL are handled: single NAL unit +packets, aggregation packets (type 48) and fragmentation units (type 49). The +receiver needs VPS, SPS and PPS in-band in front of every IRAP picture, since +the SDP carries no sprop parameter sets; parameter sets seen earlier are +inserted when an IRAP access unit arrives without them. + +The SFrame plaintext matches what Scrypted's HomeKit WebRTC plugin sends for +the same RTP input (https://github.com/koush/scrypted, ``plugins/webrtc``). + +:class:`HevcPacketizer` is the reverse direction, for sessions without +SFrame, which send plain RFC 7798 RTP. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass, field +import logging +import struct + +logger = logging.getLogger(__name__) + +NAL_HEADER_SIZE = 2 +NAL_VPS = 32 +NAL_SPS = 33 +NAL_PPS = 34 +NAL_AUD = 35 +NAL_AP = 48 +NAL_FU = 49 +IRAP_FIRST = 16 +IRAP_LAST = 23 +PARAMETER_SETS = (NAL_VPS, NAL_SPS, NAL_PPS) +MAX_ACCESS_UNIT = 16 * 1024 * 1024 + + +def nal_type(nal: bytes) -> int: + """Return the NAL unit type of an H.265 NAL unit or RTP payload.""" + return (nal[0] >> 1) & 0x3F + + +def is_irap(nal: bytes) -> bool: + """Whether a NAL unit is an IRAP (BLA, IDR or CRA) picture slice.""" + return IRAP_FIRST <= nal_type(nal) <= IRAP_LAST + + +def payload_starts_irap(payload: bytes) -> bool: + """Whether an RTP payload starts or carries an IRAP slice. + + A cheap look at the payload header (and the FU header or AP units), + without reassembling anything. + """ + if len(payload) < NAL_HEADER_SIZE + 1: + return False + kind = nal_type(payload) + if kind == NAL_FU: + return bool(payload[2] & 0x80) and IRAP_FIRST <= payload[2] & 0x3F <= IRAP_LAST + if kind == NAL_AP: + return any(is_irap(nal) for nal in split_aggregation_packet(payload) or ()) + return is_irap(payload) + + +def length_prefixed(nals: Iterable[bytes]) -> bytes: + """Join NAL units with 4-byte big-endian length prefixes.""" + return b"".join(struct.pack(">I", len(nal)) + nal for nal in nals) + + +def split_length_prefixed(data: bytes) -> list[bytes]: + """Split 4-byte length-prefixed NAL units.""" + nals = [] + offset = 0 + while offset < len(data): + if offset + 4 > len(data): + raise ValueError("Truncated NAL unit length") + (length,) = struct.unpack_from(">I", data, offset) + offset += 4 + if offset + length > len(data): + raise ValueError("Truncated NAL unit") + nals.append(data[offset : offset + length]) + offset += length + return nals + + +def split_aggregation_packet(payload: bytes) -> list[bytes] | None: + """Return the NAL units of an AP, or ``None`` when it is malformed.""" + nals = [] + offset = NAL_HEADER_SIZE + while offset < len(payload): + if offset + 2 > len(payload): + return None + (length,) = struct.unpack_from(">H", payload, offset) + offset += 2 + if length < NAL_HEADER_SIZE or offset + length > len(payload): + return None + nals.append(payload[offset : offset + length]) + offset += length + return nals or None + + +@dataclass +class AccessUnit: + """All NAL units of one picture, sharing one RTP timestamp.""" + + timestamp: int + nals: list[bytes] = field(default_factory=list) + + @property + def keyframe(self) -> bool: + """Whether the access unit contains an IRAP picture.""" + return any(is_irap(nal) for nal in self.nals) + + @property + def self_contained(self) -> bool: + """Whether a decoder can start here: IRAP with VPS, SPS and PPS.""" + kinds = {nal_type(nal) for nal in self.nals} + return self.keyframe and all(kind in kinds for kind in PARAMETER_SETS) + + def to_bytes(self) -> bytes: + """Return the 4-byte length-prefixed SFrame plaintext.""" + return length_prefixed(self.nals) + + +class HevcDepacketizer: + """Reassembles access units from RFC 7798 RTP payloads. + + An access unit ends with the RTP marker bit. A sequence gap, a broken + fragmentation unit or an unsupported packet type damages the access unit, + which is then dropped; the next one starts clean. + """ + + def __init__(self) -> None: + """Start without cached parameter sets.""" + self.parameter_sets: dict[int, bytes] = {} + self.dropped = 0 + self._timestamp: int | None = None + self._last_sequence: int | None = None + self._nals: list[bytes] = [] + self._size = 0 + self._fragment: list[bytes] | None = None + self._damaged = False + + def reset(self) -> None: + """Drop the access unit in progress, for example on a new input SSRC. + + Cached parameter sets go too: a new source sends its own in-band, and + an old set must not be put in front of its pictures. + """ + self.parameter_sets = {} + self.restart() + + def restart(self) -> None: + """Drop the access unit in progress and forget the last sequence. + + Cached parameter sets stay, for a pause in the same input stream. + """ + self._last_sequence = None + self._reset() + + def push( + self, sequence: int, timestamp: int, marker: bool, payload: bytes + ) -> AccessUnit | None: + """Add one RTP payload; return the access unit completed by it.""" + if self._timestamp is not None and timestamp != self._timestamp: + # A new timestamp without a marker: the previous unit is incomplete. + self._drop() + if self._timestamp is None: + self._timestamp = timestamp + if ( + self._last_sequence is not None + and sequence != (self._last_sequence + 1) & 0xFFFF + ): + self._damaged = True + self._last_sequence = sequence + if not self._damaged: + self._add_payload(payload) + if not marker: + return None + if self._fragment is not None: + self._damaged = True + if self._damaged or not self._nals: + self._drop() + return None + unit = AccessUnit(timestamp, self._complete(self._nals)) + self._reset() + return unit + + def _add_payload(self, payload: bytes) -> None: + if len(payload) < NAL_HEADER_SIZE: + self._damaged = True + return + kind = nal_type(payload) + if kind == NAL_FU: + self._add_fragment(payload) + return + if self._fragment is not None: + self._damaged = True + return + if kind == NAL_AP: + nals = split_aggregation_packet(payload) + if nals is None: + self._damaged = True + return + for nal in nals: + self._add_nal(nal) + return + if kind > NAL_AP: + self._damaged = True + return + self._add_nal(payload) + + def _add_fragment(self, payload: bytes) -> None: + if len(payload) < 3: + self._damaged = True + return + fu_header = payload[2] + start, end = fu_header & 0x80, fu_header & 0x40 + if start: + if self._fragment is not None: + self._damaged = True + return + header = bytes([(payload[0] & 0x81) | (fu_header & 0x3F) << 1, payload[1]]) + self._fragment = [header] + elif self._fragment is None: + self._damaged = True + return + self._fragment.append(payload[3:]) + self._size += len(payload) - 3 + if self._size > MAX_ACCESS_UNIT: + self._damaged = True + return + if end: + nal = b"".join(self._fragment) + self._fragment = None + self._size -= len(nal) - NAL_HEADER_SIZE + self._add_nal(nal) + + def _add_nal(self, nal: bytes) -> None: + self._size += 4 + len(nal) + if self._size > MAX_ACCESS_UNIT: + # Bound memory when a source never sets the marker bit. + self._damaged = True + return + kind = nal_type(nal) + if kind in PARAMETER_SETS: + self.parameter_sets[kind] = nal + self._nals.append(nal) + + def _complete(self, nals: list[bytes]) -> list[bytes]: + """Put cached parameter sets in front of an IRAP that lacks them.""" + if not any(is_irap(nal) for nal in nals): + return nals + present = {nal_type(nal) for nal in nals} + missing = [ + self.parameter_sets[kind] + for kind in PARAMETER_SETS + if kind not in present and kind in self.parameter_sets + ] + # An access unit delimiter stays the first NAL unit (H.265 7.4.2.4.4). + lead = 1 if nals and nal_type(nals[0]) == NAL_AUD else 0 + return nals[:lead] + missing + nals[lead:] + + def _drop(self) -> None: + if self._nals or self._fragment is not None or self._damaged: + self.dropped += 1 + self._reset() + + def _reset(self) -> None: + self._timestamp = None + self._nals = [] + self._size = 0 + self._fragment = None + self._damaged = False + + +def fragment(nal: bytes, max_payload: int) -> list[bytes]: + """Split one NAL unit into RFC 7798 fragmentation units.""" + kind = nal_type(nal) + header = bytes([(nal[0] & 0x81) | NAL_FU << 1, nal[1]]) + chunk = max_payload - 3 + body = nal[NAL_HEADER_SIZE:] + payloads = [] + for offset in range(0, len(body), chunk): + flags = (0x80 if offset == 0 else 0) | ( + 0x40 if offset + chunk >= len(body) else 0 + ) + payloads.append(header + bytes([flags | kind]) + body[offset : offset + chunk]) + return payloads + + +class HevcPacketizer: + """Packetizes access units into RFC 7798 RTP payloads (no DONL).""" + + def __init__(self, max_payload: int = 1200) -> None: + """Set the RTP payload budget.""" + if max_payload < 4: + raise ValueError("Payload budget too small for H.265") + self.max_payload = max_payload + + def packetize(self, nals: Iterable[bytes]) -> list[bytes]: + """Return the RTP payloads of one access unit, in order.""" + payloads: list[bytes] = [] + pending: list[bytes] = [] + for nal in nals: + if len(nal) > self.max_payload: + payloads += self._aggregate(pending) + pending = [] + payloads += self._fragment(nal) + continue + if self._ap_size([*pending, nal]) > self.max_payload: + payloads += self._aggregate(pending) + pending = [] + pending.append(nal) + payloads += self._aggregate(pending) + return payloads + + @staticmethod + def _ap_size(nals: list[bytes]) -> int: + return NAL_HEADER_SIZE + sum(2 + len(nal) for nal in nals) + + @staticmethod + def _aggregate(nals: list[bytes]) -> list[bytes]: + if len(nals) <= 1: + return list(nals) + # The AP header carries the lowest LayerId and TID (RFC 7798 4.4.2). + layer = min(((nal[0] & 1) << 5) | (nal[1] >> 3) for nal in nals) + tid = min(nal[1] & 7 for nal in nals) + header = bytes([NAL_AP << 1 | layer >> 5, (layer & 0x1F) << 3 | tid]) + return [header + b"".join(struct.pack(">H", len(n)) + n for n in nals)] + + def _fragment(self, nal: bytes) -> list[bytes]: + return fragment(nal, self.max_payload) diff --git a/pyhap/media/ice.py b/pyhap/media/ice.py new file mode 100644 index 00000000..b1f4e12b --- /dev/null +++ b/pyhap/media/ice.py @@ -0,0 +1,680 @@ +"""A small ICE agent with host candidates only (RFC 8445 subset). + +One component (BUNDLE with rtcp-mux), UDP, one socket per local address. +Both roles: the controlling agent checks every pair and nominates the best +pair that succeeded with a second check carrying USE-CANDIDATE (regular +nomination); the controlled agent answers checks, runs triggered checks and +selects the pair the controlling agent nominates. Role conflicts are +resolved with the tie-breaker and 487 (RFC 8445 section 7.3.1.1). + +No STUN or TURN server is used: HomeKit controllers reach the accessory +through a relay of their own (remote) or on the LAN, and the accessory's +outgoing check opens its NAT towards the relay. A request from an unknown +address with a valid MESSAGE-INTEGRITY becomes a peer-reflexive pair. + +After selection, consent freshness (RFC 7675) sends a check on the selected +pair every few seconds; without an authenticated response for +:data:`CONSENT_TIMEOUT` seconds the agent closes. Non-STUN datagrams are +only accepted from the remote address of the selected pair or of another +pair that passed its checks (before selection: of any pair), into a +bounded queue. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable, Coroutine, Iterable, Sequence +import contextlib +from dataclasses import dataclass, field +import enum +import ipaddress +import logging +import random +import secrets +import socket +import string +import time +from typing import Any, cast + +from . import stun +from .stun import Attr, Class, Message, Method + +logger = logging.getLogger(__name__) + +# Pacing of new checks (RFC 8445 section 14.2). +TA = 0.05 +# Retransmission timeouts of one check; about 8 s in all. +CHECK_RTOS = (0.2, 0.4, 0.8, 1.6, 1.6, 1.6, 1.6) +CONSENT_INTERVAL = 5.0 +CONSENT_TIMEOUT = 30.0 +MAX_PAIRS = 64 +MAX_QUEUED = 512 +MAX_QUEUED_BEFORE_CONNECT = 32 +HOST_TYPE_PREFERENCE = 126 +PRFLX_TYPE_PREFERENCE = 110 +COMPONENT = 1 +ICE_CHARS = string.ascii_letters + string.digits + "+/" + + +def _random_ice_string(length: int) -> str: + return "".join(secrets.choice(ICE_CHARS) for _ in range(length)) + + +def candidate_priority(type_preference: int, local_preference: int) -> int: + """The priority of a component-1 candidate (RFC 8445 section 5.1.2.1).""" + return (type_preference << 24) | (local_preference << 8) | (256 - COMPONENT) + + +def pair_priority(controlling: int, controlled: int) -> int: + """The priority of a pair (RFC 8445 section 6.1.2.3).""" + low, high = min(controlling, controlled), max(controlling, controlled) + return (low << 32) + 2 * high + (1 if controlling > controlled else 0) + + +def _address(addr: Sequence[object]) -> tuple[str, int]: + """Normalize a socket address to (compressed IP without scope, port).""" + host = str(addr[0]).split("%", 1)[0] + with contextlib.suppress(ValueError): + host = ipaddress.ip_address(host).compressed + return host, int(str(addr[1])) + + +@dataclass(frozen=True) +class Candidate: + """An ICE candidate as in an SDP ``a=candidate`` line.""" + + foundation: str + component: int + transport: str + priority: int + host: str + port: int + type: str + related_address: str | None = None + related_port: int | None = None + + @classmethod + def from_sdp(cls, value: str) -> Candidate: + """Parse the text after ``candidate:``; raises ValueError if malformed.""" + parts = value.split() + if len(parts) < 8 or parts[6] != "typ": + raise ValueError(f"Bad candidate {value!r}") + extensions = dict(zip(parts[8::2], parts[9::2], strict=False)) + rport = extensions.get("rport") + return cls( + foundation=parts[0], + component=int(parts[1]), + transport=parts[2], + priority=int(parts[3]), + host=parts[4], + port=int(parts[5]), + type=parts[7], + related_address=extensions.get("raddr"), + related_port=int(rport) if rport is not None else None, + ) + + def to_sdp(self) -> str: + """The text after ``candidate:``.""" + value = ( + f"{self.foundation} {self.component} {self.transport} {self.priority} " + f"{self.host} {self.port} typ {self.type}" + ) + if self.related_address is not None and self.related_port is not None: + value += f" raddr {self.related_address} rport {self.related_port}" + return value + + +class PairState(enum.Enum): + """The state of a candidate pair.""" + + WAITING = "waiting" + IN_PROGRESS = "in progress" + SUCCEEDED = "succeeded" + FAILED = "failed" + + +@dataclass(eq=False) +class CandidatePair: + """A local socket and a remote address.""" + + local: _Socket + remote: tuple[str, int] + remote_priority: int + state: PairState = PairState.WAITING + # The controlling peer sent USE-CANDIDATE on this pair. + remote_nominated: bool = False + last_response: float = field(default=0.0) + + def priority(self, controlling: bool) -> int: + """The pair priority from this agent's role.""" + local = self.local.candidate.priority + if controlling: + return pair_priority(local, self.remote_priority) + return pair_priority(self.remote_priority, local) + + def __str__(self) -> str: + """``local -> remote`` for logs.""" + local = self.local.candidate + return f"{local.host}:{local.port} -> {self.remote[0]}:{self.remote[1]}" + + +class _Socket(asyncio.DatagramProtocol): + """One local UDP socket and its host candidate.""" + + def __init__(self, agent: IceAgent) -> None: + self.agent = agent + self.transport: asyncio.DatagramTransport | None = None + self.candidate: Candidate + + def connection_made(self, transport: asyncio.BaseTransport) -> None: + self.transport = cast("asyncio.DatagramTransport", transport) + + def datagram_received(self, data: bytes, addr: tuple[str | object, ...]) -> None: + self.agent._datagram_received(self, data, _address(addr)) + + def error_received(self, exc: Exception) -> None: + logger.debug("ICE socket error: %s", exc) + + def sendto(self, data: bytes, addr: tuple[str, int]) -> None: + if self.transport is not None and not self.transport.is_closing(): + self.transport.sendto(data, addr) + + def close(self) -> None: + if self.transport is not None: + self.transport.close() + + +class IceAgent: + """ICE for one component with host candidates. + + Call :meth:`gather`, then set :attr:`remote_username` and + :attr:`remote_password` and :meth:`connect`. :meth:`recv` raises + ConnectionError once the agent is closed, also after consent expired. + """ + + def __init__( + self, + *, + controlling: bool, + consent_interval: float = CONSENT_INTERVAL, + consent_timeout: float = CONSENT_TIMEOUT, + ) -> None: + """Create the agent; nothing is bound until :meth:`gather`.""" + self.controlling = controlling + self.local_username = _random_ice_string(8) + self.local_password = _random_ice_string(24) + self.remote_username: str | None = None + self.remote_password: str | None = None + self.remote_is_lite = False + self.on_closed: Callable[[str], None] | None = None + self.tie_breaker = secrets.randbits(64) + self._consent_interval = consent_interval + self._consent_timeout = consent_timeout + self._sockets: list[_Socket] = [] + self._pairs: list[CandidatePair] = [] + self._selected: CandidatePair | None = None + self._transactions: dict[ + bytes, tuple[CandidatePair, asyncio.Future[Message]] + ] = {} + self._tasks: set[asyncio.Task[object]] = set() + self._queue: asyncio.Queue[bytes | None] = asyncio.Queue() + self._wake = asyncio.Event() + self._connecting = False + self._closed = False + self._close_reason = "closed" + + @property + def local_candidates(self) -> list[Candidate]: + """The host candidates gathered.""" + return [s.candidate for s in self._sockets] + + @property + def selected_pair(self) -> CandidatePair | None: + """The pair media flows on, once selected.""" + return self._selected + + @property + def write_buffer_size(self) -> int: + """Bytes the selected pair's socket could not send yet.""" + pair = self._selected + transport = pair.local.transport if pair is not None else None + if transport is None: + return 0 + # Selector datagram transports have it; typeshed only lists it on + # WriteTransport. + size = getattr(transport, "get_write_buffer_size", None) + return int(size()) if size is not None else 0 + + async def gather(self, addresses: Iterable[str]) -> list[Candidate]: + """Bind one UDP socket per address; returns the host candidates. + + An address that cannot be bound is skipped. Raises ConnectionError + when the agent is closed before or while binding. + """ + if self._closed: + raise ConnectionError("ICE agent is closed") + if self._sockets: + return self.local_candidates + loop = asyncio.get_running_loop() + for index, address in enumerate(dict.fromkeys(addresses)): + family = socket.AF_INET6 if ":" in address else socket.AF_INET + try: + transport, protocol = await loop.create_datagram_endpoint( + lambda: _Socket(self), local_addr=(address, 0), family=family + ) + except OSError as err: + logger.debug("ICE cannot bind %s: %s", address, err) + continue + if self._closed: + protocol.close() + break + port = transport.get_extra_info("sockname")[1] + protocol.candidate = Candidate( + foundation=str(index + 1), + component=COMPONENT, + transport="udp", + priority=candidate_priority( + HOST_TYPE_PREFERENCE, max(0, 65535 - index) + ), + host=address, + port=port, + type="host", + ) + self._sockets.append(protocol) + logger.debug("ICE local candidate %s", protocol.candidate.to_sdp()) + if self._closed: + self._close_sockets() + raise ConnectionError("ICE agent is closed") + return self.local_candidates + + async def connect(self, remote_candidates: Iterable[Candidate]) -> None: + """Run the checks until a pair is selected. + + Raises ConnectionError when every check failed (controlling) or the + agent is closed; the caller bounds the time. + """ + if self._closed: + raise ConnectionError("ICE agent is closed") + if self.remote_username is None or self.remote_password is None: + raise ConnectionError("Remote ICE credentials are missing") + if self.remote_is_lite and not self.controlling: + # A lite peer is always controlled (RFC 8445 section 6.1.1). + self.controlling = True + for candidate in remote_candidates: + logger.debug( + "ICE remote candidate %s:%s typ %s", + candidate.host, + candidate.port, + candidate.type, + ) + for sock in self._sockets: + self._add_pair( + sock, (candidate.host, candidate.port), candidate.priority + ) + self._connecting = True + # Also covers pairs the remote checked before connect(). + self._spawn(self._start_checks()) + while self._selected is None: + # Cleared before looking, so a change meanwhile is not missed. + self._wake.clear() + if self._closed: + raise ConnectionError(f"ICE agent is closed: {self._close_reason}") + if self.controlling: + succeeded = [p for p in self._pairs if p.state is PairState.SUCCEEDED] + if succeeded: + best = max(succeeded, key=lambda p: p.priority(True)) + logger.debug("ICE nominating %s", best) + if await self._check(best, nominate=True) and self.controlling: + self._select(best) + continue + if self._pairs and all( + p.state is PairState.FAILED for p in self._pairs + ): + raise ConnectionError("ICE checks failed") + await self._wake.wait() + self._spawn(self._consent()) + + def _add_pair( + self, sock: _Socket, remote: tuple[str, int], remote_priority: int + ) -> CandidatePair | None: + remote = _address(remote) + if (":" in remote[0]) != (":" in sock.candidate.host): + return None + for pair in self._pairs: + if pair.local is sock and pair.remote == remote: + return pair + if len(self._pairs) >= MAX_PAIRS: + return None + pair = CandidatePair(sock, remote, remote_priority) + self._pairs.append(pair) + logger.debug("ICE pair %s", pair) + return pair + + async def _start_checks(self) -> None: + ordered = sorted( + self._pairs, key=lambda p: p.priority(self.controlling), reverse=True + ) + for pair in ordered: + if self._selected is not None or self._closed: + return + if pair.state is PairState.WAITING: + self._spawn(self._check(pair)) + await asyncio.sleep(TA) + + async def _check(self, pair: CandidatePair, nominate: bool = False) -> bool: + """Run one connectivity check; True when it succeeded.""" + if not nominate: + if pair.state is not PairState.WAITING: + return pair.state is PairState.SUCCEEDED + pair.state = PairState.IN_PROGRESS + response = None + for _ in range(2): + sent_controlling = self.controlling + response = await self._transact(pair, nominate=nominate) + if response is not None and response.error_code == 487: + # Take the other role than the request had, unless a request + # of the peer already switched it (RFC 8445 section 7.2.5.1). + if self.controlling == sent_controlling: + self._switch_role("487 role conflict response") + continue + break + ok = response is not None and response.message_class is Class.SUCCESS + if ok: + pair.state = PairState.SUCCEEDED + pair.last_response = time.monotonic() + logger.debug( + "ICE check %s succeeded%s", pair, " (nominated)" if nominate else "" + ) + if not self.controlling and pair.remote_nominated: + self._select(pair) + else: + pair.state = PairState.FAILED + logger.debug("ICE check %s failed", pair) + self._wake.set() + return ok + + def _request(self, nominate: bool) -> Message: + attributes: dict[int, bytes] = { + Attr.USERNAME: f"{self.remote_username}:{self.local_username}".encode(), + Attr.PRIORITY: candidate_priority(PRFLX_TYPE_PREFERENCE, 65535).to_bytes( + 4, "big" + ), + } + role = Attr.ICE_CONTROLLING if self.controlling else Attr.ICE_CONTROLLED + attributes[role] = self.tie_breaker.to_bytes(8, "big") + if nominate and self.controlling: + attributes[Attr.USE_CANDIDATE] = b"" + return Message(Method.BINDING, Class.REQUEST, attributes=attributes) + + async def _transact( + self, + pair: CandidatePair, + nominate: bool = False, + rtos: Sequence[float] | None = None, + ) -> Message | None: + """Send a request with retransmissions; the response or None.""" + assert self.remote_password is not None # noqa: S101 - set before checks + request = self._request(nominate) + data = stun.encode(request, self.remote_password.encode()) + future: asyncio.Future[Message] = asyncio.get_running_loop().create_future() + self._transactions[request.transaction_id] = (pair, future) + try: + for rto in CHECK_RTOS if rtos is None else rtos: + if self._closed: + return None + pair.local.sendto(data, pair.remote) + with contextlib.suppress(TimeoutError): + return await asyncio.wait_for(asyncio.shield(future), rto) + return None + finally: + self._transactions.pop(request.transaction_id, None) + future.cancel() + + def _select(self, pair: CandidatePair) -> None: + if self._selected is not None: + return + self._selected = pair + logger.debug( + "ICE selected %s (%s)", + pair, + "controlling" if self.controlling else "controlled", + ) + self._wake.set() + + def _switch_role(self, why: str) -> None: + self.controlling = not self.controlling + logger.debug( + "ICE role is now %s: %s", + "controlling" if self.controlling else "controlled", + why, + ) + self._wake.set() + + async def _consent(self) -> None: + """Keep consent fresh on the selected pair (RFC 7675).""" + pair = self._selected + assert pair is not None # noqa: S101 - started after selection + pair.last_response = max(pair.last_response, time.monotonic()) + while not self._closed: + await asyncio.sleep(self._consent_interval * random.uniform(0.8, 1.2)) # noqa: S311 + if self._closed: + return + self._spawn(self._consent_check(pair)) + silent = time.monotonic() - pair.last_response + if silent > self._consent_timeout: + logger.debug("ICE consent expired on %s after %.0f s", pair, silent) + await self.close("consent expired") + return + + async def _consent_check(self, pair: CandidatePair) -> None: + # One transmission; the next round is the retransmission. + response = await self._transact(pair, rtos=(self._consent_interval,)) + if response is not None and response.message_class is Class.SUCCESS: + pair.last_response = time.monotonic() + logger.debug("ICE consent refreshed on %s", pair) + + def _spawn(self, coro: Coroutine[Any, Any, object]) -> None: + task = asyncio.ensure_future(coro) + self._tasks.add(task) + task.add_done_callback(self._task_done) + + def _task_done(self, task: asyncio.Task[object]) -> None: + self._tasks.discard(task) + if not task.cancelled() and (error := task.exception()) is not None: + logger.debug("ICE task failed: %r", error) + + def _datagram_received( + self, sock: _Socket, data: bytes, addr: tuple[str, int] + ) -> None: + if self._closed or not data: + return + if stun.is_stun(data): + try: + message = stun.parse(data) + except stun.StunError as err: + logger.debug("ICE dropping STUN from %s: %s", addr, err) + return + if message.message_class is Class.REQUEST: + self._handle_request(sock, message, addr) + elif message.message_class in (Class.SUCCESS, Class.ERROR): + self._handle_response(sock, message, addr) + return + if not self._data_allowed(sock, addr): + return + limit = MAX_QUEUED if self._selected is not None else MAX_QUEUED_BEFORE_CONNECT + if self._queue.qsize() < limit: + self._queue.put_nowait(data) + + def _data_allowed(self, sock: _Socket, addr: tuple[str, int]) -> bool: + selected = self._selected + if selected is not None and selected.local is sock and selected.remote == addr: + return True + # The peer may send on another pair that passed its checks; before + # selection, any pair of the check list is a valid source. + return any( + p.local is sock + and p.remote == addr + and (selected is None or p.state is PairState.SUCCEEDED) + for p in self._pairs + ) + + def _handle_request( + self, sock: _Socket, request: Message, addr: tuple[str, int] + ) -> None: + username = request.username or "" + if ( + request.method is not Method.BINDING + or not username.startswith(f"{self.local_username}:") + or not stun.check_integrity(request, self.local_password.encode()) + ): + logger.debug("ICE dropping unauthenticated request from %s", addr) + return + theirs_controlling = request.tie_breaker(Attr.ICE_CONTROLLING) + theirs_controlled = request.tie_breaker(Attr.ICE_CONTROLLED) + if self.controlling and theirs_controlling is not None: + if self.tie_breaker >= theirs_controlling: + self._respond_error(sock, request, addr, 487, "Role Conflict") + return + self._switch_role("peer is controlling with a higher tie-breaker") + elif not self.controlling and theirs_controlled is not None: + if self.tie_breaker >= theirs_controlled: + self._switch_role("peer is controlled with a lower tie-breaker") + else: + self._respond_error(sock, request, addr, 487, "Role Conflict") + return + response = Message( + Method.BINDING, + Class.SUCCESS, + transaction_id=request.transaction_id, + attributes={ + Attr.XOR_MAPPED_ADDRESS: stun.xor_address( + addr[0], addr[1], request.transaction_id + ) + }, + ) + sock.sendto(stun.encode(response, self.local_password.encode()), addr) + pair = self._add_pair(sock, addr, request.priority or 0) + if pair is None: + return + if Attr.USE_CANDIDATE in request and not self.controlling: + if not pair.remote_nominated: + logger.debug("ICE peer nominated %s", pair) + pair.remote_nominated = True + if pair.state is PairState.SUCCEEDED: + self._select(pair) + return + # Triggered check (RFC 8445 section 7.3.1.4); needs the remote + # password, so before connect() it runs when connect() starts. + if pair.state is PairState.WAITING and self._connecting: + self._spawn(self._check(pair)) + elif pair.state is PairState.FAILED and self._connecting: + pair.state = PairState.WAITING + self._spawn(self._check(pair)) + + def _respond_error( + self, + sock: _Socket, + request: Message, + addr: tuple[str, int], + code: int, + reason: str, + ) -> None: + response = Message( + Method.BINDING, + Class.ERROR, + transaction_id=request.transaction_id, + attributes={Attr.ERROR_CODE: stun.error_value(code, reason)}, + ) + sock.sendto(stun.encode(response, self.local_password.encode()), addr) + + def _handle_response( + self, sock: _Socket, response: Message, addr: tuple[str, int] + ) -> None: + entry = self._transactions.get(response.transaction_id) + if entry is None: + return + pair, future = entry + password = self.remote_password + if ( + pair.local is not sock + or pair.remote != addr + or password is None + or not stun.check_integrity(response, password.encode()) + ): + logger.debug("ICE dropping unauthenticated response from %s", addr) + return + if not future.done(): + future.set_result(response) + + async def send(self, data: bytes) -> None: + """Send one datagram on the selected pair.""" + pair = self._selected + if pair is None or self._closed: + raise ConnectionError("ICE is not connected") + pair.local.sendto(data, pair.remote) + + async def recv(self) -> bytes: + """Receive one non-STUN datagram; ConnectionError once closed.""" + data = await self._queue.get() + if data is None: + self._queue.put_nowait(None) + raise ConnectionError(f"ICE closed: {self._close_reason}") + return data + + async def close(self, reason: str = "closed") -> None: + """Stop checks and consent and release the sockets.""" + if self._closed: + return + self._closed = True + self._close_reason = reason + self._wake.set() + current = asyncio.current_task() + tasks = [t for t in self._tasks if t is not current] + for task in tasks: + task.cancel() + for task in tasks: + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + self._close_sockets() + self._queue.put_nowait(None) + if self.on_closed is not None and reason != "closed": + try: + self.on_closed(reason) + except Exception: + logger.exception("Error in ICE close callback") + + def _close_sockets(self) -> None: + sockets, self._sockets = self._sockets, [] + for sock in sockets: + sock.close() + + +async def resolve_mdns(names: Iterable[str], timeout: float) -> dict[str, str | None]: + """Resolve mDNS ``.local`` host names to IP addresses, all at once. + + One zeroconf instance asks for every name in parallel; a name that does + not resolve within ``timeout`` seconds maps to None. + """ + from zeroconf import AddressResolver # noqa: PLC0415 - rarely needed + from zeroconf.asyncio import AsyncZeroconf # noqa: PLC0415 + + zeroconf = AsyncZeroconf() + + async def resolve(name: str) -> str | None: + resolver = AddressResolver(name if name.endswith(".") else f"{name}.") + if not await resolver.async_request(zeroconf.zeroconf, int(timeout * 1000)): + return None + addresses = resolver.parsed_scoped_addresses() + return addresses[0] if addresses else None + + try: + unique = list(dict.fromkeys(names)) + results = await asyncio.gather( + *(resolve(name) for name in unique), return_exceptions=True + ) + finally: + await zeroconf.async_close() + return { + name: result if isinstance(result, str) else None + for name, result in zip(unique, results, strict=True) + } diff --git a/pyhap/media/mpegts.py b/pyhap/media/mpegts.py new file mode 100644 index 00000000..9b73002c --- /dev/null +++ b/pyhap/media/mpegts.py @@ -0,0 +1,264 @@ +"""A minimal MPEG-TS muxer for H.265 access units and Opus packets. + +It feeds a recording transcoder: ffmpeg reads the stream from a pipe +(``-f mpegts -i pipe:0``) with the timestamps of the media source, so the +buffered seconds before a recording go through ffmpeg faster than real time. +A pipe also gives back-pressure and cannot drop data, unlike RTP over UDP. + +One program carries H.265 (stream type 0x24) and, optionally, Opus as +private data with the ``Opus`` registration descriptor and the channel +configuration extension descriptor (ETSI TS 102 366 annex, as ffmpeg muxes +it). Each Opus packet gets the Opus control header. Timestamps are 90 kHz +PTS; video PES packets carry no DTS, the camera streams have no B-frames. +""" + +from __future__ import annotations + +from collections.abc import Iterable + +PACKET_SIZE = 188 +PAYLOAD_SIZE = PACKET_SIZE - 4 +PAT_PID = 0x0000 +PMT_PID = 0x1000 +VIDEO_PID = 0x0100 +AUDIO_PID = 0x0101 +PROGRAM_NUMBER = 1 +STREAM_TYPE_HEVC = 0x24 +STREAM_TYPE_PRIVATE = 0x06 +STREAM_ID_VIDEO = 0xE0 +STREAM_ID_PRIVATE_1 = 0xBD +START_CODE = b"\x00\x00\x00\x01" +NAL_AUD = 35 +# NAL unit types below this are VCL NAL units. +NAL_FIRST_NON_VCL = 32 +# The payload of an HEVC access unit delimiter: pic_type 2 (any), stop bit. +AUD_PAYLOAD = 0x50 +# PCR runs this far behind the PTS of the video it is sent with. +PCR_DELAY = 9000 +PTS_MASK = (1 << 33) - 1 + + +def _aud(nals: list[bytes]) -> bytes: + """An access unit delimiter with the TemporalId of the unit's pictures.""" + tid_plus1 = next( + ( + nal[1] & 0x07 + for nal in nals + if len(nal) > 1 and (nal[0] >> 1) & 0x3F < NAL_FIRST_NON_VCL + ), + 1, + ) + return bytes([NAL_AUD << 1, tid_plus1 or 1, AUD_PAYLOAD]) + + +def _crc_table() -> list[int]: + table: list[int] = [] + for byte in range(256): + crc = byte << 24 + for _ in range(8): + crc = (crc << 1) ^ 0x04C11DB7 if crc & 0x80000000 else crc << 1 + table.append(crc & 0xFFFFFFFF) + return table + + +_CRC_TABLE: list[int] = _crc_table() + + +def crc32_mpeg2(data: bytes) -> int: + """The CRC-32/MPEG-2 of a PSI section.""" + crc = 0xFFFFFFFF + for byte in data: + entry: int = _CRC_TABLE[((crc >> 24) ^ byte) & 0xFF] + crc = ((crc << 8) & 0xFFFFFFFF) ^ entry + return crc + + +def encode_pts(pts: int, marker: int = 0x2) -> bytes: + """The 5-byte PTS field of a PES header.""" + pts &= PTS_MASK + return bytes( + [ + (marker << 4) | ((pts >> 29) & 0x0E) | 0x01, + (pts >> 22) & 0xFF, + ((pts >> 14) & 0xFE) | 0x01, + (pts >> 7) & 0xFF, + ((pts << 1) & 0xFE) | 0x01, + ] + ) + + +def decode_pts(field: bytes) -> int: + """The PTS of a 5-byte PES header field.""" + return ( + ((field[0] >> 1) & 0x07) << 30 + | field[1] << 22 + | (field[2] >> 1) << 15 + | field[3] << 7 + | field[4] >> 1 + ) + + +def opus_control_header(size: int) -> bytes: + """The Opus control header that precedes each packet in MPEG-TS.""" + return b"\x7f\xe0" + b"\xff" * (size // 255) + bytes([size % 255]) + + +class MpegTsMuxer: + """Turns access units and Opus packets into 188-byte TS packets.""" + + def __init__(self, audio: bool = True, channels: int = 1) -> None: + """Configure the program; :meth:`video` sends the tables first.""" + self.audio = audio + self.channels = channels + self._continuity: dict[int, int] = {} + + def tables(self) -> bytes: + """The PAT and PMT; sent before every IRAP picture.""" + return self._section(PAT_PID, self._pat()) + self._section(PMT_PID, self._pmt()) + + def video(self, nals: Iterable[bytes], pts: int, keyframe: bool) -> bytes: + """One access unit as a PES packet in Annex B, with tables at IRAPs. + + Starts with an access unit delimiter, which HEVC in MPEG-TS requires + (ISO/IEC 13818-1 2.17): the unit's own one moved to the front, else a + new one. + """ + nals = list(nals) + auds = [nal for nal in nals if (nal[0] >> 1) & 0x3F == NAL_AUD] + if auds: + nals = [auds[0], *(nal for nal in nals if nal not in auds)] + else: + nals.insert(0, _aud(nals)) + payload = b"".join(START_CODE + nal for nal in nals) + pes = self._pes(STREAM_ID_VIDEO, payload, pts, bounded=False, aligned=True) + pcr = max(pts - PCR_DELAY, 0) + packets = self._packets(VIDEO_PID, pes, pcr=pcr, random_access=keyframe) + return self.tables() + packets if keyframe else packets + + def opus(self, packet: bytes, pts: int) -> bytes: + """One Opus packet as a PES packet; empty without audio.""" + if not self.audio: + return b"" + payload = opus_control_header(len(packet)) + packet + pes = self._pes(STREAM_ID_PRIVATE_1, payload, pts, bounded=True) + return self._packets(AUDIO_PID, pes) + + def _pat(self) -> bytes: + body = PROGRAM_NUMBER.to_bytes(2, "big") + (0xE000 | PMT_PID).to_bytes(2, "big") + return self._psi(0x00, 1, body) + + def _pmt(self) -> bytes: + streams = bytes([STREAM_TYPE_HEVC]) + self._es(VIDEO_PID, b"") + if self.audio: + descriptors = ( + b"\x05\x04Opus" # registration descriptor + + bytes([0x7F, 0x02, 0x80, self.channels]) # Opus channel config + ) + streams += bytes([STREAM_TYPE_PRIVATE]) + self._es(AUDIO_PID, descriptors) + body = (0xE000 | VIDEO_PID).to_bytes(2, "big") + b"\xf0\x00" + streams + return self._psi(0x02, PROGRAM_NUMBER, body) + + @staticmethod + def _es(pid: int, descriptors: bytes) -> bytes: + return ( + (0xE000 | pid).to_bytes(2, "big") + + (0xF000 | len(descriptors)).to_bytes(2, "big") + + descriptors + ) + + @staticmethod + def _psi(table_id: int, table_id_extension: int, body: bytes) -> bytes: + # Section syntax, version 0, current, section 0 of 0, then the CRC. + length = 5 + len(body) + 4 + section = ( + bytes([table_id]) + + (0xB000 | length).to_bytes(2, "big") + + table_id_extension.to_bytes(2, "big") + + b"\xc1\x00\x00" + + body + ) + return section + crc32_mpeg2(section).to_bytes(4, "big") + + def _section(self, pid: int, section: bytes) -> bytes: + # A pointer field, then the section; one packet holds our tables. + payload = b"\x00" + section + payload += b"\xff" * (PAYLOAD_SIZE - len(payload)) + return self._header(pid, start=True, adaptation=False) + payload + + @staticmethod + def _pes( + stream_id: int, + payload: bytes, + pts: int, + *, + bounded: bool, + aligned: bool = False, + ) -> bytes: + # '10', no scrambling or priority, data_alignment_indicator, PTS only. + flags = b"\x84" if aligned else b"\x80" + header = flags + b"\x80\x05" + encode_pts(pts) + length = len(header) + len(payload) + # Video may exceed 16 bits, so its length is left unbounded (0). + size = length if bounded and length <= 0xFFFF else 0 + return ( + b"\x00\x00\x01" + + bytes([stream_id]) + + size.to_bytes(2, "big") + + (header + payload) + ) + + def _packets( + self, + pid: int, + pes: bytes, + *, + pcr: int | None = None, + random_access: bool = False, + ) -> bytes: + out = bytearray() + offset = 0 + first = True + while offset < len(pes) or first: + adaptation = b"" + if first and (pcr is not None or random_access): + flags = (0x40 if random_access else 0) | ( + 0x10 if pcr is not None else 0 + ) + field = bytes([flags]) + if pcr is not None: + field += self._pcr(pcr) + adaptation = field + room = PAYLOAD_SIZE - (len(adaptation) + 1 if adaptation else 0) + chunk = pes[offset : offset + room] + offset += len(chunk) + if len(chunk) < room: + # Stuff the last packet through its adaptation field. + if adaptation: + adaptation += b"\xff" * (room - len(chunk)) + else: + # The length byte alone fills one byte, a flags byte and + # 0xFF stuffing the rest. + stuffing = PAYLOAD_SIZE - len(chunk) - 1 + adaptation = b"\x00" + b"\xff" * (stuffing - 1) if stuffing else b"" + out += self._header(pid, start=first, adaptation=True) + out += bytes([len(adaptation)]) + adaptation + chunk + elif adaptation: + out += self._header(pid, start=first, adaptation=True) + out += bytes([len(adaptation)]) + adaptation + chunk + else: + out += self._header(pid, start=first, adaptation=False) + chunk + first = False + return bytes(out) + + @staticmethod + def _pcr(pcr: int) -> bytes: + base = pcr & PTS_MASK + return (base << 15 | 0x3F << 9).to_bytes(6, "big") + + def _header(self, pid: int, *, start: bool, adaptation: bool) -> bytes: + counter = self._continuity.get(pid, -1) + 1 & 0x0F + self._continuity[pid] = counter + control = (0x30 if adaptation else 0x10) | counter + return bytes( + [0x47, (0x40 if start else 0) | (pid >> 8) & 0x1F, pid & 0xFF, control] + ) diff --git a/pyhap/media/multi_tier.py b/pyhap/media/multi_tier.py new file mode 100644 index 00000000..e7c79e91 --- /dev/null +++ b/pyhap/media/multi_tier.py @@ -0,0 +1,490 @@ +"""SRTP media of HomeKit multi-tier RTP sessions from a shared media source. + +A :class:`MultiTierSender` sends one camera stream to one controller on the +local network, as the Camera Multi-Tier RTP Stream Management service of a +HomeKit Secure Video 17.99 camera negotiates it. It reads the same +:class:`~pyhap.media.source.MediaSource` as the camera's other sessions, so a +camera channel is pulled once for all local and remote sessions; see +:class:`~pyhap.media.pool.SourcePool`. + +Video goes out as RFC 7798 H.265 (single NAL units, aggregation and +fragmentation units, no DONL) with the 90 kHz clock, the parameter sets +in-band before each IRAP picture as the source delivers them. Audio is the +source's Opus with the 48 kHz clock. Each packet gets the session's SSRC and +payload type and is protected with SRTP (AES_CM_128_HMAC_SHA1_80) with the +key of its stream; RTCP sender reports go out once a second as SRTCP. + +Session flow, mapped to the HAP operations: + +* Setup Endpoints: create the sender and :meth:`MultiTierSender.bind` it. + It binds the accessory's video and audio ports, which the answer + announces, and sends from them. +* Start: :meth:`MultiTierSender.start` with the source of the tier's + channel. Video starts with the next IRAP picture; ``on_keyframe_request`` + asks the channel's pull for one. +* Start again with another tier: :meth:`MultiTierSender.switch` to the + other channel's source. SSRCs, sequence numbers, RTP time and the SRTP + packet index continue. +* End: :meth:`MultiTierSender.close`. + +The controller sends its RTCP to the accessory's ports. Each datagram from +the controller's address that decrypts with the session key and does not +carry the accessory's own SSRC calls ``on_rtcp`` (at most once per +``rtcp_notify_interval``), which keeps the HAP session alive; a picture loss +indication or full intra request among them asks for a keyframe. The sender +ends on its own, calling ``on_closed(reason)``, when no video arrives for +``idle_timeout``, SRTP fails or the source is closed. A session starts, and +continues after a switch, with an IRAP picture that carries its parameter +sets. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable, Coroutine +import contextlib +import logging +import time +from typing import Any, cast + +import pylibsrtp + +from . import MediaError, rtp +from .hevc import AccessUnit, HevcPacketizer +from .sframe import DEFAULT_MAX_PAYLOAD +from .source import MediaSource + +logger = logging.getLogger(__name__) + +DEFAULT_IDLE_TIMEOUT = 10.0 +RTCP_INTERVAL = 1.0 +RTCP_NOTIFY_INTERVAL = 1.0 +# Length of an AES_CM_128_HMAC_SHA1_80 master key and salt together. +SRTP_KEY_LENGTH = 30 + + +class _Port(asyncio.DatagramProtocol): + """One of the accessory's ports: sends media, receives the controller's RTCP.""" + + def __init__(self, on_datagram: Callable[[bytes, tuple[Any, ...]], None]) -> None: + self.transport: asyncio.DatagramTransport | None = None + self._on_datagram = on_datagram + + def connection_made(self, transport: asyncio.BaseTransport) -> None: + self.transport = cast(asyncio.DatagramTransport, transport) + + def datagram_received(self, data: bytes, addr: tuple[Any, ...]) -> None: + try: + self._on_datagram(data, addr) + except Exception: + logger.exception("Error handling a multi-tier RTCP packet") + + def error_received(self, exc: Exception) -> None: + # asyncio reports send errors here, e.g. ICMP unreachable; the idle + # timeout ends a session whose controller is gone. + logger.debug("Multi-tier port error: %s", exc) + + @property + def port(self) -> int: + if self.transport is None: + return 0 + return int(self.transport.get_extra_info("sockname")[1]) + + +class _SrtpStream: + """An outbound RTP stream and its SRTP context.""" + + def __init__(self, ssrc: int, payload_type: int, clock: int, key: bytes) -> None: + if len(key) != SRTP_KEY_LENGTH: + raise ValueError("SRTP key and salt must be 30 bytes") + self.rtp = rtp.OutboundStream(payload_type, clock) + self.rtp.ssrc = ssrc + self._outbound = pylibsrtp.Session( + pylibsrtp.Policy(key=key, ssrc_type=pylibsrtp.Policy.SSRC_ANY_OUTBOUND) + ) + self._inbound = pylibsrtp.Session( + pylibsrtp.Policy(key=key, ssrc_type=pylibsrtp.Policy.SSRC_ANY_INBOUND) + ) + + def packet(self, timestamp: int, marker: bool, payload: bytes) -> bytes: + packet = self.rtp.build(timestamp, marker, payload) + # Nothing is retransmitted, no history is kept. + self.rtp.history.clear() + return self._outbound.protect(packet) + + def report(self, cname: str) -> bytes | None: + report = self.rtp.sender_report() + if report is None: + return None + return self._outbound.protect_rtcp( + report + rtp.build_sdes(self.rtp.ssrc, cname) + ) + + def bye(self, cname: str) -> bytes: + # RFC 3550 6.1: a compound packet starts with a report and has a CNAME; + # a source that has sent nothing reports as a receiver. + report = self.rtp.sender_report() or rtp.build_receiver_report(self.rtp.ssrc) + return self._outbound.protect_rtcp( + report + + rtp.build_sdes(self.rtp.ssrc, cname) + + rtp.build_bye([self.rtp.ssrc]) + ) + + def unprotect_rtcp(self, data: bytes) -> bytes | None: + """The controller's RTCP, None unless it is SRTCP from another SSRC. + + Both directions share the key, so the accessory's own packets sent + back would decrypt too; they carry the accessory's SSRC. + """ + try: + rtcp = self._inbound.unprotect_rtcp(data) + except pylibsrtp.Error: + return None + if len(rtcp) < 8 or int.from_bytes(rtcp[4:8], "big") == self.rtp.ssrc: + return None + return rtcp + + def authentic(self, data: bytes) -> bool: + """Whether ``data`` is the controller's SRTCP or SRTP under the key.""" + if self.unprotect_rtcp(data) is not None: + return True + try: + packet = self._inbound.unprotect(data) + except pylibsrtp.Error: + return False + return len(packet) >= 12 and int.from_bytes(packet[8:12], "big") != ( + self.rtp.ssrc + ) + + +class MultiTierSender: + """Sends one camera stream to one controller as SRTP, see the module docs. + + Callbacks, all optional and called on the event loop: + + * ``on_rtcp()``: the controller is still there; keep its session alive. + * ``on_keyframe_request(age, gop)``: the session needs an IRAP picture; + ``age`` is the time since the source last started one and ``gop`` the + interval between them the source measured (each None until known). + * ``on_closed(reason)``: the sender ended on its own. Not called for + :meth:`close`. + """ + + def __init__( + self, + *, + address: str, + video_port: int, + audio_port: int, + video_key: bytes, + audio_key: bytes, + video_ssrc: int, + audio_ssrc: int, + video_payload_type: int = rtp.DEFAULT_VIDEO_PAYLOAD_TYPE, + audio_payload_type: int = rtp.DEFAULT_AUDIO_PAYLOAD_TYPE, + local_video_port: int = 0, + local_audio_port: int = 0, + is_ipv6: bool | None = None, + max_payload: int = DEFAULT_MAX_PAYLOAD, + idle_timeout: float = DEFAULT_IDLE_TIMEOUT, + keyframe_request_interval: float = rtp.KEYFRAME_REQUEST_INTERVAL, + rtcp_notify_interval: float = RTCP_NOTIFY_INTERVAL, + ) -> None: + """Configure the session; :meth:`bind` opens the ports. + + ``address`` and the ports are where the controller receives. The + keys are the SRTP master key and salt (30 bytes) of each stream, as + Setup Endpoints carries them. Local ports of 0 bind free ones. + """ + self.address = address.split("%", 1)[0] + self.is_ipv6 = ":" in self.address if is_ipv6 is None else is_ipv6 + self.on_rtcp: Callable[[], None] | None = None + self.on_keyframe_request: ( + Callable[[float | None, float | None], None] | None + ) = None + self.on_closed: Callable[[str], None] | None = None + self._destination = (video_port, audio_port) + self._local = (local_video_port, local_audio_port) + self._video = _SrtpStream( + video_ssrc, video_payload_type, rtp.VIDEO_CLOCK_RATE, video_key + ) + self._audio = _SrtpStream( + audio_ssrc, audio_payload_type, rtp.AUDIO_CLOCK_RATE, audio_key + ) + self._cname = f"pyhap-{video_ssrc:08x}" + self._packetizer = HevcPacketizer(max_payload) + self._closing: asyncio.Task[None] | None = None + self._idle_timeout = idle_timeout + self._keyframe_request_interval = keyframe_request_interval + self._rtcp_notify_interval = rtcp_notify_interval + self._video_port: _Port | None = None + self._audio_port: _Port | None = None + self._source: MediaSource | None = None + self._send_audio = False + self._waiting_for_keyframe = True + self._last_keyframe_request = -keyframe_request_interval + self._last_rtcp_notify = -rtcp_notify_interval + self._last_audio_input = 0.0 + self._started_at = 0.0 + self._tasks: set[asyncio.Task[None]] = set() + self._started = False + self._closed = False + + @property + def ports(self) -> tuple[int, int]: + """The accessory's ``(video_port, audio_port)`` once bound.""" + if self._video_port is None or self._audio_port is None: + return self._local + return self._video_port.port, self._audio_port.port + + @property + def source(self) -> MediaSource | None: + """The source the sender reads, once started.""" + return self._source + + @property + def started(self) -> bool: + """Whether media flows to the controller.""" + return self._started and not self._closed + + @property + def closed(self) -> bool: + """Whether the session ended.""" + return self._closed + + @property + def wants_media(self) -> bool: + """Whether the source should reassemble media for this session.""" + return self.started + + async def bind(self) -> tuple[int, int]: + """Bind the accessory's ports on every interface and return them. + + :raises OSError: If a port cannot be bound. + """ + if self._closed: + raise MediaError("Multi-tier session closed") + if self._video_port is not None: + return self.ports + loop = asyncio.get_running_loop() + # Any interface: the controller reaches the accessory on its LAN + # address; only the controller's datagrams are read. + any_address = "::" if self.is_ipv6 else "0.0.0.0" # noqa: S104 + try: + _, self._video_port = await loop.create_datagram_endpoint( + lambda: _Port(self._on_video_port), + local_addr=(any_address, self._local[0]), + ) + if not self._closed: + _, self._audio_port = await loop.create_datagram_endpoint( + lambda: _Port(self._on_audio_port), + local_addr=(any_address, self._local[1]), + ) + except BaseException: + self._close_ports() + raise + if self._closed: + # Closed while binding: _close found no ports to release. + self._close_ports() + raise MediaError("Multi-tier session closed") + return self.ports + + async def start(self, source: MediaSource, audio: bool = True) -> None: + """Send ``source`` to the controller, with its audio if ``audio``.""" + if self._closed: + raise MediaError("Multi-tier session closed") + if self._video_port is None: + await self.bind() + if self._started: + raise MediaError("Multi-tier session already started") + self._send_audio = audio + self._attach(source) + self._started = True + self._started_at = time.monotonic() + self._spawn(self._rtcp_loop()) + self._request_keyframe() + + async def switch(self, source: MediaSource) -> None: + """Continue the stream from another source, e.g. for another tier.""" + if not self.started: + raise MediaError("Multi-tier session not started") + if source is self._source: + return + self._attach(source) + self._started_at = time.monotonic() + # The new source needs its own request, however recent the last one. + self._last_keyframe_request = -self._keyframe_request_interval + self._request_keyframe() + + def _attach(self, source: MediaSource) -> None: + if source.closed: + raise MediaError("Media source is closed") + if self._source is not None: + self._source.detach(self) + self._source = source + self._waiting_for_keyframe = True + # Another source has its own timestamps; the stream goes on smoothly. + self._video.rtp.rebase() + self._audio.rtp.rebase() + source.attach(self) + + async def close(self) -> None: + """Stop sending and release the ports, without calling ``on_closed``.""" + if self._closing is not None: + # Ending on its own already; wait until the ports are released. + await asyncio.shield(self._closing) + return + await self._close(None) + + async def _close(self, reason: str | None) -> None: + if self._closed: + return + self._closed = True + if reason is not None: + logger.debug("Multi-tier session closed: %s", reason) + if self._started: + self._send(self._video_port, self._video.bye(self._cname), 0) + if self._send_audio: + self._send(self._audio_port, self._audio.bye(self._cname), 1) + if self._source is not None: + self._source.detach(self) + current = asyncio.current_task() + tasks = [task for task in self._tasks if task is not current] + for task in tasks: + task.cancel() + try: + await asyncio.gather(*tasks, return_exceptions=True) + finally: + self._close_ports() + if reason is not None and self.on_closed: + try: + self.on_closed(reason) + except Exception: + logger.exception("Error in multi-tier close callback") + + def _close_ports(self) -> None: + for port in (self._video_port, self._audio_port): + if port is not None and port.transport is not None: + port.transport.close() + + def _close_soon(self, reason: str) -> None: + if not self._closed and self._closing is None: + self._closing = asyncio.get_running_loop().create_task(self._close(reason)) + + def _spawn(self, coro: Coroutine[Any, Any, None]) -> None: + task = asyncio.get_running_loop().create_task(coro) + self._tasks.add(task) + task.add_done_callback(self._tasks.discard) + + def _notify(self, callback: Callable[..., None] | None, *args: Any) -> None: + if callback is None: + return + try: + callback(*args) + except Exception: + logger.exception("Error in multi-tier callback") + + def _send(self, port: _Port | None, packet: bytes, index: int) -> None: + if port is None or port.transport is None or port.transport.is_closing(): + return + port.transport.sendto(packet, (self.address, self._destination[index])) + + def on_source_closed(self, reason: str) -> None: + """The media source ended: end the session.""" + self._close_soon(f"media source ended: {reason}") + + def on_source_video(self, unit: AccessUnit, ssrc: int) -> None: + """Send one access unit, starting with an IRAP picture.""" + if not self.started: + return + if self._waiting_for_keyframe and not unit.self_contained: + # Start where a decoder can: an IRAP picture with its parameters. + return + self._waiting_for_keyframe = False + timestamp = self._video.rtp.timestamp(ssrc, unit.timestamp) + payloads = self._packetizer.packetize(unit.nals) + last = len(payloads) - 1 + try: + for index, payload in enumerate(payloads): + packet = self._video.packet(timestamp, index == last, payload) + self._send(self._video_port, packet, 0) + except pylibsrtp.Error as err: + self._close_soon(f"send failed: {err!r}") + + def on_source_audio(self, packet: rtp.RtpPacket) -> None: + """Send one Opus packet.""" + now = time.monotonic() + marker = now - self._last_audio_input > rtp.AUDIO_TALKSPURT_GAP + self._last_audio_input = now + if not self.started or not self._send_audio: + return + timestamp = self._audio.rtp.timestamp(packet.ssrc, packet.timestamp) + try: + data = self._audio.packet(timestamp, marker, packet.payload) + self._send(self._audio_port, data, 1) + except pylibsrtp.Error as err: + self._close_soon(f"send failed: {err!r}") + + def _from_controller(self, addr: tuple[Any, ...]) -> bool: + # The controller's address has no scope ID, a received one may. + return str(addr[0]).split("%", 1)[0] == self.address + + def _on_video_port(self, data: bytes, addr: tuple[Any, ...]) -> None: + if not self._from_controller(addr): + return + rtcp = self._video.unprotect_rtcp(data) + if rtcp is None: + # A spoofed source address alone must not keep the session alive. + return + self._controller_alive() + for message in rtp.parse_rtcp(rtcp): + if ( + message.is_keyframe_request + and self._video.rtp.ssrc in message.media_ssrcs + ): + self._request_keyframe() + + def _on_audio_port(self, data: bytes, addr: tuple[Any, ...]) -> None: + if self._from_controller(addr) and self._audio.authentic(data): + self._controller_alive() + + def _controller_alive(self) -> None: + now = time.monotonic() + if now - self._last_rtcp_notify < self._rtcp_notify_interval: + return + self._last_rtcp_notify = now + self._notify(self.on_rtcp) + + def _request_keyframe(self) -> None: + now = time.monotonic() + if now - self._last_keyframe_request < self._keyframe_request_interval: + return + self._last_keyframe_request = now + source = self._source + age = source.keyframe_age() if source else None + self._notify(self.on_keyframe_request, age, source.gop if source else None) + + async def _rtcp_loop(self) -> None: + cname = self._cname + try: + while True: + await asyncio.sleep(RTCP_INTERVAL) + source = self._source + last_input = max( + self._started_at, source.last_video_input if source else 0.0 + ) + if time.monotonic() - last_input > self._idle_timeout: + await self._close("media input idle") + return + if self._waiting_for_keyframe: + self._request_keyframe() + if report := self._video.report(cname): + self._send(self._video_port, report, 0) + if self._send_audio and (report := self._audio.report(cname)): + self._send(self._audio_port, report, 1) + except asyncio.CancelledError: + raise + except Exception as err: # noqa: BLE001 - a broken path ends the session + logger.debug("Multi-tier RTCP send failed", exc_info=True) + with contextlib.suppress(Exception): + await self._close(f"rtcp send failed: {err!r}") diff --git a/pyhap/media/multi_tier_delegate.py b/pyhap/media/multi_tier_delegate.py new file mode 100644 index 00000000..51fdacdf --- /dev/null +++ b/pyhap/media/multi_tier_delegate.py @@ -0,0 +1,277 @@ +"""Run the media of multi-tier RTP sessions from shared media sources. + +:class:`MultiTierMediaDelegate` is a +:class:`pyhap.camera_multi_tier.MultiTierDelegate` with one +:class:`~pyhap.media.multi_tier.MultiTierSender` per session. It leases +sources from the same provider and pool as +:class:`~pyhap.media.webrtc_delegate.WebRTCMediaDelegate`, so a camera channel is +pulled once for all local and remote sessions:: + + worker = MediaWorker() + provider = FFmpegSourceProvider({"high": [...], "medium": [...]}, worker=worker) + camera = Camera( + { + ..., + "multi_tier": {"delegate": MultiTierMediaDelegate(provider)}, + "webrtc": {"delegate": WebRTCMediaDelegate(provider)}, + }, + driver, + "Camera", + ) + +Setup Endpoints binds the session's ports in the pool's worker (or in +process) and announces them; the start leases the source of the tier's +channel, starts the sender on it and pulls the channel if no other session +does. A start with another tier moves the session to that tier's channel; +SSRCs, sequence numbers and SRTP state continue. The controller's RTCP keeps +the session alive (:meth:`~pyhap.camera_multi_tier.MultiTierSession.keep_alive`); +a sender that ends on its own ends its HAP session. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass +from functools import partial +import logging +from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable + +from pyhap import hksv3 +from pyhap.camera_stream import StreamError + +from .pool import MultiTierMediaSender, SourceLease, SourcePool + +if TYPE_CHECKING: + from pyhap.camera_multi_tier import MultiTierSession + +logger = logging.getLogger(__name__) + +_SETUP_ERROR = hksv3.SetupEndpointsStatus.ERROR +_ERROR = hksv3.RTPStreamingStatus.ERROR + +MultiTierSourceProvider = Callable[ + ["MultiTierSession", hksv3.VideoStreamTier, "hksv3.AudioStreamTier | None"], + Awaitable[SourceLease], +] + + +@runtime_checkable +class PooledSourceProvider(Protocol): + """A source provider that names its pool, e.g. an + :class:`~pyhap.media.ffmpeg.FFmpegSourceProvider`. + """ + + pool: SourcePool + + +# MultiTierSender options a delegate user may set; the session sets the rest. +_SENDER_OPTIONS = frozenset( + { + "local_video_port", + "local_audio_port", + "max_payload", + "idle_timeout", + "keyframe_request_interval", + "rtcp_notify_interval", + } +) + + +@dataclass(eq=False) +class _Media: + """What the delegate holds for one session.""" + + session: MultiTierSession + sender: MultiTierMediaSender + lease: SourceLease | None = None + finished: bool = False + + +class MultiTierMediaDelegate: + """A ``MultiTierDelegate`` that streams from a :class:`SourcePool`. + + ``source_provider`` leases the source of a tier's channel, for example + an :class:`~pyhap.media.ffmpeg.FFmpegSourceProvider`; ``pool`` is the + pool its leases come from and defaults to the provider's ``pool``. + ``sender_options`` go to every + :class:`~pyhap.media.multi_tier.MultiTierSender`, e.g. ``idle_timeout`` + and ``max_payload``. + + A media worker should be started before the first session: starting it + takes longer than a HAP hook may. + """ + + def __init__( + self, + source_provider: MultiTierSourceProvider, + pool: SourcePool | None = None, + *, + sender_options: Mapping[str, Any] | None = None, + ) -> None: + """Configure the delegate.""" + if pool is None and isinstance(source_provider, PooledSourceProvider): + pool = source_provider.pool + if not isinstance(pool, SourcePool): + raise TypeError("MultiTierMediaDelegate needs the provider's SourcePool") + self.source_provider = source_provider + self.pool = pool + self.sender_options = dict(sender_options or {}) + if unknown := sorted(set(self.sender_options) - _SENDER_OPTIONS): + raise TypeError(f"Unknown sender_options: {', '.join(unknown)}") + self._media: dict[MultiTierSession, _Media] = {} + + @property + def session_count(self) -> int: + """Sessions the delegate holds media for.""" + return len(self._media) + + def sender(self, session: MultiTierSession) -> MultiTierMediaSender | None: + """The sender of a session, None if there is none.""" + media = self._media.get(session) + return None if media is None else media.sender + + async def handle_prepare(self, session: MultiTierSession) -> None: + """Bind the session's ports and announce them.""" + sender = self.pool.create_multi_tier_sender( + **self.sender_options, + address=session.address, + is_ipv6=session.is_ipv6, + video_port=session.video_port, + audio_port=session.audio_port, + video_key=session.video_srtp_key + session.video_srtp_salt, + audio_key=session.audio_srtp_key + session.audio_srtp_salt, + video_ssrc=session.video_ssrc, + audio_ssrc=session.audio_ssrc, + video_payload_type=hksv3.VIDEO_PAYLOAD_TYPE, + audio_payload_type=hksv3.AUDIO_PAYLOAD_TYPE, + ) + media = self._media[session] = _Media(session, sender) + sender.on_rtcp = session.keep_alive + sender.on_closed = partial(self._sender_closed, media) + sender.on_keyframe_request = partial(self._keyframe_requested, media) + try: + ports = await sender.bind() + except Exception as err: + raise StreamError(_SETUP_ERROR, f"Ports not bound: {err}") from err + session.accessory_video_port, session.accessory_audio_port = ports + + def _get(self, session: MultiTierSession) -> _Media: + media = self._media.get(session) + if media is None or media.finished: + raise StreamError(_ERROR, "No multi-tier media for the session") + return media + + async def _lease( + self, + session: MultiTierSession, + video_tier: hksv3.VideoStreamTier, + audio_tier: hksv3.AudioStreamTier | None, + ) -> SourceLease: + try: + return await self.source_provider(session, video_tier, audio_tier) + except Exception as err: + # A provider's status may be one of another service, e.g. the + # WebRTC offer's; a start answers with its own error. + logger.warning("[%s] No multi-tier media source: %s", session.id, err) + raise StreamError(_ERROR, f"No media source: {err}") from err + + async def handle_start( + self, + session: MultiTierSession, + video_tier: hksv3.VideoStreamTier, + audio_tier: hksv3.AudioStreamTier | None, + ) -> None: + """Lease the tier's source, pull it and start the sender on it.""" + media = self._get(session) + if (old := media.lease) is not None: + # A repeated start after a failed one: give up the earlier claim. + media.lease = None + await old.release() + lease = await self._lease(session, video_tier, audio_tier) + media.lease = lease + try: + # Pull first: a failed pull leaves the sender unstarted, so a + # repeated start can still start it; its keyframe request also + # reaches the pull. + await lease.activate() + await media.sender.start(lease.source, audio=audio_tier is not None) + except BaseException as err: + media.lease = None + await lease.release() + if not isinstance(err, Exception): + raise + raise StreamError(_ERROR, f"Stream not started: {err}") from err + logger.debug( + "[%s] Multi-tier stream of %dx%d@%d started", + session.id, + video_tier.width, + video_tier.height, + video_tier.frame_rate, + ) + + async def handle_reconfigure( + self, + session: MultiTierSession, + video_tier: hksv3.VideoStreamTier, + audio_tier: hksv3.AudioStreamTier | None, + ) -> None: + """Move the session to the source of the new tier's channel. + + Audio stays as the start set it. + """ + media = self._get(session) + old = media.lease + lease = await self._lease(session, video_tier, audio_tier) + if old is not None and lease.source is old.source: + await lease.release() + logger.debug("[%s] Tier %s uses the same source", session.id, video_tier) + return + # The switch asks the new source for a keyframe at once, through the + # session's lease: it must already be the new one. + media.lease = lease + try: + await lease.activate() + await media.sender.switch(lease.source) + except BaseException as err: + # Also when the hook is cancelled, e.g. on a timeout. + media.lease = old + await lease.release() + if not isinstance(err, Exception): + raise + raise StreamError(_ERROR, f"Tier not changed: {err}") from err + if old is not None: + await old.release() + logger.debug("[%s] Multi-tier stream moved to %s", session.id, video_tier) + + async def handle_end(self, session: MultiTierSession) -> None: + """Close the sender and release the lease.""" + media = self._media.pop(session, None) + if media is not None: + await self._finish(media) + + async def _finish(self, media: _Media) -> None: + if media.finished: + return + media.finished = True + try: + await media.sender.close() + finally: + if media.lease is not None: + await media.lease.release() + logger.debug("[%s] Multi-tier media released", media.session.id) + + def _keyframe_requested( + self, media: _Media, age: float | None, gop: float | None + ) -> None: + if media.lease is not None: + media.lease.keyframe_requested(age, gop) + + def _sender_closed(self, media: _Media, reason: str) -> None: + logger.info("[%s] Multi-tier stream ended: %s", media.session.id, reason) + media.session.end(f"multi-tier media ended: {reason}") + + async def close(self) -> None: + """Release every session left.""" + media, self._media = list(self._media.values()), {} + await asyncio.gather(*(self._finish(m) for m in media)) diff --git a/pyhap/media/pool.py b/pyhap/media/pool.py new file mode 100644 index 00000000..e2955fbf --- /dev/null +++ b/pyhap/media/pool.py @@ -0,0 +1,448 @@ +"""Media sources shared by the sessions of a camera channel. + +:class:`SourcePool` gives each WebRTC, multi-tier or recording session a +:class:`ChannelLease` on the media source of its camera channel. All +sessions of a channel share one source and one media pull +(:class:`ChannelPull`, for example an ffmpeg process), so N sessions cost one +camera connection. Sources live in a +:class:`~pyhap.media.worker.MediaWorker`, or in process without one; +a lease creates its session's sender in the same place. + +A WebRTC sender is created by its lease (:meth:`ChannelLease.create_webrtc_sender`); +a multi-tier sender binds its ports before the controller picks a tier, so +it is created by the pool (:meth:`SourcePool.create_multi_tier_sender`) and +started with a lease's source later. + +The ports of a source are bound by the first lease; the pull starts with the +first answered session (:meth:`ChannelLease.activate`) and stops with the +last one; the source is closed with the last lease. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable, Coroutine, Hashable, Iterable +from dataclasses import dataclass, field +from functools import partial +import inspect +import logging +from typing import TYPE_CHECKING, Any, Protocol, cast + +from . import MediaError + +if TYPE_CHECKING: + from .worker import MediaWorker + +logger = logging.getLogger(__name__) + + +class MediaSourceLike(Protocol): + """A :class:`~pyhap.media.source.MediaSource` or its worker proxy.""" + + on_closed: Callable[[str], None] | None + + @property + def closed(self) -> bool: + """Whether the source has ended.""" + + @property + def rtp_input(self) -> tuple[str, int, int]: + """``(host, video_port, audio_port)`` the pull sends to.""" + + @property + def has_audio(self) -> bool: + """Whether the source takes audio, i.e. has an audio input port.""" + + def close(self, reason: str = "closed") -> Awaitable[None] | None: + """Release the ports and end every session on the source.""" + + +class WebRTCMediaSender(Protocol): + """A :class:`~pyhap.media.sender.WebRTCSender` or its worker proxy.""" + + on_closed: Callable[[str], None] | None + on_connected: Callable[[], None] | None + on_keyframe_request: Callable[[float | None, float | None], None] | None + on_audio: Callable[[bytes, int], None] | None + + async def create_offer(self) -> str: + """Bind the transport and return the offer SDP.""" + + async def apply_answer( + self, answer_sdp: str, candidates: Iterable[Any] = () + ) -> None: + """Accept the controller's answer.""" + + async def reoffer(self, offer_sdp: str) -> str: + """Answer a controller re-offer.""" + + async def update_receive_keys( + self, add: Iterable[Any] = (), remove: Iterable[int] = () + ) -> None: + """Install and remove the controller's SFrame keys.""" + + async def close(self) -> None: + """End the session.""" + + +class MultiTierMediaSender(Protocol): + """A :class:`~pyhap.media.multi_tier.MultiTierSender` or its worker proxy.""" + + on_closed: Callable[[str], None] | None + on_keyframe_request: Callable[[float | None, float | None], None] | None + on_rtcp: Callable[[], None] | None + + async def bind(self) -> tuple[int, int]: + """Bind the accessory's video and audio ports and return them.""" + + async def start(self, source: MediaSourceLike, audio: bool = True) -> None: + """Send ``source``, a source of the same pool, to the controller.""" + + async def switch(self, source: MediaSourceLike) -> None: + """Continue the stream from another source of the same pool.""" + + async def close(self) -> None: + """End the session.""" + + +class RecorderLike(Protocol): + """A :class:`~pyhap.media.recording.Recorder` or its worker proxy.""" + + on_data: Callable[[bytes], None] | None + on_ended: Callable[[str | None], None] | None + on_closed: Callable[[str], None] | None + + @property + def closed(self) -> bool: + """Whether the recorder or its source ended.""" + + @property + def recording(self) -> bool: + """Whether a recording runs.""" + + async def start(self, command: list[str], prebuffer: float) -> None: + """Run the transcoder on the buffer and the live media.""" + + async def stop(self) -> None: + """End the running recording without calling ``on_ended``.""" + + async def close(self) -> None: + """Stop recording and buffering.""" + + +class ChannelPull(Protocol): + """The media pull of one camera channel, for example an ffmpeg process.""" + + async def start(self, rtp_input: tuple[str, int, int]) -> None: + """Start sending plain RTP to ``(host, video_port, audio_port)``.""" + + async def stop(self) -> None: + """Stop sending; called once after a successful :meth:`start`.""" + + def keyframe_requested(self, age: float | None, gop: float | None) -> None: + """A session needs an IRAP picture, see ``WebRTCSender``.""" + + +class SourceLease(Protocol): + """A session's claim on the media source of its camera channel.""" + + @property + def source(self) -> MediaSourceLike: + """The shared source.""" + + def create_webrtc_sender(self, **options: Any) -> WebRTCMediaSender: + """Create the session's sender on the source, where the source runs.""" + + async def activate(self) -> None: + """The session was answered: the channel must be pulled now.""" + + def keyframe_requested(self, age: float | None, gop: float | None) -> None: + """Forward a keyframe request of the session's sender.""" + + async def release(self) -> None: + """The session ended; called exactly once.""" + + +@dataclass(eq=False) +class _Channel: + key: Hashable + name: str + source: MediaSourceLike + pull: ChannelPull + leases: int = 0 + active: int = 0 + pulling: bool = False + closed: bool = False + lock: asyncio.Lock = field(default_factory=asyncio.Lock) + + +class ChannelLease: + """A :class:`SourceLease` of a :class:`SourcePool`.""" + + def __init__(self, pool: SourcePool, channel: _Channel) -> None: + """Use :meth:`SourcePool.lease`.""" + self._pool = pool + self._channel = channel + self.active = False + self.released = False + + @property + def source(self) -> MediaSourceLike: + """The channel's source.""" + return self._channel.source + + @property + def channel(self) -> str: + """The channel's name.""" + return self._channel.name + + def create_webrtc_sender(self, **options: Any) -> WebRTCMediaSender: + """Create a sender on the channel's source, in the pool's worker if any. + + ``options`` are those of :class:`~pyhap.media.sender.WebRTCSender` + except ``source``. + """ + worker = self._pool.worker + source: Any = self.source + if worker is not None: + return worker.create_webrtc_sender(source=source, **options) + from .sender import WebRTCSender # noqa: PLC0415 + + return cast(WebRTCMediaSender, WebRTCSender(source=source, **options)) + + async def open_recorder( + self, *, prebuffer: float, audio: bool = True + ) -> RecorderLike: + """Buffer the channel's source for recordings, in the pool's worker if any. + + See :class:`~pyhap.media.recording.Recorder`; the channel is only + pulled while the lease is active. + """ + worker = self._pool.worker + source: Any = self.source + if worker is not None: + return await worker.open_recorder(source, prebuffer=prebuffer, audio=audio) + from .recording import Recorder # noqa: PLC0415 + + return cast(RecorderLike, Recorder(source, prebuffer=prebuffer, audio=audio)) + + async def activate(self) -> None: + """Count the session as answered; the first one starts the pull.""" + await self._pool._activate(self) + + def keyframe_requested(self, age: float | None, gop: float | None) -> None: + """Pass the request on while the channel is pulled.""" + channel = self._channel + if self.active and channel.pulling and not channel.closed: + channel.pull.keyframe_requested(age, gop) + + async def release(self) -> None: + """Give up the claim; the last session stops the pull and the source.""" + await asyncio.shield(self._pool._release(self)) + + +class SourcePool: + """Shares one media source and one pull per camera channel. + + A channel is any hashable key; leases of equal keys share the source. + Sources and senders run in ``worker``, or in process without one. + """ + + def __init__( + self, worker: MediaWorker | None = None, host: str = "127.0.0.1" + ) -> None: + """Configure the pool; sources are opened by the first lease.""" + self.worker = worker + self.host = host + self._channels: dict[Hashable, _Channel] = {} + self._lock = asyncio.Lock() + self._tasks: set[asyncio.Task[Any]] = set() + + def create_multi_tier_sender(self, **options: Any) -> MultiTierMediaSender: + """Create a multi-tier sender where the pool's sources run. + + ``options`` are those of + :class:`~pyhap.media.multi_tier.MultiTierSender`. Start it with the + source of a lease of this pool. Unlike WebRTC senders and recorders, + which a lease creates on its source, the pool creates it: its ports + are announced in Setup Endpoints, before the controller picks the + tier, i.e. the lease. + """ + if self.worker is not None: + return cast( + MultiTierMediaSender, self.worker.create_multi_tier_sender(**options) + ) + from .multi_tier import MultiTierSender # noqa: PLC0415 + + return cast(MultiTierMediaSender, MultiTierSender(**options)) + + def counts(self) -> dict[str, tuple[int, int]]: + """Leases and answered leases per channel name.""" + return {c.name: (c.leases, c.active) for c in self._channels.values()} + + async def lease( + self, + channel: Hashable, + pull: Callable[[], ChannelPull], + *, + audio: bool = True, + name: str | None = None, + ) -> ChannelLease: + """Lease the source of ``channel``, opening it with ``pull`` if needed. + + ``pull`` makes the channel's pull when a new source is opened. + ``name`` is what the log shows instead of the key. + """ + task = asyncio.ensure_future(self._lease(channel, pull, audio, name)) + try: + return await asyncio.shield(task) + except asyncio.CancelledError: + # The lease is made anyway; give it back once it is. + task.add_done_callback(self._release_orphan) + raise + + async def _lease( + self, + key: Hashable, + pull: Callable[[], ChannelPull], + audio: bool, + name: str | None, + ) -> ChannelLease: + async with self._lock: + channel = self._channels.get(key) + if channel is None: + # Made first: a failing factory leaves no open source behind. + channel_pull = pull() + source = await self._open_source(audio) + channel = _Channel(key, name or str(key), source, channel_pull) + source.on_closed = partial(self._source_closed, channel) + if source.closed: + # It ended before on_closed was set, nothing will report it. + raise MediaError(f"Media source {channel.name} ended") + self._channels[key] = channel + logger.debug("Source %s opened at %s", channel.name, source.rtp_input) + channel.leases += 1 + self._log(channel) + return ChannelLease(self, channel) + + async def _open_source(self, audio: bool) -> MediaSourceLike: + if self.worker is not None: + return await self.worker.open_source(self.host, audio) + from .source import MediaSource # noqa: PLC0415 + + source = MediaSource(self.host, audio) + try: + await source.start() + except BaseException: + source.close("start failed") + raise + return source + + def _release_orphan(self, task: asyncio.Task[ChannelLease]) -> None: + if not task.cancelled() and task.exception() is None: + self._spawn(task.result().release()) + + def _spawn(self, coro: Coroutine[Any, Any, None]) -> None: + task = asyncio.ensure_future(coro) + self._tasks.add(task) + task.add_done_callback(self._tasks.discard) + + async def _activate(self, lease: ChannelLease) -> None: + channel = lease._channel + if lease.released or lease.active: + return + if channel.closed: + raise MediaError(f"Media source {channel.name} is closed") + lease.active = True + channel.active += 1 + try: + await self._sync_pull(channel) + except BaseException: + # A release while the pull started has already counted it off. + if lease.active: + lease.active = False + channel.active -= 1 + raise + self._log(channel) + + async def _release(self, lease: ChannelLease) -> None: + if lease.released: + return + lease.released = True + channel = lease._channel + channel.leases -= 1 + if lease.active: + lease.active = False + channel.active -= 1 + last = channel.leases == 0 and not channel.closed + if last: + # From here on a new lease opens a new source. + channel.closed = True + self._forget(channel) + self._log(channel) + await self._sync_pull(channel) + if last: + await self._close_source(channel, "no sessions left") + + async def _sync_pull(self, channel: _Channel) -> None: + """Start or stop the pull so it runs exactly while sessions are answered.""" + async with channel.lock: + wanted = channel.active > 0 and not channel.closed + if wanted and not channel.pulling: + logger.debug("Starting the pull of source %s", channel.name) + await channel.pull.start(channel.source.rtp_input) + channel.pulling = True + elif not wanted and channel.pulling: + channel.pulling = False + logger.debug("Stopping the pull of source %s", channel.name) + try: + await channel.pull.stop() + except Exception: + logger.exception("Stopping the pull of %s failed", channel.name) + + async def _close_source(self, channel: _Channel, reason: str) -> None: + source = channel.source + source.on_closed = None + try: + result = source.close(reason) + if inspect.isawaitable(result): + await result + except Exception: + logger.exception("Closing source %s failed", channel.name) + logger.debug("Source %s closed: %s", channel.name, reason) + + def _forget(self, channel: _Channel) -> None: + if self._channels.get(channel.key) is channel: + del self._channels[channel.key] + + def _source_closed(self, channel: _Channel, reason: str) -> None: + """The source ended on its own, e.g. its worker exited.""" + if channel.closed: + return + logger.warning("Source %s ended: %s", channel.name, reason) + channel.closed = True + self._forget(channel) + self._spawn(self._sync_pull(channel)) + + def _log(self, channel: _Channel) -> None: + logger.debug( + "Source %s: %d leases, %d answered%s", + channel.name, + channel.leases, + channel.active, + ", closed" if channel.closed else "", + ) + + async def close(self) -> None: + """Stop every pull and close every source, e.g. on shutdown.""" + # A lease being opened finishes first, so its channel is closed too. + async with self._lock: + channels = list(self._channels.values()) + self._channels.clear() + for channel in channels: + channel.closed = True + for channel in channels: + await self._sync_pull(channel) + await self._close_source(channel, "pool closed") + while self._tasks: + await asyncio.gather(*self._tasks, return_exceptions=True) diff --git a/pyhap/media/recording.py b/pyhap/media/recording.py new file mode 100644 index 00000000..3b098d56 --- /dev/null +++ b/pyhap/media/recording.py @@ -0,0 +1,537 @@ +"""Classic HomeKit Secure Video recordings from a shared media source. + +A :class:`Recorder` attaches a :class:`PrebufferRing` to a +:class:`~pyhap.media.source.MediaSource`: while the hub has recording on, +the ring keeps the last seconds of the channel as compressed access units +and Opus packets, starting at an IRAP picture. Nothing is decoded while no +recording runs. + +When the hub opens a recording, the recorder starts ffmpeg with +:func:`recording_ffmpeg_args`, writes the buffered media and then the live +media to its stdin as MPEG-TS (:mod:`pyhap.media.mpegts`) and hands +ffmpeg's fragmented MP4 to ``on_data``. ffmpeg reads the buffered part as +fast as it can encode, so the recording catches up with the live picture +within the first fragment or two. The hub selects H.264 and AAC in an exact +size, frame rate, profile, level and bit rate; the transcode makes exactly +that, with an IRAP picture, and so a fragment, every selected interval. + +The recorder runs where the source runs, in the media worker or in process. +""" + +from __future__ import annotations + +import asyncio +from collections import deque +from collections.abc import Callable +import contextlib +from dataclasses import dataclass +import logging +import math +import time +from typing import TYPE_CHECKING + +from pyhap import hksv_recording + +from . import rtp +from .ffmpeg import FFMPEG_STOP_TIMEOUT, StderrLines, redact_urls +from .hevc import AccessUnit +from .mpegts import MpegTsMuxer +from .rtp import VIDEO_CLOCK_RATE + +if TYPE_CHECKING: + from .source import MediaSource + +logger = logging.getLogger(__name__) + +AUDIO_CLOCK_RATE = 48000 +TS_CLOCK_RATE = 90000 +# The ring drops its oldest media beyond this many bytes, whatever the time. +MAX_BUFFER_BYTES = 64 * 1024 * 1024 +# A recording ends when ffmpeg falls this far behind its input. +MAX_PENDING_BYTES = 32 * 1024 * 1024 +# A timestamp this far from what the arrival times say starts a new anchor, +# e.g. after the pull restarted with a new SSRC. +MAX_TIMESTAMP_DRIFT = 1.0 +# The shortest IRAP interval, i.e. fragment length, a recording uses. +MIN_FRAGMENT_SECONDS = 1.0 +# The first PTS, so audio that arrived just before the video is not negative. +PTS_START = TS_CLOCK_RATE +OUTPUT_CHUNK = 64 * 1024 +# The selected recording values and their ffmpeg names. +H264_PROFILES = {0: "baseline", 1: "main", 2: "high"} +H264_LEVELS = {0: "3.1", 1: "3.2", 2: "4.0"} +SAMPLE_RATES = { + hksv_recording.AudioSampleRate.KHZ_8: 8000, + hksv_recording.AudioSampleRate.KHZ_16: 16000, + hksv_recording.AudioSampleRate.KHZ_24: 24000, + hksv_recording.AudioSampleRate.KHZ_32: 32000, + hksv_recording.AudioSampleRate.KHZ_44_1: 44100, + hksv_recording.AudioSampleRate.KHZ_48: 48000, +} +# Encoders that take ``-level`` and the x264 speed options. +X264_ENCODERS = ("libx264",) + + +@dataclass(frozen=True) +class BufferedMedia: + """One access unit or Opus packet and when it arrived.""" + + arrival: float + unit: AccessUnit | None = None + audio: rtp.RtpPacket | None = None + + @property + def keyframe(self) -> bool: + """Whether this is an IRAP picture.""" + return self.unit is not None and self.unit.keyframe + + @property + def size(self) -> int: + """Bytes of media.""" + if self.unit is not None: + return sum(len(nal) for nal in self.unit.nals) + return len(self.audio.payload) if self.audio is not None else 0 + + +class PrebufferRing: + """The last seconds of a source, starting at an IRAP picture. + + A :class:`~pyhap.media.source.MediaConsumer` that always wants media. + It keeps at least ``seconds`` before now, from the latest IRAP picture + at or before that point; ``listener`` gets each new item once set. + """ + + def __init__( + self, + seconds: float, + max_bytes: int = MAX_BUFFER_BYTES, + clock: Callable[[], float] = time.monotonic, + ) -> None: + """Configure the ring; attach it to a source to fill it.""" + self.seconds = seconds + self.max_bytes = max_bytes + self.listener: Callable[[BufferedMedia], None] | None = None + self.on_closed: Callable[[str], None] | None = None + self._clock = clock + self._items: deque[BufferedMedia] = deque() + self._bytes = 0 + self._ssrc: int | None = None + + @property + def wants_media(self) -> bool: + """Always, the ring buffers while it is attached.""" + return True + + @property + def buffered(self) -> float: + """Seconds from the first buffered IRAP picture to now, 0 without one.""" + if not self._items or not self._items[0].keyframe: + return 0.0 + return self._clock() - self._items[0].arrival + + def on_source_video(self, unit: AccessUnit, ssrc: int) -> None: + """Buffer an access unit; video before the first IRAP is dropped.""" + if ssrc != self._ssrc: + # A restarted pull: its timestamps start over. + self._ssrc = ssrc + self._clear() + if not self._items and not unit.keyframe: + return + self._add(BufferedMedia(self._clock(), unit=unit)) + + def on_source_audio(self, packet: rtp.RtpPacket) -> None: + """Buffer an Opus packet once there is video to go with it.""" + if self._items: + self._add(BufferedMedia(self._clock(), audio=packet)) + + def on_source_closed(self, reason: str) -> None: + """The source ended.""" + self._clear() + if self.on_closed is not None: + self.on_closed(reason) + + def snapshot(self, seconds: float) -> list[BufferedMedia]: + """The buffered media from the latest IRAP at least ``seconds`` ago. + + Fewer seconds when the ring holds less; empty without an IRAP. + """ + limit = self._clock() - seconds + start = 0 + for index, item in enumerate(self._items): + if item.arrival > limit: + break + if item.keyframe: + start = index + return list(self._items)[start:] + + def _add(self, item: BufferedMedia) -> None: + self._items.append(item) + self._bytes += item.size + self._trim() + if self.listener is not None: + self.listener(item) + + def _trim(self) -> None: + limit = self._clock() - self.seconds + while True: + # The second IRAP from the left: everything before it can go once + # it is old enough, or the ring is too large. + second = next( + (i for i, item in enumerate(self._items) if i and item.keyframe), None + ) + if second is None: + if self._bytes > self.max_bytes: + # No later IRAP to cut at, e.g. a very long GOP: start over + # at the next IRAP rather than grow without bound. A + # running recording misses the frames until that IRAP. + self._clear() + return + if self._items[second].arrival > limit and self._bytes <= self.max_bytes: + return + for _ in range(second): + self._bytes -= self._items.popleft().size + + def _clear(self) -> None: + self._items.clear() + self._bytes = 0 + + +class _Timestamps: + """Maps the RTP timestamps of one stream to a common 90 kHz timeline.""" + + def __init__(self, clock_rate: int, origin: float) -> None: + self._rate = clock_rate + self._origin = origin + self._anchor: tuple[int, float] | None = None + self._last = 0 + + def pts(self, timestamp: int, arrival: float) -> int: + elapsed = 0 + if self._anchor is not None: + elapsed = ( + (timestamp - self._anchor[0] + 0x80000000) & 0xFFFFFFFF + ) - 0x80000000 + expected = arrival - self._anchor[1] + if abs(elapsed / self._rate - expected) > MAX_TIMESTAMP_DRIFT: + self._anchor = None + if self._anchor is None: + self._anchor = (timestamp, arrival) + elapsed = 0 + seconds = self._anchor[1] - self._origin + elapsed / self._rate + pts = PTS_START + round(seconds * TS_CLOCK_RATE) + # Strictly increasing, ffmpeg drops repeated or earlier timestamps. + self._last = pts if pts > self._last else self._last + 1 + return self._last + + +def recording_ffmpeg_args( + selected: hksv_recording.SelectedRecordingConfiguration, + *, + audio: bool, + source_audio: bool = True, + ffmpeg: str = "ffmpeg", + video_encoder: str = "libx264", + encoder_args: tuple[str, ...] = (), +) -> list[str]: + """The ffmpeg command transcoding MPEG-TS on stdin to the hub's selection. + + ``audio`` is whether the hub wants audio; without it, or without + ``source_audio``, the recording gets a silent track, the hub expects one. + """ + video = selected.video + attributes = video.attributes[0] + fps = attributes.frame_rate + width, height = attributes.width, attributes.height + # Fragments start at each IRAP picture, so the IRAP interval is the + # fragment length the hub selected. + containers = selected.recording.media_containers + fragment_ms = ( + containers[0].fragment_length_ms if containers else video.iframe_interval_ms + ) + interval = max(fragment_ms / 1000, MIN_FRAGMENT_SECONDS) + profile = video.profile[0] if isinstance(video.profile, list) else video.profile + level = video.level[0] if isinstance(video.level, list) else video.level + sample_rate = SAMPLE_RATES[selected.audio.sample_rate] + channels = selected.audio.channels + audio_bitrate = selected.audio.max_audio_bitrate_kbps or 64 + real_audio = audio and source_audio + command = [ + *(ffmpeg, "-hide_banner", "-nostdin", "-loglevel", "warning", "-nostats"), + # Media is ready at once; do not wait seconds to probe it. + *("-analyzeduration", "1000000", "-f", "mpegts", "-i", "pipe:0"), + ] + if not real_audio: + layout = "mono" if channels == 1 else "stereo" + command += ["-f", "lavfi", "-i", f"anullsrc=r={sample_rate}:cl={layout}"] + encoder = [] + if video_encoder in X264_ENCODERS: + encoder = [ + *("-level:v", H264_LEVELS.get(level, "4.0")), + *("-preset", "veryfast", "-tune", "zerolatency"), + ] + command += [ + *("-map", "0:v:0", "-c:v", video_encoder), + *("-profile:v", H264_PROFILES.get(profile, "main")), + *encoder, + *encoder_args, + "-pix_fmt", + "yuv420p", + # The hub expects exactly the selected size, other aspect ratios are padded. + "-vf", + f"scale={width}:{height}:force_original_aspect_ratio=decrease:" + f"force_divisible_by=2,pad={width}:{height}:(ow-iw)/2:(oh-ih)/2,setsar=1", + *("-r", str(fps), "-b:v", f"{video.bitrate_kbps}k"), + *( + "-maxrate", + f"{video.bitrate_kbps}k", + "-bufsize", + f"{video.bitrate_kbps * 2}k", + ), + *("-bf", "0", "-g", str(math.ceil(fps * interval))), + *("-force_key_frames", f"expr:gte(t,n_forced*{interval:g})"), + # Scene cuts would add IRAP pictures and split fragments. + *("-sc_threshold", "0"), + *("-map", "0:a:0" if real_audio else "1:a:0"), + *("-c:a", "aac", "-profile:a", "aac_low"), + *("-ac", str(channels), "-ar", str(sample_rate), "-b:a", f"{audio_bitrate}k"), + ] + if not real_audio: + command.append("-shortest") + command += [ + *( + "-f", + "mp4", + "-movflags", + "frag_keyframe+empty_moov+default_base_moof+skip_trailer", + ), + "pipe:1", + ] + return command + + +class Recorder: + """Buffers a source and transcodes it into recordings, one at a time. + + ``on_data(chunk)`` gets ffmpeg's fragmented MP4 output and + ``on_ended(error)`` the end of a recording, None for a normal end. + ``on_closed(reason)`` fires when the source ends. + """ + + def __init__( + self, source: MediaSource, *, prebuffer: float, audio: bool = True + ) -> None: + """Attach the ring to ``source``; it buffers from now on.""" + self.on_data: Callable[[bytes], None] | None = None + self.on_ended: Callable[[str | None], None] | None = None + self.on_closed: Callable[[str], None] | None = None + self.audio = audio + self.ring = PrebufferRing(prebuffer) + self.ring.on_closed = self._source_closed + self._source = source + self._process: asyncio.subprocess.Process | None = None + self._tasks: list[asyncio.Task[None]] = [] + # The last transcoder lines, the newest says why it failed. + self._stderr_tail: deque[str] = deque(maxlen=3) + self._stderr_done: asyncio.Task[None] | None = None + # Stops started from callbacks, kept so they are not collected. + self._stops: set[asyncio.Task[None]] = set() + self._truncated = False + self._queue: asyncio.Queue[bytes | None] | None = None + self._pending = 0 + self._closed = False + source.attach(self.ring) + + @property + def closed(self) -> bool: + """Whether the recorder or its source ended.""" + return self._closed + + @property + def recording(self) -> bool: + """Whether a recording runs.""" + return self._process is not None + + async def start(self, command: list[str], prebuffer: float) -> None: + """Run ``command`` on the last ``prebuffer`` seconds and the live media.""" + if self._closed: + raise RuntimeError("Recorder is closed") + if self._process is not None: + raise RuntimeError("A recording runs already") + items = self.ring.snapshot(prebuffer) + origin = items[0].arrival if items else time.monotonic() + video = _Timestamps(VIDEO_CLOCK_RATE, origin) + audio = _Timestamps(AUDIO_CLOCK_RATE, origin) + muxer = MpegTsMuxer(audio=self.audio) + queue: asyncio.Queue[bytes | None] = asyncio.Queue() + self._queue = queue + self._pending = 0 + self._truncated = False + started = bool(items) + + def mux(item: BufferedMedia) -> None: + nonlocal started + if item.unit is not None: + if not started and not item.keyframe: + return + started = True + data = muxer.video( + item.unit.nals, + video.pts(item.unit.timestamp, item.arrival), + item.keyframe, + ) + elif item.audio is not None and started: + data = muxer.opus( + item.audio.payload, audio.pts(item.audio.timestamp, item.arrival) + ) + else: + return + self._pending += len(data) + if self._pending > MAX_PENDING_BYTES: + self.ring.listener = None + self._truncated = True + logger.warning("Recording transcoder fell behind, ending it") + queue.put_nowait(None) + return + queue.put_nowait(data) + + logger.debug( + "Recording starts with %.1f s of buffered media", + time.monotonic() - origin if items else 0.0, + ) + for item in items: + mux(item) + self.ring.listener = mux + try: + self._process = await asyncio.create_subprocess_exec( + *command, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + except BaseException: + self.ring.listener = None + self._queue = None + raise + logger.info("Recording transcoder started, pid %d", self._process.pid) + process = self._process + loop = asyncio.get_running_loop() + self._stderr_tail.clear() + self._stderr_done = loop.create_task(_log_stderr(process, self._stderr_tail)) + self._tasks = [ + loop.create_task(self._write(process, queue)), + loop.create_task(self._read(process)), + self._stderr_done, + ] + + async def _write( + self, process: asyncio.subprocess.Process, queue: asyncio.Queue[bytes | None] + ) -> None: + stdin = process.stdin + assert stdin is not None # noqa: S101 + try: + while (data := await queue.get()) is not None: + self._pending -= len(data) + stdin.write(data) + await stdin.drain() + except (BrokenPipeError, ConnectionResetError): + return + finally: + with contextlib.suppress(Exception): + stdin.close() + + async def _read(self, process: asyncio.subprocess.Process) -> None: + stdout = process.stdout + assert stdout is not None # noqa: S101 + while chunk := await stdout.read(OUTPUT_CHUNK): + if self.on_data is not None: + try: + self.on_data(chunk) + except Exception as err: + logger.exception("Error handling the recording output") + if self._process is process: + self._finish(f"recording output failed: {err!r}") + return + code = await process.wait() + if self._process is not process: + return + if code != 0: + # The last line usually says why; wait briefly for it. + if self._stderr_done is not None: + with contextlib.suppress(TimeoutError): + await asyncio.wait_for(asyncio.shield(self._stderr_done), 1.0) + reason = f": {self._stderr_tail[-1]}" if self._stderr_tail else "" + self._finish(f"ffmpeg exited with {code}{reason}") + elif self._truncated: + self._finish("the transcoder fell behind, the recording was cut short") + else: + self._finish(None) + + def _finish(self, error: str | None) -> None: + self.ring.listener = None + self._process = None + if self._queue is not None: + self._queue.put_nowait(None) + self._queue = None + if self.on_ended is not None: + self.on_ended(error) + + async def stop(self) -> None: + """End the running recording; ``on_ended`` is not called.""" + process, self._process = self._process, None + self.ring.listener = None + if self._queue is not None: + self._queue.put_nowait(None) + self._queue = None + tasks, self._tasks = self._tasks, [] + if process is not None and process.returncode is None: + with contextlib.suppress(ProcessLookupError): + process.terminate() + try: + await asyncio.wait_for(process.wait(), FFMPEG_STOP_TIMEOUT) + except TimeoutError: + with contextlib.suppress(ProcessLookupError): + process.kill() + await process.wait() + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + async def close(self) -> None: + """Stop recording and buffering.""" + if self._closed: + return + self._closed = True + await self.stop() + self._source.detach(self.ring) + + def _source_closed(self, reason: str) -> None: + if self._closed: + return + self._closed = True + if self._process is not None: + task = asyncio.get_running_loop().create_task(self.stop()) + self._stops.add(task) + task.add_done_callback(self._stops.discard) + if self.on_ended is not None: + self.on_ended(f"media source ended: {reason}") + if self.on_closed is not None: + self.on_closed(reason) + + +async def _log_stderr(process: asyncio.subprocess.Process, tail: deque[str]) -> None: + stream = process.stderr + if stream is None: + return + lines = StderrLines() + while chunk := await stream.read(4096): + for line in lines.feed(chunk): + _log_line(line, tail) + for line in lines.end(): + _log_line(line, tail) + + +def _log_line(line: bytes, tail: deque[str]) -> None: + text = redact_urls(line.decode(errors="replace").rstrip()) + if text: + tail.append(text) + logger.debug("Recording ffmpeg: %s", text) diff --git a/pyhap/media/recording_delegate.py b/pyhap/media/recording_delegate.py new file mode 100644 index 00000000..208c006e --- /dev/null +++ b/pyhap/media/recording_delegate.py @@ -0,0 +1,329 @@ +"""Classic HomeKit Secure Video recordings from a source pool. + +:class:`RecordingMediaDelegate` turns one camera channel into the hub's +recordings. Without a prebuffer (the default) it pulls the channel only for +a recording, which starts at the next IRAP picture. With ``prebuffer_ms`` +it keeps the channel buffered while the hub has recording on, so a +recording starts that many milliseconds before the hub asked:: + + worker = MediaWorker() + provider = FFmpegSourceProvider({"high": [...], "medium": [...]}, worker=worker) + recording = RecordingMediaDelegate(provider, medium_tier, prebuffer_ms=4000) + camera = Camera( + { + ..., + "recording": {"delegate": recording}, + "multi_tier": {"delegate": MultiTierMediaDelegate(provider)}, + }, + driver, + "Camera", + ) + +The channel comes from the same provider and pool as the live sessions, so +a recording channel that is also watched live is pulled once. While armed +with a prebuffer the channel is pulled and buffered as compressed media; +only a recording transcodes, see :class:`~pyhap.media.recording.Recorder`. +The camera arms the delegate while the hub has recording on and the camera +is on, and advertises :attr:`RecordingMediaDelegate.prebuffer_ms`. + +Observed with iOS and tvOS 27: the hub does not upload recordings it gets +over HDS from a camera that also offers the Multi-Tier RTP service, so they +never appear in the Home app. Use this delegate for cameras without +Multi-Tier. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncGenerator, Callable, Sequence +from functools import partial +import logging +from typing import Protocol + +from pyhap import hksv3, hksv_recording +from pyhap.hds_recording import RecordingPacket, fmp4_recording_packets + +from .pool import ChannelLease, RecorderLike +from .recording import recording_ffmpeg_args + +logger = logging.getLogger(__name__) + +DEFAULT_PREBUFFER_MS = 0 +# Seconds before a buffered channel whose recorder ended is opened again, +# doubling up to the maximum while it keeps failing. +REOPEN_DELAY = 1.0 +MAX_REOPEN_DELAY = 60.0 +# The fragmented MP4 a recording may hold while the hub reads it. +MAX_PENDING_OUTPUT = 16 * 1024 * 1024 + + +class ChannelProvider(Protocol): + """Leases camera channels, e.g. an + :class:`~pyhap.media.ffmpeg.FFmpegSourceProvider`. + """ + + ffmpeg: str + + async def lease_channel(self, tier: hksv3.VideoStreamTier) -> ChannelLease: + """Lease the channel of ``tier`` without activating it.""" + + +class _BoundedReader: + """Holds a recording's MP4 output until the hub reads it, up to a limit. + + Has the ``readexactly`` that :func:`fmp4_recording_packets` needs; + ``on_overflow`` runs once when unread output exceeds ``limit`` bytes. + """ + + def __init__(self, limit: int, on_overflow: Callable[[], None]) -> None: + self._limit = limit + self._on_overflow = on_overflow + self._buffer = bytearray() + self._eof = False + self._changed = asyncio.Event() + + def feed_data(self, data: bytes) -> None: + if self._eof: + return + self._buffer += data + self._changed.set() + if len(self._buffer) > self._limit: + self._on_overflow() + + def feed_eof(self) -> None: + self._eof = True + self._changed.set() + + def at_eof(self) -> bool: + return self._eof and not self._buffer + + async def readexactly(self, n: int) -> bytes: + size = n + while len(self._buffer) < size: + if self._eof: + partial = bytes(self._buffer) + self._buffer.clear() + raise asyncio.IncompleteReadError(partial, size) + self._changed.clear() + await self._changed.wait() + data = bytes(self._buffer[:size]) + del self._buffer[:size] + return data + + +class RecordingError(Exception): + """A recording could not be made or ended early.""" + + +class RecordingMediaDelegate: + """Transcodes a channel into recordings, buffering it while armed. + + With ``prebuffer_ms`` 0 nothing is pulled while idle. + + ``source_provider`` is a :class:`ChannelProvider`, ``tier`` selects its + recording channel. ``prebuffer_ms`` 0 is valid: hubs accept it (observed + with iOS and tvOS 26 and 27) and record from the trigger on. ``ffmpeg`` + defaults to the provider's; ``video_encoder`` and ``encoder_args`` pick + the H.264 encoder, e.g. a hardware one. + """ + + def __init__( + self, + source_provider: ChannelProvider, + tier: hksv3.VideoStreamTier, + *, + prebuffer_ms: int = DEFAULT_PREBUFFER_MS, + ffmpeg: str | None = None, + video_encoder: str = "libx264", + encoder_args: Sequence[str] = (), + ) -> None: + """Configure the delegate; :meth:`arm` starts buffering.""" + self._provider = source_provider + self.tier = tier + self.prebuffer_ms = prebuffer_ms + self.ffmpeg = ffmpeg or source_provider.ffmpeg + self.video_encoder = video_encoder + self.encoder_args = tuple(encoder_args) + self._lease: ChannelLease | None = None + self._recorder: RecorderLike | None = None + self._armed = False + self._lock = asyncio.Lock() + self._reopening: asyncio.Task[None] | None = None + + @property + def armed(self) -> bool: + """Whether the hub has recording on (see :meth:`arm`).""" + return self._armed + + @property + def buffering(self) -> bool: + """Whether the channel is kept pulled and buffered between recordings.""" + return self._armed and self.prebuffer_ms > 0 + + @property + def recording(self) -> bool: + """Whether a recording runs.""" + return self._recorder is not None and self._recorder.recording + + async def set_armed(self, armed: bool) -> None: + """Arm or disarm, see :meth:`arm` and :meth:`disarm`.""" + if armed: + await self.arm() + else: + await self.disarm() + + async def arm(self) -> None: + """Allow recordings; with a prebuffer, buffer until :meth:`disarm`.""" + async with self._lock: + self._armed = True + if self.buffering: + try: + await self._open() + except BaseException: + self._armed = False + raise + + async def disarm(self) -> None: + """Stop buffering; a running recording ends.""" + async with self._lock: + self._armed = False + await self._release() + + async def close(self) -> None: + """Disarm and release the channel, e.g. on shutdown. + + The delegate may be armed again afterwards. + """ + await self.disarm() + + async def _open(self) -> RecorderLike: + recorder = self._recorder + if recorder is not None and not recorder.closed: + return recorder + await self._release() + lease = await self._provider.lease_channel(self.tier) + try: + await lease.activate() + audio = lease.source.has_audio + recorder = await lease.open_recorder( + prebuffer=self.prebuffer_ms / 1000, audio=audio + ) + except BaseException: + await lease.release() + raise + self._lease, self._recorder = lease, recorder + recorder.on_closed = partial(self._recorder_closed, recorder) + logger.debug("Recording channel %s buffered", lease.channel) + return recorder + + def _recorder_closed(self, recorder: RecorderLike, reason: str) -> None: + """The recorder ended on its own, e.g. its worker exited.""" + if self._recorder is not recorder or not self.buffering: + return + logger.warning("Recording channel ended, buffering again: %s", reason) + if self._reopening is None or self._reopening.done(): + self._reopening = asyncio.ensure_future(self._reopen()) + + async def _reopen(self) -> None: + """Buffer again while armed, with a growing delay between failures.""" + delay = REOPEN_DELAY + while True: + async with self._lock: + recorder = self._recorder + if not self.buffering or (recorder and not recorder.closed): + return + try: + await self._open() + except Exception as err: # noqa: BLE001 - retried below + logger.warning( + "Could not buffer the recording channel, retrying in" + " %.0f s: %s", + delay, + err, + ) + else: + return + await asyncio.sleep(delay) + delay = min(delay * 2, MAX_REOPEN_DELAY) + + async def _release(self) -> None: + reopening = self._reopening + if reopening is not None and reopening is not asyncio.current_task(): + self._reopening = None + reopening.cancel() + recorder, self._recorder = self._recorder, None + lease, self._lease = self._lease, None + if recorder is not None: + ended = recorder.on_ended if recorder.recording else None + await recorder.close() + if ended is not None: + ended("recording channel released") + if lease is not None: + await lease.release() + + async def record( + self, + selected: hksv_recording.SelectedRecordingConfiguration, + audio: bool, + ) -> AsyncGenerator[RecordingPacket, None]: + """Yield the recording the hub selected, starting with the buffer. + + Without a buffered channel the channel is pulled for this recording + only, which then starts at the next IRAP picture. Raises + :class:`RecordingError` when the transcode fails. + """ + async with self._lock: + recorder = await self._open() + if recorder.recording: + raise RecordingError("A recording runs already") + errors: list[str] = [] + + def overflow() -> None: + errors.append("the hub reads the recording too slowly") + recorder.on_data = None + reader.feed_eof() + + reader = _BoundedReader(MAX_PENDING_OUTPUT, overflow) + + def ended(error: str | None) -> None: + if error is not None: + errors.append(error) + reader.feed_eof() + + try: + command = recording_ffmpeg_args( + selected, + audio=audio, + source_audio=self._lease is not None + and self._lease.source.has_audio, + ffmpeg=self.ffmpeg, + video_encoder=self.video_encoder, + encoder_args=self.encoder_args, + ) + recorder.on_data = reader.feed_data + recorder.on_ended = ended + await recorder.start( + command, selected.recording.prebuffer_length_ms / 1000 + ) + except BaseException: + recorder.on_data = None + recorder.on_ended = None + if not self.buffering: + await self._release() + raise + try: + async for packet in fmp4_recording_packets(reader): + if packet.is_last and errors: + break + yield packet + if errors: + raise RecordingError(errors[0]) + finally: + recorder.on_data = None + recorder.on_ended = None + await recorder.stop() + if not self.buffering: + async with self._lock: + # A recording started meanwhile keeps the channel. + if not self.buffering and not self.recording: + await self._release() diff --git a/pyhap/media/rtp.py b/pyhap/media/rtp.py new file mode 100644 index 00000000..26ac69e8 --- /dev/null +++ b/pyhap/media/rtp.py @@ -0,0 +1,310 @@ +"""Minimal RTP and RTCP handling for the camera media senders (RFC 3550, RFC 4585).""" + +from __future__ import annotations + +from collections import OrderedDict +from collections.abc import Iterable +from dataclasses import dataclass, field +import secrets +import struct +import time + +# RTP clock rates of H.265 video and Opus audio (RFC 7798, RFC 7587). +VIDEO_CLOCK_RATE = 90000 +AUDIO_CLOCK_RATE = 48000 +# Dynamic payload types the HomeKit streams use for H.265 and Opus. +DEFAULT_VIDEO_PAYLOAD_TYPE = 99 +DEFAULT_AUDIO_PAYLOAD_TYPE = 110 + +# Shortest interval between keyframe requests to a source. +KEYFRAME_REQUEST_INTERVAL = 1.0 +# An audio gap this long starts a new talkspurt (RTP marker bit). +AUDIO_TALKSPURT_GAP = 1.0 +# Sent packets kept for NACK retransmission. +RETRANSMIT_HISTORY = 1024 + + +RTP_VERSION = 2 +RTP_HEADER = 12 +RTCP_SR = 200 +RTCP_RR = 201 +RTCP_SDES = 202 +RTCP_BYE = 203 +RTCP_RTPFB = 205 +RTCP_PSFB = 206 +FMT_NACK = 1 +FMT_PLI = 1 +FMT_FIR = 4 +NTP_EPOCH_OFFSET = 2208988800 +# Payload types 64-95 collide with RTCP packet types (RFC 5761 section 4). +RTCP_PAYLOAD_TYPES = range(64, 96) + + +@dataclass +class RtpPacket: + """A parsed RTP packet; header extensions and CSRCs are dropped.""" + + payload_type: int + sequence: int + timestamp: int + ssrc: int + marker: bool + payload: bytes + + @property + def is_rtcp(self) -> bool: + """Whether this "RTP" packet is really RTCP sent to an RTP port.""" + return self.payload_type in RTCP_PAYLOAD_TYPES + + +def parse_rtp(packet: bytes) -> RtpPacket: + """Parse an RTP packet, skipping CSRCs, extension and padding.""" + if len(packet) < RTP_HEADER or packet[0] >> 6 != RTP_VERSION: + raise ValueError("Not an RTP packet") + first, second, sequence, timestamp, ssrc = struct.unpack_from(">BBHII", packet) + offset = RTP_HEADER + 4 * (first & 0x0F) + if first & 0x10: + if offset + 4 > len(packet): + raise ValueError("Truncated RTP header extension") + (words,) = struct.unpack_from(">H", packet, offset + 2) + offset += 4 + 4 * words + end = len(packet) + if first & 0x20: + if end == 0 or packet[-1] == 0: + raise ValueError("Invalid RTP padding") + end -= packet[-1] + if offset > end: + raise ValueError("Truncated RTP packet") + return RtpPacket( + payload_type=second & 0x7F, + sequence=sequence, + timestamp=timestamp, + ssrc=ssrc, + marker=bool(second & 0x80), + payload=packet[offset:end], + ) + + +def build_rtp(packet: RtpPacket) -> bytes: + """Serialize an RTP packet without CSRCs, extension or padding.""" + second = (0x80 if packet.marker else 0) | packet.payload_type + header = struct.pack( + ">BBHII", + RTP_VERSION << 6, + second, + packet.sequence & 0xFFFF, + packet.timestamp & 0xFFFFFFFF, + packet.ssrc, + ) + return header + packet.payload + + +@dataclass +class RtcpMessage: + """The RTCP fields the sender acts on.""" + + packet_type: int + fmt: int + sender_ssrc: int + media_ssrcs: list[int] = field(default_factory=list) + lost: list[int] = field(default_factory=list) + + @property + def is_keyframe_request(self) -> bool: + """Whether this is a PLI or FIR.""" + return self.packet_type == RTCP_PSFB and self.fmt in (FMT_PLI, FMT_FIR) + + @property + def is_nack(self) -> bool: + """Whether this is a generic NACK.""" + return self.packet_type == RTCP_RTPFB and self.fmt == FMT_NACK + + +def parse_rtcp(data: bytes) -> list[RtcpMessage]: + """Parse a compound RTCP packet; stops at the first malformed part. + + Every part must be version 2, fit in ``data`` and carry at least the + sender SSRC (RFC 3550 section 6.4), except a BYE without sources, whose + ``sender_ssrc`` is 0. + """ + messages = [] + offset = 0 + while offset + 4 <= len(data): + first, packet_type, length = struct.unpack_from(">BBH", data, offset) + end = offset + 4 * (length + 1) + if first >> 6 != RTP_VERSION or end > len(data): + break + if length == 0: + # Only a BYE without sources is just a header (RFC 3550 6.6). + if packet_type != RTCP_BYE or first & 0x1F: + break + messages.append(RtcpMessage(packet_type, 0, 0)) + else: + body = data[offset + 4 : end] + messages.append(_parse_one(first & 0x1F, packet_type, body)) + offset = end + return messages + + +def _parse_one(fmt: int, packet_type: int, body: bytes) -> RtcpMessage: + (sender,) = struct.unpack_from(">I", body) + message = RtcpMessage(packet_type, fmt, sender) + if packet_type == RTCP_BYE: + message.media_ssrcs = list( + struct.unpack_from(f">{min(fmt, len(body) // 4)}I", body) + ) + elif packet_type in (RTCP_RTPFB, RTCP_PSFB) and len(body) >= 8: + (media,) = struct.unpack_from(">I", body, 4) + fci = body[8:] + if packet_type == RTCP_PSFB and fmt == FMT_FIR: + # FIR names its targets in the FCI (RFC 5104 section 4.3.1). + message.media_ssrcs = [ + struct.unpack_from(">I", fci, i)[0] for i in range(0, len(fci) - 7, 8) + ] + else: + message.media_ssrcs = [media] + if message.is_nack: + message.lost = _nack_sequences(fci) + return message + + +def _nack_sequences(fci: bytes) -> list[int]: + lost: list[int] = [] + for i in range(0, len(fci) - 3, 4): + pid: int + blp: int + pid, blp = struct.unpack_from(">HH", fci, i) + lost.append(pid) + lost += [(pid + bit + 1) & 0xFFFF for bit in range(16) if blp >> bit & 1] + return lost + + +def ntp_time(now: float | None = None) -> int: + """Return a 64-bit NTP timestamp for a Unix time.""" + now = time.time() if now is None else now + seconds = int(now) + NTP_EPOCH_OFFSET + fraction = int((now % 1) * (1 << 32)) + return (seconds & 0xFFFFFFFF) << 32 | fraction + + +def build_sender_report( + ssrc: int, ntp: int, rtp_timestamp: int, packets: int, octets: int +) -> bytes: + """Build an RTCP SR without report blocks.""" + return struct.pack( + ">BBHIQIII", + RTP_VERSION << 6, + RTCP_SR, + 6, + ssrc, + ntp, + rtp_timestamp & 0xFFFFFFFF, + packets & 0xFFFFFFFF, + octets & 0xFFFFFFFF, + ) + + +def build_receiver_report(ssrc: int) -> bytes: + """Build an RTCP RR without report blocks.""" + return struct.pack(">BBHI", RTP_VERSION << 6, RTCP_RR, 1, ssrc) + + +def build_sdes(ssrc: int, cname: str) -> bytes: + """Build an RTCP SDES with one CNAME chunk.""" + name = cname.encode()[:255] + chunk = struct.pack(">IBB", ssrc, 1, len(name)) + name + b"\x00" + chunk += bytes(-len(chunk) % 4) + header = struct.pack(">BBH", RTP_VERSION << 6 | 1, RTCP_SDES, len(chunk) // 4) + return header + chunk + + +def build_bye(ssrcs: Iterable[int]) -> bytes: + """Build an RTCP BYE for the given sources.""" + sources = list(ssrcs) + header = struct.pack( + ">BBH", RTP_VERSION << 6 | len(sources), RTCP_BYE, len(sources) + ) + return header + b"".join(struct.pack(">I", ssrc) for ssrc in sources) + + +def build_pli(sender_ssrc: int, media_ssrc: int) -> bytes: + """Build an RTCP PLI.""" + return struct.pack( + ">BBHII", RTP_VERSION << 6 | FMT_PLI, RTCP_PSFB, 2, sender_ssrc, media_ssrc + ) + + +def build_nack(sender_ssrc: int, media_ssrc: int, lost: Iterable[int]) -> bytes: + """Build an RTCP generic NACK with one FCI per lost packet.""" + fci = b"".join(struct.pack(">HH", seq & 0xFFFF, 0) for seq in lost) + header = struct.pack( + ">BBHII", + RTP_VERSION << 6 | FMT_NACK, + RTCP_RTPFB, + 2 + len(fci) // 4, + sender_ssrc, + media_ssrc, + ) + return header + fci + + +class OutboundStream: + """SSRC, sequence and timestamp rewriting plus SR and NACK state.""" + + def __init__(self, payload_type: int, clock_rate: int) -> None: + self.ssrc = secrets.randbits(32) + self.payload_type = payload_type + self.clock_rate = clock_rate + self.sequence = secrets.randbits(16) + self.packets = 0 + self.octets = 0 + self._out_base = secrets.randbits(32) + self._in_base: int | None = None + self._in_ssrc: int | None = None + self._last_out: int | None = None + self._last_wall = 0.0 + self.history: OrderedDict[int, bytes] = OrderedDict() + + def rebase(self) -> None: + """Continue the timeline from the next input, e.g. of another source.""" + self._in_base = None + + def timestamp(self, ssrc: int, timestamp: int) -> int: + """Map an input timestamp onto this stream's timeline.""" + now = time.monotonic() + if ssrc != self._in_ssrc or self._in_base is None: + # A new or restarted source continues where the old one ended. + if self._last_out is not None: + elapsed = max(1, round((now - self._last_wall) * self.clock_rate)) + self._out_base = self._last_out + elapsed + self._in_ssrc = ssrc + self._in_base = timestamp + out = (timestamp - self._in_base + self._out_base) & 0xFFFFFFFF + self._last_out = out + self._last_wall = now + return out + + def build(self, timestamp: int, marker: bool, payload: bytes) -> bytes: + """Build the next outgoing RTP packet and remember it for NACK.""" + packet = build_rtp( + RtpPacket( + self.payload_type, self.sequence, timestamp, self.ssrc, marker, payload + ) + ) + self.history[self.sequence] = packet + if len(self.history) > RETRANSMIT_HISTORY: + self.history.popitem(last=False) + self.sequence = (self.sequence + 1) & 0xFFFF + self.packets += 1 + self.octets += len(payload) + return packet + + def sender_report(self) -> bytes | None: + """Build an SR, extrapolating the RTP time to now.""" + if self._last_out is None: + return None + elapsed = time.monotonic() - self._last_wall + rtp_now = self._last_out + round(elapsed * self.clock_rate) + return build_sender_report( + self.ssrc, ntp_time(), rtp_now, self.packets, self.octets + ) diff --git a/pyhap/media/sdp.py b/pyhap/media/sdp.py new file mode 100644 index 00000000..3e62824d --- /dev/null +++ b/pyhap/media/sdp.py @@ -0,0 +1,618 @@ +"""SDP for HomeKit WebRTC remote live view. + +The accessory is the offerer. Its offer is sendonly, BUNDLEs one H.265 video +and one Opus audio section, carries all ICE candidates (no trickle) and keeps +``a=setup:actpass`` so the controller picks the DTLS role. + +The video section offers H.265 as the only video codec and carries +``a=rid:1 send`` with ``max-width``, ``max-height``, ``max-fps`` and +``max-br`` but no RID RTP header extension. That matches the offer of +Scrypted's HomeKit WebRTC plugin (https://github.com/koush/scrypted, +``plugins/webrtc``), which is known to play remotely. + +The offer carries every ICE candidate. A controller may still send its +answer's candidates next to the SDP in HAP (RFC 8839 candidate strings with an +optional mid and m-line index, see :class:`IceCandidateLike`); they are merged +into the answer before it is parsed. At most :data:`MAX_CANDIDATES` candidates +are kept per media section. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from dataclasses import dataclass, field +from itertools import pairwise +from typing import Protocol + +from .rtp import ( + AUDIO_CLOCK_RATE, + DEFAULT_AUDIO_PAYLOAD_TYPE, + DEFAULT_VIDEO_PAYLOAD_TYPE, + VIDEO_CLOCK_RATE, +) + +OPUS_PTIME = 20 +VIDEO_MID = "0" +AUDIO_MID = "1" +RID = "1" +DIRECTIONS = ("sendrecv", "sendonly", "recvonly", "inactive") +MAX_CANDIDATES = 32 +MAX_SDP_SIZE = 32 * 1024 +MAX_MEDIA_SECTIONS = 16 + + +class SdpError(ValueError): + """An SDP could not be parsed or does not fit the session.""" + + +class IceCandidateLike(Protocol): + """An ICE candidate as HAP carries it next to the SDP. + + ``candidate`` is the RFC 8839 candidate string, with or without the + ``candidate:`` prefix. + """ + + @property + def candidate(self) -> str: + """The candidate string.""" + + @property + def sdp_mid(self) -> str | None: + """The mid of the media section it belongs to.""" + + @property + def sdp_mline_index(self) -> int | None: + """The index of the media section it belongs to.""" + + +class VideoTierLike(Protocol): + """A HAP video stream tier; bit rates in kbps.""" + + @property + def width(self) -> int: + """Pixels.""" + + @property + def height(self) -> int: + """Pixels.""" + + @property + def frame_rate(self) -> int: + """Frames per second.""" + + @property + def target_bitrate(self) -> int: + """The average bit rate.""" + + @property + def peak_bitrate(self) -> int | None: + """The peak bit rate, if known.""" + + +@dataclass(frozen=True) +class VideoParams: + """The video stream tier the offer advertises. + + ``peak_bitrate_kbps`` goes into ``max-br``; ``target_bitrate_kbps`` (the + average) paces the sender and defaults to the peak. + """ + + width: int + height: int + fps: int + peak_bitrate_kbps: int + payload_type: int = DEFAULT_VIDEO_PAYLOAD_TYPE + target_bitrate_kbps: int | None = None + + @classmethod + def from_tier( + cls, tier: VideoTierLike, payload_type: int = DEFAULT_VIDEO_PAYLOAD_TYPE + ) -> VideoParams: + """Describe a HAP video tier; the peak falls back to the target.""" + return cls( + width=tier.width, + height=tier.height, + fps=tier.frame_rate, + peak_bitrate_kbps=tier.peak_bitrate or tier.target_bitrate, + payload_type=payload_type, + target_bitrate_kbps=tier.target_bitrate, + ) + + @property + def pacing_bitrate_kbps(self) -> int: + """The bit rate the pacer is based on.""" + return self.target_bitrate_kbps or self.peak_bitrate_kbps + + +@dataclass(frozen=True) +class AudioParams: + """The mono Opus audio stream; ``talkback`` makes the section sendrecv.""" + + payload_type: int = DEFAULT_AUDIO_PAYLOAD_TYPE + talkback: bool = False + + +@dataclass(frozen=True) +class LocalParams: + """Transport and stream identifiers of the local side.""" + + ice_ufrag: str + ice_pwd: str + fingerprint: str + cname: str + stream_id: str + video_ssrc: int + audio_ssrc: int | None = None + candidates: Sequence[str] = () + session_id: int = 0 + session_version: int = 2 + + +@dataclass +class MediaDescription: + """One parsed m-section.""" + + kind: str + port: int + protocol: str + formats: list[str] + mid: str | None = None + direction: str = "sendrecv" + rtpmap: dict[int, tuple[str, int, int | None]] = field(default_factory=dict) + fmtp: dict[int, str] = field(default_factory=dict) + ssrcs: dict[int, None] = field(default_factory=dict) + candidates: list[str] = field(default_factory=list) + rids: list[str] = field(default_factory=list) + ice_ufrag: str | None = None + ice_pwd: str | None = None + fingerprints: list[tuple[str, str]] = field(default_factory=list) + setup: str | None = None + + @property + def payload_types(self) -> list[int]: + """The numeric formats of an RTP section.""" + return [int(fmt) for fmt in self.formats if fmt.isascii() and fmt.isdigit()] + + def payload_type_for( + self, codec: str, clock_rate: int, channels: int | None = None + ) -> int | None: + """Return the first payload type mapped to a codec, if any.""" + for payload_type in self.payload_types: + mapping = self.rtpmap.get(payload_type) + if ( + mapping + and mapping[0].lower() == codec.lower() + and mapping[1] == clock_rate + and (channels is None or (mapping[2] or 1) == channels) + ): + return payload_type + return None + + +@dataclass +class SessionDescription: + """A parsed SDP with transport attributes resolved per section.""" + + media: list[MediaDescription] + bundle: list[str] = field(default_factory=list) + ice_lite: bool = False + + def transport(self) -> MediaDescription: + """Return the section whose transport the session uses. + + With BUNDLE that is the first section of the group, otherwise the + first section with a non-zero port. + """ + for media in self.media: + if media.port and (not self.bundle or media.mid == self.bundle[0]): + return media + raise SdpError("SDP has no usable media section") + + @property + def candidates(self) -> list[str]: + """Candidates of the transport sections, without duplicates.""" + seen: dict[str, None] = {} + bundle = set(self.bundle) + for media in self.media: + if media.port and (not bundle or media.mid in bundle): + seen.update(dict.fromkeys(media.candidates)) + return list(seen)[:MAX_CANDIDATES] + + +def _checked_size(sdp: str) -> str: + if len(sdp) > MAX_SDP_SIZE: + raise SdpError(f"SDP larger than {MAX_SDP_SIZE} characters") + return sdp + + +def _split_lines(sdp: str) -> list[str]: + return [line for line in sdp.replace("\r\n", "\n").split("\n") if line] + + +@dataclass +class _SessionAttributes: + ice_ufrag: str | None = None + ice_pwd: str | None = None + setup: str | None = None + fingerprints: list[tuple[str, str]] = field(default_factory=list) + + +def parse(sdp: str) -> SessionDescription: + """Parse an SDP; ICE and DTLS attributes fall back to session level. + + An SDP over :data:`MAX_SDP_SIZE` characters or with more than + :data:`MAX_MEDIA_SECTIONS` media sections is rejected. + """ + session = _SessionAttributes() + desc = SessionDescription(media=[]) + current: MediaDescription | None = None + for line in _split_lines(_checked_size(sdp)): + if line.startswith("m="): + if len(desc.media) == MAX_MEDIA_SECTIONS: + raise SdpError("Too many media sections") + current = _media_line(line) + desc.media.append(current) + continue + if not line.startswith("a="): + continue + name, _, value = line[2:].partition(":") + try: + if current is None: + _session_attribute(desc, session, name, value) + else: + _media_attribute(current, name, value) + except (ValueError, IndexError) as err: + raise SdpError(f"Invalid attribute: {line[:80]!r}") from err + for media in desc.media: + media.ice_ufrag = media.ice_ufrag or session.ice_ufrag + media.ice_pwd = media.ice_pwd or session.ice_pwd + media.setup = media.setup or session.setup + media.fingerprints = media.fingerprints or list(session.fingerprints) + return desc + + +def _media_line(line: str) -> MediaDescription: + parts = line[2:].split() + try: + return MediaDescription( + kind=parts[0], + port=int(parts[1].split("/")[0]), + protocol=parts[2], + formats=parts[3:], + ) + except (ValueError, IndexError) as err: + raise SdpError(f"Invalid m-line: {line[:80]!r}") from err + + +def _fingerprint(value: str) -> tuple[str, str]: + algorithm, _, digest = value.partition(" ") + return algorithm.lower(), digest.strip().upper() + + +def _session_attribute( + desc: SessionDescription, session: _SessionAttributes, name: str, value: str +) -> None: + if name == "group" and value.startswith("BUNDLE"): + desc.bundle = value.split()[1:] + elif name == "ice-lite": + desc.ice_lite = True + elif name == "ice-ufrag": + session.ice_ufrag = value.strip() + elif name == "ice-pwd": + session.ice_pwd = value.strip() + elif name == "setup": + session.setup = value.strip() + elif name == "fingerprint": + session.fingerprints.append(_fingerprint(value)) + + +def _media_attribute(media: MediaDescription, name: str, value: str) -> None: + if name == "mid": + media.mid = value.strip() + elif name in DIRECTIONS and not value: + media.direction = name + elif name == "rtpmap": + pt, _, codec = value.partition(" ") + codec_parts = codec.strip().split("/") + media.rtpmap[int(pt)] = ( + codec_parts[0], + int(codec_parts[1]) if len(codec_parts) > 1 else 0, + int(codec_parts[2]) if len(codec_parts) > 2 else None, + ) + elif name == "fmtp": + pt, _, params = value.partition(" ") + media.fmtp[int(pt)] = params.strip() + elif name == "ssrc": + media.ssrcs[int(value.split(maxsplit=1)[0])] = None + elif name == "candidate" and len(media.candidates) < MAX_CANDIDATES: + media.candidates.append(value.strip()) + elif name == "rid": + media.rids.append(value.strip()) + elif name == "ice-ufrag": + media.ice_ufrag = value.strip() + elif name == "ice-pwd": + media.ice_pwd = value.strip() + elif name == "setup": + media.setup = value.strip() + elif name == "fingerprint": + media.fingerprints.append(_fingerprint(value)) + + +def candidate_value(candidate: str) -> str: + """Strip ``a=`` and ``candidate:`` from a candidate string.""" + value = candidate.strip() + value = value.removeprefix("a=") + return value.removeprefix("candidate:") + + +def merge_candidates(sdp: str, candidates: Iterable[IceCandidateLike]) -> str: + """Add HAP-signaled candidates to the m-sections they belong to. + + A candidate goes to the section with its mid, else to its m-line index, + else to the only section. Candidates already present are not repeated, + and a section takes at most :data:`MAX_CANDIDATES`; once every section is + full the rest is not looked at. + """ + lines = _split_lines(_checked_size(sdp)) + starts = [i for i, line in enumerate(lines) if line.startswith("m=")] + bounds = [*starts, len(lines)] + media = [lines[start:end] for start, end in pairwise(bounds)] + mids: dict[str, int] = {} + counts = [0] * len(media) + present: set[str] = set() + for index, section in enumerate(media): + for line in section: + if line.startswith("a=mid:"): + mids.setdefault(line[6:], index) + elif line.startswith("a=candidate:"): + present.add(line) + counts[index] += 1 + room = sum(max(0, MAX_CANDIDATES - count) for count in counts) + added: list[list[str]] = [[] for _ in media] + for candidate in candidates: + if not room: + break + if not candidate.candidate: + continue + if "\r" in candidate.candidate or "\n" in candidate.candidate: + raise SdpError("Invalid ICE candidate") + line = "a=candidate:" + candidate_value(candidate.candidate) + if line in present: + continue + index = _candidate_section(mids, len(media), candidate) + if counts[index] >= MAX_CANDIDATES: + continue + present.add(line) + counts[index] += 1 + room -= 1 + added[index].append(line) + for section, lines_added in zip(media, added, strict=True): + end = ( + section.index("a=end-of-candidates") + if "a=end-of-candidates" in section + else len(section) + ) + section[end:end] = lines_added + out = lines[: bounds[0]] + [line for section in media for line in section] + return "\r\n".join(out) + "\r\n" + + +def _candidate_section( + mids: dict[str, int], count: int, candidate: IceCandidateLike +) -> int: + index: int | None + if candidate.sdp_mid is not None: + index = mids.get(candidate.sdp_mid) + if index is None or candidate.sdp_mline_index not in (None, index): + raise SdpError("ICE candidate does not match a media section") + return index + index = candidate.sdp_mline_index + if index is None and count == 1: + index = 0 + if index is None or not 0 <= index < count: + raise SdpError("ICE candidate does not match a media section") + return index + + +def _default_address(candidates: Sequence[str]) -> tuple[str, int]: + """Return the connection address of the first candidate (RFC 8839 4.2.1.2).""" + for candidate in candidates: + parts = candidate_value(candidate).split() + if len(parts) >= 6: + return parts[4], int(parts[5]) + return "0.0.0.0", 9 # noqa: S104 - SDP placeholder address, not a bind + + +def _connection_line(address: str) -> str: + family = "IP6" if ":" in address else "IP4" + return f"c=IN {family} {address}" + + +def _transport_lines(local: LocalParams, setup: str) -> list[str]: + lines = [f"a=candidate:{candidate_value(c)}" for c in local.candidates] + if local.candidates: + lines.append("a=end-of-candidates") + return [ + *lines, + f"a=ice-ufrag:{local.ice_ufrag}", + f"a=ice-pwd:{local.ice_pwd}", + f"a=fingerprint:sha-256 {local.fingerprint}", + f"a=setup:{setup}", + ] + + +def _ssrc_lines(local: LocalParams, ssrc: int, track: str) -> list[str]: + return [ + f"a=msid:{local.stream_id} {track}", + f"a=ssrc:{ssrc} cname:{local.cname}", + f"a=ssrc:{ssrc} msid:{local.stream_id} {track}", + ] + + +def rid_line(video: VideoParams) -> str: + """Return the RID restriction line for the video tier.""" + return ( + f"a=rid:{RID} send max-width={video.width};max-height={video.height};" + f"max-fps={video.fps};max-br={video.peak_bitrate_kbps * 1000}" + ) + + +def _video_section( + local: LocalParams, + video: VideoParams, + payload_type: int, + mid: str, + direction: str, + setup: str, +) -> list[str]: + address, port = _default_address(local.candidates) + pt = payload_type + return [ + f"m=video {port} UDP/TLS/RTP/SAVPF {pt}", + _connection_line(address), + *_transport_lines(local, setup), + f"a=mid:{mid}", + f"a={direction}", + "a=rtcp-mux", + "a=rtcp-rsize", + f"a=rtpmap:{pt} H265/{VIDEO_CLOCK_RATE}", + f"a=rtcp-fb:{pt} nack", + f"a=rtcp-fb:{pt} nack pli", + f"a=rtcp-fb:{pt} ccm fir", + rid_line(video), + *_ssrc_lines(local, local.video_ssrc, "video"), + ] + + +def _audio_section( + local: LocalParams, payload_type: int, mid: str, direction: str, setup: str +) -> list[str]: + address, port = _default_address(local.candidates) + pt = payload_type + lines = [ + f"m=audio {port} UDP/TLS/RTP/SAVPF {pt}", + _connection_line(address), + *_transport_lines(local, setup), + f"a=mid:{mid}", + f"a={direction}", + "a=rtcp-mux", + "a=rtcp-rsize", + f"a=rtpmap:{pt} opus/{AUDIO_CLOCK_RATE}/2", + f"a=fmtp:{pt} minptime=20;useinbandfec=1;stereo=0;sprop-stereo=0", + f"a=ptime:{OPUS_PTIME}", + ] + if local.audio_ssrc is not None and direction in ("sendonly", "sendrecv"): + lines += _ssrc_lines(local, local.audio_ssrc, "audio") + return lines + + +def _session_lines(local: LocalParams, mids: Sequence[str]) -> list[str]: + return [ + "v=0", + f"o=- {local.session_id} {local.session_version} IN IP4 127.0.0.1", + "s=-", + "t=0 0", + f"a=group:BUNDLE {' '.join(mids)}", + f"a=msid-semantic: WMS {local.stream_id}", + ] + + +def build_offer( + local: LocalParams, video: VideoParams, audio: AudioParams | None +) -> str: + """Build the accessory's offer.""" + mids = [VIDEO_MID] + ([AUDIO_MID] if audio else []) + lines = _session_lines(local, mids) + lines += _video_section( + local, video, video.payload_type, VIDEO_MID, "sendonly", "actpass" + ) + if audio: + direction = "sendrecv" if audio.talkback else "sendonly" + lines += _audio_section( + local, audio.payload_type, AUDIO_MID, direction, "actpass" + ) + return "\r\n".join(lines) + "\r\n" + + +_ANSWER_DIRECTION = { + # (offered direction, we may receive) -> answered direction + ("sendrecv", True): "sendrecv", + ("sendrecv", False): "sendonly", + ("recvonly", True): "sendonly", + ("recvonly", False): "sendonly", + ("sendonly", True): "recvonly", + ("sendonly", False): "inactive", + ("inactive", True): "inactive", + ("inactive", False): "inactive", +} + + +@dataclass(frozen=True) +class Answer: + """An answer to a controller's offer and the payload types it selected.""" + + sdp: str + video_payload_type: int | None + audio_payload_type: int | None + video_mid: str | None + audio_mid: str | None + audio_direction: str | None = None + + +def build_answer( + offer: SessionDescription, + local: LocalParams, + setup: str, + video: VideoParams, + audio: AudioParams | None, +) -> Answer: + """Answer a controller's re-offer with the established transport. + + The first H.265 video section and the first Opus audio section are + accepted, every other section is rejected with port 0. + """ + accepted: list[str] = [] + body: list[str] = [] + video_pt = audio_pt = None + video_mid = audio_mid = audio_direction = None + for index, media in enumerate(offer.media): + mid = media.mid if media.mid is not None else str(index) + pt = None + if media.kind == "video" and video_pt is None and media.port: + pt = media.payload_type_for("H265", VIDEO_CLOCK_RATE) + elif media.kind == "audio" and audio and audio_pt is None and media.port: + pt = media.payload_type_for("opus", AUDIO_CLOCK_RATE, 2) + if pt is None: + body += _rejected_section(media, mid) + continue + accepted.append(mid) + if media.kind == "video": + video_pt, video_mid = pt, mid + direction = _ANSWER_DIRECTION[(media.direction, False)] + body += _video_section(local, video, pt, mid, direction, setup) + else: + audio_pt, audio_mid = pt, mid + talkback = bool(audio and audio.talkback) + direction = audio_direction = _ANSWER_DIRECTION[(media.direction, talkback)] + body += _audio_section(local, pt, mid, direction, setup) + if video_pt is None: + raise SdpError("Offer has no H.265 video") + lines = _session_lines(local, accepted) + body + return Answer( + "\r\n".join(lines) + "\r\n", + video_pt, + audio_pt, + video_mid, + audio_mid, + audio_direction, + ) + + +def _rejected_section(media: MediaDescription, mid: str) -> list[str]: + formats = " ".join(media.formats) or "0" + return [ + f"m={media.kind} 0 {media.protocol} {formats}", + "c=IN IP4 0.0.0.0", + f"a=mid:{mid}", + "a=inactive", + ] diff --git a/pyhap/media/sender.py b/pyhap/media/sender.py new file mode 100644 index 00000000..f39417dd --- /dev/null +++ b/pyhap/media/sender.py @@ -0,0 +1,802 @@ +"""A minimal WebRTC sender for HomeKit camera remote live view. + +The integrator runs a media pull per camera channel and sends plain RTP to +the loopback ports of a :class:`~pyhap.media.source.MediaSource`, for +example:: + + ffmpeg -i + -map 0:v -c:v copy -bsf:v dump_extra -f rtp rtp://127.0.0.1: