From 20064ce580a674e3c61d4556f7957244256cd44c Mon Sep 17 00:00:00 2001 From: Simon Hamp Date: Wed, 28 Oct 2026 20:59:04 +0000 Subject: [PATCH 1/6] Add the microphone and media player plugins for voice notes nativephp/mobile-microphone records them and nativephp/mobile-media-player plays them. iOS asks for the microphone in devnull's words. Co-Authored-By: Claude Opus 5.5 --- app/Providers/NativeServiceProvider.php | 4 + composer.json | 2 + composer.lock | 112 +++++++++++++++++++++++- config/nativephp.php | 1 + 4 files changed, 118 insertions(+), 1 deletion(-) diff --git a/app/Providers/NativeServiceProvider.php b/app/Providers/NativeServiceProvider.php index dfdf29b8..16703314 100644 --- a/app/Providers/NativeServiceProvider.php +++ b/app/Providers/NativeServiceProvider.php @@ -6,9 +6,11 @@ use Illuminate\Support\ServiceProvider; use Native\Mobile\Providers\BrowserServiceProvider; use Native\Mobile\Providers\CameraServiceProvider; +use Native\Mobile\Providers\MicrophoneServiceProvider; use Native\Mobile\Providers\SecureStorageServiceProvider; use Native\Mobile\Providers\ShareServiceProvider; use Native\Mobile\UI\NativeUIServiceProvider; +use NativePHP\MediaPlayer\MediaPlayerServiceProvider; use S2BR\MobileSplashscreen\MobileSplashscreenServiceProvider; use Victorycodedev\ToastKit\ToastKitServiceProvider; @@ -48,6 +50,8 @@ public function plugins(): array NativeUIServiceProvider::class, BrowserServiceProvider::class, CameraServiceProvider::class, + MicrophoneServiceProvider::class, + MediaPlayerServiceProvider::class, SecureStorageServiceProvider::class, ShareServiceProvider::class, MobileSplashscreenServiceProvider::class, diff --git a/composer.json b/composer.json index ebda5842..b127b241 100644 --- a/composer.json +++ b/composer.json @@ -38,6 +38,8 @@ "nativephp/mobile": "dev-main as 4.6.0", "nativephp/mobile-browser": "^1.2", "nativephp/mobile-camera": "^1.1", + "nativephp/mobile-media-player": "^1.0", + "nativephp/mobile-microphone": "^1.1", "nativephp/mobile-secure-storage": "^1.1", "nativephp/mobile-share": "^1.1", "nativephp/mobile-ui": "dev-main as 0.8.0", diff --git a/composer.lock b/composer.lock index fe720515..39fa58ea 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "4c795f7f152e1855d8c1e257076e3f74", + "content-hash": "a6e34837e083ae6eb8cc1df7ee22845e", "packages": [ { "name": "bacon/bacon-qr-code", @@ -2951,6 +2951,116 @@ }, "time": "2026-10-22T15:11:58+00:00" }, + { + "name": "nativephp/mobile-media-player", + "version": "1.0.2", + "source": { + "type": "git", + "url": "https://github.com/NativePHP/mobile-media-player.git", + "reference": "ff249aaa50de0623edd3faeffeeeec9d15124869" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/NativePHP/mobile-media-player/zipball/ff249aaa50de0623edd3faeffeeeec9d15124869", + "reference": "ff249aaa50de0623edd3faeffeeeec9d15124869", + "shasum": "" + }, + "require": { + "nativephp/mobile": "^3.0 || ^4.0", + "php": "^8.2" + }, + "require-dev": { + "orchestra/testbench": "^10.0", + "pestphp/pest": "^2.7|^3.0|^4.0" + }, + "type": "nativephp-plugin", + "extra": { + "laravel": { + "providers": [ + "NativePHP\\MediaPlayer\\MediaPlayerServiceProvider" + ] + }, + "nativephp": { + "manifest": "nativephp.json" + } + }, + "autoload": { + "psr-4": { + "NativePHP\\MediaPlayer\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Bifrost Technology, LLC", + "email": "support@nativephp.com" + } + ], + "description": "Media player plugin for NativePHP Mobile - shared audio/video playback, full-screen presentation, and a video-player element", + "support": { + "issues": "https://github.com/NativePHP/mobile-media-player/issues", + "source": "https://github.com/NativePHP/mobile-media-player/tree/1.0.2" + }, + "time": "2026-09-15T18:59:52+00:00" + }, + { + "name": "nativephp/mobile-microphone", + "version": "1.1.0", + "source": { + "type": "git", + "url": "https://github.com/NativePHP/mobile-microphone.git", + "reference": "94603d36388fb9b9f897fe146c0b8555e9c3c918" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/NativePHP/mobile-microphone/zipball/94603d36388fb9b9f897fe146c0b8555e9c3c918", + "reference": "94603d36388fb9b9f897fe146c0b8555e9c3c918", + "shasum": "" + }, + "require": { + "nativephp/mobile": "^3.0 || ^4.0", + "php": "^8.2" + }, + "require-dev": { + "orchestra/testbench": "^10.0", + "pestphp/pest": "^2.7|^3.0|^4.0" + }, + "type": "nativephp-plugin", + "extra": { + "laravel": { + "providers": [ + "Native\\Mobile\\Providers\\MicrophoneServiceProvider" + ] + }, + "nativephp": { + "manifest": "nativephp.json" + } + }, + "autoload": { + "psr-4": { + "Native\\Mobile\\Providers\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "NativePHP", + "email": "support@nativephp.com" + } + ], + "description": "Microphone recording plugin for NativePHP Mobile - audio recording with pause/resume support", + "support": { + "issues": "https://github.com/NativePHP/mobile-microphone/issues", + "source": "https://github.com/NativePHP/mobile-microphone/tree/1.1.0" + }, + "time": "2026-09-03T18:26:19+00:00" + }, { "name": "nativephp/mobile-secure-storage", "version": "1.1.0", diff --git a/config/nativephp.php b/config/nativephp.php index 90b5540e..9941ccc2 100644 --- a/config/nativephp.php +++ b/config/nativephp.php @@ -112,6 +112,7 @@ 'permissions' => [ 'NSCameraUsageDescription' => 'devnull uses the camera when you take a photo to attach to a commit.', 'NSPhotoLibraryUsageDescription' => 'devnull opens your photos when you pick some to attach to a commit.', + 'NSMicrophoneUsageDescription' => 'devnull uses the microphone when you record a voice note in Remotes.', ], /* From 72c809f7256f9c783c25ab75208107d81633826a Mon Sep 17 00:00:00 2001 From: Simon Hamp Date: Wed, 28 Oct 2026 21:18:27 +0000 Subject: [PATCH 2/6] Record and play voice notes in Remotes threads, as the web does The mic sits between photos and Ping, off while there's text or photos. It records 16 kHz WAV for up to a minute in the field's place, with the level and a clock, then shows the recording to listen to before Push sends it. Push while recording sends it as soon as the microphone hands it over. A voice note goes through the outbox like any message: staged at once, pushed, or rejected with retry, its recording kept on the phone until devnull has it. Voice notes play in the thread with play and pause, the waveform with what has played, a tap on it to play from there, the clock and 1x, 1.5x and 2x. They download from the thread's signed URL on first play. One plays at a time, and leaving the thread stops it. The inbox says "sent a voice note" and a reply's ghost says "Voice note", as the web words them. Co-Authored-By: Claude Opus 5.5 --- app/Devnull/Account.php | 5 +- app/Devnull/Client.php | 50 +- app/Devnull/Data/Message.php | 49 +- app/Devnull/Data/MessageVoice.php | 60 +++ app/Devnull/DownloadVoiceNote.php | 54 ++ app/Devnull/Exceptions/InvalidVoiceNote.php | 10 + app/Devnull/VoiceNotes.php | 302 +++++++++++ app/Devnull/VoicePlayback.php | 119 +++++ app/Icons/Lucide.php | 10 + app/Models/PendingWrite.php | 10 + .../Concerns/PlaysVoiceNotes.php | 270 ++++++++++ .../Concerns/RecordsVoiceNotes.php | 260 +++++++++ app/NativeComponents/Concerns/SyncsOutbox.php | 9 +- app/NativeComponents/Conversation.php | 120 ++++- resources/icons/mic.svg | 1 + resources/icons/pause-filled.svg | 1 + resources/icons/play-filled.svg | 1 + resources/icons/rotate-cw.svg | 1 + resources/icons/square-filled.svg | 1 + resources/views/native/conversation.blade.php | 15 +- .../native/partials/message-item.blade.php | 2 + .../native/partials/message-voice.blade.php | 25 + .../native/partials/pinned-message.blade.php | 8 +- .../native/partials/voice-note.blade.php | 51 ++ .../native/partials/voice-recorder.blade.php | 40 ++ tests/Feature/Devnull/ClientTest.php | 23 + tests/Feature/Screens/RemotesComposerTest.php | 8 +- .../Feature/Screens/RemotesVoiceNotesTest.php | 497 ++++++++++++++++++ 28 files changed, 1962 insertions(+), 40 deletions(-) create mode 100644 app/Devnull/Data/MessageVoice.php create mode 100644 app/Devnull/DownloadVoiceNote.php create mode 100644 app/Devnull/Exceptions/InvalidVoiceNote.php create mode 100644 app/Devnull/VoiceNotes.php create mode 100644 app/Devnull/VoicePlayback.php create mode 100644 app/NativeComponents/Concerns/PlaysVoiceNotes.php create mode 100644 app/NativeComponents/Concerns/RecordsVoiceNotes.php create mode 100644 resources/icons/mic.svg create mode 100644 resources/icons/pause-filled.svg create mode 100644 resources/icons/play-filled.svg create mode 100644 resources/icons/rotate-cw.svg create mode 100644 resources/icons/square-filled.svg create mode 100644 resources/views/native/partials/message-voice.blade.php create mode 100644 resources/views/native/partials/voice-note.blade.php create mode 100644 resources/views/native/partials/voice-recorder.blade.php create mode 100644 tests/Feature/Screens/RemotesVoiceNotesTest.php diff --git a/app/Devnull/Account.php b/app/Devnull/Account.php index f52a7866..e2ba3138 100644 --- a/app/Devnull/Account.php +++ b/app/Devnull/Account.php @@ -168,13 +168,14 @@ public function rememberSkipsPinWarning(): void } /** - * Signs this phone out: who it was, the badges, its pastéis and Remotes, every answer and image kept for a - * quick start, and your unsent drafts. + * Signs this phone out: who it was, the badges, its pastéis and Remotes, every answer, image and voice note kept + * for a quick start, and your unsent drafts and recordings. */ public function forget(): void { app(ResponseCache::class)->flush(); ImageCache::clear(); + VoiceNotes::clear(); app(Stash::class)->forget($this->user()?->id); app(Outbox::class)->forgetAll($this->user()?->id); diff --git a/app/Devnull/Client.php b/app/Devnull/Client.php index 0449451e..3ab0a4c4 100644 --- a/app/Devnull/Client.php +++ b/app/Devnull/Client.php @@ -7,6 +7,7 @@ use App\Devnull\Data\Connection; use App\Devnull\Data\CurrentUser; use App\Devnull\Data\DraftImage; +use App\Devnull\Data\MessageVoice; use App\Devnull\Data\Notification; use App\Devnull\Data\Page; use App\Devnull\Data\Profile; @@ -270,17 +271,20 @@ public function signedCommitWithImages(string $body, array $images, ?string $par * Signs a message for a Remotes thread without sending it, for the outbox * to send from the background. devnull stores it once per client id, so * it can go again. One with photos goes as a commit's do (see - * signedWithImages()), and may have no text. + * signedWithImages()), and may have no text. A voice note goes the same + * way, alone, with its waveform and no text (see signedWithVoice()). * * @param list $images */ - public function signedMessage(string $threadPath, string $body, string $clientId, ?int $replyToId = null, array $images = []): SignedRequest + public function signedMessage(string $threadPath, string $body, string $clientId, ?int $replyToId = null, array $images = [], ?MessageVoice $voice = null): SignedRequest { $payload = array_filter(['body' => $body, 'client_id' => $clientId, 'reply_to_id' => $replyToId], fn (mixed $value): bool => $value !== null); - return $images === [] - ? $this->signedWrite('POST', $threadPath.'/messages', $payload) - : $this->signedWithImages($threadPath.'/messages', $payload, $images); + return match (true) { + $voice !== null => $this->signedWithVoice($threadPath.'/messages', $clientId, $replyToId, $voice), + $images !== [] => $this->signedWithImages($threadPath.'/messages', $payload, $images), + default => $this->signedWrite('POST', $threadPath.'/messages', $payload), + }; } public function deleteCommit(string $id): void @@ -591,7 +595,35 @@ private function signedWithImages(string $path, array $payload, array $images): $target = $this->target($path, []); $descriptions = array_map(fn (DraftImage $image): ?string => $image->isDescribed() ? trim($image->description) : null, $images); $payload = self::body(array_filter($descriptions) === [] ? $payload : [...$payload, 'image_descriptions' => $descriptions]); - $contents = array_map(fn (DraftImage $image): string => (string) file_get_contents($image->path), $images); + $files = array_map(fn (DraftImage $image, int $index): array => [ + 'images[]', 'image-'.($index + 1).'.'.substr($image->mimeType, 6), $image->mimeType, (string) file_get_contents($image->path), + ], $images, array_keys($images)); + + return $this->signedMultipart($target, $payload, $files); + } + + /** + * Signs a voice note, as devnull takes one: multipart like a write with + * images, the recording alone in a voice part, its SHA-256 the line + * signed before the body, and the waveform the phone drew from it in the + * payload, since devnull never decodes it. + */ + private function signedWithVoice(string $path, string $clientId, ?int $replyToId, MessageVoice $voice): SignedRequest + { + $payload = array_filter(['client_id' => $clientId, 'reply_to_id' => $replyToId, 'voice_waveform' => $voice->waveform], fn (mixed $value): bool => $value !== null); + + return $this->signedMultipart($this->target($path, []), self::body($payload), [ + ['voice', 'voice-note.wav', 'audio/wav', (string) file_get_contents($voice->url)], + ]); + } + + /** + * The payload part, then each file, with the files' SHA-256 signed on a line before the body. + * + * @param list $files each one's field, filename, type and bytes + */ + private function signedMultipart(string $target, string $payload, array $files): SignedRequest + { $boundary = 'devnull-'.bin2hex(random_bytes(16)); $part = fn (string $name, string $filename, string $type, string $bytes): string => "--{$boundary}\r\n" @@ -600,11 +632,11 @@ private function signedWithImages(string $path, array $payload, array $images): $body = $part('payload', 'payload.json', 'application/json', $payload); - foreach ($images as $index => $image) { - $body .= $part('images[]', 'image-'.($index + 1).'.'.substr($image->mimeType, 6), $image->mimeType, $contents[$index]); + foreach ($files as [$name, $filename, $type, $bytes]) { + $body .= $part($name, $filename, $type, $bytes); } - $hashes = implode(',', array_map(fn (string $bytes): string => hash('sha256', $bytes), $contents)); + $hashes = implode(',', array_map(fn (array $file): string => hash('sha256', $file[3]), $files)); return new SignedRequest( $this->origin().$target, diff --git a/app/Devnull/Data/Message.php b/app/Devnull/Data/Message.php index f2674a78..aa6e273d 100644 --- a/app/Devnull/Data/Message.php +++ b/app/Devnull/Data/Message.php @@ -13,10 +13,13 @@ * logs who joined and left as messages with an event. A ping, as MSN * Messenger had, is an event too, in an org or with one person. * + * A voice note goes on its own, with no text or images, and plays from the + * URL devnull signed for you (see MessageVoice). + * * A message you've just pushed shows straight away, as the web's do: staged * until devnull has it, or rejected with a retry when it couldn't take it, - * its photos drawn from the phone meanwhile. Until then its id is 0 and its - * client id is what ties the two together. + * its photos or voice note drawn from the phone meanwhile. Until then its id + * is 0 and its client id is what ties the two together. */ final readonly class Message { @@ -33,9 +36,14 @@ public const string Ping = 'ping'; + /** + * The key of the voice note waiting in the composer, which no message has (see voiceKey()). + */ + public const string DraftVoice = 'draft'; + /** * @param list $content - * @param array{id: int, user_id: string, body: string, images_count: int}|null $replyTo + * @param array{id: int, user_id: string, body: string, images_count: int, has_voice: bool}|null $replyTo * @param list $images * @param list $reactions */ @@ -50,6 +58,7 @@ public function __construct( public ?LinkPreview $linkPreview = null, public bool $linkPreviewPending = false, public array $images = [], + public ?MessageVoice $voice = null, public array $reactions = [], public ?string $event = null, public ?string $subjectId = null, @@ -87,6 +96,7 @@ public static function fromArray(array $data): self 'user_id' => (string) ($replyTo['user_id'] ?? ''), 'body' => is_string($replyTo['body'] ?? null) ? $replyTo['body'] : '', 'images_count' => (int) ($replyTo['images_count'] ?? 0), + 'has_voice' => ($replyTo['has_voice'] ?? false) === true, ], linkPreview: $preview === null ? null : LinkPreview::fromArray($preview), linkPreviewPending: ($preview['is_pending'] ?? false) === true, @@ -94,6 +104,7 @@ public static function fromArray(array $data): self fn (mixed $image): ?CommitImage => is_array($image) ? CommitImage::fromArray($image) : null, is_array($data['images'] ?? null) ? $data['images'] : [], ))), + voice: MessageVoice::fromArray($data['voice'] ?? null), reactions: self::reactionsFrom($data['reactions'] ?? []), event: is_string($data['event'] ?? null) ? $data['event'] : null, subjectId: is_string($data['subject_id'] ?? null) ? $data['subject_id'] : null, @@ -108,7 +119,7 @@ public static function fromArray(array $data): self * * @param list $images */ - public static function staged(string $userId, string $clientId, string $body, ?self $replyTo = null, array $images = []): self + public static function staged(string $userId, string $clientId, string $body, ?self $replyTo = null, array $images = [], ?MessageVoice $voice = null): self { return new self( id: 0, @@ -117,8 +128,9 @@ public static function staged(string $userId, string $clientId, string $body, ?s body: $body, content: CommitBody::blocks($body, null), replyToId: $replyTo?->id, - replyTo: $replyTo === null ? null : ['id' => $replyTo->id, 'user_id' => $replyTo->userId, 'body' => $replyTo->body, 'images_count' => count($replyTo->images)], + replyTo: $replyTo === null ? null : ['id' => $replyTo->id, 'user_id' => $replyTo->userId, 'body' => $replyTo->body, 'images_count' => count($replyTo->images), 'has_voice' => $replyTo->voice !== null], images: array_map(fn (DraftImage $image, int $index): CommitImage => $image->asCommitImage($index), $images, array_keys($images)), + voice: $voice, createdAt: CarbonImmutable::now(), status: self::Staged, ); @@ -157,6 +169,15 @@ public function withLinkPreview(?LinkPreview $linkPreview, bool $pending): self return new self(...[...get_object_vars($this), 'linkPreview' => $linkPreview, 'linkPreviewPending' => $pending]); } + /** + * What the thread knows its voice note by while it plays: its client id, which a message you pushed keeps once + * devnull has it. + */ + public function voiceKey(): string + { + return $this->clientId !== '' ? $this->clientId : (string) $this->id; + } + public function isEvent(): bool { return $this->event !== null; @@ -204,8 +225,8 @@ public function hasReacted(?string $viewerId, string $emoji): bool /** * The one line a row of the inbox shows of the last message, as the web words it: "you: …" for yours, the first - * name in an org for someone else's, "sent an image" for a message that is only images, and the event for an - * org's log line. + * name in an org for someone else's, "sent a voice note" or "sent an image" for a message with no text, and the + * event for an org's log line. */ public function preview(?string $viewerId, bool $inOrg): string { @@ -214,7 +235,7 @@ public function preview(?string $viewerId, bool $inOrg): string } $text = trim((string) preg_replace('/\s+/', ' ', $this->body)); - $text = $text === '' ? 'sent an image' : $text; + $text = $text === '' ? ($this->voice !== null ? 'sent a voice note' : 'sent an image') : $text; return match (true) { $this->userId === $viewerId => 'you: '.$text, @@ -252,11 +273,11 @@ public function eventText(?string $viewerId, ?string $orgName = null): string /** * What the ghost of a reply shows of the message it answers: one line of its body with the code and links - * stripped, or that it was only images. + * stripped, or that it was a voice note or only images. */ public function replySnippet(): string { - return $this->replyTo === null ? '' : self::snippetOf($this->replyTo['body'], $this->replyTo['images_count']); + return $this->replyTo === null ? '' : self::snippetOf($this->replyTo['body'], $this->replyTo['images_count'], $this->replyTo['has_voice'] ?? false); } /** @@ -264,11 +285,15 @@ public function replySnippet(): string */ public function snippet(): string { - return self::snippetOf($this->body, count($this->images)); + return self::snippetOf($this->body, count($this->images), $this->voice !== null); } - private static function snippetOf(string $body, int $images): string + private static function snippetOf(string $body, int $images, bool $hasVoice = false): string { + if ($hasVoice) { + return 'Voice note'; + } + $text = (string) preg_replace('/```[\s\S]*?```/', ' code ', $body); $text = trim((string) preg_replace('/\s+/', ' ', (string) preg_replace(['/`/', '#https?://#i'], '', $text))); diff --git a/app/Devnull/Data/MessageVoice.php b/app/Devnull/Data/MessageVoice.php new file mode 100644 index 00000000..ec3fd902 --- /dev/null +++ b/app/Devnull/Data/MessageVoice.php @@ -0,0 +1,60 @@ + $waveform VoiceNotes::Bars levels, 0 to 100 + */ + public function __construct(public string $id, public int $durationMs, public array $waveform, public string $url) {} + + /** + * A voice note as devnull lists it in a message, or null when it isn't one. + */ + public static function fromArray(mixed $voice): ?self + { + if (! is_array($voice) || ! is_string($voice['url'] ?? null) || ! is_int($voice['duration_ms'] ?? null)) { + return null; + } + + return new self((string) ($voice['id'] ?? ''), $voice['duration_ms'], VoiceNotes::bars($voice['waveform'] ?? []), $voice['url']); + } + + /** + * A recording on this phone, as the outbox keeps it, or null when its file has gone. + */ + public static function fromOutbox(mixed $voice): ?self + { + if (! is_array($voice) || ! is_string($voice['path'] ?? null) || ! is_file($voice['path'])) { + return null; + } + + return new self('', (int) ($voice['duration_ms'] ?? 0), VoiceNotes::bars($voice['waveform'] ?? []), $voice['path']); + } + + /** + * @return array{path: string, duration_ms: int, waveform: list} + */ + public function toOutbox(): array + { + return ['path' => $this->url, 'duration_ms' => $this->durationMs, 'waveform' => $this->waveform]; + } + + /** + * Whether it's a file on this phone rather than one devnull serves. + */ + public function isOnPhone(): bool + { + return str_starts_with($this->url, '/'); + } +} diff --git a/app/Devnull/DownloadVoiceNote.php b/app/Devnull/DownloadVoiceNote.php new file mode 100644 index 00000000..8eb528ae --- /dev/null +++ b/app/Devnull/DownloadVoiceNote.php @@ -0,0 +1,54 @@ + + */ + private const array Extensions = ['audio/mp4' => 'mp4', 'audio/wav' => 'wav']; + + public function handle(string $url, string $name): ?string + { + File::ensureDirectoryExists(VoiceNotes::cacheDirectory()); + VoiceNotes::pruneCache(); + + try { + $response = Http::timeout(self::Timeout)->accept('audio/*')->get($url); + } catch (ConnectionException) { + return null; + } + + $type = strtolower(trim(explode(';', (string) $response->header('Content-Type'))[0])); + + // Signing out empties the folder (see Account::forget), and this thread can't tell it happened otherwise. + if (! $response->successful() || ! isset(self::Extensions[$type]) || app(Account::class)->user() === null) { + return null; + } + + $path = VoiceNotes::cacheDirectory().'/'.$name.'.'.self::Extensions[$type]; + + File::put($path.'.part', $response->body()); + File::move($path.'.part', $path); + + return $path; + } +} diff --git a/app/Devnull/Exceptions/InvalidVoiceNote.php b/app/Devnull/Exceptions/InvalidVoiceNote.php new file mode 100644 index 00000000..bbedd0f3 --- /dev/null +++ b/app/Devnull/Exceptions/InvalidVoiceNote.php @@ -0,0 +1,10 @@ + + */ + public const array Speeds = [1.0, 1.5, 2.0]; + + public const string TooShort = 'fatal: voice note shorter than half a second'; + + public const string Unreadable = 'fatal: could not read the recording'; + + public const string NotOne = 'Voice notes must be Opus WebM or 16 kHz mono WAV recordings between half a second and a minute long.'; + + public const string Denied = 'fatal: microphone access denied'; + + public const string CouldNotOpen = 'fatal: could not open the microphone'; + + /** + * How long a recording nothing holds any more is kept, and a downloaded voice note. + */ + private const int KeptForSeconds = 86_400; + + private const int CachedForSeconds = 7 * 86_400; + + /** + * Moves a recording the microphone made into the app's storage, as a voice + * note to push, with how long it plays and its waveform. + * + * @throws InvalidVoiceNote + */ + public function keep(string $path): MessageVoice + { + $wav = is_file($path) ? (string) file_get_contents($path) : ''; + $samples = self::samples($wav); + + try { + if ($samples === null) { + throw new InvalidVoiceNote(self::Unreadable); + } + + $milliseconds = intdiv(strlen($samples) * 1000, self::SampleRate * 2); + + if ($milliseconds < self::MinMilliseconds) { + throw new InvalidVoiceNote(self::TooShort); + } + + if ($milliseconds > self::AcceptedMilliseconds) { + throw new InvalidVoiceNote(self::NotOne); + } + } catch (InvalidVoiceNote $exception) { + @unlink($path); + + throw $exception; + } + + $this->prune(); + + $kept = $this->directory().'/'.bin2hex(random_bytes(16)).'.wav'; + + if (! @rename($path, $kept) && ! (@copy($path, $kept) && @unlink($path))) { + throw new InvalidVoiceNote(self::Unreadable); + } + + return new MessageVoice('', $milliseconds, self::waveformOf($samples), $kept); + } + + /** + * Deletes a recording that was pushed, or that you discarded. Voice notes devnull serves are left alone. + */ + public function discard(?MessageVoice $voice): void + { + if ($voice !== null && str_starts_with($voice->url, $this->directory().'/')) { + @unlink($voice->url); + } + } + + /** + * Where a voice note can be played from on this phone: the recording itself, or the copy downloaded of one + * devnull serves, or null until it's downloaded. + */ + public function playable(MessageVoice $voice): ?string + { + if ($voice->isOnPhone()) { + return is_file($voice->url) ? $voice->url : null; + } + + return array_find(glob(self::cacheDirectory().'/'.self::cacheName($voice).'.*') ?: [], fn (string $file): bool => ! str_ends_with($file, '.part')); + } + + /** + * The downloaded copies of voice notes, by their id. Each is named for what devnull serves it as, which is how + * the players tell MP4 from WAV. + */ + public static function cacheDirectory(): string + { + return storage_path('app/voice-notes-cache'); + } + + public static function cacheName(MessageVoice $voice): string + { + return preg_replace('/[^A-Za-z0-9-]/', '', $voice->id) ?: md5($voice->url); + } + + /** + * Deletes downloaded voice notes past a week, and any half-written ones. + */ + public static function pruneCache(): void + { + foreach (glob(self::cacheDirectory().'/*') ?: [] as $file) { + if (str_ends_with($file, '.part') || filemtime($file) < time() - self::CachedForSeconds) { + @unlink($file); + } + } + } + + /** + * Everything signing out leaves behind: recordings and downloaded voice notes. + */ + public static function clear(): void + { + File::deleteDirectory(storage_path('app/voice-notes')); + File::deleteDirectory(self::cacheDirectory()); + } + + /** + * A waveform as devnull lists it, or as the outbox keeps it: Bars levels from 0 to 100, flat where it's short. + * + * @return list + */ + public static function bars(mixed $levels): array + { + $levels = is_array($levels) ? array_values($levels) : []; + + return array_map(fn (int $bar): int => max(0, min(100, (int) ($levels[$bar] ?? 0))), range(0, self::Bars - 1)); + } + + /** + * The web's waveform of a WAV: the loudest sample of each slice, its square root, from 0 to 100. + * + * @return list + */ + public static function waveformOf(string $samples): array + { + $count = intdiv(strlen($samples), 2); + $bars = []; + + for ($bar = 0; $bar < self::Bars; $bar++) { + $from = intdiv($count * $bar, self::Bars); + $to = intdiv($count * ($bar + 1), self::Bars); + + // Phones are little-endian, as WAV is, so the machine's own signed shorts read it. + $slice = $to > $from ? unpack('s*', substr($samples, $from * 2, ($to - $from) * 2)) : []; + $peak = $slice === [] || $slice === false ? 0 : max(max($slice), -min($slice)); + + $bars[] = (int) round(sqrt(min($peak / 32768, 1)) * 100); + } + + return $bars; + } + + /** + * "0:07", "1:00", as the web's clock reads. + */ + public static function clock(int|float $milliseconds): string + { + $seconds = max(0, (int) floor($milliseconds / 1000)); + + return intdiv($seconds, 60).':'.str_pad((string) ($seconds % 60), 2, '0', STR_PAD_LEFT); + } + + /** + * The speed after this one, back to 1× after 2×. + */ + public static function nextSpeed(float $speed): float + { + $index = array_search($speed, self::Speeds, true); + + return self::Speeds[$index === false ? 0 : ($index + 1) % count(self::Speeds)]; + } + + /** + * "1×", "1.5×", "2×". + */ + public static function speedLabel(float $speed): string + { + return rtrim(rtrim(number_format($speed, 1), '0'), '.').'×'; + } + + /** + * The samples of a WAV the phone recorded, 16 kHz mono 16-bit as devnull takes it, or null when it isn't one. + * The chunks are walked as devnull walks them, so any a recorder adds (iOS pads with one) are stepped over. + */ + private static function samples(string $wav): ?string + { + if (! str_starts_with($wav, 'RIFF') || substr($wav, 8, 4) !== 'WAVE') { + return null; + } + + $format = null; + $offset = 12; + + while ($offset + 8 <= strlen($wav)) { + $id = substr($wav, $offset, 4); + $size = unpack('V', substr($wav, $offset + 4, 4))[1]; + + if ($offset + 8 + $size > strlen($wav)) { + return null; + } + + if ($id === 'data') { + return $format === pack('vvVVvv', 1, 1, self::SampleRate, self::SampleRate * 2, 2, 16) && $size % 2 === 0 + ? substr($wav, $offset + 8, $size) + : null; + } + + if ($id === 'fmt ' && $size >= 16) { + $format = substr($wav, $offset + 8, 16); + } + + $offset += 8 + $size + $size % 2; + } + + return null; + } + + private function directory(): string + { + $directory = storage_path('app/voice-notes'); + + if (! is_dir($directory)) { + mkdir($directory, 0700, true); + } + + return $directory; + } + + private function prune(): void + { + $held = array_flip($this->held()); + + foreach (glob($this->directory().'/*') ?: [] as $file) { + if (! isset($held[$file]) && filemtime($file) < time() - self::KeptForSeconds) { + @unlink($file); + } + } + } + + /** + * The recordings the messages in the outbox hold. + * + * @return list + */ + private function held(): array + { + return app(Outbox::class)->pending(PendingWrite::Message, withFailed: true) + ->map(fn (PendingWrite $write): ?string => $write->voice()?->url) + ->filter() + ->values() + ->all(); + } +} diff --git a/app/Devnull/VoicePlayback.php b/app/Devnull/VoicePlayback.php new file mode 100644 index 00000000..d00f7efb --- /dev/null +++ b/app/Devnull/VoicePlayback.php @@ -0,0 +1,119 @@ + $positions where each paused voice note stopped, in milliseconds + * @param array $speeds how fast each plays, for those not at 1× + */ + public function __construct( + public ?string $key = null, + public string $state = self::Idle, + public int $fromMs = 0, + public float $startedAt = 0, + public array $positions = [], + public array $speeds = [], + public float $now = 0, + ) {} + + public function stateOf(string $key): string + { + return $key === $this->key ? $this->state : self::Idle; + } + + public function isPlaying(string $key): bool + { + return $this->stateOf($key) === self::Playing; + } + + public function speedOf(string $key): float + { + return $this->speeds[$key] ?? 1.0; + } + + /** + * Where a voice note is, in milliseconds: moving on from where it started while it plays. + */ + public function positionOf(string $key, MessageVoice $voice): int + { + $position = $this->isPlaying($key) + ? $this->fromMs + (int) round(($this->now - $this->startedAt) * 1000 * $this->speedOf($key)) + : ($key === $this->key ? $this->fromMs : ($this->positions[$key] ?? 0)); + + return max(0, min($voice->durationMs, $position)); + } + + /** + * The web's clock: where it is, or how long it is before it has played, in the time it takes at its speed, so + * 0:14 at 2× reads 0:07. + */ + public function clock(string $key, MessageVoice $voice): string + { + $position = $this->positionOf($key, $voice); + + return VoiceNotes::clock(($position > 0 ? $position : $voice->durationMs) / $this->speedOf($key)); + } + + /** + * Each bar of the waveform: its height in points, and whether it's drawn played, which it is once playback reaches + * its middle. + * + * @return list + */ + public function bars(string $key, MessageVoice $voice): array + { + $position = $this->positionOf($key, $voice); + $bars = []; + + foreach ($voice->waveform as $bar => $level) { + $bars[] = [ + 'height' => (int) max(3, round(self::Height * max(self::ShortestBar, $level) / 100)), + 'played' => $position >= (int) round(($bar + 0.5) * $voice->durationMs / VoiceNotes::Bars), + ]; + } + + return $bars; + } + + public function speedLabel(string $key): string + { + return VoiceNotes::speedLabel($this->speedOf($key)); + } +} diff --git a/app/Icons/Lucide.php b/app/Icons/Lucide.php index 4fed0362..11663d40 100644 --- a/app/Icons/Lucide.php +++ b/app/Icons/Lucide.php @@ -94,6 +94,11 @@ enum Lucide case BellOff; case LogOut; case UserMinus; + case Mic; + case Play; + case Pause; + case RotateCw; + case Square; /** * The icon the web's inbox draws for a kind of notification. @@ -200,6 +205,11 @@ public function custom(bool $filled = false): ?string self::BellOff => 'bell-off', self::LogOut => 'log-out', self::UserMinus => 'user-minus', + self::Mic => 'mic', + self::Play => 'play-filled', + self::Pause => 'pause-filled', + self::RotateCw => 'rotate-cw', + self::Square => 'square-filled', default => null, }; diff --git a/app/Models/PendingWrite.php b/app/Models/PendingWrite.php index d674be5e..3710eca4 100644 --- a/app/Models/PendingWrite.php +++ b/app/Models/PendingWrite.php @@ -5,6 +5,7 @@ use App\Devnull\Data\Commit; use App\Devnull\Data\CommitBody; use App\Devnull\Data\DraftImage; +use App\Devnull\Data\MessageVoice; use App\Devnull\Data\User; use Carbon\CarbonImmutable; use Illuminate\Database\Eloquent\Model; @@ -179,6 +180,15 @@ public function images(): array return array_values(array_filter(array_map(DraftImage::fromArray(...), $images))); } + /** + * The voice note of a message still on the phone, or null for any other write, or when its recording has gone + * (see VoiceNotes). + */ + public function voice(): ?MessageVoice + { + return MessageVoice::fromOutbox($this->payload['voice'] ?? null); + } + /** * @param list $images * @return list diff --git a/app/NativeComponents/Concerns/PlaysVoiceNotes.php b/app/NativeComponents/Concerns/PlaysVoiceNotes.php new file mode 100644 index 00000000..61245341 --- /dev/null +++ b/app/NativeComponents/Concerns/PlaysVoiceNotes.php @@ -0,0 +1,270 @@ + + */ + public array $voicePositions = []; + + /** + * How fast each voice note plays, for those you changed from 1×. + * + * @var array + */ + public array $voiceSpeeds = []; + + /** + * The voice note with this key, in the thread or the composer. + */ + abstract protected function voiceNoteFor(string $key): ?MessageVoice; + + /** + * Play, pause, or after it failed, retry. + */ + public function toggleVoiceNote(string $key): void + { + if ($this->playingVoice === $key && $this->playback === VoicePlayback::Playing) { + $this->pauseVoiceNote(); + + return; + } + + if ($this->playingVoice === $key && $this->playback === VoicePlayback::Loading) { + return; + } + + $this->playVoiceNote($key, $this->playingVoice === $key ? $this->playbackFromMs : ($this->voicePositions[$key] ?? 0)); + } + + /** + * A tap on one of the waveform's bars plays the note from there. + */ + public function seekVoiceNote(string $key, int $bar): void + { + $voice = $this->voiceNoteFor($key); + + if ($voice !== null) { + $this->playVoiceNote($key, intdiv(max(0, min(VoiceNotes::Bars - 1, $bar)) * $voice->durationMs, VoiceNotes::Bars)); + } + } + + /** + * The speed button: the next speed, back to 1× after 2×. A note playing carries on from where it is at the new + * speed. + */ + public function changeVoiceSpeed(string $key): void + { + $voice = $this->voiceNoteFor($key); + $playing = $voice !== null && $this->playingVoice === $key && $this->playback === VoicePlayback::Playing; + $position = $playing ? $this->voicePlayback()->positionOf($key, $voice) : 0; + $speed = VoiceNotes::nextSpeed($this->voiceSpeeds[$key] ?? 1.0); + + if ($speed === 1.0) { + unset($this->voiceSpeeds[$key]); + } else { + $this->voiceSpeeds[$key] = $speed; + } + + if ($this->playingVoice === $key) { + MediaPlayer::setRate($speed); + } + + if ($playing) { + $this->playbackFromMs = $position; + $this->playbackStartedAt = (float) CarbonImmutable::now()->format('U.u'); + } + } + + #[On(PlaybackEnded::class)] + public function voiceNotePlayed(string $source = ''): void + { + if ($this->playingVoice === null || $source !== $this->playbackSource) { + return; + } + + unset($this->voicePositions[$this->playingVoice]); + $this->forgetPlayback(); + } + + #[On(PlaybackError::class)] + public function voiceNoteBroke(string $source = '', string $message = ''): void + { + if ($this->playingVoice !== null && ($source === $this->playbackSource || $this->playback === VoicePlayback::Loading)) { + $this->playback = VoicePlayback::Failed; + } + } + + /** + * Where the thread's voice notes are, for drawing them. + */ + protected function voicePlayback(): VoicePlayback + { + return new VoicePlayback( + key: $this->playingVoice, + state: $this->playback, + fromMs: $this->playbackFromMs, + startedAt: $this->playbackStartedAt, + positions: $this->voicePositions, + speeds: $this->voiceSpeeds, + now: (float) CarbonImmutable::now()->format('U.u'), + ); + } + + /** + * Stops whatever plays, as leaving the thread does, or pushing the recording in the composer. + */ + protected function stopVoiceNotes(): void + { + if ($this->playingVoice !== null) { + MediaPlayer::stop(); + } + + $this->forgetPlayback(); + } + + private function playVoiceNote(string $key, int $fromMs): void + { + $voice = $this->voiceNoteFor($key); + + if ($voice === null) { + return; + } + + $this->leaveNotePlaying($key); + $this->playingVoice = $key; + $this->playbackFromMs = $fromMs >= $voice->durationMs ? 0 : $fromMs; + + $file = app(VoiceNotes::class)->playable($voice); + + if ($file !== null) { + $this->startPlayer($file); + + return; + } + + if ($voice->isOnPhone()) { + $this->playback = VoicePlayback::Failed; + + return; + } + + $this->playback = VoicePlayback::Loading; + + DownloadVoiceNote::dispatch($voice->url, VoiceNotes::cacheName($voice)) + ->timeout(DownloadVoiceNote::Timeout + 5) + ->finished(fn (mixed $path) => $this->voiceNoteDownloaded($key, is_string($path) ? $path : null)) + ->failed(fn (Throwable $exception) => $this->voiceNoteDownloaded($key, null)); + } + + /** + * The download is done: it plays, unless you've moved on to another note since. + */ + private function voiceNoteDownloaded(string $key, ?string $file): void + { + if ($this->playingVoice !== $key || $this->playback !== VoicePlayback::Loading) { + return; + } + + $file === null || ! is_file($file) + ? $this->playback = VoicePlayback::Failed + : $this->startPlayer($file); + } + + private function startPlayer(string $file): void + { + $key = (string) $this->playingVoice; + + $started = MediaPlayer::play($file, ['rate' => $this->voiceSpeeds[$key] ?? 1.0, 'position' => $this->playbackFromMs / 1000]); + + $this->playbackSource = $file; + $this->playback = $started ? VoicePlayback::Playing : VoicePlayback::Failed; + $this->playbackStartedAt = (float) CarbonImmutable::now()->format('U.u'); + } + + private function pauseVoiceNote(): void + { + $key = (string) $this->playingVoice; + $voice = $this->voiceNoteFor($key); + + MediaPlayer::pause(); + + $this->playbackFromMs = $voice === null ? 0 : $this->voicePlayback()->positionOf($key, $voice); + $this->voicePositions[$key] = $this->playbackFromMs; + $this->playback = VoicePlayback::Paused; + } + + /** + * Another note is about to play: the one in the player keeps where it was, paused, as the web's do. + */ + private function leaveNotePlaying(string $key): void + { + if ($this->playingVoice === null || $this->playingVoice === $key) { + return; + } + + if ($this->playback === VoicePlayback::Playing) { + $this->pauseVoiceNote(); + } + + $this->playback = VoicePlayback::Idle; + } + + private function forgetPlayback(): void + { + $this->playingVoice = null; + $this->playback = VoicePlayback::Idle; + $this->playbackFromMs = 0; + $this->playbackStartedAt = 0; + $this->playbackSource = null; + } +} diff --git a/app/NativeComponents/Concerns/RecordsVoiceNotes.php b/app/NativeComponents/Concerns/RecordsVoiceNotes.php new file mode 100644 index 00000000..4e86cc74 --- /dev/null +++ b/app/NativeComponents/Concerns/RecordsVoiceNotes.php @@ -0,0 +1,260 @@ + + */ + public array $recordingLevels = []; + + /** + * Whether the composer can take a voice note right now: the thread takes messages, and there's no text or + * photos on the draft, since voice notes go on their own. + */ + abstract protected function takesVoiceNotes(): bool; + + /** + * A recorded voice note to push, with whatever it replies to. + */ + abstract protected function pushVoiceNote(MessageVoice $voice): void; + + public function recordVoiceNote(): void + { + if ($this->recorder !== null || ! $this->takesVoiceNotes()) { + return; + } + + $this->recordingId = (string) Str::uuid(); + + $started = Microphone::record() + ->id($this->recordingId) + ->wav(VoiceNotes::SampleRate) + ->maxDuration(VoiceNotes::MaxMilliseconds / 1000) + ->start(); + + if (! $started) { + $this->recordingId = null; + $this->voiceNoteFailed(VoiceNotes::CouldNotOpen); + + return; + } + + $this->recorder = self::Recording; + $this->recordingStarted = Microphone::getStatus() === 'recording'; + $this->recordingStartedAt = (float) CarbonImmutable::now()->format('U.u'); + $this->recordingLevels = array_fill(0, VoiceNotes::Bars, 0); + $this->pushesWhenRecorded = false; + } + + /** + * Stop: the recording, to listen to before it's pushed. + */ + public function stopVoiceNote(): void + { + if ($this->recorder !== self::Recording) { + return; + } + + $this->recorder = self::Stopping; + Microphone::stop(); + } + + /** + * Push: a recording goes as soon as the microphone hands it over, and a recorded one at once. + */ + public function pushRecording(): void + { + if ($this->recorder === self::Recorded && $this->recorded !== null) { + $voice = $this->recorded; + $this->forgetRecording(); + $this->pushVoiceNote($voice); + + return; + } + + if ($this->recorder === self::Recording || $this->recorder === self::Stopping) { + $this->pushesWhenRecorded = true; + $this->stopVoiceNote(); + } + } + + /** + * The trash: the recording goes, from the composer and off the phone. + */ + public function discardVoiceNote(): void + { + if ($this->recorder === self::Recording) { + Microphone::stop(); + } + + app(VoiceNotes::class)->discard($this->recorded); + $this->forgetRecording(); + } + + /** + * The microphone handed over a recording, or reached its minute and stopped by itself. + */ + #[On(MicrophoneRecorded::class)] + public function voiceNoteRecorded(string $path = '', string $mimeType = '', ?string $id = null): void + { + if ($id === null || $id !== $this->recordingId || $this->recorder === null || $this->recorder === self::Recorded) { + @unlink($path); + + return; + } + + try { + $voice = app(VoiceNotes::class)->keep($path); + } catch (InvalidVoiceNote $exception) { + $this->forgetRecording(); + $this->voiceNoteFailed($exception->getMessage()); + + return; + } + + if ($this->pushesWhenRecorded) { + $this->forgetRecording(); + $this->pushVoiceNote($voice); + + return; + } + + $this->recorder = self::Recorded; + $this->recorded = $voice; + } + + /** + * The microphone couldn't be used: not allowed, or it wouldn't start. + */ + #[On(MicrophoneCancelled::class)] + public function voiceNoteCancelled(?string $id = null, string $reason = ''): void + { + if ($id === null || $id !== $this->recordingId) { + return; + } + + $this->forgetRecording(); + $this->voiceNoteFailed($reason === 'permission_denied' ? VoiceNotes::Denied : VoiceNotes::CouldNotOpen); + } + + /** + * How long it has been recording, up to the limit, in milliseconds. + */ + protected function recordingElapsed(): int + { + return $this->recorder === self::Recording && $this->recordingStarted + ? (int) min(VoiceNotes::MaxMilliseconds, round(((float) CarbonImmutable::now()->format('U.u') - $this->recordingStartedAt) * 1000)) + : 0; + } + + /** + * Takes the microphone's level since the last time for the meter's newest bar, while it records. Until the + * microphone really records, the clock waits. The thread renders every MeterInterval for it. + */ + protected function sampleRecording(): void + { + if ($this->recorder !== self::Recording) { + return; + } + + if (! $this->recordingStarted) { + $this->recordingStarted = Microphone::getStatus() === 'recording'; + $this->recordingStartedAt = (float) CarbonImmutable::now()->format('U.u'); + + return; + } + + $level = (int) round(sqrt(Microphone::level()) * 100); + $this->recordingLevels = [...array_slice($this->recordingLevels, 1 - VoiceNotes::Bars), $level]; + } + + /** + * Leaving the thread lets go of the microphone and throws the recording away, as the web's composer going does. + */ + protected function letGoOfRecording(): void + { + if ($this->recorder !== null) { + $this->discardVoiceNote(); + } + } + + /** + * Why a voice note couldn't be recorded, in the web's words. The thread says it with a toast. + */ + abstract protected function voiceNoteFailed(string $error): void; + + private function forgetRecording(): void + { + $this->recorder = null; + $this->recordingId = null; + $this->recordingStartedAt = 0; + $this->recordingStarted = false; + $this->recorded = null; + $this->pushesWhenRecorded = false; + $this->recordingLevels = []; + } +} diff --git a/app/NativeComponents/Concerns/SyncsOutbox.php b/app/NativeComponents/Concerns/SyncsOutbox.php index 2ffe5444..52b9a63e 100644 --- a/app/NativeComponents/Concerns/SyncsOutbox.php +++ b/app/NativeComponents/Concerns/SyncsOutbox.php @@ -15,6 +15,7 @@ use App\Devnull\SiteLink; use App\Devnull\Stash; use App\Devnull\Toasts; +use App\Devnull\VoiceNotes; use App\Models\PendingWrite; use Native\Mobile\Attributes\On; use Native\Mobile\Facades\Browser; @@ -47,7 +48,8 @@ * toast offers Edit, which opens it here when it can, or the commit the * reply or quote was for. * - A message stays in its thread as rejected, with its Retry, and the toast - * offers Retry too. Its photos stay on the phone with it until it goes. + * offers Retry too. Its photos or voice note stay on the phone with it + * until it goes. * * A 401 signs the phone out, as any request does, which empties the outbox. * @@ -114,7 +116,7 @@ protected function sendFromOutbox(PendingWrite $write): void app(Outbox::class)->sending($write, $taskId); - $task->timeout($write->images() === [] ? Outbox::Timeout : Outbox::UploadTimeout) + $task->timeout($write->images() === [] && $write->voice() === null ? Outbox::Timeout : Outbox::UploadTimeout) ->shared(Outbox::Answered) ->finished(fn (mixed $result) => $this->outboxAnswered($taskId, 'finished', $result)) ->failed(fn (Throwable $exception) => $this->outboxAnswered($taskId, 'failed', null, $exception->getMessage())) @@ -240,6 +242,7 @@ private function writeWent(PendingWrite $write, array $json): void $outbox->forget($write); app(DraftImages::class)->discard($write->images()); + app(VoiceNotes::class)->discard($write->voice()); if ($write->isCommit()) { app(ResponseCache::class)->commitAdded($data); @@ -328,7 +331,7 @@ private function signWrite(PendingWrite $write): SignedRequest return match ($write->kind) { PendingWrite::Star => $devnull->signedWrite('POST', $commit), PendingWrite::Unstar => $devnull->signedWrite('DELETE', $commit), - PendingWrite::Message => $devnull->signedMessage((string) $write->target, $write->body(), $write->client_id, $write->replyToId(), $write->images()), + PendingWrite::Message => $devnull->signedMessage((string) $write->target, $write->body(), $write->client_id, $write->replyToId(), $write->images(), $write->voice()), default => $write->images() === [] ? $devnull->signedCommit($write->body(), $write->parentId(), $write->quotedCommitId(), $write->poll()) : $devnull->signedCommitWithImages($write->body(), $write->images(), $write->parentId(), $write->quotedCommitId()), diff --git a/app/NativeComponents/Conversation.php b/app/NativeComponents/Conversation.php index 1e3cacac..3efec185 100644 --- a/app/NativeComponents/Conversation.php +++ b/app/NativeComponents/Conversation.php @@ -8,6 +8,7 @@ use App\Devnull\Data\Conversation as ConversationData; use App\Devnull\Data\DraftImage; use App\Devnull\Data\Message; +use App\Devnull\Data\MessageVoice; use App\Devnull\Data\Thread; use App\Devnull\Data\User; use App\Devnull\Display; @@ -18,10 +19,13 @@ use App\Devnull\ResponseCache; use App\Devnull\SiteLink; use App\Devnull\Toasts; +use App\Devnull\VoiceNotes; use App\Icons\Lucide; use App\Models\PendingWrite; use App\NativeComponents\Concerns\AttachesPhotos; use App\NativeComponents\Concerns\OpensInTab; +use App\NativeComponents\Concerns\PlaysVoiceNotes; +use App\NativeComponents\Concerns\RecordsVoiceNotes; use App\NativeComponents\Concerns\RefreshesInBackground; use App\NativeComponents\Concerns\SyncsOutbox; use App\NativeComponents\Concerns\TalksToDevnull; @@ -73,6 +77,11 @@ * beside each one in the thread. Pinning and unpinning show at once and go * back if devnull says no, and pins from the others come with the polls. * + * The composer's microphone records a voice note of up to a minute, which + * goes on its own, through the outbox like any message (see + * RecordsVoiceNotes), and voice notes play in the thread as the web's do, + * one at a time, at the speed you pick (see PlaysVoiceNotes). + * * The composer's Ping pings them, or everyone in an org, as MSN Messenger * had: it lands in the thread as a log line. A ping coming in or going out * while you're on the thread buzzes the phone, as the web shakes the thread @@ -86,6 +95,8 @@ class Conversation extends NativeComponent { use AttachesPhotos; use OpensInTab; + use PlaysVoiceNotes; + use RecordsVoiceNotes; use RefreshesInBackground; use SyncsOutbox; use TalksToDevnull; @@ -462,10 +473,17 @@ protected function showsContent(): bool } /** - * Pushes what's in the composer, its text, its photos or both, staged at once, as the web does. + * Pushes what's in the composer, its text, its photos or both, staged at once, as the web does. While a voice + * note is in its place, the voice note goes. */ public function send(): void { + if ($this->recorder !== null) { + $this->pushRecording(); + + return; + } + $body = trim($this->draft); $me = $this->me(); @@ -507,6 +525,100 @@ public function retry(string $clientId): void } } + /** + * A recorded voice note goes as a message of its own, staged at once with whatever it replies to, its recording + * kept on the phone until devnull has it. + */ + protected function pushVoiceNote(MessageVoice $voice): void + { + $me = $this->me(); + + if ($me === null || $this->thread?->access !== Thread::Open) { + app(VoiceNotes::class)->discard($voice); + + return; + } + + if ($this->playingVoice === Message::DraftVoice) { + $this->stopVoiceNotes(); + } + + $message = Message::staged($me->id, (string) Str::uuid(), '', $this->findMessage($this->replyToId), voice: $voice); + $write = app(Outbox::class)->add(PendingWrite::Message, $this->threadPath(), [ + 'body' => '', + 'reply_to_id' => $message->replyToId, + 'voice' => $voice->toOutbox(), + ], $message->clientId); + + if ($write === null) { + return; + } + + $this->pending[] = $message; + $this->replyToId = null; + $this->lastActivityAt = self::now(); + + $this->sendFromOutbox($write); + } + + protected function takesVoiceNotes(): bool + { + return $this->thread?->access === Thread::Open && trim($this->draft) === '' && $this->photos === []; + } + + /** + * Why a voice note couldn't be recorded comes as a toast, from the top, as a photo's does. + */ + protected function voiceNoteFailed(string $error): void + { + Toasts::failed($error); + } + + protected function voiceNoteFor(string $key): ?MessageVoice + { + if ($key === Message::DraftVoice) { + return $this->recorded; + } + + foreach ([...$this->messages, ...$this->pending, ...$this->pinned] as $message) { + if ($message->voice !== null && $message->voiceKey() === $key) { + return $message->voice; + } + } + + return null; + } + + /** + * Leaving the thread stops the voice note playing, and throws away one being recorded, as leaving the web's does. + */ + public function navigate(string $uri, array $data = []): static + { + $this->leaveVoiceNotes(); + + return parent::navigate($uri, $data); + } + + public function back(): static + { + $this->leaveVoiceNotes(); + + return parent::back(); + } + + public function unmount(): void + { + $this->leaveVoiceNotes(); + + parent::unmount(); + } + + private function leaveVoiceNotes(): void + { + $this->stopVoiceNotes(); + $this->letGoOfRecording(); + } + /** * The composer's Ping, as the web's: it lands in the thread as a log line, and buzzes as one coming in does. When * devnull won't take it, such as before anyone has said anything or after three in a minute, a toast says why @@ -1147,7 +1259,7 @@ private function showPending(): void $this->pending = app(Outbox::class)->pending(PendingWrite::Message, $this->threadPath(), withFailed: true) ->map(function (PendingWrite $write) use ($me): Message { - $message = Message::staged($me->id, $write->client_id, $write->body(), $this->findMessage($write->replyToId()), $write->images()); + $message = Message::staged($me->id, $write->client_id, $write->body(), $this->findMessage($write->replyToId()), $write->images(), $write->voice()); return $write->isFailed() ? $message->withStatus(Message::Rejected) : $message; }) @@ -1590,6 +1702,8 @@ private function subtitle(): ?string public function render(): View { + $this->sampleRecording(); + $viewer = $this->me(); $thread = $this->thread; @@ -1621,6 +1735,8 @@ public function render(): View 'pingLabel' => $this->isOrg() ? 'Ping everyone in the org' : 'Ping @'.$this->username, 'muting' => $this->choosingMute ? 'thread' : null, 'mutingTitle' => $this->menuTitle(), + 'player' => $this->voicePlayback(), + 'recordingElapsed' => $this->recordingElapsed(), ]); } diff --git a/resources/icons/mic.svg b/resources/icons/mic.svg new file mode 100644 index 00000000..66a2077a --- /dev/null +++ b/resources/icons/mic.svg @@ -0,0 +1 @@ + diff --git a/resources/icons/pause-filled.svg b/resources/icons/pause-filled.svg new file mode 100644 index 00000000..95314f45 --- /dev/null +++ b/resources/icons/pause-filled.svg @@ -0,0 +1 @@ + diff --git a/resources/icons/play-filled.svg b/resources/icons/play-filled.svg new file mode 100644 index 00000000..3465d223 --- /dev/null +++ b/resources/icons/play-filled.svg @@ -0,0 +1 @@ + diff --git a/resources/icons/rotate-cw.svg b/resources/icons/rotate-cw.svg new file mode 100644 index 00000000..0e9d389f --- /dev/null +++ b/resources/icons/rotate-cw.svg @@ -0,0 +1 @@ + diff --git a/resources/icons/square-filled.svg b/resources/icons/square-filled.svg new file mode 100644 index 00000000..2ae95a1e --- /dev/null +++ b/resources/icons/square-filled.svg @@ -0,0 +1 @@ + diff --git a/resources/views/native/conversation.blade.php b/resources/views/native/conversation.blade.php index 977db61f..bd1502e3 100644 --- a/resources/views/native/conversation.blade.php +++ b/resources/views/native/conversation.blade.php @@ -1,6 +1,6 @@ {{-- A Remotes thread: the messages, oldest first, anchored to the latest at the bottom as a chat is, then the - composer in the bottom bar, with photos and Ping. Who's typing shows under the title in the nav bar, so the bar never changes height + composer in the bottom bar, with photos, voice notes and Ping. Who's typing shows under the title in the nav bar, so the bar never changes height for it. While the thread can't take a message, the bar says why instead, in the web's words, with Merge and Drop for a request. An org's pinned messages lie in a bar over the top of the thread. --}} @@ -75,7 +75,7 @@ {{-- The composer, drawn as the reply bar on a commit's page is (see partials/composer-plate): on iOS the bar is bare and the field is a Liquid Glass plate with the tools inside it, on Android the bar is the surface with - a hairline and the tools lead it. The tools are the web's, in its order: photos, then Ping. Above the field + a hairline and the tools lead it. The tools are the web's, in its order: photos, the microphone, then Ping. Above the field sit what you're replying to and the photos on the draft, on glass of their own on iOS. While the thread can't take a message, the bar is solid on both, with why in the field's place. --}} @@ -94,7 +94,7 @@ Replying to {{ $replyTo->userId === $viewer?->id ? 'yourself' : ($authors[$replyTo->userId]->name ?? 'Someone') }} - {{ $replyTo->body !== '' ? $replyTo->body : 'Image' }} + {{ $replyTo->body !== '' ? $replyTo->body : $replyTo->snippet() }} @@ -107,11 +107,14 @@ {{-- The same room above and below the field, so the bar sits evenly on the keyboard. --}} + {{-- A voice note takes the field's place, and the tools', while it's recorded and waits to be pushed. --}} + @php($voiceNotesGoAlone = trim($draft) !== '' || $photos !== []) @include('native.partials.composer-plate', [ 'ref' => 'message', - 'field' => 'native.partials.message-field', - 'tools' => [ + 'field' => $recorder === null ? 'native.partials.message-field' : 'native.partials.voice-recorder', + 'tools' => $recorder !== null ? [] : [ ['ref' => 'attach-photos', 'icon' => \App\Icons\Lucide::ImagePlus, 'label' => 'Attach up to 3 photos', 'press' => 'pickPhotos', 'disabled' => count($photos) >= \App\Devnull\DraftImages::MaxImages], + ['ref' => 'record-voice-note', 'icon' => \App\Icons\Lucide::Mic, 'label' => $voiceNotesGoAlone ? 'Voice notes go on their own' : 'Record a voice note (max. 1 min)', 'press' => 'recordVoiceNote', 'disabled' => $voiceNotesGoAlone], ['ref' => 'ping', 'icon' => \App\Icons\Lucide::Vibrate, 'label' => $pingLabel, 'press' => 'ping'], ], ]) @@ -120,7 +123,7 @@ {{ number_format($length) }}/2,000 @endif - Push + Push @else diff --git a/resources/views/native/partials/message-item.blade.php b/resources/views/native/partials/message-item.blade.php index c9ccb63b..3e1255d3 100644 --- a/resources/views/native/partials/message-item.blade.php +++ b/resources/views/native/partials/message-item.blade.php @@ -62,6 +62,8 @@ @include('native.partials.message-images') @endif + @include('native.partials.message-voice') + @if ($message->body !== '') {{-- The bubble shrinks to wrap short of the slot for its actions, which keeps its width either way. --}} diff --git a/resources/views/native/partials/message-voice.blade.php b/resources/views/native/partials/message-voice.blade.php new file mode 100644 index 00000000..75fa70fa --- /dev/null +++ b/resources/views/native/partials/message-voice.blade.php @@ -0,0 +1,25 @@ +{{-- + A message's voice note, in a bubble of its own as the web draws it, with the message's actions beside it as a text + bubble has them (see partials/message-item). Yours is in your lane, staged in it faded until devnull has it, or + outlined in red once it turned the note down; everyone else's is on the wash. The player is partials/voice-note. +--}} +@if ($message->voice !== null) + @php($voiceInk = $message->status === 'rejected' ? 'ink' : ($mine ? 'on-'.$myColour : 'ink')) + + @if ($mine) + @include('native.partials.message-actions') + @endif + @if ($message->isPending()) + + @include('native.partials.voice-note', ['voice' => $message->voice, 'voiceKey' => $message->voiceKey(), 'ref' => 'voice-'.$key, 'ink' => $voiceInk]) + + @else + + @include('native.partials.voice-note', ['voice' => $message->voice, 'voiceKey' => $message->voiceKey(), 'ref' => 'voice-'.$key, 'ink' => $voiceInk]) + + @endif + @if (! $mine) + @include('native.partials.message-actions') + @endif + +@endif diff --git a/resources/views/native/partials/pinned-message.blade.php b/resources/views/native/partials/pinned-message.blade.php index 0cf9ddc6..cdf8219a 100644 --- a/resources/views/native/partials/pinned-message.blade.php +++ b/resources/views/native/partials/pinned-message.blade.php @@ -1,7 +1,7 @@ {{-- One pinned message in the pinned sheet, as the web's panel lists it: who wrote it and when, Reply, React and, for a - maintainer, Unpin, which always show; then up to three lines of it with the rest a tap on Show more away, how many - images it has, and its reactions. + maintainer, Unpin, which always show; then up to three lines of it with the rest a tap on Show more away, its voice + note to play, how many images it has, and its reactions. --}} @php($author = $message->user ?? ($authors[$message->userId] ?? null)) @php($isLong = isset($longPins[$message->id])) @@ -63,6 +63,10 @@ @endif @endif + @if ($message->voice !== null) + @include('native.partials.voice-note', ['voice' => $message->voice, 'voiceKey' => $message->voiceKey(), 'ref' => $key.'-voice', 'ink' => 'ink']) + @endif + @if ($message->images !== []) diff --git a/resources/views/native/partials/voice-note.blade.php b/resources/views/native/partials/voice-note.blade.php new file mode 100644 index 00000000..1ba81a1e --- /dev/null +++ b/resources/views/native/partials/voice-note.blade.php @@ -0,0 +1,51 @@ +{{-- + A voice note, as the web's player draws it: play and pause, with a spinner while it downloads and retry when it + couldn't play; the waveform, played bars in full ink and the rest faded; the clock; and the speed, faded at 1×. + A tap on the waveform plays from there, in twelve stretches of four bars, so a screen reader has a dozen places to + play from rather than every bar. The volume button the web shows a mouse is left off, as it is on a phone, which + has its own buttons. + + While it plays, the thread renders four times a second for the clock and the waveform (VoicePlayback::Interval). + + $voice is the MessageVoice, $voiceKey what the thread knows it by, $ref names its parts, $ink the theme token it's + drawn in (the bubble's ink), and $player the thread's VoicePlayback. +--}} +@php($voiceState = $player->stateOf($voiceKey)) +@php($bars = $player->bars($voiceKey, $voice)) +@php($speed = $player->speedOf($voiceKey)) +@if ($voiceState === \App\Devnull\VoicePlayback::Playing) + +@else + +@endif + + + @if ($voiceState === \App\Devnull\VoicePlayback::Loading) + + @else + @php($voiceIcon = match ($voiceState) { 'playing' => \App\Icons\Lucide::Pause, 'failed' => \App\Icons\Lucide::RotateCw, default => \App\Icons\Lucide::Play }) + + @endif + + + + @foreach (array_chunk($bars, 4, true) as $stretch) + @php($from = array_key_first($stretch)) + + + @foreach ($stretch as $bar) + + + + @endforeach + + + @endforeach + + {{ $player->clock($voiceKey, $voice) }} + + + {{ $player->speedLabel($voiceKey) }} + + + diff --git a/resources/views/native/partials/voice-recorder.blade.php b/resources/views/native/partials/voice-recorder.blade.php new file mode 100644 index 00000000..3c19a267 --- /dev/null +++ b/resources/views/native/partials/voice-recorder.blade.php @@ -0,0 +1,40 @@ +{{-- + The voice note in the composer, in the field's place (see partials/composer-plate), as the web's recorder draws + it: the trash, then "opening microphone…" until it records, then a pulsing dot, the microphone's level coming in + on the right, and the clock counting to the minute, amber in the last ten seconds, with Stop; then the recording + itself, to listen to before Push sends it. The same height as the field with one line, so the bar doesn't move. + + While it records, the thread renders five times a second for the meter (RecordsVoiceNotes::MeterInterval). +--}} +@if ($recorder === 'recording') + +@else + +@endif + + + + + + @if ($recorder === 'recorded' && $recorded !== null) + @include('native.partials.voice-note', ['voice' => $recorded, 'voiceKey' => \App\Devnull\Data\Message::DraftVoice, 'ref' => 'voice-draft', 'ink' => 'ink']) + @elseif ($recorder === 'recording' && $recordingStarted) + + + @foreach ($recordingLevels as $level) + + @endforeach + + {{ \App\Devnull\VoiceNotes::clock($recordingElapsed) }}/{{ \App\Devnull\VoiceNotes::clock(\App\Devnull\VoiceNotes::MaxMilliseconds) }} + + + + + + @else + + + {{ $recorder === 'recording' ? 'opening microphone…' : 'encoding…' }} + + @endif + diff --git a/tests/Feature/Devnull/ClientTest.php b/tests/Feature/Devnull/ClientTest.php index 16283840..6d7115d5 100644 --- a/tests/Feature/Devnull/ClientTest.php +++ b/tests/Feature/Devnull/ClientTest.php @@ -4,6 +4,7 @@ use App\Devnull\Data\Commit; use App\Devnull\Data\CurrentUser; use App\Devnull\Data\DraftImage; +use App\Devnull\Data\MessageVoice; use App\Devnull\Data\PollDraft; use App\Devnull\Data\User; use App\Devnull\DeviceKey; @@ -156,6 +157,28 @@ function receivedMessage(Request $request): string ->and($request->headers['X-SSH-Signature'])->toBe(testKey()->sign($message, 'devnull-api-images')); }); +it('signs a voice note alone as multipart, with its SHA-256 on a line before the JSON body and its waveform in it', function () { + $path = sys_get_temp_dir().'/devnull-voice-'.bin2hex(random_bytes(6)).'.wav'; + file_put_contents($path, 'RIFF fake wav'); + $voice = new MessageVoice('', 2500, array_fill(0, 48, 50), $path); + + $request = app(Client::class)->signedMessage('remotes/@ada', 'ignored', '9f0f6f3e-0000-4000-8000-000000000001', 7, voice: $voice); + + $boundary = substr($request->contentType, strlen('multipart/form-data; boundary=')); + $payload = '{"client_id":"9f0f6f3e-0000-4000-8000-000000000001","reply_to_id":7,"voice_waveform":['.implode(',', array_fill(0, 48, 50)).']}'; + + expect($request->url)->toBe('http://devnull.test/api/v1/remotes/@ada/messages') + ->and(array_slice(explode("--{$boundary}", $request->body), 1))->toBe([ + "\r\nContent-Disposition: form-data; name=\"payload\"; filename=\"payload.json\"\r\nContent-Type: application/json\r\n\r\n{$payload}\r\n", + "\r\nContent-Disposition: form-data; name=\"voice\"; filename=\"voice-note.wav\"\r\nContent-Type: audio/wav\r\n\r\nRIFF fake wav\r\n", + "--\r\n", + ]); + + $message = RequestSigner::message($request->headers['X-Devnull-Timestamp'], $request->headers['X-Devnull-Nonce'], 'POST', '/api/v1/remotes/@ada/messages', hash('sha256', 'RIFF fake wav')."\n".$payload); + + expect($request->headers['X-SSH-Signature'])->toBe(testKey()->sign($message, 'devnull-api-images')); +}); + it('sends a signed request with the content type it was signed with', function () { Http::fake(['devnull.test/*' => Http::response(['data' => commitJson()], 201)]); diff --git a/tests/Feature/Screens/RemotesComposerTest.php b/tests/Feature/Screens/RemotesComposerTest.php index 1f7e9544..1388fa44 100644 --- a/tests/Feature/Screens/RemotesComposerTest.php +++ b/tests/Feature/Screens/RemotesComposerTest.php @@ -252,25 +252,25 @@ function threadWithAda(string $platform = 'ios'): TestableComponent ->and(childRefs($screen, 'composer-bar'))->toBe($rows); }); -it('puts the photo and Ping buttons inside the Liquid Glass field on iOS, as the commit reply bar does', function () { +it('puts the photo, microphone and Ping buttons inside the Liquid Glass field on iOS, as the commit reply bar does', function () { serveThreadWithAda([remoteMessageJson(['id' => 1])], ['read_through_id' => 1]); $screen = threadWithAda('ios'); $bar = findNode($screen->tree(), fn (array $node): bool => ($node['type'] ?? null) === 'bottom_bar'); - expect(childRefs($screen, 'message-plate'))->toBe(['message-field', 'attach-photos', 'ping']) + expect(childRefs($screen, 'message-plate'))->toBe(['message-field', 'attach-photos', 'record-voice-note', 'ping']) ->and(nodeByRef($screen, 'message-plate')['props']['glass'] ?? null)->toBe(1) ->and($bar['children'][0]['ref'] ?? null)->toBe('composer-bar') ->and($bar['children'][0]['style']['bg_color'] ?? null)->toBeNull() ->and(nodeByRef($screen, 'ping')['props']['a11y_label'])->toBe('Ping @ada'); }); -it('leads the bar with the photo and Ping buttons on Android, on a solid bar', function () { +it('leads the bar with the photo, microphone and Ping buttons on Android, on a solid bar', function () { serveThreadWithAda([remoteMessageJson(['id' => 1])], ['read_through_id' => 1]); $screen = threadWithAda('android'); - expect(array_slice(childRefs($screen, 'message-row'), 0, 3))->toBe(['attach-photos', 'ping', 'message-plate']) + expect(array_slice(childRefs($screen, 'message-row'), 0, 4))->toBe(['attach-photos', 'record-voice-note', 'ping', 'message-plate']) ->and(childRefs($screen, 'message-plate'))->toBe(['message-field']) ->and(nodeByRef($screen, 'message-plate')['props']['glass'] ?? null)->toBeNull() ->and(nodeByRef($screen, 'composer-bar')['style']['bg_color'] ?? null)->toBe(config('native-ui.theme.light.surface')); diff --git a/tests/Feature/Screens/RemotesVoiceNotesTest.php b/tests/Feature/Screens/RemotesVoiceNotesTest.php new file mode 100644 index 00000000..12ec183b --- /dev/null +++ b/tests/Feature/Screens/RemotesVoiceNotesTest.php @@ -0,0 +1,497 @@ + 'http://devnull.test']); + signIn() + ->respondTo('Microphone.Start', []) + ->respondTo('Microphone.GetStatus', ['status' => 'recording']) + ->respondTo('MediaPlayer.Play', ['success' => true]); + $this->travelTo('2026-10-19 15:30:00'); +}); + +afterEach(function () { + VoiceNotes::clear(); +}); + +/** + * Devnull answering a thread with Ada: its first page, a push (which it echoes back, with the voice note it was sent) + * and Ada's voice note, served as $voiceType. + * + * @param list> $messages + */ +function serveVoiceThread(array $messages, int $pushStatus = 201, string $voiceType = 'audio/mp4', int $voiceStatus = 200): void +{ + serveApi(function (Request $request) use ($messages, $pushStatus, $voiceType, $voiceStatus) { + $path = parse_url($request->url(), PHP_URL_PATH); + $query = (string) parse_url($request->url(), PHP_URL_QUERY); + + return match (true) { + $request->method() === 'GET' && $path === '/api/v1/remotes/@ada/messages' => Http::response(str_contains($query, 'after=') ? threadJson([]) : threadJson($messages, ['read_through_id' => 1])), + $request->method() === 'POST' && $path === '/api/v1/remotes/@ada/messages' => $pushStatus >= 300 + ? Http::response(['message' => 'The voice note is not one.'], $pushStatus) + : Http::response(['data' => echoedVoiceNote($request)], $pushStatus), + $path === '/api/v1/remotes/voices/01a18436-9e52-7015-be72-4153d1cbabb4' => Http::response('ftypisom opus', $voiceStatus, ['Content-Type' => $voiceType]), + $path === '/api/v1/me' => Http::response(currentUserJson()), + default => Http::response(['usage' => []], $request->method() === 'GET' ? 200 : 204), + }; + }); +} + +/** + * The message devnull stores for a pushed voice note, as it lists it. + * + * @return array + */ +function echoedVoiceNote(Request $request): array +{ + $payload = json_decode(multipartParts($request)[0]['contents'], true); + + return remoteMessageJson([ + 'id' => 99, + 'user_id' => VoiceViewerId, + 'client_id' => $payload['client_id'], + 'body' => '', + 'voice' => ['id' => 'v99', 'duration_ms' => 2500, 'waveform' => $payload['voice_waveform'], 'url' => 'http://devnull.test/api/v1/remotes/voices/v99?signature=abc'], + ]); +} + +/** + * A message from Ada holding a voice note of $milliseconds, as devnull lists it. + * + * @param array $overrides + * @return array + */ +function voiceMessageJson(int $id = 2, int $milliseconds = 6000, array $overrides = []): array +{ + return remoteMessageJson([ + 'id' => $id, + 'client_id' => '9f0f6f3e-0000-4000-8000-00000000000'.$id, + 'body' => '', + 'voice' => ['id' => '01a18436-9e52-7015-be72-4153d1cbabb4', 'duration_ms' => $milliseconds, 'waveform' => range(2, 96, 2), 'url' => AdaVoiceUrl], + ...$overrides, + ]); +} + +/** + * A recording as the microphone leaves one in its temporary folder: 16 kHz mono 16-bit WAV of $milliseconds, a + * tone whose loudness rises from silence to $peak, with iOS's padding chunk before its samples. + */ +function testRecording(int $milliseconds = 2500, int $peak = 16384, string $format = ''): string +{ + $count = intdiv(VoiceNotes::SampleRate * $milliseconds, 1000); + $samples = ''; + + for ($index = 0; $index < $count; $index++) { + $samples .= pack('v', (int) round($peak * $index / max(1, $count - 1)) * ($index % 2 === 0 ? 1 : -1) & 0xFFFF); + } + + $format = $format !== '' ? $format : pack('vvVVvv', 1, 1, VoiceNotes::SampleRate, VoiceNotes::SampleRate * 2, 2, 16); + $filler = 'FLLR'.pack('V', 12).str_repeat("\0", 12); + $directory = sys_get_temp_dir().'/devnull-recorded-'.bin2hex(random_bytes(6)); + mkdir($directory); + $path = $directory.'/NativePHP_1.wav'; + + file_put_contents($path, 'RIFF'.pack('V', 4 + 24 + strlen($filler) + 8 + strlen($samples)).'WAVEfmt '.pack('V', 16).$format.$filler.'data'.pack('V', strlen($samples)).$samples); + + return $path; +} + +/** + * The microphone handing over what it recorded, for the recording under way. + */ +function voiceRecorded(TestableComponent $screen, string $path): TestableComponent +{ + return $screen->emitNative(MicrophoneRecorded::class, ['path' => $path, 'mimeType' => 'audio/wav', 'id' => $screen->instance()->recordingId]); +} + +function voiceThread(string $platform = 'ios'): TestableComponent +{ + return finishRefreshes(Native::visit('/remotes/conversations/ada', platform: $platform)); +} + +/** + * The parts of each voice note pushed to Ada. + * + * @return list> + */ +function voiceUploads(): array +{ + return collect(Http::recorded()) + ->map(fn (array $sent): Request => $sent[0]) + ->filter(fn (Request $request): bool => $request->method() === 'POST' && str_ends_with($request->url(), '/remotes/@ada/messages')) + ->map(fn (Request $request): array => multipartParts($request)) + ->values() + ->all(); +} + +/** + * How often the screen renders for its Blade `native:poll`s, in milliseconds. + * + * @return list + */ +function rendersEvery(TestableComponent $screen): array +{ + return array_keys(Closure::bind(fn (): array => $this->bladePollDeadlines, $screen->instance(), NativeComponent::class)()); +} + +/** + * How many of a voice note's bars are drawn played. + */ +function playedBars(TestableComponent $screen, string $ref): int +{ + $bars = findNodes(nodeByRef($screen, $ref.'-waveform'), fn (array $node): bool => str_starts_with((string) ($node['style']['bg_color'] ?? ''), '#') && ($node['layout']['width'] ?? null) == 1.5); + + return count(array_filter($bars, fn (array $node): bool => ($node['style']['opacity'] ?? 1) == 1)); +} + +it('offers the microphone between photos and Ping while the draft is empty, since voice notes go on their own', function (Closure $draft, bool $offered, string $label) { + serveVoiceThread([remoteMessageJson(['id' => 1])]); + + $screen = $draft(voiceThread()); + + expect(childRefs($screen, 'message-plate'))->toBe(['message-field', 'attach-photos', 'record-voice-note', 'ping']) + ->and(nodeByRef($screen, 'record-voice-note')['type'] === 'pressable')->toBe($offered) + ->and(nodeByRef($screen, 'record-voice-note')['props']['a11y_label'])->toBe($label); +})->with([ + 'an empty draft' => [fn (TestableComponent $screen) => $screen, true, 'Record a voice note (max. 1 min)'], + 'some text' => [fn (TestableComponent $screen) => $screen->input('message-input', 'one sec'), false, 'Voice notes go on their own'], + 'a photo' => [fn (TestableComponent $screen) => photosPicked($screen, [testPhoto()]), false, 'Voice notes go on their own'], +]); + +it("records 16 kHz WAV for up to a minute, with the microphone's level and the clock in the field's place", function () { + serveVoiceThread([remoteMessageJson(['id' => 1])]); + Native::fakeBridge()->respondTo('Microphone.GetLevel', ['level' => 0.25]); + + $screen = voiceThread()->tap('record-voice-note'); + + $screen->assertNativeCalled('Microphone.Start', fn (array $params): bool => $params['format'] === 'wav' + && $params['sampleRate'] === 16000 + && $params['maxDuration'] == 60 + && $params['id'] === $screen->instance()->recordingId); + + expect(childRefs($screen, 'message-plate'))->toBe(['voice-recorder']) + ->and(nodeByRef($screen, 'voice-elapsed')['props']['text'] ?? null)->toBe('0:00/1:00') + ->and(rendersEvery($screen))->toBe([200]) + ->and(nodeByRef($screen, 'push')['props']['disabled'] ?? false)->toBeFalse(); + + // The level is the loudest sample since the last look, square-rooted as the web draws it, and the clock turns amber + // in the last ten seconds. + $this->travel(52)->seconds(); + $screen->call('poll'); + $meter = nodeByRef($screen, 'voice-meter')['children']; + + expect($meter)->toHaveCount(VoiceNotes::Bars) + ->and($meter[VoiceNotes::Bars - 1]['layout']['height'] ?? null)->toEqual(12) + ->and(nodeByRef($screen, 'voice-elapsed')['props']['text'])->toBe('0:52/1:00') + ->and(strtolower(nodeByRef($screen, 'voice-elapsed')['props']['color'] ?? ''))->toBe(strtolower(config('native-ui.theme.light.pending'))); +}); + +it('says it is opening the microphone until it records, as while the phone asks for it', function () { + serveVoiceThread([remoteMessageJson(['id' => 1])]); + Native::fakeBridge()->withMicrophoneStatus('idle'); + + $screen = voiceThread()->tap('record-voice-note')->assertSee('opening microphone…'); + + expect(nodeByRef($screen, 'push')['props']['disabled'] ?? false)->toBeTrue(); + + // The clock counts from when the microphone starts, not from the tap. + $this->travel(5)->seconds(); + $screen->bridge()->withMicrophoneStatus('recording'); + $screen->call('poll'); + $this->travel(2)->seconds(); + $screen->call('poll'); + + expect(nodeByRef($screen, 'voice-elapsed')['props']['text'])->toBe('0:02/1:00') + ->and(nodeByRef($screen, 'push')['props']['disabled'] ?? false)->toBeFalse(); +}); + +it('stops to listen to the recording first, then pushes it staged at once and pushed once devnull has it', function () { + serveVoiceThread([remoteMessageJson(['id' => 1])]); + $recording = testRecording(2500); + $bytes = file_get_contents($recording); + + $screen = voiceThread()->tap('record-voice-note')->tap('stop-voice-note'); + $screen->assertNativeCalled('Microphone.Stop')->assertSee('encoding…'); + + expect(nodeByRef($screen, 'push')['props']['disabled'] ?? false)->toBeTrue(); + + voiceRecorded($screen, $recording); + $kept = $screen->instance()->recorded->url; + + // The recording itself, to play from the phone before it goes. + expect(file_exists($recording))->toBeFalse() + ->and(nodeByRef($screen, 'voice-draft-clock')['props']['text'])->toBe('0:02') + ->and($screen->instance()->recorded->waveform)->toHaveCount(VoiceNotes::Bars) + ->and($screen->instance()->recorded->waveform[VoiceNotes::Bars - 1])->toBe(71); + + $screen->tap('voice-draft-toggle')->assertNativeCalled('MediaPlayer.Play', fn (array $params): bool => $params['source'] === $kept); + + holdRefreshes()->respondTo('MediaPlayer.Play', ['success' => true]); + $screen->tap('push')->assertSee('staged')->assertSet('recorder', null)->assertNativeCalled('MediaPlayer.Stop'); + $clientId = $screen->instance()->pending[0]->clientId; + + expect(childRefs($screen, 'message-plate'))->toBe(['message-field', 'attach-photos', 'record-voice-note', 'ping']) + ->and(nodeByRef($screen, 'voice-'.$clientId.'-clock')['props']['text'])->toBe('0:02'); + + finishRefreshes($screen)->assertSee('pushed')->assertSet('pending', []); + + [$payload, $voice] = voiceUploads()[0]; + + expect(json_decode($payload['contents'], true))->toBe(['client_id' => $clientId, 'voice_waveform' => $screen->instance()->messages[1]->voice->waveform]) + ->and($voice)->toBe(['name' => 'voice', 'type' => 'audio/wav', 'contents' => $bytes]) + ->and(nodeByRef($screen, 'voice-99-clock')['props']['text'])->toBe('0:02') + ->and(file_exists($kept))->toBeFalse(); +}); + +it('pushes straight from the recording, as soon as the microphone hands it over', function () { + serveVoiceThread([remoteMessageJson(['id' => 1, 'body' => 'say it'])]); + + $screen = voiceThread()->dragEnd('swipe-1', -200)->tap('reply-1')->tap('record-voice-note')->tap('push'); + $screen->assertNativeCalled('Microphone.Stop')->assertSet('pending', []); + + finishRefreshes(voiceRecorded($screen, testRecording()))->assertSet('pending', [])->assertSet('recorder', null); + + expect(json_decode(voiceUploads()[0][0]['contents'], true)['reply_to_id'])->toBe(1); +}); + +it('stops by itself at a minute, ready to listen to, as the web does', function () { + serveVoiceThread([remoteMessageJson(['id' => 1])]); + + $screen = voiceRecorded(voiceThread()->tap('record-voice-note'), testRecording(60_000)); + + expect($screen->instance()->recorder)->toBe('recorded') + ->and(nodeByRef($screen, 'voice-draft-clock')['props']['text'])->toBe('1:00'); +}); + +it('keeps a refused voice note on the phone, showing it, and sends it again with its retry', function () { + serveVoiceThread([remoteMessageJson(['id' => 1])], pushStatus: 422); + + $screen = voiceRecorded(voiceThread()->tap('record-voice-note')->tap('stop-voice-note'), testRecording()); + $kept = $screen->instance()->recorded->url; + + finishRefreshes($screen->tap('push'))->assertSee('rejected ·'); + $clientId = $screen->instance()->pending[0]->clientId; + + // Back on the thread later, it's still there to play, and stays on the phone until it goes. + $again = voiceThread()->assertSee('rejected ·'); + + expect(nodeByRef($again, 'voice-'.$clientId.'-clock')['props']['text'])->toBe('0:02') + ->and(file_exists($kept))->toBeTrue(); + + serveVoiceThread([remoteMessageJson(['id' => 1])]); + finishRefreshes($again->call('retry', $clientId))->assertSet('pending', []); + + expect(voiceUploads())->toHaveCount(2) + ->and(voiceUploads()[1][1]['name'])->toBe('voice') + ->and(file_exists($kept))->toBeFalse(); +}); + +it('throws a recording away with the trash, and one that arrives after that too', function (bool $whileRecording) { + serveVoiceThread([remoteMessageJson(['id' => 1])]); + + $screen = voiceThread()->tap('record-voice-note'); + $recording = testRecording(); + + if (! $whileRecording) { + voiceRecorded($screen->tap('stop-voice-note'), $recording); + $recording = $screen->instance()->recorded->url; + } + + $id = $screen->instance()->recordingId; + $screen->tap('discard-voice-note')->assertSet('recorder', null); + + if ($whileRecording) { + $screen->assertNativeCalled('Microphone.Stop') + ->emitNative(MicrophoneRecorded::class, ['path' => $recording, 'mimeType' => 'audio/wav', 'id' => $id]); + } + + expect(file_exists($recording))->toBeFalse() + ->and(childRefs($screen, 'message-plate'))->toBe(['message-field', 'attach-photos', 'record-voice-note', 'ping']) + ->and(voiceUploads())->toBe([]); +})->with(['while recording' => [true], 'once recorded' => [false]]); + +it("says why a voice note can't be recorded, in the web's words", function (Closure $fail, string $message) { + serveVoiceThread([remoteMessageJson(['id' => 1])]); + + $screen = $fail(voiceThread()); + + $screen->assertNativeCalled('ToastKit.Show', fn (array $params): bool => $params['message'] === $message) + ->assertSet('recorder', null) + ->assertSet('pending', []); +})->with([ + 'shorter than half a second' => [fn (TestableComponent $screen) => voiceRecorded($screen->tap('record-voice-note')->tap('stop-voice-note'), testRecording(400)), VoiceNotes::TooShort], + 'not the WAV it asked for' => [fn (TestableComponent $screen) => voiceRecorded($screen->tap('record-voice-note')->tap('stop-voice-note'), testRecording(format: pack('vvVVvv', 1, 2, 44100, 176400, 4, 16))), VoiceNotes::Unreadable], + 'the microphone not allowed' => [fn (TestableComponent $screen) => $screen->tap('record-voice-note')->emitNative(MicrophoneCancelled::class, ['cancelled' => true, 'reason' => 'permission_denied', 'id' => $screen->instance()->recordingId]), VoiceNotes::Denied], + 'the microphone not starting' => [function (TestableComponent $screen) { + $screen->bridge()->respondTo('Microphone.Start', null); + + return $screen->tap('record-voice-note'); + }, VoiceNotes::CouldNotOpen], +]); + +it("plays someone's voice note once it's downloaded, and pauses it where it is", function () { + serveVoiceThread([remoteMessageJson(['id' => 1]), voiceMessageJson()]); + holdRefreshes()->respondTo('MediaPlayer.Play', ['success' => true]); + + $screen = voiceThread(); + + expect(nodeByRef($screen, 'voice-2-clock')['props']['text'])->toBe('0:06') + ->and(nodeByRef($screen, 'voice-2-toggle')['props']['a11y_label'])->toBe('Play voice note') + ->and(playedBars($screen, 'voice-2'))->toBe(0); + + // Nothing is downloaded until play is pressed, and a spinner shows meanwhile. + $screen->tap('voice-2-toggle'); + + expect(findNode(nodeByRef($screen, 'voice-2-toggle'), fn (array $node): bool => $node['type'] === 'activity_indicator'))->not->toBeNull(); + + finishRefreshes($screen)->assertNativeCalled('MediaPlayer.Play', fn (array $params): bool => str_ends_with($params['source'], '/01a18436-9e52-7015-be72-4153d1cbabb4.mp4') + && file_get_contents($params['source']) === 'ftypisom opus' + && $params['rate'] == 1 + && $params['position'] == 0); + + expect(nodeByRef($screen, 'voice-2-toggle')['props']['a11y_label'])->toBe('Pause voice note') + ->and(rendersEvery($screen))->toBe([250]); + + // A second in, the first eight bars are played. + $this->travel(1)->seconds(); + $screen->call('poll'); + + expect(nodeByRef($screen, 'voice-2-clock')['props']['text'])->toBe('0:01') + ->and(playedBars($screen, 'voice-2'))->toBe(8); + + $screen->tap('voice-2-toggle')->assertNativeCalled('MediaPlayer.Pause'); + + expect(nodeByRef($screen, 'voice-2-toggle')['props']['a11y_label'])->toBe('Play voice note') + ->and(nodeByRef($screen, 'voice-2-clock')['props']['text'])->toBe('0:01') + ->and(playedBars($screen, 'voice-2'))->toBe(8); + + // It carries on from there, from the copy already on the phone. + $screen->tap('voice-2-toggle'); + + $downloads = collect(Http::recorded())->filter(fn (array $sent): bool => str_contains($sent[0]->url(), '/remotes/voices/')); + + expect(array_column(Native::fakeBridge()->callsTo('MediaPlayer.Play'), 'params')[1]['position'])->toEqual(1.0) + ->and($downloads)->toHaveCount(1); +}); + +it('plays a voice note from where its waveform is tapped', function () { + serveVoiceThread([remoteMessageJson(['id' => 1]), voiceMessageJson()]); + + $screen = finishRefreshes(voiceThread()->tap('voice-2-from-24')); + + $screen->assertNativeCalled('MediaPlayer.Play', fn (array $params): bool => $params['position'] == 3.0); + + expect(nodeByRef($screen, 'voice-2-clock')['props']['text'])->toBe('0:03') + ->and(nodeByRef($screen, 'voice-2-from-24')['props']['a11y_label'])->toBe('Play from 0:03'); +}); + +it('steps through 1×, 1.5× and 2×, carrying on from where it is, and counts the clock at that speed', function () { + serveVoiceThread([remoteMessageJson(['id' => 1]), voiceMessageJson()]); + + $screen = voiceThread()->tap('voice-2-speed'); + + // At 1.5×, six seconds take four. + expect(nodeByRef($screen, 'voice-2-speed')['props']['a11y_label'])->toBe('Playback speed, 1.5×') + ->and(nodeByRef($screen, 'voice-2-clock')['props']['text'])->toBe('0:04'); + + finishRefreshes($screen->tap('voice-2-toggle'))->assertNativeCalled('MediaPlayer.Play', fn (array $params): bool => $params['rate'] == 1.5); + + $this->travel(2)->seconds(); + $screen->tap('voice-2-speed')->assertNativeCalled('MediaPlayer.SetRate', fn (array $params): bool => $params['rate'] == 2); + + // Three seconds in after two at 1.5×, which reads 0:01 at 2×. + expect(nodeByRef($screen, 'voice-2-clock')['props']['text'])->toBe('0:01') + ->and(nodeByRef($screen, 'voice-2-speed')['props']['a11y_label'])->toBe('Playback speed, 2×'); + + $screen->tap('voice-2-speed'); + + expect(nodeByRef($screen, 'voice-2-speed')['props']['a11y_label'])->toBe('Playback speed, 1×') + ->and($screen->instance()->voiceSpeeds)->toBe([]); +}); + +it('plays one voice note at a time, leaving the last one paused where it was', function () { + serveVoiceThread([remoteMessageJson(['id' => 1]), voiceMessageJson(2), voiceMessageJson(3)]); + + $screen = finishRefreshes(voiceThread()->tap('voice-2-toggle')); + $this->travel(2)->seconds(); + finishRefreshes($screen->tap('voice-3-toggle'))->assertNativeCalled('MediaPlayer.Pause'); + + expect(nodeByRef($screen, 'voice-2-toggle')['props']['a11y_label'])->toBe('Play voice note') + ->and(nodeByRef($screen, 'voice-2-clock')['props']['text'])->toBe('0:02') + ->and(nodeByRef($screen, 'voice-3-toggle')['props']['a11y_label'])->toBe('Pause voice note'); +}); + +it('goes back to the start once it has played', function () { + serveVoiceThread([remoteMessageJson(['id' => 1]), voiceMessageJson()]); + + $screen = finishRefreshes(voiceThread()->tap('voice-2-toggle')); + $this->travel(6)->seconds(); + $screen->emitNative(PlaybackEnded::class, ['source' => $screen->instance()->playbackSource]); + + expect(nodeByRef($screen, 'voice-2-toggle')['props']['a11y_label'])->toBe('Play voice note') + ->and(nodeByRef($screen, 'voice-2-clock')['props']['text'])->toBe('0:06') + ->and(playedBars($screen, 'voice-2'))->toBe(0); +}); + +it("offers the web's retry when a voice note can't be downloaded or played", function (Closure $fail) { + serveVoiceThread([remoteMessageJson(['id' => 1]), voiceMessageJson()], voiceStatus: 404); + + $screen = $fail(voiceThread()); + + expect(nodeByRef($screen, 'voice-2-toggle')['props']['a11y_label'])->toBe('Retry voice note'); +})->with([ + 'not served' => [fn (TestableComponent $screen) => finishRefreshes($screen->tap('voice-2-toggle'))], + 'not playable' => [function (TestableComponent $screen) { + serveVoiceThread([remoteMessageJson(['id' => 1]), voiceMessageJson()], voiceType: 'audio/wav'); + finishRefreshes($screen->tap('voice-2-toggle')); + + return $screen->emitNative(PlaybackError::class, ['source' => $screen->instance()->playbackSource, 'message' => 'Cannot Open']); + }], + 'not something devnull serves' => [function (TestableComponent $screen) { + serveVoiceThread([remoteMessageJson(['id' => 1]), voiceMessageJson()], voiceType: 'text/html'); + + return finishRefreshes($screen->tap('voice-2-toggle')); + }], +]); + +it('stops the voice note playing, and throws away one being recorded, when you leave the thread', function (Closure $leave) { + serveVoiceThread([remoteMessageJson(['id' => 1]), voiceMessageJson()]); + + $screen = finishRefreshes(voiceThread()->tap('voice-2-toggle'))->tap('record-voice-note'); + + $leave($screen)->assertNativeCalled('MediaPlayer.Stop')->assertNativeCalled('Microphone.Stop'); +})->with([ + 'back' => [fn (TestableComponent $screen) => $screen->pressBack()], + "to Ada's profile" => [fn (TestableComponent $screen) => $screen->call('viewProfile')], +]); + +it('says someone sent a voice note in the inbox, as the web does', function () { + fakeApi(['GET /api/v1/remotes' => [remotesInboxJson([conversationJson(['last_message' => voiceMessageJson()])])]]); + + finishRefreshes(Native::visit('/remotes'))->assertSee('sent a voice note'); +}); + +it('calls a voice note a voice note above a reply to it, and while replying to it', function () { + serveVoiceThread([voiceMessageJson(), remoteMessageJson(['id' => 3, 'body' => 'got it', 'reply_to_id' => 2, 'reply_to' => ['id' => 2, 'user_id' => 'u2', 'body' => '', 'images_count' => 0, 'has_voice' => true]])]); + + $screen = voiceThread(); + + expect(nodeByRef($screen, 'reply-ghost-3')['props']['a11y_label'])->toBe('Replying to Ada Lovelace: Voice note'); + + $screen->dragEnd('swipe-2', -200)->tap('reply-2')->assertSee('Voice note'); +}); From 0262fb0566bb18d70161a3ba4ab2e2e5d47b3bdf Mon Sep 17 00:00:00 2001 From: Simon Hamp Date: Thu, 29 Oct 2026 10:18:22 +0000 Subject: [PATCH 3/6] Use the released microphone and media player plugins nativephp/mobile-microphone 1.2.0 and nativephp/mobile-media-player 1.1.0 have the WAV recording, level and playback speed voice notes need, and nativephp/mobile main has the microphone's wav() and maxDuration(). The media player needs mobile 4.6.1, so main is aliased as that now. Co-Authored-By: Claude Opus 5.5 --- composer.json | 6 +++--- composer.lock | 40 ++++++++++++++++++++-------------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/composer.json b/composer.json index b127b241..0ec1282b 100644 --- a/composer.json +++ b/composer.json @@ -35,11 +35,11 @@ "laravel/boost": "^2.0", "laravel/framework": "^13.0", "laravel/tinker": "^3.0", - "nativephp/mobile": "dev-main as 4.6.0", + "nativephp/mobile": "dev-main as 4.6.1", "nativephp/mobile-browser": "^1.2", "nativephp/mobile-camera": "^1.1", - "nativephp/mobile-media-player": "^1.0", - "nativephp/mobile-microphone": "^1.1", + "nativephp/mobile-media-player": "^1.1", + "nativephp/mobile-microphone": "^1.2", "nativephp/mobile-secure-storage": "^1.1", "nativephp/mobile-share": "^1.1", "nativephp/mobile-ui": "dev-main as 0.8.0", diff --git a/composer.lock b/composer.lock index 39fa58ea..bc72fdd5 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "a6e34837e083ae6eb8cc1df7ee22845e", + "content-hash": "79323358e7e37b25e59d3cb701ccd8f5", "packages": [ { "name": "bacon/bacon-qr-code", @@ -2723,12 +2723,12 @@ "source": { "type": "git", "url": "https://github.com/NativePHP/mobile-air.git", - "reference": "40825383f7a3e381280d83d559ea08f4dd0e2ab5" + "reference": "9da9a848ac70140fd5c248c53dd921f70e13381c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/NativePHP/mobile-air/zipball/40825383f7a3e381280d83d559ea08f4dd0e2ab5", - "reference": "40825383f7a3e381280d83d559ea08f4dd0e2ab5", + "url": "https://api.github.com/repos/NativePHP/mobile-air/zipball/9da9a848ac70140fd5c248c53dd921f70e13381c", + "reference": "9da9a848ac70140fd5c248c53dd921f70e13381c", "shasum": "" }, "require": { @@ -2839,7 +2839,7 @@ "source": "https://github.com/NativePHP/mobile-air/tree/main", "issues": "https://github.com/NativePHP/mobile-air/issues" }, - "time": "2026-10-28T19:54:03+00:00" + "time": "2026-10-29T09:23:33+00:00" }, { "name": "nativephp/mobile-browser", @@ -2953,20 +2953,20 @@ }, { "name": "nativephp/mobile-media-player", - "version": "1.0.2", + "version": "1.1.0", "source": { "type": "git", "url": "https://github.com/NativePHP/mobile-media-player.git", - "reference": "ff249aaa50de0623edd3faeffeeeec9d15124869" + "reference": "28653560ea5cf458bf30a6c0f7f64c7f9c003cb1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/NativePHP/mobile-media-player/zipball/ff249aaa50de0623edd3faeffeeeec9d15124869", - "reference": "ff249aaa50de0623edd3faeffeeeec9d15124869", + "url": "https://api.github.com/repos/NativePHP/mobile-media-player/zipball/28653560ea5cf458bf30a6c0f7f64c7f9c003cb1", + "reference": "28653560ea5cf458bf30a6c0f7f64c7f9c003cb1", "shasum": "" }, "require": { - "nativephp/mobile": "^3.0 || ^4.0", + "nativephp/mobile": "^3.3.11 || ^4.6.1", "php": "^8.2" }, "require-dev": { @@ -3002,22 +3002,22 @@ "description": "Media player plugin for NativePHP Mobile - shared audio/video playback, full-screen presentation, and a video-player element", "support": { "issues": "https://github.com/NativePHP/mobile-media-player/issues", - "source": "https://github.com/NativePHP/mobile-media-player/tree/1.0.2" + "source": "https://github.com/NativePHP/mobile-media-player/tree/1.1.0" }, - "time": "2026-09-15T18:59:52+00:00" + "time": "2026-10-29T09:15:16+00:00" }, { "name": "nativephp/mobile-microphone", - "version": "1.1.0", + "version": "1.2.0", "source": { "type": "git", "url": "https://github.com/NativePHP/mobile-microphone.git", - "reference": "94603d36388fb9b9f897fe146c0b8555e9c3c918" + "reference": "9234a6cbb9f5bca853f5ee0c95d8781e10afde08" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/NativePHP/mobile-microphone/zipball/94603d36388fb9b9f897fe146c0b8555e9c3c918", - "reference": "94603d36388fb9b9f897fe146c0b8555e9c3c918", + "url": "https://api.github.com/repos/NativePHP/mobile-microphone/zipball/9234a6cbb9f5bca853f5ee0c95d8781e10afde08", + "reference": "9234a6cbb9f5bca853f5ee0c95d8781e10afde08", "shasum": "" }, "require": { @@ -3057,9 +3057,9 @@ "description": "Microphone recording plugin for NativePHP Mobile - audio recording with pause/resume support", "support": { "issues": "https://github.com/NativePHP/mobile-microphone/issues", - "source": "https://github.com/NativePHP/mobile-microphone/tree/1.1.0" + "source": "https://github.com/NativePHP/mobile-microphone/tree/1.2.0" }, - "time": "2026-09-03T18:26:19+00:00" + "time": "2026-10-29T09:22:03+00:00" }, { "name": "nativephp/mobile-secure-storage", @@ -11394,8 +11394,8 @@ { "package": "nativephp/mobile", "version": "dev-main", - "alias": "4.6.0", - "alias_normalized": "4.6.0.0" + "alias": "4.6.1", + "alias_normalized": "4.6.1.0" }, { "package": "nativephp/mobile-ui", From bc002b90800bbefaab8b658e094a8acd4878e99a Mon Sep 17 00:00:00 2001 From: Simon Hamp Date: Thu, 29 Oct 2026 10:20:53 +0000 Subject: [PATCH 4/6] Open photos, voice notes and Ping from one + in the message bar Three buttons beside the field took too much of its room, so they're one + that opens them in a menu, in the web's order. Attach photos stops at three, and the voice note says it goes on its own while there's text or photos, as the web's button does. Co-Authored-By: Claude Opus 5.5 --- app/Devnull/ComposerMenu.php | 40 +++++++++++++++++++ resources/views/native/conversation.blade.php | 13 +++--- .../native/partials/composer-tool.blade.php | 10 ++++- tests/Feature/Screens/RemotesComposerTest.php | 15 +++---- .../Feature/Screens/RemotesVoiceNotesTest.php | 12 +++--- 5 files changed, 70 insertions(+), 20 deletions(-) create mode 100644 app/Devnull/ComposerMenu.php diff --git a/app/Devnull/ComposerMenu.php b/app/Devnull/ComposerMenu.php new file mode 100644 index 00000000..f5b56afd --- /dev/null +++ b/app/Devnull/ComposerMenu.php @@ -0,0 +1,40 @@ + + */ + public static function items(bool $photosFull, bool $voiceNotesGoAlone, string $pingLabel): array + { + return [ + self::item('attach-photos', 'Attach up to 3 photos', Lucide::ImagePlus, 'pickPhotos', disabled: $photosFull), + self::item('record-voice-note', $voiceNotesGoAlone ? 'Voice notes go on their own' : 'Record a voice note (max. 1 min)', Lucide::Mic, 'recordVoiceNote', disabled: $voiceNotesGoAlone), + self::item('ping', $pingLabel, Lucide::Vibrate, 'ping'), + ]; + } + + private static function item(string $ref, string $label, Lucide $icon, string $method, bool $disabled = false): TopBarAction + { + return NavAction::make($ref) + ->label($label) + ->icon(ios: $icon->ios(), android: $icon->android()) + ->disabled($disabled) + ->press($method) + ->toElement() + ->ref($ref); + } +} diff --git a/resources/views/native/conversation.blade.php b/resources/views/native/conversation.blade.php index bd1502e3..79dc4321 100644 --- a/resources/views/native/conversation.blade.php +++ b/resources/views/native/conversation.blade.php @@ -75,7 +75,7 @@ {{-- The composer, drawn as the reply bar on a commit's page is (see partials/composer-plate): on iOS the bar is bare and the field is a Liquid Glass plate with the tools inside it, on Android the bar is the surface with - a hairline and the tools lead it. The tools are the web's, in its order: photos, the microphone, then Ping. Above the field + a hairline and the tools lead it. The web's tools, photos, the microphone and Ping, open from one + here. Above the field sit what you're replying to and the photos on the draft, on glass of their own on iOS. While the thread can't take a message, the bar is solid on both, with why in the field's place. --}} @@ -107,15 +107,16 @@ {{-- The same room above and below the field, so the bar sits evenly on the keyboard. --}} - {{-- A voice note takes the field's place, and the tools', while it's recorded and waits to be pushed. --}} - @php($voiceNotesGoAlone = trim($draft) !== '' || $photos !== []) + {{-- + The web's tools, photos, a voice note and Ping, are one + here, which opens them in a menu (see + ComposerMenu), so the field keeps its room. A voice note takes the field's place, and the +'s, + while it's recorded and waits to be pushed. + --}} @include('native.partials.composer-plate', [ 'ref' => 'message', 'field' => $recorder === null ? 'native.partials.message-field' : 'native.partials.voice-recorder', 'tools' => $recorder !== null ? [] : [ - ['ref' => 'attach-photos', 'icon' => \App\Icons\Lucide::ImagePlus, 'label' => 'Attach up to 3 photos', 'press' => 'pickPhotos', 'disabled' => count($photos) >= \App\Devnull\DraftImages::MaxImages], - ['ref' => 'record-voice-note', 'icon' => \App\Icons\Lucide::Mic, 'label' => $voiceNotesGoAlone ? 'Voice notes go on their own' : 'Record a voice note (max. 1 min)', 'press' => 'recordVoiceNote', 'disabled' => $voiceNotesGoAlone], - ['ref' => 'ping', 'icon' => \App\Icons\Lucide::Vibrate, 'label' => $pingLabel, 'press' => 'ping'], + ['ref' => 'composer-tools', 'icon' => \App\Icons\Lucide::Plus, 'label' => 'Photos, a voice note or a ping', 'menu' => \App\Devnull\ComposerMenu::items(count($photos) >= \App\Devnull\DraftImages::MaxImages, trim($draft) !== '' || $photos !== [], $pingLabel)], ], ]) @php($length = mb_strlen($draft)) diff --git a/resources/views/native/partials/composer-tool.blade.php b/resources/views/native/partials/composer-tool.blade.php index 475cb1f1..a5318333 100644 --- a/resources/views/native/partials/composer-tool.blade.php +++ b/resources/views/native/partials/composer-tool.blade.php @@ -1,10 +1,18 @@ {{-- One of the composer's tools, as the web's ghost icon buttons: a 44pt target with the icon in muted ink, on Blue Wash in Commit Blue while it's on ($active). While it can't be used ($disabled), it's faded, with nothing to press. + With $menu (TopBarAction items), a tap opens that menu instead of pressing. --}} @php($active ??= false) @php($disabled ??= false) -@if ($disabled) +@php($menu ??= null) +@if ($menu !== null) + + + + + +@elseif ($disabled) diff --git a/tests/Feature/Screens/RemotesComposerTest.php b/tests/Feature/Screens/RemotesComposerTest.php index 1388fa44..731d9788 100644 --- a/tests/Feature/Screens/RemotesComposerTest.php +++ b/tests/Feature/Screens/RemotesComposerTest.php @@ -190,7 +190,7 @@ function threadWithAda(string $platform = 'ios'): TestableComponent photosPicked($screen, [testPhoto(), testPhoto()]); - expect(nodeByRef($screen, 'attach-photos')['type'])->not->toBe('pressable'); + expect(nodeByRef($screen, 'attach-photos')['props']['disabled'] ?? false)->toBeTrue(); }); it('takes a photo off the draft and the phone with its ✕', function () { @@ -252,25 +252,26 @@ function threadWithAda(string $platform = 'ios'): TestableComponent ->and(childRefs($screen, 'composer-bar'))->toBe($rows); }); -it('puts the photo, microphone and Ping buttons inside the Liquid Glass field on iOS, as the commit reply bar does', function () { +it('puts a + for photos, a voice note and Ping inside the Liquid Glass field on iOS', function () { serveThreadWithAda([remoteMessageJson(['id' => 1])], ['read_through_id' => 1]); $screen = threadWithAda('ios'); $bar = findNode($screen->tree(), fn (array $node): bool => ($node['type'] ?? null) === 'bottom_bar'); - expect(childRefs($screen, 'message-plate'))->toBe(['message-field', 'attach-photos', 'record-voice-note', 'ping']) + expect(childRefs($screen, 'message-plate'))->toBe(['message-field', 'composer-tools']) + ->and(array_slice(childRefs($screen, 'composer-tools'), 0, 3))->toBe(['attach-photos', 'record-voice-note', 'ping']) ->and(nodeByRef($screen, 'message-plate')['props']['glass'] ?? null)->toBe(1) ->and($bar['children'][0]['ref'] ?? null)->toBe('composer-bar') ->and($bar['children'][0]['style']['bg_color'] ?? null)->toBeNull() - ->and(nodeByRef($screen, 'ping')['props']['a11y_label'])->toBe('Ping @ada'); + ->and(nodeByRef($screen, 'ping')['props']['label'])->toBe('Ping @ada'); }); -it('leads the bar with the photo, microphone and Ping buttons on Android, on a solid bar', function () { +it('leads the bar with the + on Android, on a solid bar', function () { serveThreadWithAda([remoteMessageJson(['id' => 1])], ['read_through_id' => 1]); $screen = threadWithAda('android'); - expect(array_slice(childRefs($screen, 'message-row'), 0, 4))->toBe(['attach-photos', 'record-voice-note', 'ping', 'message-plate']) + expect(array_slice(childRefs($screen, 'message-row'), 0, 2))->toBe(['composer-tools', 'message-plate']) ->and(childRefs($screen, 'message-plate'))->toBe(['message-field']) ->and(nodeByRef($screen, 'message-plate')['props']['glass'] ?? null)->toBeNull() ->and(nodeByRef($screen, 'composer-bar')['style']['bg_color'] ?? null)->toBe(config('native-ui.theme.light.surface')); @@ -354,7 +355,7 @@ function threadWithAda(string $platform = 'ios'): TestableComponent $screen = finishRefreshes(Native::visit('/remotes/orgs/org-1'))->assertSee('* @ada sent a ping'); - expect(nodeByRef($screen, 'ping')['props']['a11y_label'])->toBe('Ping everyone in the org'); + expect(nodeByRef($screen, 'ping')['props']['label'])->toBe('Ping everyone in the org'); finishRefreshes($screen->tap('ping'))->assertSee('* you sent a ping'); diff --git a/tests/Feature/Screens/RemotesVoiceNotesTest.php b/tests/Feature/Screens/RemotesVoiceNotesTest.php index 12ec183b..6d342706 100644 --- a/tests/Feature/Screens/RemotesVoiceNotesTest.php +++ b/tests/Feature/Screens/RemotesVoiceNotesTest.php @@ -159,14 +159,14 @@ function playedBars(TestableComponent $screen, string $ref): int return count(array_filter($bars, fn (array $node): bool => ($node['style']['opacity'] ?? 1) == 1)); } -it('offers the microphone between photos and Ping while the draft is empty, since voice notes go on their own', function (Closure $draft, bool $offered, string $label) { +it("offers a voice note in the +'s menu, between photos and Ping, while the draft is empty, since voice notes go on their own", function (Closure $draft, bool $offered, string $label) { serveVoiceThread([remoteMessageJson(['id' => 1])]); $screen = $draft(voiceThread()); - expect(childRefs($screen, 'message-plate'))->toBe(['message-field', 'attach-photos', 'record-voice-note', 'ping']) - ->and(nodeByRef($screen, 'record-voice-note')['type'] === 'pressable')->toBe($offered) - ->and(nodeByRef($screen, 'record-voice-note')['props']['a11y_label'])->toBe($label); + expect(array_slice(childRefs($screen, 'composer-tools'), 0, 3))->toBe(['attach-photos', 'record-voice-note', 'ping']) + ->and(nodeByRef($screen, 'record-voice-note')['props']['disabled'] ?? false)->toBe(! $offered) + ->and(nodeByRef($screen, 'record-voice-note')['props']['label'])->toBe($label); })->with([ 'an empty draft' => [fn (TestableComponent $screen) => $screen, true, 'Record a voice note (max. 1 min)'], 'some text' => [fn (TestableComponent $screen) => $screen->input('message-input', 'one sec'), false, 'Voice notes go on their own'], @@ -245,7 +245,7 @@ function playedBars(TestableComponent $screen, string $ref): int $screen->tap('push')->assertSee('staged')->assertSet('recorder', null)->assertNativeCalled('MediaPlayer.Stop'); $clientId = $screen->instance()->pending[0]->clientId; - expect(childRefs($screen, 'message-plate'))->toBe(['message-field', 'attach-photos', 'record-voice-note', 'ping']) + expect(childRefs($screen, 'message-plate'))->toBe(['message-field', 'composer-tools']) ->and(nodeByRef($screen, 'voice-'.$clientId.'-clock')['props']['text'])->toBe('0:02'); finishRefreshes($screen)->assertSee('pushed')->assertSet('pending', []); @@ -321,7 +321,7 @@ function playedBars(TestableComponent $screen, string $ref): int } expect(file_exists($recording))->toBeFalse() - ->and(childRefs($screen, 'message-plate'))->toBe(['message-field', 'attach-photos', 'record-voice-note', 'ping']) + ->and(childRefs($screen, 'message-plate'))->toBe(['message-field', 'composer-tools']) ->and(voiceUploads())->toBe([]); })->with(['while recording' => [true], 'once recorded' => [false]]); From ba270860b4e965d72100e39b8eb60281005171da Mon Sep 17 00:00:00 2001 From: Simon Hamp Date: Thu, 29 Oct 2026 10:24:47 +0000 Subject: [PATCH 5/6] Give the + menu's items a menu's short labels Co-Authored-By: Claude Opus 5.5 --- app/Devnull/ComposerMenu.php | 13 +++++++------ tests/Feature/Screens/RemotesVoiceNotesTest.php | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/app/Devnull/ComposerMenu.php b/app/Devnull/ComposerMenu.php index f5b56afd..71e5716e 100644 --- a/app/Devnull/ComposerMenu.php +++ b/app/Devnull/ComposerMenu.php @@ -8,10 +8,11 @@ /** * What the + beside a Remotes thread's message field opens: the web's - * composer tools, which there sit in a row of their own, in its order. - * Attach photos, until there are three; Record a voice note, while the - * draft has no text or photos, since voice notes go on their own, which - * it then says in its place; and Ping. + * composer tools, which there sit in a row of their own, in its order, with + * shorter labels than its tooltips, as a menu's are. Attach photos, until + * there are three; Record a voice note, while the draft has no text or + * photos, since voice notes go on their own, which it then says in its + * place; and Ping. */ final class ComposerMenu { @@ -21,8 +22,8 @@ final class ComposerMenu public static function items(bool $photosFull, bool $voiceNotesGoAlone, string $pingLabel): array { return [ - self::item('attach-photos', 'Attach up to 3 photos', Lucide::ImagePlus, 'pickPhotos', disabled: $photosFull), - self::item('record-voice-note', $voiceNotesGoAlone ? 'Voice notes go on their own' : 'Record a voice note (max. 1 min)', Lucide::Mic, 'recordVoiceNote', disabled: $voiceNotesGoAlone), + self::item('attach-photos', 'Attach photos', Lucide::ImagePlus, 'pickPhotos', disabled: $photosFull), + self::item('record-voice-note', $voiceNotesGoAlone ? 'Voice notes go on their own' : 'Record a voice note', Lucide::Mic, 'recordVoiceNote', disabled: $voiceNotesGoAlone), self::item('ping', $pingLabel, Lucide::Vibrate, 'ping'), ]; } diff --git a/tests/Feature/Screens/RemotesVoiceNotesTest.php b/tests/Feature/Screens/RemotesVoiceNotesTest.php index 6d342706..83759e73 100644 --- a/tests/Feature/Screens/RemotesVoiceNotesTest.php +++ b/tests/Feature/Screens/RemotesVoiceNotesTest.php @@ -168,7 +168,7 @@ function playedBars(TestableComponent $screen, string $ref): int ->and(nodeByRef($screen, 'record-voice-note')['props']['disabled'] ?? false)->toBe(! $offered) ->and(nodeByRef($screen, 'record-voice-note')['props']['label'])->toBe($label); })->with([ - 'an empty draft' => [fn (TestableComponent $screen) => $screen, true, 'Record a voice note (max. 1 min)'], + 'an empty draft' => [fn (TestableComponent $screen) => $screen, true, 'Record a voice note'], 'some text' => [fn (TestableComponent $screen) => $screen->input('message-input', 'one sec'), false, 'Voice notes go on their own'], 'a photo' => [fn (TestableComponent $screen) => photosPicked($screen, [testPhoto()]), false, 'Voice notes go on their own'], ]); From f90ad5c5b1cd8404b232a4f72eeee9b0e0c0459f Mon Sep 17 00:00:00 2001 From: Simon Hamp Date: Thu, 29 Oct 2026 10:31:08 +0000 Subject: [PATCH 6/6] Let VoiceOver read the + by its label iOS's menu reads its icon rather than the pressable's label, so the icon carries the label too. Co-Authored-By: Claude Opus 5.5 --- resources/views/native/partials/composer-tool.blade.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/resources/views/native/partials/composer-tool.blade.php b/resources/views/native/partials/composer-tool.blade.php index a5318333..92027763 100644 --- a/resources/views/native/partials/composer-tool.blade.php +++ b/resources/views/native/partials/composer-tool.blade.php @@ -7,9 +7,10 @@ @php($disabled ??= false) @php($menu ??= null) @if ($menu !== null) + {{-- iOS's menu reads its icon to VoiceOver rather than the pressable's label, so the icon carries it too. --}} - + @elseif ($disabled)