From 93829cd638a55bc597010d9f732d66248aa30805 Mon Sep 17 00:00:00 2001 From: Salastil Date: Mon, 28 Sep 2026 17:11:33 -0400 Subject: [PATCH 01/18] Key the build cache on the runner image too Pinning the Linux job to ubuntu-22.04, so the daemon links against a glibc old enough for the distributions moho claims to support, changed what builds the artifacts without changing anything the cache key looked at. The next push restored a target directory compiled on 24.04 under an exact-match key, cargo took the crates it named to be already built, and the build died on "can't find crate for tor_bytes" - a dependency whose rlib the restored tree did not contain. The Windows job passed the same run. It is still on ubuntu-latest and restored a cache written on ubuntu-latest, which is how the image turned out to be the variable rather than anything in the tree: the lockfile, the manifest and the compiler were identical to the last green build. Both target caches now carry the image, including the Windows one - it cross-compiles the binary but still builds host-side build scripts and proc macros into the same directory. The restore-keys are scoped the same way, since falling back across images is the identical trap one step down. Co-Authored-By: Claude Opus 5 --- .github/workflows/rolling-release.yml | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/.github/workflows/rolling-release.yml b/.github/workflows/rolling-release.yml index ddca021..dd4ba33 100644 --- a/.github/workflows/rolling-release.yml +++ b/.github/workflows/rolling-release.yml @@ -166,14 +166,26 @@ jobs: # hit on every push after that, so the only thing recompiled per push is # nobilis itself, which is the few minutes that genuinely cannot be # avoided. + # + # ImageOS is in the key because the runner image is part of what built + # these artifacts, and pinning this job to 22.04 changed it without + # changing anything the key looked at. The next run restored a tree + # compiled on 24.04 under an exact-match key, cargo believed the crates + # it named were already built, and the build died on "can't find crate + # for tor_bytes" - a dependency whose rlib the restored tree did not + # have. The Windows job, still on ubuntu-latest, restored its own + # matching cache and passed, which is how the image turned out to be + # the variable. - name: cache the cargo target directory uses: actions/cache@v4 with: path: nobilis/target - key: cargo-target-linux-${{ hashFiles('nobilis/Cargo.lock') }}-${{ env.RUSTC_VERSION }} + key: cargo-target-linux-${{ env.ImageOS }}-${{ hashFiles('nobilis/Cargo.lock') }}-${{ env.RUSTC_VERSION }} # A lockfile bump should start from the previous tree and rebuild - # what actually changed, rather than starting from nothing. - restore-keys: cargo-target-linux- + # what actually changed, rather than starting from nothing. Scoped + # to the image for the reason above: falling back across images is + # the same trap one step further down. + restore-keys: cargo-target-linux-${{ env.ImageOS }}- # The Electron binary, and the appimagetool and nsis bundles # electron-builder fetches at pack time. Small, and it removes a real @@ -378,8 +390,11 @@ jobs: uses: actions/cache@v4 with: path: nobilis/target - key: cargo-target-windows-gnu-${{ hashFiles('nobilis/Cargo.lock') }}-${{ env.RUSTC_VERSION }} - restore-keys: cargo-target-windows-gnu- + # Scoped to the image for the same reason as the Linux job's: this + # directory also holds host-built artifacts - build scripts and + # proc macros - even though the binary itself is cross-compiled. + key: cargo-target-windows-gnu-${{ env.ImageOS }}-${{ hashFiles('nobilis/Cargo.lock') }}-${{ env.RUSTC_VERSION }} + restore-keys: cargo-target-windows-gnu-${{ env.ImageOS }}- - name: cache electron downloads uses: actions/cache@v4 From fa353fbf35c25b4746a0e273728a35217996c2c1 Mon Sep 17 00:00:00 2001 From: Salastil Date: Mon, 28 Sep 2026 19:46:42 -0400 Subject: [PATCH 02/18] Pick up the warning cleanup Co-Authored-By: Claude Opus 5 --- nobilis | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nobilis b/nobilis index bda039d..d0efc2a 160000 --- a/nobilis +++ b/nobilis @@ -1 +1 @@ -Subproject commit bda039d6939495664d78eded47bd983a27dd19d3 +Subproject commit d0efc2ad17b9c28dfd1cd34a6a496c0b6f23a755 From fb7f22409195060616c64ae877a1dcea83d1564b Mon Sep 17 00:00:00 2001 From: Salastil Date: Mon, 28 Sep 2026 20:36:47 -0400 Subject: [PATCH 03/18] Pick up DCC RESUME Co-Authored-By: Claude Opus 5 --- nobilis | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nobilis b/nobilis index d0efc2a..39216b8 160000 --- a/nobilis +++ b/nobilis @@ -1 +1 @@ -Subproject commit d0efc2ad17b9c28dfd1cd34a6a496c0b6f23a755 +Subproject commit 39216b884053e7b7f7e1a4d1921a5986dbbe3af1 From d250d984568c255d22785a3a870e10a8be849ccd Mon Sep 17 00:00:00 2001 From: Salastil Date: Mon, 28 Sep 2026 20:48:26 -0400 Subject: [PATCH 04/18] Pick up passive DCC and wide acknowledgements Co-Authored-By: Claude Opus 5 --- nobilis | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nobilis b/nobilis index 39216b8..6f11176 160000 --- a/nobilis +++ b/nobilis @@ -1 +1 @@ -Subproject commit 39216b884053e7b7f7e1a4d1921a5986dbbe3af1 +Subproject commit 6f11176909f66cdc1d7109f4e4bff5bf22eae621 From 61a5b83f691c4aae3a47cc83b5254a9c613ea4f7 Mon Sep 17 00:00:00 2001 From: Salastil Date: Mon, 28 Sep 2026 21:03:13 -0400 Subject: [PATCH 05/18] Pick up automatic DCC retry Co-Authored-By: Claude Opus 5 --- nobilis | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nobilis b/nobilis index 6f11176..a84c6cc 160000 --- a/nobilis +++ b/nobilis @@ -1 +1 @@ -Subproject commit 6f11176909f66cdc1d7109f4e4bff5bf22eae621 +Subproject commit a84c6cc274d1b49c215b2dd944c2d3a23687f3b9 From fe576caa23ec5872765484c0cc4a85f5528c9608 Mon Sep 17 00:00:00 2001 From: Salastil Date: Tue, 29 Sep 2026 18:54:45 -0400 Subject: [PATCH 06/18] Offer suggested room directories in public room search Co-Authored-By: Claude Sonnet 5.5 --- nobilis | 2 +- src/renderer/src/app.css | 8 +++ src/renderer/src/components/RoomSearch.tsx | 57 ++++++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/nobilis b/nobilis index a84c6cc..ebcac09 160000 --- a/nobilis +++ b/nobilis @@ -1 +1 @@ -Subproject commit a84c6cc274d1b49c215b2dd944c2d3a23687f3b9 +Subproject commit ebcac096cb61ca7fd7725a76055c4e162de01ea8 diff --git a/src/renderer/src/app.css b/src/renderer/src/app.css index 6776e74..35ff22b 100644 --- a/src/renderer/src/app.css +++ b/src/renderer/src/app.css @@ -7085,3 +7085,11 @@ video.lightbox-media { gap: var(--spacing-xs); color: var(--surface-variant-text); } + +.room-search-suggested { padding: var(--spacing-xs) var(--spacing-m) var(--spacing-s); display: flex; flex-direction: column; gap: var(--spacing-xs); } +.room-search-suggested-toggle, +.room-search-suggested-all { align-self: flex-start; background: none; border: 0; padding: 0; color: var(--primary); cursor: pointer; font: inherit; } +.room-search-suggested-list { display: grid; grid-template-columns: repeat(auto-fill, minmax(260px, 1fr)); gap: var(--spacing-xxs) var(--spacing-m); max-height: 180px; overflow-y: auto; } +.room-search-suggestion { display: flex; flex-direction: column; align-items: flex-start; text-align: left; background: none; border: 0; padding: var(--spacing-xxs) 0; color: inherit; cursor: pointer; font: inherit; } +.room-search-suggestion:hover .room-search-suggestion-name { text-decoration: underline; } +.room-search-suggestion-name { font-weight: 600; } diff --git a/src/renderer/src/components/RoomSearch.tsx b/src/renderer/src/components/RoomSearch.tsx index 44dfef1..6888654 100644 --- a/src/renderer/src/components/RoomSearch.tsx +++ b/src/renderer/src/components/RoomSearch.tsx @@ -89,6 +89,19 @@ export function RoomSearch({ account, onClose }: { account: Account; onClose: () * across restarts, until it is taken off the line. */ const [extraServers, setExtraServers] = usePref('roomSearchServers', []) + /** + * Directories the daemon suggests, for somebody who has not yet heard of + * any of them. Asked of the daemon because the list is knowledge about + * Matrix rather than about this window, and the next client will want it too. + */ + const [suggested, setSuggested] = useState<{ server: string; about: string }[]>([]) + const [showSuggested, setShowSuggested] = useState(false) + useEffect(() => { + void window.moho + .rpc<{ server: string; about: string }[]>('suggestedMatrixDirectories', {}) + .then((list) => setSuggested(Array.isArray(list) ? list : [])) + .catch(() => setSuggested([])) + }, []) /** What is in the box, which is not yet a server until Enter says so. */ const [pendingServer, setPendingServer] = useState('') const [answer, setAnswer] = useState(null) @@ -143,6 +156,9 @@ export function RoomSearch({ account, onClose }: { account: Account; onClose: () if (!extraServers.includes(name)) setExtraServers([...extraServers, name]) } + /** Suggestions not yet on the line - the ones still worth offering. */ + const unadded = suggested.filter((d) => !extraServers.includes(d.server)) + /** * Every server worth a chip: the ones that answered, and the ones added by * hand that have not yet. @@ -377,6 +393,47 @@ export function RoomSearch({ account, onClose }: { account: Account; onClose: () })} + {/* Where to look beyond your own server. Closed until asked for, and + nothing in it is searched until it is added: each one is a request + to somebody else's server. */} + {unadded.length > 0 && ( +
+ + {showSuggested && ( + <> +
+ {unadded.map((d) => ( + + ))} +
+ + + )} +
+ )} +
{rooms.map((room) => (
From be810265dbcdac8b3a5e58aaa6a4f701bb2df8e9 Mon Sep 17 00:00:00 2001 From: Salastil Date: Tue, 29 Sep 2026 19:02:25 -0400 Subject: [PATCH 07/18] Open Look under its room, and only where the room allows it Co-Authored-By: Claude Sonnet 5.5 --- nobilis | 2 +- src/renderer/src/components/RoomPeek.tsx | 12 +++++++-- src/renderer/src/components/RoomSearch.tsx | 31 +++++++++++++--------- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/nobilis b/nobilis index ebcac09..5c8f8ac 160000 --- a/nobilis +++ b/nobilis @@ -1 +1 @@ -Subproject commit ebcac096cb61ca7fd7725a76055c4e162de01ea8 +Subproject commit 5c8f8ac4436ff7eddf870135a72ee9d718ef4a3c diff --git a/src/renderer/src/components/RoomPeek.tsx b/src/renderer/src/components/RoomPeek.tsx index d486e9e..f3f4edc 100644 --- a/src/renderer/src/components/RoomPeek.tsx +++ b/src/renderer/src/components/RoomPeek.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { Icon } from './Icon' import { formatTime } from '../lib/util' import type { Account } from '../../../shared/wire' @@ -53,6 +53,11 @@ export function RoomPeek({ const [lines, setLines] = useState(null) const [failed, setFailed] = useState('') const [refused, setRefused] = useState('') + const box = useRef(null) + + // Brought into view: a panel that opens below the fold is indistinguishable + // from a button that does nothing. + useEffect(() => box.current?.scrollIntoView({ block: 'nearest', behavior: 'smooth' }), []) useEffect(() => { void window.moho @@ -76,7 +81,7 @@ export function RoomPeek({ }, [account.id, room]) return ( -
+
{summary?.name || summary?.alias || room} @@ -113,6 +118,9 @@ export function RoomPeek({
)} {refused &&
{refused}
} + {summary.worldReadable && !lines && !refused && ( +
Reading the latest messages…
+ )} {lines?.length === 0 &&
Nothing has been said here.
} {lines && lines.length > 0 && ( diff --git a/src/renderer/src/components/RoomSearch.tsx b/src/renderer/src/components/RoomSearch.tsx index 6888654..4016cb8 100644 --- a/src/renderer/src/components/RoomSearch.tsx +++ b/src/renderer/src/components/RoomSearch.tsx @@ -1,4 +1,4 @@ -import { useEffect, useMemo, useRef, useState } from 'react' +import { Fragment, useEffect, useMemo, useRef, useState } from 'react' import { ReasonPrompt } from './ReasonPrompt' import { createPortal } from 'react-dom' import { Avatar } from './Avatar' @@ -23,6 +23,8 @@ interface PublicRoom { * One button for all three is a button that fails for two of them. */ joinRule?: string + /** Whether the room itself allows being read without joining it. */ + worldReadable?: boolean } /** One homeserver that was asked, and what came of asking it. */ @@ -436,7 +438,8 @@ export function RoomSearch({ account, onClose }: { account: Account; onClose: ()
{rooms.map((room) => ( -
+ +
@@ -461,10 +464,11 @@ export function RoomSearch({ account, onClose }: { account: Account; onClose: () and leaving again leaves both behind - so the two decisions have to be separable, and on a room already joined there is nothing to separate. */} - {!room.joined && ( + {!room.joined && room.worldReadable && (
+ {/* Under the room it is about. It used to open after the last row of + results, which for anything but the bottom room was off-screen - + so pressing Look appeared to do nothing at all. */} + {peeking?.roomId === room.roomId && ( + setPeeking(null)} + /> + )} + ))} - {peeking && ( - setPeeking(null)} - /> - )} - {knocking && ( Date: Tue, 29 Sep 2026 19:37:17 -0400 Subject: [PATCH 08/18] Stop the peek panel handing React a cleanup that is not a function Co-Authored-By: Claude Sonnet 5.5 --- src/renderer/src/components/RoomPeek.tsx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/renderer/src/components/RoomPeek.tsx b/src/renderer/src/components/RoomPeek.tsx index f3f4edc..487ab0c 100644 --- a/src/renderer/src/components/RoomPeek.tsx +++ b/src/renderer/src/components/RoomPeek.tsx @@ -57,7 +57,12 @@ export function RoomPeek({ // Brought into view: a panel that opens below the fold is indistinguishable // from a button that does nothing. - useEffect(() => box.current?.scrollIntoView({ block: 'nearest', behavior: 'smooth' }), []) + // A block body, and no return: an effect's return value is its cleanup, so + // an expression body here handed React whatever scrollIntoView answered and + // it tried to call that when the panel closed. + useEffect(() => { + box.current?.scrollIntoView({ block: 'nearest', behavior: 'smooth' }) + }, []) useEffect(() => { void window.moho From 654a56075679e9ebf16bceb421364b33c33e0fd3 Mon Sep 17 00:00:00 2001 From: Salastil Date: Tue, 29 Sep 2026 19:43:39 -0400 Subject: [PATCH 09/18] Look in the conversation pane, with a bar in place of the input box The peek used to open inside the search dialog. It now stands where a conversation does, read-only, with a bar where the composer would be that says what it is and holds Join. Pressing Join takes the bar away and brings the box back, and the room opens for real when it arrives. Co-Authored-By: Claude Sonnet 5.5 --- src/renderer/src/App.tsx | 42 ++-- src/renderer/src/app.css | 80 +------- src/renderer/src/components/PeekView.tsx | 222 +++++++++++++++++++++ src/renderer/src/components/RoomPeek.tsx | 146 -------------- src/renderer/src/components/RoomSearch.tsx | 35 ++-- src/renderer/src/state/store.ts | 77 ++++++- 6 files changed, 350 insertions(+), 252 deletions(-) create mode 100644 src/renderer/src/components/PeekView.tsx delete mode 100644 src/renderer/src/components/RoomPeek.tsx diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index 222a2ac..c58eb8f 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -18,6 +18,7 @@ import { AccountsPanel } from './components/AccountsPanel' import { SettingsPanel } from './components/settings/SettingsPanel' import { DownloadsPanel } from './components/DownloadsPanel' import { JoinPanel } from './components/JoinPanel' +import { PeekBar, PeekView } from './components/PeekView' import { Toasts } from './components/Toasts' import { IncomingCallPanel } from './components/IncomingCallPanel' import { CallStage, IncomingMatrixCall, ScreenPicker } from './components/CallStage' @@ -82,6 +83,10 @@ export default function App(): JSX.Element { * composer all belong to a buffer that is not on screen. */ const onMentionsPage = activePanel === '' && activeGroupId === MENTIONS_GROUP_ID + // A room being looked into stands where the conversation would. It has no + // buffer, so everything below that acts on the open conversation is off. + const peek = useChat((s) => s.peek) + const peeking = !!peek && activePanel === '' // Which Kick channels are actually being watched, told to the daemon as it // changes. Kick counts watch time from an authenticated subscription and @@ -99,7 +104,7 @@ export default function App(): JSX.Element { accounts.find((a) => a.id === allBuffers.find((b) => b.id === id)?.accountId)?.service const now = watchedKickBuffers( allBuffers, - activePanel === '' && !onMentionsPage ? activeBufferId : '', + activePanel === '' && !onMentionsPage && !peeking ? activeBufferId : '', watching?.bufferId, serviceOf ) @@ -113,17 +118,19 @@ export default function App(): JSX.Element { // daemon answers rather than treats as a failure. void window.moho.rpc('setKickWatching', { bufferId, watching: true }).catch(() => {}) } - }, [booted, allBuffers, accounts, activeBufferId, activePanel, onMentionsPage, watching]) + }, [booted, allBuffers, accounts, activeBufferId, activePanel, onMentionsPage, peeking, watching]) const openThread = useChat((s) => s.openThread) // A stale restored id is harmless: it simply resolves to no buffer and the // empty state shows, exactly as it would for "". const showNickList = - !userListFolded && activePanel === '' && !onMentionsPage && buffer?.kind === 'channel' + !userListFolded && activePanel === '' && !onMentionsPage && !peeking && buffer?.kind === 'channel' const headerTitle = - activePanel === 'accounts' + peeking + ? `Looking · ${peek?.name || peek?.alias || ''}` + : activePanel === 'accounts' ? 'Accounts' : activePanel === 'settings' ? 'Settings' @@ -169,7 +176,7 @@ export default function App(): JSX.Element { somewhere you go to find a conversation; this is where you already are when you want to do something to the one you are reading. */} - {activePanel === '' && !onMentionsPage && buffer ? ( + {activePanel === '' && !onMentionsPage && !peeking && buffer ? ( ) : ( {headerTitle} @@ -179,7 +186,7 @@ export default function App(): JSX.Element { conversation, which the mentions page is not showing - leaving them there would offer to search a channel that isn't on screen. */} - {activePanel === '' && !onMentionsPage && buffer && ( + {activePanel === '' && !onMentionsPage && !peeking && buffer && ( )} @@ -191,7 +198,7 @@ export default function App(): JSX.Element { direct message. */} {activePanel === '' && } - {activePanel === '' && !onMentionsPage && buffer?.kind === 'channel' && ( + {activePanel === '' && !onMentionsPage && !peeking && buffer?.kind === 'channel' && ( } - {activePanel !== '' && ( + {activePanel === '' && !onMentionsPage && !peeking && buffer && } + {(activePanel !== '' || peeking) && ( store.setActivePanel('')} + title={peeking ? 'Stop looking' : 'Back to chat'} + onClick={() => (peeking ? store.stopPeek() : store.setActivePanel(''))} /> )}
@@ -219,10 +226,15 @@ export default function App(): JSX.Element { hasAccounts={accounts.length > 0} hasBuffer={!!activeBufferId} activeGroupId={activeGroupId} + peeking={peeking} />
- {activePanel === '' && activeBufferId !== '' && activeGroupId !== MENTIONS_GROUP_ID && ( + {/* Where the box would be: what this pane is, and the way in. */} + {peeking && } + + {/* No box while looking: there is nobody here to say it as. */} + {activePanel === '' && !peeking && activeBufferId !== '' && activeGroupId !== MENTIONS_GROUP_ID && ( <> {/* Above the box, saying why it will not work here yet. */} {buffer && } @@ -311,12 +323,14 @@ function Body({ activePanel, hasAccounts, hasBuffer, - activeGroupId + activeGroupId, + peeking }: { activePanel: string hasAccounts: boolean hasBuffer: boolean activeGroupId: string + peeking: boolean }): JSX.Element { if (activePanel === 'settings') return // Above the no-accounts case below: a finished download is still worth @@ -326,6 +340,8 @@ function Body({ // show - there is nothing to chat in yet. if (activePanel === 'accounts' || !hasAccounts) return if (activePanel === 'join') return + // A room being looked into: the pane's own view, with nothing to type into. + if (peeking) return // The mentions page replaces the log rather than sitting beside it: it is a // list of places to go, and every row leads into a conversation. if (activeGroupId === MENTIONS_GROUP_ID) return diff --git a/src/renderer/src/app.css b/src/renderer/src/app.css index 35ff22b..e6f4e16 100644 --- a/src/renderer/src/app.css +++ b/src/renderer/src/app.css @@ -6800,77 +6800,6 @@ video.lightbox-media { color: var(--error); } -/* --- Looking into a room ---------------------------------------------- */ - -/* What is in a room, read from outside it. Sits under the list rather than - over it, so the row it came from stays visible - the point of looking is to - decide about that room, and a panel covering it takes away what the decision - is about. */ -.room-peek { - display: flex; - flex-direction: column; - gap: var(--spacing-xs); - margin: var(--spacing-s) 0; - padding: var(--spacing-s); - border: 1px solid var(--outline-medium); - border-radius: var(--corner-radius-small); - background: var(--surface-container-lowest); -} - -.room-peek-head { - display: flex; - align-items: center; - gap: var(--spacing-s); - font-weight: 500; -} - -.room-peek-head .ellipsis { - flex: 1; - min-width: 0; -} - -.room-peek-topic { - color: var(--surface-variant-text); - /* A topic can be a paragraph. Three lines is enough to judge a room by and - short enough that the conversation under it stays on screen. */ - display: -webkit-box; - -webkit-line-clamp: 3; - -webkit-box-orient: vertical; - overflow: hidden; -} - -.room-peek-log { - display: flex; - flex-direction: column; - gap: 2px; - max-height: 240px; - overflow-y: auto; -} - -.room-peek-line { - display: flex; - gap: var(--spacing-s); - align-items: baseline; -} - -.room-peek-from { - flex: none; - max-width: 12ch; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; - color: var(--surface-variant-text); - font-size: 0.82rem; -} - -.room-peek-body { - min-width: 0; - font-size: 0.85rem; - /* Somebody else's text, from a room nobody here has joined: it wraps and - it never stretches the dialog. */ - overflow-wrap: anywhere; -} - /* --- Link previews ---------------------------------------------------- */ /* The picture a homeserver found behind a link. Capped rather than sized to @@ -7093,3 +7022,12 @@ video.lightbox-media { .room-search-suggestion { display: flex; flex-direction: column; align-items: flex-start; text-align: left; background: none; border: 0; padding: var(--spacing-xxs) 0; color: inherit; cursor: pointer; font: inherit; } .room-search-suggestion:hover .room-search-suggestion-name { text-decoration: underline; } .room-search-suggestion-name { font-weight: 600; } + +/* A room being looked into, in the pane a conversation would fill. */ +.peek-view { flex: 1; min-height: 0; display: flex; flex-direction: column; } +.peek-facts { padding: var(--spacing-xs) var(--spacing-m); border-bottom: 1px solid var(--outline-light); } +.peek-bar { flex: none; background: var(--surface-container-high); } +.peek-bar-row { display: flex; align-items: center; gap: var(--spacing-s); padding: var(--spacing-s) var(--spacing-m); } +.peek-bar-text { flex: 1; min-width: 0; display: flex; flex-direction: column; } +.peek-topic { padding: var(--spacing-xs) var(--spacing-m); color: var(--surface-variant-text); border-bottom: 1px solid var(--outline-light); } +.peek-log { flex: 1; min-height: 0; overflow-y: auto; overflow-x: hidden; padding: var(--spacing-s) 0 var(--spacing-m); } diff --git a/src/renderer/src/components/PeekView.tsx b/src/renderer/src/components/PeekView.tsx new file mode 100644 index 0000000..510397f --- /dev/null +++ b/src/renderer/src/components/PeekView.tsx @@ -0,0 +1,222 @@ +import { useEffect, useRef, useState } from 'react' +import { Icon } from './Icon' +import { useChat, useStore } from '../state/hooks' +import { formatFullTime, formatTime, nickColor } from '../lib/util' + +interface Summary { + roomId: string + name: string + topic: string + alias: string + members: number + joinRule: string + membership: string + encrypted: boolean + worldReadable: boolean +} + +interface PeekLine { + id: string + from: string + body: string + isAction: boolean + ts: number +} + +/** + * A look inside a room nobody here has joined, in the place a room is read. + * + * This used to open inside the search dialog, under the row it was about - + * which put a conversation in a window built for choosing one, and left the + * log somewhere it could not be read the way a log is read. It stands where + * the conversation would instead: the same pane, the same row layout, and + * nothing to type into, because there is nobody here to say it as. + * + * Joining is a membership event everybody in the room can see, and leaving + * again leaves both of them behind - so looking and joining are separate acts, + * and the bar under this says which of the two this is. + * + * Two halves, because homeservers permit them separately. The summary is + * answered for any room the server can reach; the conversation needs the room + * to be world-readable *and* this homeserver to hold a copy of it. Where it + * does not, that is said in as many words, because silence there reads as an + * empty room rather than as a door that is shut. + */ +export function PeekView(): JSX.Element | null { + const peek = useChat((s) => s.peek) + const [summary, setSummary] = useState(null) + const [lines, setLines] = useState(null) + const [failed, setFailed] = useState('') + const [refused, setRefused] = useState('') + const scroller = useRef(null) + + const accountId = peek?.accountId + const roomId = peek?.roomId + const via = peek?.via + + useEffect(() => { + if (!accountId || !roomId) return + // Whatever was on screen belongs to the room before this one. + setSummary(null) + setLines(null) + setFailed('') + setRefused('') + let current = true + void window.moho + .rpc('matrixRoomSummary', { accountId, room: roomId, via: via ? [via] : [] }) + .then((answer) => { + if (!current) return + setSummary(answer) + // Only worth asking where the room itself would allow it - a room + // that is not world-readable refuses however willing the homeserver + // is, and an error for that would blame the wrong thing. + if (!answer.worldReadable) return + void window.moho + .rpc<{ messages: PeekLine[] }>('matrixPeekRoom', { + accountId, + roomId: answer.roomId, + limit: 50 + }) + .then((r) => current && setLines(r.messages ?? [])) + .catch((e: Error) => current && setRefused(e.message)) + }) + .catch((e: Error) => current && setFailed(e.message)) + return () => { + current = false + } + }, [accountId, roomId, via]) + + // A log is read from its end. + useEffect(() => { + const el = scroller.current + if (el) el.scrollTop = el.scrollHeight + }, [lines]) + + if (!peek) return null + + return ( +
+ {summary && ( +
+ {summary.members.toLocaleString()} members + {summary.joinRule === 'knock' + ? ' · asks to be knocked on' + : summary.joinRule === 'restricted' + ? ' · open to a space’s members' + : summary.joinRule === 'invite' + ? ' · invitation only' + : ''} + {summary.encrypted ? ' · encrypted' : ''} + {peek.via ? ` · ${peek.via}` : ''} +
+ )} + + {summary?.topic &&
{summary.topic}
} + +
+ {!summary && !failed && ( +
+ + Looking… +
+ )} + {failed &&
Couldn’t look: {failed}
} + + {summary && !summary.worldReadable && ( +
+ This room’s history is only for people who have joined it, so there is nothing to read + from out here. +
+ )} + {refused &&
{refused}
} + {summary?.worldReadable && !lines && !refused && ( +
+ + Reading the latest messages… +
+ )} + {lines?.length === 0 && ( +
Nothing has been said here.
+ )} + + {/* The same row the log draws, without what acts on a message: nothing + here can be replied to, reacted to or quoted from a room you are + not in. */} + {lines?.map((line) => ( +
+ + {formatTime(line.ts)} + +
+ + + {line.isAction ? `* ${line.from}` : line.from} + + + {line.body} +
+
+ ))} +
+
+ ) +} + +/** + * Where the box would be, while a room is only being looked at. + * + * There is nobody here to say anything as, so the box is not offered - this + * stands in its place and says what the pane is, with the one thing that + * turns it into a conversation. Pressing Join takes the banner away and puts + * the box back, inert until the room has actually arrived: the box belongs to + * a conversation, and the room is not one for the moment or two the server + * takes to let somebody in. + */ +export function PeekBar(): JSX.Element | null { + const store = useStore() + const peek = useChat((s) => s.peek) + if (!peek) return null + const title = peek.name || peek.alias || peek.roomId + + if (peek.joining) { + return ( +
+
+
+
+
+ Joining {title}… +
+
+
+
+ ) + } + + return ( +
+
+
+ +
+ + You’re looking at {title} without having joined it. + + Nobody in it can see you here. +
+ {peek.joinRule !== 'knock' && ( + + )} + +
+
+ ) +} diff --git a/src/renderer/src/components/RoomPeek.tsx b/src/renderer/src/components/RoomPeek.tsx deleted file mode 100644 index 487ab0c..0000000 --- a/src/renderer/src/components/RoomPeek.tsx +++ /dev/null @@ -1,146 +0,0 @@ -import { useEffect, useRef, useState } from 'react' -import { Icon } from './Icon' -import { formatTime } from '../lib/util' -import type { Account } from '../../../shared/wire' - -interface Summary { - roomId: string - name: string - topic: string - alias: string - members: number - joinRule: string - membership: string - encrypted: boolean - worldReadable: boolean -} - -interface PeekLine { - id: string - from: string - body: string - isAction: boolean - ts: number -} - -/** - * A look inside a room nobody here has joined. - * - * Joining is a membership event everybody in the room can see, and leaving - * again leaves both of them behind - so without this, the decision to look - * and the decision to join were the same act. - * - * Two halves, because homeservers permit them separately. The summary - - * topic, size, how the room is entered, whether it is encrypted - is - * answered for any room the server can reach, and is most of what somebody - * decides on. The conversation needs the room to be world-readable *and* the - * homeserver to allow peeking at all, which many do not; where it refuses, - * that is said in as many words, because silence there reads as an empty - * room rather than as a door that is shut. - */ -export function RoomPeek({ - account, - room, - via, - onClose -}: { - account: Account - room: string - via?: string - onClose: () => void -}): JSX.Element { - const [summary, setSummary] = useState(null) - const [lines, setLines] = useState(null) - const [failed, setFailed] = useState('') - const [refused, setRefused] = useState('') - const box = useRef(null) - - // Brought into view: a panel that opens below the fold is indistinguishable - // from a button that does nothing. - // A block body, and no return: an effect's return value is its cleanup, so - // an expression body here handed React whatever scrollIntoView answered and - // it tried to call that when the panel closed. - useEffect(() => { - box.current?.scrollIntoView({ block: 'nearest', behavior: 'smooth' }) - }, []) - - useEffect(() => { - void window.moho - .rpc('matrixRoomSummary', { accountId: account.id, room, via: via ? [via] : [] }) - .then((answer) => { - setSummary(answer) - // Only worth asking where the room itself would allow it - a room - // that is not world-readable refuses however willing the homeserver - // is, and an error for that would blame the wrong thing. - if (!answer.worldReadable) return - void window.moho - .rpc<{ messages: PeekLine[] }>('matrixPeekRoom', { - accountId: account.id, - roomId: answer.roomId, - limit: 20 - }) - .then((r) => setLines(r.messages)) - .catch((e: Error) => setRefused(e.message)) - }) - .catch((e: Error) => setFailed(e.message)) - }, [account.id, room]) - - return ( -
-
- - {summary?.name || summary?.alias || room} - -
- - {!summary && !failed &&
Looking…
} - {failed &&
Couldn’t look: {failed}
} - - {summary && ( - <> -
- {summary.members.toLocaleString()} members - {summary.joinRule === 'knock' - ? ' · asks to be knocked on' - : summary.joinRule === 'restricted' - ? ' · open to a space’s members' - : summary.joinRule === 'invite' - ? ' · invitation only' - : ''} - {summary.encrypted ? ' · encrypted' : ''} -
- {summary.topic &&
{summary.topic}
} - - {/* Why there is no conversation below, in each of the two cases - - the room's own rule, and the homeserver's. They are different - answers and only one of them is about this room. */} - {!summary.worldReadable && ( -
- This room’s history is only for people who have joined it, so there is nothing to - read from out here. -
- )} - {refused &&
{refused}
} - {summary.worldReadable && !lines && !refused && ( -
Reading the latest messages…
- )} - - {lines?.length === 0 &&
Nothing has been said here.
} - {lines && lines.length > 0 && ( -
- {lines.map((line) => ( -
- {formatTime(line.ts)} - {line.isAction ? `* ${line.from}` : line.from} - {line.body} -
- ))} -
- )} - - )} -
- ) -} diff --git a/src/renderer/src/components/RoomSearch.tsx b/src/renderer/src/components/RoomSearch.tsx index 4016cb8..25d710f 100644 --- a/src/renderer/src/components/RoomSearch.tsx +++ b/src/renderer/src/components/RoomSearch.tsx @@ -1,8 +1,7 @@ -import { Fragment, useEffect, useMemo, useRef, useState } from 'react' +import { useEffect, useMemo, useRef, useState } from 'react' import { ReasonPrompt } from './ReasonPrompt' import { createPortal } from 'react-dom' import { Avatar } from './Avatar' -import { RoomPeek } from './RoomPeek' import { Icon, IconButton } from './Icon' import { usePref, useStore } from '../state/hooks' import { classes } from '../lib/util' @@ -117,8 +116,6 @@ export function RoomSearch({ account, onClose }: { account: Account; onClose: () /** The room being knocked on, while the reason is being written. */ const [knocking, setKnocking] = useState(null) const [joining, setJoining] = useState>({}) - /** Which room somebody has asked to look into without joining it. */ - const [peeking, setPeeking] = useState(null) const inputRef = useRef(null) const resultsRef = useRef(null) // Which search the answer in flight belongs to. A slow server answering a @@ -438,8 +435,7 @@ export function RoomSearch({ account, onClose }: { account: Account; onClose: ()
{rooms.map((room) => ( - -
+
@@ -468,9 +464,20 @@ export function RoomSearch({ account, onClose }: { account: Account; onClose: () @@ -490,18 +497,6 @@ export function RoomSearch({ account, onClose }: { account: Account; onClose: () : 'Join'}
- {/* Under the room it is about. It used to open after the last row of - results, which for anything but the bottom room was off-screen - - so pressing Look appeared to do nothing at all. */} - {peeking?.roomId === room.roomId && ( - setPeeking(null)} - /> - )} - ))} {knocking && ( diff --git a/src/renderer/src/state/store.ts b/src/renderer/src/state/store.ts index b31dd6d..6bb8b61 100644 --- a/src/renderer/src/state/store.ts +++ b/src/renderer/src/state/store.ts @@ -316,6 +316,19 @@ export interface DiscordModal { export type ActivePanel = '' | 'accounts' | 'settings' | 'join' | 'downloads' +/** A room being looked into without having been joined. */ +export interface PeekTarget { + accountId: string + roomId: string + alias: string + name: string + /** The homeserver that listed it, which is where to ask about it. */ + via: string + joinRule?: string + /** Set once Join has been pressed, until the room arrives or the join fails. */ + joining?: boolean +} + export interface ChatState { /** Who is composing, per buffer, with when to stop believing it. */ typingByBuffer: Record @@ -431,6 +444,11 @@ export interface ChatState { /** Set in a popped-out window, naming the one conversation it may show. */ pinnedBufferId: string activePanel: ActivePanel + /** + * The room being looked at, if any. Shown in the conversation's own pane in + * place of a conversation, and gone the moment one is selected. + */ + peek: PeekTarget | null joinPanelAccountId: string messagesByBuffer: Record presenceByBuffer: Record @@ -610,6 +628,7 @@ const INITIAL: ChatState = { popouts: { open: [], watched: [] }, pinnedBufferId: '', activePanel: '', + peek: null, joinPanelAccountId: '', messagesByBuffer: {}, presenceByBuffer: {}, @@ -1768,7 +1787,7 @@ export class ChatStore { selectGroup(groupId: string): void { if (groupId === this.state.activeGroupId) return - this.set({ activeGroupId: groupId }) + this.set({ activeGroupId: groupId, peek: null }) if (!this.state.pinnedBufferId) void window.moho.prefs.set('ui.activeGroupId', groupId) } @@ -3120,9 +3139,11 @@ export class ChatStore { // bumped). Merge the fresh server fields, keeping local-only unread and // highlight rather than resetting them. this.set({ buffers: buffers.map((b) => (b.id === data.id ? { ...b, ...data } : b)) }) + this.followPeekJoin(data) return } this.set({ buffers: [...buffers, { unread: 0, highlight: false, ...data }] }) + this.followPeekJoin(data) bestEffort(window.moho.rpc('subscribe', { bufferId: data.id }), `subscribe ${data.id}`) // A channel this client had not heard of may already hold mentions in the // stored history, so the inbox has to ask again now that the daemon can @@ -3301,6 +3322,7 @@ export class ChatStore { const patch: Partial = { activeBufferId: bufferId, activePanel: '', + peek: null, replyingTo: null, buffers: this.state.buffers.map((b) => b.id === bufferId ? { ...b, unread: 0, highlight: false } : b @@ -3960,7 +3982,58 @@ export class ChatStore { } setActivePanel(panel: ActivePanel, joinPanelAccountId = ''): void { - this.set({ activePanel: panel, joinPanelAccountId }) + // A panel is somewhere else to be, and coming back should not find a room + // still being looked at that nobody asked to keep. + this.set({ activePanel: panel, joinPanelAccountId, ...(panel === '' ? {} : { peek: null }) }) + } + + /** Look into a room in the conversation pane, without joining it. */ + startPeek(target: PeekTarget): void { + this.set({ peek: target, activePanel: '' }) + } + + stopPeek(): void { + this.set({ peek: null }) + } + + /** + * Join the room being looked at. + * + * The banner gives way to the box the moment this starts, but the box + * belongs to a conversation and the room is not one yet - so the pane holds + * the log it was showing until the room arrives from the daemon, and + * [`followPeekJoin`] then opens it properly. A join that fails puts the + * banner back, because the room is still only being looked at. + */ + async joinPeeked(): Promise { + const peek = this.state.peek + if (!peek || peek.joining) return + this.set({ peek: { ...peek, joining: true } }) + // Not for ever: a join the server accepted and then never delivered would + // otherwise leave a box that can never be typed into. + setTimeout(() => { + const now = this.state.peek + if (now?.joining && now.roomId === peek.roomId) this.set({ peek: null }) + }, JOIN_GIVE_UP_MS + 1000) + try { + await this.joinMatrixRoom( + peek.accountId, + peek.alias || peek.roomId, + peek.name || peek.alias || peek.roomId, + peek.alias ? [] : [peek.via] + ) + } catch (e) { + const now = this.state.peek + if (now?.roomId === peek.roomId) this.set({ peek: { ...now, joining: false } }) + throw e + } + } + + /** The room that was being joined has turned up: open it for real. */ + private followPeekJoin(data: { id: string; remoteId?: string }): void { + const peek = this.state.peek + if (!peek?.joining || !data.remoteId) return + if (data.remoteId === peek.roomId || data.remoteId === peek.alias) void this.selectBuffer(data.id) } startReply(id: string, from: string, body: string): void { From fb73c16c69553018f6f29f5a1ad2fd639c6fef7a Mon Sep 17 00:00:00 2001 From: Salastil Date: Tue, 29 Sep 2026 21:02:24 -0400 Subject: [PATCH 10/18] Pick up naming of unanswered direct messages Co-Authored-By: Claude Sonnet 5.5 --- nobilis | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nobilis b/nobilis index 5c8f8ac..f2401fb 160000 --- a/nobilis +++ b/nobilis @@ -1 +1 @@ -Subproject commit 5c8f8ac4436ff7eddf870135a72ee9d718ef4a3c +Subproject commit f2401fb16d5e9cd4042d3fa9b97e6e11f3980c8a From db0f6516f08ed2b07b7c38588392653898ec0e9f Mon Sep 17 00:00:00 2001 From: Salastil Date: Tue, 29 Sep 2026 21:39:37 -0400 Subject: [PATCH 11/18] Call diagnostics, and make call renegotiation wait for its answer Every call now leaves a [call] trace in moho.log: signalling both ways, session description summaries, ICE and connection state, tracks, and media counters every five seconds. Renegotiation used to move on once an offer was sent rather than answered, so a camera followed by a screen share put two offers in flight, and one failed change silently disabled every later one. It now waits for the answer (ten seconds at most), catches and logs failures, and settles two ends offering at once the way other clients do. Co-Authored-By: Claude Sonnet 5.5 --- src/main/index.ts | 12 ++ src/renderer/src/lib/calllog.ts | 171 ++++++++++++++++++++ src/renderer/src/lib/matrixcall.ts | 26 ++++ src/renderer/src/lib/webrtc.ts | 242 ++++++++++++++++++++++++++--- 4 files changed, 431 insertions(+), 20 deletions(-) create mode 100644 src/renderer/src/lib/calllog.ts diff --git a/src/main/index.ts b/src/main/index.ts index 821cc4d..a8a56c2 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -841,6 +841,18 @@ app.whenReady().then(() => { log.toDirectory(path.join(cacheRoot(), 'moho')) log.info('moho starting, logging to', log.file() ?? '(nowhere)') + // What a call did, from the window that held it. The renderer writes one + // `[call] {...}` line per event to its console - see lib/calllog.ts - and + // this is the only place they are kept: a console dies with its window, and + // the record of why a call had no picture is wanted after the call is over. + // Only that prefix, so nothing else a page prints ends up in the file. + app.on('web-contents-created', (_e, contents) => { + contents.on('console-message', (event) => { + const message = (event as unknown as { message?: string }).message + if (typeof message === 'string' && message.startsWith('[call] ')) log.info(message) + }) + }) + // Said once, in the log and in the window, and never in the way. // // Chromium's sandbox needs unprivileged user namespaces, and an AppImage diff --git a/src/renderer/src/lib/calllog.ts b/src/renderer/src/lib/calllog.ts new file mode 100644 index 0000000..6ffdbf1 --- /dev/null +++ b/src/renderer/src/lib/calllog.ts @@ -0,0 +1,171 @@ +/** + * What a call did, written down where somebody can read it afterwards. + * + * A call fails in one of a dozen places that all look the same from the + * window - "no picture" - and the honest answer to which one is in the + * signalling, the descriptions each end sent, the network path that was + * chosen and whether any media crossed it. None of that is visible in a + * running call, and most of it is gone once the call ends, so this keeps it. + * + * Lines are written to the console with a `[call]` prefix; the main process + * copies exactly those into moho.log, which outlives the window. One JSON + * object per line, so a person can read it and a script can too. + * + * What it will not write: ICE credentials, DTLS fingerprints, the ufrag and + * password lines of a description, or addresses. A log somebody sends to + * another person to explain a fault should not also be a key to the call. + */ + +const started = new Map() + +/** One thing that happened in a call. */ +export function callLog(callId: string, event: string, detail: Record = {}): void { + const now = Date.now() + if (!started.has(callId)) started.set(callId, now) + const line = { call: callId.slice(0, 10), t: now - (started.get(callId) ?? now), event, ...detail } + try { + console.info(`[call] ${JSON.stringify(line)}`) + } catch { + /* a log line must never be the thing that breaks a call */ + } +} + +/** A call is over: its clock and its counters are no longer needed. */ +export function callLogEnd(callId: string): void { + started.delete(callId) +} + +/** + * What a session description says, without what it holds. + * + * The parts that decide whether a picture can flow: which sections exist, + * which way each one points, whether it was refused (port 0), and which + * codecs it offers. `dir=inactive` or `port=0` on a video section is the + * whole answer to "why was there no video" more often than anything deeper. + */ +export function describeSdp(sdp: string | undefined | null): Record { + if (!sdp) return { sdp: 'none' } + const lines = sdp.split(/\r?\n/) + const sections: Record[] = [] + let current: Record | null = null + const rtpmap = new Map() + const candidates: Record = {} + let bundle = '' + let ended = false + + const finish = (): void => { + if (!current) return + const payloads = (current.payloads as string[]).map((p) => rtpmap.get(p) ?? p) + // The codec names once each: VP8 appears with its retransmission and + // forward-error-correction companions and listing all of them is noise. + const names = [...new Set(payloads.map((p) => p.split('/')[0]).filter((n) => !/^(rtx|red|ulpfec|flexfec-03|telephone-event|CN)$/i.test(n)))] + current.codecs = names + delete current.payloads + sections.push(current) + current = null + rtpmap.clear() + } + + for (const line of lines) { + if (line.startsWith('m=')) { + finish() + const [kind, port, , ...payloads] = line.slice(2).split(' ') + current = { kind, port: Number(port), payloads, dir: 'sendrecv' } + } else if (line.startsWith('a=group:BUNDLE')) { + bundle = line.slice('a=group:BUNDLE'.length).trim() + } else if (line.startsWith('a=end-of-candidates')) { + ended = true + } else if (current) { + const m = /^a=(sendrecv|sendonly|recvonly|inactive)$/.exec(line) + if (m) current.dir = m[1] + else if (line.startsWith('a=mid:')) current.mid = line.slice(6) + else if (line.startsWith('a=msid:')) current.msid = line.slice(7).split(' ')[0].slice(0, 12) + else if (line.startsWith('a=rtpmap:')) { + const [pt, name] = line.slice(9).split(' ') + rtpmap.set(pt, name) + } else if (line.startsWith('a=candidate:')) { + const type = /typ (\w+)/.exec(line)?.[1] ?? '?' + candidates[type] = (candidates[type] ?? 0) + 1 + } + } + } + finish() + return { + sections: sections.map((s) => `${s.kind}${s.mid !== undefined ? `#${s.mid}` : ''} ${s.dir}${s.port === 0 ? ' REFUSED' : ''} [${(s.codecs as string[]).join(',')}]${s.msid ? ` msid=${s.msid}` : ''}`), + bundle, + ...(Object.keys(candidates).length ? { candidatesInSdp: candidates, endOfCandidates: ended } : {}) + } +} + +/** The relays and stun servers a call was given, by address only. */ +export function describeIce(servers: RTCIceServer[]): string[] { + return servers.flatMap((s) => (Array.isArray(s.urls) ? s.urls : [s.urls])).map((u) => u.replace(/^(\w+:)/, '$1')) +} + +interface Counters { + bytes: number + packets: number + frames: number +} + +const previous = new Map() + +/** + * A reading of what is actually crossing the connection. + * + * The chosen path (a relay, or a direct route) and, per direction and kind, + * how much has moved since the last reading. "Video track exists, no bytes in + * five seconds" is a network or a sender fault; "bytes arriving, no frames + * decoded" is a codec fault; "no bytes leaving" is this end. Those are the + * distinctions a silent picture needs made. + */ +export async function sampleStats(callId: string, pc: RTCPeerConnection): Promise> { + const report = await pc.getStats() + const byId = new Map() + report.forEach((s) => byId.set(s.id, s)) + const out: Record = { + ice: pc.iceConnectionState, + conn: pc.connectionState, + signalling: pc.signalingState + } + report.forEach((s) => { + if (s.type === 'candidate-pair' && s.nominated && s.state === 'succeeded') { + const l = byId.get(s.localCandidateId) + const r = byId.get(s.remoteCandidateId) + out.path = `${l?.candidateType}/${l?.protocol} -> ${r?.candidateType}/${r?.protocol}` + if (typeof s.currentRoundTripTime === 'number') out.rttMs = Math.round(s.currentRoundTripTime * 1000) + } + if (s.type === 'transport') out.dtls = s.dtlsState + const inbound = s.type === 'inbound-rtp' + const outbound = s.type === 'outbound-rtp' + if (!inbound && !outbound) return + const kind = s.kind ?? s.mediaType + const key = `${callId}|${s.id}` + const now: Counters = { + bytes: inbound ? s.bytesReceived ?? 0 : s.bytesSent ?? 0, + packets: inbound ? s.packetsReceived ?? 0 : s.packetsSent ?? 0, + frames: inbound ? s.framesDecoded ?? 0 : s.framesEncoded ?? 0 + } + const before = previous.get(key) ?? { bytes: 0, packets: 0, frames: 0 } + previous.set(key, now) + const entry: Record = { + bytes: now.bytes, + new: now.bytes - before.bytes, + ...(kind === 'video' ? { frames: now.frames, newFrames: now.frames - before.frames } : {}) + } + const codec = s.codecId ? byId.get(s.codecId)?.mimeType : undefined + if (codec) entry.codec = codec + if (kind === 'video' && s.frameWidth) entry.size = `${s.frameWidth}x${s.frameHeight}` + if (inbound && typeof s.packetsLost === 'number') entry.lost = s.packetsLost + if (outbound && kind === 'video' && s.qualityLimitationReason && s.qualityLimitationReason !== 'none') { + entry.limited = s.qualityLimitationReason + } + out[`${inbound ? 'in' : 'out'}.${kind}`] = entry + }) + return out +} + +/** Forget a call's counters. */ +export function forgetStats(callId: string): void { + for (const key of previous.keys()) if (key.startsWith(`${callId}|`)) previous.delete(key) +} diff --git a/src/renderer/src/lib/matrixcall.ts b/src/renderer/src/lib/matrixcall.ts index 8bb2919..6bdaece 100644 --- a/src/renderer/src/lib/matrixcall.ts +++ b/src/renderer/src/lib/matrixcall.ts @@ -13,6 +13,7 @@ */ import { Call, type CallPhase } from './webrtc' +import { callLog, describeSdp } from './calllog' import { SfuCall, type SfuParticipant } from './livekit' /** What the daemon sends up when a call event arrives in a room. */ @@ -677,6 +678,7 @@ export class MatrixCalls { handlers: { send: (signal) => { const base = { call_id: callId, version: '1', party_id: call.partyId } + callLog(callId, 'signal-out', { kind: signal.kind, keys: Object.keys(signal.content) }) // Each signal is its own event type; the shapes differ enough that // naming them here is clearer than a table. if (signal.kind === 'invite') { @@ -722,6 +724,24 @@ export class MatrixCalls { */ handle(event: MatrixCallEvent): void { const { kind, callId, content } = event + // Everything that arrives, including what is about to be ignored: the + // events a client does not answer are exactly the ones a fault hides in. + // `metadata` is the stream description Element sends to say which stream + // is a camera and which a screen; a peer that sends none is an older one. + callLog(callId, 'signal-in', { + kind, + own: event.own, + party: content.party_id ?? null, + version: content.version ?? null, + metadata: !!content.sdp_stream_metadata, + // Whether a renegotiation is an offer or the answer to one, and what it + // holds: the phone's changes to a call are only readable from this. + ...(content.description ? { description: content.description.type, ...describeSdp(content.description.sdp) } : {}), + ...(content.offer ? { offerSdp: describeSdp(content.offer.sdp) } : {}), + ...(content.answer ? { answerSdp: describeSdp(content.answer.sdp) } : {}), + keys: Object.keys(content).filter((k) => k !== 'offer' && k !== 'answer' && k !== 'description' && k !== 'candidates'), + active: this.active?.call.id === callId + }) // A group call's own signalling, which is addressed rather than broadcast // even though every event in the room is seen by everybody. @@ -794,6 +814,12 @@ export class MatrixCalls { this.surface.onRinging(null) } if (this.active?.call.id === callId) this.end(false) + return } + + // Anything else in the call namespace is ours to say nothing about. Logged + // because `select_answer`, `reject` and a `negotiate` for a call this + // window does not hold are the ones whose silence looks like a fault. + callLog(callId, 'signal-unhandled', { kind, active: this.active?.call.id === callId }) } } diff --git a/src/renderer/src/lib/webrtc.ts b/src/renderer/src/lib/webrtc.ts index c9cc83a..f27d570 100644 --- a/src/renderer/src/lib/webrtc.ts +++ b/src/renderer/src/lib/webrtc.ts @@ -19,6 +19,8 @@ * needs from whatever is carrying it. */ +import { callLog, callLogEnd, describeIce, describeSdp, forgetStats, sampleStats } from './calllog' + /** What a call is doing, in the words a person would use. */ export type CallPhase = 'ringing' | 'connecting' | 'connected' | 'ended' @@ -38,6 +40,15 @@ export interface CallHandlers { onError: (message: string) => void } +/** + * How long a change to a call in progress waits for its answer. + * + * Short beside the ring timeout: the other end is already connected and + * answering is a matter of milliseconds, so ten seconds of nothing means it + * is not going to. + */ +const NEGOTIATE_TIMEOUT_MS = 10_000 + /** How long to wait for an answer before giving up on an unanswered call. */ const RING_TIMEOUT_MS = 60_000 @@ -81,6 +92,8 @@ export class Call { /** Media opened by somebody else and lent to this leg - see the * constructor. Never stopped here, because it is not this leg's to stop. */ private shared: MediaStream | null = null + /** Reads what is crossing the connection, for the log. */ + private statsTimer: ReturnType | null = null constructor(opts: { id: string @@ -104,13 +117,29 @@ export class Call { this.partyId = `moho-${Math.random().toString(36).slice(2, 10)}` this.shared = opts.local ?? null this.pc = new RTCPeerConnection({ iceServers: opts.iceServers }) + callLog(this.id, 'created', { + outgoing: this.outgoing, + party: this.partyId, + ice: describeIce(opts.iceServers), + lentMedia: !!opts.local + }) + this.watchConnection() // Candidates are found for as long as a call lasts, not only at the // start: a network changing under a call is exactly when the later ones // matter, and a client that stopped listening after connecting would drop // the call rather than move it. this.pc.onicecandidate = (e) => { - if (!e.candidate) return + if (!e.candidate) { + callLog(this.id, 'ice-gathered') + return + } + // Only what kind of route it is - the address is not the log's to keep. + callLog(this.id, 'ice-candidate', { + type: e.candidate.type, + protocol: e.candidate.protocol, + ...(e.candidate.type === 'relay' ? { relay: true } : {}) + }) const candidate = { candidate: e.candidate.candidate, sdpMid: e.candidate.sdpMid, @@ -122,6 +151,13 @@ export class Call { } this.pc.ontrack = (e) => { + callLog(this.id, 'remote-track', { + kind: e.track.kind, + mid: e.transceiver.mid, + muted: e.track.muted, + streams: e.streams.length, + direction: e.transceiver.currentDirection + }) for (const track of e.streams[0]?.getTracks() ?? [e.track]) { if (this.remote.getTracks().some((t) => t.id === track.id)) continue this.remote.addTrack(track) @@ -131,6 +167,7 @@ export class Call { // before it reports it as an end, and on a removed track the mute is // often all there is. const drop = (): void => { + callLog(this.id, 'remote-track-gone', { kind: track.kind, state: track.readyState, muted: track.muted }) this.remote.removeTrack(track) this.handlers.onRemoteStream(this.remote) } @@ -139,6 +176,7 @@ export class Call { // And back again when they turn it on: the same transceiver is // reused, so the track unmutes rather than a new one arriving. track.onunmute = () => { + callLog(this.id, 'remote-track-live', { kind: track.kind }) if (!this.remote.getTracks().some((t) => t.id === track.id)) this.remote.addTrack(track) this.handlers.onRemoteStream(this.remote) } @@ -148,6 +186,7 @@ export class Call { this.pc.onconnectionstatechange = () => { const state = this.pc.connectionState + callLog(this.id, 'connection', { state }) if (state === 'connected') this.setPhase('connected') // "failed" is over; "disconnected" is not, and often comes back on its // own when a network hiccups. Treating the second as the first is how @@ -194,9 +233,37 @@ export class Call { return !!this.camera && this.camera.readyState === 'live' } + /** + * Everything the connection says about itself, and a reading of the media + * every few seconds. + * + * The readings are the point. A call whose picture never arrives has a + * connection that says "connected" all the way through; only counting what + * crossed it says whether the fault is a route, a sender or a decoder. + */ + private watchConnection(): void { + this.pc.oniceconnectionstatechange = () => callLog(this.id, 'ice', { state: this.pc.iceConnectionState }) + this.pc.onsignalingstatechange = () => callLog(this.id, 'signalling', { state: this.pc.signalingState }) + this.pc.onicegatheringstatechange = () => callLog(this.id, 'gathering', { state: this.pc.iceGatheringState }) + this.pc.onicecandidateerror = (e) => { + const err = e as RTCPeerConnectionIceErrorEvent + // Silent unless something went wrong: this is the one that says a relay + // refused the credentials or could not be reached at all. + callLog(this.id, 'ice-error', { url: err.url, code: err.errorCode, text: err.errorText }) + } + this.statsTimer = setInterval(() => { + if (this.phase === 'ended') return + void sampleStats(this.id, this.pc) + .then((reading) => callLog(this.id, 'stats', reading)) + .catch(() => {}) + }, 5000) + } + private setPhase(phase: CallPhase): void { if (this.phase === phase || this.phase === 'ended') return this.phase = phase + callLog(this.id, 'phase', { phase }) + if (phase === 'ended' && this.statsTimer) clearInterval(this.statsTimer) this.handlers.onPhase(phase) } @@ -219,8 +286,10 @@ export class Call { const audio = { echoCancellation: true, noiseSuppression: true } try { this.local = await navigator.mediaDevices.getUserMedia({ audio, video }) + callLog(this.id, 'local-media', { wantedVideo: video, ...describeStream(this.local) }) } catch (e) { const name = (e as Error).name + callLog(this.id, 'local-media-failed', { wantedVideo: video, name, message: (e as Error).message }) const noCamera = name === 'NotFoundError' || name === 'NotAllowedError' || name === 'OverconstrainedError' if (!video || !noCamera) throw e this.handlers.onError('No camera here - calling with audio only') @@ -235,6 +304,7 @@ export class Call { await this.openLocal(video) const offer = await this.pc.createOffer() await this.pc.setLocalDescription(offer) + callLog(this.id, 'offer-made', describeSdp(offer.sdp)) this.setPhase('ringing') this.handlers.send({ kind: 'invite', @@ -254,10 +324,12 @@ export class Call { /** Answers a call that is ringing here. */ async accept(offerSdp: string, video: boolean): Promise { this.setPhase('connecting') + callLog(this.id, 'offer-received', { answeringWithVideo: video, ...describeSdp(offerSdp) }) await this.pc.setRemoteDescription({ type: 'offer', sdp: offerSdp }) await this.openLocal(video) const answer = await this.pc.createAnswer() await this.pc.setLocalDescription(answer) + callLog(this.id, 'answer-made', describeSdp(answer.sdp)) this.answered = true this.handlers.send({ kind: 'answer', content: { answer: { type: 'answer', sdp: answer.sdp } } }) this.flushCandidates() @@ -269,6 +341,7 @@ export class Call { this.answered = true if (this.ringTimer) clearTimeout(this.ringTimer) this.setPhase('connecting') + callLog(this.id, 'answer-received', describeSdp(answerSdp)) await this.pc.setRemoteDescription({ type: 'answer', sdp: answerSdp }) this.flushCandidates() } @@ -322,6 +395,7 @@ export class Call { */ async setCamera(on: boolean, source?: MediaStream | null): Promise { if (on === this.cameraOn) return this.cameraOn + callLog(this.id, 'camera', { turning: on ? 'on' : 'off' }) if (!on) { const track = this.camera @@ -343,10 +417,16 @@ export class Call { // Its own request rather than reopening the microphone with it: the // microphone is already open and already being listened to, and asking // for both again would interrupt the audio to add a picture. - stream = await navigator.mediaDevices.getUserMedia({ video: true }) + try { + stream = await navigator.mediaDevices.getUserMedia({ video: true }) + } catch (e) { + callLog(this.id, 'camera-failed', { name: (e as Error).name, message: (e as Error).message }) + throw e + } } const track = stream.getVideoTracks()[0] if (!track) return false + callLog(this.id, 'camera-opened', describeStream(stream)) this.camera = track // The browser's own "stop" - a camera unplugged, a privacy shutter, the // device taken by something else - has to reach the call, or the far end @@ -368,6 +448,7 @@ export class Call { */ async shareScreen(source?: MediaStream): Promise { if (this.screen) { + callLog(this.id, 'screen', { turning: 'off' }) this.screen.stop() const sender = this.pc.getSenders().find((s) => s.track === this.screen) if (sender) this.pc.removeTrack(sender) @@ -375,9 +456,17 @@ export class Call { await this.renegotiate() return false } - const stream = source ?? (await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false })) + callLog(this.id, 'screen', { turning: 'on' }) + let stream: MediaStream + try { + stream = source ?? (await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false })) + } catch (e) { + callLog(this.id, 'screen-failed', { name: (e as Error).name, message: (e as Error).message }) + throw e + } const track = stream.getVideoTracks()[0] if (!track) return false + callLog(this.id, 'screen-opened', describeStream(stream)) this.screen = track // Stopping from the browser's own "stop sharing" control has to reach the // call too, or the other end keeps a frozen last frame. @@ -398,14 +487,63 @@ export class Call { * have to happen. */ private pendingNegotiation: Promise = Promise.resolve() + /** Offers from the other end, one at a time, and not behind our own. */ + private incomingOffers: Promise = Promise.resolve() + /** Set while one of our offers is out and its answer is awaited. */ + private answerWaiter: (() => void) | null = null + /** An offer of ours was given up for theirs, and still has to be made. */ + private owesOffer = false + + /** + * Waits for the answer to the offer just sent. + * + * This is what "one at a time" has to mean. The chain used to move on the + * moment an offer was *sent*, so turning a camera on and starting a share a + * moment later put two offers in flight; the far end answered both, and the + * second answer arrived in a connection that had already settled and was + * refused. Waiting for the answer is the only way there is ever one. + * + * With a limit, because some clients never answer a change at all - and + * "the other end ignored it" is exactly the thing worth being able to see, + * so it is logged rather than waited on for ever. + */ + private waitForAnswer(): Promise { + return new Promise((resolve) => { + const timer = setTimeout(() => { + this.answerWaiter = null + callLog(this.id, 'negotiate-unanswered', { afterMs: NEGOTIATE_TIMEOUT_MS, signalling: this.pc.signalingState }) + this.handlers.onError("The other end didn't answer the change to the call") + // Our offer is still standing; leaving it there would refuse every + // change that came after. + this.pc.setLocalDescription({ type: 'rollback' }).catch(() => {}).finally(resolve) + }, NEGOTIATE_TIMEOUT_MS) + this.answerWaiter = () => { + clearTimeout(timer) + this.answerWaiter = null + resolve() + } + }) + } private renegotiate(): Promise { - this.pendingNegotiation = this.pendingNegotiation.then(async () => { - if (this.phase === 'ended') return - const offer = await this.pc.createOffer() - await this.pc.setLocalDescription(offer) - this.handlers.send({ kind: 'negotiate', content: { description: { type: 'offer', sdp: offer.sdp } } }) - }) + this.pendingNegotiation = this.pendingNegotiation + .then(async () => { + if (this.phase === 'ended') return + const offer = await this.pc.createOffer() + await this.pc.setLocalDescription(offer) + callLog(this.id, 'negotiate-offer-made', describeSdp(offer.sdp)) + this.handlers.send({ kind: 'negotiate', content: { description: { type: 'offer', sdp: offer.sdp } } }) + await this.waitForAnswer() + }) + // Caught here, and not passed on. A rejected promise left in this chain + // is a rejected promise every later step is chained onto, and a `then` + // on one never runs - so a single failed renegotiation used to switch + // off camera and screen sharing for the rest of the call, without a + // word about why. The failure is logged and the chain carries on. + .catch((e: Error) => { + callLog(this.id, 'negotiate-failed', { direction: 'ours', name: e.name, message: e.message, signalling: this.pc.signalingState }) + this.handlers.onError(`Couldn't change the call: ${e.message}`) + }) return this.pendingNegotiation } @@ -413,24 +551,74 @@ export class Call { * A renegotiation from the other end - somebody turning a camera on, or * starting to share. * - * Queued behind this end's own, for the reason above: two descriptions - * being applied to one connection at once is how a call ends up describing - * something neither end is sending. + * An answer goes straight to the offer waiting for it; only offers queue. + * They cannot queue behind our own change, which is itself waiting for the + * answer these are holding up. + * + * Two ends offering at once - a camera pressed on both sides in the same + * second - is settled the way every other client settles it: the end that + * made the call ignores the other's offer, and the end that answered it + * gives its own up, takes theirs, and offers again afterwards. Which is the + * one that yields is a property of the call, not of who was quicker, so both + * ends reach the same answer without talking. */ takeNegotiation(description: RTCSessionDescriptionInit): Promise { - this.pendingNegotiation = this.pendingNegotiation.then(async () => { - if (this.phase === 'ended') return + if (description.type === 'answer') return this.takeNegotiatedAnswer(description) + + this.incomingOffers = this.incomingOffers + .then(async () => { + if (this.phase === 'ended') return + const collision = this.pc.signalingState !== 'stable' + callLog(this.id, 'negotiate-received', { + type: description.type, + signalling: this.pc.signalingState, + ...(collision ? { glare: this.outgoing ? 'ignored (we made the call)' : 'yielding (we answered the call)' } : {}), + ...describeSdp(description.sdp) + }) + if (collision && this.outgoing) return + if (collision) { + this.owesOffer = true + this.answerWaiter?.() + } + await this.pc.setRemoteDescription(description) + const answer = await this.pc.createAnswer() + await this.pc.setLocalDescription(answer) + callLog(this.id, 'negotiate-answer-made', describeSdp(answer.sdp)) + this.handlers.send({ kind: 'negotiate', content: { description: { type: 'answer', sdp: answer.sdp } } }) + if (this.owesOffer) { + this.owesOffer = false + void this.renegotiate() + } + }) + .catch((e: Error) => { + callLog(this.id, 'negotiate-failed', { direction: 'theirs', name: e.name, message: e.message, signalling: this.pc.signalingState }) + this.handlers.onError(`The other end's change to the call failed: ${e.message}`) + }) + return this.incomingOffers + } + + private async takeNegotiatedAnswer(description: RTCSessionDescriptionInit): Promise { + callLog(this.id, 'negotiate-answer-received', { signalling: this.pc.signalingState, ...describeSdp(description.sdp) }) + try { + // An answer to an offer that is no longer standing - ours was given up + // in a collision, or timed out - has nothing to answer. + if (this.pc.signalingState !== 'have-local-offer') { + callLog(this.id, 'negotiate-answer-stale', { signalling: this.pc.signalingState }) + return + } await this.pc.setRemoteDescription(description) - if (description.type !== 'offer') return - const answer = await this.pc.createAnswer() - await this.pc.setLocalDescription(answer) - this.handlers.send({ kind: 'negotiate', content: { description: { type: 'answer', sdp: answer.sdp } } }) - }) - return this.pendingNegotiation + } catch (e) { + callLog(this.id, 'negotiate-failed', { direction: 'answer', name: (e as Error).name, message: (e as Error).message, signalling: this.pc.signalingState }) + this.handlers.onError(`The other end's answer failed: ${(e as Error).message}`) + } finally { + this.answerWaiter?.() + } } /** Ends the call, telling the other end unless it was them who ended it. */ async hangUp(reason?: string, tell = true): Promise { + callLog(this.id, 'hangup', { reason: reason ?? null, toldThem: tell, phase: this.phase }) + if (this.statsTimer) clearInterval(this.statsTimer) if (this.ringTimer) clearTimeout(this.ringTimer) if (tell) this.handlers.send({ kind: 'hangup', content: reason ? { reason } : {} }) // Not the shared capture: other legs of the same call are still using it, @@ -446,5 +634,19 @@ export class Call { this.camera = null this.pc.close() this.setPhase('ended') + forgetStats(this.id) + callLogEnd(this.id) + } +} + +/** What a capture is, in a line: the tracks it holds and what they are. */ +function describeStream(stream: MediaStream): Record { + return { + tracks: stream.getTracks().map((t) => { + const settings = t.getSettings() + return t.kind === 'video' + ? `video ${settings.width ?? '?'}x${settings.height ?? '?'}@${Math.round(settings.frameRate ?? 0)} ${t.readyState}${t.label ? ` "${t.label.slice(0, 40)}"` : ''}` + : `audio ${t.readyState}` + }) } } From cf87069aab002c84d2cf75d9df8311a3427ee058 Mon Sep 17 00:00:00 2001 From: Salastil Date: Tue, 29 Sep 2026 21:58:01 -0400 Subject: [PATCH 12/18] Warn when a camera or screen capture produces no picture A capture can open, report itself live, be negotiated and accepted, and still deliver no frames, which from outside is a call with every indicator green and no picture. Count frames for four seconds after a camera or screen opens and say so when there are none. Also read the media counters shortly after tracks appear or changes are answered, so a share that lasts two seconds is not lost between the five-second readings. Co-Authored-By: Claude Sonnet 5.5 --- src/renderer/src/lib/webrtc.ts | 60 ++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/renderer/src/lib/webrtc.ts b/src/renderer/src/lib/webrtc.ts index f27d570..4fe077a 100644 --- a/src/renderer/src/lib/webrtc.ts +++ b/src/renderer/src/lib/webrtc.ts @@ -177,6 +177,7 @@ export class Call { // reused, so the track unmutes rather than a new one arriving. track.onunmute = () => { callLog(this.id, 'remote-track-live', { kind: track.kind }) + if (track.kind === 'video') this.sampleSoon('remote video live') if (!this.remote.getTracks().some((t) => t.id === track.id)) this.remote.addTrack(track) this.handlers.onRemoteStream(this.remote) } @@ -259,6 +260,61 @@ export class Call { }, 5000) } + /** + * A reading shortly after something changed. + * + * The regular one is every five seconds, and a screen share the other end + * starts and stops inside that window is invisible to it - which is exactly + * how the first phone share was lost to the log: it lasted two seconds. So a + * track arriving, a change being answered, a capture opening each ask for + * their own readings a moment later, when there is something to read. + */ + private sampleSoon(reason: string): void { + for (const after of [1500, 4000]) { + setTimeout(() => { + if (this.phase === 'ended') return + void sampleStats(this.id, this.pc) + .then((reading) => callLog(this.id, 'stats', { after: reason, ...reading })) + .catch(() => {}) + }, after) + } + } + + /** + * Whether something captured here is producing a picture at all. + * + * A camera or a screen can open, report itself live, be negotiated and be + * accepted by the other end, and still deliver no frames - a portal that is + * waiting for a choice nobody sees, a device another program holds. From + * outside that is a call in which the picture simply never appears, with + * every indicator green. Counting frames for a few seconds is the only way + * to tell it from one that works, and it turns silence into a sentence. + */ + private watchCapture(source: 'camera' | 'screen', track: MediaStreamTrack): void { + const probe = document.createElement('video') + probe.muted = true + probe.srcObject = new MediaStream([track]) + let frames = 0 + const tick = (): void => { + frames++ + probe.requestVideoFrameCallback?.(tick) + } + probe.requestVideoFrameCallback?.(tick) + void probe.play().catch(() => {}) + setTimeout(() => { + const width = probe.videoWidth + probe.srcObject = null + callLog(this.id, 'capture', { source, frames, width, state: track.readyState }) + if (frames === 0 && track.readyState === 'live') { + this.handlers.onError( + source === 'screen' + ? 'Screen capture started but is producing no picture - the other end will see nothing' + : 'The camera opened but is producing no picture' + ) + } + }, 4000) + } + private setPhase(phase: CallPhase): void { if (this.phase === phase || this.phase === 'ended') return this.phase = phase @@ -427,6 +483,7 @@ export class Call { const track = stream.getVideoTracks()[0] if (!track) return false callLog(this.id, 'camera-opened', describeStream(stream)) + this.watchCapture('camera', track) this.camera = track // The browser's own "stop" - a camera unplugged, a privacy shutter, the // device taken by something else - has to reach the call, or the far end @@ -467,6 +524,7 @@ export class Call { const track = stream.getVideoTracks()[0] if (!track) return false callLog(this.id, 'screen-opened', describeStream(stream)) + this.watchCapture('screen', track) this.screen = track // Stopping from the browser's own "stop sharing" control has to reach the // call too, or the other end keeps a frozen last frame. @@ -585,6 +643,7 @@ export class Call { await this.pc.setLocalDescription(answer) callLog(this.id, 'negotiate-answer-made', describeSdp(answer.sdp)) this.handlers.send({ kind: 'negotiate', content: { description: { type: 'answer', sdp: answer.sdp } } }) + this.sampleSoon('their change answered') if (this.owesOffer) { this.owesOffer = false void this.renegotiate() @@ -607,6 +666,7 @@ export class Call { return } await this.pc.setRemoteDescription(description) + this.sampleSoon('our change answered') } catch (e) { callLog(this.id, 'negotiate-failed', { direction: 'answer', name: (e as Error).name, message: (e as Error).message, signalling: this.pc.signalingState }) this.handlers.onError(`The other end's answer failed: ${(e as Error).message}`) From 2c146d956fb98b009c3449dc1623d75aee0931f9 Mon Sep 17 00:00:00 2001 From: Salastil Date: Tue, 29 Sep 2026 23:04:15 -0400 Subject: [PATCH 13/18] Share a screen through the Wayland portal with one dialog, not two On Wayland desktopCapturer.getSources opens a portal session to draw thumbnails and opening the chosen source opens another, so a share needed two 'Select what to share' windows answered and no picture existed until the second. getDisplayMedia is now answered with a bare screen id and no source list is requested, so the portal asks once. The stream comes back live before anyone has answered that window and stays live for ever if it is cancelled, so moho waits for a first frame before it announces a share, and treats a closed or unanswered window as backing out. MOHO_SCREENSHARE=picker|portal overrides the detection. Co-Authored-By: Claude Sonnet 5.5 --- src/main/index.ts | 32 +++++++++ src/preload/index.ts | 11 +++ src/renderer/src/lib/capture.ts | 45 ++++++++++++ src/renderer/src/lib/webrtc.ts | 2 +- src/renderer/src/state/store.ts | 117 +++++++++++++++++++++++--------- src/shared/ipc.ts | 1 + 6 files changed, 176 insertions(+), 32 deletions(-) create mode 100644 src/renderer/src/lib/capture.ts diff --git a/src/main/index.ts b/src/main/index.ts index a8a56c2..3555590 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -196,6 +196,22 @@ function send(channel: string, ...args: unknown[]): void { } /** The window an IPC call came from, so a handler acts on its own caller. */ +/** + * Whether a screen is captured through the desktop portal, which is the case + * in any Wayland session: X11 offers a list of windows and screens to draw a + * picker from, and Wayland deliberately does not. + * + * `MOHO_SCREENSHARE=picker` forces the list-and-choose path and `portal` + * forces the other, for somebody whose session reports itself wrongly. + */ +function capturesThroughPortal(): boolean { + if (process.platform !== 'linux') return false + const forced = process.env.MOHO_SCREENSHARE + if (forced === 'picker') return false + if (forced === 'portal') return true + return process.env.XDG_SESSION_TYPE === 'wayland' || !!process.env.WAYLAND_DISPLAY +} + function callerWindow(event: Electron.IpcMainInvokeEvent): BrowserWindow | null { return BrowserWindow.fromWebContents(event.sender) } @@ -662,6 +678,8 @@ function wireIpc(): void { })) }) + ipcMain.handle(IPC.screenCaptureViaPortal, () => capturesThroughPortal()) + ipcMain.handle(IPC.pickSavePath, async (e, suggested?: string) => { const parent = callerWindow(e) ?? mainWindow if (!parent) return null @@ -891,6 +909,20 @@ app.whenReady().then(() => { }) session.defaultSession.setPermissionCheckHandler((_contents, permission) => allowed.has(permission)) + // Screen capture on Wayland goes through the compositor's portal, which puts + // up its own "what to share" window whatever it is told. Answering + // getDisplayMedia with a bare screen id - and never asking for a list of + // sources first - makes that window appear once. Listing sources opens a + // portal session just to draw thumbnails, and opening the chosen one opens + // a second, so a share needed two windows answered and the picture only + // began after the second. Measured on Electron 43 and 44: one window, and + // the first frame arrives as soon as it is answered. + if (capturesThroughPortal()) { + session.defaultSession.setDisplayMediaRequestHandler((_request, callback) => { + callback({ video: { id: 'screen:0:0', name: 'Entire Screen' } }) + }) + } + // Someone tried to launch a second copy: treat it as "show me moho", which // is almost always what they meant - especially when the window is hidden // to the tray and looks like nothing is running. diff --git a/src/preload/index.ts b/src/preload/index.ts index 8912bec..5c55b0f 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -145,6 +145,17 @@ const api = { */ screenSources: (): Promise<{ id: string; name: string; thumbnail: string }[]> => ipcRenderer.invoke(IPC.screenSources), + /** + * Whether a screen is chosen in the desktop's own window rather than ours. + * + * True on Wayland, where the compositor's portal is the only way to capture + * a screen and it always shows its own dialog. Asking for a list of sources + * first shows that dialog once for the list, and opening the capture shows + * it again - two windows for one share, the second of which nothing warns + * you about. Where this is true the list is skipped and the portal is asked + * once. + */ + screenCaptureViaPortal: (): Promise => ipcRenderer.invoke(IPC.screenCaptureViaPortal), pickSavePath: (suggested?: string): Promise => ipcRenderer.invoke(IPC.pickSavePath, suggested), pickDirectory: (): Promise => ipcRenderer.invoke(IPC.pickDirectory), diff --git a/src/renderer/src/lib/capture.ts b/src/renderer/src/lib/capture.ts new file mode 100644 index 0000000..398fc62 --- /dev/null +++ b/src/renderer/src/lib/capture.ts @@ -0,0 +1,45 @@ +/** + * Whether a capture is actually producing a picture. + * + * A screen opened through the desktop portal on Wayland is handed back as a + * live track at once - before anybody has answered the portal's own "what do + * you want to share" window - and if that window is cancelled, or never + * noticed, the track stays live for ever and delivers nothing. It does not + * end and it does not fail. So a returned stream says nothing about whether + * there is a screen behind it; only a frame does. + * + * Resolves true on the first frame and false if the track ends or nothing + * arrives in time, which is what "the person backed out" looks like from + * here. The time allowed is a person's time - they are choosing a window - + * not a network's. + */ +export function firstFrame(stream: MediaStream, timeoutMs: number): Promise { + return new Promise((resolve) => { + const track = stream.getVideoTracks()[0] + if (!track) return resolve(false) + + const probe = document.createElement('video') + probe.muted = true + probe.srcObject = stream + + let settled = false + const finish = (arrived: boolean): void => { + if (settled) return + settled = true + clearTimeout(timer) + track.removeEventListener('ended', onEnded) + probe.srcObject = null + resolve(arrived) + } + const onEnded = (): void => finish(false) + const timer = setTimeout(() => finish(false), timeoutMs) + track.addEventListener('ended', onEnded) + + if (probe.requestVideoFrameCallback) probe.requestVideoFrameCallback(() => finish(true)) + else probe.onloadeddata = () => finish(true) + void probe.play().catch(() => {}) + }) +} + +/** How long somebody is given to choose what to share. */ +export const SCREEN_CHOICE_MS = 60_000 diff --git a/src/renderer/src/lib/webrtc.ts b/src/renderer/src/lib/webrtc.ts index 4fe077a..ad3a242 100644 --- a/src/renderer/src/lib/webrtc.ts +++ b/src/renderer/src/lib/webrtc.ts @@ -308,7 +308,7 @@ export class Call { if (frames === 0 && track.readyState === 'live') { this.handlers.onError( source === 'screen' - ? 'Screen capture started but is producing no picture - the other end will see nothing' + ? 'Screen capture started but is producing no picture - the other end will see nothing. If a "Select what to share" window is open anywhere, answer it' : 'The camera opened but is producing no picture' ) } diff --git a/src/renderer/src/state/store.ts b/src/renderer/src/state/store.ts index 6bb8b61..0408e23 100644 --- a/src/renderer/src/state/store.ts +++ b/src/renderer/src/state/store.ts @@ -30,6 +30,7 @@ import { parseDeepLink, type IrcLink } from '../../../shared/deeplink' import { MatrixCalls, type CallMember, type MatrixCallEvent, type RingingCall } from '../lib/matrixcall' import type { CallPhase } from '../lib/webrtc' import { Levels } from '../lib/levels' +import { firstFrame, SCREEN_CHOICE_MS } from '../lib/capture' import type { PopoutState } from '../../../shared/ipc' /** @@ -1067,6 +1068,10 @@ export class ChatStore { const stored = await window.moho.prefs.getAll() this.set({ lastReadTs: (stored.lastReadTs as Record) ?? {} }) + // Known before anybody presses share, so the capture can be asked for in + // the same breath as the click: a screen is a request that needs the + // gesture that made it, and a round trip first would spend it. + this.screenViaPortal = await window.moho.screenCaptureViaPortal().catch(() => false) window.moho.onLinkChange((up) => { this.set({ linkUp: up }) @@ -2231,6 +2236,64 @@ export class ChatStore { }) } + /** Whether the desktop's portal picks the screen - see the preload. */ + private screenViaPortal = false + + /** + * Decides what is to be shared, or that nothing is. + * + * Two shapes, because two desktops. Where the portal chooses, there is + * nothing to ask here: the choice is made in its window when the capture is + * opened, and asking for a list first would put that window up twice. Where + * it does not, a list of windows is drawn as our own picker and one is + * picked. + */ + private async chooseScreen(): Promise<{ portal: true } | { portal: false; id: string } | null> { + if (this.screenViaPortal) return { portal: true } + const sources = await window.moho.screenSources() + if (sources.length === 0) { + this.toast('info', 'Nothing to share') + return null + } + // Chosen in the window rather than here: the picker is a component, and + // this waits for it to answer. + const source = await this.askForScreenSource(sources) + return source ? { portal: false, id: source.id } : null + } + + /** + * Opens what was chosen, or null if the person backed out of it. + * + * On the portal path the stream comes back at once and means nothing until + * a frame does - see `firstFrame` - so this waits for one, and a window that + * was closed, or never answered, ends here as a quiet "no" rather than as a + * share that was announced to the room and shows nothing. + */ + private async openScreen(choice: { portal: true } | { portal: false; id: string }): Promise { + if (!choice.portal) { + return await navigator.mediaDevices.getUserMedia({ + audio: false, + video: { + // Chromium's own desktop-capture constraint, which is how an + // Electron window turns a source id into a stream. + mandatory: { chromeMediaSource: 'desktop', chromeMediaSourceId: choice.id } + } + } as unknown as MediaStreamConstraints) + } + let stream: MediaStream + try { + stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false }) + } catch (e) { + // Refusing is an answer, not a fault. + if ((e as Error).name === 'NotAllowedError' || (e as Error).name === 'AbortError') return null + throw e + } + if (await firstFrame(stream, SCREEN_CHOICE_MS)) return stream + for (const track of stream.getTracks()) track.stop() + this.toast('info', 'No screen was shared') + return null + } + /** The picker answering, with a source or with nothing. */ chooseScreenSource(source: { id: string; name: string } | null): void { const resolve = this.screenChoice @@ -2433,23 +2496,10 @@ export class ChatStore { this.set({ activeCall: { ...this.state.activeCall, sharingScreen: false } }) return } - const sources = await window.moho.screenSources() - if (sources.length === 0) { - this.toast('info', 'Nothing to share') - return - } - // Chosen in the window rather than here: the picker is a component, and - // this waits for it to answer. - const source = await this.askForScreenSource(sources) - if (!source) return - const stream = await navigator.mediaDevices.getUserMedia({ - audio: false, - video: { - // Chromium's own desktop-capture constraint, which is how an - // Electron window turns a source id into a stream. - mandatory: { chromeMediaSource: 'desktop', chromeMediaSourceId: source.id } - } - } as unknown as MediaStreamConstraints) + const choice = await this.chooseScreen() + if (!choice) return + const stream = await this.openScreen(choice) + if (!stream) return await call.call.shareScreen(stream) this.set({ activeCall: { ...this.state.activeCall, sharingScreen: true } }) } catch (e) { @@ -2477,15 +2527,19 @@ export class ChatStore { } try { - const sources = await window.moho.screenSources() - if (sources.length === 0) { - this.toast('info', 'Nothing to share') - return + // The same choice the Matrix path makes, for the same reason. + const choice = await this.chooseScreen() + if (!choice) return + + // On the portal the person answers its window now, before Discord is + // asked for a stream: a stream opened for a share that is then + // cancelled is a connection made for nothing. Where the capture cannot + // be opened until the connection is up, it waits for it as before. + let stream: MediaStream | null = null + if (choice.portal) { + stream = await this.openScreen(choice) + if (!stream) return } - // The same picker the Matrix path uses, for the same reason: Electron - // will not answer getDisplayMedia without being told which source. - const source = await this.askForScreenSource(sources) - if (!source) return await window.moho.rpc('startDiscordScreenShare', { bufferId }) this.toast('info', 'Setting up the stream…') @@ -2501,15 +2555,16 @@ export class ChatStore { // or a server it could not find is a sentence somebody can act on. this.toast('error', ready.error ? `Couldn’t open the stream: ${ready.error}` : 'Discord never opened the stream') await window.moho.rpc('stopDiscordScreenShare', { bufferId }).catch(() => {}) + // A capture already open has nobody to send it to. + for (const track of stream?.getTracks() ?? []) track.stop() return } - const stream = await navigator.mediaDevices.getUserMedia({ - audio: false, - video: { - mandatory: { chromeMediaSource: 'desktop', chromeMediaSourceId: source.id } - } - } as unknown as MediaStreamConstraints) + stream ??= await this.openScreen(choice) + if (!stream) { + await window.moho.rpc('stopDiscordScreenShare', { bufferId }).catch(() => {}) + return + } const { shareScreen } = await import('../lib/discordscreen') this.discordShare = await shareScreen(accountId, stream, (message) => { this.toast('error', message) diff --git a/src/shared/ipc.ts b/src/shared/ipc.ts index f9aac73..cece9ee 100644 --- a/src/shared/ipc.ts +++ b/src/shared/ipc.ts @@ -25,6 +25,7 @@ export const IPC = { sandboxState: 'moho:sandboxState', /** The screens and windows that could be shared into a call. */ screenSources: 'moho:screenSources', + screenCaptureViaPortal: 'moho:screenCaptureViaPortal', markBufferRead: 'moho:markBufferRead', browserLogin: 'moho:browserLogin', solveCaptcha: 'moho:solveCaptcha', From 90d7d5a9fdbf1f9f1ec3cdb927aea386bc83b651 Mon Sep 17 00:00:00 2001 From: Salastil Date: Tue, 29 Sep 2026 23:09:03 -0400 Subject: [PATCH 14/18] Use matrix.org's STUN server as a call backup, with a setting A homeserver can advertise a relay nobody can reach - poa.st's answers turn.poast.org and then says nothing - and moho then offered only local addresses, so a call between two networks never connected. The daemon now returns a public STUN fallback beside the relay and the client adds it to the call's servers unless matrix.fallbackStun is switched off. STUN only mirrors an address back; it relays nothing. Co-Authored-By: Claude Sonnet 5.5 --- nobilis | 2 +- .../src/components/settings/SettingsPanel.tsx | 11 ++++++++ src/renderer/src/state/store.ts | 25 ++++++++++++++----- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/nobilis b/nobilis index f2401fb..d14edbc 160000 --- a/nobilis +++ b/nobilis @@ -1 +1 @@ -Subproject commit f2401fb16d5e9cd4042d3fa9b97e6e11f3980c8a +Subproject commit d14edbcb954841801c661c51a0619b63a53e4385 diff --git a/src/renderer/src/components/settings/SettingsPanel.tsx b/src/renderer/src/components/settings/SettingsPanel.tsx index 37d310c..c635da7 100644 --- a/src/renderer/src/components/settings/SettingsPanel.tsx +++ b/src/renderer/src/components/settings/SettingsPanel.tsx @@ -815,6 +815,17 @@ function MatrixSettings(): JSX.Element { defaultValue={true} /> + + + ) diff --git a/src/renderer/src/state/store.ts b/src/renderer/src/state/store.ts index 0408e23..68b03db 100644 --- a/src/renderer/src/state/store.ts +++ b/src/renderer/src/state/store.ts @@ -923,15 +923,28 @@ export class ChatStore { }, iceServers: async (accountId) => { try { - const answer = await window.moho.rpc<{ uris?: string[]; username?: string; password?: string }>( - 'matrixTurnServers', - { accountId } - ) + const answer = await window.moho.rpc<{ + uris?: string[] + username?: string + password?: string + fallback?: string[] + }>('matrixTurnServers', { accountId }) + const servers: RTCIceServer[] = [] const uris = answer.uris ?? [] - if (uris.length === 0) return [] // One entry carrying every uri, which is how a browser wants them: // the credentials are the same for all of a homeserver's relays. - return [{ urls: uris, username: answer.username, credential: answer.password }] + if (uris.length > 0) servers.push({ urls: uris, username: answer.username, credential: answer.password }) + // A public STUN server as well, unless it has been switched off. It is + // listed apart from the relay and never given its credentials: it is a + // mirror that says what this address looks like from outside, and it + // is what gets a call across two networks when the homeserver's own + // relay is missing - or is advertised and answers nobody, which looks + // the same from here. + const prefs = await window.moho.prefs.getAll() + if (prefs['matrix.fallbackStun'] !== false && (answer.fallback?.length ?? 0) > 0) { + servers.push({ urls: answer.fallback! }) + } + return servers } catch { // No relay is a working configuration for two people on the same // network, and a call worth trying is better than a refusal. From dd547d8a5863a7ddcd0eef70dbd3902dc301c08b Mon Sep 17 00:00:00 2001 From: Salastil Date: Wed, 30 Sep 2026 09:44:40 -0400 Subject: [PATCH 15/18] One screen-share path for every desktop Matrix calls, LiveKit calls and Discord streams now all open a screen the same way: getDisplayMedia, then wait for a first frame before anything is announced. Who chooses what is shown is answered once, in the main process's display-media handler: the desktop portal on Wayland (Chromium's own rule, not any one compositor's), moho's own picker on X11, Windows and older macOS, and the system picker on macOS 15+. Before this the Wayland path and the rest used different capture calls, the LiveKit path had no handler at all outside Wayland, and Discord opened its capture after a server round trip that could spend the click's user activation. Tested on Wayland (Hyprland: one portal dialog, cancel handled) and X11 (moho's picker: 4 sources with thumbnails, a chosen window captured, cancel handled). Windows pending. Co-Authored-By: Claude Opus 5.5 --- src/main/index.ts | 49 +----------- src/main/screenshare.ts | 130 ++++++++++++++++++++++++++++++++ src/preload/index.ts | 33 ++++---- src/renderer/src/lib/livekit.ts | 31 ++++++-- src/renderer/src/lib/webrtc.ts | 2 +- src/renderer/src/state/store.ts | 124 ++++++++++++------------------ src/shared/ipc.ts | 14 +++- 7 files changed, 233 insertions(+), 150 deletions(-) create mode 100644 src/main/screenshare.ts diff --git a/src/main/index.ts b/src/main/index.ts index 3555590..a1a41ed 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -6,7 +6,6 @@ import { app, BrowserWindow, clipboard, - desktopCapturer, dialog, globalShortcut, ipcMain, @@ -30,6 +29,7 @@ import { clearnetLinks } from '../shared/clearnet' import { readCapped, pictureNamedIn } from './imagepage' import { DEEP_LINK_SCHEMES, isDeepLink } from '../shared/deeplink' import { allowPickedFile, allowRoot, installMediaHandler, registerMediaScheme } from './media-protocol' +import { installScreenShare } from './screenshare' import { defaultDownloadDir, saveMedia } from './downloads' import type { Buffer as ChatBuffer } from '../shared/wire' import { log } from './log' @@ -196,22 +196,6 @@ function send(channel: string, ...args: unknown[]): void { } /** The window an IPC call came from, so a handler acts on its own caller. */ -/** - * Whether a screen is captured through the desktop portal, which is the case - * in any Wayland session: X11 offers a list of windows and screens to draw a - * picker from, and Wayland deliberately does not. - * - * `MOHO_SCREENSHARE=picker` forces the list-and-choose path and `portal` - * forces the other, for somebody whose session reports itself wrongly. - */ -function capturesThroughPortal(): boolean { - if (process.platform !== 'linux') return false - const forced = process.env.MOHO_SCREENSHARE - if (forced === 'picker') return false - if (forced === 'portal') return true - return process.env.XDG_SESSION_TYPE === 'wayland' || !!process.env.WAYLAND_DISPLAY -} - function callerWindow(event: Electron.IpcMainInvokeEvent): BrowserWindow | null { return BrowserWindow.fromWebContents(event.sender) } @@ -664,22 +648,6 @@ function wireIpc(): void { // far. Separate from pickFile because the two dialogs ask opposite // questions, and a save dialog that cannot name a default file is a save // dialog people cancel. - // What could be shared into a call. Thumbnails at a size worth looking at - // but not worth waiting for - this is a picker, not a preview. - ipcMain.handle(IPC.screenSources, async () => { - const sources = await desktopCapturer.getSources({ - types: ['screen', 'window'], - thumbnailSize: { width: 320, height: 180 } - }) - return sources.map((s) => ({ - id: s.id, - name: s.name, - thumbnail: s.thumbnail.toDataURL() - })) - }) - - ipcMain.handle(IPC.screenCaptureViaPortal, () => capturesThroughPortal()) - ipcMain.handle(IPC.pickSavePath, async (e, suggested?: string) => { const parent = callerWindow(e) ?? mainWindow if (!parent) return null @@ -909,19 +877,8 @@ app.whenReady().then(() => { }) session.defaultSession.setPermissionCheckHandler((_contents, permission) => allowed.has(permission)) - // Screen capture on Wayland goes through the compositor's portal, which puts - // up its own "what to share" window whatever it is told. Answering - // getDisplayMedia with a bare screen id - and never asking for a list of - // sources first - makes that window appear once. Listing sources opens a - // portal session just to draw thumbnails, and opening the chosen one opens - // a second, so a share needed two windows answered and the picture only - // began after the second. Measured on Electron 43 and 44: one window, and - // the first frame arrives as soon as it is answered. - if (capturesThroughPortal()) { - session.defaultSession.setDisplayMediaRequestHandler((_request, callback) => { - callback({ video: { id: 'screen:0:0', name: 'Entire Screen' } }) - }) - } + // Every screen share in the app, on every desktop - see screenshare.ts. + installScreenShare(session.defaultSession, () => mainWindow?.webContents) // Someone tried to launch a second copy: treat it as "show me moho", which // is almost always what they meant - especially when the window is hidden diff --git a/src/main/screenshare.ts b/src/main/screenshare.ts new file mode 100644 index 0000000..8ecd1ba --- /dev/null +++ b/src/main/screenshare.ts @@ -0,0 +1,130 @@ +import { desktopCapturer, ipcMain, webContents, type Session, type WebContents } from 'electron' +import { IPC, type ScreenSource } from '../shared/ipc' +import { log } from './log' + +/** + * Sharing a screen, the same way on every desktop. + * + * Every share in the app - a Matrix call, a Matrix call on a media server, + * a Discord stream - asks the one standard question, `getDisplayMedia`, and + * this answers it. What differs between desktops is only who chooses what is + * shown, and that is decided here, once, rather than in each place that + * shares: + * + * - **The desktop chooses** on Wayland. Every Wayland compositor - GNOME, + * KDE, sway, Hyprland - hands screens out through xdg-desktop-portal, which + * always puts up its own "what do you want to share" window and gives an + * application no list to draw one of its own from. So the request is + * answered straight away and the portal asks. + * - **moho chooses** everywhere else: X11, Windows, and macOS before 15. The + * screens and windows are listed and moho's own picker is shown in the + * window that asked. + * - **macOS 15 and later** have a system picker of their own, which Electron + * uses instead of calling this at all. + * + * The rule that must never be broken is that there is exactly one question. + * Listing sources on Wayland is not free - it opens a portal session to draw + * the thumbnails, which is a portal window of its own - and opening the chosen + * source opens a second. That was two windows for one share, and no picture + * until the second was answered. + */ + +/** + * Whether the desktop, rather than moho, chooses what a share shows. + * + * This has to agree with Chromium, because it is Chromium that decides which + * capturer to use, and if the two disagree a share asks twice or not at all. + * Chromium's rule (WebRTC's `DesktopCapturer::IsRunningUnderWayland`) is a + * Linux session whose XDG_SESSION_TYPE says Wayland *and* which has a Wayland + * display to talk to; it captures through the portal whether or not the + * window itself is drawn through XWayland. So this is that rule and nothing + * else - no compositor, distribution or GPU comes into it. + * + * `MOHO_SCREENSHARE=picker` or `=portal` overrides it, for a session that + * describes itself wrongly. + */ +export function desktopPicksScreen( + env: NodeJS.ProcessEnv = process.env, + platform: NodeJS.Platform = process.platform +): boolean { + if (platform !== 'linux') return false + if (env.MOHO_SCREENSHARE === 'picker') return false + if (env.MOHO_SCREENSHARE === 'portal') return true + return (env.XDG_SESSION_TYPE ?? '').startsWith('wayland') && !!env.WAYLAND_DISPLAY +} + +/** Questions put to a window's picker, waiting for its answer. */ +const waiting = new Map void>() +let nextQuestion = 1 + +/** + * Shows the picker in `contents` and waits for a choice. + * + * Answered with null if the window closes first, so a request is never left + * hanging on a picker nobody can see any more. + */ +function askWindow(contents: WebContents, sources: ScreenSource[]): Promise { + return new Promise((resolve) => { + const question = nextQuestion++ + const gone = (): void => answer(null) + const answer = (id: string | null): void => { + waiting.delete(question) + contents.off('destroyed', gone) + resolve(id) + } + waiting.set(question, answer) + contents.once('destroyed', gone) + contents.send(IPC.screenPick, question, sources) + }) +} + +/** + * Answers every `getDisplayMedia` in `ses`. + * + * `fallback` is the window to ask when the request cannot say which one it + * came from - a frame that has already navigated away. + */ +export function installScreenShare(ses: Session, fallback: () => WebContents | undefined): void { + ipcMain.on(IPC.screenPicked, (_event, question: number, id: string | null) => { + waiting.get(question)?.(typeof id === 'string' ? id : null) + }) + + const portal = desktopPicksScreen() + log.info(`[screenshare] ${portal ? 'the desktop portal chooses what is shared' : 'moho lists screens and windows to choose from'}`) + + ses.setDisplayMediaRequestHandler( + (request, callback) => { + if (portal) { + // Any id will do: the portal capturer ignores it and asks. + callback({ video: { id: 'screen:0:0', name: 'Entire screen' } }) + return + } + void (async () => { + try { + const sources = await desktopCapturer.getSources({ + types: ['screen', 'window'], + // A still worth recognising a window by, not worth waiting for. + thumbnailSize: { width: 320, height: 180 } + }) + const contents = (request.frame && webContents.fromFrame(request.frame)) || fallback() + if (!contents || sources.length === 0) { + callback({}) + return + } + const chosen = await askWindow( + contents, + sources.map((s) => ({ id: s.id, name: s.name, thumbnail: s.thumbnail.toDataURL() })) + ) + const source = sources.find((s) => s.id === chosen) + // No stream is a refusal, which is what backing out of the picker is. + callback(source ? { video: source } : {}) + } catch (e) { + log.warn('[screenshare] could not list what to share:', (e as Error).message) + callback({}) + } + })() + }, + // macOS 15+: the system's own picker, and this handler is not called. + { useSystemPicker: true } + ) +} diff --git a/src/preload/index.ts b/src/preload/index.ts index 5c55b0f..099968b 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -1,5 +1,5 @@ import { contextBridge, ipcRenderer, webUtils } from 'electron' -import { IPC, POPOUT_FLAG, type PopoutState } from '../shared/ipc' +import { IPC, POPOUT_FLAG, type PopoutState, type ScreenSource } from '../shared/ipc' import type { NobilisEvent } from '../shared/wire' /** @@ -136,26 +136,21 @@ const api = { pickFile: (): Promise => ipcRenderer.invoke(IPC.pickFile), /** - * What could be shared into a call: every screen and window, each with a - * still of what is on it. + * Answers the main process when it needs a screen or window chosen. * - * Asked of the desktop rather than of the browser. Electron refuses - * getDisplayMedia's own picker, and a person choosing what to show a room - * needs to see which window they are choosing. + * Asked in the middle of a `getDisplayMedia`, on the desktops where moho + * draws its own picker - see main/screenshare.ts. The handler resolves with + * the chosen source's id, or null for nothing. */ - screenSources: (): Promise<{ id: string; name: string; thumbnail: string }[]> => - ipcRenderer.invoke(IPC.screenSources), - /** - * Whether a screen is chosen in the desktop's own window rather than ours. - * - * True on Wayland, where the compositor's portal is the only way to capture - * a screen and it always shows its own dialog. Asking for a list of sources - * first shows that dialog once for the list, and opening the capture shows - * it again - two windows for one share, the second of which nothing warns - * you about. Where this is true the list is skipped and the portal is asked - * once. - */ - screenCaptureViaPortal: (): Promise => ipcRenderer.invoke(IPC.screenCaptureViaPortal), + onScreenPick(cb: (sources: ScreenSource[]) => Promise): () => void { + const handler = (_e: unknown, question: number, sources: ScreenSource[]): void => { + void cb(sources) + .catch(() => null) + .then((id) => ipcRenderer.send(IPC.screenPicked, question, id)) + } + ipcRenderer.on(IPC.screenPick, handler) + return () => ipcRenderer.off(IPC.screenPick, handler) + }, pickSavePath: (suggested?: string): Promise => ipcRenderer.invoke(IPC.pickSavePath, suggested), pickDirectory: (): Promise => ipcRenderer.invoke(IPC.pickDirectory), diff --git a/src/renderer/src/lib/livekit.ts b/src/renderer/src/lib/livekit.ts index 43f0863..1c26755 100644 --- a/src/renderer/src/lib/livekit.ts +++ b/src/renderer/src/lib/livekit.ts @@ -234,12 +234,33 @@ export class SfuCall { return this.room.localParticipant.isCameraEnabled } - /** Shares a screen, or stops. Returns whether one is being shared. */ - async toggleScreen(): Promise { - const on = this.room.localParticipant.isScreenShareEnabled - await this.room.localParticipant.setScreenShareEnabled(!on) + /** Whether this end is sharing a screen. */ + get sharingScreen(): boolean { + return this.room.localParticipant.isScreenShareEnabled + } + + /** + * Shares a screen that has already been opened, or stops with null. + * Returns whether one is being shared. + * + * Opened by the caller rather than by LiveKit, so that a share here is + * chosen and checked exactly as it is in a call between two people or on + * Discord - one picker, and nothing published until a picture exists. + */ + async setScreen(stream: MediaStream | null): Promise { + const participant = this.room.localParticipant + if (!stream) { + for (const publication of [...participant.trackPublications.values()]) { + if (publication.source === Track.Source.ScreenShare && publication.track) { + await participant.unpublishTrack(publication.track, true) + } + } + } else { + const track = stream.getVideoTracks()[0] + if (track) await participant.publishTrack(track, { source: Track.Source.ScreenShare, name: 'screen' }) + } this.local = this.ownStream() - return !on + return participant.isScreenShareEnabled } /** diff --git a/src/renderer/src/lib/webrtc.ts b/src/renderer/src/lib/webrtc.ts index ad3a242..9c63eaf 100644 --- a/src/renderer/src/lib/webrtc.ts +++ b/src/renderer/src/lib/webrtc.ts @@ -308,7 +308,7 @@ export class Call { if (frames === 0 && track.readyState === 'live') { this.handlers.onError( source === 'screen' - ? 'Screen capture started but is producing no picture - the other end will see nothing. If a "Select what to share" window is open anywhere, answer it' + ? 'The shared screen has stopped producing a picture - the other end will see nothing' : 'The camera opened but is producing no picture' ) } diff --git a/src/renderer/src/state/store.ts b/src/renderer/src/state/store.ts index 68b03db..3dee95d 100644 --- a/src/renderer/src/state/store.ts +++ b/src/renderer/src/state/store.ts @@ -1081,10 +1081,10 @@ export class ChatStore { const stored = await window.moho.prefs.getAll() this.set({ lastReadTs: (stored.lastReadTs as Record) ?? {} }) - // Known before anybody presses share, so the capture can be asked for in - // the same breath as the click: a screen is a request that needs the - // gesture that made it, and a round trip first would spend it. - this.screenViaPortal = await window.moho.screenCaptureViaPortal().catch(() => false) + // The picker is drawn here but asked for by the main process, halfway + // through a capture request - see main/screenshare.ts. Every window + // registers, because the question goes to whichever one asked to share. + window.moho.onScreenPick(async (sources) => (await this.askForScreenSource(sources))?.id ?? null) window.moho.onLinkChange((up) => { this.set({ linkUp: up }) @@ -2243,61 +2243,37 @@ export class ChatStore { private askForScreenSource( sources: { id: string; name: string; thumbnail: string }[] ): Promise<{ id: string; name: string } | null> { + // A picker still open from an earlier request is answered "nothing" + // rather than left waiting for ever behind this one. + this.screenChoice?.(null) return new Promise((resolve) => { this.screenChoice = resolve this.set({ screenSources: sources }) }) } - /** Whether the desktop's portal picks the screen - see the preload. */ - private screenViaPortal = false - /** - * Decides what is to be shared, or that nothing is. + * Opens a screen to share, or returns null if the person backed out. * - * Two shapes, because two desktops. Where the portal chooses, there is - * nothing to ask here: the choice is made in its window when the capture is - * opened, and asking for a list first would put that window up twice. Where - * it does not, a list of windows is drawn as our own picker and one is - * picked. - */ - private async chooseScreen(): Promise<{ portal: true } | { portal: false; id: string } | null> { - if (this.screenViaPortal) return { portal: true } - const sources = await window.moho.screenSources() - if (sources.length === 0) { - this.toast('info', 'Nothing to share') - return null - } - // Chosen in the window rather than here: the picker is a component, and - // this waits for it to answer. - const source = await this.askForScreenSource(sources) - return source ? { portal: false, id: source.id } : null - } - - /** - * Opens what was chosen, or null if the person backed out of it. + * The same everywhere: Matrix, Matrix on a media server, and Discord all + * come through here, on every desktop. Who chooses the screen - moho's own + * picker or the desktop's - is the main process's business, answered inside + * this one request (see main/screenshare.ts), so nothing here knows or asks + * which desktop it is on. * - * On the portal path the stream comes back at once and means nothing until - * a frame does - see `firstFrame` - so this waits for one, and a window that - * was closed, or never answered, ends here as a quiet "no" rather than as a - * share that was announced to the room and shows nothing. + * A stream coming back is not a screen being shared. Through the Wayland + * portal it comes back live before the portal's window has been answered, + * and stays live and empty for ever if that window is closed. So this waits + * for a first frame, and only then is the share real; a choice never made + * ends as a quiet "no" rather than as a share announced to a room and + * showing nothing. */ - private async openScreen(choice: { portal: true } | { portal: false; id: string }): Promise { - if (!choice.portal) { - return await navigator.mediaDevices.getUserMedia({ - audio: false, - video: { - // Chromium's own desktop-capture constraint, which is how an - // Electron window turns a source id into a stream. - mandatory: { chromeMediaSource: 'desktop', chromeMediaSourceId: choice.id } - } - } as unknown as MediaStreamConstraints) - } + private async openScreen(): Promise { let stream: MediaStream try { stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false }) } catch (e) { - // Refusing is an answer, not a fault. + // Backing out of a picker is an answer, not a fault. if ((e as Error).name === 'NotAllowedError' || (e as Error).name === 'AbortError') return null throw e } @@ -2470,9 +2446,8 @@ export class ChatStore { /** * Shares a screen or a window into the call. * - * The picker is the desktop's rather than the browser's: Electron will not - * answer getDisplayMedia without being told which source, and a list of the - * windows somebody actually has open is the only way to choose one. + * Opened by `openScreen`, which is where every share on every desktop + * starts. */ async toggleScreenShare(): Promise { // A Discord call is not a Matrix one and has no activeCall behind it - @@ -2490,15 +2465,24 @@ export class ChatStore { return } if (!this.state.activeCall) return - // On a media server the picker is LiveKit's own job, and Electron's - // source list is what answers it - see the permission handler in main. + // On a media server the screen is opened here, exactly as for the other + // two, and handed to LiveKit to publish. const group = this.matrixCalls.currentGroup if (group?.sfu) { - const sharing = await group.sfu.toggleScreen().catch((e: Error) => { - this.toast('error', e.message) - return false - }) - this.set({ activeCall: { ...this.state.activeCall, sharingScreen: sharing } }) + const sfu = group.sfu + try { + let sharing: boolean + if (sfu.sharingScreen) { + sharing = await sfu.setScreen(null) + } else { + const stream = await this.openScreen() + if (!stream) return + sharing = await sfu.setScreen(stream) + } + if (this.state.activeCall) this.set({ activeCall: { ...this.state.activeCall, sharingScreen: sharing } }) + } catch (e) { + this.toast('error', `Couldn't share the screen: ${(e as Error).message}`) + } return } const call = this.matrixCalls.current @@ -2509,9 +2493,7 @@ export class ChatStore { this.set({ activeCall: { ...this.state.activeCall, sharingScreen: false } }) return } - const choice = await this.chooseScreen() - if (!choice) return - const stream = await this.openScreen(choice) + const stream = await this.openScreen() if (!stream) return await call.call.shareScreen(stream) this.set({ activeCall: { ...this.state.activeCall, sharingScreen: true } }) @@ -2540,19 +2522,12 @@ export class ChatStore { } try { - // The same choice the Matrix path makes, for the same reason. - const choice = await this.chooseScreen() - if (!choice) return - - // On the portal the person answers its window now, before Discord is - // asked for a stream: a stream opened for a share that is then - // cancelled is a connection made for nothing. Where the capture cannot - // be opened until the connection is up, it waits for it as before. - let stream: MediaStream | null = null - if (choice.portal) { - stream = await this.openScreen(choice) - if (!stream) return - } + // The screen first, then Discord. The person chooses while the click + // that asked is still fresh - a browser will not open a capture for a + // gesture spent on a twenty-second wait for a server - and a share + // they back out of never costs a stream connection. + const stream = await this.openScreen() + if (!stream) return await window.moho.rpc('startDiscordScreenShare', { bufferId }) this.toast('info', 'Setting up the stream…') @@ -2569,15 +2544,10 @@ export class ChatStore { this.toast('error', ready.error ? `Couldn’t open the stream: ${ready.error}` : 'Discord never opened the stream') await window.moho.rpc('stopDiscordScreenShare', { bufferId }).catch(() => {}) // A capture already open has nobody to send it to. - for (const track of stream?.getTracks() ?? []) track.stop() + for (const track of stream.getTracks()) track.stop() return } - stream ??= await this.openScreen(choice) - if (!stream) { - await window.moho.rpc('stopDiscordScreenShare', { bufferId }).catch(() => {}) - return - } const { shareScreen } = await import('../lib/discordscreen') this.discordShare = await shareScreen(accountId, stream, (message) => { this.toast('error', message) diff --git a/src/shared/ipc.ts b/src/shared/ipc.ts index cece9ee..00efbca 100644 --- a/src/shared/ipc.ts +++ b/src/shared/ipc.ts @@ -24,8 +24,10 @@ export const IPC = { /** Whether this process got Chromium's sandbox, or is running without it. */ sandboxState: 'moho:sandboxState', /** The screens and windows that could be shared into a call. */ - screenSources: 'moho:screenSources', - screenCaptureViaPortal: 'moho:screenCaptureViaPortal', + /** main -> window: show the screen picker (question id, sources). */ + screenPick: 'moho:screenPick', + /** window -> main: the picker's answer (question id, source id or null). */ + screenPicked: 'moho:screenPicked', markBufferRead: 'moho:markBufferRead', browserLogin: 'moho:browserLogin', solveCaptcha: 'moho:solveCaptcha', @@ -66,3 +68,11 @@ export interface PopoutState { */ watched: string[] } + +/** Something that can be shared into a call, as the picker draws it. */ +export interface ScreenSource { + id: string + name: string + /** A data: URL of what is on it. */ + thumbnail: string +} From 2acebd49d38aa92a285d5880c3ece9f96afb9ca7 Mon Sep 17 00:00:00 2001 From: Salastil Date: Wed, 30 Sep 2026 11:05:29 -0400 Subject: [PATCH 16/18] Show Discord cameras in calls, and pick up the new voice connection Co-Authored-By: Claude Opus 5.5 --- nobilis | 2 +- src/renderer/src/app.css | 6 +++ src/renderer/src/components/CallView.tsx | 14 +++++ .../src/components/DiscordCameraTile.tsx | 37 +++++++++++++ src/renderer/src/lib/discordwatch.ts | 54 +++++++++++++++---- src/renderer/src/state/store.ts | 24 +++++++++ 6 files changed, 126 insertions(+), 11 deletions(-) create mode 100644 src/renderer/src/components/DiscordCameraTile.tsx diff --git a/nobilis b/nobilis index d14edbc..cbb044d 160000 --- a/nobilis +++ b/nobilis @@ -1 +1 @@ -Subproject commit d14edbcb954841801c661c51a0619b63a53e4385 +Subproject commit cbb044dd96bbaaf3bc5e7df69694f0212fac4735 diff --git a/src/renderer/src/app.css b/src/renderer/src/app.css index e6f4e16..71f5adb 100644 --- a/src/renderer/src/app.css +++ b/src/renderer/src/app.css @@ -7031,3 +7031,9 @@ video.lightbox-media { .peek-bar-text { flex: 1; min-width: 0; display: flex; flex-direction: column; } .peek-topic { padding: var(--spacing-xs) var(--spacing-m); color: var(--surface-variant-text); border-bottom: 1px solid var(--outline-light); } .peek-log { flex: 1; min-height: 0; overflow-y: auto; overflow-x: hidden; padding: var(--spacing-s) 0 var(--spacing-m); } + +/* Discord cameras in a call: a row of pictures above the faces. */ +.discord-camera-row { display: flex; flex-wrap: wrap; gap: var(--spacing-s); padding: var(--spacing-s) 0; } +.discord-camera-tile { position: relative; width: min(320px, 100%); aspect-ratio: 16 / 9; background: #000; border-radius: var(--corner-radius-small); overflow: hidden; } +.discord-camera-canvas { width: 100%; height: 100%; object-fit: contain; display: block; } +.discord-camera-name { position: absolute; left: var(--spacing-xs); bottom: var(--spacing-xs); padding: 1px var(--spacing-xs); border-radius: var(--corner-radius-small); background: rgba(0, 0, 0, 0.55); color: #fff; max-width: calc(100% - 2 * var(--spacing-xs)); } diff --git a/src/renderer/src/components/CallView.tsx b/src/renderer/src/components/CallView.tsx index 809c13e..afe85b4 100644 --- a/src/renderer/src/components/CallView.tsx +++ b/src/renderer/src/components/CallView.tsx @@ -3,6 +3,7 @@ import { Icon, IconButton } from './Icon' import { Avatar } from './Avatar' import { useChat, useStore } from '../state/hooks' import { DiscordStreamTile } from './DiscordStreamTile' +import { DiscordCameraTile } from './DiscordCameraTile' import type { VoiceMember } from '../../../shared/wire' /** How often to ask who is talking. */ @@ -36,6 +37,7 @@ export function CallView({ bufferId }: { bufferId: string }): JSX.Element | null const [folded, setFolded] = useState(false) const sharing = useChat((st) => st.discordSharing) const watching = useChat((st) => st.discordWatching) + const cameras = useChat((st) => st.discordCameras) /** * The call this conversation should be showing. @@ -147,6 +149,18 @@ export function CallView({ bufferId }: { bufferId: string }): JSX.Element | null {!folded && } + {/* Cameras above the faces, larger than them: a camera is the one thing + in the call that is worth looking at rather than glancing at. */} + {!folded && accountId && members.some((m) => cameras[`${accountId}|${m.userId}`]) && ( +
+ {members + .filter((m) => cameras[`${accountId}|${m.userId}`]) + .map((m) => ( + + ))} +
+ )} + {!folded && (
{/* Nobody at all means the roster has not arrived yet, not that the diff --git a/src/renderer/src/components/DiscordCameraTile.tsx b/src/renderer/src/components/DiscordCameraTile.tsx new file mode 100644 index 0000000..f1f9191 --- /dev/null +++ b/src/renderer/src/components/DiscordCameraTile.tsx @@ -0,0 +1,37 @@ +import { useEffect, useRef } from 'react' +import { useStore } from '../state/hooks' +import { watchCamera } from '../lib/discordwatch' + +/** + * One person's camera in a Discord call. + * + * A canvas for the same reason as the Go Live tile: frames arrive from the + * daemon as encoded VP8 rather than as a MediaStream, and this window does the + * decoding. Mounted only while the camera is on - the store drops the entry + * when the daemon says it has gone off. + */ +export function DiscordCameraTile({ + accountId, + userId, + name +}: { + accountId: string + userId: string + name: string +}): JSX.Element { + const store = useStore() + const canvas = useRef(null) + + useEffect(() => { + if (!canvas.current) return + const watch = watchCamera(accountId, userId, canvas.current, (message) => store.toast('error', message)) + return () => watch.stop() + }, [accountId, userId]) + + return ( +
+ + {name} +
+ ) +} diff --git a/src/renderer/src/lib/discordwatch.ts b/src/renderer/src/lib/discordwatch.ts index 7e52c54..b35e82b 100644 --- a/src/renderer/src/lib/discordwatch.ts +++ b/src/renderer/src/lib/discordwatch.ts @@ -34,6 +34,47 @@ export function watchStream( streamKey: string, canvas: HTMLCanvasElement, onError: (message: string) => void +): StreamWatch { + return decodeInto( + canvas, + 'discordStreamFrame', + (d) => d.accountId === accountId && d.streamKey === streamKey, + onError + ) +} + +/** + * Somebody's camera in a voice call, decoded and drawn. + * + * The same decoding as a Go Live stream: the voice connection hands over + * whole VP8 frames exactly as the stream connection does, keyed by the person + * rather than by a stream. The daemon only sends frames once a keyframe has + * arrived, and an `ended` frame when the camera goes off. + */ +export function watchCamera( + accountId: string, + userId: string, + canvas: HTMLCanvasElement, + onError: (message: string) => void +): StreamWatch { + return decodeInto(canvas, 'discordCameraFrame', (d) => d.accountId === accountId && d.userId === userId, onError) +} + +interface FrameEvent { + accountId?: string + streamKey?: string + userId?: string + frame?: string + keyframe?: boolean + timestampMicros?: number + ended?: boolean +} + +function decodeInto( + canvas: HTMLCanvasElement, + eventName: string, + mine: (d: FrameEvent) => boolean, + onError: (message: string) => void ): StreamWatch { const context = canvas.getContext('2d') let stopped = false @@ -79,16 +120,9 @@ export function watchStream( decoder.configure({ codec: 'vp8', optimizeForLatency: true }) const unsubscribe = window.moho.onEvent((event: NobilisEvent) => { - if (stopped || event.event !== 'discordStreamFrame') return - const d = event.data as { - accountId?: string - streamKey?: string - frame?: string - keyframe?: boolean - timestampMicros?: number - ended?: boolean - } - if (d.accountId !== accountId || d.streamKey !== streamKey) return + if (stopped || event.event !== eventName) return + const d = event.data as FrameEvent + if (!mine(d)) return if (d.ended) { stop() return diff --git a/src/renderer/src/state/store.ts b/src/renderer/src/state/store.ts index 3dee95d..8c433d1 100644 --- a/src/renderer/src/state/store.ts +++ b/src/renderer/src/state/store.ts @@ -545,6 +545,12 @@ export interface ChatState { * key is what watching one is asked for by. */ discordStreams: Record + /** + * Whose camera is on in a Discord call, as `accountId|userId`. Set by the + * first frame and cleared by the `ended` one - not touched in between, so + * thirty frames a second do not redraw the window. + */ + discordCameras: Record /** The stream being watched, if any. One at a time, deliberately. */ discordWatching: { accountId: string; streamKey: string; userId: string; nick: string } | null activeCall: { @@ -658,6 +664,7 @@ const INITIAL: ChatState = { activeCall: null, discordSharing: false, discordStreams: {}, + discordCameras: {}, discordWatching: null, ignoredByAccount: {}, matrixWidgets: {}, @@ -2775,6 +2782,23 @@ export class ChatStore { break } + // Somebody's camera in a Discord call. The frames themselves are the + // tile's business (see watchCamera); here only whether there is one. + case 'discordCameraFrame': { + const d = data as { accountId?: string; userId?: string; ended?: boolean } + if (!d.accountId || !d.userId) break + const key = `${d.accountId}|${d.userId}` + const on = !!this.state.discordCameras[key] + if (d.ended && on) { + const cameras = { ...this.state.discordCameras } + delete cameras[key] + this.set({ discordCameras: cameras }) + } else if (!d.ended && !on) { + this.set({ discordCameras: { ...this.state.discordCameras, [key]: true } }) + } + break + } + // How far an attachment's upload has got. Matched to the row by the // id the send named, so a second message sent while the first is // still uploading updates its own row rather than the newest one. From af3354da546d22859eeaece8388f64301b925ab1 Mon Sep 17 00:00:00 2001 From: Salastil Date: Wed, 30 Sep 2026 11:47:56 -0400 Subject: [PATCH 17/18] A call stage like Discord's client, which can pop out into its own window Matrix and Discord calls now draw on one service-blind stage: a black, resizable panel above the conversation with everybody as a rounded 16:9 tile (their camera or screen where they have one, otherwise their picture on a panel tinted with their colour), a green ring on whoever is talking, name pills with mute and deafen, a LIVE tile with Watch stream for a shared screen, click-to-focus with a strip underneath, collapse, full screen, and the round microphone / deafen / camera / share / hang-up row. Pop out opens a child window that this renderer draws into, so a Matrix call's WebRTC streams and Discord's decoded cameras carry straight over with nothing reconnected. Camera and stream decoding moved to one decoder per picture fed from the store (lib/framefeed.ts), so a tile moved between the grid, the strip and the other window is never left waiting for a keyframe. Matrix call audio plays from one hidden element per person in the main window, so where the pictures are drawn never decides whether anybody can be heard. Co-Authored-By: Claude Opus 5.5 --- src/main/index.ts | 18 +- src/renderer/src/App.tsx | 7 +- src/renderer/src/app.css | 484 +++++++++++++----- src/renderer/src/components/CallStage.tsx | 323 ++++++------ src/renderer/src/components/CallView.tsx | 316 +++++------- .../src/components/DiscordCameraTile.tsx | 37 -- .../src/components/DiscordStreamTile.tsx | 51 -- .../src/components/stage/CallWindowHost.tsx | 32 ++ .../src/components/stage/FeedPicture.tsx | 16 + src/renderer/src/components/stage/Stage.tsx | 377 ++++++++++++++ .../src/components/stage/StageWindow.tsx | 73 +++ src/renderer/src/lib/discordwatch.ts | 182 ------- src/renderer/src/lib/framefeed.ts | 158 ++++++ src/renderer/src/state/store.ts | 19 + 14 files changed, 1352 insertions(+), 741 deletions(-) delete mode 100644 src/renderer/src/components/DiscordCameraTile.tsx delete mode 100644 src/renderer/src/components/DiscordStreamTile.tsx create mode 100644 src/renderer/src/components/stage/CallWindowHost.tsx create mode 100644 src/renderer/src/components/stage/FeedPicture.tsx create mode 100644 src/renderer/src/components/stage/Stage.tsx create mode 100644 src/renderer/src/components/stage/StageWindow.tsx delete mode 100644 src/renderer/src/lib/discordwatch.ts create mode 100644 src/renderer/src/lib/framefeed.ts diff --git a/src/main/index.ts b/src/main/index.ts index a1a41ed..9d56e2a 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -273,7 +273,23 @@ function createWindow(): void { mainWindow = null }) - mainWindow.webContents.setWindowOpenHandler(({ url }) => { + mainWindow.webContents.setWindowOpenHandler(({ url, frameName }) => { + // The one window the page may open itself: a call popped out of the + // conversation. The page draws into it directly - see StageWindow - so it + // is blank, frameless like this one, and black before anything arrives. + if (frameName === 'moho-call' && (url === '' || url === 'about:blank')) { + return { + action: 'allow', + overrideBrowserWindowOptions: { + title: 'moho — call', + frame: false, + backgroundColor: '#000000', + minWidth: 360, + minHeight: 240, + autoHideMenuBar: true + } + } + } shell.openExternal(url) return { action: 'deny' } }) diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index c58eb8f..e7e43f3 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -21,7 +21,8 @@ import { JoinPanel } from './components/JoinPanel' import { PeekBar, PeekView } from './components/PeekView' import { Toasts } from './components/Toasts' import { IncomingCallPanel } from './components/IncomingCallPanel' -import { CallStage, IncomingMatrixCall, ScreenPicker } from './components/CallStage' +import { CallAudio, CallStage, IncomingMatrixCall, ScreenPicker } from './components/CallStage' +import { CallWindowHost } from './components/stage/CallWindowHost' import { StreamStage } from './components/StreamStage' import { DiscordModal } from './components/DiscordModal' import { FileDrop } from './components/FileDrop' @@ -286,6 +287,10 @@ export default function App(): JSX.Element { it belongs to it sits above the log instead - see MessageList - so the two are never both on screen. */} {callElsewhere && } + {/* The call's sound, played from here whatever is drawing its pictures, + and the window a call can be popped out into. */} + + {/* A stream keeps playing when you go and read something else, and goes to the same corner - it is the same surface, differently fed. */} {streamElsewhere && } diff --git a/src/renderer/src/app.css b/src/renderer/src/app.css index 71f5adb..cfdb35e 100644 --- a/src/renderer/src/app.css +++ b/src/renderer/src/app.css @@ -2034,136 +2034,6 @@ img.composer-emoji { gap: var(--spacing-s); } -/* --- the call, while it is happening ------------------------------------ */ - -.call-view { - flex: none; - border-bottom: 1px solid var(--outline-medium); - background: var(--surface-container-low); -} - -.call-view-head { - display: flex; - align-items: center; - gap: var(--spacing-s); - padding: var(--spacing-xs) var(--spacing-m); -} - -/* The title takes the slack so the buttons stay at the right edge. */ -.call-view-head > .ellipsis { - flex: 1; - min-width: 0; - font-weight: 500; -} - -.call-view-head .calling { - color: var(--error); -} - -.call-tiles { - display: flex; - flex-wrap: wrap; - gap: var(--spacing-m); - padding: var(--spacing-s) var(--spacing-m) var(--spacing-m); -} - -.call-tile { - display: flex; - flex-direction: column; - align-items: center; - gap: var(--spacing-xs); - width: 88px; - text-align: center; -} - -/* The ring, drawn on the picture rather than the tile so it reads as being - around the person and not around a box. Outline rather than border: a - border would resize the avatar every time somebody started talking, and - the whole row would twitch. */ -.call-tile img, -.call-tile .avatar-fallback { - outline: 3px solid transparent; - outline-offset: 2px; - transition: outline-color 0.12s ease; -} - -.call-tile.talking img, -.call-tile.talking .avatar-fallback { - outline-color: var(--success); -} - -/* Holds the badges against the picture. */ -.call-face { - position: relative; - display: inline-flex; -} - -.call-badges { - position: absolute; - right: -2px; - bottom: -2px; - display: flex; - gap: 1px; - padding: 2px; - border-radius: 999px; - /* Its own plate, because these sit over an avatar that could be any - colour and a bare glyph would vanish against half of them. */ - background: var(--surface-container-highest); - color: var(--surface-text); -} - -/* Says a stream exists; deliberately not a button that opens one. */ -.call-watch { - padding: 0 var(--spacing-xs); - border: 1px solid var(--outline-medium); - border-radius: 999px; - background: none; - color: var(--surface-variant-text); - font-size: 10px; - text-transform: uppercase; - letter-spacing: 0.04em; -} - -.call-watch[disabled] { - opacity: 0.75; - cursor: not-allowed; -} - -/* The one being watched, so a row of Live badges says which one is open. */ -.call-watch.active { - border-color: var(--primary); - color: var(--primary); -} - -/* Somebody else's shared screen, above the row of faces rather than among - them: it is the thing being looked at, and a picture sized like an avatar - is not worth having. */ -.discord-stream-tile { - display: flex; - flex-direction: column; - gap: var(--spacing-xxs); - margin-bottom: var(--spacing-xs); -} - -.discord-stream-bar { - display: flex; - align-items: center; - justify-content: space-between; - gap: var(--spacing-xs); -} - -/* Height-capped rather than free: the canvas takes its pixel size from the - stream, and a 1080p share would otherwise push the roster off the panel. - `contain` because the sender chooses the shape and letterboxing a picture - is better than stretching somebody's desktop. */ -.discord-stream-canvas { - width: 100%; - max-height: 320px; - object-fit: contain; - border-radius: var(--corner-radius-small); - background: #000; -} - /* The bar that appears while picking people for a group message. */ .group-dm-bar { display: flex; @@ -7032,8 +6902,352 @@ video.lightbox-media { .peek-topic { padding: var(--spacing-xs) var(--spacing-m); color: var(--surface-variant-text); border-bottom: 1px solid var(--outline-light); } .peek-log { flex: 1; min-height: 0; overflow-y: auto; overflow-x: hidden; padding: var(--spacing-s) 0 var(--spacing-m); } -/* Discord cameras in a call: a row of pictures above the faces. */ -.discord-camera-row { display: flex; flex-wrap: wrap; gap: var(--spacing-s); padding: var(--spacing-s) 0; } -.discord-camera-tile { position: relative; width: min(320px, 100%); aspect-ratio: 16 / 9; background: #000; border-radius: var(--corner-radius-small); overflow: hidden; } -.discord-camera-canvas { width: 100%; height: 100%; object-fit: contain; display: block; } -.discord-camera-name { position: absolute; left: var(--spacing-xs); bottom: var(--spacing-xs); padding: 1px var(--spacing-xs); border-radius: var(--corner-radius-small); background: rgba(0, 0, 0, 0.55); color: #fff; max-width: calc(100% - 2 * var(--spacing-xs)); } +/* --- The call stage ---------------------------------------------------- */ + +/* A call, drawn the way Discord's client draws one: a black stage above the + conversation that looks like a video before there is any video in it. + Deliberately the same in both themes - it is a picture, and pictures are + shown on black. */ +.stage { + position: relative; + flex: none; + background: #000; + color: #f2f3f5; + overflow: hidden; + user-select: none; +} + +.stage.in-window, +.stage.fullscreen { + position: fixed; + inset: 0; + height: auto; +} + +.stage-window-root { + position: fixed; + inset: 0; + background: #000; +} + +.stage.collapsed { + display: flex; + align-items: center; + gap: var(--spacing-s); + padding: var(--spacing-xs) var(--spacing-s); +} + +.stage.collapsed .stage-head { + position: static; + flex: 1; + min-width: 0; + padding: 0; + background: none; +} + +.stage-head { + position: absolute; + top: 0; + left: 0; + right: 0; + z-index: 2; + display: flex; + align-items: center; + gap: var(--spacing-s); + height: 44px; + padding: 0 var(--spacing-s) 0 var(--spacing-m); + background: linear-gradient(to bottom, rgba(0, 0, 0, 0.7), transparent); +} + +/* A popped-out stage is its window's title bar too: frameless, so the head + is what the window is moved by. Buttons stay pressable. */ +.stage-head.window-drag { + -webkit-app-region: drag; +} + +.stage-head.window-drag button { + -webkit-app-region: no-drag; +} + +.stage-title { + flex: 1; + min-width: 0; + display: flex; + align-items: center; + gap: var(--spacing-xs); + font-weight: 600; +} + +.stage-subtitle { + font-weight: 400; + font-size: 12px; + color: #b5bac1; + margin-left: var(--spacing-xs); +} + +.stage-head-buttons { + display: flex; + gap: 2px; +} + +.stage-head-button { + display: inline-flex; + align-items: center; + justify-content: center; + width: 32px; + height: 32px; + border: 0; + border-radius: 6px; + background: transparent; + color: #dbdee1; + cursor: pointer; +} + +.stage-head-button:hover { + background: rgba(255, 255, 255, 0.12); + color: #fff; +} + +.stage-area { + position: absolute; + left: var(--spacing-m); + right: var(--spacing-m); + display: flex; + align-items: center; + justify-content: center; +} + +.stage-empty { + color: #949ba4; +} + +.stage-grid { + display: grid; + gap: 8px; + justify-content: center; + align-content: center; +} + +.stage-focus { + display: flex; + flex-direction: column; + align-items: center; + gap: 8px; + width: 100%; + height: 100%; + justify-content: center; +} + +.stage-strip { + display: flex; + gap: 8px; + justify-content: center; + max-width: 100%; + overflow-x: auto; +} + +.stage-tile { + position: relative; + aspect-ratio: 16 / 9; + max-width: 100%; + border-radius: 8px; + overflow: hidden; + display: flex; + align-items: center; + justify-content: center; + cursor: pointer; + outline: none; + transition: box-shadow 0.12s ease; +} + +.stage-tile.strip { + flex: none; +} + +.stage-tile:focus-visible { + box-shadow: inset 0 0 0 2px #00a8fc; +} + +/* Who is talking: the green ring Discord draws. */ +.stage-tile.speaking { + box-shadow: inset 0 0 0 3px #23a55a; +} + +.stage-tile-face .avatar-fallback, +.stage-tile-face img { + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.35); +} + +.stage-picture { + position: absolute; + inset: 0; + width: 100%; + height: 100%; + object-fit: contain; + background: #000; +} + +.stage-tile-name { + position: absolute; + left: 8px; + bottom: 8px; + z-index: 1; + display: flex; + align-items: center; + gap: 4px; + max-width: calc(100% - 16px); + padding: 2px 8px; + border-radius: 4px; + background: rgba(0, 0, 0, 0.6); + color: #fff; + font-size: 13px; + font-weight: 500; +} + +.stage-tile.strip .stage-tile-name { + left: 4px; + bottom: 4px; + font-size: 11px; + padding: 1px 6px; +} + +.stage-tile-state { + color: #f23f43; +} + +.stage-tile-live { + position: absolute; + top: 8px; + left: 8px; + z-index: 1; + padding: 1px 6px; + border-radius: 4px; + background: #da373c; + color: #fff; + font-size: 11px; + font-weight: 700; + letter-spacing: 0.04em; +} + +.stage-tile-action { + position: absolute; + z-index: 1; + padding: 8px 16px; + border: 0; + border-radius: 4px; + background: rgba(0, 0, 0, 0.72); + color: #fff; + font: inherit; + font-weight: 600; + cursor: pointer; +} + +.stage-tile-action:hover { + background: #5865f2; +} + +.stage-tile-dismiss { + position: absolute; + top: 6px; + right: 6px; + z-index: 1; + display: none; + align-items: center; + justify-content: center; + width: 28px; + height: 28px; + border: 0; + border-radius: 50%; + background: rgba(0, 0, 0, 0.6); + color: #fff; + cursor: pointer; +} + +.stage-tile:hover .stage-tile-dismiss { + display: inline-flex; +} + +/* The buttons, along the bottom middle. */ +.stage-controls { + position: absolute; + left: 0; + right: 0; + bottom: 14px; + z-index: 2; + display: flex; + justify-content: center; + gap: 10px; +} + +.stage-controls.compact { + position: static; + gap: 4px; +} + +.stage-button { + display: inline-flex; + align-items: center; + justify-content: center; + width: 52px; + height: 52px; + border: 0; + border-radius: 50%; + background: #2b2d31; + color: #f2f3f5; + cursor: pointer; + transition: background 0.12s ease; +} + +.stage-button.compact { + width: 32px; + height: 32px; +} + +.stage-button:hover { + background: #3a3c42; +} + +/* Something switched on that is doing something: a screen being shared. */ +.stage-button.active { + background: #f2f3f5; + color: #111214; +} + +/* Something switched off that normally is on: a muted microphone. */ +.stage-button.off { + background: #f2f3f5; + color: #da373c; +} + +.stage-button.danger { + background: #da373c; + color: #fff; +} + +.stage-button.danger:hover { + background: #a12828; +} + +.stage-resize { + position: absolute; + left: 0; + right: 0; + bottom: 0; + z-index: 3; + height: 6px; + cursor: ns-resize; +} + +.stage-resize:hover { + background: rgba(255, 255, 255, 0.12); +} + +/* The line left above the conversation while the call is in its own window. */ +.stage-away { + display: flex; + align-items: center; + gap: var(--spacing-s); + padding: var(--spacing-xs) var(--spacing-m); + background: var(--surface-container-high); + border-bottom: 1px solid var(--outline-light); +} diff --git a/src/renderer/src/components/CallStage.tsx b/src/renderer/src/components/CallStage.tsx index 4de851f..08e722e 100644 --- a/src/renderer/src/components/CallStage.tsx +++ b/src/renderer/src/components/CallStage.tsx @@ -1,6 +1,7 @@ import { useEffect, useRef, useState } from 'react' import { Icon, IconButton } from './Icon' import { VideoStage } from './VideoStage' +import { Stage, fitTiles, type StageButton, type StageTile } from './stage/Stage' import { Avatar } from './Avatar' import { useChat, useStore } from '../state/hooks' import type { CallTile } from '../state/store' @@ -22,141 +23,159 @@ import { bufferDisplayName, classes } from '../lib/util' * which is what a call is on any service - the screen-share button here is the * same button Discord's will be, and the tiles are the same tiles. */ -/** Sixteen by nine, which is what cameras and screens both are. */ -const TILE_RATIO = 16 / 9 -/** No tile gets bigger than this, however much room there is. */ -const TILE_MAX = 420 +export { fitTiles } from './stage/Stage' + +/** The gap between tiles in the corner player. */ const TILE_GAP = 6 -/** - * How to lay out a call of this size in the space there is. - * - * Every column count is tried and the one that makes the tiles biggest wins, - * subject to them all fitting - which is what a grid of faces should do and - * what a fixed table of counts cannot: three people in a wide short box want - * one row, and the same three in a tall narrow one want three. - * - * Capped, because a call between two people on a large screen should not be - * two enormous windows - past a certain size a face stops being easier to - * read and the conversation underneath just disappears. - */ -export function fitTiles( - count: number, - width: number, - height: number, - max: number = TILE_MAX -): { columns: number; tileWidth: number } { - if (count < 1 || width < 1) return { columns: 1, tileWidth: 0 } - let best = { columns: 1, tileWidth: 0 } - for (let columns = 1; columns <= count; columns++) { - const rows = Math.ceil(count / columns) - // What the width allows, and what the height allows; the smaller governs. - const byWidth = (width - TILE_GAP * (columns - 1)) / columns - const byHeight = ((height - TILE_GAP * (rows - 1)) / rows) * TILE_RATIO - const tileWidth = Math.min(byWidth, byHeight, max) - if (tileWidth > best.tileWidth) best = { columns, tileWidth } - } - return best -} - -export function CallStage({ mode = 'inline' }: { mode?: 'inline' | 'pip' }): JSX.Element | null { +export function CallStage({ mode = 'inline' }: { mode?: 'inline' | 'pip' | 'window' }): JSX.Element | null { const store = useStore() const call = useChat((s) => s.activeCall) const minimized = useChat((s) => s.callMinimized) + const poppedOut = useChat((s) => s.callPoppedOut) const buffers = useChat((s) => s.buffers) const videos = useRef(new Map()) const grid = useRef(null) const [tiles, setTiles] = useState([]) - /** How wide the grid is, which decides how the tiles are laid out. */ - const [room, setRoom] = useState({ width: 640, height: 260 }) - /** - * Whose picture is being looked at, if anybody. - * - * Null is the grid, where nobody is more important than anybody else. Naming - * one person is a decision the person watching made, so it is undone the - * same way it was made rather than by anything the call does. - */ + /** How wide the corner player's grid is, which decides its layout. */ + const [room, setRoom] = useState({ width: 320, height: 180 }) + /** Whose picture the corner player is showing large, if anybody. */ const [focused, setFocused] = useState(null) - // Measured rather than assumed: the same call is a different shape in a - // narrow window, in a full-screen one, and in the corner it retreats to. useEffect(() => { const el = grid.current - if (!el) return - const measure = (): void => - setRoom({ - width: el.clientWidth, - // As much height as a call may take before it is taking over the - // conversation rather than sitting above it. - height: Math.min(window.innerHeight * (mode === 'pip' ? 0.3 : 0.45), mode === 'pip' ? 260 : 460) - }) + if (!el || mode !== 'pip') return + const measure = (): void => setRoom({ width: el.clientWidth, height: Math.min(window.innerHeight * 0.3, 260) }) measure() const observer = new ResizeObserver(measure) observer.observe(el) - window.addEventListener('resize', measure) - return () => { - observer.disconnect() - window.removeEventListener('resize', measure) - } + return () => observer.disconnect() }, [mode, minimized]) // The streams are not state: they are live objects whose tracks change under // a call - somebody turning a camera on, somebody starting to share - so the - // elements are pointed at them rather than re-rendered from them. + // elements are pointed at them rather than re-rendered from them. Every + // element showing a person is kept pointed at their stream, however many + // places they are drawn at once. useEffect(() => { if (!call) return const attach = (): void => { const current = store.callTiles() setTiles((was) => - was.length === current.length && was.every((t, i) => t.id === current[i].id && t.hasVideo === current[i].hasVideo) + was.length === current.length && + was.every( + (t, i) => + t.id === current[i].id && + t.hasVideo === current[i].hasVideo && + t.speaking === current[i].speaking && + t.muted === current[i].muted + ) ? was : current ) - // Both slots: in the focused layout the same person is on the stage and - // in the strip underneath, which is two elements showing one stream. for (const tile of current) { - for (const slot of ['stage', 'strip']) { - const el = videos.current.get(`${tile.id}#${slot}`) - if (el && tile.stream && el.srcObject !== tile.stream) el.srcObject = tile.stream + for (const [key, el] of videos.current) { + if (key.startsWith(`${tile.id}#`) && tile.stream && el.srcObject !== tile.stream) el.srcObject = tile.stream } } } attach() - const timer = setInterval(attach, 700) + const timer = setInterval(attach, 300) return () => clearInterval(timer) }, [call, store]) if (!call) return null + // In a window of its own: the conversation keeps a line saying so, and the + // corner keeps nothing. + if (mode !== 'window' && poppedOut) { + if (mode === 'pip') return null + return ( +
+ + The call is in its own window. + +
+ ) + } + const where = buffers.find((b) => b.id === call.bufferId)?.name ?? '' - // Somebody who has left the call cannot go on being the one looked at. - const stage = tiles.find((t) => t.id === focused) ?? null - const layout = fitTiles(Math.max(tiles.length, 1), room.width, room.height) - /** The row of everybody else under the stage, and how tall it is. */ - const stripHeight = mode === 'pip' ? 42 : 74 - const stageFit = fitTiles( - 1, - room.width, - room.height - (tiles.length > 1 ? stripHeight + TILE_GAP : 0), - // Uncapped: the whole point of picking somebody out is to see them larger - // than the grid would ever draw them. - Infinity - ) const phase = call.phase === 'ringing' ? 'Ringing…' : call.phase === 'connecting' ? 'Connecting…' : 'Connected' - // Who is in it, named. Only in the corner: in the conversation the tiles are - // large enough to read the names off, and a second list of them would be the - // same information twice. + /** + * A video element for one person in one place. Always muted: the call's + * sound is played once, by CallAudio, not by every picture of every person + * - two pictures of one person would otherwise be two voices. + */ + const video = (tile: CallTile, slot: string, className: string): JSX.Element => ( +