From 4e9f6ddbb218aa2e8ca4c5f23b832b194ca1d76f Mon Sep 17 00:00:00 2001 From: Salastil Date: Sun, 6 Sep 2026 20:45:59 -0400 Subject: [PATCH] A LiveKit client, for the calls held on a media server One stream up and everybody else's back down, rather than a connection per person - which is what Element's calls are and, past a handful of people, the only shape that works. Deliberately not a second implementation of the call: it hands back the same people-and-streams the mesh does, so the grid, the speaking ring and the corner view do not know which kind they are drawing, and mute and screen share reach whichever is up. Two things learned by hitting them. The renderer's own CSP refused the websocket - a signalling connection is neither http nor https - so `wss:` is allowed now, and plaintext only to this machine, since a server across a network should not be carrying signalling in the clear. And the stage now goes up before the joining rather than after it: connecting to a media server takes a moment, and setting the phase afterwards overwrote the "connected" the call had already reported. Co-Authored-By: Claude Opus 5 --- nobilis | 2 +- package-lock.json | 158 +++++++++++++++++++ package.json | 1 + src/renderer/index.html | 9 +- src/renderer/src/components/CallStage.tsx | 26 ++-- src/renderer/src/lib/livekit.ts | 175 ++++++++++++++++++++++ src/renderer/src/lib/matrixcall.ts | 79 +++++++++- src/renderer/src/state/store.ts | 57 ++++++- 8 files changed, 480 insertions(+), 27 deletions(-) create mode 100644 src/renderer/src/lib/livekit.ts diff --git a/nobilis b/nobilis index 9e07dba..2d96e89 160000 --- a/nobilis +++ b/nobilis @@ -1 +1 @@ -Subproject commit 9e07dba3c2aa7618a679ffe46105dfe9cee2898b +Subproject commit 2d96e89a34f503a76a88d27d373fc320e18edbdd diff --git a/package-lock.json b/package-lock.json index 586dbe3..0ad228c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,7 @@ "electron-builder": "^25.1.8", "electron-vite": "^2.3.0", "hls.js": "^1.7.2", + "livekit-client": "^2.22.2", "react": "^18.3.1", "react-dom": "^18.3.1", "typescript": "^5.7.2", @@ -325,6 +326,13 @@ "node": ">=6.9.0" } }, + "node_modules/@bufbuild/protobuf": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-1.10.1.tgz", + "integrity": "sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ==", + "dev": true, + "license": "(Apache-2.0 AND BSD-3-Clause)" + }, "node_modules/@develar/schema-utils": { "version": "2.6.5", "resolved": "https://registry.npmjs.org/@develar/schema-utils/-/schema-utils-2.6.5.tgz", @@ -1279,6 +1287,23 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@livekit/mutex": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@livekit/mutex/-/mutex-1.1.1.tgz", + "integrity": "sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/@livekit/protocol": { + "version": "1.50.4", + "resolved": "https://registry.npmjs.org/@livekit/protocol/-/protocol-1.50.4.tgz", + "integrity": "sha512-L1uggNQAqyY21smQY8AllyOYbcv9Me9TaxwuLytL1R8ck9nbYPmQLNwEDi3pOFGAMa5F8I2nUi2Jc59W5awxlA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@bufbuild/protobuf": "^1.10.0" + } + }, "node_modules/@malept/cross-spawn-promise": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@malept/cross-spawn-promise/-/cross-spawn-promise-2.0.0.tgz", @@ -1927,6 +1952,14 @@ "@types/ms": "*" } }, + "node_modules/@types/dom-mediacapture-record": { + "version": "1.0.22", + "resolved": "https://registry.npmjs.org/@types/dom-mediacapture-record/-/dom-mediacapture-record-1.0.22.tgz", + "integrity": "sha512-mUMZLK3NvwRLcAAT9qmcK+9p7tpU2FHdDsntR3YI4+GY88XrgG4XiE7u1Q2LAN2/FZOz/tdMDC3GQCR4T8nFuw==", + "dev": true, + "license": "MIT", + "peer": true + }, "node_modules/@types/estree": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", @@ -3969,6 +4002,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/exponential-backoff": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.3.tgz", @@ -4816,6 +4859,16 @@ "node": ">=10" } }, + "node_modules/jose": { + "version": "6.2.12", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.12.tgz", + "integrity": "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -4967,6 +5020,28 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/livekit-client": { + "version": "2.22.2", + "resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-2.22.2.tgz", + "integrity": "sha512-o3yLYoH2eHVR/DmIZzgC96FMvtR4+g/DjeqNwS5zp12EjqrqiRULxm6Oq/YsTE97WnA6fhy23+a2hGkbce5SPA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@livekit/mutex": "1.1.1", + "@livekit/protocol": "1.50.4", + "events": "^3.3.0", + "jose": "^6.1.0", + "loglevel": "^1.9.2", + "machina": "^7.0.0", + "sdp-transform": "^2.15.0", + "tslib": "2.8.1", + "typed-emitter": "^2.1.0", + "webrtc-adapter": "9.0.6" + }, + "peerDependencies": { + "@types/dom-mediacapture-record": "^1" + } + }, "node_modules/lodash": { "version": "4.18.1", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", @@ -5031,6 +5106,20 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/loglevel": { + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.9.2.tgz", + "integrity": "sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + }, + "funding": { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/loglevel" + } + }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -5063,6 +5152,16 @@ "yallist": "^3.0.2" } }, + "node_modules/machina": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/machina/-/machina-7.0.1.tgz", + "integrity": "sha512-0Xkgs8NeBvO2hW6bjJT8ewLWrMfh1haMPstaghpASWjXkLjRhtDiaJG1Kb7mslaOHxZYxpH7PIjpmirRVYbMMA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=22.22" + } + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -6141,6 +6240,17 @@ "fsevents": "~2.3.2" } }, + "node_modules/rxjs": { + "version": "7.8.2", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", + "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "tslib": "^2.1.0" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -6199,6 +6309,23 @@ "loose-envify": "^1.1.0" } }, + "node_modules/sdp": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/sdp/-/sdp-3.2.2.tgz", + "integrity": "sha512-xZocWwfyp4hkbN4hLWxMjmv2Q8aNa9MhmOZ7L9aCZPT+dZsgRr6wZRrSYE3HTdyk/2pZKPSgqI7ns7Een1xMSA==", + "dev": true, + "license": "MIT" + }, + "node_modules/sdp-transform": { + "version": "2.15.0", + "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.15.0.tgz", + "integrity": "sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw==", + "dev": true, + "license": "MIT", + "bin": { + "sdp-verify": "checker.js" + } + }, "node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", @@ -6651,6 +6778,13 @@ "utf8-byte-length": "^1.0.1" } }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD" + }, "node_modules/type-fest": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", @@ -6664,6 +6798,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/typed-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/typed-emitter/-/typed-emitter-2.1.0.tgz", + "integrity": "sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==", + "dev": true, + "license": "MIT", + "optionalDependencies": { + "rxjs": "*" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -6860,6 +7004,20 @@ "defaults": "^1.0.3" } }, + "node_modules/webrtc-adapter": { + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/webrtc-adapter/-/webrtc-adapter-9.0.6.tgz", + "integrity": "sha512-CHbl2ZQbxx164IgWRgzJno4hWtM4tFbRam1QfI3Yxhs3w/DvqluVxVWeXs3oL5/fbGkSNLKo0Ty5MgUWceNhog==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "sdp": "^3.2.0" + }, + "engines": { + "node": ">=6.0.0", + "npm": ">=3.10.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index de9f846..ae666d8 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "electron-builder": "^25.1.8", "electron-vite": "^2.3.0", "hls.js": "^1.7.2", + "livekit-client": "^2.22.2", "react": "^18.3.1", "react-dom": "^18.3.1", "typescript": "^5.7.2", diff --git a/src/renderer/index.html b/src/renderer/index.html index 0dd1e0f..ccd050a 100644 --- a/src/renderer/index.html +++ b/src/renderer/index.html @@ -1,11 +1,18 @@ + moho diff --git a/src/renderer/src/components/CallStage.tsx b/src/renderer/src/components/CallStage.tsx index 9601d0b..f3e7dd9 100644 --- a/src/renderer/src/components/CallStage.tsx +++ b/src/renderer/src/components/CallStage.tsx @@ -299,28 +299,24 @@ export function RoomCallBar({ bufferId }: { bufferId: string }): JSX.Element | n ? `${others[0].user_id} is in a call` : `${others.length} people are in a call` - // A call held on a media server - which is what Element's own calls are. - // moho can see who is in one and cannot be in it yet, so it says so here - // rather than offering a button that fails. - const needsFocus = members.some((m) => m.transports?.includes('livekit')) + // Where the call is held, which is worth saying: on a media server it is + // the same call Element's people are in, and on the mesh it is one between + // whoever is in the room. + const onServer = members.some((m) => m.transports?.includes('livekit')) return (
{who} - {needsFocus ? ' · through a media server moho cannot join yet' : ''} + {onServer ? ' · on a media server' : ''} - {!needsFocus && ( - <> - - - - )} + +
) } diff --git a/src/renderer/src/lib/livekit.ts b/src/renderer/src/lib/livekit.ts new file mode 100644 index 0000000..ba38a4d --- /dev/null +++ b/src/renderer/src/lib/livekit.ts @@ -0,0 +1,175 @@ +/** + * A call held on a media server, which is what Element's calls are. + * + * The mesh beside this connects everybody to everybody: fine for a handful of + * people, and nothing any other Matrix client speaks. Element Call sends one + * stream up to a LiveKit SFU and gets everybody else's back down, which costs + * one upload however many people are in the room - and, more to the point, is + * where the people using Element already are. + * + * moho is a LiveKit client here and nothing more: the daemon has already + * turned this account's identity into a URL and a token (see calls.rs), and + * what happens after that is the SFU's own protocol rather than Matrix's. + * + * Deliberately not a second implementation of the tiles. It hands back the + * same shape the mesh does - a list of people and their streams - so the grid, + * the speaking ring and the corner view do not know which kind of call they + * are drawing. + */ + +import { + ConnectionState, + Room, + RoomEvent, + Track, + type RemoteTrack, + type RemoteTrackPublication +} from 'livekit-client' + +/** One person on the media server, and what they are sending. */ +export interface SfuParticipant { + /** LiveKit's own identity, which the JWT service sets to `user:device`. */ + key: string + userId: string + stream: MediaStream + hasVideo: boolean +} + +export interface SfuHandlers { + /** Somebody joined, left, or started sending something different. */ + onParticipants: (people: SfuParticipant[]) => void + /** Whether this end is on the server, which is what "connected" means for + * a call held on one - being alone in it is still being in it. */ + onConnected: (connected: boolean) => void + onError: (message: string) => void +} + +/** + * The Matrix user behind a LiveKit identity. + * + * The JWT service sets the identity to `@user:server:DEVICEID`, so the user id + * is everything up to the last colon - which is not the same as splitting on + * the first, because a Matrix id has a colon in it already. + */ +export function userFromIdentity(identity: string): string { + const cut = identity.lastIndexOf(':') + return cut > 0 ? identity.slice(0, cut) : identity +} + +export class SfuCall { + private room: Room + private handlers: SfuHandlers + /** This end's own capture, so the tile shows what is being sent. */ + local: MediaStream | null = null + + constructor(handlers: SfuHandlers) { + this.handlers = handlers + this.room = new Room({ + // Let LiveKit decide what to send as the window and the connection + // change: it can see the packet loss and this cannot. + adaptiveStream: true, + dynacast: true + }) + const announce = (): void => this.handlers.onParticipants(this.people()) + this.room + .on(RoomEvent.TrackSubscribed, announce) + .on(RoomEvent.TrackUnsubscribed, announce) + .on(RoomEvent.ParticipantConnected, announce) + .on(RoomEvent.ParticipantDisconnected, announce) + .on(RoomEvent.Disconnected, () => { + this.handlers.onParticipants([]) + this.handlers.onConnected(false) + }) + .on(RoomEvent.ConnectionStateChanged, (state) => { + // Being on the server is what connected means here, whether or not + // anybody else has arrived - unlike the mesh, where there is nothing + // to be connected *to* until somebody answers. + this.handlers.onConnected(state === ConnectionState.Connected) + announce() + }) + } + + /** Joins the media server and starts sending. */ + async join(url: string, token: string, video: boolean): Promise { + await this.room.connect(url, token) + try { + await this.room.localParticipant.setMicrophoneEnabled(true) + if (video) await this.room.localParticipant.setCameraEnabled(true) + } catch (e) { + // A machine with no camera still belongs in the call. + this.handlers.onError(`Couldn't open the camera: ${(e as Error).message}`) + await this.room.localParticipant.setMicrophoneEnabled(true).catch(() => {}) + } + this.local = this.ownStream() + this.handlers.onConnected(this.room.state === ConnectionState.Connected) + this.handlers.onParticipants(this.people()) + } + + /** Everybody else, with whatever they are currently sending. */ + people(): SfuParticipant[] { + const out: SfuParticipant[] = [] + for (const participant of this.room.remoteParticipants.values()) { + const stream = new MediaStream() + let hasVideo = false + for (const publication of participant.trackPublications.values()) { + const track = (publication as RemoteTrackPublication).track as RemoteTrack | undefined + if (!track?.mediaStreamTrack) continue + stream.addTrack(track.mediaStreamTrack) + if (track.kind === Track.Kind.Video) hasVideo = true + } + if (stream.getTracks().length === 0) continue + out.push({ + key: participant.identity, + userId: userFromIdentity(participant.identity), + stream, + hasVideo + }) + } + return out + } + + /** What this end is sending, for its own tile. */ + private ownStream(): MediaStream | null { + const stream = new MediaStream() + for (const publication of this.room.localParticipant.trackPublications.values()) { + const track = publication.track?.mediaStreamTrack + if (track) stream.addTrack(track) + } + return stream.getTracks().length > 0 ? stream : null + } + + /** Silences the microphone, or lets it speak again. Returns muted. */ + async toggleMute(): Promise { + const on = this.room.localParticipant.isMicrophoneEnabled + await this.room.localParticipant.setMicrophoneEnabled(!on) + return on + } + + /** Shares a screen, or stops. Returns whether one is being shared. */ + async toggleScreen(): Promise { + const on = this.room.localParticipant.isScreenShareEnabled + await this.room.localParticipant.setScreenShareEnabled(!on) + this.local = this.ownStream() + return !on + } + + /** Leaves, letting go of the camera and the microphone. */ + async leave(): Promise { + await this.room.disconnect() + this.local = null + } + + /** + * Whether the media server is telling us it cannot read what is arriving. + * + * Element encrypts a call's media per sender in an encrypted room, with the + * keys shared over Matrix. moho does not do that yet, so a call in such a + * room is one where everybody is connected and nobody can hear anybody - + * which is worth saying rather than leaving people to wonder. + */ + get anyoneUndecodable(): boolean { + return [...this.room.remoteParticipants.values()].some((p) => + [...p.trackPublications.values()].some((pub) => pub.isEncrypted && !pub.isSubscribed) + ) + } +} diff --git a/src/renderer/src/lib/matrixcall.ts b/src/renderer/src/lib/matrixcall.ts index 44e130b..b810ba4 100644 --- a/src/renderer/src/lib/matrixcall.ts +++ b/src/renderer/src/lib/matrixcall.ts @@ -13,6 +13,7 @@ */ import { Call, type CallPhase } from './webrtc' +import { SfuCall, type SfuParticipant } from './livekit' /** What the daemon sends up when a call event arrives in a room. */ export interface MatrixCallEvent { @@ -78,7 +79,15 @@ export interface CallSurface { iceServers: (accountId: string) => Promise /** Publishes this end's membership of the room's call, and says who we are * in it - the user and the device, since one person may be in from two. */ - joinMembership: (bufferId: string) => Promise<{ userId: string; deviceId: string; roomId?: string } | null> + joinMembership: (bufferId: string) => Promise<{ + userId: string + deviceId: string + roomId?: string + /** Where the media goes, when the call is held on a media server. */ + media?: { url: string; jwt: string; identity: string } | null + /** Whether the room is encrypted, which decides whether the media is. */ + encrypted?: boolean + } | null> leaveMembership: (bufferId: string) => Promise /** Who the room says is in the call. */ callMembers: (bufferId: string) => Promise @@ -136,6 +145,15 @@ export interface GroupCall { peers: Map /** This end's camera and microphone, opened once and lent to every leg. */ local: MediaStream | null + /** + * The media server, where the call is held on one. + * + * Set means this is an SFU call - Element's kind - and `peers` stays empty + * because nobody is connected to anybody directly. Absent means the mesh. + */ + sfu: SfuCall | null + /** Who the media server says is here, for the SFU kind. */ + sfuPeople: SfuParticipant[] /** The whole call's id, on every signal, so a second call in the same room * is a second call rather than a crossed line. */ confId: string @@ -235,8 +253,50 @@ export class MatrixCalls { } const own = await this.surface.joinMembership(bufferId) if (!own) return - this.iceForGroup = await this.surface.iceServers(accountId) const ownKey = memberKey(own.userId, own.deviceId) + + // A call held on a media server, which is what Element's are. Nobody + // connects to anybody directly, so none of the mesh below runs. + if (own.media) { + const sfu = new SfuCall({ + onParticipants: (people) => { + if (this.group) this.group.sfuPeople = people + }, + onConnected: (connected) => this.surface.onPhase(connected ? 'connected' : 'connecting', null), + onError: (message) => this.surface.onError(message) + }) + this.group = { + accountId, + bufferId, + ownKey, + video, + local: null, + peers: new Map(), + sfu, + sfuPeople: [], + confId: `moho-conf-${own.roomId ?? bufferId}` + } + this.heartbeat = setInterval(() => void this.surface.joinMembership(bufferId), 30_000) + this.surface.onPhase('connecting', null) + try { + await sfu.join(own.media.url, own.media.jwt, video) + this.group.local = sfu.local + if (own.encrypted) { + // Element encrypts a call's media per sender in an encrypted room, + // with the keys shared over Matrix. moho does not do that yet, so + // this is a call everybody is in and nobody can hear. + this.surface.onError( + 'This room is encrypted, and moho cannot yet read the media in an encrypted call - the others will not hear you' + ) + } + } catch (e) { + this.surface.onError(`Couldn't reach the media server: ${(e as Error).message}`) + this.leaveGroup() + } + return + } + + this.iceForGroup = await this.surface.iceServers(accountId) // Opened once here rather than per leg: five connections asking the // machine for the same camera get one capture and four refusals. let local: MediaStream | null = null @@ -262,6 +322,8 @@ export class MatrixCalls { video, local, peers: new Map(), + sfu: null, + sfuPeople: [], // Named for the room rather than for the buffer: each account calls the // same room by a different buffer id, and a conference named after one // of those is a conference the other end never joins. @@ -285,6 +347,9 @@ export class MatrixCalls { async meet(members: CallMember[]): Promise { const group = this.group if (!group) return + // On a media server there is nobody to meet: the server is what everybody + // is connected to, and it says who is there. + if (group.sfu) return const present = new Set() for (const member of members) { const key = memberKey(member.user_id, member.membership?.device_id ?? '') @@ -329,8 +394,12 @@ export class MatrixCalls { if (this.heartbeat) clearInterval(this.heartbeat) this.heartbeat = null for (const peer of group.peers.values()) void peer.call.hangUp('user_hangup', true) - // The shared capture is nobody's leg to stop, so it is stopped here. - for (const track of group.local?.getTracks() ?? []) track.stop() + if (group.sfu) { + void group.sfu.leave() + } else { + // The shared capture is nobody's leg to stop, so it is stopped here. + for (const track of group.local?.getTracks() ?? []) track.stop() + } void this.surface.leaveMembership(group.bufferId) this.surface.onPhase('ended', null) } @@ -402,7 +471,7 @@ export class MatrixCalls { */ private handleGroup(event: MatrixCallEvent): boolean { const group = this.group - if (!group) return false + if (!group || group.sfu) return false const { content, kind, callId } = event if (content.conf_id !== group.confId) return false // Somebody else's leg of the mesh, seen because room events are seen by diff --git a/src/renderer/src/state/store.ts b/src/renderer/src/state/store.ts index 6a78f52..14e20a2 100644 --- a/src/renderer/src/state/store.ts +++ b/src/renderer/src/state/store.ts @@ -1837,9 +1837,12 @@ export class ChatStore { async joinMatrixGroupCall(bufferId: string, video: boolean): Promise { const account = this.accountFor(bufferId) if (!account) return - await this.matrixCalls.joinGroup(account.id, bufferId, video) - const group = this.matrixCalls.currentGroup - if (!group) return + // The stage goes up before the joining rather than after it: connecting + // to a media server takes a moment, and a window that shows nothing for + // that moment looks like a button that did nothing. Set first so the + // phases the call reports afterwards land on top of it rather than + // underneath - reporting "connected" and then overwriting it with + // "connecting" was exactly the bug this shape avoids. this.set({ callMinimized: false, activeCall: { @@ -1851,6 +1854,8 @@ export class ChatStore { sharingScreen: false } }) + await this.matrixCalls.joinGroup(account.id, bufferId, video) + if (!this.matrixCalls.currentGroup) this.set({ activeCall: null }) } /** @@ -1914,6 +1919,18 @@ export class ChatStore { speaking: !this.state.activeCall?.muted && this.levels.speaking('self') } ] + // On a media server, everybody arrives from the server rather than from + // a connection of their own - the same tiles either way. + for (const person of group.sfuPeople) { + this.levels.watch(person.key, person.stream) + tiles.push({ + id: person.key, + label: this.matrixDisplayName(group.bufferId, person.userId), + stream: person.stream, + hasVideo: person.hasVideo, + speaking: this.levels.speaking(person.key) + }) + } for (const peer of group.peers.values()) { this.levels.watch(peer.key, peer.remote) // Only once there is something to show: a tile for somebody who has @@ -1973,8 +1990,26 @@ export class ChatStore { } toggleCallMute(): void { + if (!this.state.activeCall) return + // Three kinds of call reach this button and the button does not care: + // one to one, a mesh in a room, and a room call held on a media server. + const group = this.matrixCalls.currentGroup + if (group?.sfu) { + void group.sfu.toggleMute().then((muted) => { + if (this.state.activeCall) this.set({ activeCall: { ...this.state.activeCall, muted } }) + }) + return + } + if (group) { + // Every leg carries the same capture, so muting it mutes all of them. + const track = group.local?.getAudioTracks()[0] + if (!track) return + track.enabled = !track.enabled + this.set({ activeCall: { ...this.state.activeCall, muted: !track.enabled } }) + return + } const call = this.matrixCalls.current - if (!call || !this.state.activeCall) return + if (!call) return const muted = call.call.toggleMute() this.set({ activeCall: { ...this.state.activeCall, muted } }) } @@ -1987,8 +2022,20 @@ export class ChatStore { * windows somebody actually has open is the only way to choose one. */ async toggleScreenShare(): Promise { + if (!this.state.activeCall) return + // On a media server the picker is LiveKit's own job, and Electron's + // source list is what answers it - see the permission handler in main. + const group = this.matrixCalls.currentGroup + if (group?.sfu) { + const sharing = await group.sfu.toggleScreen().catch((e: Error) => { + this.toast('error', e.message) + return false + }) + this.set({ activeCall: { ...this.state.activeCall, sharingScreen: sharing } }) + return + } const call = this.matrixCalls.current - if (!call || !this.state.activeCall) return + if (!call) return try { if (call.call.sharingScreen) { await call.call.shareScreen()