A LiveKit client, for the calls held on a media server
One stream up and everybody else's back down, rather than a connection per person - which is what Element's calls are and, past a handful of people, the only shape that works. Deliberately not a second implementation of the call: it hands back the same people-and-streams the mesh does, so the grid, the speaking ring and the corner view do not know which kind they are drawing, and mute and screen share reach whichever is up. Two things learned by hitting them. The renderer's own CSP refused the websocket - a signalling connection is neither http nor https - so `wss:` is allowed now, and plaintext only to this machine, since a server across a network should not be carrying signalling in the clear. And the stage now goes up before the joining rather than after it: connecting to a media server takes a moment, and setting the phase afterwards overwrote the "connected" the call had already reported. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
Claude Opus 5
+1
-1
Submodule nobilis updated: 9e07dba3c2...2d96e89a34
Generated
+158
@@ -21,6 +21,7 @@
|
||||
"electron-builder": "^25.1.8",
|
||||
"electron-vite": "^2.3.0",
|
||||
"hls.js": "^1.7.2",
|
||||
"livekit-client": "^2.22.2",
|
||||
"react": "^18.3.1",
|
||||
"react-dom": "^18.3.1",
|
||||
"typescript": "^5.7.2",
|
||||
@@ -325,6 +326,13 @@
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@bufbuild/protobuf": {
|
||||
"version": "1.10.1",
|
||||
"resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-1.10.1.tgz",
|
||||
"integrity": "sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ==",
|
||||
"dev": true,
|
||||
"license": "(Apache-2.0 AND BSD-3-Clause)"
|
||||
},
|
||||
"node_modules/@develar/schema-utils": {
|
||||
"version": "2.6.5",
|
||||
"resolved": "https://registry.npmjs.org/@develar/schema-utils/-/schema-utils-2.6.5.tgz",
|
||||
@@ -1279,6 +1287,23 @@
|
||||
"@jridgewell/sourcemap-codec": "^1.4.14"
|
||||
}
|
||||
},
|
||||
"node_modules/@livekit/mutex": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@livekit/mutex/-/mutex-1.1.1.tgz",
|
||||
"integrity": "sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/@livekit/protocol": {
|
||||
"version": "1.50.4",
|
||||
"resolved": "https://registry.npmjs.org/@livekit/protocol/-/protocol-1.50.4.tgz",
|
||||
"integrity": "sha512-L1uggNQAqyY21smQY8AllyOYbcv9Me9TaxwuLytL1R8ck9nbYPmQLNwEDi3pOFGAMa5F8I2nUi2Jc59W5awxlA==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@bufbuild/protobuf": "^1.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@malept/cross-spawn-promise": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@malept/cross-spawn-promise/-/cross-spawn-promise-2.0.0.tgz",
|
||||
@@ -1927,6 +1952,14 @@
|
||||
"@types/ms": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/dom-mediacapture-record": {
|
||||
"version": "1.0.22",
|
||||
"resolved": "https://registry.npmjs.org/@types/dom-mediacapture-record/-/dom-mediacapture-record-1.0.22.tgz",
|
||||
"integrity": "sha512-mUMZLK3NvwRLcAAT9qmcK+9p7tpU2FHdDsntR3YI4+GY88XrgG4XiE7u1Q2LAN2/FZOz/tdMDC3GQCR4T8nFuw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true
|
||||
},
|
||||
"node_modules/@types/estree": {
|
||||
"version": "1.0.9",
|
||||
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz",
|
||||
@@ -3969,6 +4002,16 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/events": {
|
||||
"version": "3.3.0",
|
||||
"resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
|
||||
"integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.8.x"
|
||||
}
|
||||
},
|
||||
"node_modules/exponential-backoff": {
|
||||
"version": "3.1.3",
|
||||
"resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.3.tgz",
|
||||
@@ -4816,6 +4859,16 @@
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/jose": {
|
||||
"version": "6.2.12",
|
||||
"resolved": "https://registry.npmjs.org/jose/-/jose-6.2.12.tgz",
|
||||
"integrity": "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/panva"
|
||||
}
|
||||
},
|
||||
"node_modules/js-tokens": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
|
||||
@@ -4967,6 +5020,28 @@
|
||||
"safe-buffer": "~5.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/livekit-client": {
|
||||
"version": "2.22.2",
|
||||
"resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-2.22.2.tgz",
|
||||
"integrity": "sha512-o3yLYoH2eHVR/DmIZzgC96FMvtR4+g/DjeqNwS5zp12EjqrqiRULxm6Oq/YsTE97WnA6fhy23+a2hGkbce5SPA==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@livekit/mutex": "1.1.1",
|
||||
"@livekit/protocol": "1.50.4",
|
||||
"events": "^3.3.0",
|
||||
"jose": "^6.1.0",
|
||||
"loglevel": "^1.9.2",
|
||||
"machina": "^7.0.0",
|
||||
"sdp-transform": "^2.15.0",
|
||||
"tslib": "2.8.1",
|
||||
"typed-emitter": "^2.1.0",
|
||||
"webrtc-adapter": "9.0.6"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@types/dom-mediacapture-record": "^1"
|
||||
}
|
||||
},
|
||||
"node_modules/lodash": {
|
||||
"version": "4.18.1",
|
||||
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz",
|
||||
@@ -5031,6 +5106,20 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/loglevel": {
|
||||
"version": "1.9.2",
|
||||
"resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.9.2.tgz",
|
||||
"integrity": "sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 0.6.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "tidelift",
|
||||
"url": "https://tidelift.com/funding/github/npm/loglevel"
|
||||
}
|
||||
},
|
||||
"node_modules/loose-envify": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
|
||||
@@ -5063,6 +5152,16 @@
|
||||
"yallist": "^3.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/machina": {
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/machina/-/machina-7.0.1.tgz",
|
||||
"integrity": "sha512-0Xkgs8NeBvO2hW6bjJT8ewLWrMfh1haMPstaghpASWjXkLjRhtDiaJG1Kb7mslaOHxZYxpH7PIjpmirRVYbMMA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=22.22"
|
||||
}
|
||||
},
|
||||
"node_modules/magic-string": {
|
||||
"version": "0.30.21",
|
||||
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz",
|
||||
@@ -6141,6 +6240,17 @@
|
||||
"fsevents": "~2.3.2"
|
||||
}
|
||||
},
|
||||
"node_modules/rxjs": {
|
||||
"version": "7.8.2",
|
||||
"resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz",
|
||||
"integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/safe-buffer": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
|
||||
@@ -6199,6 +6309,23 @@
|
||||
"loose-envify": "^1.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/sdp": {
|
||||
"version": "3.2.2",
|
||||
"resolved": "https://registry.npmjs.org/sdp/-/sdp-3.2.2.tgz",
|
||||
"integrity": "sha512-xZocWwfyp4hkbN4hLWxMjmv2Q8aNa9MhmOZ7L9aCZPT+dZsgRr6wZRrSYE3HTdyk/2pZKPSgqI7ns7Een1xMSA==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/sdp-transform": {
|
||||
"version": "2.15.0",
|
||||
"resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.15.0.tgz",
|
||||
"integrity": "sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"sdp-verify": "checker.js"
|
||||
}
|
||||
},
|
||||
"node_modules/semver": {
|
||||
"version": "6.3.1",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
|
||||
@@ -6651,6 +6778,13 @@
|
||||
"utf8-byte-length": "^1.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/tslib": {
|
||||
"version": "2.8.1",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
|
||||
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
|
||||
"dev": true,
|
||||
"license": "0BSD"
|
||||
},
|
||||
"node_modules/type-fest": {
|
||||
"version": "0.13.1",
|
||||
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz",
|
||||
@@ -6664,6 +6798,16 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/typed-emitter": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/typed-emitter/-/typed-emitter-2.1.0.tgz",
|
||||
"integrity": "sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optionalDependencies": {
|
||||
"rxjs": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/typescript": {
|
||||
"version": "5.9.3",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
|
||||
@@ -6860,6 +7004,20 @@
|
||||
"defaults": "^1.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/webrtc-adapter": {
|
||||
"version": "9.0.6",
|
||||
"resolved": "https://registry.npmjs.org/webrtc-adapter/-/webrtc-adapter-9.0.6.tgz",
|
||||
"integrity": "sha512-CHbl2ZQbxx164IgWRgzJno4hWtM4tFbRam1QfI3Yxhs3w/DvqluVxVWeXs3oL5/fbGkSNLKo0Ty5MgUWceNhog==",
|
||||
"dev": true,
|
||||
"license": "BSD-3-Clause",
|
||||
"dependencies": {
|
||||
"sdp": "^3.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.0.0",
|
||||
"npm": ">=3.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/which": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
"electron-builder": "^25.1.8",
|
||||
"electron-vite": "^2.3.0",
|
||||
"hls.js": "^1.7.2",
|
||||
"livekit-client": "^2.22.2",
|
||||
"react": "^18.3.1",
|
||||
"react-dom": "^18.3.1",
|
||||
"typescript": "^5.7.2",
|
||||
|
||||
@@ -1,11 +1,18 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<!--
|
||||
Why the websocket schemes are in connect-src: a Matrix room call held on
|
||||
a media server is a LiveKit connection, and LiveKit signals over a
|
||||
websocket. `wss:` for a real one; plaintext only to this machine, since
|
||||
a server reached over a network should not be carrying signalling in
|
||||
the clear and allowing `ws:` outright would say it may.
|
||||
-->
|
||||
<meta charset="UTF-8" />
|
||||
<title>moho</title>
|
||||
<meta
|
||||
http-equiv="Content-Security-Policy"
|
||||
content="default-src 'self'; img-src 'self' data: blob: https: moho-media:; media-src 'self' blob: https: moho-media:; font-src 'self'; style-src 'self' 'unsafe-inline'; connect-src 'self' https:"
|
||||
content="default-src 'self'; img-src 'self' data: blob: https: moho-media:; media-src 'self' blob: https: moho-media:; font-src 'self'; style-src 'self' 'unsafe-inline'; connect-src 'self' https: wss: ws://localhost:* http://localhost:*"
|
||||
/>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
@@ -299,28 +299,24 @@ export function RoomCallBar({ bufferId }: { bufferId: string }): JSX.Element | n
|
||||
? `${others[0].user_id} is in a call`
|
||||
: `${others.length} people are in a call`
|
||||
|
||||
// A call held on a media server - which is what Element's own calls are.
|
||||
// moho can see who is in one and cannot be in it yet, so it says so here
|
||||
// rather than offering a button that fails.
|
||||
const needsFocus = members.some((m) => m.transports?.includes('livekit'))
|
||||
// Where the call is held, which is worth saying: on a media server it is
|
||||
// the same call Element's people are in, and on the mesh it is one between
|
||||
// whoever is in the room.
|
||||
const onServer = members.some((m) => m.transports?.includes('livekit'))
|
||||
|
||||
return (
|
||||
<div className="room-call-bar">
|
||||
<Icon name="videocam" size={16} />
|
||||
<span className="small ellipsis">
|
||||
{who}
|
||||
{needsFocus ? ' · through a media server moho cannot join yet' : ''}
|
||||
{onServer ? ' · on a media server' : ''}
|
||||
</span>
|
||||
{!needsFocus && (
|
||||
<>
|
||||
<button type="button" className="button" onClick={() => void store.joinMatrixGroupCall(bufferId, false)}>
|
||||
Join
|
||||
</button>
|
||||
<button type="button" className="button" onClick={() => void store.joinMatrixGroupCall(bufferId, true)}>
|
||||
Join with video
|
||||
</button>
|
||||
</>
|
||||
)}
|
||||
<button type="button" className="button" onClick={() => void store.joinMatrixGroupCall(bufferId, false)}>
|
||||
Join
|
||||
</button>
|
||||
<button type="button" className="button" onClick={() => void store.joinMatrixGroupCall(bufferId, true)}>
|
||||
Join with video
|
||||
</button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
/**
|
||||
* A call held on a media server, which is what Element's calls are.
|
||||
*
|
||||
* The mesh beside this connects everybody to everybody: fine for a handful of
|
||||
* people, and nothing any other Matrix client speaks. Element Call sends one
|
||||
* stream up to a LiveKit SFU and gets everybody else's back down, which costs
|
||||
* one upload however many people are in the room - and, more to the point, is
|
||||
* where the people using Element already are.
|
||||
*
|
||||
* moho is a LiveKit client here and nothing more: the daemon has already
|
||||
* turned this account's identity into a URL and a token (see calls.rs), and
|
||||
* what happens after that is the SFU's own protocol rather than Matrix's.
|
||||
*
|
||||
* Deliberately not a second implementation of the tiles. It hands back the
|
||||
* same shape the mesh does - a list of people and their streams - so the grid,
|
||||
* the speaking ring and the corner view do not know which kind of call they
|
||||
* are drawing.
|
||||
*/
|
||||
|
||||
import {
|
||||
ConnectionState,
|
||||
Room,
|
||||
RoomEvent,
|
||||
Track,
|
||||
type RemoteTrack,
|
||||
type RemoteTrackPublication
|
||||
} from 'livekit-client'
|
||||
|
||||
/** One person on the media server, and what they are sending. */
|
||||
export interface SfuParticipant {
|
||||
/** LiveKit's own identity, which the JWT service sets to `user:device`. */
|
||||
key: string
|
||||
userId: string
|
||||
stream: MediaStream
|
||||
hasVideo: boolean
|
||||
}
|
||||
|
||||
export interface SfuHandlers {
|
||||
/** Somebody joined, left, or started sending something different. */
|
||||
onParticipants: (people: SfuParticipant[]) => void
|
||||
/** Whether this end is on the server, which is what "connected" means for
|
||||
* a call held on one - being alone in it is still being in it. */
|
||||
onConnected: (connected: boolean) => void
|
||||
onError: (message: string) => void
|
||||
}
|
||||
|
||||
/**
|
||||
* The Matrix user behind a LiveKit identity.
|
||||
*
|
||||
* The JWT service sets the identity to `@user:server:DEVICEID`, so the user id
|
||||
* is everything up to the last colon - which is not the same as splitting on
|
||||
* the first, because a Matrix id has a colon in it already.
|
||||
*/
|
||||
export function userFromIdentity(identity: string): string {
|
||||
const cut = identity.lastIndexOf(':')
|
||||
return cut > 0 ? identity.slice(0, cut) : identity
|
||||
}
|
||||
|
||||
export class SfuCall {
|
||||
private room: Room
|
||||
private handlers: SfuHandlers
|
||||
/** This end's own capture, so the tile shows what is being sent. */
|
||||
local: MediaStream | null = null
|
||||
|
||||
constructor(handlers: SfuHandlers) {
|
||||
this.handlers = handlers
|
||||
this.room = new Room({
|
||||
// Let LiveKit decide what to send as the window and the connection
|
||||
// change: it can see the packet loss and this cannot.
|
||||
adaptiveStream: true,
|
||||
dynacast: true
|
||||
})
|
||||
const announce = (): void => this.handlers.onParticipants(this.people())
|
||||
this.room
|
||||
.on(RoomEvent.TrackSubscribed, announce)
|
||||
.on(RoomEvent.TrackUnsubscribed, announce)
|
||||
.on(RoomEvent.ParticipantConnected, announce)
|
||||
.on(RoomEvent.ParticipantDisconnected, announce)
|
||||
.on(RoomEvent.Disconnected, () => {
|
||||
this.handlers.onParticipants([])
|
||||
this.handlers.onConnected(false)
|
||||
})
|
||||
.on(RoomEvent.ConnectionStateChanged, (state) => {
|
||||
// Being on the server is what connected means here, whether or not
|
||||
// anybody else has arrived - unlike the mesh, where there is nothing
|
||||
// to be connected *to* until somebody answers.
|
||||
this.handlers.onConnected(state === ConnectionState.Connected)
|
||||
announce()
|
||||
})
|
||||
}
|
||||
|
||||
/** Joins the media server and starts sending. */
|
||||
async join(url: string, token: string, video: boolean): Promise<void> {
|
||||
await this.room.connect(url, token)
|
||||
try {
|
||||
await this.room.localParticipant.setMicrophoneEnabled(true)
|
||||
if (video) await this.room.localParticipant.setCameraEnabled(true)
|
||||
} catch (e) {
|
||||
// A machine with no camera still belongs in the call.
|
||||
this.handlers.onError(`Couldn't open the camera: ${(e as Error).message}`)
|
||||
await this.room.localParticipant.setMicrophoneEnabled(true).catch(() => {})
|
||||
}
|
||||
this.local = this.ownStream()
|
||||
this.handlers.onConnected(this.room.state === ConnectionState.Connected)
|
||||
this.handlers.onParticipants(this.people())
|
||||
}
|
||||
|
||||
/** Everybody else, with whatever they are currently sending. */
|
||||
people(): SfuParticipant[] {
|
||||
const out: SfuParticipant[] = []
|
||||
for (const participant of this.room.remoteParticipants.values()) {
|
||||
const stream = new MediaStream()
|
||||
let hasVideo = false
|
||||
for (const publication of participant.trackPublications.values()) {
|
||||
const track = (publication as RemoteTrackPublication).track as RemoteTrack | undefined
|
||||
if (!track?.mediaStreamTrack) continue
|
||||
stream.addTrack(track.mediaStreamTrack)
|
||||
if (track.kind === Track.Kind.Video) hasVideo = true
|
||||
}
|
||||
if (stream.getTracks().length === 0) continue
|
||||
out.push({
|
||||
key: participant.identity,
|
||||
userId: userFromIdentity(participant.identity),
|
||||
stream,
|
||||
hasVideo
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** What this end is sending, for its own tile. */
|
||||
private ownStream(): MediaStream | null {
|
||||
const stream = new MediaStream()
|
||||
for (const publication of this.room.localParticipant.trackPublications.values()) {
|
||||
const track = publication.track?.mediaStreamTrack
|
||||
if (track) stream.addTrack(track)
|
||||
}
|
||||
return stream.getTracks().length > 0 ? stream : null
|
||||
}
|
||||
|
||||
/** Silences the microphone, or lets it speak again. Returns muted. */
|
||||
async toggleMute(): Promise<boolean> {
|
||||
const on = this.room.localParticipant.isMicrophoneEnabled
|
||||
await this.room.localParticipant.setMicrophoneEnabled(!on)
|
||||
return on
|
||||
}
|
||||
|
||||
/** Shares a screen, or stops. Returns whether one is being shared. */
|
||||
async toggleScreen(): Promise<boolean> {
|
||||
const on = this.room.localParticipant.isScreenShareEnabled
|
||||
await this.room.localParticipant.setScreenShareEnabled(!on)
|
||||
this.local = this.ownStream()
|
||||
return !on
|
||||
}
|
||||
|
||||
/** Leaves, letting go of the camera and the microphone. */
|
||||
async leave(): Promise<void> {
|
||||
await this.room.disconnect()
|
||||
this.local = null
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the media server is telling us it cannot read what is arriving.
|
||||
*
|
||||
* Element encrypts a call's media per sender in an encrypted room, with the
|
||||
* keys shared over Matrix. moho does not do that yet, so a call in such a
|
||||
* room is one where everybody is connected and nobody can hear anybody -
|
||||
* which is worth saying rather than leaving people to wonder.
|
||||
*/
|
||||
get anyoneUndecodable(): boolean {
|
||||
return [...this.room.remoteParticipants.values()].some((p) =>
|
||||
[...p.trackPublications.values()].some((pub) => pub.isEncrypted && !pub.isSubscribed)
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@
|
||||
*/
|
||||
|
||||
import { Call, type CallPhase } from './webrtc'
|
||||
import { SfuCall, type SfuParticipant } from './livekit'
|
||||
|
||||
/** What the daemon sends up when a call event arrives in a room. */
|
||||
export interface MatrixCallEvent {
|
||||
@@ -78,7 +79,15 @@ export interface CallSurface {
|
||||
iceServers: (accountId: string) => Promise<RTCIceServer[]>
|
||||
/** Publishes this end's membership of the room's call, and says who we are
|
||||
* in it - the user and the device, since one person may be in from two. */
|
||||
joinMembership: (bufferId: string) => Promise<{ userId: string; deviceId: string; roomId?: string } | null>
|
||||
joinMembership: (bufferId: string) => Promise<{
|
||||
userId: string
|
||||
deviceId: string
|
||||
roomId?: string
|
||||
/** Where the media goes, when the call is held on a media server. */
|
||||
media?: { url: string; jwt: string; identity: string } | null
|
||||
/** Whether the room is encrypted, which decides whether the media is. */
|
||||
encrypted?: boolean
|
||||
} | null>
|
||||
leaveMembership: (bufferId: string) => Promise<void>
|
||||
/** Who the room says is in the call. */
|
||||
callMembers: (bufferId: string) => Promise<CallMember[]>
|
||||
@@ -136,6 +145,15 @@ export interface GroupCall {
|
||||
peers: Map<string, Peer>
|
||||
/** This end's camera and microphone, opened once and lent to every leg. */
|
||||
local: MediaStream | null
|
||||
/**
|
||||
* The media server, where the call is held on one.
|
||||
*
|
||||
* Set means this is an SFU call - Element's kind - and `peers` stays empty
|
||||
* because nobody is connected to anybody directly. Absent means the mesh.
|
||||
*/
|
||||
sfu: SfuCall | null
|
||||
/** Who the media server says is here, for the SFU kind. */
|
||||
sfuPeople: SfuParticipant[]
|
||||
/** The whole call's id, on every signal, so a second call in the same room
|
||||
* is a second call rather than a crossed line. */
|
||||
confId: string
|
||||
@@ -235,8 +253,50 @@ export class MatrixCalls {
|
||||
}
|
||||
const own = await this.surface.joinMembership(bufferId)
|
||||
if (!own) return
|
||||
this.iceForGroup = await this.surface.iceServers(accountId)
|
||||
const ownKey = memberKey(own.userId, own.deviceId)
|
||||
|
||||
// A call held on a media server, which is what Element's are. Nobody
|
||||
// connects to anybody directly, so none of the mesh below runs.
|
||||
if (own.media) {
|
||||
const sfu = new SfuCall({
|
||||
onParticipants: (people) => {
|
||||
if (this.group) this.group.sfuPeople = people
|
||||
},
|
||||
onConnected: (connected) => this.surface.onPhase(connected ? 'connected' : 'connecting', null),
|
||||
onError: (message) => this.surface.onError(message)
|
||||
})
|
||||
this.group = {
|
||||
accountId,
|
||||
bufferId,
|
||||
ownKey,
|
||||
video,
|
||||
local: null,
|
||||
peers: new Map(),
|
||||
sfu,
|
||||
sfuPeople: [],
|
||||
confId: `moho-conf-${own.roomId ?? bufferId}`
|
||||
}
|
||||
this.heartbeat = setInterval(() => void this.surface.joinMembership(bufferId), 30_000)
|
||||
this.surface.onPhase('connecting', null)
|
||||
try {
|
||||
await sfu.join(own.media.url, own.media.jwt, video)
|
||||
this.group.local = sfu.local
|
||||
if (own.encrypted) {
|
||||
// Element encrypts a call's media per sender in an encrypted room,
|
||||
// with the keys shared over Matrix. moho does not do that yet, so
|
||||
// this is a call everybody is in and nobody can hear.
|
||||
this.surface.onError(
|
||||
'This room is encrypted, and moho cannot yet read the media in an encrypted call - the others will not hear you'
|
||||
)
|
||||
}
|
||||
} catch (e) {
|
||||
this.surface.onError(`Couldn't reach the media server: ${(e as Error).message}`)
|
||||
this.leaveGroup()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
this.iceForGroup = await this.surface.iceServers(accountId)
|
||||
// Opened once here rather than per leg: five connections asking the
|
||||
// machine for the same camera get one capture and four refusals.
|
||||
let local: MediaStream | null = null
|
||||
@@ -262,6 +322,8 @@ export class MatrixCalls {
|
||||
video,
|
||||
local,
|
||||
peers: new Map(),
|
||||
sfu: null,
|
||||
sfuPeople: [],
|
||||
// Named for the room rather than for the buffer: each account calls the
|
||||
// same room by a different buffer id, and a conference named after one
|
||||
// of those is a conference the other end never joins.
|
||||
@@ -285,6 +347,9 @@ export class MatrixCalls {
|
||||
async meet(members: CallMember[]): Promise<void> {
|
||||
const group = this.group
|
||||
if (!group) return
|
||||
// On a media server there is nobody to meet: the server is what everybody
|
||||
// is connected to, and it says who is there.
|
||||
if (group.sfu) return
|
||||
const present = new Set<string>()
|
||||
for (const member of members) {
|
||||
const key = memberKey(member.user_id, member.membership?.device_id ?? '')
|
||||
@@ -329,8 +394,12 @@ export class MatrixCalls {
|
||||
if (this.heartbeat) clearInterval(this.heartbeat)
|
||||
this.heartbeat = null
|
||||
for (const peer of group.peers.values()) void peer.call.hangUp('user_hangup', true)
|
||||
// The shared capture is nobody's leg to stop, so it is stopped here.
|
||||
for (const track of group.local?.getTracks() ?? []) track.stop()
|
||||
if (group.sfu) {
|
||||
void group.sfu.leave()
|
||||
} else {
|
||||
// The shared capture is nobody's leg to stop, so it is stopped here.
|
||||
for (const track of group.local?.getTracks() ?? []) track.stop()
|
||||
}
|
||||
void this.surface.leaveMembership(group.bufferId)
|
||||
this.surface.onPhase('ended', null)
|
||||
}
|
||||
@@ -402,7 +471,7 @@ export class MatrixCalls {
|
||||
*/
|
||||
private handleGroup(event: MatrixCallEvent): boolean {
|
||||
const group = this.group
|
||||
if (!group) return false
|
||||
if (!group || group.sfu) return false
|
||||
const { content, kind, callId } = event
|
||||
if (content.conf_id !== group.confId) return false
|
||||
// Somebody else's leg of the mesh, seen because room events are seen by
|
||||
|
||||
@@ -1837,9 +1837,12 @@ export class ChatStore {
|
||||
async joinMatrixGroupCall(bufferId: string, video: boolean): Promise<void> {
|
||||
const account = this.accountFor(bufferId)
|
||||
if (!account) return
|
||||
await this.matrixCalls.joinGroup(account.id, bufferId, video)
|
||||
const group = this.matrixCalls.currentGroup
|
||||
if (!group) return
|
||||
// The stage goes up before the joining rather than after it: connecting
|
||||
// to a media server takes a moment, and a window that shows nothing for
|
||||
// that moment looks like a button that did nothing. Set first so the
|
||||
// phases the call reports afterwards land on top of it rather than
|
||||
// underneath - reporting "connected" and then overwriting it with
|
||||
// "connecting" was exactly the bug this shape avoids.
|
||||
this.set({
|
||||
callMinimized: false,
|
||||
activeCall: {
|
||||
@@ -1851,6 +1854,8 @@ export class ChatStore {
|
||||
sharingScreen: false
|
||||
}
|
||||
})
|
||||
await this.matrixCalls.joinGroup(account.id, bufferId, video)
|
||||
if (!this.matrixCalls.currentGroup) this.set({ activeCall: null })
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1914,6 +1919,18 @@ export class ChatStore {
|
||||
speaking: !this.state.activeCall?.muted && this.levels.speaking('self')
|
||||
}
|
||||
]
|
||||
// On a media server, everybody arrives from the server rather than from
|
||||
// a connection of their own - the same tiles either way.
|
||||
for (const person of group.sfuPeople) {
|
||||
this.levels.watch(person.key, person.stream)
|
||||
tiles.push({
|
||||
id: person.key,
|
||||
label: this.matrixDisplayName(group.bufferId, person.userId),
|
||||
stream: person.stream,
|
||||
hasVideo: person.hasVideo,
|
||||
speaking: this.levels.speaking(person.key)
|
||||
})
|
||||
}
|
||||
for (const peer of group.peers.values()) {
|
||||
this.levels.watch(peer.key, peer.remote)
|
||||
// Only once there is something to show: a tile for somebody who has
|
||||
@@ -1973,8 +1990,26 @@ export class ChatStore {
|
||||
}
|
||||
|
||||
toggleCallMute(): void {
|
||||
if (!this.state.activeCall) return
|
||||
// Three kinds of call reach this button and the button does not care:
|
||||
// one to one, a mesh in a room, and a room call held on a media server.
|
||||
const group = this.matrixCalls.currentGroup
|
||||
if (group?.sfu) {
|
||||
void group.sfu.toggleMute().then((muted) => {
|
||||
if (this.state.activeCall) this.set({ activeCall: { ...this.state.activeCall, muted } })
|
||||
})
|
||||
return
|
||||
}
|
||||
if (group) {
|
||||
// Every leg carries the same capture, so muting it mutes all of them.
|
||||
const track = group.local?.getAudioTracks()[0]
|
||||
if (!track) return
|
||||
track.enabled = !track.enabled
|
||||
this.set({ activeCall: { ...this.state.activeCall, muted: !track.enabled } })
|
||||
return
|
||||
}
|
||||
const call = this.matrixCalls.current
|
||||
if (!call || !this.state.activeCall) return
|
||||
if (!call) return
|
||||
const muted = call.call.toggleMute()
|
||||
this.set({ activeCall: { ...this.state.activeCall, muted } })
|
||||
}
|
||||
@@ -1987,8 +2022,20 @@ export class ChatStore {
|
||||
* windows somebody actually has open is the only way to choose one.
|
||||
*/
|
||||
async toggleScreenShare(): Promise<void> {
|
||||
if (!this.state.activeCall) return
|
||||
// On a media server the picker is LiveKit's own job, and Electron's
|
||||
// source list is what answers it - see the permission handler in main.
|
||||
const group = this.matrixCalls.currentGroup
|
||||
if (group?.sfu) {
|
||||
const sharing = await group.sfu.toggleScreen().catch((e: Error) => {
|
||||
this.toast('error', e.message)
|
||||
return false
|
||||
})
|
||||
this.set({ activeCall: { ...this.state.activeCall, sharingScreen: sharing } })
|
||||
return
|
||||
}
|
||||
const call = this.matrixCalls.current
|
||||
if (!call || !this.state.activeCall) return
|
||||
if (!call) return
|
||||
try {
|
||||
if (call.call.sharingScreen) {
|
||||
await call.call.shareScreen()
|
||||
|
||||
Reference in New Issue
Block a user