diff --git a/nobilis b/nobilis index 2d96e89..96e096a 160000 --- a/nobilis +++ b/nobilis @@ -1 +1 @@ -Subproject commit 2d96e89a34f503a76a88d27d373fc320e18edbdd +Subproject commit 96e096adf4887341b1110dfff0c71ad4fd4c4e0e diff --git a/src/renderer/src/lib/livekit.ts b/src/renderer/src/lib/livekit.ts index ba38a4d..054b407 100644 --- a/src/renderer/src/lib/livekit.ts +++ b/src/renderer/src/lib/livekit.ts @@ -18,6 +18,7 @@ */ import { + BaseKeyProvider, ConnectionState, Room, RoomEvent, @@ -25,6 +26,7 @@ import { type RemoteTrack, type RemoteTrackPublication } from 'livekit-client' +import E2EEWorker from 'livekit-client/e2ee-worker?worker' /** One person on the media server, and what they are sending. */ export interface SfuParticipant { @@ -35,6 +37,38 @@ export interface SfuParticipant { hasVideo: boolean } +/** + * The keys a call's media is encrypted with, as Matrix hands them around. + * + * The media server forwards frames it cannot read: each participant makes a + * key, sends it to the devices in the call over Matrix, and everybody's + * player is told which key belongs to which participant. This is the provider + * LiveKit asks for those keys, filled from the daemon's side of that + * exchange - see calls.rs. + * + * The window sizes are Element's own, because a key provider that ratchets + * differently from the other clients in a call is a key provider that decodes + * nothing. + */ +class MatrixKeyProvider extends BaseKeyProvider { + constructor() { + super({ ratchetWindowSize: 10, keyringSize: 256 }) + } + + /** Somebody's key, for the participant the media server knows them as. */ + async setParticipantKey(identity: string, index: number, keyBase64: string): Promise { + const raw = Uint8Array.from(atob(keyBase64.replace(/-/g, '+').replace(/_/g, '/')), (c) => c.charCodeAt(0)) + // Imported as HKDF material rather than as a key: LiveKit derives the + // cipher key from it, and Element hands it the same material, which is + // what makes the two agree. + const material = await crypto.subtle.importKey('raw', raw as BufferSource, 'HKDF', false, [ + 'deriveBits', + 'deriveKey' + ]) + this.onSetEncryptionKey(material, identity, index) + } +} + export interface SfuHandlers { /** Somebody joined, left, or started sending something different. */ onParticipants: (people: SfuParticipant[]) => void @@ -59,16 +93,26 @@ export function userFromIdentity(identity: string): string { export class SfuCall { private room: Room private handlers: SfuHandlers + /** Where the keys go, for an encrypted call. Absent means one in the clear. */ + private keys: MatrixKeyProvider | null = null /** This end's own capture, so the tile shows what is being sent. */ local: MediaStream | null = null - constructor(handlers: SfuHandlers) { + constructor(handlers: SfuHandlers, encrypted: boolean) { this.handlers = handlers + this.keys = encrypted ? new MatrixKeyProvider() : null this.room = new Room({ // Let LiveKit decide what to send as the window and the connection // change: it can see the packet loss and this cannot. adaptiveStream: true, - dynacast: true + dynacast: true, + // A call in an encrypted room is encrypted between the people in it, + // with the server forwarding frames it cannot read. The worker is where + // the frames are actually enciphered - the browser will only let a + // transform touch them off the main thread. + ...(this.keys + ? { e2ee: { keyProvider: this.keys as unknown as BaseKeyProvider, worker: new E2EEWorker() } } + : {}) }) const announce = (): void => this.handlers.onParticipants(this.people()) this.room @@ -80,6 +124,13 @@ export class SfuCall { this.handlers.onParticipants([]) this.handlers.onConnected(false) }) + // Frames arriving that cannot be read. In an encrypted call that means + // somebody's key never got here, or theirs and ours disagree - which + // looks exactly like a working call with silence in it, so it is worth + // saying rather than leaving people to wonder why nobody is talking. + .on(RoomEvent.EncryptionError, (error) => + this.handlers.onError(`Cannot read what somebody in the call is sending: ${error.message}`) + ) .on(RoomEvent.ConnectionStateChanged, (state) => { // Being on the server is what connected means here, whether or not // anybody else has arrived - unlike the mesh, where there is nothing @@ -89,6 +140,23 @@ export class SfuCall { }) } + /** Whether this call's media is encrypted end to end. */ + get encrypted(): boolean { + return this.keys !== null + } + + /** This end's own key, which everybody else has to be told about. */ + async useOwnKey(identity: string, index: number, keyBase64: string): Promise { + if (!this.keys) return + await this.keys.setParticipantKey(identity, index, keyBase64) + await this.room.setE2EEEnabled(true) + } + + /** Somebody else's key, as it arrives over Matrix. */ + async useTheirKey(identity: string, index: number, keyBase64: string): Promise { + await this.keys?.setParticipantKey(identity, index, keyBase64) + } + /** Joins the media server and starts sending. */ async join(url: string, token: string, video: boolean): Promise { await this.room.connect(url, token) @@ -153,6 +221,33 @@ export class SfuCall { return !on } + /** + * What the connection says is actually arriving, per participant. + * + * Tracks existing is not the same as tracks being readable: an undecodable + * stream is still a stream, with silence in it. The receiver's own counters + * are what tell the two apart. + */ + async arriving(): Promise<{ identity: string; packets: number; samples: number }[]> { + const out: { identity: string; packets: number; samples: number }[] = [] + for (const participant of this.room.remoteParticipants.values()) { + let packets = 0 + let samples = 0 + for (const publication of participant.trackPublications.values()) { + const receiver = (publication as RemoteTrackPublication).track?.receiver + if (!receiver) continue + const stats = await receiver.getStats() + stats.forEach((report) => { + if (report.type !== 'inbound-rtp') return + packets += (report as { packetsReceived?: number }).packetsReceived ?? 0 + samples += (report as { totalSamplesReceived?: number }).totalSamplesReceived ?? 0 + }) + } + out.push({ identity: participant.identity, packets, samples }) + } + return out + } + /** Leaves, letting go of the camera and the microphone. */ async leave(): Promise { await this.room.disconnect() diff --git a/src/renderer/src/lib/matrixcall.ts b/src/renderer/src/lib/matrixcall.ts index b810ba4..5a219bc 100644 --- a/src/renderer/src/lib/matrixcall.ts +++ b/src/renderer/src/lib/matrixcall.ts @@ -89,6 +89,8 @@ export interface CallSurface { encrypted?: boolean } | null> leaveMembership: (bufferId: string) => Promise + /** Hands this end's media key to the devices in the call. */ + sendCallKey: (bufferId: string, key: string, index: number) => Promise /** Who the room says is in the call. */ callMembers: (bufferId: string) => Promise onRinging: (call: RingingCall | null) => void @@ -154,6 +156,15 @@ export interface GroupCall { sfu: SfuCall | null /** Who the media server says is here, for the SFU kind. */ sfuPeople: SfuParticipant[] + /** This end's own media key and where it is in the cycle. */ + ownKeyBase64: string + keyIndex: number + ownIdentity: string + /** Who was in the call last time it was looked at, so an arrival and a + * departure can be told apart - they are answered differently. */ + knownMembers: Set + /** Whose media this end can actually read, by media-server identity. */ + haveKeyFrom: Set /** The whole call's id, on every signal, so a second call in the same room * is a second call rather than a crossed line. */ confId: string @@ -258,13 +269,18 @@ export class MatrixCalls { // A call held on a media server, which is what Element's are. Nobody // connects to anybody directly, so none of the mesh below runs. if (own.media) { - const sfu = new SfuCall({ - onParticipants: (people) => { - if (this.group) this.group.sfuPeople = people + const sfu = new SfuCall( + { + onParticipants: (people) => { + if (this.group) this.group.sfuPeople = people + }, + onConnected: (connected) => this.surface.onPhase(connected ? 'connected' : 'connecting', null), + onError: (message) => this.surface.onError(message) }, - onConnected: (connected) => this.surface.onPhase(connected ? 'connected' : 'connecting', null), - onError: (message) => this.surface.onError(message) - }) + // A call in an encrypted room is encrypted; one in an open room is + // not, and pretending otherwise would leave everybody silent. + !!own.encrypted + ) this.group = { accountId, bufferId, @@ -274,21 +290,23 @@ export class MatrixCalls { peers: new Map(), sfu, sfuPeople: [], + ownKeyBase64: '', + // Starts below zero so the first roll is index 0, which is where + // every other client starts too. + keyIndex: -1, + ownIdentity: own.media.identity, + knownMembers: new Set(), + haveKeyFrom: new Set(), confId: `moho-conf-${own.roomId ?? bufferId}` } this.heartbeat = setInterval(() => void this.surface.joinMembership(bufferId), 30_000) this.surface.onPhase('connecting', null) try { + // The key first, so the first frame that goes out is already + // encrypted with something the others will be told about. + if (sfu.encrypted) await this.rollKey(own.media.identity) await sfu.join(own.media.url, own.media.jwt, video) this.group.local = sfu.local - if (own.encrypted) { - // Element encrypts a call's media per sender in an encrypted room, - // with the keys shared over Matrix. moho does not do that yet, so - // this is a call everybody is in and nobody can hear. - this.surface.onError( - 'This room is encrypted, and moho cannot yet read the media in an encrypted call - the others will not hear you' - ) - } } catch (e) { this.surface.onError(`Couldn't reach the media server: ${(e as Error).message}`) this.leaveGroup() @@ -324,6 +342,11 @@ export class MatrixCalls { peers: new Map(), sfu: null, sfuPeople: [], + ownKeyBase64: '', + keyIndex: -1, + ownIdentity: '', + knownMembers: new Set(), + haveKeyFrom: new Set(), // Named for the room rather than for the buffer: each account calls the // same room by a different buffer id, and a conference named after one // of those is a conference the other end never joins. @@ -337,6 +360,60 @@ export class MatrixCalls { await this.meet(await this.surface.callMembers(bufferId)) } + /** + * Makes a new key for this end, uses it, and hands it to everybody else. + * + * Sixteen random bytes and an index that cycles, which is what Element + * makes: a key provider that generates differently is one whose media + * nobody else can read. + * + * Rolled again whenever somebody leaves, so that what is said after they go + * cannot be read by them - the whole reason a call's key is per session + * rather than per room. + */ + private async rollKey(identity: string): Promise { + const group = this.group + if (!group?.sfu) return + const raw = new Uint8Array(16) + crypto.getRandomValues(raw) + const key = btoa(String.fromCharCode(...raw)).replace(/=+$/, '') + group.keyIndex = (group.keyIndex + 1) % 256 + group.ownIdentity = identity + await group.sfu.useOwnKey(identity, group.keyIndex, key) + group.ownKeyBase64 = key + await this.surface.sendCallKey(group.bufferId, key, group.keyIndex) + } + + /** Somebody else's key, arriving over Matrix. */ + async takeKey(userId: string, deviceId: string, keyBase64: string, index: number): Promise { + const group = this.group + if (!group?.sfu) return + // The media server knows people by the identity its token service gave + // them, which is the user and the device with a colon between. + const identity = `${userId}:${deviceId}` + group.haveKeyFrom.add(identity) + await group.sfu.useTheirKey(identity, index, keyBase64) + } + + /** + * Names anybody whose media cannot be read, once they have had time to say. + * + * An encrypted call with a missing key does not fail: it connects, draws + * everybody's tile, and is silent - which is indistinguishable from a quiet + * room unless somebody says so. Called a little after each arrival, when a + * key that was coming would have come. + */ + private warnAboutSilentPeople(): void { + const group = this.group + if (!group?.sfu?.encrypted) return + const unreadable = group.sfuPeople.filter((p) => !group.haveKeyFrom.has(p.key)) + if (unreadable.length === 0) return + const who = unreadable.map((p) => p.userId).join(', ') + this.surface.onError( + `No key from ${who} - this call is encrypted and you will not hear them until their client sends one` + ) + } + /** * Brings the peer list into line with who the room says is in the call. * @@ -347,9 +424,29 @@ export class MatrixCalls { async meet(members: CallMember[]): Promise { const group = this.group if (!group) return - // On a media server there is nobody to meet: the server is what everybody - // is connected to, and it says who is there. - if (group.sfu) return + // On a media server there is nobody to meet - the server is what + // everybody is connected to - but for an encrypted call the keys still + // have to follow the roster: a new arrival needs the key in use, and + // somebody leaving needs a new one, or what is said after they go is + // still readable by them. + if (group.sfu) { + if (!group.sfu.encrypted) return + const now = new Set( + members.map((m) => `${m.user_id}:${m.membership?.device_id ?? ''}`) + ) + const left = [...group.knownMembers].some((who) => !now.has(who)) + const arrived = [...now].some((who) => !group.knownMembers.has(who)) + group.knownMembers = now + if (left) { + await this.rollKey(group.ownIdentity) + } else if (arrived && group.ownKeyBase64) { + await this.surface.sendCallKey(group.bufferId, group.ownKeyBase64, group.keyIndex) + // And if theirs never arrives, say so rather than leaving somebody + // wondering why the call is quiet. + setTimeout(() => this.warnAboutSilentPeople(), 12_000) + } + return + } const present = new Set() for (const member of members) { const key = memberKey(member.user_id, member.membership?.device_id ?? '') diff --git a/src/renderer/src/state/store.ts b/src/renderer/src/state/store.ts index 14e20a2..ec3fdcf 100644 --- a/src/renderer/src/state/store.ts +++ b/src/renderer/src/state/store.ts @@ -799,6 +799,14 @@ export class ChatStore { leaveMembership: async (bufferId) => { await window.moho.rpc('setMatrixCallMembership', { bufferId, joined: false }).catch(() => {}) }, + // The daemon holds the encryption, so it is what hands a media key to the + // devices in the call - one Olm-encrypted event each, the way Element + // does it. + sendCallKey: async (bufferId, key, index) => { + await window.moho.rpc('sendMatrixCallKey', { bufferId, key, index }).catch((e: Error) => { + this.toast('error', `Couldn't share the call key: ${e.message}`) + }) + }, callMembers: async (bufferId) => window.moho.rpc('listMatrixCallMembers', { bufferId }).catch(() => []), onRinging: (call) => this.set({ ringingCall: call }), @@ -1817,6 +1825,13 @@ export class ChatStore { this.set({ watchMinimized: minimized }) } + /** What is actually arriving in a call held on a media server, if one is + * up: a check that the media is readable rather than merely connected. */ + async callArrivals(): Promise { + const group = this.matrixCalls.currentGroup + return group?.sfu ? group.sfu.arriving() : [] + } + hangUpMatrixCall(): void { // Whichever kind is up: leaving a room's call withdraws the membership // that says this end is in it, as well as hanging up on everybody. @@ -2146,6 +2161,12 @@ export class ChatStore { // stopped - so a list replaces, and a single nick is merged in. // Somebody joined or left a room's call. Both the offer of a call to // join and, while in one, the signal to meet whoever just arrived. + // Somebody's media key, for a call whose frames the server cannot read. + case 'matrixCallKey': { + void this.matrixCalls.takeKey(data.userId, data.deviceId, data.key, data.index ?? 0) + break + } + case 'matrixCallMembers': { const members = (data.members ?? []) as CallMember[] this.set({ callMembers: { ...this.state.callMembers, [data.bufferId]: members } })