Encrypt the media, with the keys the room hands round

Sixteen random bytes and an index that cycles, which is what Element makes - a client that generates differently is one whose media nobody else can read. Handed to LiveKit as HKDF material rather than as a key, because that is what it derives its cipher key from and what Element gives it, and the ratchet window and keyring size are Element's too. Rolled when somebody leaves, so what is said after they go is not readable by them, and re-sent when somebody arrives, which is the difference between the two events. And a call that is connected but unreadable now says so: an encrypted call with a missing key looks exactly like a quiet room, so anybody whose key has not arrived a few seconds after they joined is named. Co-Authored-By: Claude Opus 5 <[email protected]>
2026-09-06 22:06:06 -04:00
parent 4e9f6ddbb2
commit c33bdc812b
4 changed files with 233 additions and 20 deletions
@@ -18,6 +18,7 @@
 */

 import {
+  BaseKeyProvider,
  ConnectionState,
  Room,
  RoomEvent,
@@ -25,6 +26,7 @@ import {
  type RemoteTrack,
  type RemoteTrackPublication
 } from 'livekit-client'
+import E2EEWorker from 'livekit-client/e2ee-worker?worker'

 /** One person on the media server, and what they are sending. */
 export interface SfuParticipant {
@@ -35,6 +37,38 @@ export interface SfuParticipant {
  hasVideo: boolean
 }

+/**
+ * The keys a call's media is encrypted with, as Matrix hands them around.
+ *
+ * The media server forwards frames it cannot read: each participant makes a
+ * key, sends it to the devices in the call over Matrix, and everybody's
+ * player is told which key belongs to which participant. This is the provider
+ * LiveKit asks for those keys, filled from the daemon's side of that
+ * exchange - see calls.rs.
+ *
+ * The window sizes are Element's own, because a key provider that ratchets
+ * differently from the other clients in a call is a key provider that decodes
+ * nothing.
+ */
+class MatrixKeyProvider extends BaseKeyProvider {
+  constructor() {
+    super({ ratchetWindowSize: 10, keyringSize: 256 })
+  }
+
+  /** Somebody's key, for the participant the media server knows them as. */
+  async setParticipantKey(identity: string, index: number, keyBase64: string): Promise<void> {
+    const raw = Uint8Array.from(atob(keyBase64.replace(/-/g, '+').replace(/_/g, '/')), (c) => c.charCodeAt(0))
+    // Imported as HKDF material rather than as a key: LiveKit derives the
+    // cipher key from it, and Element hands it the same material, which is
+    // what makes the two agree.
+    const material = await crypto.subtle.importKey('raw', raw as BufferSource, 'HKDF', false, [
+      'deriveBits',
+      'deriveKey'
+    ])
+    this.onSetEncryptionKey(material, identity, index)
+  }
+}
+
 export interface SfuHandlers {
  /** Somebody joined, left, or started sending something different. */
  onParticipants: (people: SfuParticipant[]) => void
@@ -59,16 +93,26 @@ export function userFromIdentity(identity: string): string {
 export class SfuCall {
  private room: Room
  private handlers: SfuHandlers
+  /** Where the keys go, for an encrypted call. Absent means one in the clear. */
+  private keys: MatrixKeyProvider | null = null
  /** This end's own capture, so the tile shows what is being sent. */
  local: MediaStream | null = null

-  constructor(handlers: SfuHandlers) {
+  constructor(handlers: SfuHandlers, encrypted: boolean) {
    this.handlers = handlers
+    this.keys = encrypted ? new MatrixKeyProvider() : null
    this.room = new Room({
      // Let LiveKit decide what to send as the window and the connection
      // change: it can see the packet loss and this cannot.
      adaptiveStream: true,
-      dynacast: true
+      dynacast: true,
+      // A call in an encrypted room is encrypted between the people in it,
+      // with the server forwarding frames it cannot read. The worker is where
+      // the frames are actually enciphered - the browser will only let a
+      // transform touch them off the main thread.
+      ...(this.keys
+        ? { e2ee: { keyProvider: this.keys as unknown as BaseKeyProvider, worker: new E2EEWorker() } }
+        : {})
    })
    const announce = (): void => this.handlers.onParticipants(this.people())
    this.room
@@ -80,6 +124,13 @@ export class SfuCall {
        this.handlers.onParticipants([])
        this.handlers.onConnected(false)
      })
+      // Frames arriving that cannot be read. In an encrypted call that means
+      // somebody's key never got here, or theirs and ours disagree - which
+      // looks exactly like a working call with silence in it, so it is worth
+      // saying rather than leaving people to wonder why nobody is talking.
+      .on(RoomEvent.EncryptionError, (error) =>
+        this.handlers.onError(`Cannot read what somebody in the call is sending: ${error.message}`)
+      )
      .on(RoomEvent.ConnectionStateChanged, (state) => {
        // Being on the server is what connected means here, whether or not
        // anybody else has arrived - unlike the mesh, where there is nothing
@@ -89,6 +140,23 @@ export class SfuCall {
      })
  }

+  /** Whether this call's media is encrypted end to end. */
+  get encrypted(): boolean {
+    return this.keys !== null
+  }
+
+  /** This end's own key, which everybody else has to be told about. */
+  async useOwnKey(identity: string, index: number, keyBase64: string): Promise<void> {
+    if (!this.keys) return
+    await this.keys.setParticipantKey(identity, index, keyBase64)
+    await this.room.setE2EEEnabled(true)
+  }
+
+  /** Somebody else's key, as it arrives over Matrix. */
+  async useTheirKey(identity: string, index: number, keyBase64: string): Promise<void> {
+    await this.keys?.setParticipantKey(identity, index, keyBase64)
+  }
+
  /** Joins the media server and starts sending. */
  async join(url: string, token: string, video: boolean): Promise<void> {
    await this.room.connect(url, token)
@@ -153,6 +221,33 @@ export class SfuCall {
    return !on
  }

+  /**
+   * What the connection says is actually arriving, per participant.
+   *
+   * Tracks existing is not the same as tracks being readable: an undecodable
+   * stream is still a stream, with silence in it. The receiver's own counters
+   * are what tell the two apart.
+   */
+  async arriving(): Promise<{ identity: string; packets: number; samples: number }[]> {
+    const out: { identity: string; packets: number; samples: number }[] = []
+    for (const participant of this.room.remoteParticipants.values()) {
+      let packets = 0
+      let samples = 0
+      for (const publication of participant.trackPublications.values()) {
+        const receiver = (publication as RemoteTrackPublication).track?.receiver
+        if (!receiver) continue
+        const stats = await receiver.getStats()
+        stats.forEach((report) => {
+          if (report.type !== 'inbound-rtp') return
+          packets += (report as { packetsReceived?: number }).packetsReceived ?? 0
+          samples += (report as { totalSamplesReceived?: number }).totalSamplesReceived ?? 0
+        })
+      }
+      out.push({ identity: participant.identity, packets, samples })
+    }
+    return out
+  }
+
  /** Leaves, letting go of the camera and the microphone. */
  async leave(): Promise<void> {
    await this.room.disconnect()
@@ -89,6 +89,8 @@ export interface CallSurface {
    encrypted?: boolean
  } | null>
  leaveMembership: (bufferId: string) => Promise<void>
+  /** Hands this end's media key to the devices in the call. */
+  sendCallKey: (bufferId: string, key: string, index: number) => Promise<void>
  /** Who the room says is in the call. */
  callMembers: (bufferId: string) => Promise<CallMember[]>
  onRinging: (call: RingingCall | null) => void
@@ -154,6 +156,15 @@ export interface GroupCall {
  sfu: SfuCall | null
  /** Who the media server says is here, for the SFU kind. */
  sfuPeople: SfuParticipant[]
+  /** This end's own media key and where it is in the cycle. */
+  ownKeyBase64: string
+  keyIndex: number
+  ownIdentity: string
+  /** Who was in the call last time it was looked at, so an arrival and a
+   *  departure can be told apart - they are answered differently. */
+  knownMembers: Set<string>
+  /** Whose media this end can actually read, by media-server identity. */
+  haveKeyFrom: Set<string>
  /** The whole call's id, on every signal, so a second call in the same room
   *  is a second call rather than a crossed line. */
  confId: string
@@ -258,13 +269,18 @@ export class MatrixCalls {
    // A call held on a media server, which is what Element's are. Nobody
    // connects to anybody directly, so none of the mesh below runs.
    if (own.media) {
-      const sfu = new SfuCall({
-        onParticipants: (people) => {
-          if (this.group) this.group.sfuPeople = people
+      const sfu = new SfuCall(
+        {
+          onParticipants: (people) => {
+            if (this.group) this.group.sfuPeople = people
+          },
+          onConnected: (connected) => this.surface.onPhase(connected ? 'connected' : 'connecting', null),
+          onError: (message) => this.surface.onError(message)
        },
-        onConnected: (connected) => this.surface.onPhase(connected ? 'connected' : 'connecting', null),
-        onError: (message) => this.surface.onError(message)
-      })
+        // A call in an encrypted room is encrypted; one in an open room is
+        // not, and pretending otherwise would leave everybody silent.
+        !!own.encrypted
+      )
      this.group = {
        accountId,
        bufferId,
@@ -274,21 +290,23 @@ export class MatrixCalls {
        peers: new Map(),
        sfu,
        sfuPeople: [],
+        ownKeyBase64: '',
+        // Starts below zero so the first roll is index 0, which is where
+        // every other client starts too.
+        keyIndex: -1,
+        ownIdentity: own.media.identity,
+        knownMembers: new Set(),
+        haveKeyFrom: new Set(),
        confId: `moho-conf-${own.roomId ?? bufferId}`
      }
      this.heartbeat = setInterval(() => void this.surface.joinMembership(bufferId), 30_000)
      this.surface.onPhase('connecting', null)
      try {
+        // The key first, so the first frame that goes out is already
+        // encrypted with something the others will be told about.
+        if (sfu.encrypted) await this.rollKey(own.media.identity)
        await sfu.join(own.media.url, own.media.jwt, video)
        this.group.local = sfu.local
-        if (own.encrypted) {
-          // Element encrypts a call's media per sender in an encrypted room,
-          // with the keys shared over Matrix. moho does not do that yet, so
-          // this is a call everybody is in and nobody can hear.
-          this.surface.onError(
-            'This room is encrypted, and moho cannot yet read the media in an encrypted call - the others will not hear you'
-          )
-        }
      } catch (e) {
        this.surface.onError(`Couldn't reach the media server: ${(e as Error).message}`)
        this.leaveGroup()
@@ -324,6 +342,11 @@ export class MatrixCalls {
      peers: new Map(),
      sfu: null,
      sfuPeople: [],
+      ownKeyBase64: '',
+      keyIndex: -1,
+      ownIdentity: '',
+      knownMembers: new Set(),
+      haveKeyFrom: new Set(),
      // Named for the room rather than for the buffer: each account calls the
      // same room by a different buffer id, and a conference named after one
      // of those is a conference the other end never joins.
@@ -337,6 +360,60 @@ export class MatrixCalls {
    await this.meet(await this.surface.callMembers(bufferId))
  }

+  /**
+   * Makes a new key for this end, uses it, and hands it to everybody else.
+   *
+   * Sixteen random bytes and an index that cycles, which is what Element
+   * makes: a key provider that generates differently is one whose media
+   * nobody else can read.
+   *
+   * Rolled again whenever somebody leaves, so that what is said after they go
+   * cannot be read by them - the whole reason a call's key is per session
+   * rather than per room.
+   */
+  private async rollKey(identity: string): Promise<void> {
+    const group = this.group
+    if (!group?.sfu) return
+    const raw = new Uint8Array(16)
+    crypto.getRandomValues(raw)
+    const key = btoa(String.fromCharCode(...raw)).replace(/=+$/, '')
+    group.keyIndex = (group.keyIndex + 1) % 256
+    group.ownIdentity = identity
+    await group.sfu.useOwnKey(identity, group.keyIndex, key)
+    group.ownKeyBase64 = key
+    await this.surface.sendCallKey(group.bufferId, key, group.keyIndex)
+  }
+
+  /** Somebody else's key, arriving over Matrix. */
+  async takeKey(userId: string, deviceId: string, keyBase64: string, index: number): Promise<void> {
+    const group = this.group
+    if (!group?.sfu) return
+    // The media server knows people by the identity its token service gave
+    // them, which is the user and the device with a colon between.
+    const identity = `${userId}:${deviceId}`
+    group.haveKeyFrom.add(identity)
+    await group.sfu.useTheirKey(identity, index, keyBase64)
+  }
+
+  /**
+   * Names anybody whose media cannot be read, once they have had time to say.
+   *
+   * An encrypted call with a missing key does not fail: it connects, draws
+   * everybody's tile, and is silent - which is indistinguishable from a quiet
+   * room unless somebody says so. Called a little after each arrival, when a
+   * key that was coming would have come.
+   */
+  private warnAboutSilentPeople(): void {
+    const group = this.group
+    if (!group?.sfu?.encrypted) return
+    const unreadable = group.sfuPeople.filter((p) => !group.haveKeyFrom.has(p.key))
+    if (unreadable.length === 0) return
+    const who = unreadable.map((p) => p.userId).join(', ')
+    this.surface.onError(
+      `No key from ${who} - this call is encrypted and you will not hear them until their client sends one`
+    )
+  }
+
  /**
   * Brings the peer list into line with who the room says is in the call.
   *
@@ -347,9 +424,29 @@ export class MatrixCalls {
  async meet(members: CallMember[]): Promise<void> {
    const group = this.group
    if (!group) return
-    // On a media server there is nobody to meet: the server is what everybody
-    // is connected to, and it says who is there.
-    if (group.sfu) return
+    // On a media server there is nobody to meet - the server is what
+    // everybody is connected to - but for an encrypted call the keys still
+    // have to follow the roster: a new arrival needs the key in use, and
+    // somebody leaving needs a new one, or what is said after they go is
+    // still readable by them.
+    if (group.sfu) {
+      if (!group.sfu.encrypted) return
+      const now = new Set(
+        members.map((m) => `${m.user_id}:${m.membership?.device_id ?? ''}`)
+      )
+      const left = [...group.knownMembers].some((who) => !now.has(who))
+      const arrived = [...now].some((who) => !group.knownMembers.has(who))
+      group.knownMembers = now
+      if (left) {
+        await this.rollKey(group.ownIdentity)
+      } else if (arrived && group.ownKeyBase64) {
+        await this.surface.sendCallKey(group.bufferId, group.ownKeyBase64, group.keyIndex)
+        // And if theirs never arrives, say so rather than leaving somebody
+        // wondering why the call is quiet.
+        setTimeout(() => this.warnAboutSilentPeople(), 12_000)
+      }
+      return
+    }
    const present = new Set<string>()
    for (const member of members) {
      const key = memberKey(member.user_id, member.membership?.device_id ?? '')
@@ -799,6 +799,14 @@ export class ChatStore {
    leaveMembership: async (bufferId) => {
      await window.moho.rpc('setMatrixCallMembership', { bufferId, joined: false }).catch(() => {})
    },
+    // The daemon holds the encryption, so it is what hands a media key to the
+    // devices in the call - one Olm-encrypted event each, the way Element
+    // does it.
+    sendCallKey: async (bufferId, key, index) => {
+      await window.moho.rpc('sendMatrixCallKey', { bufferId, key, index }).catch((e: Error) => {
+        this.toast('error', `Couldn't share the call key: ${e.message}`)
+      })
+    },
    callMembers: async (bufferId) =>
      window.moho.rpc<CallMember[]>('listMatrixCallMembers', { bufferId }).catch(() => []),
    onRinging: (call) => this.set({ ringingCall: call }),
@@ -1817,6 +1825,13 @@ export class ChatStore {
    this.set({ watchMinimized: minimized })
  }

+  /** What is actually arriving in a call held on a media server, if one is
+   *  up: a check that the media is readable rather than merely connected. */
+  async callArrivals(): Promise<unknown> {
+    const group = this.matrixCalls.currentGroup
+    return group?.sfu ? group.sfu.arriving() : []
+  }
+
  hangUpMatrixCall(): void {
    // Whichever kind is up: leaving a room's call withdraws the membership
    // that says this end is in it, as well as hanging up on everybody.
@@ -2146,6 +2161,12 @@ export class ChatStore {
      // stopped - so a list replaces, and a single nick is merged in.
      // Somebody joined or left a room's call. Both the offer of a call to
      // join and, while in one, the signal to meet whoever just arrived.
+      // Somebody's media key, for a call whose frames the server cannot read.
+      case 'matrixCallKey': {
+        void this.matrixCalls.takeKey(data.userId, data.deviceId, data.key, data.index ?? 0)
+        break
+      }
+
      case 'matrixCallMembers': {
        const members = (data.members ?? []) as CallMember[]
        this.set({ callMembers: { ...this.state.callMembers, [data.bufferId]: members } })