A call with everybody in it, and a way to look at one
A call between two people needed two tiles. A call between five needs a grid, and the grid has to be worked out rather than tabulated: fitTiles tries every column count and keeps the one that makes the faces largest in the space there actually is, so the same five people fill a wide window differently from a narrow one and neither leaves slivers. Sixteen by nine, because that is what cameras are and what a face fits in. Capped, because two people on a large screen do not want two enormous windows and the conversation underneath should survive. Who is talking is measured here rather than asked for: the audio is in this window, so an AnalyserNode reads it and lights the tile in the same green the Discord voice view uses - a ring rather than a reordering, since a grid that rearranged itself every time somebody spoke would be unreadable. Pressing a face makes that person large with everybody else in a strip underneath, and pressing them again puts everybody back. It is a decision the person watching made, so nothing in the call undoes it - only their leaving, which takes the reason with it. Walking away from the conversation leaves the call in the corner, with a title bar naming who is in it, a way back to the room it is held in, and a minimise that puts the picture away without hanging up. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
Claude Opus 5
@@ -37,10 +37,17 @@ export default function App(): JSX.Element {
|
||||
const accounts = useChat((s) => s.accounts)
|
||||
const activePanel = useChat((s) => s.activePanel)
|
||||
const activeBufferId = useChat((s) => s.activeBufferId)
|
||||
// A call whose conversation is not the one on screen. The stage lives in
|
||||
// the conversation; this is the corner it retreats to.
|
||||
const activeCall = useChat((s) => s.activeCall)
|
||||
const activeGroupId = useChat((s) => s.activeGroupId)
|
||||
const joinPanelAccountId = useChat((s) => s.joinPanelAccountId)
|
||||
const buffer = useActiveBuffer()
|
||||
const joinAccount = accounts.find((a) => a.id === joinPanelAccountId)
|
||||
// Shown in the corner only when the conversation it belongs to is not the
|
||||
// one being read - and not while a panel is covering the log either, since
|
||||
// that is walking away from it too.
|
||||
const callElsewhere = !!activeCall && (activeCall.bufferId !== activeBufferId || activePanel !== '')
|
||||
|
||||
const [sidebarFolded, setSidebarFolded] = usePref<boolean>('ui.sidebarFolded', false)
|
||||
const [userListFolded, setUserListFolded] = usePref<boolean>('ui.userListFolded', false)
|
||||
@@ -208,11 +215,11 @@ export default function App(): JSX.Element {
|
||||
than by the daemon - the media is the window's. */}
|
||||
<IncomingMatrixCall />
|
||||
<ScreenPicker />
|
||||
{/* The call this window is holding, wherever the window has wandered
|
||||
to. It does not live under the conversation: walking away from a
|
||||
conversation does not hang up, and a call that vanished when you
|
||||
looked at something else would be a call you could not hang up. */}
|
||||
<CallStage />
|
||||
{/* Where a call goes when you walk away from it: a corner of its own,
|
||||
the way a phone keeps the picture in the corner. In the conversation
|
||||
it belongs to it sits above the log instead - see MessageList - so
|
||||
the two are never both on screen. */}
|
||||
{callElsewhere && <CallStage mode="pip" />}
|
||||
{/* Over everything for the same reason: a file is offered while you are
|
||||
reading something else, and often in a conversation you are not. */}
|
||||
<TransferPanel />
|
||||
|
||||
@@ -40,6 +40,7 @@ export function Popout({ bufferId }: { bufferId: string }): JSX.Element {
|
||||
const [booted, setBooted] = useState(false)
|
||||
const buffer = useActiveBuffer()
|
||||
const linkUp = useChat((s) => s.linkUp)
|
||||
const activeCall = useChat((s) => s.activeCall)
|
||||
// Folded by default: a popout is a narrow window opened to read one
|
||||
// conversation, and a member list would take a third of it before anything
|
||||
// has been read.
|
||||
@@ -135,7 +136,9 @@ export function Popout({ bufferId }: { bufferId: string }): JSX.Element {
|
||||
answered here is answered here. */}
|
||||
<IncomingMatrixCall />
|
||||
<ScreenPicker />
|
||||
<CallStage />
|
||||
{/* A popped-out conversation is one conversation, so a call that is not
|
||||
this one is in the corner and this one is above the log. */}
|
||||
{activeCall && activeCall.bufferId !== bufferId && <CallStage mode="pip" />}
|
||||
<Toasts />
|
||||
</div>
|
||||
)
|
||||
|
||||
+155
-38
@@ -852,67 +852,180 @@
|
||||
* Sized so it can be looked at without swallowing the log: a call is watched
|
||||
* and the conversation is read, and the two happen at once.
|
||||
*/
|
||||
/*
|
||||
* In the conversation it belongs to, above the log - the same place the voice
|
||||
* call view sits, because a call and the conversation it is in are the same
|
||||
* conversation and people type into a channel while talking in it.
|
||||
*/
|
||||
.call-stage {
|
||||
/* Over everything, because a call is not part of the conversation it
|
||||
started in: it follows the window rather than the buffer, and has to be
|
||||
hangable-up from wherever somebody has wandered to. */
|
||||
position: absolute;
|
||||
right: var(--spacing-m);
|
||||
bottom: 90px;
|
||||
z-index: 40;
|
||||
width: 380px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
flex: none;
|
||||
gap: var(--spacing-xs);
|
||||
margin: var(--spacing-xs) var(--spacing-m) 0;
|
||||
padding: var(--spacing-xs);
|
||||
border: 1px solid var(--outline-medium);
|
||||
border-radius: var(--corner-radius);
|
||||
background: var(--surface-container);
|
||||
}
|
||||
|
||||
.call-stage-video {
|
||||
/*
|
||||
* And in a corner of its own once you walk away from that conversation, the
|
||||
* way a phone keeps the picture in the corner rather than ending the call.
|
||||
*/
|
||||
.call-stage.pip {
|
||||
position: absolute;
|
||||
right: var(--spacing-m);
|
||||
bottom: 90px;
|
||||
z-index: 40;
|
||||
width: 340px;
|
||||
margin: 0;
|
||||
box-shadow: 0 8px 28px rgba(0, 0, 0, 0.45);
|
||||
}
|
||||
|
||||
/*
|
||||
* The corner view's own title bar: who is in the call, and the control that
|
||||
* puts it away.
|
||||
*
|
||||
* Only in the corner, where the panel is a window in all but name and wants a
|
||||
* window's furniture. In the conversation the tiles carry the names
|
||||
* themselves and a second list would be the same thing twice.
|
||||
*/
|
||||
.call-stage-title {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--spacing-xs);
|
||||
padding: 0 2px 2px;
|
||||
color: var(--surface-variant-text);
|
||||
}
|
||||
|
||||
.call-stage-title .small {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
/* Put away: one line that says a call is still running, and offers it back. */
|
||||
.call-stage.minimized {
|
||||
padding: 2px var(--spacing-xs);
|
||||
}
|
||||
|
||||
.call-stage.minimized.pip {
|
||||
width: auto;
|
||||
max-width: 320px;
|
||||
}
|
||||
|
||||
/*
|
||||
* One tile per person, in as square a grid as the number allows.
|
||||
*
|
||||
* The grid grows downwards rather than squeezing: five people in a row would
|
||||
* be five slivers, and the whole point of a tile is that a face is legible in
|
||||
* it. Bounded in height so a call never takes the conversation over.
|
||||
*/
|
||||
.video-grid {
|
||||
display: grid;
|
||||
gap: 6px;
|
||||
align-content: start;
|
||||
/* Centred, because the tiles are sized to fit rather than stretched: a row
|
||||
of three in a wide window leaves room at the sides, and that room belongs
|
||||
equally on both. */
|
||||
justify-content: center;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
/*
|
||||
* Landscape, like every camera and every other client: a face fits in a
|
||||
* sixteen-by-nine tile the way it does not fit in a tall one, and the whole
|
||||
* row reads as one thing.
|
||||
*/
|
||||
.video-tile {
|
||||
position: relative;
|
||||
height: 214px;
|
||||
width: 100%;
|
||||
aspect-ratio: 16 / 9;
|
||||
min-width: 0;
|
||||
/* It is a button now - pressing a face is how somebody is singled out - so
|
||||
the button's own chrome has to go before it can look like a picture. */
|
||||
padding: 0;
|
||||
border: none;
|
||||
border-radius: var(--corner-radius-small);
|
||||
overflow: hidden;
|
||||
background: #000;
|
||||
cursor: pointer;
|
||||
background: var(--surface-container-highest);
|
||||
}
|
||||
|
||||
.call-remote {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: contain;
|
||||
}
|
||||
|
||||
/* Nothing to look at yet, so the element is not what is on screen. */
|
||||
.call-remote.audio-only {
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
.call-stage-placeholder {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
/*
|
||||
* One person, large, with everybody else in a row underneath.
|
||||
*
|
||||
* What a grid cannot say: that this is the person being watched. Discord and
|
||||
* every other client draw it the same way, and the strip is what keeps it a
|
||||
* call rather than a broadcast - the others are still there, still lit when
|
||||
* they talk, and still one press away from being the large one.
|
||||
*/
|
||||
.video-stage {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: var(--spacing-s);
|
||||
gap: 6px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
/* Yourself, small and out of the way: it is there to check you are in frame,
|
||||
not to be watched. */
|
||||
.call-local {
|
||||
position: absolute;
|
||||
right: var(--spacing-s);
|
||||
bottom: var(--spacing-s);
|
||||
width: 140px;
|
||||
max-height: 100px;
|
||||
border-radius: var(--corner-radius-small);
|
||||
border: 1px solid var(--outline-medium);
|
||||
background: #000;
|
||||
.video-strip {
|
||||
display: flex;
|
||||
gap: 6px;
|
||||
justify-content: center;
|
||||
max-width: 100%;
|
||||
/* More people than fit: scrolled rather than shrunk to nothing, since a
|
||||
thumbnail below a certain size stops being a person. */
|
||||
overflow-x: auto;
|
||||
overflow-y: hidden;
|
||||
}
|
||||
|
||||
.video-strip .video-tile {
|
||||
flex: none;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
/* Who is talking, in the same green the voice call view uses - said with a
|
||||
ring rather than by moving anybody, since a grid that reordered itself
|
||||
every time somebody spoke would be unreadable. */
|
||||
.video-tile.speaking {
|
||||
outline: 3px solid var(--success);
|
||||
outline-offset: -3px;
|
||||
}
|
||||
|
||||
.video-tile-picture {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
/* Nothing to look at: the name and a face, which is what a call without
|
||||
cameras actually is. */
|
||||
.video-tile-picture.audio-only {
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
.video-tile-face {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.video-tile-name {
|
||||
position: absolute;
|
||||
left: 4px;
|
||||
bottom: 4px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 3px;
|
||||
max-width: calc(100% - 8px);
|
||||
padding: 1px 6px;
|
||||
border-radius: var(--corner-radius-small);
|
||||
background: rgba(0, 0, 0, 0.55);
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.call-stage-controls {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
@@ -924,6 +1037,10 @@
|
||||
margin-right: auto;
|
||||
}
|
||||
|
||||
/* Smaller in the corner than in the conversation, and shorter, so a call
|
||||
somebody has walked away from stays out of the way. */
|
||||
|
||||
|
||||
/*
|
||||
* Which window to show a room.
|
||||
*
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
import { Icon, IconButton } from './Icon'
|
||||
import { Avatar } from './Avatar'
|
||||
import { useChat, useStore } from '../state/hooks'
|
||||
import type { CallTile } from '../state/store'
|
||||
import { bufferDisplayName, classes } from '../lib/util'
|
||||
|
||||
/**
|
||||
@@ -19,32 +21,106 @@ import { bufferDisplayName, classes } from '../lib/util'
|
||||
* which is what a call is on any service - the screen-share button here is the
|
||||
* same button Discord's will be, and the tiles are the same tiles.
|
||||
*/
|
||||
export function CallStage(): JSX.Element | null {
|
||||
/** Sixteen by nine, which is what cameras and screens both are. */
|
||||
const TILE_RATIO = 16 / 9
|
||||
/** No tile gets bigger than this, however much room there is. */
|
||||
const TILE_MAX = 420
|
||||
const TILE_GAP = 6
|
||||
|
||||
/**
|
||||
* How to lay out a call of this size in the space there is.
|
||||
*
|
||||
* Every column count is tried and the one that makes the tiles biggest wins,
|
||||
* subject to them all fitting - which is what a grid of faces should do and
|
||||
* what a fixed table of counts cannot: three people in a wide short box want
|
||||
* one row, and the same three in a tall narrow one want three.
|
||||
*
|
||||
* Capped, because a call between two people on a large screen should not be
|
||||
* two enormous windows - past a certain size a face stops being easier to
|
||||
* read and the conversation underneath just disappears.
|
||||
*/
|
||||
export function fitTiles(
|
||||
count: number,
|
||||
width: number,
|
||||
height: number,
|
||||
max: number = TILE_MAX
|
||||
): { columns: number; tileWidth: number } {
|
||||
if (count < 1 || width < 1) return { columns: 1, tileWidth: 0 }
|
||||
let best = { columns: 1, tileWidth: 0 }
|
||||
for (let columns = 1; columns <= count; columns++) {
|
||||
const rows = Math.ceil(count / columns)
|
||||
// What the width allows, and what the height allows; the smaller governs.
|
||||
const byWidth = (width - TILE_GAP * (columns - 1)) / columns
|
||||
const byHeight = ((height - TILE_GAP * (rows - 1)) / rows) * TILE_RATIO
|
||||
const tileWidth = Math.min(byWidth, byHeight, max)
|
||||
if (tileWidth > best.tileWidth) best = { columns, tileWidth }
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
export function CallStage({ mode = 'inline' }: { mode?: 'inline' | 'pip' }): JSX.Element | null {
|
||||
const store = useStore()
|
||||
const call = useChat((s) => s.activeCall)
|
||||
const minimized = useChat((s) => s.callMinimized)
|
||||
const buffers = useChat((s) => s.buffers)
|
||||
const localVideo = useRef<HTMLVideoElement>(null)
|
||||
const remoteVideo = useRef<HTMLVideoElement>(null)
|
||||
const [hasRemoteVideo, setHasRemoteVideo] = useState(false)
|
||||
const videos = useRef(new Map<string, HTMLVideoElement>())
|
||||
const grid = useRef<HTMLDivElement>(null)
|
||||
const [tiles, setTiles] = useState<CallTile[]>([])
|
||||
/** How wide the grid is, which decides how the tiles are laid out. */
|
||||
const [room, setRoom] = useState({ width: 640, height: 260 })
|
||||
/**
|
||||
* Whose picture is being looked at, if anybody.
|
||||
*
|
||||
* Null is the grid, where nobody is more important than anybody else. Naming
|
||||
* one person is a decision the person watching made, so it is undone the
|
||||
* same way it was made rather than by anything the call does.
|
||||
*/
|
||||
const [focused, setFocused] = useState<string | null>(null)
|
||||
|
||||
// The streams are not state: they are live objects that change tracks under
|
||||
// a call, so the elements are pointed at them rather than re-rendered from
|
||||
// them.
|
||||
// Measured rather than assumed: the same call is a different shape in a
|
||||
// narrow window, in a full-screen one, and in the corner it retreats to.
|
||||
useEffect(() => {
|
||||
const el = grid.current
|
||||
if (!el) return
|
||||
const measure = (): void =>
|
||||
setRoom({
|
||||
width: el.clientWidth,
|
||||
// As much height as a call may take before it is taking over the
|
||||
// conversation rather than sitting above it.
|
||||
height: Math.min(window.innerHeight * (mode === 'pip' ? 0.3 : 0.45), mode === 'pip' ? 260 : 460)
|
||||
})
|
||||
measure()
|
||||
const observer = new ResizeObserver(measure)
|
||||
observer.observe(el)
|
||||
window.addEventListener('resize', measure)
|
||||
return () => {
|
||||
observer.disconnect()
|
||||
window.removeEventListener('resize', measure)
|
||||
}
|
||||
}, [mode, minimized])
|
||||
|
||||
// The streams are not state: they are live objects whose tracks change under
|
||||
// a call - somebody turning a camera on, somebody starting to share - so the
|
||||
// elements are pointed at them rather than re-rendered from them.
|
||||
useEffect(() => {
|
||||
if (!call) return
|
||||
const attach = (): void => {
|
||||
const { local, remote } = store.callStreams()
|
||||
if (localVideo.current && local && localVideo.current.srcObject !== local) {
|
||||
localVideo.current.srcObject = local
|
||||
const current = store.callTiles()
|
||||
setTiles((was) =>
|
||||
was.length === current.length && was.every((t, i) => t.id === current[i].id && t.hasVideo === current[i].hasVideo)
|
||||
? was
|
||||
: current
|
||||
)
|
||||
// Both slots: in the focused layout the same person is on the stage and
|
||||
// in the strip underneath, which is two elements showing one stream.
|
||||
for (const tile of current) {
|
||||
for (const slot of ['stage', 'strip']) {
|
||||
const el = videos.current.get(`${tile.id}#${slot}`)
|
||||
if (el && tile.stream && el.srcObject !== tile.stream) el.srcObject = tile.stream
|
||||
}
|
||||
}
|
||||
if (remoteVideo.current && remote && remoteVideo.current.srcObject !== remote) {
|
||||
remoteVideo.current.srcObject = remote
|
||||
}
|
||||
setHasRemoteVideo(!!remote?.getVideoTracks().some((t) => t.readyState === 'live'))
|
||||
}
|
||||
attach()
|
||||
// Tracks arrive after the call connects, and again when somebody starts
|
||||
// sharing a screen - so this keeps looking rather than attaching once.
|
||||
const timer = setInterval(attach, 700)
|
||||
return () => clearInterval(timer)
|
||||
}, [call, store])
|
||||
@@ -52,23 +128,155 @@ export function CallStage(): JSX.Element | null {
|
||||
if (!call) return null
|
||||
|
||||
const where = buffers.find((b) => b.id === call.bufferId)?.name ?? ''
|
||||
// Somebody who has left the call cannot go on being the one looked at.
|
||||
const stage = tiles.find((t) => t.id === focused) ?? null
|
||||
const layout = fitTiles(Math.max(tiles.length, 1), room.width, room.height)
|
||||
/** The row of everybody else under the stage, and how tall it is. */
|
||||
const stripHeight = mode === 'pip' ? 42 : 74
|
||||
const stageFit = fitTiles(
|
||||
1,
|
||||
room.width,
|
||||
room.height - (tiles.length > 1 ? stripHeight + TILE_GAP : 0),
|
||||
// Uncapped: the whole point of picking somebody out is to see them larger
|
||||
// than the grid would ever draw them.
|
||||
Infinity
|
||||
)
|
||||
const phase =
|
||||
call.phase === 'ringing' ? 'Ringing…' : call.phase === 'connecting' ? 'Connecting…' : 'Connected'
|
||||
|
||||
return (
|
||||
<div className="call-stage">
|
||||
<div className="call-stage-video">
|
||||
{/* Muted, because this is our own microphone coming back: playing it
|
||||
would be the oldest mistake in video calling. */}
|
||||
<video ref={remoteVideo} className={classes('call-remote', !hasRemoteVideo && 'audio-only')} autoPlay playsInline />
|
||||
{!hasRemoteVideo && (
|
||||
<div className="call-stage-placeholder muted">
|
||||
<Icon name="call" size={28} />
|
||||
<span>{phase}</span>
|
||||
</div>
|
||||
)}
|
||||
<video ref={localVideo} className="call-local" autoPlay playsInline muted />
|
||||
// Put away without being hung up. A call somebody is listening to while
|
||||
// reading something else does not need a picture on screen, and closing the
|
||||
// picture must not close the call - so this is a bar with a way back.
|
||||
if (minimized) {
|
||||
return (
|
||||
<div className={classes('call-stage', 'minimized', mode === 'pip' && 'pip')}>
|
||||
<div className="call-stage-controls">
|
||||
<Icon name="call" size={16} />
|
||||
<span className="small muted ellipsis">
|
||||
{phase}
|
||||
{where ? ` · ${bufferDisplayName(where)}` : ''}
|
||||
</span>
|
||||
<IconButton name="expand_less" title="Show the call" onClick={() => store.setCallMinimized(false)} />
|
||||
<IconButton name="call_end" title="Hang up" className="calling" onClick={() => store.hangUpMatrixCall()} />
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// Who is in it, named. Only in the corner: in the conversation the tiles are
|
||||
// large enough to read the names off, and a second list of them would be the
|
||||
// same information twice.
|
||||
const everybody = tiles.map((t) => t.label).join(', ')
|
||||
|
||||
/**
|
||||
* One person's tile, wherever it is being drawn.
|
||||
*
|
||||
* A button rather than a div with a click on it: it is a control now, so it
|
||||
* is reached by keyboard and says what it does to anything reading the
|
||||
* window out loud.
|
||||
*/
|
||||
const drawTile = (
|
||||
tile: CallTile,
|
||||
slot: 'stage' | 'strip',
|
||||
opts: { width?: number; onClick: () => void }
|
||||
): JSX.Element => (
|
||||
<button
|
||||
key={`${tile.id}#${slot}`}
|
||||
type="button"
|
||||
className={classes('video-tile', tile.speaking && 'speaking', slot === 'strip' && 'small-tile')}
|
||||
style={opts.width ? { width: `${opts.width}px` } : undefined}
|
||||
title={stage?.id === tile.id ? 'Back to everybody' : `Look at ${tile.label}`}
|
||||
onClick={opts.onClick}
|
||||
>
|
||||
<video
|
||||
ref={(el) => {
|
||||
if (el) videos.current.set(`${tile.id}#${slot}`, el)
|
||||
else videos.current.delete(`${tile.id}#${slot}`)
|
||||
}}
|
||||
className={classes('video-tile-picture', !tile.hasVideo && 'audio-only')}
|
||||
autoPlay
|
||||
playsInline
|
||||
// Our own microphone coming back would be the oldest mistake in video
|
||||
// calling.
|
||||
muted={tile.self}
|
||||
/>
|
||||
{!tile.hasVideo && (
|
||||
<div className="video-tile-face">
|
||||
<Avatar name={tile.label} size={slot === 'strip' ? 24 : mode === 'pip' ? 32 : 44} />
|
||||
</div>
|
||||
)}
|
||||
{slot !== 'strip' && (
|
||||
<span className="video-tile-name small ellipsis">
|
||||
{tile.muted && <Icon name="mic_off" size={12} />}
|
||||
{tile.label}
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
)
|
||||
|
||||
return (
|
||||
<div className={classes('call-stage', mode === 'pip' && 'pip')}>
|
||||
{mode === 'pip' && (
|
||||
<div className="call-stage-title">
|
||||
<Icon name="call" size={14} />
|
||||
<span className="small ellipsis" title={everybody}>
|
||||
{everybody || phase}
|
||||
</span>
|
||||
{/* Back to where the call is being held, which turns this corner
|
||||
back into the conversation - the picture stops being a corner of
|
||||
somewhere else and becomes the room you are in. */}
|
||||
<IconButton
|
||||
name="open_in_full"
|
||||
size={14}
|
||||
title="Go to the call"
|
||||
onClick={() => void store.selectBuffer(call.bufferId)}
|
||||
/>
|
||||
{/* Where a window's own control would be, because in the corner this
|
||||
is a window: the thing you close is closed from its top right. */}
|
||||
<IconButton name="remove" size={14} title="Minimise" onClick={() => store.setCallMinimized(true)} />
|
||||
</div>
|
||||
)}
|
||||
{/* Everybody in the call, one tile each, in as square a grid as the
|
||||
number allows - which is what every client with more than two people
|
||||
in a call has arrived at, because faces are roughly square and a row
|
||||
of six is six unreadable slivers.
|
||||
|
||||
Not a big-one-and-a-strip layout by default: that decides for
|
||||
somebody which person matters, and in a conversation between five
|
||||
people it is wrong most of the time. Pressing a face is how that
|
||||
decision gets made, and pressing it again is how it is unmade. */}
|
||||
{stage ? (
|
||||
<div className="video-stage" ref={grid}>
|
||||
{drawTile(stage, 'stage', {
|
||||
width: Math.floor(stageFit.tileWidth),
|
||||
onClick: () => setFocused(null)
|
||||
})}
|
||||
{tiles.length > 1 && (
|
||||
/* Everybody else, still there and still watchable: a call does not
|
||||
become a broadcast because one person is being looked at. */
|
||||
<div className="video-strip" style={{ height: `${stripHeight}px` }}>
|
||||
{tiles
|
||||
.filter((t) => t.id !== stage.id)
|
||||
.map((tile) =>
|
||||
drawTile(tile, 'strip', {
|
||||
width: Math.round((stripHeight * 16) / 9),
|
||||
onClick: () => setFocused(tile.id)
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<div
|
||||
className="video-grid"
|
||||
ref={grid}
|
||||
style={{
|
||||
gridTemplateColumns: `repeat(${layout.columns}, ${Math.floor(layout.tileWidth)}px)`
|
||||
}}
|
||||
>
|
||||
{tiles.map((tile) => drawTile(tile, 'stage', { onClick: () => setFocused(tile.id) }))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="call-stage-controls">
|
||||
{/* Which conversation this call is in, because the stage no longer
|
||||
@@ -89,6 +297,12 @@ export function CallStage(): JSX.Element | null {
|
||||
className={call.sharingScreen ? 'active' : undefined}
|
||||
onClick={() => void store.toggleScreenShare()}
|
||||
/>
|
||||
{/* Out of the way without ending: the call keeps going, and the bar
|
||||
that is left says so and offers it back. In the corner this sits in
|
||||
the title bar instead, where a window's own controls live. */}
|
||||
{mode !== 'pip' && (
|
||||
<IconButton name="expand_more" title="Minimise" onClick={() => store.setCallMinimized(true)} />
|
||||
)}
|
||||
<IconButton name="call_end" title="Hang up" className="calling" onClick={() => store.hangUpMatrixCall()} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -3,6 +3,7 @@ import { Icon } from './Icon'
|
||||
import { MessageRow, type MessageMode } from './MessageRow'
|
||||
import { LiveCards } from './LiveCards'
|
||||
import { CallView } from './CallView'
|
||||
import { CallStage } from './CallStage'
|
||||
import { useChat, usePref, useStore } from '../state/hooks'
|
||||
import { bufferDisplayName, isChatKind } from '../lib/util'
|
||||
import type { ChannelIndex } from '../lib/format'
|
||||
@@ -115,6 +116,7 @@ export function MessageList(): JSX.Element {
|
||||
const store = useStore()
|
||||
const bufferId = useChat((s) => s.activeBufferId)
|
||||
const messagesByBuffer = useChat((s) => s.messagesByBuffer)
|
||||
const activeCall = useChat((s) => s.activeCall)
|
||||
const loadingMore = useChat((s) => s.loadingMore)
|
||||
const loadingNewer = useChat((s) => s.loadingNewer)
|
||||
const historyGapAfter = useChat((s) => s.historyGapAfter)
|
||||
@@ -448,6 +450,10 @@ export function MessageList(): JSX.Element {
|
||||
useful while it can be seen, and a panel that scrolled away with
|
||||
the backlog would be gone the moment anybody read anything. */}
|
||||
<CallView bufferId={bufferId} />
|
||||
{/* A call with pictures, in the same place as the one without: a call
|
||||
and the conversation it is in are the same conversation. It moves to
|
||||
a corner of its own only when you walk away from it - see App. */}
|
||||
{activeCall?.bufferId === bufferId && <CallStage mode="inline" />}
|
||||
{/* Over the log rather than in it: a poll is one question being
|
||||
answered while the chat keeps moving underneath, and a line in the
|
||||
log would scroll away mid-vote. */}
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
/**
|
||||
* Who is talking, measured from the audio itself.
|
||||
*
|
||||
* Discord's voice call view is told this by the daemon, which holds that
|
||||
* connection and can hear it. A call held in this window has no such helper:
|
||||
* the audio is here, so the measuring is here too, which is the same division
|
||||
* the rest of the call follows.
|
||||
*
|
||||
* Root-mean-square rather than peak, sampled a few times a second: a peak
|
||||
* lights up on a chair creaking, and an indicator that flickers is worse than
|
||||
* none because it teaches people to ignore it.
|
||||
*/
|
||||
|
||||
/** Below this is a room with somebody in it, not somebody talking. */
|
||||
export const SPEAKING_FLOOR = 0.015
|
||||
|
||||
/** How often to look. Fast enough to land on the right person mid-sentence. */
|
||||
const SAMPLE_MS = 120
|
||||
|
||||
/**
|
||||
* How long a voice keeps the light on after it stops.
|
||||
*
|
||||
* Speech is full of gaps - between words, between breaths - and an indicator
|
||||
* that follows them exactly strobes. This is the shortest hold that reads as
|
||||
* one person talking rather than as a flashing box.
|
||||
*/
|
||||
const HOLD_MS = 400
|
||||
|
||||
interface Watched {
|
||||
stream: MediaStream
|
||||
analyser: AnalyserNode
|
||||
data: Uint8Array<ArrayBuffer>
|
||||
/** When this voice was last above the floor. */
|
||||
lastHeard: number
|
||||
peak: number
|
||||
}
|
||||
|
||||
export class Levels {
|
||||
private context: AudioContext | null = null
|
||||
private watched = new Map<string, Watched>()
|
||||
private timer: ReturnType<typeof setInterval> | null = null
|
||||
|
||||
/** Starts measuring one person's audio, or leaves it alone if already on. */
|
||||
watch(id: string, stream: MediaStream | null): void {
|
||||
if (!stream || stream.getAudioTracks().length === 0) {
|
||||
this.watched.delete(id)
|
||||
return
|
||||
}
|
||||
const existing = this.watched.get(id)
|
||||
if (existing?.stream === stream) return
|
||||
|
||||
try {
|
||||
// Created on first use rather than at startup: a window that never makes
|
||||
// a call should not hold an audio device open.
|
||||
this.context = this.context ?? new AudioContext()
|
||||
const analyser = this.context.createAnalyser()
|
||||
analyser.fftSize = 512
|
||||
this.context.createMediaStreamSource(stream).connect(analyser)
|
||||
this.watched.set(id, {
|
||||
stream,
|
||||
analyser,
|
||||
data: new Uint8Array(new ArrayBuffer(analyser.fftSize)),
|
||||
lastHeard: 0,
|
||||
peak: 0
|
||||
})
|
||||
this.start()
|
||||
} catch {
|
||||
// A stream with nothing to measure is not worth an error: the call
|
||||
// still works, it just has no light beside the name.
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether this person is talking now. */
|
||||
speaking(id: string): boolean {
|
||||
const watched = this.watched.get(id)
|
||||
if (!watched) return false
|
||||
return Date.now() - watched.lastHeard < HOLD_MS
|
||||
}
|
||||
|
||||
/** Stops measuring everybody and lets the audio device go. */
|
||||
dispose(): void {
|
||||
if (this.timer) clearInterval(this.timer)
|
||||
this.timer = null
|
||||
this.watched.clear()
|
||||
void this.context?.close()
|
||||
this.context = null
|
||||
}
|
||||
|
||||
private start(): void {
|
||||
if (this.timer) return
|
||||
this.timer = setInterval(() => {
|
||||
const now = Date.now()
|
||||
for (const watched of this.watched.values()) {
|
||||
watched.analyser.getByteTimeDomainData(watched.data)
|
||||
let sum = 0
|
||||
for (const sample of watched.data) {
|
||||
// The bytes are centred on 128; the distance from there is the
|
||||
// signal.
|
||||
const value = (sample - 128) / 128
|
||||
sum += value * value
|
||||
}
|
||||
watched.peak = Math.sqrt(sum / watched.data.length)
|
||||
if (watched.peak > SPEAKING_FLOOR) watched.lastHeard = now
|
||||
}
|
||||
}, SAMPLE_MS)
|
||||
}
|
||||
}
|
||||
@@ -115,7 +115,18 @@ export class Call {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* What to show this end of the call.
|
||||
*
|
||||
* The screen when one is being shared, the camera otherwise: every client
|
||||
* previews what it is sending, and a black square while sharing is the one
|
||||
* moment somebody most wants to check what the room can see.
|
||||
*/
|
||||
get stream(): MediaStream | null {
|
||||
if (this.screen) {
|
||||
const preview = new MediaStream([this.screen])
|
||||
return preview
|
||||
}
|
||||
return this.local
|
||||
}
|
||||
|
||||
@@ -123,6 +134,17 @@ export class Call {
|
||||
return this.remote
|
||||
}
|
||||
|
||||
/**
|
||||
* This end's microphone, separate from what the tile shows.
|
||||
*
|
||||
* `stream` above answers "what should be drawn", which is the screen while
|
||||
* one is being shared. Measuring whether somebody is talking has to follow
|
||||
* the microphone wherever the picture goes.
|
||||
*/
|
||||
get localAudio(): MediaStream | null {
|
||||
return this.local
|
||||
}
|
||||
|
||||
get sharingScreen(): boolean {
|
||||
return !!this.screen
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ import { ircNetworkFor } from '../lib/networks'
|
||||
import { parseDeepLink, type IrcLink } from '../../../shared/deeplink'
|
||||
import { MatrixCalls, type MatrixCallEvent, type RingingCall } from '../lib/matrixcall'
|
||||
import type { CallPhase } from '../lib/webrtc'
|
||||
import { Levels } from '../lib/levels'
|
||||
import type { PopoutState } from '../../../shared/ipc'
|
||||
|
||||
/**
|
||||
@@ -47,6 +48,19 @@ export interface BufferEntry extends WireBuffer {
|
||||
}
|
||||
|
||||
/** A wire Message plus local optimistic-send bookkeeping. */
|
||||
/** One person in a call, as the call view draws them. */
|
||||
export interface CallTile {
|
||||
id: string
|
||||
label: string
|
||||
stream: MediaStream | null
|
||||
/** Whether there is a live picture, as opposed to a name and a microphone. */
|
||||
hasVideo: boolean
|
||||
/** This end, which is drawn silent and is the one that can be muted here. */
|
||||
self?: boolean
|
||||
muted?: boolean
|
||||
speaking?: boolean
|
||||
}
|
||||
|
||||
export interface ChatMessage extends Message {
|
||||
/** Set on a locally-echoed message that nobilis hasn't confirmed yet. */
|
||||
pending?: boolean
|
||||
@@ -401,6 +415,8 @@ export interface ChatState {
|
||||
* the panel that shows it has to be able to find it again.
|
||||
*/
|
||||
ringingCall: RingingCall | null
|
||||
/** The call put away without being hung up. */
|
||||
callMinimized: boolean
|
||||
/** The screens and windows on offer, while somebody is choosing one. */
|
||||
screenSources: { id: string; name: string; thumbnail: string }[] | null
|
||||
activeCall: {
|
||||
@@ -495,6 +511,7 @@ const INITIAL: ChatState = {
|
||||
pinnedMessages: {},
|
||||
discordFriends: {},
|
||||
ringingCall: null,
|
||||
callMinimized: false,
|
||||
screenSources: null,
|
||||
activeCall: null,
|
||||
matrixIgnored: {},
|
||||
@@ -704,6 +721,8 @@ export class ChatStore {
|
||||
/** clientId -> the optimistic message awaiting its real echo. */
|
||||
private pendingSends = new Map<string, { bufferId: string; ts: number }>()
|
||||
private toastSeq = 0
|
||||
/** Who is talking in a call this window is holding. */
|
||||
private levels = new Levels()
|
||||
/**
|
||||
* The Matrix call engine, one per window.
|
||||
*
|
||||
@@ -741,6 +760,7 @@ export class ChatStore {
|
||||
return
|
||||
}
|
||||
this.set({
|
||||
callMinimized: this.state.activeCall ? this.state.callMinimized : false,
|
||||
activeCall: {
|
||||
accountId: call.accountId,
|
||||
bufferId: call.bufferId,
|
||||
@@ -1680,6 +1700,11 @@ export class ChatStore {
|
||||
this.matrixCalls.decline()
|
||||
}
|
||||
|
||||
/** Puts the call away, or brings it back. It keeps running either way. */
|
||||
setCallMinimized(minimized: boolean): void {
|
||||
this.set({ callMinimized: minimized })
|
||||
}
|
||||
|
||||
hangUpMatrixCall(): void {
|
||||
this.matrixCalls.end()
|
||||
}
|
||||
@@ -1710,10 +1735,55 @@ export class ChatStore {
|
||||
resolve?.(source)
|
||||
}
|
||||
|
||||
/** The two streams a call view draws: ours and theirs. */
|
||||
callStreams(): { local: MediaStream | null; remote: MediaStream | null } {
|
||||
/**
|
||||
* Everybody in the call, one entry per tile.
|
||||
*
|
||||
* A list rather than "ours and theirs", because a call is not always two
|
||||
* people: a 1:1 Matrix call is the two below, and a group call - when there
|
||||
* is one - is the same list, longer. The view draws whatever it is given
|
||||
* and does not need to learn anything new to draw five.
|
||||
*/
|
||||
callTiles(): CallTile[] {
|
||||
const call = this.matrixCalls.current
|
||||
return { local: call?.call.stream ?? null, remote: call?.remote ?? null }
|
||||
if (!call) {
|
||||
this.levels.dispose()
|
||||
return []
|
||||
}
|
||||
const active = this.state.activeCall
|
||||
const live = (stream: MediaStream | null): boolean =>
|
||||
!!stream?.getVideoTracks().some((t) => t.readyState === 'live' && !t.muted)
|
||||
const ours = call.call.stream
|
||||
const theirs = call.remote
|
||||
const them = this.state.buffers.find((b) => b.id === call.bufferId)?.name
|
||||
// Who is talking, measured here because the audio is here - the daemon
|
||||
// does this for a Discord call because it holds that connection.
|
||||
this.levels.watch('self', call.call.localAudio)
|
||||
this.levels.watch('them', theirs)
|
||||
const tiles: CallTile[] = [
|
||||
{
|
||||
id: 'self',
|
||||
label: this.ownNameIn(call.bufferId) || 'You',
|
||||
stream: ours,
|
||||
hasVideo: live(ours),
|
||||
self: true,
|
||||
muted: !!active?.muted,
|
||||
// A muted microphone is not a quiet one, and lighting it up would be
|
||||
// telling somebody they are being heard when they are not.
|
||||
speaking: !active?.muted && this.levels.speaking('self')
|
||||
}
|
||||
]
|
||||
// Only once there is somebody there: a tile for a person who has not
|
||||
// answered is a picture of nobody.
|
||||
if (theirs) {
|
||||
tiles.push({
|
||||
id: 'them',
|
||||
label: them ? bufferDisplayName(them) : 'Them',
|
||||
stream: theirs,
|
||||
hasVideo: live(theirs),
|
||||
speaking: this.levels.speaking('them')
|
||||
})
|
||||
}
|
||||
return tiles
|
||||
}
|
||||
|
||||
toggleCallMute(): void {
|
||||
|
||||
Reference in New Issue
Block a user