Discord: camera button, Go Live quality, sticker picker, scrollback repair
Camera button on the Discord stage, your own camera drawn from the capture. Sharing a screen asks resolution and frame rate beside the button, offering only what the account's tier allows; the width follows the capture's shape and frames are paced to the rate chosen. The sticker button now opens on Discord too. Settings gains a Discord page whose Repair scrollback runs refillDiscordHistory over every conversation (#201). Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
+1
-1
Submodule nobilis updated: a5751c0b68...ac6ef9cbb5
@@ -7314,3 +7314,66 @@ video.lightbox-media {
|
||||
background: var(--surface-container-high);
|
||||
border-bottom: 1px solid var(--outline-light);
|
||||
}
|
||||
|
||||
/* Something a call button asks before it acts, above the button. */
|
||||
.stage-popover {
|
||||
position: fixed;
|
||||
z-index: 1000;
|
||||
transform: translateX(-50%);
|
||||
}
|
||||
|
||||
/* Discord's Go Live question: how good a stream. */
|
||||
.go-live {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--spacing-s);
|
||||
width: max-content;
|
||||
max-width: 92vw;
|
||||
padding: var(--spacing-m);
|
||||
border: 1px solid var(--outline-medium);
|
||||
border-radius: var(--corner-radius);
|
||||
background: var(--surface-container-high);
|
||||
box-shadow: 0 12px 40px rgba(0, 0, 0, 0.5);
|
||||
}
|
||||
|
||||
.go-live-row {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.go-live-note {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.segmented {
|
||||
display: inline-flex;
|
||||
flex-wrap: wrap;
|
||||
border: 1px solid var(--outline-medium);
|
||||
border-radius: var(--corner-radius-small);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.segmented button {
|
||||
padding: 4px 10px;
|
||||
border: none;
|
||||
background: transparent;
|
||||
color: inherit;
|
||||
font: inherit;
|
||||
font-variant-numeric: tabular-nums;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.segmented button + button {
|
||||
border-left: 1px solid var(--outline-medium);
|
||||
}
|
||||
|
||||
.segmented button.active {
|
||||
background: var(--primary);
|
||||
color: var(--primary-text);
|
||||
}
|
||||
|
||||
/* Your own camera, as you would see yourself in a mirror. */
|
||||
.stage-picture.mirrored {
|
||||
transform: scaleX(-1);
|
||||
}
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
import { Icon } from './Icon'
|
||||
import { useChat, useStore } from '../state/hooks'
|
||||
import { Stage, type StageButton, type StageTile } from './stage/Stage'
|
||||
import { FeedPicture } from './stage/FeedPicture'
|
||||
import { cameraKey, streamKey } from '../lib/framefeed'
|
||||
import type { VideoQuality } from '../lib/discordscreen'
|
||||
import { usePref } from '../state/hooks'
|
||||
import type { VoiceMember, VoiceSession } from '../../../shared/wire'
|
||||
|
||||
/** How often to ask who is talking. */
|
||||
@@ -76,6 +78,8 @@ export function DiscordStage({ session, where }: { session: VoiceSession; where:
|
||||
const buffers = useChat((s) => s.buffers)
|
||||
const cameras = useChat((s) => s.discordCameras)
|
||||
const sharing = useChat((s) => s.discordSharing)
|
||||
const ownCamera = useChat((s) => s.discordCamera)
|
||||
const [choosing, setChoosing] = useState(false)
|
||||
const watching = useChat((s) => s.discordWatching)
|
||||
const voicePrefs = useChat((s) => s.voicePrefs)
|
||||
const [members, setMembers] = useState<VoiceMember[]>([])
|
||||
@@ -139,7 +143,14 @@ export function DiscordStage({ session, where }: { session: VoiceSession; where:
|
||||
speaking: m.isSelf ? speaking.self !== undefined : (speaking[m.userId] ?? 0) > 0,
|
||||
muted: m.muted,
|
||||
deafened: m.deafened,
|
||||
picture: camera ? () => <FeedPicture feedKey={cameraKey(accountId, m.userId)} /> : undefined
|
||||
// Your own camera is drawn from the capture rather than from the
|
||||
// call: Discord does not send a picture back to the one sending it.
|
||||
picture:
|
||||
m.isSelf && ownCamera
|
||||
? () => <LocalPicture stream={ownCamera} />
|
||||
: camera
|
||||
? () => <FeedPicture feedKey={cameraKey(accountId, m.userId)} />
|
||||
: undefined
|
||||
})
|
||||
// A shared screen is a tile of its own, the way Discord shows it: dark
|
||||
// until somebody chooses to watch, because watching opens a connection.
|
||||
@@ -178,11 +189,30 @@ export function DiscordStage({ session, where }: { session: VoiceSession; where:
|
||||
off: voicePrefs.deafened,
|
||||
onClick: () => void store.setVoiceMuted({ deafened: !voicePrefs.deafened })
|
||||
},
|
||||
{
|
||||
icon: ownCamera ? 'videocam' : 'videocam_off',
|
||||
label: ownCamera ? 'Turn the camera off' : 'Turn the camera on',
|
||||
off: !ownCamera,
|
||||
onClick: () => void store.toggleDiscordCamera(accountId)
|
||||
},
|
||||
{
|
||||
icon: sharing ? 'stop_screen_share' : 'screen_share',
|
||||
label: sharing ? 'Stop sharing' : 'Share your screen',
|
||||
active: sharing,
|
||||
onClick: () => void store.toggleScreenShare()
|
||||
// Stopping needs no questions; starting asks how good a stream, the
|
||||
// way Discord's own client does, beside the button that asked.
|
||||
onClick: () => (sharing ? void store.toggleScreenShare() : setChoosing(!choosing)),
|
||||
popover:
|
||||
choosing && !sharing ? (
|
||||
<GoLiveChooser
|
||||
accountId={accountId}
|
||||
onStart={(quality) => {
|
||||
setChoosing(false)
|
||||
void store.toggleScreenShare(quality)
|
||||
}}
|
||||
onClose={() => setChoosing(false)}
|
||||
/>
|
||||
) : null
|
||||
},
|
||||
{ icon: 'call_end', label: 'Disconnect', danger: true, onClick: () => void store.leaveVoice(accountId) }
|
||||
]
|
||||
@@ -201,3 +231,119 @@ export function DiscordStage({ session, where }: { session: VoiceSession; where:
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
/** This end's own camera, from the capture. Mirrored, as a mirror is. */
|
||||
function LocalPicture({ stream }: { stream: MediaStream }): JSX.Element {
|
||||
return (
|
||||
<video
|
||||
ref={(el) => {
|
||||
if (el && el.srcObject !== stream) el.srcObject = stream
|
||||
}}
|
||||
className="stage-picture mirrored"
|
||||
autoPlay
|
||||
playsInline
|
||||
muted
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
const HEIGHTS: (number | 'source')[] = [480, 720, 1080, 1440, 'source']
|
||||
const FRAMERATES = [15, 30, 60]
|
||||
|
||||
/**
|
||||
* How good a stream to send: resolution and frame rate.
|
||||
*
|
||||
* Asked each time, because the right answer depends on what is shared -
|
||||
* text wants resolution, a game wants frames, a thin connection wants
|
||||
* neither - and remembered, because it is usually the same answer as last
|
||||
* time. Only what the account's tier allows is offered: Discord holds an
|
||||
* account without Nitro to 720p30, and its own client offers nothing above.
|
||||
*/
|
||||
function GoLiveChooser({
|
||||
accountId,
|
||||
onStart,
|
||||
onClose
|
||||
}: {
|
||||
accountId: string
|
||||
onStart: (quality: VideoQuality) => void
|
||||
onClose: () => void
|
||||
}): JSX.Element {
|
||||
const [limits, setLimits] = useState<{ maxHeight: number; maxFramerate: number; source: boolean } | null>(null)
|
||||
const [height, setHeight] = usePref<number | 'source'>('discord.streamHeight', 720)
|
||||
const [framerate, setFramerate] = usePref<number>('discord.streamFramerate', 30)
|
||||
const box = useRef<HTMLDivElement>(null)
|
||||
|
||||
useEffect(() => {
|
||||
void window.moho
|
||||
.rpc<{ maxHeight: number; maxFramerate: number; source: boolean }>('discordStreamLimits', { accountId })
|
||||
.then(setLimits)
|
||||
.catch(() => setLimits({ maxHeight: 720, maxFramerate: 30, source: false }))
|
||||
}, [accountId])
|
||||
|
||||
useEffect(() => {
|
||||
const onKey = (e: KeyboardEvent): void => {
|
||||
if (e.key === 'Escape') onClose()
|
||||
}
|
||||
// A press anywhere else is a change of mind. Except on the button that
|
||||
// opened this, which closes it by itself and would otherwise reopen it.
|
||||
const onDown = (e: MouseEvent): void => {
|
||||
const target = e.target as HTMLElement
|
||||
if (box.current?.contains(target) || target.closest('.stage-button')) return
|
||||
onClose()
|
||||
}
|
||||
window.addEventListener('keydown', onKey)
|
||||
window.addEventListener('mousedown', onDown)
|
||||
return () => {
|
||||
window.removeEventListener('keydown', onKey)
|
||||
window.removeEventListener('mousedown', onDown)
|
||||
}
|
||||
}, [onClose])
|
||||
|
||||
const heights = HEIGHTS.filter((h) => (h === 'source' ? !!limits?.source : h <= (limits?.maxHeight ?? 720)))
|
||||
const rates = FRAMERATES.filter((r) => r <= (limits?.maxFramerate ?? 30))
|
||||
// A remembered choice above what this account may send falls back to the
|
||||
// best it may.
|
||||
const chosenHeight = heights.includes(height) ? height : heights[heights.length - 1]
|
||||
const chosenRate = rates.includes(framerate) ? framerate : rates[rates.length - 1]
|
||||
|
||||
return (
|
||||
<div className="go-live" ref={box} role="dialog" aria-label="Stream quality">
|
||||
<div className="go-live-row">
|
||||
<span className="small muted">Resolution</span>
|
||||
<div className="segmented">
|
||||
{heights.map((h) => (
|
||||
<button
|
||||
key={String(h)}
|
||||
type="button"
|
||||
className={h === chosenHeight ? 'active' : ''}
|
||||
onClick={() => setHeight(h)}
|
||||
>
|
||||
{h === 'source' ? 'Source' : `${h}p`}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
<div className="go-live-row">
|
||||
<span className="small muted">Frame rate</span>
|
||||
<div className="segmented">
|
||||
{rates.map((r) => (
|
||||
<button key={r} type="button" className={r === chosenRate ? 'active' : ''} onClick={() => setFramerate(r)}>
|
||||
{r} fps
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
{limits && !limits.source && limits.maxHeight <= 720 && (
|
||||
<p className="small muted go-live-note">Higher than 720p30 needs Nitro on this account.</p>
|
||||
)}
|
||||
<button
|
||||
type="button"
|
||||
className="button primary"
|
||||
disabled={!limits}
|
||||
onClick={() => onStart({ height: chosenHeight, framerate: chosenRate })}
|
||||
>
|
||||
Go live
|
||||
</button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -779,9 +779,9 @@ export function Composer(): JSX.Element | null {
|
||||
|
||||
{/* Its own button beside the emoji one, because it is its own
|
||||
gesture: an emoji goes into the line being written and a sticker
|
||||
is the message. Only where the service has them - which today is
|
||||
Matrix, and where a pack is what supplies them. */}
|
||||
{service === 'matrix' && (
|
||||
is the message. Only where the service has them: Matrix, where a
|
||||
pack supplies them, and Discord, where each guild has its own. */}
|
||||
{(service === 'matrix' || service === 'discord') && (
|
||||
<button
|
||||
ref={stickerButtonRef}
|
||||
type="button"
|
||||
@@ -796,7 +796,9 @@ export function Composer(): JSX.Element | null {
|
||||
// images are cached by the time they are drawn twice.
|
||||
if (opening) {
|
||||
void window.moho
|
||||
.rpc<StickerEntry[]>('listMatrixStickers', { bufferId: buffer.id })
|
||||
.rpc<StickerEntry[]>(service === 'discord' ? 'listDiscordStickers' : 'listMatrixStickers', {
|
||||
bufferId: buffer.id
|
||||
})
|
||||
.then(setStickers)
|
||||
.catch(() => setStickers([]))
|
||||
}
|
||||
@@ -873,8 +875,10 @@ export function Composer(): JSX.Element | null {
|
||||
accountId={account?.id}
|
||||
onSticker={(sticker) => {
|
||||
setStickerPicker(false)
|
||||
void window.moho
|
||||
.rpc('sendMatrixSticker', { bufferId: buffer.id, mxc: sticker.mxc, body: sticker.body })
|
||||
const sent = sticker.id
|
||||
? window.moho.rpc('sendDiscordSticker', { bufferId: buffer.id, stickerId: sticker.id })
|
||||
: window.moho.rpc('sendMatrixSticker', { bufferId: buffer.id, mxc: sticker.mxc, body: sticker.body })
|
||||
void sent
|
||||
.catch((e: Error) => store.toast('error', e.message))
|
||||
}}
|
||||
onSelect={() => setStickerPicker(false)}
|
||||
|
||||
@@ -113,14 +113,23 @@ function travels(accountId?: string): boolean {
|
||||
return !!accountId?.startsWith('matrix:')
|
||||
}
|
||||
|
||||
/** One image out of a Matrix sticker pack, as the daemon offers it. */
|
||||
/**
|
||||
* One sticker, as the daemon offers it: out of a Matrix pack, or one of a
|
||||
* Discord guild's.
|
||||
*/
|
||||
export interface StickerEntry {
|
||||
name: string
|
||||
/** The pack, or on Discord the guild it belongs to. */
|
||||
pack: string
|
||||
mxc: string
|
||||
/** Matrix: the upload it is. */
|
||||
mxc?: string
|
||||
/** Discord: the sticker's id. */
|
||||
id?: string
|
||||
body: string
|
||||
/** A local path the daemon already fetched, absent while it is fetching. */
|
||||
/** A picture of it, absent while fetching or where it cannot be drawn. */
|
||||
url?: string | null
|
||||
/** Discord: belongs to a guild this conversation is not in, without Nitro. */
|
||||
locked?: boolean
|
||||
}
|
||||
|
||||
/** One place emoji come from, as `listAllEmoji` answers. */
|
||||
@@ -527,10 +536,15 @@ export function EmojiPicker({
|
||||
<Section key={pack} title={pack}>
|
||||
{entries.map((sticker) => (
|
||||
<button
|
||||
key={sticker.mxc}
|
||||
key={sticker.mxc ?? sticker.id}
|
||||
type="button"
|
||||
className="emoji-cell sticker-cell"
|
||||
title={`${sticker.name} · ${pack}`}
|
||||
disabled={sticker.locked}
|
||||
title={
|
||||
sticker.locked
|
||||
? `${sticker.name} · only in ${pack} without Nitro`
|
||||
: `${sticker.name} · ${pack}`
|
||||
}
|
||||
onClick={() => onSticker?.(sticker)}
|
||||
>
|
||||
{sticker.url ? (
|
||||
@@ -545,8 +559,9 @@ export function EmojiPicker({
|
||||
|
||||
{stickersOnly && packs.length === 0 && (
|
||||
<p className="small muted emoji-empty">
|
||||
No sticker packs on this account. Packs added in another client - your own, or
|
||||
one a room shares - turn up here.
|
||||
{accountId?.startsWith('discord:')
|
||||
? 'None of your servers has stickers of its own.'
|
||||
: 'No sticker packs on this account. Packs added in another client - your own, or one a room shares - turn up here.'}
|
||||
</p>
|
||||
)}
|
||||
|
||||
|
||||
@@ -104,6 +104,7 @@ const CATEGORIES: { id: string; label: string; service?: Account['service'] }[]
|
||||
{ id: 'irc', label: 'IRC', service: 'irc' },
|
||||
{ id: 'sneedchat', label: 'Sneedchat', service: 'sneedchat' },
|
||||
{ id: 'tor', label: 'Tor' },
|
||||
{ id: 'discord', label: 'Discord', service: 'discord' },
|
||||
{ id: 'matrix', label: 'Matrix', service: 'matrix' },
|
||||
{ id: 'kick', label: 'Kick', service: 'kick' },
|
||||
{ id: 'about', label: 'About' }
|
||||
@@ -240,6 +241,7 @@ export function SettingsPanel(): JSX.Element {
|
||||
<MatrixReceiptSettings />
|
||||
</>
|
||||
)}
|
||||
{selected === 'discord' && <DiscordSettings />}
|
||||
{selected === 'kick' && <KickSettings />}
|
||||
{selected === 'about' && <AboutSettings />}
|
||||
</div>
|
||||
@@ -802,6 +804,52 @@ function TorSettings(): JSX.Element {
|
||||
* account's own row in the Accounts pane instead - a room-open toggle isn't
|
||||
* tied to any one account the way those are.
|
||||
*/
|
||||
/**
|
||||
* Mending Discord scrollback.
|
||||
*
|
||||
* A sweep over every Discord conversation rather than a button in each,
|
||||
* because the messages a storage fault lost are by definition the ones
|
||||
* nobody knows to go looking for. It re-reads recent history and keeps only
|
||||
* what is missing, so running it when nothing is wrong costs a minute of
|
||||
* requests and changes nothing.
|
||||
*/
|
||||
function DiscordSettings(): JSX.Element {
|
||||
const [running, setRunning] = useState(false)
|
||||
const [result, setResult] = useState<string | null>(null)
|
||||
|
||||
const repair = (): void => {
|
||||
setRunning(true)
|
||||
setResult(null)
|
||||
void window.moho
|
||||
.rpc<{ scanned: number; recovered: number; failed: number }>('refillDiscordHistory', {})
|
||||
.then(({ scanned, recovered, failed }) => {
|
||||
const found =
|
||||
recovered === 0
|
||||
? 'nothing was missing'
|
||||
: `${recovered} missing message${recovered === 1 ? ' was' : 's were'} put back`
|
||||
const skipped = failed > 0 ? `; ${failed} could not be read` : ''
|
||||
setResult(`Checked ${scanned} conversation${scanned === 1 ? '' : 's'}: ${found}${skipped}.`)
|
||||
})
|
||||
.catch((e: Error) => setResult(`The repair stopped: ${e.message}`))
|
||||
.finally(() => setRunning(false))
|
||||
}
|
||||
|
||||
return (
|
||||
<SettingsSection
|
||||
title="Scrollback"
|
||||
description="Re-reads the last 50 messages of every Discord conversation on every account and stores any that are missing here. Nothing already stored is changed. A conversation takes about a third of a second, so a large account takes a few minutes."
|
||||
>
|
||||
<div className="setting-row">
|
||||
<button type="button" className="button subtle" disabled={running} onClick={repair}>
|
||||
{running ? 'Repairing…' : 'Repair scrollback'}
|
||||
</button>
|
||||
{running && <span className="spinner" aria-label="Repairing" />}
|
||||
</div>
|
||||
{result && <p className="small muted">{result}</p>}
|
||||
</SettingsSection>
|
||||
)
|
||||
}
|
||||
|
||||
function MatrixSettings(): JSX.Element {
|
||||
return (
|
||||
<>
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useEffect, useRef, useState, type PointerEvent, type ReactNode } from 'react'
|
||||
import { useEffect, useLayoutEffect, useRef, useState, type PointerEvent, type ReactNode } from 'react'
|
||||
import { createPortal } from 'react-dom'
|
||||
import { Icon } from '../Icon'
|
||||
import { Avatar } from '../Avatar'
|
||||
import { usePref } from '../../state/hooks'
|
||||
@@ -52,6 +53,12 @@ export interface StageButton {
|
||||
off?: boolean
|
||||
danger?: boolean
|
||||
onClick: () => void
|
||||
/**
|
||||
* Something to ask before the button does its thing, drawn above it while
|
||||
* present - next to the gesture that asked, rather than in a dialog
|
||||
* somewhere else on the screen.
|
||||
*/
|
||||
popover?: JSX.Element | null
|
||||
}
|
||||
|
||||
/** Sixteen by nine, which is what cameras and screens both are. */
|
||||
@@ -356,8 +363,45 @@ function HeadButton({ icon, label, onClick }: { icon: string; label: string; onC
|
||||
}
|
||||
|
||||
function ControlButton({ button, compact }: { button: StageButton; compact?: boolean }): JSX.Element {
|
||||
const ref = useRef<HTMLButtonElement>(null)
|
||||
const [at, setAt] = useState<{ left: number; bottom: number } | null>(null)
|
||||
// Placed against the button on screen, in a portal: the stage clips what
|
||||
// overflows it, and a collapsed stage is a bar with no room above it.
|
||||
useLayoutEffect(() => {
|
||||
if (!button.popover || !ref.current) {
|
||||
setAt(null)
|
||||
return
|
||||
}
|
||||
const rect = ref.current.getBoundingClientRect()
|
||||
setAt({ left: rect.left + rect.width / 2, bottom: window.innerHeight - rect.top + 8 })
|
||||
}, [button.popover])
|
||||
return (
|
||||
<>
|
||||
{button.popover &&
|
||||
at &&
|
||||
createPortal(
|
||||
<div className="stage-popover" style={{ left: `${at.left}px`, bottom: `${at.bottom}px` }}>
|
||||
{button.popover}
|
||||
</div>,
|
||||
document.body
|
||||
)}
|
||||
<ControlButtonFace button={button} compact={compact} buttonRef={ref} />
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
function ControlButtonFace({
|
||||
button,
|
||||
compact,
|
||||
buttonRef
|
||||
}: {
|
||||
button: StageButton
|
||||
compact?: boolean
|
||||
buttonRef: React.Ref<HTMLButtonElement>
|
||||
}): JSX.Element {
|
||||
return (
|
||||
<button
|
||||
ref={buttonRef}
|
||||
type="button"
|
||||
className={classes(
|
||||
'stage-button',
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
/**
|
||||
* A screen, encoded here and sent to the daemon to put on the wire.
|
||||
* A screen or a camera, encoded here and sent to the daemon to put on the
|
||||
* wire.
|
||||
*
|
||||
* The division is the one the Matrix calls already draw, from the other side.
|
||||
* There the browser holds the whole call, because WebRTC is the browser's.
|
||||
@@ -25,11 +26,55 @@ declare class MediaStreamTrackProcessor {
|
||||
readonly readable: ReadableStream<VideoFrame>
|
||||
}
|
||||
|
||||
/** What to aim for. Discord's own Go Live default for a free account. */
|
||||
const WIDTH = 1280
|
||||
const HEIGHT = 720
|
||||
const FRAMERATE = 30
|
||||
const BITRATE = 2_500_000
|
||||
/**
|
||||
* What a picture is sent at. The height is a ceiling, not a size: the width
|
||||
* follows from the capture's own shape, so a tall window or an ultrawide
|
||||
* screen is scaled once, in proportion, rather than squeezed into 16:9.
|
||||
*/
|
||||
export interface VideoQuality {
|
||||
/** The tallest it may be, or `source` for the capture's own height. */
|
||||
height: number | 'source'
|
||||
framerate: number
|
||||
}
|
||||
|
||||
/** Discord's own default, and the ceiling without Nitro. */
|
||||
export const DEFAULT_QUALITY: VideoQuality = { height: 720, framerate: 30 }
|
||||
|
||||
/** What the encoder is configured with, worked out from the capture. */
|
||||
export interface EncodeSettings {
|
||||
width: number
|
||||
height: number
|
||||
framerate: number
|
||||
bitrate: number
|
||||
}
|
||||
|
||||
/** No picture is ever sent larger than this, whatever the source. */
|
||||
const LARGEST_HEIGHT = 2160
|
||||
|
||||
/**
|
||||
* The size, rate and bitrate to encode a capture at.
|
||||
*
|
||||
* Even dimensions, because VP8 works in 16-pixel blocks and an odd one is
|
||||
* refused by some encoders. The bitrate is about 0.09 bits a pixel a frame,
|
||||
* which is what puts 720p30 at Discord's 2.5 Mbit, kept between half a
|
||||
* megabit - below which text is mush - and eight.
|
||||
*/
|
||||
export function encodeSettings(
|
||||
capture: { width?: number; height?: number },
|
||||
quality: VideoQuality
|
||||
): EncodeSettings {
|
||||
const sourceWidth = capture.width || 1280
|
||||
const sourceHeight = capture.height || 720
|
||||
const ceiling = Math.min(quality.height === 'source' ? sourceHeight : quality.height, LARGEST_HEIGHT)
|
||||
// Never scaled up: sending a 600-pixel window as 1080p costs bandwidth and
|
||||
// adds nothing.
|
||||
const height = Math.min(sourceHeight, ceiling)
|
||||
const width = Math.round((sourceWidth * height) / sourceHeight)
|
||||
const even = (n: number): number => Math.max(16, n - (n % 2))
|
||||
const settings = { width: even(width), height: even(height), framerate: quality.framerate }
|
||||
const bitrate = Math.round(settings.width * settings.height * settings.framerate * 0.09)
|
||||
return { ...settings, bitrate: Math.min(8_000_000, Math.max(500_000, bitrate)) }
|
||||
}
|
||||
|
||||
/**
|
||||
* How often to send a keyframe unasked.
|
||||
@@ -58,17 +103,45 @@ export interface ScreenShare {
|
||||
export async function shareScreen(
|
||||
accountId: string,
|
||||
source: MediaStream,
|
||||
settings: EncodeSettings,
|
||||
onError: (message: string) => void
|
||||
): Promise<ScreenShare> {
|
||||
return sendVideo(accountId, 'screen', source, settings, onError)
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts encoding a camera for the call this account is in.
|
||||
*
|
||||
* The same encoder as a screen: VP8 in software, keyframes on a clock. What
|
||||
* differs is only where the daemon puts the frames - on the voice
|
||||
* connection's camera SSRC rather than on a stream connection of its own.
|
||||
*/
|
||||
export async function sendCamera(
|
||||
accountId: string,
|
||||
source: MediaStream,
|
||||
settings: EncodeSettings,
|
||||
onError: (message: string) => void
|
||||
): Promise<ScreenShare> {
|
||||
return sendVideo(accountId, 'camera', source, settings, onError)
|
||||
}
|
||||
|
||||
async function sendVideo(
|
||||
accountId: string,
|
||||
kind: 'screen' | 'camera',
|
||||
source: MediaStream,
|
||||
settings: EncodeSettings,
|
||||
onError: (message: string) => void
|
||||
): Promise<ScreenShare> {
|
||||
const track = source.getVideoTracks()[0]
|
||||
if (!track) throw new Error('that capture has no picture in it')
|
||||
const { width, height, framerate, bitrate } = settings
|
||||
|
||||
const supported = await VideoEncoder.isConfigSupported({
|
||||
codec: 'vp8',
|
||||
width: WIDTH,
|
||||
height: HEIGHT,
|
||||
bitrate: BITRATE,
|
||||
framerate: FRAMERATE
|
||||
width,
|
||||
height,
|
||||
bitrate,
|
||||
framerate
|
||||
})
|
||||
if (!supported.supported) throw new Error('this machine has no VP8 encoder')
|
||||
|
||||
@@ -82,6 +155,7 @@ export async function shareScreen(
|
||||
void window.moho
|
||||
.rpc('sendDiscordVideoFrame', {
|
||||
accountId,
|
||||
kind,
|
||||
frame: toBase64(bytes),
|
||||
timestampMicros: chunk.timestamp
|
||||
})
|
||||
@@ -98,10 +172,10 @@ export async function shareScreen(
|
||||
})
|
||||
encoder.configure({
|
||||
codec: 'vp8',
|
||||
width: WIDTH,
|
||||
height: HEIGHT,
|
||||
bitrate: BITRATE,
|
||||
framerate: FRAMERATE,
|
||||
width,
|
||||
height,
|
||||
bitrate,
|
||||
framerate,
|
||||
// Latency over quality, which is what a shared screen is for: somebody
|
||||
// is watching a pointer move.
|
||||
latencyMode: 'realtime'
|
||||
@@ -109,6 +183,8 @@ export async function shareScreen(
|
||||
|
||||
const reader = new MediaStreamTrackProcessor({ track }).readable.getReader()
|
||||
let lastKeyframe = 0
|
||||
let lastFrame = 0
|
||||
const spacing = 1000 / framerate
|
||||
|
||||
const pump = async (): Promise<void> => {
|
||||
while (!stopped) {
|
||||
@@ -122,6 +198,15 @@ export async function shareScreen(
|
||||
continue
|
||||
}
|
||||
const now = performance.now()
|
||||
// Held to the rate chosen. A screen captures at whatever the display
|
||||
// runs at, and 60 frames encoded for a 30 frame stream is twice the
|
||||
// bandwidth for a picture nobody sees move faster. A little slack, so
|
||||
// a capture running exactly at the rate is not halved by jitter.
|
||||
if (now - lastFrame < spacing * 0.85) {
|
||||
value.close()
|
||||
continue
|
||||
}
|
||||
lastFrame = now
|
||||
const key = now - lastKeyframe > KEYFRAME_EVERY_MS
|
||||
if (key) lastKeyframe = now
|
||||
encoder.encode(value, { keyFrame: key })
|
||||
|
||||
@@ -32,6 +32,7 @@ import type { CallPhase } from '../lib/webrtc'
|
||||
import { Levels } from '../lib/levels'
|
||||
import { firstFrame, SCREEN_CHOICE_MS } from '../lib/capture'
|
||||
import { cameraKey, closeFeeds, feedFrame, streamKey as feedStreamKey } from '../lib/framefeed'
|
||||
import { DEFAULT_QUALITY, encodeSettings, sendCamera, shareScreen, type VideoQuality } from '../lib/discordscreen'
|
||||
import type { PopoutState } from '../../../shared/ipc'
|
||||
|
||||
/**
|
||||
@@ -539,6 +540,11 @@ export interface ChatState {
|
||||
screenSources: { id: string; name: string; thumbnail: string }[] | null
|
||||
/** Whether this window is sharing a screen into a Discord call. */
|
||||
discordSharing: boolean
|
||||
/**
|
||||
* This end's camera in a Discord call, while it is on: the capture, so the
|
||||
* call can show you what everybody else sees.
|
||||
*/
|
||||
discordCamera: MediaStream | null
|
||||
/**
|
||||
* Who is streaming right now, by Discord user id, and the key their stream
|
||||
* is named by.
|
||||
@@ -667,6 +673,7 @@ const INITIAL: ChatState = {
|
||||
discordModal: null,
|
||||
activeCall: null,
|
||||
discordSharing: false,
|
||||
discordCamera: null,
|
||||
discordStreams: {},
|
||||
discordCameras: {},
|
||||
discordWatching: null,
|
||||
@@ -930,6 +937,8 @@ export class ChatStore {
|
||||
* `discordSharing` beside it.
|
||||
*/
|
||||
private discordShare: { stop: () => void; stream: MediaStream } | null = null
|
||||
/** The Discord camera's encoder, held for the same reason. */
|
||||
private discordCameraSend: { stop: () => void; stream: MediaStream } | null = null
|
||||
/**
|
||||
* The Matrix call engine, one per window.
|
||||
*
|
||||
@@ -1763,6 +1772,15 @@ export class ChatStore {
|
||||
}
|
||||
|
||||
async leaveVoice(accountId: string): Promise<void> {
|
||||
// What this end was sending goes with it. Left running, the camera light
|
||||
// would stay on and the encoder would go on handing frames to a call
|
||||
// that has ended.
|
||||
this.stopDiscordCamera()
|
||||
if (this.discordShare) {
|
||||
this.discordShare.stop()
|
||||
this.discordShare = null
|
||||
this.set({ discordSharing: false })
|
||||
}
|
||||
try {
|
||||
await window.moho.rpc('leaveVoiceChannel', { accountId })
|
||||
await this.refreshVoiceSessions()
|
||||
@@ -2476,7 +2494,7 @@ export class ChatStore {
|
||||
* Opened by `openScreen`, which is where every share on every desktop
|
||||
* starts.
|
||||
*/
|
||||
async toggleScreenShare(): Promise<void> {
|
||||
async toggleScreenShare(quality?: VideoQuality): Promise<void> {
|
||||
// A Discord call is not a Matrix one and has no activeCall behind it -
|
||||
// its audio lives in the daemon - so it is answered first and on its own
|
||||
// terms. The gesture and the button are the same either way, which is
|
||||
@@ -2488,7 +2506,7 @@ export class ChatStore {
|
||||
(s) => s.accountId.startsWith('discord:') && !!s.bufferId
|
||||
)
|
||||
if (discord?.bufferId && !this.state.activeCall) {
|
||||
await this.toggleDiscordScreenShare(discord.accountId, discord.bufferId)
|
||||
await this.toggleDiscordScreenShare(discord.accountId, discord.bufferId, quality)
|
||||
return
|
||||
}
|
||||
if (!this.state.activeCall) return
|
||||
@@ -2539,7 +2557,7 @@ export class ChatStore {
|
||||
* draw from the other side, where the browser holds the whole call because
|
||||
* WebRTC is the browser's.
|
||||
*/
|
||||
async toggleDiscordScreenShare(accountId: string, bufferId: string): Promise<void> {
|
||||
async toggleDiscordScreenShare(accountId: string, bufferId: string, quality: VideoQuality = DEFAULT_QUALITY): Promise<void> {
|
||||
if (this.discordShare) {
|
||||
this.discordShare.stop()
|
||||
this.discordShare = null
|
||||
@@ -2556,7 +2574,11 @@ export class ChatStore {
|
||||
const stream = await this.openScreen()
|
||||
if (!stream) return
|
||||
|
||||
await window.moho.rpc('startDiscordScreenShare', { bufferId })
|
||||
// Worked out from the capture, now that there is one: the width
|
||||
// follows the screen's own shape, and the daemon tells viewers exactly
|
||||
// what the encoder below will send.
|
||||
const settings = encodeSettings(stream.getVideoTracks()[0]?.getSettings() ?? {}, quality)
|
||||
await window.moho.rpc('startDiscordScreenShare', { bufferId, ...settings })
|
||||
this.toast('info', 'Setting up the stream…')
|
||||
|
||||
// The connection is opened by the daemon when Discord answers, which
|
||||
@@ -2575,8 +2597,7 @@ export class ChatStore {
|
||||
return
|
||||
}
|
||||
|
||||
const { shareScreen } = await import('../lib/discordscreen')
|
||||
this.discordShare = await shareScreen(accountId, stream, (message) => {
|
||||
this.discordShare = await shareScreen(accountId, stream, settings, (message) => {
|
||||
this.toast('error', message)
|
||||
this.discordShare = null
|
||||
this.set({ discordSharing: false })
|
||||
@@ -2589,6 +2610,73 @@ export class ChatStore {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Turns this end's camera on or off in a Discord call.
|
||||
*
|
||||
* The camera is opened first, so a machine without one is told so before
|
||||
* anybody in the call sees a tile appear. Then the daemon tells Discord a
|
||||
* picture is coming, and only once the call's encryption group can be
|
||||
* encrypted for does the encoder start - a frame before that is one nobody
|
||||
* can read.
|
||||
*/
|
||||
async toggleDiscordCamera(accountId: string): Promise<void> {
|
||||
if (this.state.discordCamera) {
|
||||
this.stopDiscordCamera(accountId)
|
||||
return
|
||||
}
|
||||
let stream: MediaStream
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getUserMedia({
|
||||
video: { width: { ideal: 1280 }, height: { ideal: 720 }, frameRate: { ideal: 30 } },
|
||||
audio: false
|
||||
})
|
||||
} catch (e) {
|
||||
this.toast('error', `Couldn't turn the camera on: ${(e as Error).message}`)
|
||||
return
|
||||
}
|
||||
// Discord's own client sends a camera at 720p30 whatever the account.
|
||||
const settings = encodeSettings(stream.getVideoTracks()[0]?.getSettings() ?? {}, DEFAULT_QUALITY)
|
||||
const giveUp = (message: string): void => {
|
||||
for (const track of stream.getTracks()) track.stop()
|
||||
this.toast('error', message)
|
||||
void window.moho.rpc('setDiscordCamera', { accountId, on: false }).catch(() => {})
|
||||
}
|
||||
try {
|
||||
await window.moho.rpc('setDiscordCamera', { accountId, on: true, ...settings })
|
||||
let ready = false
|
||||
for (let i = 0; i < 50 && !ready; i++) {
|
||||
ready = await window.moho
|
||||
.rpc<{ ready: boolean }>('discordCameraReady', { accountId })
|
||||
.then((r) => r.ready)
|
||||
.catch(() => false)
|
||||
if (!ready) await new Promise((r) => setTimeout(r, 200))
|
||||
}
|
||||
if (!ready) {
|
||||
giveUp("Couldn't turn the camera on: the call's encryption never got ready")
|
||||
return
|
||||
}
|
||||
this.discordCameraSend = await sendCamera(accountId, stream, settings, (message) => {
|
||||
this.discordCameraSend = null
|
||||
this.set({ discordCamera: null })
|
||||
giveUp(message)
|
||||
})
|
||||
this.set({ discordCamera: stream })
|
||||
} catch (e) {
|
||||
giveUp(`Couldn't turn the camera on: ${(e as Error).message}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** Turns the Discord camera off, if it is on. */
|
||||
private stopDiscordCamera(accountId?: string): void {
|
||||
const sending = this.discordCameraSend
|
||||
this.discordCameraSend = null
|
||||
sending?.stop()
|
||||
const stream = this.state.discordCamera
|
||||
if (stream) for (const track of stream.getTracks()) track.stop()
|
||||
if (stream || sending) this.set({ discordCamera: null })
|
||||
if (accountId) void window.moho.rpc('setDiscordCamera', { accountId, on: false }).catch(() => {})
|
||||
}
|
||||
|
||||
/** Waits for the daemon to say the stream connection is up. */
|
||||
private async waitForDiscordStream(
|
||||
accountId: string
|
||||
|
||||
Reference in New Issue
Block a user