Discord calls: echo cancellation and noise suppression, and the soundboard
Echo cancellation (#232): a call's microphone was a raw capture, so anybody on speakers sent the call back into itself, where a Matrix call gets Chromium's processing for free. sonora - WebRTC's audio processing ported to Rust, no C++ in any build - now sits between the capture and the speaking gate: AEC3 with the call's own mix, as actually played, for its reference; noise suppression; a high-pass filter. Both switchable in voice.toml (echoCancellation, noiseSuppression; on by default) through setVoiceProcessing, followed by a call in progress within a tick. Soundboard (#183): Discord plays sounds in the clients. Playing one is send-soundboard-sound; hearing one is VOICE_CHANNEL_EFFECT_SEND, after which the sound is fetched (kept in the cache), decoded with symphonia (MP3, Ogg Vorbis), resampled to 48kHz stereo and mixed into the call as one more voice - at the call's volume, and in the echo canceller's reference so it is not sent back. Default sounds come from REST; a guild's own are asked of the gateway (op 31) on joining its voice channel and kept up to date from the GUILD_SOUNDBOARD_* dispatches. listSoundboard marks what needs Nitro here. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
Generated
+199
@@ -573,6 +573,20 @@ name = "bytemuck"
|
||||
version = "1.25.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
|
||||
dependencies = [
|
||||
"bytemuck_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bytemuck_derive"
|
||||
version = "1.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
@@ -822,6 +836,12 @@ dependencies = [
|
||||
"rand 0.9.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core_detect"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48"
|
||||
|
||||
[[package]]
|
||||
name = "coreaudio-rs"
|
||||
version = "0.14.2"
|
||||
@@ -1499,6 +1519,20 @@ dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.42"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"core_detect",
|
||||
"multiversion_no_op",
|
||||
"rustversion",
|
||||
"scopeguard",
|
||||
"simdutf8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "enum-ordinalize"
|
||||
version = "3.1.15"
|
||||
@@ -3469,6 +3503,12 @@ dependencies = [
|
||||
"pxfm",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "multiversion_no_op"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d"
|
||||
|
||||
[[package]]
|
||||
name = "ndk"
|
||||
version = "0.9.0"
|
||||
@@ -3549,6 +3589,8 @@ dependencies = [
|
||||
"sha1 0.11.0",
|
||||
"sha2",
|
||||
"socket2",
|
||||
"sonora",
|
||||
"symphonia",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tokio-socks",
|
||||
@@ -5588,6 +5630,79 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sonora"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e1ce442ab5cb392c4067b8f2cced20be4cf12e479f6dfbf44acbe9b14ac9250"
|
||||
dependencies = [
|
||||
"sonora-aec3",
|
||||
"sonora-agc2",
|
||||
"sonora-common-audio",
|
||||
"sonora-ns",
|
||||
"sonora-simd",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sonora-aec3"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "50280e31256d09f7838aff6715ed4b7640e514dc1092a1e14013c01e552380db"
|
||||
dependencies = [
|
||||
"sonora-common-audio",
|
||||
"sonora-fft",
|
||||
"sonora-simd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sonora-agc2"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a93a9fa8373a33e61a7fb441d8a9e725ac6024079d2c77a845cf6abe1790b9f"
|
||||
dependencies = [
|
||||
"bytemuck",
|
||||
"derive_more",
|
||||
"sonora-common-audio",
|
||||
"sonora-fft",
|
||||
"sonora-simd",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sonora-common-audio"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6635bc61b24807ceac93ad0807b1292b6b0620caa6cd360d3be89fc616b833cc"
|
||||
dependencies = [
|
||||
"derive_more",
|
||||
"sonora-simd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sonora-fft"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "27e272c70f34363afb5382dd1818c05f8f63ef6785635736a14e9ccbbb5ba012"
|
||||
|
||||
[[package]]
|
||||
name = "sonora-ns"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d511feef19302c716d245c4555087d161ba91c178735ea11107dcce0d0b7a86f"
|
||||
dependencies = [
|
||||
"sonora-fft",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sonora-simd"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7cf0804e4752a5562262e143014f53043080ead4cd5a98f9278d3ceaf32d868"
|
||||
dependencies = [
|
||||
"cpufeatures 0.3.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spin"
|
||||
version = "0.9.9"
|
||||
@@ -5727,6 +5842,90 @@ version = "2.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||
|
||||
[[package]]
|
||||
name = "symphonia"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"symphonia-bundle-mp3",
|
||||
"symphonia-codec-vorbis",
|
||||
"symphonia-core",
|
||||
"symphonia-format-ogg",
|
||||
"symphonia-metadata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-bundle-mp3"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4872dd6bb56bf5eac799e3e957aa1981086c3e613b27e0ac23b176054f7c57ed"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"log",
|
||||
"symphonia-core",
|
||||
"symphonia-metadata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-codec-vorbis"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f025837c309cd69ffef572750b4a2257b59552c5399a5e49707cc5b1b85d1c73"
|
||||
dependencies = [
|
||||
"log",
|
||||
"symphonia-core",
|
||||
"symphonia-utils-xiph",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-core"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ea00cc4f79b7f6bb7ff87eddc065a1066f3a43fe1875979056672c9ef948c2af"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"bitflags 1.3.2",
|
||||
"bytemuck",
|
||||
"lazy_static",
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-format-ogg"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b4955c67c1ed3aa8ae8428d04ca8397fbef6a19b2b051e73b5da8b1435639cb"
|
||||
dependencies = [
|
||||
"log",
|
||||
"symphonia-core",
|
||||
"symphonia-metadata",
|
||||
"symphonia-utils-xiph",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-metadata"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "36306ff42b9ffe6e5afc99d49e121e0bd62fe79b9db7b9681d48e29fa19e6b16"
|
||||
dependencies = [
|
||||
"encoding_rs",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"symphonia-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symphonia-utils-xiph"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee27c85ab799a338446b68eec77abf42e1a6f1bb490656e121c6e27bfbab9f16"
|
||||
dependencies = [
|
||||
"symphonia-core",
|
||||
"symphonia-metadata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.109"
|
||||
|
||||
@@ -103,6 +103,12 @@ js_int = "=0.2.2"
|
||||
cpal = "0.18.2"
|
||||
# The Opus codec, for calls and for recording a voice message.
|
||||
opus2 = "0.4"
|
||||
# Echo cancellation, noise suppression and a high-pass filter for a call's
|
||||
# microphone: WebRTC's audio processing, ported to Rust, so a Discord call gets
|
||||
# what Chromium gives a Matrix one without a C++ library in every build.
|
||||
sonora = "0.2"
|
||||
# Soundboard sounds are MP3 or Ogg Vorbis, decoded here to be mixed into a call.
|
||||
symphonia = { version = "0.5", default-features = false, features = ["mp3", "ogg", "vorbis"] }
|
||||
|
||||
# Two calls, both in main.rs: see keep_the_heap_honest() there for why a chat
|
||||
# daemon has anything to say to its allocator.
|
||||
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
//! What a call's microphone goes through before it is sent.
|
||||
//!
|
||||
//! A Matrix call takes its microphone from the window, where Chromium has
|
||||
//! already run WebRTC's audio processing over it - echo cancellation, noise
|
||||
//! suppression, a high-pass filter. A Discord call's microphone is a raw
|
||||
//! capture in this process, and had none of that: anybody on speakers sent
|
||||
//! the call back into itself (#232). This is the same processing, from a
|
||||
//! pure-Rust port of it, between the capture and the encoder.
|
||||
//!
|
||||
//! The echo canceller needs to hear what the speakers are playing, which is
|
||||
//! the call's own mix: the voice clock hands each 20ms of it here as it plays
|
||||
//! it, and each 20ms of microphone after. Both are 48kHz stereo, interleaved,
|
||||
//! in the 10ms frames WebRTC works in.
|
||||
|
||||
use sonora::config::{EchoCanceller, HighPassFilter, NoiseSuppression};
|
||||
use sonora::{AudioProcessing, Config, StreamConfig};
|
||||
|
||||
const RATE: usize = 48_000;
|
||||
const CHANNELS: usize = 2;
|
||||
/// One 10ms frame, interleaved.
|
||||
const CHUNK: usize = RATE / 100 * CHANNELS;
|
||||
|
||||
pub struct CallProcessor {
|
||||
apm: AudioProcessing,
|
||||
echo: bool,
|
||||
noise: bool,
|
||||
}
|
||||
|
||||
impl CallProcessor {
|
||||
pub fn new(echo: bool, noise: bool) -> Self {
|
||||
let apm = AudioProcessing::builder()
|
||||
.config(config(echo, noise))
|
||||
.capture_config(StreamConfig::new(RATE as u32, CHANNELS as u16))
|
||||
.render_config(StreamConfig::new(RATE as u32, CHANNELS as u16))
|
||||
.build();
|
||||
CallProcessor { apm, echo, noise }
|
||||
}
|
||||
|
||||
/// Follows the settings, which can change during a call.
|
||||
pub fn set(&mut self, echo: bool, noise: bool) {
|
||||
if (echo, noise) != (self.echo, self.noise) {
|
||||
self.echo = echo;
|
||||
self.noise = noise;
|
||||
self.apm.apply_config(config(echo, noise));
|
||||
}
|
||||
}
|
||||
|
||||
/// What the speakers are playing right now - silence included, because
|
||||
/// the canceller keeps time by it.
|
||||
pub fn render(&mut self, played: &[i16]) {
|
||||
if !self.echo {
|
||||
return;
|
||||
}
|
||||
let mut scratch = [0i16; CHUNK];
|
||||
for chunk in played.chunks_exact(CHUNK) {
|
||||
let _ = self.apm.process_render_i16(chunk, &mut scratch);
|
||||
}
|
||||
}
|
||||
|
||||
/// The microphone, cleaned in place.
|
||||
pub fn capture(&mut self, frame: &mut [f32]) {
|
||||
if !self.echo && !self.noise {
|
||||
return;
|
||||
}
|
||||
let mut src = [0i16; CHUNK];
|
||||
let mut out = [0i16; CHUNK];
|
||||
for chunk in frame.chunks_exact_mut(CHUNK) {
|
||||
for (s, f) in src.iter_mut().zip(chunk.iter()) {
|
||||
*s = (f.clamp(-1.0, 1.0) * i16::MAX as f32) as i16;
|
||||
}
|
||||
if self.apm.process_capture_i16(&src, &mut out).is_ok() {
|
||||
for (f, s) in chunk.iter_mut().zip(out.iter()) {
|
||||
*f = *s as f32 / i16::MAX as f32;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn config(echo: bool, noise: bool) -> Config {
|
||||
Config {
|
||||
// Always on: it takes out the rumble below speech, which nobody
|
||||
// wants sent, and the echo canceller expects it.
|
||||
high_pass_filter: Some(HighPassFilter::default()),
|
||||
echo_canceller: echo.then(EchoCanceller::default),
|
||||
noise_suppression: noise.then(NoiseSuppression::default),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A tone played out of the speakers and heard back by the microphone
|
||||
/// comes out of the canceller much quieter than it went in.
|
||||
#[test]
|
||||
fn an_echo_of_what_was_played_is_taken_out() {
|
||||
let mut apm = CallProcessor::new(true, false);
|
||||
let tone = |n: usize| -> f32 { (n as f32 * 2.0 * std::f32::consts::PI * 440.0 / RATE as f32).sin() * 0.3 };
|
||||
let mut heard_in = 0.0f32;
|
||||
let mut heard_out = 0.0f32;
|
||||
// Ten seconds, so the canceller has time to converge; measured over
|
||||
// the last two.
|
||||
for tick in 0..500usize {
|
||||
let base = tick * 960;
|
||||
let played: Vec<i16> = (0..960).flat_map(|i| {
|
||||
let v = (tone(base + i) * i16::MAX as f32) as i16;
|
||||
[v, v]
|
||||
}).collect();
|
||||
apm.render(&played);
|
||||
// The room: the same tone, a little later and quieter.
|
||||
let mut mic: Vec<f32> = (0..960).flat_map(|i| {
|
||||
let v = if base + i >= 480 { tone(base + i - 480) * 0.5 } else { 0.0 };
|
||||
[v, v]
|
||||
}).collect();
|
||||
let before: f32 = mic.iter().map(|s| s * s).sum();
|
||||
apm.capture(&mut mic);
|
||||
if tick >= 400 {
|
||||
heard_in += before;
|
||||
heard_out += mic.iter().map(|s| s * s).sum::<f32>();
|
||||
}
|
||||
}
|
||||
assert!(heard_out < heard_in * 0.1, "echo energy went {heard_in} -> {heard_out}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn off_is_untouched() {
|
||||
let mut apm = CallProcessor::new(false, false);
|
||||
let mut mic = vec![0.25f32; 1920];
|
||||
apm.capture(&mut mic);
|
||||
assert!(mic.iter().all(|s| *s == 0.25));
|
||||
}
|
||||
}
|
||||
+22
-1
@@ -63,6 +63,18 @@ pub struct VoicePrefs {
|
||||
/// Discord keeps it: the person who is too loud is too loud every time.
|
||||
#[serde(default)]
|
||||
pub user_volumes: std::collections::HashMap<String, f32>,
|
||||
/// Taking what the speakers play back out of the microphone, so somebody
|
||||
/// on speakers does not send the call back into itself. On by default,
|
||||
/// as it is in Discord and in every browser.
|
||||
#[serde(default = "on")]
|
||||
pub echo_cancellation: bool,
|
||||
/// Taking steady background noise - a fan, a hum - out of the microphone.
|
||||
#[serde(default = "on")]
|
||||
pub noise_suppression: bool,
|
||||
}
|
||||
|
||||
fn on() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn unity() -> f32 {
|
||||
@@ -71,7 +83,16 @@ fn unity() -> f32 {
|
||||
|
||||
impl Default for VoicePrefs {
|
||||
fn default() -> Self {
|
||||
VoicePrefs { input: None, output: None, mic_muted: false, deafened: false, output_volume: 1.0, user_volumes: Default::default() }
|
||||
VoicePrefs {
|
||||
input: None,
|
||||
output: None,
|
||||
mic_muted: false,
|
||||
deafened: false,
|
||||
output_volume: 1.0,
|
||||
user_volumes: Default::default(),
|
||||
echo_cancellation: true,
|
||||
noise_suppression: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -900,6 +900,23 @@ pub(super) async fn run_gateway(state: &AppState, config: &DiscordAccountConfig,
|
||||
// at connect, so the cheapest correct answer is to
|
||||
// register it again - which is what the role events
|
||||
// already do for the same reason.
|
||||
// A guild's soundboard: the whole list, asked for or
|
||||
// replaced, and single sounds coming and going.
|
||||
"SOUNDBOARD_SOUNDS" | "GUILD_SOUNDBOARD_SOUNDS_UPDATE" => {
|
||||
if let Some(guild) = d["guild_id"].as_str() {
|
||||
super::soundboard::note_guild_sounds(&account_id, guild, &d["soundboard_sounds"]);
|
||||
}
|
||||
}
|
||||
"GUILD_SOUNDBOARD_SOUND_CREATE" | "GUILD_SOUNDBOARD_SOUND_UPDATE" | "GUILD_SOUNDBOARD_SOUND_DELETE" => {
|
||||
super::soundboard::note_sound_change(&account_id, t, d);
|
||||
}
|
||||
// Somebody set a soundboard sound off in a voice channel.
|
||||
// Each client plays it itself; this is how it is told to.
|
||||
"VOICE_CHANNEL_EFFECT_SEND" => {
|
||||
let (s, a, d) = (state.clone(), account_id.clone(), d.clone());
|
||||
tokio::spawn(async move { super::soundboard::heard(&s, &a, &d).await });
|
||||
}
|
||||
|
||||
// A stage going live, changing its topic, or ending.
|
||||
// The channel exists either way; what this says is
|
||||
// whether anything is on in it.
|
||||
@@ -1084,6 +1101,11 @@ pub(super) async fn run_gateway(state: &AppState, config: &DiscordAccountConfig,
|
||||
// Into a stage's audience or out of it. Held off
|
||||
// the microphone while in it.
|
||||
state.voice.set_suppressed(&account_id, channel_id.is_some() && flags.suppressed);
|
||||
// Into a guild's voice channel: its own soundboard
|
||||
// is asked for, so the board can offer it.
|
||||
if let (Some(guild), Some(_)) = (d["guild_id"].as_str(), channel_id) {
|
||||
super::soundboard::request(state, &account_id, &[guild.to_string()]);
|
||||
}
|
||||
// Our own move. The session id here is half of what
|
||||
// a voice connection needs; VOICE_SERVER_UPDATE
|
||||
// carries the other half.
|
||||
|
||||
@@ -70,6 +70,7 @@ pub mod videorx;
|
||||
pub mod search;
|
||||
pub mod send;
|
||||
pub mod stickers;
|
||||
pub mod soundboard;
|
||||
|
||||
pub use calls::*;
|
||||
pub use commands::*;
|
||||
|
||||
@@ -0,0 +1,320 @@
|
||||
//! The soundboard: short sounds anybody in a voice channel can set off.
|
||||
//!
|
||||
//! Discord does this in the clients, not the voice server. Playing one is a
|
||||
//! REST call; everybody in the channel is then told by the gateway
|
||||
//! (`VOICE_CHANNEL_EFFECT_SEND`) which sound it was, and each client fetches
|
||||
//! the file and plays it itself. So hearing them here means fetching, decoding
|
||||
//! and mixing them into the call - which also puts them in what the echo
|
||||
//! canceller listens for, so a sound played out of the speakers is not sent
|
||||
//! back into the call.
|
||||
//!
|
||||
//! What can be played: Discord's default sounds, from REST, and each guild's
|
||||
//! own, asked of the gateway (op 31) when a voice channel is joined. A guild's
|
||||
//! sounds can be played in that guild; anywhere with Nitro.
|
||||
|
||||
use super::*;
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Sound {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
/// Discord's own level for it, 0 to 1.
|
||||
pub volume: f32,
|
||||
/// A Unicode emoji, or a custom one's id.
|
||||
pub emoji_name: Option<String>,
|
||||
pub emoji_id: Option<String>,
|
||||
/// Absent for a default sound.
|
||||
pub guild_id: Option<String>,
|
||||
}
|
||||
|
||||
pub fn read_sound(v: &Value) -> Option<Sound> {
|
||||
if v["available"].as_bool() == Some(false) {
|
||||
return None;
|
||||
}
|
||||
Some(Sound {
|
||||
id: v["sound_id"].as_str()?.to_string(),
|
||||
name: v["name"].as_str().unwrap_or("sound").to_string(),
|
||||
volume: v["volume"].as_f64().unwrap_or(1.0) as f32,
|
||||
emoji_name: v["emoji_name"].as_str().filter(|s| !s.is_empty()).map(str::to_string),
|
||||
emoji_id: v["emoji_id"].as_str().map(str::to_string),
|
||||
guild_id: v["guild_id"].as_str().map(str::to_string),
|
||||
})
|
||||
}
|
||||
|
||||
fn defaults() -> &'static Mutex<Option<Vec<Sound>>> {
|
||||
static DEFAULTS: OnceLock<Mutex<Option<Vec<Sound>>>> = OnceLock::new();
|
||||
DEFAULTS.get_or_init(Default::default)
|
||||
}
|
||||
|
||||
/// Account and guild to that guild's sounds.
|
||||
fn guilds() -> &'static Mutex<HashMap<(String, String), Vec<Sound>>> {
|
||||
static GUILDS: OnceLock<Mutex<HashMap<(String, String), Vec<Sound>>>> = OnceLock::new();
|
||||
GUILDS.get_or_init(Default::default)
|
||||
}
|
||||
|
||||
/// A guild's whole list, as `SOUNDBOARD_SOUNDS` or
|
||||
/// `GUILD_SOUNDBOARD_SOUNDS_UPDATE` carry it.
|
||||
pub fn note_guild_sounds(account_id: &str, guild_id: &str, list: &Value) {
|
||||
let sounds: Vec<Sound> = list.as_array().into_iter().flatten().filter_map(read_sound).collect();
|
||||
guilds().lock().unwrap().insert((account_id.to_string(), guild_id.to_string()), sounds);
|
||||
}
|
||||
|
||||
/// One sound added, changed or removed.
|
||||
pub fn note_sound_change(account_id: &str, dispatch: &str, d: &Value) {
|
||||
let Some(guild_id) = d["guild_id"].as_str() else { return };
|
||||
let Some(id) = d["sound_id"].as_str() else { return };
|
||||
let mut all = guilds().lock().unwrap();
|
||||
let list = all.entry((account_id.to_string(), guild_id.to_string())).or_default();
|
||||
list.retain(|s| s.id != id);
|
||||
if dispatch != "GUILD_SOUNDBOARD_SOUND_DELETE" {
|
||||
if let Some(sound) = read_sound(d) {
|
||||
list.push(sound);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Asks the gateway for these guilds' sounds; they arrive as
|
||||
/// `SOUNDBOARD_SOUNDS`, one dispatch per guild.
|
||||
pub fn request(state: &AppState, account_id: &str, guild_ids: &[String]) {
|
||||
if guild_ids.is_empty() {
|
||||
return;
|
||||
}
|
||||
if let Some(sender) = state.runtime.discord_gateway_sender(account_id) {
|
||||
let _ = sender.send(json!({ "op": 31, "d": { "guild_ids": guild_ids } }).to_string());
|
||||
}
|
||||
}
|
||||
|
||||
async fn default_sounds(token: &str) -> Vec<Sound> {
|
||||
if let Some(list) = defaults().lock().unwrap().clone() {
|
||||
return list;
|
||||
}
|
||||
let fetched = async {
|
||||
let resp = http_client_for(token)
|
||||
.get(format!("{API_BASE}/soundboard-default-sounds"))
|
||||
.header("Authorization", token)
|
||||
.send()
|
||||
.await
|
||||
.ok()?;
|
||||
resp.status().is_success().then_some(())?;
|
||||
let body: Value = resp.json().await.ok()?;
|
||||
Some(body.as_array().into_iter().flatten().filter_map(read_sound).collect::<Vec<_>>())
|
||||
}
|
||||
.await;
|
||||
match fetched {
|
||||
Some(list) => {
|
||||
*defaults().lock().unwrap() = Some(list.clone());
|
||||
list
|
||||
}
|
||||
None => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Everything this account could play, from the guild it is in: that guild's
|
||||
/// sounds first, then Discord's defaults, then the rest - each marked with
|
||||
/// whether it can be played here.
|
||||
pub async fn list(state: &AppState, account_id: &str, guild_id: Option<&str>, token: &str) -> Vec<Value> {
|
||||
let defaults = default_sounds(token).await;
|
||||
let nitro = state.runtime.emoji_unrestricted(account_id);
|
||||
// This guild's own, then Discord's defaults, then every other guild's.
|
||||
let mut here: Vec<(String, Vec<Sound>, bool)> = Vec::new();
|
||||
let mut elsewhere: Vec<(String, Vec<Sound>, bool)> = Vec::new();
|
||||
for ((a, g), sounds) in guilds().lock().unwrap().iter() {
|
||||
if a != account_id || sounds.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let name = state
|
||||
.runtime
|
||||
.get_buffer_group(&super::guilds::guild_group_id(account_id, g))
|
||||
.map(|grp| grp.name)
|
||||
.unwrap_or_else(|| "Server".to_string());
|
||||
if Some(g.as_str()) == guild_id {
|
||||
here.push((name, sounds.clone(), true));
|
||||
} else {
|
||||
elsewhere.push((name, sounds.clone(), nitro));
|
||||
}
|
||||
}
|
||||
elsewhere.sort_by_key(|(name, _, _)| name.to_lowercase());
|
||||
let groups: Vec<(String, Vec<Sound>, bool)> =
|
||||
here.into_iter().chain([("Default".to_string(), defaults, true)]).chain(elsewhere).collect();
|
||||
groups
|
||||
.into_iter()
|
||||
.flat_map(|(group, sounds, usable)| {
|
||||
sounds.into_iter().map(move |s| {
|
||||
json!({
|
||||
"id": s.id,
|
||||
"name": s.name,
|
||||
"group": group,
|
||||
"guildId": s.guild_id,
|
||||
"emojiName": s.emoji_name,
|
||||
"emojiId": s.emoji_id,
|
||||
"locked": !usable,
|
||||
})
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Sets a sound off in the channel this account is in. Everybody there -
|
||||
/// this account included - hears it when the gateway says it was played.
|
||||
pub async fn play(state: &AppState, account_id: &str, channel_id: &str, sound_id: &str, source_guild_id: Option<&str>) -> Result<()> {
|
||||
let cfg = state.accounts.get_discord(account_id).context("account not connected")?;
|
||||
let mut body = json!({ "sound_id": sound_id });
|
||||
if let Some(g) = source_guild_id {
|
||||
body["source_guild_id"] = json!(g);
|
||||
}
|
||||
let resp = send_write(
|
||||
http_client_for(&cfg.token)
|
||||
.post(format!("{API_BASE}/channels/{channel_id}/send-soundboard-sound"))
|
||||
.header("Authorization", &cfg.token)
|
||||
.json(&body),
|
||||
)
|
||||
.await
|
||||
.context("playing the sound")?;
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let text = resp.text().await.unwrap_or_default();
|
||||
bail!("Discord API error {status}: {text}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A sound somebody played in a channel this account is in: fetched,
|
||||
/// decoded and handed to the call to mix in.
|
||||
pub async fn heard(state: &AppState, account_id: &str, d: &Value) {
|
||||
// A string for a guild's sound and, for some default ones, a number.
|
||||
let Some(sound_id) = d["sound_id"].as_str().map(str::to_string).or_else(|| d["sound_id"].as_u64().map(|n| n.to_string())) else {
|
||||
return;
|
||||
};
|
||||
let sound_id = sound_id.as_str();
|
||||
let ours = state.voice.current_channel(account_id).map(|(_, c)| c);
|
||||
if ours.as_deref() != d["channel_id"].as_str() {
|
||||
return;
|
||||
}
|
||||
let volume = d["sound_volume"].as_f64().unwrap_or(1.0) as f32;
|
||||
let token = state.accounts.get_discord(account_id).map(|c| c.token).unwrap_or_default();
|
||||
match fetch(&token, sound_id).await.and_then(|bytes| decode(&bytes)) {
|
||||
Ok(mut pcm) => {
|
||||
crate::audio::apply_gain(&mut pcm, volume);
|
||||
state.voice.queue_effect(account_id, &pcm);
|
||||
}
|
||||
Err(e) => tracing::warn!("discord[{account_id}]: soundboard sound {sound_id}: {e:#}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn cache_dir() -> std::path::PathBuf {
|
||||
dirs::cache_dir()
|
||||
.unwrap_or_else(|| dirs::home_dir().unwrap_or_default().join(".cache"))
|
||||
.join("nobilis")
|
||||
.join("discord-sounds")
|
||||
}
|
||||
|
||||
/// The sound's file, kept: a sound id names one recording for good.
|
||||
async fn fetch(token: &str, id: &str) -> Result<Vec<u8>> {
|
||||
if !id.bytes().all(|b| b.is_ascii_digit()) {
|
||||
bail!("that is not a sound id");
|
||||
}
|
||||
let path = cache_dir().join(id);
|
||||
if let Ok(bytes) = tokio::fs::read(&path).await {
|
||||
return Ok(bytes);
|
||||
}
|
||||
let resp = http_client_for(token)
|
||||
.get(format!("https://cdn.discordapp.com/soundboard-sounds/{id}"))
|
||||
.send()
|
||||
.await
|
||||
.context("fetching the sound")?;
|
||||
if !resp.status().is_success() {
|
||||
bail!("Discord has no file for that sound ({})", resp.status());
|
||||
}
|
||||
let bytes = resp.bytes().await.context("reading the sound")?.to_vec();
|
||||
tokio::fs::create_dir_all(cache_dir()).await.ok();
|
||||
let _ = tokio::fs::write(&path, &bytes).await;
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
/// The longest a sound is allowed to be, as Discord allows: 5.2 seconds.
|
||||
const LONGEST_SECS: usize = 6;
|
||||
|
||||
/// MP3 or Ogg Vorbis to 48kHz stereo, interleaved, which is what the call
|
||||
/// plays. Resampled linearly: a soundboard sound is a few seconds of effect,
|
||||
/// not music anybody is listening to closely.
|
||||
pub fn decode(bytes: &[u8]) -> Result<Vec<i16>> {
|
||||
use symphonia::core::audio::SampleBuffer;
|
||||
use symphonia::core::codecs::DecoderOptions;
|
||||
use symphonia::core::formats::FormatOptions;
|
||||
use symphonia::core::io::MediaSourceStream;
|
||||
use symphonia::core::meta::MetadataOptions;
|
||||
use symphonia::core::probe::Hint;
|
||||
|
||||
let source = MediaSourceStream::new(Box::new(std::io::Cursor::new(bytes.to_vec())), Default::default());
|
||||
let probed = symphonia::default::get_probe()
|
||||
.format(&Hint::new(), source, &FormatOptions::default(), &MetadataOptions::default())
|
||||
.map_err(|e| anyhow!("not a sound this can read: {e}"))?;
|
||||
let mut format = probed.format;
|
||||
let track = format.default_track().context("the file has no audio in it")?;
|
||||
let track_id = track.id;
|
||||
let rate = track.codec_params.sample_rate.unwrap_or(48_000) as usize;
|
||||
let mut decoder = symphonia::default::get_codecs()
|
||||
.make(&track.codec_params, &DecoderOptions::default())
|
||||
.map_err(|e| anyhow!("no decoder for it: {e}"))?;
|
||||
|
||||
let mut stereo: Vec<f32> = Vec::new();
|
||||
while let Ok(packet) = format.next_packet() {
|
||||
if packet.track_id() != track_id {
|
||||
continue;
|
||||
}
|
||||
let Ok(decoded) = decoder.decode(&packet) else { continue };
|
||||
let spec = *decoded.spec();
|
||||
let channels = spec.channels.count().max(1);
|
||||
let mut buf = SampleBuffer::<f32>::new(decoded.capacity() as u64, spec);
|
||||
buf.copy_interleaved_ref(decoded);
|
||||
for frame in buf.samples().chunks(channels) {
|
||||
let (l, r) = if channels == 1 { (frame[0], frame[0]) } else { (frame[0], frame[1]) };
|
||||
stereo.push(l);
|
||||
stereo.push(r);
|
||||
}
|
||||
if stereo.len() / 2 > rate * LONGEST_SECS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(resample(&stereo, rate, 48_000))
|
||||
}
|
||||
|
||||
fn resample(stereo: &[f32], from: usize, to: usize) -> Vec<i16> {
|
||||
let frames = stereo.len() / 2;
|
||||
let out_frames = if from == to { frames } else { frames * to / from.max(1) };
|
||||
let mut out = Vec::with_capacity(out_frames * 2);
|
||||
for i in 0..out_frames {
|
||||
let pos = i as f64 * from as f64 / to as f64;
|
||||
let a = pos.floor() as usize;
|
||||
let b = (a + 1).min(frames.saturating_sub(1));
|
||||
let t = (pos - a as f64) as f32;
|
||||
for c in 0..2 {
|
||||
let x = stereo.get(a * 2 + c).copied().unwrap_or(0.0);
|
||||
let y = stereo.get(b * 2 + c).copied().unwrap_or(0.0);
|
||||
out.push(((x + (y - x) * t).clamp(-1.0, 1.0) * i16::MAX as f32) as i16);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_sound_is_read_with_its_emoji() {
|
||||
let v = json!({ "sound_id": "1", "name": "quack", "volume": 0.5, "emoji_name": "🦆", "guild_id": "9", "available": true });
|
||||
let s = read_sound(&v).unwrap();
|
||||
assert_eq!((s.id.as_str(), s.name.as_str(), s.volume), ("1", "quack", 0.5));
|
||||
assert_eq!(s.emoji_name.as_deref(), Some("🦆"));
|
||||
assert!(read_sound(&json!({ "sound_id": "2", "available": false })).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resampling_keeps_the_length_in_time() {
|
||||
let one_second_44k = vec![0.1f32; 44_100 * 2];
|
||||
assert_eq!(resample(&one_second_44k, 44_100, 48_000).len(), 48_000 * 2);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,7 +105,8 @@ pub struct VoiceState {
|
||||
suppressed: Mutex<std::collections::HashSet<String>>,
|
||||
/// The mute button's own state, so leaving the audience restores it
|
||||
/// rather than opening a microphone somebody had closed.
|
||||
mic_wanted_muted: Mutex<HashMap<String, bool>>,
|
||||
mic_wanted_muted: Mutex<HashMap<String, bool>>, /// Soundboard sounds waiting to be mixed into the call, as 48kHz stereo.
|
||||
effects: Mutex<HashMap<String, std::collections::VecDeque<i16>>>,
|
||||
}
|
||||
|
||||
/// Who is audible in one call, and how loudly.
|
||||
@@ -439,6 +440,30 @@ impl VoiceState {
|
||||
playbacks.get(account_id).map(|p| p.take_level())
|
||||
}
|
||||
|
||||
/// A soundboard sound to play into this account's call. Sounds that
|
||||
/// overlap are summed, as two people pressing at once would be heard.
|
||||
pub fn queue_effect(&self, account_id: &str, pcm: &[i16]) {
|
||||
let mut all = self.effects.lock().unwrap();
|
||||
let queue = all.entry(account_id.to_string()).or_default();
|
||||
for (i, sample) in pcm.iter().enumerate() {
|
||||
match queue.get_mut(i) {
|
||||
Some(slot) => *slot = slot.saturating_add(*sample),
|
||||
None => queue.push_back(*sample),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The next stretch of soundboard audio for this call, if any is playing.
|
||||
pub fn take_effect(&self, account_id: &str, samples: usize) -> Option<Vec<i16>> {
|
||||
let mut all = self.effects.lock().unwrap();
|
||||
let queue = all.get_mut(account_id)?;
|
||||
if queue.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let n = samples.min(queue.len());
|
||||
Some(queue.drain(..n).collect())
|
||||
}
|
||||
|
||||
/// Whether this account's camera is on.
|
||||
pub fn camera_on(&self, account_id: &str) -> bool {
|
||||
self.cameras.lock().unwrap().contains_key(account_id)
|
||||
@@ -601,6 +626,7 @@ pub fn set_camera(state: &AppState, account_id: &str, on: bool, quality: VideoQu
|
||||
} else {
|
||||
state.voice.cameras.lock().unwrap().remove(account_id);
|
||||
state.voice.suppressed.lock().unwrap().remove(account_id);
|
||||
state.voice.effects.lock().unwrap().remove(account_id);
|
||||
}
|
||||
conn.set_camera(on, quality);
|
||||
let (muted, deafened) = state.voice.flags(account_id);
|
||||
|
||||
@@ -1057,6 +1057,8 @@ async fn clock(
|
||||
let _ = encoder.set_bitrate(opus2::Bitrate::Bits(64_000));
|
||||
}
|
||||
let mut gate = Gate::default();
|
||||
let prefs = shared.state.voice_prefs.get();
|
||||
let mut processor = crate::apm::CallProcessor::new(prefs.echo_cancellation, prefs.noise_suppression);
|
||||
let mut sequence: u16 = rand::random();
|
||||
let mut timestamp: u32 = rand::random();
|
||||
let mut next_keepalive = Instant::now();
|
||||
@@ -1076,12 +1078,16 @@ async fn clock(
|
||||
for (ssrc, pcm) in &voices {
|
||||
shared.tracker.heard_from(*ssrc, super::voice::peak_of(pcm));
|
||||
}
|
||||
let prefs = shared.state.voice_prefs.get();
|
||||
processor.set(prefs.echo_cancellation, prefs.noise_suppression);
|
||||
// What the speakers play this tick, which is what the echo canceller
|
||||
// listens for in the microphone - silence when nothing is.
|
||||
let mut played: Vec<i16> = Vec::new();
|
||||
if let Some(playback) = playback.as_ref() {
|
||||
// Each person at the volume they were set to, before the mix, and
|
||||
// the whole call at its own after it. The speaking rings above
|
||||
// were fed before either: turning somebody down is not the same
|
||||
// as them going quiet.
|
||||
let prefs = shared.state.voice_prefs.get();
|
||||
let mut voices = voices;
|
||||
if !prefs.user_volumes.is_empty() {
|
||||
let owners = shared.owners.lock().unwrap();
|
||||
@@ -1091,21 +1097,35 @@ async fn clock(
|
||||
}
|
||||
}
|
||||
}
|
||||
// A soundboard sound is one more voice in the mix, at the call's
|
||||
// volume like everybody else.
|
||||
if let Some(effect) = shared.state.voice.take_effect(&shared.account, FRAME_SAMPLES) {
|
||||
voices.push((0, effect));
|
||||
}
|
||||
let refs: Vec<&[i16]> = voices.iter().map(|(_, pcm)| pcm.as_slice()).collect();
|
||||
let mut mixed = super::voice::mix(&refs);
|
||||
crate::audio::apply_gain(&mut mixed, prefs.output_volume);
|
||||
if !mixed.is_empty() {
|
||||
playback.push(&mixed);
|
||||
if !crate::audio::playback_muted() {
|
||||
played = mixed;
|
||||
}
|
||||
}
|
||||
}
|
||||
played.resize(FRAME_SAMPLES, 0);
|
||||
processor.render(&played);
|
||||
// Decoders for people who have gone quiet for good.
|
||||
if last_sweep.elapsed() > Duration::from_secs(30) {
|
||||
last_sweep = Instant::now();
|
||||
shared.speakers.lock().unwrap().retain(|_, jitter| jitter.last_arrival.elapsed() < Duration::from_secs(60));
|
||||
}
|
||||
|
||||
// Us.
|
||||
let frame = mic.as_ref().and_then(|mic| mic.take(FRAME_SAMPLES));
|
||||
// Us, with what the speakers played taken back out, before anything
|
||||
// decides from it whether we are talking.
|
||||
let mut frame = mic.as_ref().and_then(|mic| mic.take(FRAME_SAMPLES));
|
||||
if let Some(frame) = frame.as_mut() {
|
||||
processor.capture(frame);
|
||||
}
|
||||
let peak = frame.as_ref().map(|f| f.iter().fold(0.0f32, |m, s| m.max(s.abs()))).unwrap_or(0.0);
|
||||
let action = gate.step(peak);
|
||||
if let Some(speaking) = action.announce {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
mod accounts;
|
||||
mod apm;
|
||||
mod audio;
|
||||
mod backend;
|
||||
mod commands;
|
||||
|
||||
@@ -2670,6 +2670,47 @@ pub async fn dispatch(
|
||||
(Some(serde_json::to_value(prefs).unwrap()), None)
|
||||
}
|
||||
|
||||
// The soundboard of the call this account is in: what can be played,
|
||||
// and playing one.
|
||||
"listSoundboard" => {
|
||||
let Some(account_id) = p_str_opt(params, "accountId") else {
|
||||
return (None, Some("listSoundboard requires \"accountId\"".to_string()));
|
||||
};
|
||||
let token = state.accounts.get_discord(account_id).map(|c| c.token).unwrap_or_default();
|
||||
let guild = state.voice.current_channel(account_id).and_then(|(g, _)| g);
|
||||
(Some(serde_json::json!(backend::discord::soundboard::list(state, account_id, guild.as_deref(), &token).await)), None)
|
||||
}
|
||||
|
||||
"playSoundboard" => {
|
||||
let (account_id, sound_id) = match (p_str_opt(params, "accountId"), p_str_opt(params, "soundId")) {
|
||||
(Some(a), Some(s)) => (a, s),
|
||||
_ => return (None, Some("playSoundboard requires \"accountId\" and \"soundId\"".to_string())),
|
||||
};
|
||||
let Some((_, channel)) = state.voice.current_channel(account_id) else {
|
||||
return (None, Some("not in a call".to_string()));
|
||||
};
|
||||
match backend::discord::soundboard::play(state, account_id, &channel, sound_id, p_str_opt(params, "guildId")).await {
|
||||
Ok(()) => (Some(ok_node()), None),
|
||||
Err(e) => (None, Some(format!("{e:#}"))),
|
||||
}
|
||||
}
|
||||
|
||||
// Echo cancellation and noise suppression on a call's microphone.
|
||||
// Either or both; a call in progress follows within 20ms.
|
||||
"setVoiceProcessing" => {
|
||||
let echo = params.get("echoCancellation").and_then(|v| v.as_bool());
|
||||
let noise = params.get("noiseSuppression").and_then(|v| v.as_bool());
|
||||
let prefs = state.voice_prefs.update(|p| {
|
||||
if let Some(e) = echo {
|
||||
p.echo_cancellation = e;
|
||||
}
|
||||
if let Some(n) = noise {
|
||||
p.noise_suppression = n;
|
||||
}
|
||||
});
|
||||
(Some(serde_json::to_value(prefs).unwrap()), None)
|
||||
}
|
||||
|
||||
// A stage: asking to speak, and moving between the audience and the
|
||||
// speakers. Both are this account's own voice state in that guild.
|
||||
"setStageHand" | "setStageSpeaker" => {
|
||||
|
||||
Reference in New Issue
Block a user