From ffdf63563bab53a65b2a1a318f0f05e7917de002 Mon Sep 17 00:00:00 2001 From: Justice Almanzar Date: Tue, 15 Aug 2023 23:32:11 +0000 Subject: feat(plugins): Web/Vesktop AI Noise Suppression powered by RNNoise (#1477) Co-authored-by: V --- src/plugins/rnnoise.web/icons.tsx | 21 +++ src/plugins/rnnoise.web/index.tsx | 249 +++++++++++++++++++++++++++++++ src/plugins/rnnoise.web/styles.css | 29 ++++ src/utils/dependencies.ts | 4 + src/webpack/common/types/components.d.ts | 10 +- 5 files changed, 308 insertions(+), 5 deletions(-) create mode 100644 src/plugins/rnnoise.web/icons.tsx create mode 100644 src/plugins/rnnoise.web/index.tsx create mode 100644 src/plugins/rnnoise.web/styles.css (limited to 'src') diff --git a/src/plugins/rnnoise.web/icons.tsx b/src/plugins/rnnoise.web/icons.tsx new file mode 100644 index 0000000..8fda983 --- /dev/null +++ b/src/plugins/rnnoise.web/icons.tsx @@ -0,0 +1,21 @@ +/* + * Vencord, a modification for Discord's desktop app + * Copyright (c) 2023 Vendicated and contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +export const SupressionIcon = ({ enabled }: { enabled: boolean; }) => enabled + ? + : ; diff --git a/src/plugins/rnnoise.web/index.tsx b/src/plugins/rnnoise.web/index.tsx new file mode 100644 index 0000000..7117ca2 --- /dev/null +++ b/src/plugins/rnnoise.web/index.tsx @@ -0,0 +1,249 @@ +/* + * Vencord, a modification for Discord's desktop app + * Copyright (c) 2023 Vendicated and contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +import "./styles.css"; + +import { definePluginSettings } from "@api/Settings"; +import { classNameFactory } from "@api/Styles"; +import { Switch } from "@components/Switch"; +import { loadRnnoise, RnnoiseWorkletNode } from "@sapphi-red/web-noise-suppressor"; +import { Devs } from "@utils/constants"; +import { rnnoiseWasmSrc, rnnoiseWorkletSrc } from "@utils/dependencies"; +import { makeLazy } from "@utils/lazy"; +import { Logger } from "@utils/Logger"; +import { LazyComponent } from "@utils/react"; +import definePlugin from "@utils/types"; +import { findByCode } from "@webpack"; +import { FluxDispatcher, Popout, React } from "@webpack/common"; +import { MouseEvent, ReactNode } from "react"; + +import { SupressionIcon } from "./icons"; + +const RNNOISE_OPTION = "RNNOISE"; + +interface PanelButtonProps { + tooltipText: string; + icon: () => ReactNode; + onClick: (event: MouseEvent) => void; + tooltipClassName?: string; + disabled?: boolean; + shouldShow?: boolean; +} +const PanelButton = LazyComponent(() => findByCode("Masks.PANEL_BUTTON")); +const enum SpinnerType { + SpinningCircle = "spinningCircle", + ChasingDots = "chasingDots", + LowMotion = "lowMotion", + PulsingEllipsis = "pulsingEllipsis", + WanderingCubes = "wanderingCubes", +} +export interface SpinnerProps { + type: SpinnerType; + animated?: boolean; + className?: string; + itemClassName?: string; +} +const Spinner = LazyComponent(() => findByCode(".spinningCircleInner")); + +function createExternalStore(init: () => S) { + const subscribers = new Set<() => void>(); + let state = init(); + + return { + get: () => state, + set: (newStateGetter: (oldState: S) => S) => { + state = newStateGetter(state); + for (const cb of subscribers) cb(); + }, + use: () => { + return React.useSyncExternalStore(onStoreChange => { + subscribers.add(onStoreChange); + return () => subscribers.delete(onStoreChange); + }, () => state); + }, + } as const; +} + +const cl = classNameFactory("vc-rnnoise-"); + +const loadedStore = createExternalStore(() => ({ + isLoaded: false, + isLoading: false, + isError: false, +})); +const getRnnoiseWasm = makeLazy(() => { + loadedStore.set(s => ({ ...s, isLoading: true })); + return loadRnnoise({ + url: rnnoiseWasmSrc(), + simdUrl: rnnoiseWasmSrc(true), + }).then(buffer => { + // Check WASM magic number cus fetch doesnt throw on 4XX or 5XX + if (new DataView(buffer.slice(0, 4)).getUint32(0) !== 0x0061736D) throw buffer; + + loadedStore.set(s => ({ ...s, isLoaded: true })); + return buffer; + }).catch(error => { + if (error instanceof ArrayBuffer) error = new TextDecoder().decode(error); + logger.error("Failed to load RNNoise WASM:", error); + loadedStore.set(s => ({ ...s, isError: true })); + return null; + }).finally(() => { + loadedStore.set(s => ({ ...s, isLoading: false })); + }); +}); + +const logger = new Logger("RNNoise"); +const settings = definePluginSettings({}).withPrivateSettings<{ isEnabled: boolean; }>(); +const setEnabled = (enabled: boolean) => { + settings.store.isEnabled = enabled; + FluxDispatcher.dispatch({ type: "AUDIO_SET_NOISE_SUPPRESSION", enabled }); +}; + +function NoiseSupressionPopout() { + const { isEnabled } = settings.use(); + const { isLoading, isError } = loadedStore.use(); + const isWorking = isEnabled && !isError; + + return
+
+ Noise Supression +
+ {isLoading && } + +
+
+ Enable AI noise suppression! Make some noise—like becoming an air conditioner, or a vending machine fan—while speaking. Your friends will hear nothing but your beautiful voice ✨ +
+
; +} + +export default definePlugin({ + name: "AI Noise Suppression", + description: "Uses an open-source AI model (RNNoise) to remove background noise from your microphone", + authors: [Devs.Vap], + settings, + enabledByDefault: true, + + patches: [ + { + // Pass microphone stream to RNNoise + find: "window.webkitAudioContext", + replacement: { + match: /(?<=\i\.acquire=function\((\i)\)\{return )navigator\.mediaDevices\.getUserMedia\(\1\)(?=\})/, + replace: m => `${m}.then(stream => $self.connectRnnoise(stream))` + }, + }, + { + // Noise suppression button in call modal + find: "renderNoiseCancellation()", + replacement: { + match: /(?<=(\i)\.jsxs?.{0,70}children:\[)(?=\i\?\i\.renderNoiseCancellation\(\))/, + replace: (_, react) => `${react}.jsx($self.NoiseSupressionButton, {}),` + }, + }, + { + // Give noise suppression component a "shouldShow" prop + find: "Masks.PANEL_BUTTON", + replacement: { + match: /(?<==(\i)\.tooltipForceOpen.{0,100})(?=tooltipClassName:)/, + replace: (_, props) => `shouldShow: ${props}.shouldShow,` + } + }, + { + // Noise suppression option in voice settings + find: "Messages.USER_SETTINGS_NOISE_CANCELLATION_KRISP", + replacement: [{ + match: /(?<=(\i)=\i\?\i\.KRISP:\i.{1,20}?;)/, + replace: (_, option) => `if ($self.isEnabled()) ${option} = ${JSON.stringify(RNNOISE_OPTION)};`, + }, { + match: /(?=\i&&(\i)\.push\(\{name:(?:\i\.){1,2}Messages.USER_SETTINGS_NOISE_CANCELLATION_KRISP)/, + replace: (_, options) => `${options}.push({ name: "AI (RNNoise)", value: "${RNNOISE_OPTION}" });`, + }, { + match: /(?<=onChange:function\((\i)\)\{)(?=(?:\i\.){1,2}setNoiseCancellation)/, + replace: (_, option) => `$self.setEnabled(${option}.value === ${JSON.stringify(RNNOISE_OPTION)});`, + }], + }, + ], + + setEnabled, + isEnabled: () => settings.store.isEnabled, + async connectRnnoise(stream: MediaStream): Promise { + if (!settings.store.isEnabled) return stream; + + const audioCtx = new AudioContext(); + await audioCtx.audioWorklet.addModule(rnnoiseWorkletSrc); + + const rnnoiseWasm = await getRnnoiseWasm(); + if (!rnnoiseWasm) { + logger.warn("Failed to load RNNoise, noise suppression won't work"); + return stream; + } + + const rnnoise = new RnnoiseWorkletNode(audioCtx, { + wasmBinary: rnnoiseWasm, + maxChannels: 1, + }); + + const source = audioCtx.createMediaStreamSource(stream); + source.connect(rnnoise); + + const dest = audioCtx.createMediaStreamDestination(); + rnnoise.connect(dest); + + // Cleanup + const onEnded = () => { + rnnoise.disconnect(); + source.disconnect(); + audioCtx.close(); + rnnoise.destroy(); + }; + stream.addEventListener("inactive", onEnded, { once: true }); + + return dest.stream; + }, + NoiseSupressionButton(): ReactNode { + const { isEnabled } = settings.use(); + const { isLoading, isError } = loadedStore.use(); + + return } + spacing={8} + > + {(props, { isShown }) => ( +
+ +
} + /> + )} + ; + }, +}); diff --git a/src/plugins/rnnoise.web/styles.css b/src/plugins/rnnoise.web/styles.css new file mode 100644 index 0000000..7945c49 --- /dev/null +++ b/src/plugins/rnnoise.web/styles.css @@ -0,0 +1,29 @@ +.vc-rnnoise-popout { + background: var(--background-floating); + border-radius: 0.25em; + padding: 1em; + width: 16em; +} + +.vc-rnnoise-popout-heading { + color: var(--text-normal); + font-weight: 500; + display: flex; + justify-content: space-between; + align-items: center; + font-size: 1.1em; + margin-bottom: 1em; + gap: 0.5em; +} + +.vc-rnnoise-popout-desc { + color: var(--text-muted); + font-size: 0.9em; + display: flex; + align-items: center; + line-height: 1.5; +} + +.vc-rnnoise-tooltip { + text-align: center; +} diff --git a/src/utils/dependencies.ts b/src/utils/dependencies.ts index 3ef84b7..f1a5c26 100644 --- a/src/utils/dependencies.ts +++ b/src/utils/dependencies.ts @@ -79,5 +79,9 @@ const shikiWorkerDist = "https://unpkg.com/@vap/shiki-worker@0.0.8/dist"; export const shikiWorkerSrc = `${shikiWorkerDist}/${IS_DEV ? "index.js" : "index.min.js"}`; export const shikiOnigasmSrc = "https://unpkg.com/@vap/shiki@0.10.3/dist/onig.wasm"; +export const rnnoiseDist = "https://unpkg.com/@sapphi-red/web-noise-suppressor@0.3.3/dist"; +export const rnnoiseWasmSrc = (simd = false) => `${rnnoiseDist}/rnnoise${simd ? "_simd" : ""}.wasm`; +export const rnnoiseWorkletSrc = `${rnnoiseDist}/rnnoise/workletProcessor.js`; + // @ts-expect-error SHUT UP export const getStegCloak = makeLazy(() => import("https://unpkg.com/stegcloak-dist@1.0.0/index.js")); diff --git a/src/webpack/common/types/components.d.ts b/src/webpack/common/types/components.d.ts index d6d19fe..4b316aa 100644 --- a/src/webpack/common/types/components.d.ts +++ b/src/webpack/common/types/components.d.ts @@ -17,7 +17,7 @@ */ import type { Moment } from "moment"; -import type { ComponentType, CSSProperties, FunctionComponent, HtmlHTMLAttributes, HTMLProps, PropsWithChildren, PropsWithRef, ReactNode, Ref } from "react"; +import type { ComponentType, CSSProperties, FunctionComponent, HtmlHTMLAttributes, HTMLProps, KeyboardEvent, MouseEvent, PropsWithChildren, PropsWithRef, ReactNode, Ref } from "react"; export type TextVariant = "heading-sm/normal" | "heading-sm/medium" | "heading-sm/semibold" | "heading-sm/bold" | "heading-md/normal" | "heading-md/medium" | "heading-md/semibold" | "heading-md/bold" | "heading-lg/normal" | "heading-lg/medium" | "heading-lg/semibold" | "heading-lg/bold" | "heading-xl/normal" | "heading-xl/medium" | "heading-xl/bold" | "heading-xxl/normal" | "heading-xxl/medium" | "heading-xxl/bold" | "eyebrow" | "heading-deprecated-14/normal" | "heading-deprecated-14/medium" | "heading-deprecated-14/bold" | "text-xxs/normal" | "text-xxs/medium" | "text-xxs/semibold" | "text-xxs/bold" | "text-xs/normal" | "text-xs/medium" | "text-xs/semibold" | "text-xs/bold" | "text-sm/normal" | "text-sm/medium" | "text-sm/semibold" | "text-sm/bold" | "text-md/normal" | "text-md/medium" | "text-md/semibold" | "text-md/bold" | "text-lg/normal" | "text-lg/medium" | "text-lg/semibold" | "text-lg/bold" | "display-sm" | "display-md" | "display-lg" | "code"; export type FormTextTypes = Record<"DEFAULT" | "INPUT_PLACEHOLDER" | "DESCRIPTION" | "LABEL_BOLD" | "LABEL_SELECTED" | "LABEL_DESCRIPTOR" | "ERROR" | "SUCCESS", string>; @@ -338,16 +338,16 @@ export type Popout = ComponentType<{ thing: { "aria-controls": string; "aria-expanded": boolean; - onClick(event: MouseEvent): void; - onKeyDown(event: KeyboardEvent): void; - onMouseDown(event: MouseEvent): void; + onClick(event: MouseEvent): void; + onKeyDown(event: KeyboardEvent): void; + onMouseDown(event: MouseEvent): void; }, data: { isShown: boolean; position: string; } ): ReactNode; - shouldShow: boolean; + shouldShow?: boolean; renderPopout(args: { closePopout(): void; isPositioned: boolean; -- cgit