feat(plugins): Web/Vesktop AI Noise Suppression powered by RNNoise (#1477)

Co-authored-by: V <vendicated@riseup.net>

5 files changed, 308 insertions, 5 deletions

diff --git a/src/plugins/rnnoise.web/icons.tsx b/src/plugins/rnnoise.web/icons.tsx
new file mode 100644
index 0000000..8fda983
--- /dev/null
+++ b/src/plugins/rnnoise.web/icons.tsx
@@ -0,0 +1,21 @@
+/*
+ * Vencord, a modification for Discord's desktop app
+ * Copyright (c) 2023 Vendicated and contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+export const SupressionIcon = ({ enabled }: { enabled: boolean; }) => enabled
+    ? <svg aria-hidden="true" role="img" width="20" height="20" viewBox="0 0 24 24"><path d="M10.889 4C10.889 3.44772 11.3367 3 11.889 3H12.1112C12.6635 3 13.1112 3.44772 13.1112 4V20C13.1112 20.5523 12.6635 21 12.1112 21H11.889C11.3367 21 10.889 20.5523 10.889 20V4Z" fill="currentColor"></path><path d="M6.44439 6.25C6.44439 5.69772 6.89211 5.25 7.44439 5.25H7.66661C8.2189 5.25 8.66661 5.69772 8.66661 6.25V17.75C8.66661 18.3023 8.2189 18.75 7.66661 18.75H7.44439C6.89211 18.75 6.44439 18.3023 6.44439 17.75V6.25Z" fill="currentColor"></path><path d="M3.22222 15.375C3.77451 15.375 4.22222 14.9273 4.22222 14.375L4.22222 9.625C4.22222 9.07272 3.77451 8.625 3.22222 8.625H3C2.44772 8.625 2 9.07272 2 9.625V14.375C2 14.9273 2.44772 15.375 3 15.375H3.22222Z" fill="currentColor"></path><path d="M22.0001 13.25C22.0001 13.8023 21.5523 14.25 21.0001 14.25H20.7778C20.2255 14.25 19.7778 13.8023 19.7778 13.25V10.75C19.7778 10.1977 20.2255 9.75 20.7778 9.75H21.0001C21.5523 9.75 22.0001 10.1977 22.0001 10.75V13.25Z" fill="currentColor"></path><path d="M16.3333 7.5C15.781 7.5 15.3333 7.94772 15.3333 8.5V15.5C15.3333 16.0523 15.781 16.5 16.3333 16.5H16.5555C17.1078 16.5 17.5555 16.0523 17.5555 15.5V8.5C17.5555 7.94772 17.1078 7.5 16.5555 7.5H16.3333Z" fill="currentColor"></path></svg>
+    : <svg aria-hidden="true" role="img" width="20" height="20" viewBox="0 0 48 48"><path d="M30.6666 24.9644L35.1111 20.5199V31C35.1111 32.1046 34.2156 33 33.1111 33H32.6666C31.562 33 30.6666 32.1046 30.6666 31V24.9644Z" fill="currentColor"></path><path d="M26.2224 14.1463V8C26.2224 6.89543 25.327 6 24.2224 6H23.7779C22.6734 6 21.7779 6.89543 21.7779 8V18.5907L26.2224 14.1463Z" fill="currentColor"></path><path d="M21.7779 33.8543L21.9254 33.7056L26.2224 29.4086V40C26.2224 41.1046 25.327 42 24.2224 42H23.7779C22.6734 42 21.7779 41.1046 21.7779 40V33.8543Z" fill="currentColor"></path><path d="M17.3332 23.0354L12.8888 27.4799V12.5C12.8888 11.3954 13.7842 10.5 14.8888 10.5H15.3332C16.4378 10.5 17.3332 11.3954 17.3332 12.5V23.0354Z" fill="currentColor"></path><path d="M8.44445 28.75C8.44445 29.8546 7.54902 30.75 6.44445 30.75H6C4.89543 30.75 4 29.8546 4 28.75V19.25C4 18.1454 4.89543 17.25 6 17.25H6.44445C7.54902 17.25 8.44445 18.1454 8.44445 19.25L8.44445 28.75Z" fill="currentColor"></path><path d="M44.0001 26.5C44.0001 27.6046 43.1047 28.5 42.0001 28.5H41.5557C40.4511 28.5 39.5557 27.6046 39.5557 26.5V21.5C39.5557 20.3954 40.4511 19.5 41.5557 19.5H42.0001C43.1047 19.5 44.0001 20.3954 44.0001 21.5V26.5Z" fill="currentColor"></path><path d="M42 8.54L39.46 6L6 39.46L8.54 42L16.92 33.64L19.38 31.16L22.7 27.84L29.98 20.56L42 8.54Z" fill="currentColor"></path></svg>;
diff --git a/src/plugins/rnnoise.web/index.tsx b/src/plugins/rnnoise.web/index.tsx
new file mode 100644
index 0000000..7117ca2
--- /dev/null
+++ b/src/plugins/rnnoise.web/index.tsx
@@ -0,0 +1,249 @@
+/*
+ * Vencord, a modification for Discord's desktop app
+ * Copyright (c) 2023 Vendicated and contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import "./styles.css";
+
+import { definePluginSettings } from "@api/Settings";
+import { classNameFactory } from "@api/Styles";
+import { Switch } from "@components/Switch";
+import { loadRnnoise, RnnoiseWorkletNode } from "@sapphi-red/web-noise-suppressor";
+import { Devs } from "@utils/constants";
+import { rnnoiseWasmSrc, rnnoiseWorkletSrc } from "@utils/dependencies";
+import { makeLazy } from "@utils/lazy";
+import { Logger } from "@utils/Logger";
+import { LazyComponent } from "@utils/react";
+import definePlugin from "@utils/types";
+import { findByCode } from "@webpack";
+import { FluxDispatcher, Popout, React } from "@webpack/common";
+import { MouseEvent, ReactNode } from "react";
+
+import { SupressionIcon } from "./icons";
+
+const RNNOISE_OPTION = "RNNOISE";
+
+interface PanelButtonProps {
+    tooltipText: string;
+    icon: () => ReactNode;
+    onClick: (event: MouseEvent<HTMLElement>) => void;
+    tooltipClassName?: string;
+    disabled?: boolean;
+    shouldShow?: boolean;
+}
+const PanelButton = LazyComponent<PanelButtonProps>(() => findByCode("Masks.PANEL_BUTTON"));
+const enum SpinnerType {
+    SpinningCircle = "spinningCircle",
+    ChasingDots = "chasingDots",
+    LowMotion = "lowMotion",
+    PulsingEllipsis = "pulsingEllipsis",
+    WanderingCubes = "wanderingCubes",
+}
+export interface SpinnerProps {
+    type: SpinnerType;
+    animated?: boolean;
+    className?: string;
+    itemClassName?: string;
+}
+const Spinner = LazyComponent<SpinnerProps>(() => findByCode(".spinningCircleInner"));
+
+function createExternalStore<S>(init: () => S) {
+    const subscribers = new Set<() => void>();
+    let state = init();
+
+    return {
+        get: () => state,
+        set: (newStateGetter: (oldState: S) => S) => {
+            state = newStateGetter(state);
+            for (const cb of subscribers) cb();
+        },
+        use: () => {
+            return React.useSyncExternalStore<S>(onStoreChange => {
+                subscribers.add(onStoreChange);
+                return () => subscribers.delete(onStoreChange);
+            }, () => state);
+        },
+    } as const;
+}
+
+const cl = classNameFactory("vc-rnnoise-");
+
+const loadedStore = createExternalStore(() => ({
+    isLoaded: false,
+    isLoading: false,
+    isError: false,
+}));
+const getRnnoiseWasm = makeLazy(() => {
+    loadedStore.set(s => ({ ...s, isLoading: true }));
+    return loadRnnoise({
+        url: rnnoiseWasmSrc(),
+        simdUrl: rnnoiseWasmSrc(true),
+    }).then(buffer => {
+        // Check WASM magic number cus fetch doesnt throw on 4XX or 5XX
+        if (new DataView(buffer.slice(0, 4)).getUint32(0) !== 0x0061736D) throw buffer;
+
+        loadedStore.set(s => ({ ...s, isLoaded: true }));
+        return buffer;
+    }).catch(error => {
+        if (error instanceof ArrayBuffer) error = new TextDecoder().decode(error);
+        logger.error("Failed to load RNNoise WASM:", error);
+        loadedStore.set(s => ({ ...s, isError: true }));
+        return null;
+    }).finally(() => {
+        loadedStore.set(s => ({ ...s, isLoading: false }));
+    });
+});
+
+const logger = new Logger("RNNoise");
+const settings = definePluginSettings({}).withPrivateSettings<{ isEnabled: boolean; }>();
+const setEnabled = (enabled: boolean) => {
+    settings.store.isEnabled = enabled;
+    FluxDispatcher.dispatch({ type: "AUDIO_SET_NOISE_SUPPRESSION", enabled });
+};
+
+function NoiseSupressionPopout() {
+    const { isEnabled } = settings.use();
+    const { isLoading, isError } = loadedStore.use();
+    const isWorking = isEnabled && !isError;
+
+    return <div className={cl("popout")}>
+        <div className={cl("popout-heading")}>
+            <span>Noise Supression</span>
+            <div style={{ flex: 1 }} />
+            {isLoading && <Spinner type={SpinnerType.PulsingEllipsis} />}
+            <Switch checked={isWorking} onChange={setEnabled} disabled={isError} />
+        </div>
+        <div className={cl("popout-desc")}>
+            Enable AI noise suppression! Make some noise&mdash;like becoming an air conditioner, or a vending machine fan&mdash;while speaking. Your friends will hear nothing but your beautiful voice ✨
+        </div>
+    </div>;
+}
+
+export default definePlugin({
+    name: "AI Noise Suppression",
+    description: "Uses an open-source AI model (RNNoise) to remove background noise from your microphone",
+    authors: [Devs.Vap],
+    settings,
+    enabledByDefault: true,
+
+    patches: [
+        {
+            // Pass microphone stream to RNNoise
+            find: "window.webkitAudioContext",
+            replacement: {
+                match: /(?<=\i\.acquire=function\((\i)\)\{return )navigator\.mediaDevices\.getUserMedia\(\1\)(?=\})/,
+                replace: m => `${m}.then(stream => $self.connectRnnoise(stream))`
+            },
+        },
+        {
+            // Noise suppression button in call modal
+            find: "renderNoiseCancellation()",
+            replacement: {
+                match: /(?<=(\i)\.jsxs?.{0,70}children:\[)(?=\i\?\i\.renderNoiseCancellation\(\))/,
+                replace: (_, react) => `${react}.jsx($self.NoiseSupressionButton, {}),`
+            },
+        },
+        {
+            // Give noise suppression component a "shouldShow" prop
+            find: "Masks.PANEL_BUTTON",
+            replacement: {
+                match: /(?<==(\i)\.tooltipForceOpen.{0,100})(?=tooltipClassName:)/,
+                replace: (_, props) => `shouldShow: ${props}.shouldShow,`
+            }
+        },
+        {
+            // Noise suppression option in voice settings
+            find: "Messages.USER_SETTINGS_NOISE_CANCELLATION_KRISP",
+            replacement: [{
+                match: /(?<=(\i)=\i\?\i\.KRISP:\i.{1,20}?;)/,
+                replace: (_, option) => `if ($self.isEnabled()) ${option} = ${JSON.stringify(RNNOISE_OPTION)};`,
+            }, {
+                match: /(?=\i&&(\i)\.push\(\{name:(?:\i\.){1,2}Messages.USER_SETTINGS_NOISE_CANCELLATION_KRISP)/,
+                replace: (_, options) => `${options}.push({ name: "AI (RNNoise)", value: "${RNNOISE_OPTION}" });`,
+            }, {
+                match: /(?<=onChange:function\((\i)\)\{)(?=(?:\i\.){1,2}setNoiseCancellation)/,
+                replace: (_, option) => `$self.setEnabled(${option}.value === ${JSON.stringify(RNNOISE_OPTION)});`,
+            }],
+        },
+    ],
+
+    setEnabled,
+    isEnabled: () => settings.store.isEnabled,
+    async connectRnnoise(stream: MediaStream): Promise<MediaStream> {
+        if (!settings.store.isEnabled) return stream;
+
+        const audioCtx = new AudioContext();
+        await audioCtx.audioWorklet.addModule(rnnoiseWorkletSrc);
+
+        const rnnoiseWasm = await getRnnoiseWasm();
+        if (!rnnoiseWasm) {
+            logger.warn("Failed to load RNNoise, noise suppression won't work");
+            return stream;
+        }
+
+        const rnnoise = new RnnoiseWorkletNode(audioCtx, {
+            wasmBinary: rnnoiseWasm,
+            maxChannels: 1,
+        });
+
+        const source = audioCtx.createMediaStreamSource(stream);
+        source.connect(rnnoise);
+
+        const dest = audioCtx.createMediaStreamDestination();
+        rnnoise.connect(dest);
+
+        // Cleanup
+        const onEnded = () => {
+            rnnoise.disconnect();
+            source.disconnect();
+            audioCtx.close();
+            rnnoise.destroy();
+        };
+        stream.addEventListener("inactive", onEnded, { once: true });
+
+        return dest.stream;
+    },
+    NoiseSupressionButton(): ReactNode {
+        const { isEnabled } = settings.use();
+        const { isLoading, isError } = loadedStore.use();
+
+        return <Popout
+            key="rnnoise-popout"
+            align="center"
+            animation={Popout.Animation.TRANSLATE}
+            autoInvert={true}
+            nudgeAlignIntoViewport={true}
+            position="top"
+            renderPopout={() => <NoiseSupressionPopout />}
+            spacing={8}
+        >
+            {(props, { isShown }) => (
+                <PanelButton
+                    {...props}
+                    tooltipText="Noise Suppression powered by RNNoise"
+                    tooltipClassName={cl("tooltip")}
+                    shouldShow={!isShown}
+                    icon={() => <div style={{
+                        color: isError ? "var(--status-danger)" : "inherit",
+                        opacity: isLoading ? 0.5 : 1,
+                    }}>
+                        <SupressionIcon enabled={isEnabled} />
+                    </div>}
+                />
+            )}
+        </Popout>;
+    },
+});
diff --git a/src/plugins/rnnoise.web/styles.css b/src/plugins/rnnoise.web/styles.css
new file mode 100644
index 0000000..7945c49
--- /dev/null
+++ b/src/plugins/rnnoise.web/styles.css
@@ -0,0 +1,29 @@
+.vc-rnnoise-popout {
+    background: var(--background-floating);
+    border-radius: 0.25em;
+    padding: 1em;
+    width: 16em;
+}
+
+.vc-rnnoise-popout-heading {
+    color: var(--text-normal);
+    font-weight: 500;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    font-size: 1.1em;
+    margin-bottom: 1em;
+    gap: 0.5em;
+}
+
+.vc-rnnoise-popout-desc {
+    color: var(--text-muted);
+    font-size: 0.9em;
+    display: flex;
+    align-items: center;
+    line-height: 1.5;
+}
+
+.vc-rnnoise-tooltip {
+    text-align: center;
+}
diff --git a/src/utils/dependencies.ts b/src/utils/dependencies.ts
index 3ef84b7..f1a5c26 100644
--- a/src/utils/dependencies.ts
+++ b/src/utils/dependencies.ts
@@ -79,5 +79,9 @@ const shikiWorkerDist = "https://unpkg.com/@vap/shiki-worker@0.0.8/dist";
 export const shikiWorkerSrc = `${shikiWorkerDist}/${IS_DEV ? "index.js" : "index.min.js"}`;
 export const shikiOnigasmSrc = "https://unpkg.com/@vap/shiki@0.10.3/dist/onig.wasm";
 
+export const rnnoiseDist = "https://unpkg.com/@sapphi-red/web-noise-suppressor@0.3.3/dist";
+export const rnnoiseWasmSrc = (simd = false) => `${rnnoiseDist}/rnnoise${simd ? "_simd" : ""}.wasm`;
+export const rnnoiseWorkletSrc = `${rnnoiseDist}/rnnoise/workletProcessor.js`;
+
 // @ts-expect-error SHUT UP
 export const getStegCloak = makeLazy(() => import("https://unpkg.com/stegcloak-dist@1.0.0/index.js"));
diff --git a/src/webpack/common/types/components.d.ts b/src/webpack/common/types/components.d.ts
index d6d19fe..4b316aa 100644
--- a/src/webpack/common/types/components.d.ts
+++ b/src/webpack/common/types/components.d.ts
@@ -17,7 +17,7 @@
 */
 
 import type { Moment } from "moment";
-import type { ComponentType, CSSProperties, FunctionComponent, HtmlHTMLAttributes, HTMLProps, PropsWithChildren, PropsWithRef, ReactNode, Ref } from "react";
+import type { ComponentType, CSSProperties, FunctionComponent, HtmlHTMLAttributes, HTMLProps, KeyboardEvent, MouseEvent, PropsWithChildren, PropsWithRef, ReactNode, Ref } from "react";
 
 export type TextVariant = "heading-sm/normal" | "heading-sm/medium" | "heading-sm/semibold" | "heading-sm/bold" | "heading-md/normal" | "heading-md/medium" | "heading-md/semibold" | "heading-md/bold" | "heading-lg/normal" | "heading-lg/medium" | "heading-lg/semibold" | "heading-lg/bold" | "heading-xl/normal" | "heading-xl/medium" | "heading-xl/bold" | "heading-xxl/normal" | "heading-xxl/medium" | "heading-xxl/bold" | "eyebrow" | "heading-deprecated-14/normal" | "heading-deprecated-14/medium" | "heading-deprecated-14/bold" | "text-xxs/normal" | "text-xxs/medium" | "text-xxs/semibold" | "text-xxs/bold" | "text-xs/normal" | "text-xs/medium" | "text-xs/semibold" | "text-xs/bold" | "text-sm/normal" | "text-sm/medium" | "text-sm/semibold" | "text-sm/bold" | "text-md/normal" | "text-md/medium" | "text-md/semibold" | "text-md/bold" | "text-lg/normal" | "text-lg/medium" | "text-lg/semibold" | "text-lg/bold" | "display-sm" | "display-md" | "display-lg" | "code";
 export type FormTextTypes = Record<"DEFAULT" | "INPUT_PLACEHOLDER" | "DESCRIPTION" | "LABEL_BOLD" | "LABEL_SELECTED" | "LABEL_DESCRIPTOR" | "ERROR" | "SUCCESS", string>;
@@ -338,16 +338,16 @@ export type Popout = ComponentType<{
         thing: {
             "aria-controls": string;
             "aria-expanded": boolean;
-            onClick(event: MouseEvent): void;
-            onKeyDown(event: KeyboardEvent): void;
-            onMouseDown(event: MouseEvent): void;
+            onClick(event: MouseEvent<HTMLElement>): void;
+            onKeyDown(event: KeyboardEvent<HTMLElement>): void;
+            onMouseDown(event: MouseEvent<HTMLElement>): void;
         },
         data: {
             isShown: boolean;
             position: string;
         }
     ): ReactNode;
-    shouldShow: boolean;
+    shouldShow?: boolean;
     renderPopout(args: {
         closePopout(): void;
         isPositioned: boolean;