Merge matrix-react-sdk into element-web

Merge remote-tracking branch 'repomerge/t3chguy/repomerge' into t3chguy/repo-merge

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

test/unit-tests/audio/Playback-test.ts

@@ -0,0 +1,163 @@
+/*
+Copyright 2024 New Vector Ltd.
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { mocked } from "jest-mock";
+import { logger } from "matrix-js-sdk/src/logger";
+
+import { createAudioContext, decodeOgg } from "../../../src/audio/compat";
+import { Playback, PlaybackState } from "../../../src/audio/Playback";
+
+jest.mock("../../../src/WorkerManager", () => ({
+    WorkerManager: jest.fn(() => ({
+        call: jest.fn().mockResolvedValue({ waveform: [0, 0, 1, 1] }),
+    })),
+}));
+
+jest.mock("../../../src/audio/compat", () => ({
+    createAudioContext: jest.fn(),
+    decodeOgg: jest.fn(),
+}));
+
+describe("Playback", () => {
+    const mockAudioBufferSourceNode = {
+        addEventListener: jest.fn(),
+        connect: jest.fn(),
+        start: jest.fn(),
+    };
+    const mockAudioContext = {
+        decodeAudioData: jest.fn(),
+        suspend: jest.fn(),
+        resume: jest.fn(),
+        createBufferSource: jest.fn().mockReturnValue(mockAudioBufferSourceNode),
+        currentTime: 1337,
+    };
+
+    const mockAudioBuffer = {
+        duration: 99,
+        getChannelData: jest.fn(),
+    };
+
+    const mockChannelData = new Float32Array();
+
+    beforeEach(() => {
+        jest.spyOn(logger, "error").mockRestore();
+        mockAudioBuffer.getChannelData.mockClear().mockReturnValue(mockChannelData);
+        mockAudioContext.decodeAudioData.mockReset().mockImplementation((_b, callback) => callback(mockAudioBuffer));
+        mockAudioContext.resume.mockClear().mockResolvedValue(undefined);
+        mockAudioContext.suspend.mockClear().mockResolvedValue(undefined);
+        mocked(decodeOgg).mockClear().mockResolvedValue(new ArrayBuffer(1));
+        mocked(createAudioContext).mockReturnValue(mockAudioContext as unknown as AudioContext);
+    });
+
+    it("initialises correctly", () => {
+        const buffer = new ArrayBuffer(8);
+
+        const playback = new Playback(buffer);
+        playback.clockInfo.durationSeconds = mockAudioBuffer.duration;
+
+        expect(playback.sizeBytes).toEqual(8);
+        expect(playback.clockInfo).toBeTruthy();
+        expect(playback.liveData).toBe(playback.clockInfo.liveData);
+        expect(playback.timeSeconds).toBe(1337 % 99);
+        expect(playback.currentState).toEqual(PlaybackState.Decoding);
+    });
+
+    it("toggles playback on from stopped state", async () => {
+        const buffer = new ArrayBuffer(8);
+        const playback = new Playback(buffer);
+        await playback.prepare();
+        // state is Stopped
+        await playback.toggle();
+
+        expect(mockAudioBufferSourceNode.start).toHaveBeenCalled();
+        expect(mockAudioContext.resume).toHaveBeenCalled();
+        expect(playback.currentState).toEqual(PlaybackState.Playing);
+    });
+
+    it("toggles playback to paused from playing state", async () => {
+        const buffer = new ArrayBuffer(8);
+        const playback = new Playback(buffer);
+        await playback.prepare();
+        await playback.toggle();
+        expect(playback.currentState).toEqual(PlaybackState.Playing);
+
+        await playback.toggle();
+
+        expect(mockAudioContext.suspend).toHaveBeenCalled();
+        expect(playback.currentState).toEqual(PlaybackState.Paused);
+    });
+
+    it("stop playbacks", async () => {
+        const buffer = new ArrayBuffer(8);
+        const playback = new Playback(buffer);
+        await playback.prepare();
+        await playback.toggle();
+        expect(playback.currentState).toEqual(PlaybackState.Playing);
+
+        await playback.stop();
+
+        expect(mockAudioContext.suspend).toHaveBeenCalled();
+        expect(playback.currentState).toEqual(PlaybackState.Stopped);
+    });
+
+    describe("prepare()", () => {
+        it("decodes audio data when not greater than 5mb", async () => {
+            const buffer = new ArrayBuffer(8);
+
+            const playback = new Playback(buffer);
+
+            await playback.prepare();
+
+            expect(mockAudioContext.decodeAudioData).toHaveBeenCalledTimes(1);
+            expect(mockAudioBuffer.getChannelData).toHaveBeenCalledWith(0);
+
+            // clock was updated
+            expect(playback.clockInfo.durationSeconds).toEqual(mockAudioBuffer.duration);
+            expect(playback.durationSeconds).toEqual(mockAudioBuffer.duration);
+
+            expect(playback.currentState).toEqual(PlaybackState.Stopped);
+        });
+
+        it("tries to decode ogg when decodeAudioData fails", async () => {
+            // stub logger to keep console clean from expected error
+            jest.spyOn(logger, "error").mockReturnValue(undefined);
+            jest.spyOn(logger, "warn").mockReturnValue(undefined);
+
+            const buffer = new ArrayBuffer(8);
+            const decodingError = new Error("test");
+            mockAudioContext.decodeAudioData
+                .mockImplementationOnce((_b, _callback, error) => error(decodingError))
+                .mockImplementationOnce((_b, callback) => callback(mockAudioBuffer));
+
+            const playback = new Playback(buffer);
+
+            await playback.prepare();
+
+            expect(mockAudioContext.decodeAudioData).toHaveBeenCalledTimes(2);
+            expect(decodeOgg).toHaveBeenCalled();
+
+            // clock was updated
+            expect(playback.clockInfo.durationSeconds).toEqual(mockAudioBuffer.duration);
+            expect(playback.durationSeconds).toEqual(mockAudioBuffer.duration);
+
+            expect(playback.currentState).toEqual(PlaybackState.Stopped);
+        });
+
+        it("does not try to re-decode audio", async () => {
+            const buffer = new ArrayBuffer(8);
+            const playback = new Playback(buffer);
+            await playback.prepare();
+            expect(playback.currentState).toEqual(PlaybackState.Stopped);
+
+            await playback.prepare();
+
+            // only called once in first prepare
+            expect(mockAudioContext.decodeAudioData).toHaveBeenCalledTimes(1);
+        });
+    });
+});

test/unit-tests/audio/VoiceMessageRecording-test.ts

@@ -0,0 +1,211 @@
+/*
+Copyright 2024 New Vector Ltd.
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { mocked } from "jest-mock";
+import { UploadOpts, MatrixClient } from "matrix-js-sdk/src/matrix";
+import { EncryptedFile } from "matrix-js-sdk/src/types";
+
+import { createVoiceMessageRecording, VoiceMessageRecording } from "../../../src/audio/VoiceMessageRecording";
+import { RecordingState, VoiceRecording } from "../../../src/audio/VoiceRecording";
+import { uploadFile } from "../../../src/ContentMessages";
+import { stubClient } from "../../test-utils";
+import { Playback } from "../../../src/audio/Playback";
+
+jest.mock("../../../src/ContentMessages", () => ({
+    uploadFile: jest.fn(),
+}));
+
+jest.mock("../../../src/audio/Playback", () => ({
+    Playback: jest.fn(),
+}));
+
+describe("VoiceMessageRecording", () => {
+    const roomId = "!room:example.com";
+    const contentType = "test content type";
+    const durationSeconds = 23;
+    const testBuf = new Uint8Array([1, 2, 3]);
+    const testAmplitudes = [4, 5, 6];
+
+    let voiceRecording: VoiceRecording;
+    let voiceMessageRecording: VoiceMessageRecording;
+    let client: MatrixClient;
+
+    beforeEach(() => {
+        client = stubClient();
+        voiceRecording = {
+            contentType,
+            durationSeconds,
+            start: jest.fn().mockResolvedValue(undefined),
+            stop: jest.fn().mockResolvedValue(undefined),
+            on: jest.fn(),
+            off: jest.fn(),
+            emit: jest.fn(),
+            isRecording: true,
+            isSupported: true,
+            liveData: jest.fn(),
+            amplitudes: testAmplitudes,
+        } as unknown as VoiceRecording;
+        voiceMessageRecording = new VoiceMessageRecording(client, voiceRecording);
+    });
+
+    it("hasRecording should return false", () => {
+        expect(voiceMessageRecording.hasRecording).toBe(false);
+    });
+
+    it("createVoiceMessageRecording should return a VoiceMessageRecording", () => {
+        expect(createVoiceMessageRecording(client)).toBeInstanceOf(VoiceMessageRecording);
+    });
+
+    it("durationSeconds should return the VoiceRecording value", () => {
+        expect(voiceMessageRecording.durationSeconds).toBe(durationSeconds);
+    });
+
+    it("contentType should return the VoiceRecording value", () => {
+        expect(voiceMessageRecording.contentType).toBe(contentType);
+    });
+
+    it.each([true, false])("isRecording should return %s from VoiceRecording", (value: boolean) => {
+        // @ts-ignore
+        voiceRecording.isRecording = value;
+        expect(voiceMessageRecording.isRecording).toBe(value);
+    });
+
+    it.each([true, false])("isSupported should return %s from VoiceRecording", (value: boolean) => {
+        // @ts-ignore
+        voiceRecording.isSupported = value;
+        expect(voiceMessageRecording.isSupported).toBe(value);
+    });
+
+    it("should return liveData from VoiceRecording", () => {
+        expect(voiceMessageRecording.liveData).toBe(voiceRecording.liveData);
+    });
+
+    it("start should forward the call to VoiceRecording.start", async () => {
+        await voiceMessageRecording.start();
+        expect(voiceRecording.start).toHaveBeenCalled();
+    });
+
+    it("on should forward the call to VoiceRecording", () => {
+        const callback = () => {};
+        const result = voiceMessageRecording.on("test on", callback);
+        expect(voiceRecording.on).toHaveBeenCalledWith("test on", callback);
+        expect(result).toBe(voiceMessageRecording);
+    });
+
+    it("off should forward the call to VoiceRecording", () => {
+        const callback = () => {};
+        const result = voiceMessageRecording.off("test off", callback);
+        expect(voiceRecording.off).toHaveBeenCalledWith("test off", callback);
+        expect(result).toBe(voiceMessageRecording);
+    });
+
+    it("emit should forward the call to VoiceRecording", () => {
+        voiceMessageRecording.emit("test emit", 42);
+        expect(voiceRecording.emit).toHaveBeenCalledWith("test emit", 42);
+    });
+
+    it("upload should raise an error", async () => {
+        await expect(voiceMessageRecording.upload(roomId)).rejects.toThrow("No recording available to upload");
+    });
+
+    describe("when the first data has been received", () => {
+        const uploadUrl = "https://example.com/content123";
+        const encryptedFile = {} as unknown as EncryptedFile;
+
+        beforeEach(() => {
+            voiceRecording.onDataAvailable!(testBuf);
+        });
+
+        it("contentLength should return the buffer length", () => {
+            expect(voiceMessageRecording.contentLength).toBe(testBuf.length);
+        });
+
+        it("stop should return a copy of the data buffer", async () => {
+            const result = await voiceMessageRecording.stop();
+            expect(voiceRecording.stop).toHaveBeenCalled();
+            expect(result).toEqual(testBuf);
+        });
+
+        it("hasRecording should return true", () => {
+            expect(voiceMessageRecording.hasRecording).toBe(true);
+        });
+
+        describe("upload", () => {
+            let uploadFileClient: MatrixClient | null;
+            let uploadFileRoomId: string | null;
+            let uploadBlob: Blob | null;
+
+            beforeEach(() => {
+                uploadFileClient = null;
+                uploadFileRoomId = null;
+                uploadBlob = null;
+
+                mocked(uploadFile).mockImplementation(
+                    (
+                        matrixClient: MatrixClient,
+                        roomId: string,
+                        file: File | Blob,
+                        _progressHandler?: UploadOpts["progressHandler"],
+                    ): Promise<{ url?: string; file?: EncryptedFile }> => {
+                        uploadFileClient = matrixClient;
+                        uploadFileRoomId = roomId;
+                        uploadBlob = file;
+                        // @ts-ignore
+                        return Promise.resolve({
+                            url: uploadUrl,
+                            file: encryptedFile,
+                        });
+                    },
+                );
+            });
+
+            it("should upload the file and trigger the upload events", async () => {
+                const result = await voiceMessageRecording.upload(roomId);
+                expect(voiceRecording.emit).toHaveBeenNthCalledWith(1, RecordingState.Uploading);
+                expect(voiceRecording.emit).toHaveBeenNthCalledWith(2, RecordingState.Uploaded);
+
+                expect(result.mxc).toBe(uploadUrl);
+                expect(result.encrypted).toBe(encryptedFile);
+
+                expect(mocked(uploadFile)).toHaveBeenCalled();
+                expect(uploadFileClient).toBe(client);
+                expect(uploadFileRoomId).toBe(roomId);
+                expect(uploadBlob?.type).toBe(contentType);
+                const blobArray = await uploadBlob!.arrayBuffer();
+                expect(new Uint8Array(blobArray)).toEqual(testBuf);
+            });
+
+            it("should reuse the result", async () => {
+                const result1 = await voiceMessageRecording.upload(roomId);
+                const result2 = await voiceMessageRecording.upload(roomId);
+                expect(result1).toBe(result2);
+            });
+        });
+
+        describe("getPlayback", () => {
+            beforeEach(() => {
+                mocked(Playback).mockImplementation((buf: ArrayBuffer, seedWaveform): any => {
+                    expect(new Uint8Array(buf)).toEqual(testBuf);
+                    expect(seedWaveform).toEqual(testAmplitudes);
+                    return {} as Playback;
+                });
+            });
+
+            it("should return a Playback with the data", () => {
+                voiceMessageRecording.getPlayback();
+                expect(mocked(Playback)).toHaveBeenCalled();
+            });
+
+            it("should reuse the result", () => {
+                const playback1 = voiceMessageRecording.getPlayback();
+                const playback2 = voiceMessageRecording.getPlayback();
+                expect(playback1).toBe(playback2);
+            });
+        });
+    });
+});

test/unit-tests/audio/VoiceRecording-test.ts

@@ -0,0 +1,175 @@
+/*
+Copyright 2024 New Vector Ltd.
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { mocked } from "jest-mock";
+// @ts-ignore
+import Recorder from "opus-recorder/dist/recorder.min.js";
+
+import { VoiceRecording, voiceRecorderOptions, highQualityRecorderOptions } from "../../../src/audio/VoiceRecording";
+import { createAudioContext } from "../../..//src/audio/compat";
+import MediaDeviceHandler from "../../../src/MediaDeviceHandler";
+import { useMockMediaDevices } from "../../test-utils";
+
+jest.mock("opus-recorder/dist/recorder.min.js");
+const RecorderMock = mocked(Recorder);
+
+jest.mock("../../../src/audio/compat", () => ({
+    createAudioContext: jest.fn(),
+}));
+const createAudioContextMock = mocked(createAudioContext);
+
+jest.mock("../../../src/MediaDeviceHandler");
+const MediaDeviceHandlerMock = mocked(MediaDeviceHandler);
+
+/**
+ * The tests here are heavily using access to private props.
+ * While this is not so great, we can at lest test some behaviour easily this way.
+ */
+describe("VoiceRecording", () => {
+    let recording: VoiceRecording;
+    let recorderSecondsSpy: jest.SpyInstance;
+
+    const itShouldNotCallStop = () => {
+        it("should not call stop", () => {
+            expect(recording.stop).not.toHaveBeenCalled();
+        });
+    };
+
+    const simulateUpdate = (recorderSeconds: number) => {
+        beforeEach(() => {
+            recorderSecondsSpy.mockReturnValue(recorderSeconds);
+            // @ts-ignore
+            recording.processAudioUpdate(recorderSeconds);
+        });
+    };
+
+    beforeEach(() => {
+        useMockMediaDevices();
+        recording = new VoiceRecording();
+        // @ts-ignore
+        recording.observable = {
+            update: jest.fn(),
+            close: jest.fn(),
+        };
+        jest.spyOn(recording, "stop").mockImplementation();
+        recorderSecondsSpy = jest.spyOn(recording, "recorderSeconds", "get");
+    });
+
+    afterEach(() => {
+        jest.resetAllMocks();
+    });
+
+    describe("when starting a recording", () => {
+        beforeEach(() => {
+            const mockAudioContext = {
+                createMediaStreamSource: jest.fn().mockReturnValue({
+                    connect: jest.fn(),
+                    disconnect: jest.fn(),
+                }),
+                createScriptProcessor: jest.fn().mockReturnValue({
+                    connect: jest.fn(),
+                    disconnect: jest.fn(),
+                    addEventListener: jest.fn(),
+                    removeEventListener: jest.fn(),
+                }),
+                destination: {},
+                close: jest.fn(),
+            };
+            createAudioContextMock.mockReturnValue(mockAudioContext as unknown as AudioContext);
+        });
+
+        afterEach(async () => {
+            await recording.stop();
+        });
+
+        it("should record high-quality audio if voice processing is disabled", async () => {
+            MediaDeviceHandlerMock.getAudioNoiseSuppression.mockReturnValue(false);
+            await recording.start();
+
+            expect(navigator.mediaDevices.getUserMedia).toHaveBeenCalledWith(
+                expect.objectContaining({
+                    audio: expect.objectContaining({ noiseSuppression: { ideal: false } }),
+                }),
+            );
+            expect(RecorderMock).toHaveBeenCalledWith(
+                expect.objectContaining({
+                    encoderBitRate: highQualityRecorderOptions.bitrate,
+                    encoderApplication: highQualityRecorderOptions.encoderApplication,
+                }),
+            );
+        });
+
+        it("should record normal-quality voice if voice processing is enabled", async () => {
+            MediaDeviceHandlerMock.getAudioNoiseSuppression.mockReturnValue(true);
+            await recording.start();
+
+            expect(navigator.mediaDevices.getUserMedia).toHaveBeenCalledWith(
+                expect.objectContaining({
+                    audio: expect.objectContaining({ noiseSuppression: { ideal: true } }),
+                }),
+            );
+            expect(RecorderMock).toHaveBeenCalledWith(
+                expect.objectContaining({
+                    encoderBitRate: voiceRecorderOptions.bitrate,
+                    encoderApplication: voiceRecorderOptions.encoderApplication,
+                }),
+            );
+        });
+    });
+
+    describe("when recording", () => {
+        beforeEach(() => {
+            // @ts-ignore
+            recording.recording = true;
+        });
+
+        describe("and there is an audio update and time left", () => {
+            simulateUpdate(42);
+            itShouldNotCallStop();
+        });
+
+        describe("and there is an audio update and time is up", () => {
+            // one second above the limit
+            simulateUpdate(901);
+
+            it("should call stop", () => {
+                expect(recording.stop).toHaveBeenCalled();
+            });
+        });
+
+        describe("and the max length limit has been disabled", () => {
+            beforeEach(() => {
+                recording.disableMaxLength();
+            });
+
+            describe("and there is an audio update and time left", () => {
+                simulateUpdate(42);
+                itShouldNotCallStop();
+            });
+
+            describe("and there is an audio update and time is up", () => {
+                // one second above the limit
+                simulateUpdate(901);
+                itShouldNotCallStop();
+            });
+        });
+    });
+
+    describe("when not recording", () => {
+        describe("and there is an audio update and time left", () => {
+            simulateUpdate(42);
+            itShouldNotCallStop();
+        });
+
+        describe("and there is an audio update and time is up", () => {
+            // one second above the limit
+            simulateUpdate(901);
+            itShouldNotCallStop();
+        });
+    });
+});