Generalise VoiceRecording (#9304)

src/audio/VoiceMessageRecording.ts

@@ -0,0 +1,166 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { IEncryptedFile, MatrixClient } from "matrix-js-sdk/src/matrix";
+import { SimpleObservable } from "matrix-widget-api";
+
+import { uploadFile } from "../ContentMessages";
+import { IDestroyable } from "../utils/IDestroyable";
+import { Singleflight } from "../utils/Singleflight";
+import { Playback } from "./Playback";
+import { IRecordingUpdate, RecordingState, VoiceRecording } from "./VoiceRecording";
+
+export interface IUpload {
+    mxc?: string; // for unencrypted uploads
+    encrypted?: IEncryptedFile;
+}
+
+/**
+ * This class can be used to record a single voice message.
+ */
+export class VoiceMessageRecording implements IDestroyable {
+    private lastUpload: IUpload;
+    private buffer = new Uint8Array(0); // use this.audioBuffer to access
+    private playback: Playback;
+
+    public constructor(
+        private matrixClient: MatrixClient,
+        private voiceRecording: VoiceRecording,
+    ) {
+        this.voiceRecording.onDataAvailable = this.onDataAvailable;
+    }
+
+    public async start(): Promise<void> {
+        if (this.lastUpload || this.hasRecording) {
+            throw new Error("Recording already prepared");
+        }
+
+        return this.voiceRecording.start();
+    }
+
+    public async stop(): Promise<Uint8Array> {
+        await this.voiceRecording.stop();
+        return this.audioBuffer;
+    }
+
+    public on(event: string | symbol, listener: (...args: any[]) => void): this {
+        this.voiceRecording.on(event, listener);
+        return this;
+    }
+
+    public off(event: string | symbol, listener: (...args: any[]) => void): this {
+        this.voiceRecording.off(event, listener);
+        return this;
+    }
+
+    public emit(event: string, ...args: any[]): boolean {
+        return this.voiceRecording.emit(event, ...args);
+    }
+
+    public get hasRecording(): boolean {
+        return this.buffer.length > 0;
+    }
+
+    public get isRecording(): boolean {
+        return this.voiceRecording.isRecording;
+    }
+
+    /**
+     * Gets a playback instance for this voice recording. Note that the playback will not
+     * have been prepared fully, meaning the `prepare()` function needs to be called on it.
+     *
+     * The same playback instance is returned each time.
+     *
+     * @returns {Playback} The playback instance.
+     */
+    public getPlayback(): Playback {
+        this.playback = Singleflight.for(this, "playback").do(() => {
+            return new Playback(this.audioBuffer.buffer, this.voiceRecording.amplitudes); // cast to ArrayBuffer proper;
+        });
+        return this.playback;
+    }
+
+    public async upload(inRoomId: string): Promise<IUpload> {
+        if (!this.hasRecording) {
+            throw new Error("No recording available to upload");
+        }
+
+        if (this.lastUpload) return this.lastUpload;
+
+        try {
+            this.emit(RecordingState.Uploading);
+            const { url: mxc, file: encrypted } = await uploadFile(
+                this.matrixClient,
+                inRoomId,
+                new Blob(
+                    [this.audioBuffer],
+                    {
+                        type: this.contentType,
+                    },
+                ),
+            );
+            this.lastUpload = { mxc, encrypted };
+            this.emit(RecordingState.Uploaded);
+        } catch (e) {
+            this.emit(RecordingState.Ended);
+            throw e;
+        }
+        return this.lastUpload;
+    }
+
+    public get durationSeconds(): number {
+        return this.voiceRecording.durationSeconds;
+    }
+
+    public get contentType(): string {
+        return this.voiceRecording.contentType;
+    }
+
+    public get contentLength(): number {
+        return this.buffer.length;
+    }
+
+    public get liveData(): SimpleObservable<IRecordingUpdate> {
+        return this.voiceRecording.liveData;
+    }
+
+    public get isSupported(): boolean {
+        return this.voiceRecording.isSupported;
+    }
+
+    destroy(): void {
+        this.playback?.destroy();
+        this.voiceRecording.destroy();
+    }
+
+    private onDataAvailable = (data: ArrayBuffer) => {
+        const buf = new Uint8Array(data);
+        const newBuf = new Uint8Array(this.buffer.length + buf.length);
+        newBuf.set(this.buffer, 0);
+        newBuf.set(buf, this.buffer.length);
+        this.buffer = newBuf;
+    };
+
+    private get audioBuffer(): Uint8Array {
+        // We need a clone of the buffer to avoid accidentally changing the position
+        // on the real thing.
+        return this.buffer.slice(0);
+    }
+}
+
+export const createVoiceMessageRecording = (matrixClient: MatrixClient) => {
+    return new VoiceMessageRecording(matrixClient, new VoiceRecording());
+};

src/audio/VoiceRecording.ts

@@ -16,10 +16,8 @@ limitations under the License.
 
 import * as Recorder from 'opus-recorder';
 import encoderPath from 'opus-recorder/dist/encoderWorker.min.js';
-import { MatrixClient } from "matrix-js-sdk/src/client";
 import { SimpleObservable } from "matrix-widget-api";
 import EventEmitter from "events";
-import { IEncryptedFile } from "matrix-js-sdk/src/@types/event";
 import { logger } from "matrix-js-sdk/src/logger";
 
 import MediaDeviceHandler from "../MediaDeviceHandler";
@@ -27,9 +25,7 @@ import { IDestroyable } from "../utils/IDestroyable";
 import { Singleflight } from "../utils/Singleflight";
 import { PayloadEvent, WORKLET_NAME } from "./consts";
 import { UPDATE_EVENT } from "../stores/AsyncStore";
-import { Playback } from "./Playback";
 import { createAudioContext } from "./compat";
-import { uploadFile } from "../ContentMessages";
 import { FixedRollingArray } from "../utils/FixedRollingArray";
 import { clamp } from "../utils/numbers";
 import mxRecorderWorkletPath from "./RecorderWorklet";
@@ -55,11 +51,6 @@ export enum RecordingState {
     Uploaded = "uploaded",
 }
 
-export interface IUpload {
-    mxc?: string; // for unencrypted uploads
-    encrypted?: IEncryptedFile;
-}
-
 export class VoiceRecording extends EventEmitter implements IDestroyable {
     private recorder: Recorder;
     private recorderContext: AudioContext;
@@ -67,26 +58,16 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
     private recorderStream: MediaStream;
     private recorderWorklet: AudioWorkletNode;
     private recorderProcessor: ScriptProcessorNode;
-    private buffer = new Uint8Array(0); // use this.audioBuffer to access
-    private lastUpload: IUpload;
     private recording = false;
     private observable: SimpleObservable<IRecordingUpdate>;
-    private amplitudes: number[] = []; // at each second mark, generated
-    private playback: Playback;
+    public amplitudes: number[] = []; // at each second mark, generated
     private liveWaveform = new FixedRollingArray(RECORDING_PLAYBACK_SAMPLES, 0);
-
-    public constructor(private client: MatrixClient) {
-        super();
-    }
+    public onDataAvailable: (data: ArrayBuffer) => void;
 
     public get contentType(): string {
         return "audio/ogg";
     }
 
-    public get contentLength(): number {
-        return this.buffer.length;
-    }
-
     public get durationSeconds(): number {
         if (!this.recorder) throw new Error("Duration not available without a recording");
         return this.recorderContext.currentTime;
@@ -165,13 +146,9 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
                 encoderComplexity: 3, // 0-10, 10 is slow and high quality.
                 resampleQuality: 3, // 0-10, 10 is slow and high quality
             });
-            this.recorder.ondataavailable = (a: ArrayBuffer) => {
-                const buf = new Uint8Array(a);
-                const newBuf = new Uint8Array(this.buffer.length + buf.length);
-                newBuf.set(this.buffer, 0);
-                newBuf.set(buf, this.buffer.length);
-                this.buffer = newBuf;
-            };
+
+            // not using EventEmitter here because it leads to detached bufferes
+            this.recorder.ondataavailable = (data: ArrayBuffer) => this?.onDataAvailable(data);
         } catch (e) {
             logger.error("Error starting recording: ", e);
             if (e instanceof DOMException) { // Unhelpful DOMExceptions are common - parse them sanely
@@ -191,12 +168,6 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
         }
     }
 
-    private get audioBuffer(): Uint8Array {
-        // We need a clone of the buffer to avoid accidentally changing the position
-        // on the real thing.
-        return this.buffer.slice(0);
-    }
-
     public get liveData(): SimpleObservable<IRecordingUpdate> {
         if (!this.recording) throw new Error("No observable when not recording");
         return this.observable;
@@ -206,10 +177,6 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
         return !!Recorder.isRecordingSupported();
     }
 
-    public get hasRecording(): boolean {
-        return this.buffer.length > 0;
-    }
-
     private onAudioProcess = (ev: AudioProcessingEvent) => {
         this.processAudioUpdate(ev.playbackTime);
 
@@ -251,9 +218,6 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
     };
 
     public async start(): Promise<void> {
-        if (this.lastUpload || this.hasRecording) {
-            throw new Error("Recording already prepared");
-        }
         if (this.recording) {
             throw new Error("Recording already in progress");
         }
@@ -267,7 +231,7 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
         this.emit(RecordingState.Started);
     }
 
-    public async stop(): Promise<Uint8Array> {
+    public async stop(): Promise<void> {
         return Singleflight.for(this, "stop").do(async () => {
             if (!this.recording) {
                 throw new Error("No recording to stop");
@@ -293,54 +257,16 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
             this.recording = false;
             await this.recorder.close();
             this.emit(RecordingState.Ended);
-
-            return this.audioBuffer;
         });
     }
 
-    /**
-     * Gets a playback instance for this voice recording. Note that the playback will not
-     * have been prepared fully, meaning the `prepare()` function needs to be called on it.
-     *
-     * The same playback instance is returned each time.
-     *
-     * @returns {Playback} The playback instance.
-     */
-    public getPlayback(): Playback {
-        this.playback = Singleflight.for(this, "playback").do(() => {
-            return new Playback(this.audioBuffer.buffer, this.amplitudes); // cast to ArrayBuffer proper;
-        });
-        return this.playback;
-    }
-
     public destroy() {
         // noinspection JSIgnoredPromiseFromCall - not concerned about stop() being called async here
         this.stop();
         this.removeAllListeners();
+        this.onDataAvailable = undefined;
         Singleflight.forgetAllFor(this);
         // noinspection JSIgnoredPromiseFromCall - not concerned about being called async here
-        this.playback?.destroy();
         this.observable.close();
     }
-
-    public async upload(inRoomId: string): Promise<IUpload> {
-        if (!this.hasRecording) {
-            throw new Error("No recording available to upload");
-        }
-
-        if (this.lastUpload) return this.lastUpload;
-
-        try {
-            this.emit(RecordingState.Uploading);
-            const { url: mxc, file: encrypted } = await uploadFile(this.client, inRoomId, new Blob([this.audioBuffer], {
-                type: this.contentType,
-            }));
-            this.lastUpload = { mxc, encrypted };
-            this.emit(RecordingState.Uploaded);
-        } catch (e) {
-            this.emit(RecordingState.Ended);
-            throw e;
-        }
-        return this.lastUpload;
-    }
 }

src/components/views/audio_messages/LiveRecordingClock.tsx

@@ -16,12 +16,13 @@ limitations under the License.
 
 import React from "react";
 
-import { IRecordingUpdate, VoiceRecording } from "../../../audio/VoiceRecording";
+import { IRecordingUpdate } from "../../../audio/VoiceRecording";
 import Clock from "./Clock";
 import { MarkedExecution } from "../../../utils/MarkedExecution";
+import { VoiceMessageRecording } from "../../../audio/VoiceMessageRecording";
 
 interface IProps {
-    recorder: VoiceRecording;
+    recorder: VoiceMessageRecording;
 }
 
 interface IState {

src/components/views/audio_messages/LiveRecordingWaveform.tsx

@@ -16,13 +16,14 @@ limitations under the License.
 
 import React from "react";
 
-import { IRecordingUpdate, RECORDING_PLAYBACK_SAMPLES, VoiceRecording } from "../../../audio/VoiceRecording";
+import { IRecordingUpdate, RECORDING_PLAYBACK_SAMPLES } from "../../../audio/VoiceRecording";
 import { arrayFastResample, arraySeed } from "../../../utils/arrays";
 import Waveform from "./Waveform";
 import { MarkedExecution } from "../../../utils/MarkedExecution";
+import { VoiceMessageRecording } from "../../../audio/VoiceMessageRecording";
 
 interface IProps {
-    recorder: VoiceRecording;
+    recorder: VoiceMessageRecording;
 }
 
 interface IState {

src/components/views/rooms/MessageComposer.tsx

@@ -37,7 +37,7 @@ import ReplyPreview from "./ReplyPreview";
 import { UPDATE_EVENT } from "../../../stores/AsyncStore";
 import VoiceRecordComposerTile from "./VoiceRecordComposerTile";
 import { VoiceRecordingStore } from "../../../stores/VoiceRecordingStore";
-import { RecordingState, VoiceRecording } from "../../../audio/VoiceRecording";
+import { RecordingState } from "../../../audio/VoiceRecording";
 import Tooltip, { Alignment } from "../elements/Tooltip";
 import ResizeNotifier from "../../../utils/ResizeNotifier";
 import { E2EStatus } from '../../../utils/ShieldUtils';
@@ -53,6 +53,7 @@ import { ButtonEvent } from '../elements/AccessibleButton';
 import { ViewRoomPayload } from "../../../dispatcher/payloads/ViewRoomPayload";
 import { isLocalRoom } from '../../../utils/localRoom/isLocalRoom';
 import { Features } from '../../../settings/Settings';
+import { VoiceMessageRecording } from '../../../audio/VoiceMessageRecording';
 
 let instanceCount = 0;
 
@@ -101,7 +102,7 @@ export default class MessageComposer extends React.Component<IProps, IState> {
     private ref: React.RefObject<HTMLDivElement> = createRef();
     private instanceId: number;
 
-    private _voiceRecording: Optional<VoiceRecording>;
+    private _voiceRecording: Optional<VoiceMessageRecording>;
 
     public static contextType = RoomContext;
     public context!: React.ContextType<typeof RoomContext>;
@@ -133,11 +134,11 @@ export default class MessageComposer extends React.Component<IProps, IState> {
         SettingsStore.monitorSetting(Features.VoiceBroadcast, null);
     }
 
-    private get voiceRecording(): Optional<VoiceRecording> {
+    private get voiceRecording(): Optional<VoiceMessageRecording> {
         return this._voiceRecording;
     }
 
-    private set voiceRecording(rec: Optional<VoiceRecording>) {
+    private set voiceRecording(rec: Optional<VoiceMessageRecording>) {
         if (this._voiceRecording) {
             this._voiceRecording.off(RecordingState.Started, this.onRecordingStarted);
             this._voiceRecording.off(RecordingState.EndingSoon, this.onRecordingEndingSoon);

src/components/views/rooms/VoiceRecordComposerTile.tsx

@@ -23,7 +23,7 @@ import { IEventRelation, MatrixEvent } from "matrix-js-sdk/src/models/event";
 
 import AccessibleTooltipButton from "../elements/AccessibleTooltipButton";
 import { _t } from "../../../languageHandler";
-import { IUpload, RecordingState, VoiceRecording } from "../../../audio/VoiceRecording";
+import { RecordingState } from "../../../audio/VoiceRecording";
 import { MatrixClientPeg } from "../../../MatrixClientPeg";
 import LiveRecordingWaveform from "../audio_messages/LiveRecordingWaveform";
 import LiveRecordingClock from "../audio_messages/LiveRecordingClock";
@@ -44,6 +44,7 @@ import { attachRelation } from "./SendMessageComposer";
 import { addReplyToMessageContent } from "../../../utils/Reply";
 import { RoomPermalinkCreator } from "../../../utils/permalinks/Permalinks";
 import RoomContext from "../../../contexts/RoomContext";
+import { IUpload, VoiceMessageRecording } from "../../../audio/VoiceMessageRecording";
 
 interface IProps {
     room: Room;
@@ -53,7 +54,7 @@ interface IProps {
 }
 
 interface IState {
-    recorder?: VoiceRecording;
+    recorder?: VoiceMessageRecording;
     recordingPhase?: RecordingState;
     didUploadFail?: boolean;
 }
@@ -250,7 +251,7 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
         }
     };
 
-    private bindNewRecorder(recorder: Optional<VoiceRecording>) {
+    private bindNewRecorder(recorder: Optional<VoiceMessageRecording>) {
         if (this.state.recorder) {
             this.state.recorder.off(UPDATE_EVENT, this.onRecordingUpdate);
         }

src/stores/VoiceRecordingStore.ts

@@ -22,12 +22,12 @@ import { IEventRelation } from "matrix-js-sdk/src/models/event";
 import { AsyncStoreWithClient } from "./AsyncStoreWithClient";
 import defaultDispatcher from "../dispatcher/dispatcher";
 import { ActionPayload } from "../dispatcher/payloads";
-import { VoiceRecording } from "../audio/VoiceRecording";
+import { createVoiceMessageRecording, VoiceMessageRecording } from "../audio/VoiceMessageRecording";
 
 const SEPARATOR = "|";
 
 interface IState {
-    [voiceRecordingId: string]: Optional<VoiceRecording>;
+    [voiceRecordingId: string]: Optional<VoiceMessageRecording>;
 }
 
 export class VoiceRecordingStore extends AsyncStoreWithClient<IState> {
@@ -63,7 +63,7 @@ export class VoiceRecordingStore extends AsyncStoreWithClient<IState> {
      * @param {string} voiceRecordingId The room ID (with optionally the thread ID if in one) to get the recording in.
      * @returns {Optional<VoiceRecording>} The recording, if any.
      */
-    public getActiveRecording(voiceRecordingId: string): Optional<VoiceRecording> {
+    public getActiveRecording(voiceRecordingId: string): Optional<VoiceMessageRecording> {
         return this.state[voiceRecordingId];
     }
 
@@ -74,12 +74,12 @@ export class VoiceRecordingStore extends AsyncStoreWithClient<IState> {
      * @param {string} voiceRecordingId The room ID (with optionally the thread ID if in one) to start recording in.
      * @returns {VoiceRecording} The recording.
      */
-    public startRecording(voiceRecordingId: string): VoiceRecording {
+    public startRecording(voiceRecordingId: string): VoiceMessageRecording {
         if (!this.matrixClient) throw new Error("Cannot start a recording without a MatrixClient");
         if (!voiceRecordingId) throw new Error("Recording must be associated with a room");
         if (this.state[voiceRecordingId]) throw new Error("A recording is already in progress");
 
-        const recording = new VoiceRecording(this.matrixClient);
+        const recording = createVoiceMessageRecording(this.matrixClient);
 
         // noinspection JSIgnoredPromiseFromCall - we can safely run this async
         this.updateState({ ...this.state, [voiceRecordingId]: recording });