imterah/element-portable
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Generalise VoiceRecording (#9304)

Browse source
This commit is contained in:
Michael Weimann 2022-09-21 18:46:28 +02:00 • committed by GitHub
parent 71cf9bf932
commit c182c1c706
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 422 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

166
src/audio/VoiceMessageRecording.ts Normal file
View file

@ -0,0 +1,166 @@
/*
Copyright 2022 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { IEncryptedFile, MatrixClient } from "matrix-js-sdk/src/matrix";
import { SimpleObservable } from "matrix-widget-api";
import { uploadFile } from "../ContentMessages";
import { IDestroyable } from "../utils/IDestroyable";
import { Singleflight } from "../utils/Singleflight";
import { Playback } from "./Playback";
import { IRecordingUpdate, RecordingState, VoiceRecording } from "./VoiceRecording";
export interface IUpload {
mxc?: string; // for unencrypted uploads
encrypted?: IEncryptedFile;
}
/**
* This class can be used to record a single voice message.
*/
export class VoiceMessageRecording implements IDestroyable {
private lastUpload: IUpload;
private buffer = new Uint8Array(0); // use this.audioBuffer to access
private playback: Playback;
public constructor(
private matrixClient: MatrixClient,
private voiceRecording: VoiceRecording,
) {
this.voiceRecording.onDataAvailable = this.onDataAvailable;
}
public async start(): Promise<void> {
if (this.lastUpload || this.hasRecording) {
throw new Error("Recording already prepared");
}
return this.voiceRecording.start();
}
public async stop(): Promise<Uint8Array> {
await this.voiceRecording.stop();
return this.audioBuffer;
}
public on(event: string | symbol, listener: (...args: any[]) => void): this {
this.voiceRecording.on(event, listener);
return this;
}
public off(event: string | symbol, listener: (...args: any[]) => void): this {
this.voiceRecording.off(event, listener);
return this;
}
public emit(event: string, ...args: any[]): boolean {
return this.voiceRecording.emit(event, ...args);
}
public get hasRecording(): boolean {
return this.buffer.length > 0;
}
public get isRecording(): boolean {
return this.voiceRecording.isRecording;
}
/**
* Gets a playback instance for this voice recording. Note that the playback will not
* have been prepared fully, meaning the `prepare()` function needs to be called on it.
*
* The same playback instance is returned each time.
*
* @returns {Playback} The playback instance.
*/
public getPlayback(): Playback {
this.playback = Singleflight.for(this, "playback").do(() => {
return new Playback(this.audioBuffer.buffer, this.voiceRecording.amplitudes); // cast to ArrayBuffer proper;
});
return this.playback;
}
public async upload(inRoomId: string): Promise<IUpload> {
if (!this.hasRecording) {
throw new Error("No recording available to upload");
}
if (this.lastUpload) return this.lastUpload;
try {
this.emit(RecordingState.Uploading);
const { url: mxc, file: encrypted } = await uploadFile(
this.matrixClient,
inRoomId,
new Blob(
[this.audioBuffer],
{
type: this.contentType,
},
),
);
this.lastUpload = { mxc, encrypted };
this.emit(RecordingState.Uploaded);
} catch (e) {
this.emit(RecordingState.Ended);
throw e;
}
return this.lastUpload;
}
public get durationSeconds(): number {
return this.voiceRecording.durationSeconds;
}
public get contentType(): string {
return this.voiceRecording.contentType;
}
public get contentLength(): number {
return this.buffer.length;
}
public get liveData(): SimpleObservable<IRecordingUpdate> {
return this.voiceRecording.liveData;
}
public get isSupported(): boolean {
return this.voiceRecording.isSupported;
}
destroy(): void {
this.playback?.destroy();
this.voiceRecording.destroy();
}
private onDataAvailable = (data: ArrayBuffer) => {
const buf = new Uint8Array(data);
const newBuf = new Uint8Array(this.buffer.length + buf.length);
newBuf.set(this.buffer, 0);
newBuf.set(buf, this.buffer.length);
this.buffer = newBuf;
};
private get audioBuffer(): Uint8Array {
// We need a clone of the buffer to avoid accidentally changing the position
// on the real thing.
return this.buffer.slice(0);
}
}
export const createVoiceMessageRecording = (matrixClient: MatrixClient) => {
return new VoiceMessageRecording(matrixClient, new VoiceRecording());
};

88
src/audio/VoiceRecording.ts
View file

@ -16,10 +16,8 @@ limitations under the License.
import * as Recorder from 'opus-recorder';
import encoderPath from 'opus-recorder/dist/encoderWorker.min.js';
import { MatrixClient } from "matrix-js-sdk/src/client";
import { SimpleObservable } from "matrix-widget-api";
import EventEmitter from "events";
import { IEncryptedFile } from "matrix-js-sdk/src/@types/event";
import { logger } from "matrix-js-sdk/src/logger";
import MediaDeviceHandler from "../MediaDeviceHandler";
@ -27,9 +25,7 @@ import { IDestroyable } from "../utils/IDestroyable";
import { Singleflight } from "../utils/Singleflight";
import { PayloadEvent, WORKLET_NAME } from "./consts";
import { UPDATE_EVENT } from "../stores/AsyncStore";
import { Playback } from "./Playback";
import { createAudioContext } from "./compat";
import { uploadFile } from "../ContentMessages";
import { FixedRollingArray } from "../utils/FixedRollingArray";
import { clamp } from "../utils/numbers";
import mxRecorderWorkletPath from "./RecorderWorklet";
@ -55,11 +51,6 @@ export enum RecordingState {
Uploaded = "uploaded",
}
export interface IUpload {
mxc?: string; // for unencrypted uploads
encrypted?: IEncryptedFile;
}
export class VoiceRecording extends EventEmitter implements IDestroyable {
private recorder: Recorder;
private recorderContext: AudioContext;
@ -67,26 +58,16 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
private recorderStream: MediaStream;
private recorderWorklet: AudioWorkletNode;
private recorderProcessor: ScriptProcessorNode;
private buffer = new Uint8Array(0); // use this.audioBuffer to access
private lastUpload: IUpload;
private recording = false;
private observable: SimpleObservable<IRecordingUpdate>;
private amplitudes: number[] = []; // at each second mark, generated
private playback: Playback;
public amplitudes: number[] = []; // at each second mark, generated
private liveWaveform = new FixedRollingArray(RECORDING_PLAYBACK_SAMPLES, 0);
public constructor(private client: MatrixClient) {
super();
}
public onDataAvailable: (data: ArrayBuffer) => void;
public get contentType(): string {
return "audio/ogg";
}
public get contentLength(): number {
return this.buffer.length;
}
public get durationSeconds(): number {
if (!this.recorder) throw new Error("Duration not available without a recording");
return this.recorderContext.currentTime;
@ -165,13 +146,9 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
encoderComplexity: 3, // 0-10, 10 is slow and high quality.
resampleQuality: 3, // 0-10, 10 is slow and high quality
});
this.recorder.ondataavailable = (a: ArrayBuffer) => {
const buf = new Uint8Array(a);
const newBuf = new Uint8Array(this.buffer.length + buf.length);
newBuf.set(this.buffer, 0);
newBuf.set(buf, this.buffer.length);
this.buffer = newBuf;
};
// not using EventEmitter here because it leads to detached bufferes
this.recorder.ondataavailable = (data: ArrayBuffer) => this?.onDataAvailable(data);
} catch (e) {
logger.error("Error starting recording: ", e);
if (e instanceof DOMException) { // Unhelpful DOMExceptions are common - parse them sanely
@ -191,12 +168,6 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
}
}
private get audioBuffer(): Uint8Array {
// We need a clone of the buffer to avoid accidentally changing the position
// on the real thing.
return this.buffer.slice(0);
}
public get liveData(): SimpleObservable<IRecordingUpdate> {
if (!this.recording) throw new Error("No observable when not recording");
return this.observable;
@ -206,10 +177,6 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
return !!Recorder.isRecordingSupported();
}
public get hasRecording(): boolean {
return this.buffer.length > 0;
}
private onAudioProcess = (ev: AudioProcessingEvent) => {
this.processAudioUpdate(ev.playbackTime);
@ -251,9 +218,6 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
};
public async start(): Promise<void> {
if (this.lastUpload || this.hasRecording) {
throw new Error("Recording already prepared");
}
if (this.recording) {
throw new Error("Recording already in progress");
}
@ -267,7 +231,7 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
this.emit(RecordingState.Started);
}
public async stop(): Promise<Uint8Array> {
public async stop(): Promise<void> {
return Singleflight.for(this, "stop").do(async () => {
if (!this.recording) {
throw new Error("No recording to stop");
@ -293,54 +257,16 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
this.recording = false;
await this.recorder.close();
this.emit(RecordingState.Ended);
return this.audioBuffer;
});
}
/**
* Gets a playback instance for this voice recording. Note that the playback will not
* have been prepared fully, meaning the `prepare()` function needs to be called on it.
*
* The same playback instance is returned each time.
*
* @returns {Playback} The playback instance.
*/
public getPlayback(): Playback {
this.playback = Singleflight.for(this, "playback").do(() => {
return new Playback(this.audioBuffer.buffer, this.amplitudes); // cast to ArrayBuffer proper;
});
return this.playback;
}
public destroy() {
// noinspection JSIgnoredPromiseFromCall - not concerned about stop() being called async here
this.stop();
this.removeAllListeners();
this.onDataAvailable = undefined;
Singleflight.forgetAllFor(this);
// noinspection JSIgnoredPromiseFromCall - not concerned about being called async here
this.playback?.destroy();
this.observable.close();
}
public async upload(inRoomId: string): Promise<IUpload> {
if (!this.hasRecording) {
throw new Error("No recording available to upload");
}
if (this.lastUpload) return this.lastUpload;
try {
this.emit(RecordingState.Uploading);
const { url: mxc, file: encrypted } = await uploadFile(this.client, inRoomId, new Blob([this.audioBuffer], {
type: this.contentType,
}));
this.lastUpload = { mxc, encrypted };
this.emit(RecordingState.Uploaded);
} catch (e) {
this.emit(RecordingState.Ended);
throw e;
}
return this.lastUpload;
}
}