imterah/element-portable
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement Voice Broadcast recording (#9307)

Browse source 
* Implement VoiceBroadcastRecording

* Implement PR feedback

* Add voice broadcast recording stores

* Refactor startNewVoiceBroadcastRecording

* Refactor VoiceBroadcastRecordingsStore to VoiceBroadcastRecording

* Rename VoiceBroadcastRecording to VoiceBroadcastRecorder

* Return remaining chunk on stop

* Extract createVoiceMessageContent

* Implement recording

* Replace dev value with config

* Fix clientInformation-test

* Refactor VoiceBroadcastRecording

* Fix VoiceBroadcastRecording types

* Re-order getter

* Mark voice_broadcast config as optional

* Merge voice-broadcast modules

* Remove underscore props

* Add Optional types

* Add return types everywhere

* Remove test casts

* Add magic comments

* Trigger CI

* Switch VoiceBroadcastRecorder to TypedEventEmitter

* Trigger CI

* Add voice broadcast chunk event content

Co-authored-by: Travis Ralston <travisr@matrix.org>
This commit is contained in:
Michael Weimann 2022-10-12 00:31:28 +02:00 • committed by GitHub
parent 03182d03be
commit bac6e12946
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 773 additions and 104 deletions
Download patch file Download diff file Expand all files Collapse all files

41
test/SdkConfig-test.ts Normal file
View file

@ -0,0 +1,41 @@
/*
Copyright 2022 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import SdkConfig, { DEFAULTS } from "../src/SdkConfig";
describe("SdkConfig", () => {
describe("with default values", () => {
it("should return the default config", () => {
expect(SdkConfig.get()).toEqual(DEFAULTS);
});
});
describe("with custom values", () => {
beforeEach(() => {
SdkConfig.put({
voice_broadcast: {
chunk_length: 1337,
},
});
});
it("should return the custom config", () => {
const customConfig = JSON.parse(JSON.stringify(DEFAULTS));
customConfig.voice_broadcast.chunk_length = 1337;
expect(SdkConfig.get()).toEqual(customConfig);
});
});
});

209
test/voice-broadcast/audio/VoiceBroadcastRecorder-test.ts Normal file
View file

@ -0,0 +1,209 @@
/*
Copyright 2022 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { mocked } from "jest-mock";
import { VoiceRecording } from "../../../src/audio/VoiceRecording";
import SdkConfig from "../../../src/SdkConfig";
import { concat } from "../../../src/utils/arrays";
import {
ChunkRecordedPayload,
createVoiceBroadcastRecorder,
VoiceBroadcastRecorder,
VoiceBroadcastRecorderEvent,
} from "../../../src/voice-broadcast";
describe("VoiceBroadcastRecorder", () => {
describe("createVoiceBroadcastRecorder", () => {
beforeEach(() => {
jest.spyOn(SdkConfig, "get").mockImplementation((key: string) => {
if (key === "voice_broadcast") {
return {
chunk_length: 1337,
};
}
});
});
afterEach(() => {
mocked(SdkConfig.get).mockRestore();
});
it("should return a VoiceBroadcastRecorder instance with targetChunkLength from config", () => {
const voiceBroadcastRecorder = createVoiceBroadcastRecorder();
expect(voiceBroadcastRecorder).toBeInstanceOf(VoiceBroadcastRecorder);
expect(voiceBroadcastRecorder.targetChunkLength).toBe(1337);
});
});
describe("instance", () => {
const chunkLength = 30;
const headers1 = new Uint8Array([1, 2]);
const headers2 = new Uint8Array([3, 4]);
const chunk1 = new Uint8Array([5, 6]);
const chunk2a = new Uint8Array([7, 8]);
const chunk2b = new Uint8Array([9, 10]);
const contentType = "test content type";
let voiceRecording: VoiceRecording;
let voiceBroadcastRecorder: VoiceBroadcastRecorder;
let onChunkRecorded: (chunk: ChunkRecordedPayload) => void;
const itShouldNotEmitAChunkRecordedEvent = () => {
it("should not emit a ChunkRecorded event", () => {
expect(voiceRecording.emit).not.toHaveBeenCalledWith(
VoiceBroadcastRecorderEvent.ChunkRecorded,
expect.anything(),
);
});
};
beforeEach(() => {
voiceRecording = {
contentType,
start: jest.fn().mockResolvedValue(undefined),
stop: jest.fn().mockResolvedValue(undefined),
on: jest.fn(),
off: jest.fn(),
emit: jest.fn(),
destroy: jest.fn(),
recorderSeconds: 23,
} as unknown as VoiceRecording;
voiceBroadcastRecorder = new VoiceBroadcastRecorder(voiceRecording, chunkLength);
jest.spyOn(voiceBroadcastRecorder, "removeAllListeners");
onChunkRecorded = jest.fn();
voiceBroadcastRecorder.on(VoiceBroadcastRecorderEvent.ChunkRecorded, onChunkRecorded);
});
afterEach(() => {
voiceBroadcastRecorder.destroy();
});
it("start should forward the call to VoiceRecording.start", async () => {
await voiceBroadcastRecorder.start();
expect(voiceRecording.start).toHaveBeenCalled();
});
describe("stop", () => {
beforeEach(async () => {
await voiceBroadcastRecorder.stop();
});
it("should forward the call to VoiceRecording.stop", async () => {
expect(voiceRecording.stop).toHaveBeenCalled();
});
itShouldNotEmitAChunkRecordedEvent();
});
describe("when calling destroy", () => {
beforeEach(() => {
voiceBroadcastRecorder.destroy();
});
it("should call VoiceRecording.destroy", () => {
expect(voiceRecording.destroy).toHaveBeenCalled();
});
it("should remove all listeners", () => {
expect(voiceBroadcastRecorder.removeAllListeners).toHaveBeenCalled();
});
});
it("contentType should return the value from VoiceRecording", () => {
expect(voiceBroadcastRecorder.contentType).toBe(contentType);
});
describe("when the first page from recorder has been received", () => {
beforeEach(() => {
voiceRecording.onDataAvailable(headers1);
});
itShouldNotEmitAChunkRecordedEvent();
});
describe("when a second page from recorder has been received", () => {
beforeEach(() => {
voiceRecording.onDataAvailable(headers1);
voiceRecording.onDataAvailable(headers2);
});
itShouldNotEmitAChunkRecordedEvent();
});
describe("when a third page from recorder has been received", () => {
beforeEach(() => {
voiceRecording.onDataAvailable(headers1);
voiceRecording.onDataAvailable(headers2);
voiceRecording.onDataAvailable(chunk1);
});
itShouldNotEmitAChunkRecordedEvent();
describe("stop", () => {
let stopPayload: ChunkRecordedPayload;
beforeEach(async () => {
stopPayload = await voiceBroadcastRecorder.stop();
});
it("should return the remaining chunk", () => {
expect(stopPayload).toEqual({
buffer: concat(headers1, headers2, chunk1),
length: 23,
});
});
});
});
describe("when some chunks have been received", () => {
beforeEach(() => {
// simulate first chunk
voiceRecording.onDataAvailable(headers1);
voiceRecording.onDataAvailable(headers2);
// set recorder seconds to something greater than the test chunk length of 30
// @ts-ignore
voiceRecording.recorderSeconds = 42;
voiceRecording.onDataAvailable(chunk1);
// simulate a second chunk
voiceRecording.onDataAvailable(chunk2a);
// add another 30 seconds for the next chunk
// @ts-ignore
voiceRecording.recorderSeconds = 72;
voiceRecording.onDataAvailable(chunk2b);
});
it("should emit ChunkRecorded events", () => {
expect(onChunkRecorded).toHaveBeenNthCalledWith(
1,
{
buffer: concat(headers1, headers2, chunk1),
length: 42,
},
);
expect(onChunkRecorded).toHaveBeenNthCalledWith(
2,
{
buffer: concat(headers1, headers2, chunk2a, chunk2b),
length: 72 - 42, // 72 (position at second chunk) - 42 (position of first chunk)
},
);
});
});
});
});

2
test/voice-broadcast/components/VoiceBroadcastBody-test.tsx
View file

@ -155,7 +155,7 @@ describe("VoiceBroadcastBody", () => {
itShouldRenderANonLiveVoiceBroadcast();
it("should call stop on the recording", () => {
expect(recording.state).toBe(VoiceBroadcastInfoState.Stopped);
expect(recording.getState()).toBe(VoiceBroadcastInfoState.Stopped);
expect(onRecordingStateChanged).toHaveBeenCalledWith(VoiceBroadcastInfoState.Stopped);
});
});

242
test/voice-broadcast/models/VoiceBroadcastRecording-test.ts
View file

@ -15,25 +15,57 @@ limitations under the License.
*/
import { mocked } from "jest-mock";
import { EventTimelineSet, EventType, MatrixClient, MatrixEvent, RelationType, Room } from "matrix-js-sdk/src/matrix";
import {
EventTimelineSet,
EventType,
MatrixClient,
MatrixEvent,
MsgType,
RelationType,
Room,
} from "matrix-js-sdk/src/matrix";
import { Relations } from "matrix-js-sdk/src/models/relations";
import { uploadFile } from "../../../src/ContentMessages";
import { IEncryptedFile } from "../../../src/customisations/models/IMediaEventContent";
import { createVoiceMessageContent } from "../../../src/utils/createVoiceMessageContent";
import {
ChunkRecordedPayload,
createVoiceBroadcastRecorder,
VoiceBroadcastInfoEventContent,
VoiceBroadcastInfoEventType,
VoiceBroadcastInfoState,
VoiceBroadcastRecorder,
VoiceBroadcastRecorderEvent,
VoiceBroadcastRecording,
VoiceBroadcastRecordingEvent,
} from "../../../src/voice-broadcast";
import { mkEvent, mkStubRoom, stubClient } from "../../test-utils";
jest.mock("../../../src/voice-broadcast/audio/VoiceBroadcastRecorder", () => ({
...jest.requireActual("../../../src/voice-broadcast/audio/VoiceBroadcastRecorder") as object,
createVoiceBroadcastRecorder: jest.fn(),
}));
jest.mock("../../../src/ContentMessages", () => ({
uploadFile: jest.fn(),
}));
jest.mock("../../../src/utils/createVoiceMessageContent", () => ({
createVoiceMessageContent: jest.fn(),
}));
describe("VoiceBroadcastRecording", () => {
const roomId = "!room:example.com";
const uploadedUrl = "mxc://example.com/vb";
const uploadedFile = { file: true } as unknown as IEncryptedFile;
let room: Room;
let client: MatrixClient;
let infoEvent: MatrixEvent;
let voiceBroadcastRecording: VoiceBroadcastRecording;
let onStateChanged: (state: VoiceBroadcastInfoState) => void;
let voiceBroadcastRecorder: VoiceBroadcastRecorder;
let onChunkRecorded: (chunk: ChunkRecordedPayload) => Promise<void>;
const mkVoiceBroadcastInfoEvent = (content: VoiceBroadcastInfoEventContent) => {
return mkEvent({
@ -48,6 +80,7 @@ describe("VoiceBroadcastRecording", () => {
const setUpVoiceBroadcastRecording = () => {
voiceBroadcastRecording = new VoiceBroadcastRecording(infoEvent, client);
voiceBroadcastRecording.on(VoiceBroadcastRecordingEvent.StateChanged, onStateChanged);
jest.spyOn(voiceBroadcastRecording, "removeAllListeners");
};
beforeEach(() => {
@ -59,6 +92,65 @@ describe("VoiceBroadcastRecording", () => {
}
});
onStateChanged = jest.fn();
voiceBroadcastRecorder = {
contentType: "audio/ogg",
on: jest.fn(),
off: jest.fn(),
start: jest.fn(),
stop: jest.fn(),
} as unknown as VoiceBroadcastRecorder;
mocked(createVoiceBroadcastRecorder).mockReturnValue(voiceBroadcastRecorder);
onChunkRecorded = jest.fn();
mocked(voiceBroadcastRecorder.on).mockImplementation(
(event: VoiceBroadcastRecorderEvent, listener: any): VoiceBroadcastRecorder => {
if (event === VoiceBroadcastRecorderEvent.ChunkRecorded) {
onChunkRecorded = listener;
}
return voiceBroadcastRecorder;
},
);
mocked(uploadFile).mockResolvedValue({
url: uploadedUrl,
file: uploadedFile,
});
mocked(createVoiceMessageContent).mockImplementation((
mxc: string,
mimetype: string,
duration: number,
size: number,
file?: IEncryptedFile,
waveform?: number[],
) => {
return {
body: "Voice message",
msgtype: MsgType.Audio,
url: mxc,
file,
info: {
duration,
mimetype,
size,
},
["org.matrix.msc1767.text"]: "Voice message",
["org.matrix.msc1767.file"]: {
url: mxc,
file,
name: "Voice message.ogg",
mimetype,
size,
},
["org.matrix.msc1767.audio"]: {
duration,
// https://github.com/matrix-org/matrix-doc/pull/3246
waveform,
},
["org.matrix.msc3245.voice"]: {}, // No content, this is a rendering hint
};
});
});
afterEach(() => {
@ -74,7 +166,7 @@ describe("VoiceBroadcastRecording", () => {
});
it("should be in Started state", () => {
expect(voiceBroadcastRecording.state).toBe(VoiceBroadcastInfoState.Started);
expect(voiceBroadcastRecording.getState()).toBe(VoiceBroadcastInfoState.Started);
});
describe("and calling stop()", () => {
@ -98,13 +190,155 @@ describe("VoiceBroadcastRecording", () => {
});
it("should be in state stopped", () => {
expect(voiceBroadcastRecording.state).toBe(VoiceBroadcastInfoState.Stopped);
expect(voiceBroadcastRecording.getState()).toBe(VoiceBroadcastInfoState.Stopped);
});
it("should emit a stopped state changed event", () => {
expect(onStateChanged).toHaveBeenCalledWith(VoiceBroadcastInfoState.Stopped);
});
});
describe("and calling start", () => {
beforeEach(async () => {
await voiceBroadcastRecording.start();
});
it("should start the recorder", () => {
expect(voiceBroadcastRecorder.start).toHaveBeenCalled();
});
describe("and a chunk has been recorded", () => {
beforeEach(async () => {
await onChunkRecorded({
buffer: new Uint8Array([1, 2, 3]),
length: 23,
});
});
it("should send a voice message", () => {
expect(uploadFile).toHaveBeenCalledWith(
client,
roomId,
new Blob([new Uint8Array([1, 2, 3])], { type: voiceBroadcastRecorder.contentType }),
);
expect(mocked(client.sendMessage)).toHaveBeenCalledWith(
roomId,
{
body: "Voice message",
file: {
file: true,
},
info: {
duration: 23000,
mimetype: "audio/ogg",
size: 3,
},
["m.relates_to"]: {
event_id: infoEvent.getId(),
rel_type: "m.reference",
},
msgtype: "m.audio",
["org.matrix.msc1767.audio"]: {
duration: 23000,
waveform: undefined,
},
["org.matrix.msc1767.file"]: {
file: {
file: true,
},
mimetype: "audio/ogg",
name: "Voice message.ogg",
size: 3,
url: "mxc://example.com/vb",
},
["org.matrix.msc1767.text"]: "Voice message",
["org.matrix.msc3245.voice"]: {},
url: "mxc://example.com/vb",
["io.element.voice_broadcast_chunk"]: {
sequence: 1,
},
},
);
});
});
describe("and calling stop", () => {
beforeEach(async () => {
await onChunkRecorded({
buffer: new Uint8Array([1, 2, 3]),
length: 23,
});
mocked(voiceBroadcastRecorder.stop).mockResolvedValue({
buffer: new Uint8Array([4, 5, 6]),
length: 42,
});
await voiceBroadcastRecording.stop();
});
it("should send the last chunk", () => {
expect(uploadFile).toHaveBeenCalledWith(
client,
roomId,
new Blob([new Uint8Array([4, 5, 6])], { type: voiceBroadcastRecorder.contentType }),
);
expect(mocked(client.sendMessage)).toHaveBeenCalledWith(
roomId,
{
body: "Voice message",
file: {
file: true,
},
info: {
duration: 42000,
mimetype: "audio/ogg",
size: 3,
},
["m.relates_to"]: {
event_id: infoEvent.getId(),
rel_type: "m.reference",
},
msgtype: "m.audio",
["org.matrix.msc1767.audio"]: {
duration: 42000,
waveform: undefined,
},
["org.matrix.msc1767.file"]: {
file: {
file: true,
},
mimetype: "audio/ogg",
name: "Voice message.ogg",
size: 3,
url: "mxc://example.com/vb",
},
["org.matrix.msc1767.text"]: "Voice message",
["org.matrix.msc3245.voice"]: {},
url: "mxc://example.com/vb",
["io.element.voice_broadcast_chunk"]: {
sequence: 2,
},
},
);
});
});
describe("and calling destroy", () => {
beforeEach(() => {
voiceBroadcastRecording.destroy();
});
it("should stop the recorder and remove all listeners", () => {
expect(mocked(voiceBroadcastRecorder.stop)).toHaveBeenCalled();
expect(mocked(voiceBroadcastRecorder.off)).toHaveBeenCalledWith(
VoiceBroadcastRecorderEvent.ChunkRecorded,
onChunkRecorded,
);
expect(mocked(voiceBroadcastRecording.removeAllListeners)).toHaveBeenCalled();
});
});
});
});
describe("when created for a Voice Broadcast Info with a Stopped relation", () => {
@ -152,7 +386,7 @@ describe("VoiceBroadcastRecording", () => {
});
it("should be in Stopped state", () => {
expect(voiceBroadcastRecording.state).toBe(VoiceBroadcastInfoState.Stopped);
expect(voiceBroadcastRecording.getState()).toBe(VoiceBroadcastInfoState.Stopped);
});
});
});

2
test/voice-broadcast/stores/VoiceBroadcastRecordingsStore-test.ts
View file

@ -76,7 +76,7 @@ describe("VoiceBroadcastRecordingsStore", () => {
});
it("should return it as current", () => {
expect(recordings.current).toBe(recording);
expect(recordings.getCurrent()).toBe(recording);
});
it("should return it by id", () => {

2
test/voice-broadcast/utils/startNewVoiceBroadcastRecording-test.ts
View file

@ -109,6 +109,7 @@ describe("startNewVoiceBroadcastRecording", () => {
return {
infoEvent,
client,
start: jest.fn(),
} as unknown as VoiceBroadcastRecording;
});
});
@ -120,6 +121,7 @@ describe("startNewVoiceBroadcastRecording", () => {
expect(ok).toBe(true);
expect(mocked(room.off)).toHaveBeenCalledWith(RoomStateEvent.Events, roomOnStateEventsCallback);
expect(recording.infoEvent).toBe(infoEvent);
expect(recording.start).toHaveBeenCalled();
done();
});