imterah/element-portable
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #5769 from matrix-org/travis/voice-messages/exp

Browse source 
Labs feature: Early implementation of voice messages
This commit is contained in:
Travis Ralston 2021-03-24 09:56:34 -06:00 • committed by GitHub
commit 8587ec888b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 378 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/@types/global.d.ts vendored
View file

@ -39,6 +39,7 @@ import {ModalWidgetStore} from "../stores/ModalWidgetStore";
import { WidgetLayoutStore } from "../stores/widgets/WidgetLayoutStore";
import VoipUserMapper from "../VoipUserMapper";
import {SpaceStoreClass} from "../stores/SpaceStore";
import {VoiceRecorder} from "../voice/VoiceRecorder";
declare global {
interface Window {
@ -70,6 +71,7 @@ declare global {
mxModalWidgetStore: ModalWidgetStore;
mxVoipUserMapper: VoipUserMapper;
mxSpaceStore: SpaceStoreClass;
mxVoiceRecorder: typeof VoiceRecorder;
}
interface Document {

4
src/components/views/messages/MessageEvent.js
View file

@ -71,6 +71,10 @@ export default class MessageEvent extends React.Component {
'm.file': sdk.getComponent('messages.MFileBody'),
'm.audio': sdk.getComponent('messages.MAudioBody'),
'm.video': sdk.getComponent('messages.MVideoBody'),
// TODO: @@ TravisR: Use labs flag determination.
// MSC: https://github.com/matrix-org/matrix-doc/pull/2516
'org.matrix.msc2516.voice': sdk.getComponent('messages.MAudioBody'),
};
const evTypes = {
'm.sticker': sdk.getComponent('messages.MStickerBody'),

5
src/components/views/rooms/BasicMessageComposer.tsx
View file

@ -93,6 +93,7 @@ interface IProps {
placeholder?: string;
label?: string;
initialCaret?: DocumentOffset;
disabled?: boolean;
onChange?();
onPaste?(event: ClipboardEvent<HTMLDivElement>, model: EditorModel): boolean;
@ -672,6 +673,9 @@ export default class BasicMessageEditor extends React.Component<IProps, IState>
});
const classes = classNames("mx_BasicMessageComposer_input", {
"mx_BasicMessageComposer_input_shouldShowPillAvatar": this.state.showPillAvatar,
// TODO: @@ TravisR: This doesn't work properly. The composer resets in a strange way.
"mx_BasicMessageComposer_input_disabled": this.props.disabled,
});
const shortcuts = {
@ -704,6 +708,7 @@ export default class BasicMessageEditor extends React.Component<IProps, IState>
aria-expanded={Boolean(this.state.autoComplete)}
aria-activedescendant={completionIndex >= 0 ? generateCompletionDomId(completionIndex) : undefined}
dir="auto"
aria-disabled={this.props.disabled}
/>
</div>);
}

29
src/components/views/rooms/MessageComposer.js
View file

@ -33,6 +33,7 @@ import WidgetStore from "../../../stores/WidgetStore";
import {UPDATE_EVENT} from "../../../stores/AsyncStore";
import ActiveWidgetStore from "../../../stores/ActiveWidgetStore";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import VoiceRecordComposerTile from "./VoiceRecordComposerTile";
function ComposerAvatar(props) {
const MemberStatusMessageAvatar = sdk.getComponent('avatars.MemberStatusMessageAvatar');
@ -187,6 +188,7 @@ export default class MessageComposer extends React.Component {
hasConference: WidgetStore.instance.doesRoomHaveConference(this.props.room),
joinedConference: WidgetStore.instance.isJoinedToConferenceIn(this.props.room),
isComposerEmpty: true,
haveRecording: false,
};
}
@ -325,6 +327,10 @@ export default class MessageComposer extends React.Component {
});
}
onVoiceUpdate = (haveRecording: boolean) => {
this.setState({haveRecording});
};
render() {
const controls = [
this.state.me ? <ComposerAvatar key="controls_avatar" me={this.state.me} /> : null,
@ -346,17 +352,32 @@ export default class MessageComposer extends React.Component {
permalinkCreator={this.props.permalinkCreator}
replyToEvent={this.props.replyToEvent}
onChange={this.onChange}
// TODO: @@ TravisR - Disabling the composer doesn't work
disabled={this.state.haveRecording}
/>,
<UploadButton key="controls_upload" roomId={this.props.room.roomId} />,
<EmojiButton key="emoji_button" addEmoji={this.addEmoji} />,
);
if (!this.state.haveRecording) {
controls.push(
<UploadButton key="controls_upload" roomId={this.props.room.roomId} />,
<EmojiButton key="emoji_button" addEmoji={this.addEmoji} />,
);
}
if (SettingsStore.getValue(UIFeature.Widgets) &&
SettingsStore.getValue("MessageComposerInput.showStickersButton")) {
SettingsStore.getValue("MessageComposerInput.showStickersButton") &&
!this.state.haveRecording) {
controls.push(<Stickerpicker key="stickerpicker_controls_button" room={this.props.room} />);
}
if (!this.state.isComposerEmpty) {
if (SettingsStore.getValue("feature_voice_messages")) {
controls.push(<VoiceRecordComposerTile
key="controls_voice_record"
room={this.props.room}
onRecording={this.onVoiceUpdate} />);
}
if (!this.state.isComposerEmpty || this.state.haveRecording) {
controls.push(
<SendButton key="controls_send" onClick={this.sendMessage} />,
);

2
src/components/views/rooms/SendMessageComposer.js
View file

@ -120,6 +120,7 @@ export default class SendMessageComposer extends React.Component {
permalinkCreator: PropTypes.object.isRequired,
replyToEvent: PropTypes.object,
onChange: PropTypes.func,
disabled: PropTypes.bool,
};
static contextType = MatrixClientContext;
@ -556,6 +557,7 @@ export default class SendMessageComposer extends React.Component {
label={this.props.placeholder}
placeholder={this.props.placeholder}
onPaste={this._onPaste}
disabled={this.props.disabled}
/>
</div>
);

88
src/components/views/rooms/VoiceRecordComposerTile.tsx Normal file
View file

@ -0,0 +1,88 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import AccessibleTooltipButton from "../elements/AccessibleTooltipButton";
import {_t} from "../../../languageHandler";
import React from "react";
import {VoiceRecorder} from "../../../voice/VoiceRecorder";
import {Room} from "matrix-js-sdk/src/models/room";
import {MatrixClientPeg} from "../../../MatrixClientPeg";
import classNames from "classnames";
interface IProps {
room: Room;
onRecording: (haveRecording: boolean) => void;
}
interface IState {
recorder?: VoiceRecorder;
}
export default class VoiceRecordComposerTile extends React.PureComponent<IProps, IState> {
public constructor(props) {
super(props);
this.state = {
recorder: null, // not recording by default
};
}
private onStartStopVoiceMessage = async () => {
// TODO: @@ TravisR: We do not want to auto-send on stop.
if (this.state.recorder) {
await this.state.recorder.stop();
const mxc = await this.state.recorder.upload();
MatrixClientPeg.get().sendMessage(this.props.room.roomId, {
body: "Voice message",
msgtype: "org.matrix.msc2516.voice",
url: mxc,
});
this.setState({recorder: null});
this.props.onRecording(false);
return;
}
const recorder = new VoiceRecorder(MatrixClientPeg.get());
await recorder.start();
this.props.onRecording(true);
// TODO: @@ TravisR: Run through EQ component
// recorder.frequencyData.onUpdate((freq) => {
// console.log('@@ UPDATE', freq);
// });
this.setState({recorder});
};
public render() {
const classes = classNames({
'mx_MessageComposer_button': !this.state.recorder,
'mx_MessageComposer_voiceMessage': !this.state.recorder,
'mx_VoiceRecordComposerTile_stop': !!this.state.recorder,
});
let tooltip = _t("Record a voice message");
if (!!this.state.recorder) {
// TODO: @@ TravisR: Change to match behaviour
tooltip = _t("Stop & send recording");
}
return (
<AccessibleTooltipButton
className={classes}
onClick={this.onStartStopVoiceMessage}
title={tooltip}
/>
);
}
}

3
src/i18n/strings/en_EN.json
View file

@ -783,6 +783,7 @@
"%(senderName)s: %(stickerName)s": "%(senderName)s: %(stickerName)s",
"Change notification settings": "Change notification settings",
"Spaces prototype. Incompatible with Communities, Communities v2 and Custom Tags. Requires compatible homeserver for some features.": "Spaces prototype. Incompatible with Communities, Communities v2 and Custom Tags. Requires compatible homeserver for some features.",
"Send and receive voice messages (in development)": "Send and receive voice messages (in development)",
"Render LaTeX maths in messages": "Render LaTeX maths in messages",
"Communities v2 prototypes. Requires compatible homeserver. Highly experimental - use with caution.": "Communities v2 prototypes. Requires compatible homeserver. Highly experimental - use with caution.",
"New spinner design": "New spinner design",
@ -1636,6 +1637,8 @@
"Invited by %(sender)s": "Invited by %(sender)s",
"Jump to first unread message.": "Jump to first unread message.",
"Mark all as read": "Mark all as read",
"Record a voice message": "Record a voice message",
"Stop & send recording": "Stop & send recording",
"Error updating main address": "Error updating main address",
"There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's main address. It may not be allowed by the server or a temporary failure occurred.",
"There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.": "There was an error updating the room's alternative addresses. It may not be allowed by the server or a temporary failure occurred.",

6
src/settings/Settings.ts
View file

@ -128,6 +128,12 @@ export const SETTINGS: {[setting: string]: ISetting} = {
default: false,
controller: new ReloadOnChangeController(),
},
"feature_voice_messages": {
isFeature: true,
displayName: _td("Send and receive voice messages (in development)"),
supportedLevels: LEVELS_FEATURE,
default: false,
},
"feature_latex_maths": {
isFeature: true,
displayName: _td("Render LaTeX maths in messages"),

180
src/voice/VoiceRecorder.ts Normal file
View file

@ -0,0 +1,180 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import * as Recorder from 'opus-recorder';
import encoderPath from 'opus-recorder/dist/encoderWorker.min.js';
import {MatrixClient} from "matrix-js-sdk/src/client";
import CallMediaHandler from "../CallMediaHandler";
import {SimpleObservable} from "matrix-widget-api";
const CHANNELS = 1; // stereo isn't important
const SAMPLE_RATE = 48000; // 48khz is what WebRTC uses. 12khz is where we lose quality.
const BITRATE = 24000; // 24kbps is pretty high quality for our use case in opus.
const FREQ_SAMPLE_RATE = 4; // Target rate of frequency data (samples / sec). We don't need this super often.
export interface IFrequencyPackage {
dbBars: Float32Array;
dbMin: number;
dbMax: number;
// TODO: @@ TravisR: Generalize this for a timing package?
}
export class VoiceRecorder {
private recorder: Recorder;
private recorderContext: AudioContext;
private recorderSource: MediaStreamAudioSourceNode;
private recorderStream: MediaStream;
private recorderFreqNode: AnalyserNode;
private buffer = new Uint8Array(0);
private mxc: string;
private recording = false;
private observable: SimpleObservable<IFrequencyPackage>;
private freqTimerId: number;
public constructor(private client: MatrixClient) {
}
private async makeRecorder() {
this.recorderStream = await navigator.mediaDevices.getUserMedia({
audio: {
// specify some audio settings so we're feeding the recorder with the
// best possible values. The browser will handle resampling for us.
sampleRate: SAMPLE_RATE,
channelCount: CHANNELS,
noiseSuppression: true, // browsers ignore constraints they can't honour
deviceId: CallMediaHandler.getAudioInput(),
},
});
this.recorderContext = new AudioContext({
latencyHint: "interactive",
sampleRate: SAMPLE_RATE, // once again, the browser will resample for us
});
this.recorderSource = this.recorderContext.createMediaStreamSource(this.recorderStream);
this.recorderFreqNode = this.recorderContext.createAnalyser();
this.recorderSource.connect(this.recorderFreqNode);
this.recorder = new Recorder({
encoderPath, // magic from webpack
encoderSampleRate: SAMPLE_RATE,
encoderApplication: 2048, // voice (default is "audio")
streamPages: true, // this speeds up the encoding process by using CPU over time
encoderFrameSize: 20, // ms, arbitrary frame size we send to the encoder
numberOfChannels: CHANNELS,
sourceNode: this.recorderSource,
encoderBitRate: BITRATE,
// We use low values for the following to ease CPU usage - the resulting waveform
// is indistinguishable for a voice message. Note that the underlying library will
// pick defaults which prefer the highest possible quality, CPU be damned.
encoderComplexity: 3, // 0-10, 10 is slow and high quality.
resampleQuality: 3, // 0-10, 10 is slow and high quality
});
this.recorder.ondataavailable = (a: ArrayBuffer) => {
const buf = new Uint8Array(a);
const newBuf = new Uint8Array(this.buffer.length + buf.length);
newBuf.set(this.buffer, 0);
newBuf.set(buf, this.buffer.length);
this.buffer = newBuf;
};
}
public get frequencyData(): SimpleObservable<IFrequencyPackage> {
if (!this.recording) throw new Error("No observable when not recording");
return this.observable;
}
public get isSupported(): boolean {
return !!Recorder.isRecordingSupported();
}
public get hasRecording(): boolean {
return this.buffer.length > 0;
}
public get mxcUri(): string {
if (!this.mxc) {
throw new Error("Recording has not been uploaded yet");
}
return this.mxc;
}
public async start(): Promise<void> {
if (this.mxc || this.hasRecording) {
throw new Error("Recording already prepared");
}
if (this.recording) {
throw new Error("Recording already in progress");
}
if (this.observable) {
this.observable.close();
}
this.observable = new SimpleObservable<IFrequencyPackage>();
await this.makeRecorder();
this.freqTimerId = setInterval(() => {
if (!this.recording) return;
const data = new Float32Array(this.recorderFreqNode.frequencyBinCount);
this.recorderFreqNode.getFloatFrequencyData(data);
this.observable.update({
dbBars: data,
dbMin: this.recorderFreqNode.minDecibels,
dbMax: this.recorderFreqNode.maxDecibels,
});
}, 1000 / FREQ_SAMPLE_RATE) as any as number; // XXX: Linter doesn't understand timer environment
await this.recorder.start();
this.recording = true;
}
public async stop(): Promise<Uint8Array> {
if (!this.recording) {
throw new Error("No recording to stop");
}
// Disconnect the source early to start shutting down resources
this.recorderSource.disconnect();
await this.recorder.stop();
// close the context after the recorder so the recorder doesn't try to
// connect anything to the context (this would generate a warning)
await this.recorderContext.close();
// Now stop all the media tracks so we can release them back to the user/OS
this.recorderStream.getTracks().forEach(t => t.stop());
// Finally do our post-processing and clean up
clearInterval(this.freqTimerId);
this.recording = false;
await this.recorder.close();
return this.buffer;
}
public async upload(): Promise<string> {
if (!this.hasRecording) {
throw new Error("No recording available to upload");
}
if (this.mxc) return this.mxc;
this.mxc = await this.client.uploadContent(new Blob([this.buffer], {
type: "audio/ogg",
}), {
onlyContentUri: false, // to stop the warnings in the console
}).then(r => r['content_uri']);
return this.mxc;
}
}
window.mxVoiceRecorder = VoiceRecorder;