imterah/element-portable
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #5935 from matrix-org/travis/voice/control-bar

Browse source 
Voice messages: Composer controls
This commit is contained in:
Travis Ralston 2021-04-28 09:38:28 -06:00 • committed by GitHub
commit 8d8525cdeb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 897 additions and 124 deletions
Download patch file Download diff file Expand all files Collapse all files

12
src/components/views/rooms/MessageComposer.tsx
View file

@ -198,6 +198,7 @@ interface IState {
export default class MessageComposer extends React.Component<IProps, IState> {
private dispatcherRef: string;
private messageComposerInput: SendMessageComposer;
private voiceRecordingButton: VoiceRecordComposerTile;
constructor(props) {
super(props);
@ -322,7 +323,15 @@ export default class MessageComposer extends React.Component<IProps, IState> {
});
}
sendMessage = () => {
sendMessage = async () => {
if (this.state.haveRecording && this.voiceRecordingButton) {
// There shouldn't be any text message to send when a voice recording is active, so
// just send out the voice recording.
await this.voiceRecordingButton.send();
return;
}
// XXX: Private function access
this.messageComposerInput._sendMessage();
}
@ -387,6 +396,7 @@ export default class MessageComposer extends React.Component<IProps, IState> {
if (SettingsStore.getValue("feature_voice_messages")) {
controls.push(<VoiceRecordComposerTile
key="controls_voice_record"
ref={c => this.voiceRecordingButton = c}
room={this.props.room} />);
}

179
src/components/views/rooms/VoiceRecordComposerTile.tsx
View file

@ -16,8 +16,8 @@ limitations under the License.
import AccessibleTooltipButton from "../elements/AccessibleTooltipButton";
import {_t} from "../../../languageHandler";
import React from "react";
import {VoiceRecording} from "../../../voice/VoiceRecording";
import React, {ReactNode} from "react";
import {RecordingState, VoiceRecording} from "../../../voice/VoiceRecording";
import {Room} from "matrix-js-sdk/src/models/room";
import {MatrixClientPeg} from "../../../MatrixClientPeg";
import classNames from "classnames";
@ -25,6 +25,8 @@ import LiveRecordingWaveform from "../voice_messages/LiveRecordingWaveform";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import LiveRecordingClock from "../voice_messages/LiveRecordingClock";
import {VoiceRecordingStore} from "../../../stores/VoiceRecordingStore";
import {UPDATE_EVENT} from "../../../stores/AsyncStore";
import RecordingPlayback from "../voice_messages/RecordingPlayback";
interface IProps {
room: Room;
@ -32,6 +34,7 @@ interface IProps {
interface IState {
recorder?: VoiceRecording;
recordingPhase?: RecordingState;
}
/**
@ -43,87 +46,141 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
super(props);
this.state = {
recorder: null, // not recording by default
recorder: null, // no recording started by default
};
}
private onStartStopVoiceMessage = async () => {
// TODO: @@ TravisR: We do not want to auto-send on stop.
public async componentWillUnmount() {
await VoiceRecordingStore.instance.disposeRecording();
}
// called by composer
public async send() {
if (!this.state.recorder) {
throw new Error("No recording started - cannot send anything");
}
await this.state.recorder.stop();
const mxc = await this.state.recorder.upload();
MatrixClientPeg.get().sendMessage(this.props.room.roomId, {
"body": "Voice message",
"msgtype": "org.matrix.msc2516.voice",
//"msgtype": MsgType.Audio,
"url": mxc,
"info": {
duration: Math.round(this.state.recorder.durationSeconds * 1000),
mimetype: this.state.recorder.contentType,
size: this.state.recorder.contentLength,
},
// MSC1767 experiment
"org.matrix.msc1767.text": "Voice message",
"org.matrix.msc1767.file": {
url: mxc,
name: "Voice message.ogg",
mimetype: this.state.recorder.contentType,
size: this.state.recorder.contentLength,
},
"org.matrix.msc1767.audio": {
duration: Math.round(this.state.recorder.durationSeconds * 1000),
// TODO: @@ TravisR: Waveform? (MSC1767 decision)
},
"org.matrix.experimental.msc2516.voice": { // MSC2516+MSC1767 experiment
duration: Math.round(this.state.recorder.durationSeconds * 1000),
// Events can't have floats, so we try to maintain resolution by using 1024
// as a maximum value. The waveform contains values between zero and 1, so this
// should come out largely sane.
//
// We're expecting about one data point per second of audio.
waveform: this.state.recorder.getPlayback().waveform.map(v => Math.round(v * 1024)),
},
});
await this.disposeRecording();
}
private async disposeRecording() {
await VoiceRecordingStore.instance.disposeRecording();
// Reset back to no recording, which means no phase (ie: restart component entirely)
this.setState({recorder: null, recordingPhase: null});
}
private onCancel = async () => {
await this.disposeRecording();
};
private onRecordStartEndClick = async () => {
if (this.state.recorder) {
await this.state.recorder.stop();
const mxc = await this.state.recorder.upload();
MatrixClientPeg.get().sendMessage(this.props.room.roomId, {
"body": "Voice message",
"msgtype": "org.matrix.msc2516.voice",
//"msgtype": MsgType.Audio,
"url": mxc,
"info": {
duration: Math.round(this.state.recorder.durationSeconds * 1000),
mimetype: this.state.recorder.contentType,
size: this.state.recorder.contentLength,
},
// MSC1767 experiment
"org.matrix.msc1767.text": "Voice message",
"org.matrix.msc1767.file": {
url: mxc,
name: "Voice message.ogg",
mimetype: this.state.recorder.contentType,
size: this.state.recorder.contentLength,
},
"org.matrix.msc1767.audio": {
duration: Math.round(this.state.recorder.durationSeconds * 1000),
// TODO: @@ TravisR: Waveform? (MSC1767 decision)
},
"org.matrix.experimental.msc2516.voice": { // MSC2516+MSC1767 experiment
duration: Math.round(this.state.recorder.durationSeconds * 1000),
// Events can't have floats, so we try to maintain resolution by using 1024
// as a maximum value. The waveform contains values between zero and 1, so this
// should come out largely sane.
//
// We're expecting about one data point per second of audio.
waveform: this.state.recorder.finalWaveform.map(v => Math.round(v * 1024)),
},
});
await VoiceRecordingStore.instance.disposeRecording();
this.setState({recorder: null});
return;
}
const recorder = VoiceRecordingStore.instance.startRecording();
await recorder.start();
this.setState({recorder});
// We don't need to remove the listener: the recorder will clean that up for us.
recorder.on(UPDATE_EVENT, (ev: RecordingState) => {
if (ev === RecordingState.EndingSoon) return; // ignore this state: it has no UI purpose here
this.setState({recordingPhase: ev});
});
this.setState({recorder, recordingPhase: RecordingState.Started});
};
private renderWaveformArea() {
if (!this.state.recorder) return null;
private renderWaveformArea(): ReactNode {
if (!this.state.recorder) return null; // no recorder means we're not recording: no waveform
return <div className='mx_VoiceRecordComposerTile_waveformContainer'>
if (this.state.recordingPhase !== RecordingState.Started) {
// TODO: @@ TR: Should we disable this during upload? What does a failed upload look like?
return <RecordingPlayback playback={this.state.recorder.getPlayback()} />;
}
// only other UI is the recording-in-progress UI
return <div className="mx_VoiceMessagePrimaryContainer mx_VoiceRecordComposerTile_recording">
<LiveRecordingClock recorder={this.state.recorder} />
<LiveRecordingWaveform recorder={this.state.recorder} />
</div>;
}
public render() {
const classes = classNames({
'mx_MessageComposer_button': !this.state.recorder,
'mx_MessageComposer_voiceMessage': !this.state.recorder,
'mx_VoiceRecordComposerTile_stop': !!this.state.recorder,
});
public render(): ReactNode {
let recordingInfo;
let deleteButton;
if (!this.state.recordingPhase || this.state.recordingPhase === RecordingState.Started) {
const classes = classNames({
'mx_MessageComposer_button': !this.state.recorder,
'mx_MessageComposer_voiceMessage': !this.state.recorder,
'mx_VoiceRecordComposerTile_stop': this.state.recorder?.isRecording,
});
let tooltip = _t("Record a voice message");
if (!!this.state.recorder) {
// TODO: @@ TravisR: Change to match behaviour
tooltip = _t("Stop & send recording");
let tooltip = _t("Record a voice message");
if (!!this.state.recorder) {
tooltip = _t("Stop the recording");
}
let stopOrRecordBtn = <AccessibleTooltipButton
className={classes}
onClick={this.onRecordStartEndClick}
title={tooltip}
/>;
if (this.state.recorder && !this.state.recorder?.isRecording) {
stopOrRecordBtn = null;
}
recordingInfo = stopOrRecordBtn;
}
if (this.state.recorder && this.state.recordingPhase !== RecordingState.Uploading) {
deleteButton = <AccessibleTooltipButton
className='mx_VoiceRecordComposerTile_delete'
title={_t("Delete recording")}
onClick={this.onCancel}
/>;
}
return (<>
{deleteButton}
{this.renderWaveformArea()}
<AccessibleTooltipButton
className={classes}
onClick={this.onStartStopVoiceMessage}
title={tooltip}
/>
{recordingInfo}
</>);
}
}

10
src/components/views/voice_messages/Clock.tsx
View file

@ -29,14 +29,20 @@ interface IState {
* displayed, making it possible to see "82:29".
*/
@replaceableComponent("views.voice_messages.Clock")
export default class Clock extends React.PureComponent<IProps, IState> {
export default class Clock extends React.Component<IProps, IState> {
public constructor(props) {
super(props);
}
shouldComponentUpdate(nextProps: Readonly<IProps>, nextState: Readonly<IState>, nextContext: any): boolean {
const currentFloor = Math.floor(this.props.seconds);
const nextFloor = Math.floor(nextProps.seconds);
return currentFloor !== nextFloor;
}
public render() {
const minutes = Math.floor(this.props.seconds / 60).toFixed(0).padStart(2, '0');
const seconds = Math.round(this.props.seconds % 60).toFixed(0).padStart(2, '0'); // hide millis
const seconds = Math.floor(this.props.seconds % 60).toFixed(0).padStart(2, '0'); // hide millis
return <span className='mx_Clock'>{minutes}:{seconds}</span>;
}
}

8
src/components/views/voice_messages/LiveRecordingClock.tsx
View file

@ -31,7 +31,7 @@ interface IState {
* A clock for a live recording.
*/
@replaceableComponent("views.voice_messages.LiveRecordingClock")
export default class LiveRecordingClock extends React.Component<IProps, IState> {
export default class LiveRecordingClock extends React.PureComponent<IProps, IState> {
public constructor(props) {
super(props);
@ -39,12 +39,6 @@ export default class LiveRecordingClock extends React.Component<IProps, IState>
this.props.recorder.liveData.onUpdate(this.onRecordingUpdate);
}
shouldComponentUpdate(nextProps: Readonly<IProps>, nextState: Readonly<IState>, nextContext: any): boolean {
const currentFloor = Math.floor(this.state.seconds);
const nextFloor = Math.floor(nextState.seconds);
return currentFloor !== nextFloor;
}
private onRecordingUpdate = (update: IRecordingUpdate) => {
this.setState({seconds: update.timeSeconds});
};

7
src/components/views/voice_messages/LiveRecordingWaveform.tsx
View file

@ -20,6 +20,7 @@ import {replaceableComponent} from "../../../utils/replaceableComponent";
import {arrayFastResample, arraySeed} from "../../../utils/arrays";
import {percentageOf} from "../../../utils/numbers";
import Waveform from "./Waveform";
import {PLAYBACK_WAVEFORM_SAMPLES} from "../../../voice/Playback";
interface IProps {
recorder: VoiceRecording;
@ -29,8 +30,6 @@ interface IState {
heights: number[];
}
const DOWNSAMPLE_TARGET = 35; // number of bars we want
/**
* A waveform which shows the waveform of a live recording
*/
@ -39,14 +38,14 @@ export default class LiveRecordingWaveform extends React.PureComponent<IProps, I
public constructor(props) {
super(props);
this.state = {heights: arraySeed(0, DOWNSAMPLE_TARGET)};
this.state = {heights: arraySeed(0, PLAYBACK_WAVEFORM_SAMPLES)};
this.props.recorder.liveData.onUpdate(this.onRecordingUpdate);
}
private onRecordingUpdate = (update: IRecordingUpdate) => {
// The waveform and the downsample target are pretty close, so we should be fine to
// do this, despite the docs on arrayFastResample.
const bars = arrayFastResample(Array.from(update.waveform), DOWNSAMPLE_TARGET);
const bars = arrayFastResample(Array.from(update.waveform), PLAYBACK_WAVEFORM_SAMPLES);
this.setState({
// The incoming data is between zero and one, but typically even screaming into a
// microphone won't send you over 0.6, so we artificially adjust the gain for the

61
src/components/views/voice_messages/PlayPauseButton.tsx Normal file
View file

@ -0,0 +1,61 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React, {ReactNode} from "react";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import AccessibleTooltipButton from "../elements/AccessibleTooltipButton";
import {_t} from "../../../languageHandler";
import {Playback, PlaybackState} from "../../../voice/Playback";
import classNames from "classnames";
interface IProps {
// Playback instance to manipulate. Cannot change during the component lifecycle.
playback: Playback;
// The playback phase to render. Able to change during the component lifecycle.
playbackPhase: PlaybackState;
}
/**
* Displays a play/pause button (activating the play/pause function of the recorder)
* to be displayed in reference to a recording.
*/
@replaceableComponent("views.voice_messages.PlayPauseButton")
export default class PlayPauseButton extends React.PureComponent<IProps> {
public constructor(props) {
super(props);
}
private onClick = async () => {
await this.props.playback.toggle();
};
public render(): ReactNode {
const isPlaying = this.props.playback.isPlaying;
const isDisabled = this.props.playbackPhase === PlaybackState.Decoding;
const classes = classNames('mx_PlayPauseButton', {
'mx_PlayPauseButton_play': !isPlaying,
'mx_PlayPauseButton_pause': isPlaying,
'mx_PlayPauseButton_disabled': isDisabled,
});
return <AccessibleTooltipButton
className={classes}
title={isPlaying ? _t("Pause") : _t("Play")}
onClick={this.onClick}
disabled={isDisabled}
/>;
}
}

71
src/components/views/voice_messages/PlaybackClock.tsx Normal file
View file

@ -0,0 +1,71 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React from "react";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import Clock from "./Clock";
import {Playback, PlaybackState} from "../../../voice/Playback";
import {UPDATE_EVENT} from "../../../stores/AsyncStore";
interface IProps {
playback: Playback;
}
interface IState {
seconds: number;
durationSeconds: number;
playbackPhase: PlaybackState;
}
/**
* A clock for a playback of a recording.
*/
@replaceableComponent("views.voice_messages.PlaybackClock")
export default class PlaybackClock extends React.PureComponent<IProps, IState> {
public constructor(props) {
super(props);
this.state = {
seconds: this.props.playback.clockInfo.timeSeconds,
// we track the duration on state because we won't really know what the clip duration
// is until the first time update, and as a PureComponent we are trying to dedupe state
// updates as much as possible. This is just the easiest way to avoid a forceUpdate() or
// member property to track "did we get a duration".
durationSeconds: this.props.playback.clockInfo.durationSeconds,
playbackPhase: PlaybackState.Stopped, // assume not started, so full clock
};
this.props.playback.on(UPDATE_EVENT, this.onPlaybackUpdate);
this.props.playback.clockInfo.liveData.onUpdate(this.onTimeUpdate);
}
private onPlaybackUpdate = (ev: PlaybackState) => {
// Convert Decoding -> Stopped because we don't care about the distinction here
if (ev === PlaybackState.Decoding) ev = PlaybackState.Stopped;
this.setState({playbackPhase: ev});
};
private onTimeUpdate = (time: number[]) => {
this.setState({seconds: time[0], durationSeconds: time[1]});
};
public render() {
let seconds = this.state.seconds;
if (this.state.playbackPhase === PlaybackState.Stopped) {
seconds = this.state.durationSeconds;
}
return <Clock seconds={seconds} />;
}
}

68
src/components/views/voice_messages/PlaybackWaveform.tsx Normal file
View file

@ -0,0 +1,68 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React from "react";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import {arraySeed, arrayTrimFill} from "../../../utils/arrays";
import Waveform from "./Waveform";
import {Playback, PLAYBACK_WAVEFORM_SAMPLES} from "../../../voice/Playback";
import {percentageOf} from "../../../utils/numbers";
interface IProps {
playback: Playback;
}
interface IState {
heights: number[];
progress: number;
}
/**
* A waveform which shows the waveform of a previously recorded recording
*/
@replaceableComponent("views.voice_messages.PlaybackWaveform")
export default class PlaybackWaveform extends React.PureComponent<IProps, IState> {
public constructor(props) {
super(props);
this.state = {
heights: this.toHeights(this.props.playback.waveform),
progress: 0, // default no progress
};
this.props.playback.waveformData.onUpdate(this.onWaveformUpdate);
this.props.playback.clockInfo.liveData.onUpdate(this.onTimeUpdate);
}
private toHeights(waveform: number[]) {
const seed = arraySeed(0, PLAYBACK_WAVEFORM_SAMPLES);
return arrayTrimFill(waveform, PLAYBACK_WAVEFORM_SAMPLES, seed);
}
private onWaveformUpdate = (waveform: number[]) => {
this.setState({heights: this.toHeights(waveform)});
};
private onTimeUpdate = (time: number[]) => {
// Track percentages to very coarse precision, otherwise 0.002 ends up highlighting a bar.
const progress = Number(percentageOf(time[0], 0, time[1]).toFixed(1));
this.setState({progress});
};
public render() {
return <Waveform relHeights={this.state.heights} progress={this.state.progress} />;
}
}

62
src/components/views/voice_messages/RecordingPlayback.tsx Normal file
View file

@ -0,0 +1,62 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import {Playback, PlaybackState} from "../../../voice/Playback";
import React, {ReactNode} from "react";
import {UPDATE_EVENT} from "../../../stores/AsyncStore";
import PlaybackWaveform from "./PlaybackWaveform";
import PlayPauseButton from "./PlayPauseButton";
import PlaybackClock from "./PlaybackClock";
interface IProps {
// Playback instance to render. Cannot change during component lifecycle: create
// an all-new component instead.
playback: Playback;
}
interface IState {
playbackPhase: PlaybackState;
}
export default class RecordingPlayback extends React.PureComponent<IProps, IState> {
constructor(props: IProps) {
super(props);
this.state = {
playbackPhase: PlaybackState.Decoding, // default assumption
};
// We don't need to de-register: the class handles this for us internally
this.props.playback.on(UPDATE_EVENT, this.onPlaybackUpdate);
// Don't wait for the promise to complete - it will emit a progress update when it
// is done, and it's not meant to take long anyhow.
// noinspection JSIgnoredPromiseFromCall
this.props.playback.prepare();
}
private onPlaybackUpdate = (ev: PlaybackState) => {
this.setState({playbackPhase: ev});
};
public render(): ReactNode {
return <div className='mx_VoiceMessagePrimaryContainer'>
<PlayPauseButton playback={this.props.playback} playbackPhase={this.state.playbackPhase} />
<PlaybackClock playback={this.props.playback} />
<PlaybackWaveform playback={this.props.playback} />
</div>
}
}

17
src/components/views/voice_messages/Waveform.tsx
View file

@ -16,9 +16,11 @@ limitations under the License.
import React from "react";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import classNames from "classnames";
interface IProps {
relHeights: number[]; // relative heights (0-1)
progress: number; // percent complete, 0-1, default 100%
}
interface IState {
@ -28,9 +30,16 @@ interface IState {
* A simple waveform component. This renders bars (centered vertically) for each
* height provided in the component properties. Updating the properties will update
* the rendered waveform.
*
* For CSS purposes, a mx_Waveform_bar_100pct class is added when the bar should be
* "filled", as a demonstration of the progress property.
*/
@replaceableComponent("views.voice_messages.Waveform")
export default class Waveform extends React.PureComponent<IProps, IState> {
public static defaultProps = {
progress: 1,
};
public constructor(props) {
super(props);
}
@ -38,7 +47,13 @@ export default class Waveform extends React.PureComponent<IProps, IState> {
public render() {
return <div className='mx_Waveform'>
{this.props.relHeights.map((h, i) => {
return <span key={i} style={{height: (h * 100) + '%'}} className='mx_Waveform_bar' />;
const progress = this.props.progress;
const isCompleteBar = (i / this.props.relHeights.length) <= progress && progress > 0;
const classes = classNames({
'mx_Waveform_bar': true,
'mx_Waveform_bar_100pct': isCompleteBar,
});
return <span key={i} style={{height: (h * 100) + '%'}} className={classes} />;
})}
</div>;
}