From 4c98c0bc230e6678810865574a925c51de5ce04d Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 12 Jul 2021 14:02:51 -0600 Subject: [PATCH 1/3] Increase sample count in voice message thumbnail Fixes https://github.com/vector-im/element-web/issues/17817 Technically requires https://github.com/matrix-org/matrix-react-sdk/pull/6357 for sample sizing. --- src/components/views/rooms/VoiceRecordComposerTile.tsx | 2 +- src/voice/Playback.ts | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx index 5d984eacfa..701a5e99ef 100644 --- a/src/components/views/rooms/VoiceRecordComposerTile.tsx +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -95,7 +95,7 @@ export default class VoiceRecordComposerTile extends React.PureComponent Math.round(v * 1024)), + waveform: this.state.recorder.getPlayback().waveformThumbnail.map(v => Math.round(v * 1024)), }, "org.matrix.msc3245.voice": {}, // No content, this is a rendering hint }); diff --git a/src/voice/Playback.ts b/src/voice/Playback.ts index 6a120bf924..10e4ad6b7d 100644 --- a/src/voice/Playback.ts +++ b/src/voice/Playback.ts @@ -56,6 +56,7 @@ export class Playback extends EventEmitter implements IDestroyable { private state = PlaybackState.Decoding; private audioBuf: AudioBuffer; private resampledWaveform: number[]; + private thumbnailWaveform: number[]; private waveformObservable = new SimpleObservable(); private readonly clock: PlaybackClock; private readonly fileSize: number; @@ -72,6 +73,7 @@ export class Playback extends EventEmitter implements IDestroyable { this.fileSize = this.buf.byteLength; this.context = createAudioContext(); this.resampledWaveform = arrayFastResample(seedWaveform ?? DEFAULT_WAVEFORM, PLAYBACK_WAVEFORM_SAMPLES); + this.thumbnailWaveform = arrayFastResample(seedWaveform ?? DEFAULT_WAVEFORM, 100); this.waveformObservable.update(this.resampledWaveform); this.clock = new PlaybackClock(this.context); } @@ -92,6 +94,14 @@ export class Playback extends EventEmitter implements IDestroyable { return this.resampledWaveform; } + /** + * Stable waveform for representing a thumbnail of the media. Values are + * guaranteed to be between zero and one, inclusive. + */ + public get waveformThumbnail(): number[] { + return this.thumbnailWaveform; + } + public get waveformData(): SimpleObservable { return this.waveformObservable; } From 79aa205a95e49fb2171b18dadf36fe1ec7b190fe Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 12 Jul 2021 14:05:59 -0600 Subject: [PATCH 2/3] variablize --- src/voice/Playback.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/voice/Playback.ts b/src/voice/Playback.ts index 10e4ad6b7d..9f46530de4 100644 --- a/src/voice/Playback.ts +++ b/src/voice/Playback.ts @@ -31,6 +31,7 @@ export enum PlaybackState { } export const PLAYBACK_WAVEFORM_SAMPLES = 39; +const THUMBNAIL_WAVEFORM_SAMPLES = 100; // arbitrary: [30,120] const DEFAULT_WAVEFORM = arraySeed(0, PLAYBACK_WAVEFORM_SAMPLES); function makePlaybackWaveform(input: number[]): number[] { @@ -73,7 +74,7 @@ export class Playback extends EventEmitter implements IDestroyable { this.fileSize = this.buf.byteLength; this.context = createAudioContext(); this.resampledWaveform = arrayFastResample(seedWaveform ?? DEFAULT_WAVEFORM, PLAYBACK_WAVEFORM_SAMPLES); - this.thumbnailWaveform = arrayFastResample(seedWaveform ?? DEFAULT_WAVEFORM, 100); + this.thumbnailWaveform = arrayFastResample(seedWaveform ?? DEFAULT_WAVEFORM, THUMBNAIL_WAVEFORM_SAMPLES); this.waveformObservable.update(this.resampledWaveform); this.clock = new PlaybackClock(this.context); } From e045aa940e98a216f7443e1c60e2192208d3f88a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 12 Jul 2021 14:11:24 -0600 Subject: [PATCH 3/3] Be smart with readonly --- .../views/rooms/VoiceRecordComposerTile.tsx | 2 +- src/voice/Playback.ts | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx index 701a5e99ef..709eab82a0 100644 --- a/src/components/views/rooms/VoiceRecordComposerTile.tsx +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -95,7 +95,7 @@ export default class VoiceRecordComposerTile extends React.PureComponent Math.round(v * 1024)), + waveform: this.state.recorder.getPlayback().thumbnailWaveform.map(v => Math.round(v * 1024)), }, "org.matrix.msc3245.voice": {}, // No content, this is a rendering hint }); diff --git a/src/voice/Playback.ts b/src/voice/Playback.ts index 9f46530de4..1a1ee54466 100644 --- a/src/voice/Playback.ts +++ b/src/voice/Playback.ts @@ -52,12 +52,17 @@ function makePlaybackWaveform(input: number[]): number[] { } export class Playback extends EventEmitter implements IDestroyable { + /** + * Stable waveform for representing a thumbnail of the media. Values are + * guaranteed to be between zero and one, inclusive. + */ + public readonly thumbnailWaveform: number[]; + private readonly context: AudioContext; private source: AudioBufferSourceNode; private state = PlaybackState.Decoding; private audioBuf: AudioBuffer; private resampledWaveform: number[]; - private thumbnailWaveform: number[]; private waveformObservable = new SimpleObservable(); private readonly clock: PlaybackClock; private readonly fileSize: number; @@ -95,14 +100,6 @@ export class Playback extends EventEmitter implements IDestroyable { return this.resampledWaveform; } - /** - * Stable waveform for representing a thumbnail of the media. Values are - * guaranteed to be between zero and one, inclusive. - */ - public get waveformThumbnail(): number[] { - return this.thumbnailWaveform; - } - public get waveformData(): SimpleObservable { return this.waveformObservable; }