imterah/element-portable
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Populate info.duration for audio & video file uploads (#11225)

Browse source 
* Improve m.file m.image m.audio m.video types

* Populate `info.duration` for audio & video file uploads

* Fix tests

* Iterate types

* Improve coverage

* Fix test

* Add small delay to stabilise cypress test

* Fix test idempotency

* Improve coverage

* Slow down

* iterate
This commit is contained in:
Michael Telatynski 2023-07-17 13:07:58 +01:00 • committed by GitHub
parent 8b8ca425d7
commit f04a0e2860
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 556 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

87
src/ContentMessages.ts
View file

@ -33,7 +33,14 @@ import {
import { THREAD_RELATION_TYPE } from "matrix-js-sdk/src/models/thread";
import { removeElement } from "matrix-js-sdk/src/utils";
import { IEncryptedFile, IMediaEventContent, IMediaEventInfo } from "./customisations/models/IMediaEventContent";
import {
AudioInfo,
EncryptedFile,
ImageInfo,
IMediaEventContent,
IMediaEventInfo,
VideoInfo,
} from "./customisations/models/IMediaEventContent";
import dis from "./dispatcher/dispatcher";
import { _t } from "./languageHandler";
import Modal from "./Modal";
@ -146,11 +153,7 @@ const ALWAYS_INCLUDE_THUMBNAIL = ["image/avif", "image/webp"];
* @param {File} imageFile The image to read and thumbnail.
* @return {Promise} A promise that resolves with the attachment info.
*/
async function infoForImageFile(
matrixClient: MatrixClient,
roomId: string,
imageFile: File,
): Promise<Partial<IMediaEventInfo>> {
async function infoForImageFile(matrixClient: MatrixClient, roomId: string, imageFile: File): Promise<ImageInfo> {
let thumbnailType = "image/png";
if (imageFile.type === "image/jpeg") {
thumbnailType = "image/jpeg";
@ -184,16 +187,59 @@ async function infoForImageFile(
return imageInfo;
}
/**
* Load a file into a newly created audio element and load the metadata
*
* @param {File} audioFile The file to load in an audio element.
* @return {Promise} A promise that resolves with the audio element.
*/
function loadAudioElement(audioFile: File): Promise<HTMLAudioElement> {
return new Promise((resolve, reject) => {
// Load the file into a html element
const audio = document.createElement("audio");
audio.preload = "metadata";
audio.muted = true;
const reader = new FileReader();
reader.onload = function (ev): void {
audio.onloadedmetadata = async function (): Promise<void> {
resolve(audio);
};
audio.onerror = function (e): void {
reject(e);
};
audio.src = ev.target?.result as string;
};
reader.onerror = function (e): void {
reject(e);
};
reader.readAsDataURL(audioFile);
});
}
/**
* Read the metadata for an audio file.
*
* @param {File} audioFile The audio to read.
* @return {Promise} A promise that resolves with the attachment info.
*/
async function infoForAudioFile(audioFile: File): Promise<AudioInfo> {
const audio = await loadAudioElement(audioFile);
return { duration: Math.ceil(audio.duration * 1000) };
}
/**
* Load a file into a newly created video element and pull some strings
* in an attempt to guarantee the first frame will be showing.
*
* @param {File} videoFile The file to load in an video element.
* @return {Promise} A promise that resolves with the video image element.
* @param {File} videoFile The file to load in a video element.
* @return {Promise} A promise that resolves with the video element.
*/
function loadVideoElement(videoFile: File): Promise<HTMLVideoElement> {
return new Promise((resolve, reject) => {
// Load the file into an html element
// Load the file into a html element
const video = document.createElement("video");
video.preload = "metadata";
video.playsInline = true;
@ -237,20 +283,17 @@ function loadVideoElement(videoFile: File): Promise<HTMLVideoElement> {
* @param {File} videoFile The video to read and thumbnail.
* @return {Promise} A promise that resolves with the attachment info.
*/
function infoForVideoFile(
matrixClient: MatrixClient,
roomId: string,
videoFile: File,
): Promise<Partial<IMediaEventInfo>> {
function infoForVideoFile(matrixClient: MatrixClient, roomId: string, videoFile: File): Promise<VideoInfo> {
const thumbnailType = "image/jpeg";
let videoInfo: Partial<IMediaEventInfo>;
const videoInfo: VideoInfo = {};
return loadVideoElement(videoFile)
.then((video) => {
videoInfo.duration = Math.ceil(video.duration * 1000);
return createThumbnail(video, video.videoWidth, video.videoHeight, thumbnailType);
})
.then((result) => {
videoInfo = result.info;
Object.assign(videoInfo, result.info);
return uploadFile(matrixClient, roomId, result.thumbnail);
})
.then((result) => {
@ -299,7 +342,7 @@ export async function uploadFile(
file: File | Blob,
progressHandler?: UploadOpts["progressHandler"],
controller?: AbortController,
): Promise<{ url?: string; file?: IEncryptedFile }> {
): Promise<{ url?: string; file?: EncryptedFile }> {
const abortController = controller ?? new AbortController();
// If the room is encrypted then encrypt the file before uploading it.
@ -329,7 +372,7 @@ export async function uploadFile(
file: {
...encryptResult.info,
url,
} as IEncryptedFile,
} as EncryptedFile,
};
} else {
const { content_uri: url } = await matrixClient.uploadContent(file, { progressHandler, abortController });
@ -546,6 +589,14 @@ export default class ContentMessages {
}
} else if (file.type.indexOf("audio/") === 0) {
content.msgtype = MsgType.Audio;
try {
const audioInfo = await infoForAudioFile(file);
Object.assign(content.info, audioInfo);
} catch (e) {
// Failed to process audio file, fall back to uploading an m.file
logger.error(e);
content.msgtype = MsgType.File;
}
} else if (file.type.indexOf("video/") === 0) {
content.msgtype = MsgType.Video;
try {