imterah/element-portable
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Populate info.duration for audio & video file uploads (#11225)

Browse source 
* Improve m.file m.image m.audio m.video types

* Populate `info.duration` for audio & video file uploads

* Fix tests

* Iterate types

* Improve coverage

* Fix test

* Add small delay to stabilise cypress test

* Fix test idempotency

* Improve coverage

* Slow down

* iterate
This commit is contained in:
Michael Telatynski 2023-07-17 13:07:58 +01:00 • committed by GitHub
parent 8b8ca425d7
commit f04a0e2860
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 556 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

87
src/ContentMessages.ts
View file

@ -33,7 +33,14 @@ import {
import { THREAD_RELATION_TYPE } from "matrix-js-sdk/src/models/thread";
import { removeElement } from "matrix-js-sdk/src/utils";
import { IEncryptedFile, IMediaEventContent, IMediaEventInfo } from "./customisations/models/IMediaEventContent";
import {
AudioInfo,
EncryptedFile,
ImageInfo,
IMediaEventContent,
IMediaEventInfo,
VideoInfo,
} from "./customisations/models/IMediaEventContent";
import dis from "./dispatcher/dispatcher";
import { _t } from "./languageHandler";
import Modal from "./Modal";
@ -146,11 +153,7 @@ const ALWAYS_INCLUDE_THUMBNAIL = ["image/avif", "image/webp"];
* @param {File} imageFile The image to read and thumbnail.
* @return {Promise} A promise that resolves with the attachment info.
*/
async function infoForImageFile(
matrixClient: MatrixClient,
roomId: string,
imageFile: File,
): Promise<Partial<IMediaEventInfo>> {
async function infoForImageFile(matrixClient: MatrixClient, roomId: string, imageFile: File): Promise<ImageInfo> {
let thumbnailType = "image/png";
if (imageFile.type === "image/jpeg") {
thumbnailType = "image/jpeg";
@ -184,16 +187,59 @@ async function infoForImageFile(
return imageInfo;
}
/**
* Load a file into a newly created audio element and load the metadata
*
* @param {File} audioFile The file to load in an audio element.
* @return {Promise} A promise that resolves with the audio element.
*/
function loadAudioElement(audioFile: File): Promise<HTMLAudioElement> {
return new Promise((resolve, reject) => {
// Load the file into a html element
const audio = document.createElement("audio");
audio.preload = "metadata";
audio.muted = true;
const reader = new FileReader();
reader.onload = function (ev): void {
audio.onloadedmetadata = async function (): Promise<void> {
resolve(audio);
};
audio.onerror = function (e): void {
reject(e);
};
audio.src = ev.target?.result as string;
};
reader.onerror = function (e): void {
reject(e);
};
reader.readAsDataURL(audioFile);
});
}
/**
* Read the metadata for an audio file.
*
* @param {File} audioFile The audio to read.
* @return {Promise} A promise that resolves with the attachment info.
*/
async function infoForAudioFile(audioFile: File): Promise<AudioInfo> {
const audio = await loadAudioElement(audioFile);
return { duration: Math.ceil(audio.duration * 1000) };
}
/**
* Load a file into a newly created video element and pull some strings
* in an attempt to guarantee the first frame will be showing.
*
* @param {File} videoFile The file to load in an video element.
* @return {Promise} A promise that resolves with the video image element.
* @param {File} videoFile The file to load in a video element.
* @return {Promise} A promise that resolves with the video element.
*/
function loadVideoElement(videoFile: File): Promise<HTMLVideoElement> {
return new Promise((resolve, reject) => {
// Load the file into an html element
// Load the file into a html element
const video = document.createElement("video");
video.preload = "metadata";
video.playsInline = true;
@ -237,20 +283,17 @@ function loadVideoElement(videoFile: File): Promise<HTMLVideoElement> {
* @param {File} videoFile The video to read and thumbnail.
* @return {Promise} A promise that resolves with the attachment info.
*/
function infoForVideoFile(
matrixClient: MatrixClient,
roomId: string,
videoFile: File,
): Promise<Partial<IMediaEventInfo>> {
function infoForVideoFile(matrixClient: MatrixClient, roomId: string, videoFile: File): Promise<VideoInfo> {
const thumbnailType = "image/jpeg";
let videoInfo: Partial<IMediaEventInfo>;
const videoInfo: VideoInfo = {};
return loadVideoElement(videoFile)
.then((video) => {
videoInfo.duration = Math.ceil(video.duration * 1000);
return createThumbnail(video, video.videoWidth, video.videoHeight, thumbnailType);
})
.then((result) => {
videoInfo = result.info;
Object.assign(videoInfo, result.info);
return uploadFile(matrixClient, roomId, result.thumbnail);
})
.then((result) => {
@ -299,7 +342,7 @@ export async function uploadFile(
file: File | Blob,
progressHandler?: UploadOpts["progressHandler"],
controller?: AbortController,
): Promise<{ url?: string; file?: IEncryptedFile }> {
): Promise<{ url?: string; file?: EncryptedFile }> {
const abortController = controller ?? new AbortController();
// If the room is encrypted then encrypt the file before uploading it.
@ -329,7 +372,7 @@ export async function uploadFile(
file: {
...encryptResult.info,
url,
} as IEncryptedFile,
} as EncryptedFile,
};
} else {
const { content_uri: url } = await matrixClient.uploadContent(file, { progressHandler, abortController });
@ -546,6 +589,14 @@ export default class ContentMessages {
}
} else if (file.type.indexOf("audio/") === 0) {
content.msgtype = MsgType.Audio;
try {
const audioInfo = await infoForAudioFile(file);
Object.assign(content.info, audioInfo);
} catch (e) {
// Failed to process audio file, fall back to uploading an m.file
logger.error(e);
content.msgtype = MsgType.File;
}
} else if (file.type.indexOf("video/") === 0) {
content.msgtype = MsgType.Video;
try {

2
src/components/views/messages/MFileBody.tsx
View file

@ -198,7 +198,7 @@ export default class MFileBody extends React.Component<IProps, IState> {
const isEncrypted = this.props.mediaEventHelper?.media.isEncrypted;
const contentUrl = this.getContentUrl();
const contentFileSize = this.content.info ? this.content.info.size : null;
const fileType = this.content.info ? this.content.info.mimetype : "application/octet-stream";
const fileType = this.content.info?.mimetype ?? "application/octet-stream";
let placeholder: React.ReactNode = null;
if (this.props.showGenericPlaceholder) {

22
src/components/views/messages/MImageBody.tsx
View file

@ -29,7 +29,7 @@ import SettingsStore from "../../../settings/SettingsStore";
import Spinner from "../elements/Spinner";
import { Media, mediaFromContent } from "../../../customisations/Media";
import { BLURHASH_FIELD, createThumbnail } from "../../../utils/image-media";
import { IMediaEventContent } from "../../../customisations/models/IMediaEventContent";
import { ImageContent } from "../../../customisations/models/IMediaEventContent";
import ImageView from "../elements/ImageView";
import { IBodyProps } from "./IBodyProps";
import { ImageSize, suggestedSize as suggestedImageSize } from "../../../settings/enums/ImageSize";
@ -102,7 +102,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
return;
}
const content = this.props.mxEvent.getContent<IMediaEventContent>();
const content = this.props.mxEvent.getContent<ImageContent>();
const httpUrl = this.state.contentUrl;
if (!httpUrl) return;
const params: Omit<ComponentProps<typeof ImageView>, "onFinished"> = {
@ -212,7 +212,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
const thumbWidth = 800;
const thumbHeight = 600;
const content = this.props.mxEvent.getContent<IMediaEventContent>();
const content = this.props.mxEvent.getContent<ImageContent>();
const media = mediaFromContent(content);
const info = content.info;
@ -287,7 +287,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
contentUrl = this.getContentUrl();
}
const content = this.props.mxEvent.getContent<IMediaEventContent>();
const content = this.props.mxEvent.getContent<ImageContent>();
let isAnimated = mayBeAnimated(content.info?.mimetype);
// If there is no included non-animated thumbnail then we will generate our own, we can't depend on the server
@ -317,7 +317,13 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
}
if (isAnimated) {
const thumb = await createThumbnail(img, img.width, img.height, content.info!.mimetype, false);
const thumb = await createThumbnail(
img,
img.width,
img.height,
content.info?.mimetype ?? "image/jpeg",
false,
);
thumbUrl = URL.createObjectURL(thumb.thumbnail);
}
} catch (error) {
@ -381,7 +387,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
}
}
protected getBanner(content: IMediaEventContent): ReactNode {
protected getBanner(content: ImageContent): ReactNode {
// Hide it for the threads list & the file panel where we show it as text anyway.
if (
[TimelineRenderingType.ThreadsList, TimelineRenderingType.File].includes(this.context.timelineRenderingType)
@ -395,7 +401,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
protected messageContent(
contentUrl: string | null,
thumbUrl: string | null,
content: IMediaEventContent,
content: ImageContent,
forcedHeight?: number,
): ReactNode {
if (!thumbUrl) thumbUrl = contentUrl; // fallback
@ -591,7 +597,7 @@ export default class MImageBody extends React.Component<IBodyProps, IState> {
}
public render(): React.ReactNode {
const content = this.props.mxEvent.getContent<IMediaEventContent>();
const content = this.props.mxEvent.getContent<ImageContent>();
if (this.state.error) {
let errorText = _t("Unable to show image due to error");

4
src/components/views/messages/MImageReplyBody.tsx
View file

@ -17,7 +17,7 @@ limitations under the License.
import React from "react";
import MImageBody from "./MImageBody";
import { IMediaEventContent } from "../../../customisations/models/IMediaEventContent";
import { ImageContent } from "../../../customisations/models/IMediaEventContent";
const FORCED_IMAGE_HEIGHT = 44;
@ -35,7 +35,7 @@ export default class MImageReplyBody extends MImageBody {
return super.render();
}
const content = this.props.mxEvent.getContent<IMediaEventContent>();
const content = this.props.mxEvent.getContent<ImageContent>();
const thumbnail = this.state.contentUrl
? this.messageContent(this.state.contentUrl, this.state.thumbUrl, content, FORCED_IMAGE_HEIGHT)
: undefined;

223
src/customisations/models/IMediaEventContent.ts
View file

@ -16,8 +16,21 @@
// TODO: These types should be elsewhere.
export interface IEncryptedFile {
import { MsgType } from "matrix-js-sdk/src/matrix";
import { BLURHASH_FIELD } from "../../utils/image-media";
/**
* @see https://spec.matrix.org/v1.7/client-server-api/#extensions-to-mroommessage-msgtypes
*/
export interface EncryptedFile {
/**
* The URL to the file.
*/
url: string;
/**
* A JSON Web Key object.
*/
key: {
alg: string;
key_ops: string[]; // eslint-disable-line camelcase
@ -25,43 +38,204 @@ export interface IEncryptedFile {
k: string;
ext: boolean;
};
/**
* The 128-bit unique counter block used by AES-CTR, encoded as unpadded base64.
*/
iv: string;
/**
* A map from an algorithm name to a hash of the ciphertext, encoded as unpadded base64.
* Clients should support the SHA-256 hash, which uses the key sha256.
*/
hashes: { [alg: string]: string };
/**
* Version of the encrypted attachment's protocol. Must be v2.
*/
v: string;
}
export interface IMediaEventInfo {
thumbnail_url?: string; // eslint-disable-line camelcase
thumbnail_file?: IEncryptedFile; // eslint-disable-line camelcase
thumbnail_info?: {
// eslint-disable-line camelcase
mimetype: string;
w?: number;
h?: number;
size?: number;
};
mimetype: string;
interface ThumbnailInfo {
/**
* The mimetype of the image, e.g. image/jpeg.
*/
mimetype?: string;
/**
* The intended display width of the image in pixels.
* This may differ from the intrinsic dimensions of the image file.
*/
w?: number;
/**
* The intended display height of the image in pixels.
* This may differ from the intrinsic dimensions of the image file.
*/
h?: number;
/**
* Size of the image in bytes.
*/
size?: number;
}
export interface IMediaEventContent {
msgtype: string;
body?: string;
filename?: string; // `m.file` optional field
url?: string; // required on unencrypted media
file?: IEncryptedFile; // required for *encrypted* media
info?: IMediaEventInfo;
interface BaseInfo {
mimetype?: string;
size?: number;
}
/**
* @see https://spec.matrix.org/v1.7/client-server-api/#mfile
*/
export interface FileInfo extends BaseInfo {
/**
* @see https://github.com/matrix-org/matrix-spec-proposals/pull/2448
*/
[BLURHASH_FIELD]?: string;
/**
* Information on the encrypted thumbnail file, as specified in End-to-end encryption.
* Only present if the thumbnail is encrypted.
* @see https://spec.matrix.org/v1.7/client-server-api/#sending-encrypted-attachments
*/
thumbnail_file?: EncryptedFile;
/**
* Metadata about the image referred to in thumbnail_url.
*/
thumbnail_info?: ThumbnailInfo;
/**
* The URL to the thumbnail of the file. Only present if the thumbnail is unencrypted.
*/
thumbnail_url?: string;
}
/**
* @see https://spec.matrix.org/v1.7/client-server-api/#mimage
*
*/
export interface ImageInfo extends FileInfo, ThumbnailInfo {}
/**
* @see https://spec.matrix.org/v1.7/client-server-api/#mimage
*/
export interface AudioInfo extends BaseInfo {
/**
* The duration of the audio in milliseconds.
*/
duration?: number;
}
/**
* @see https://spec.matrix.org/v1.7/client-server-api/#mvideo
*/
export interface VideoInfo extends AudioInfo, ImageInfo {
/**
* The duration of the video in milliseconds.
*/
duration?: number;
}
export type IMediaEventInfo = FileInfo | ImageInfo | AudioInfo | VideoInfo;
interface BaseContent {
/**
* Required if the file is encrypted. Information on the encrypted file, as specified in End-to-end encryption.
* @see https://spec.matrix.org/v1.7/client-server-api/#sending-encrypted-attachments
*/
file?: EncryptedFile;
/**
* Required if the file is unencrypted. The URL (typically mxc:// URI) to the file.
*/
url?: string;
}
/**
* @see https://spec.matrix.org/v1.7/client-server-api/#mfile
*/
export interface FileContent extends BaseContent {
/**
* A human-readable description of the file.
* This is recommended to be the filename of the original upload.
*/
body: string;
/**
* The original filename of the uploaded file.
*/
filename?: string;
/**
* Information about the file referred to in url.
*/
info?: FileInfo;
/**
* One of: [m.file].
*/
msgtype: MsgType.File;
}
/**
* @see https://spec.matrix.org/v1.7/client-server-api/#mimage
*/
export interface ImageContent extends BaseContent {
/**
* A textual representation of the image.
* This could be the alt text of the image, the filename of the image,
* or some kind of content description for accessibility e.g. ‘image attachment’.
*/
body: string;
/**
* Metadata about the image referred to in url.
*/
info?: ImageInfo;
/**
* One of: [m.image].
*/
msgtype: MsgType.Image;
}
/**
* @see https://spec.matrix.org/v1.7/client-server-api/#maudio
*/
export interface AudioContent extends BaseContent {
/**
* A description of the audio e.g. ‘Bee Gees - Stayin’ Alive’,
* or some kind of content description for accessibility e.g. ‘audio attachment’.
*/
body: string;
/**
* Metadata for the audio clip referred to in url.
*/
info?: AudioInfo;
/**
* One of: [m.audio].
*/
msgtype: MsgType.Audio;
}
/**
* @see https://spec.matrix.org/v1.7/client-server-api/#mvideo
*/
export interface VideoContent extends BaseContent {
/**
* A description of the video e.g. ‘Gangnam style’,
* or some kind of content description for accessibility e.g. ‘video attachment’.
*/
body: string;
/**
* Metadata about the video clip referred to in url.
*/
info?: VideoInfo;
/**
* One of: [m.video].
*/
msgtype: MsgType.Video;
}
/**
* Type representing media event contents for `m.room.message` events listed in the Matrix specification
*/
export type IMediaEventContent = FileContent | ImageContent | AudioContent | VideoContent;
export interface IPreparedMedia extends IMediaObject {
thumbnail?: IMediaObject;
}
export interface IMediaObject {
mxc: string;
file?: IEncryptedFile;
file?: EncryptedFile;
}
/**
@ -73,12 +247,17 @@ export interface IMediaObject {
*/
export function prepEventContentAsMedia(content: Partial<IMediaEventContent>): IPreparedMedia {
let thumbnail: IMediaObject | undefined;
if (content?.info?.thumbnail_url) {
if (typeof content?.info === "object" && "thumbnail_url" in content.info && content.info.thumbnail_url) {
thumbnail = {
mxc: content.info.thumbnail_url,
file: content.info.thumbnail_file,
};
} else if (content?.info?.thumbnail_file?.url) {
} else if (
typeof content?.info === "object" &&
"thumbnail_file" in content.info &&
typeof content?.info?.thumbnail_file === "object" &&
content?.info?.thumbnail_file?.url
) {
thumbnail = {
mxc: content.info.thumbnail_file.url,
file: content.info.thumbnail_file,

4
src/models/RoomUpload.ts
View file

@ -16,11 +16,11 @@ limitations under the License.
import { IEventRelation, UploadProgress } from "matrix-js-sdk/src/matrix";
import { IEncryptedFile } from "../customisations/models/IMediaEventContent";
import { EncryptedFile } from "../customisations/models/IMediaEventContent";
export class RoomUpload {
public readonly abortController = new AbortController();
public promise?: Promise<{ url?: string; file?: IEncryptedFile }>;
public promise?: Promise<{ url?: string; file?: EncryptedFile }>;
private uploaded = 0;
public constructor(

6
src/utils/DecryptFile.ts
View file

@ -19,7 +19,7 @@ import encrypt from "matrix-encrypt-attachment";
import { parseErrorResponse } from "matrix-js-sdk/src/http-api";
import { mediaFromContent } from "../customisations/Media";
import { IEncryptedFile, IMediaEventInfo } from "../customisations/models/IMediaEventContent";
import { EncryptedFile, IMediaEventInfo } from "../customisations/models/IMediaEventContent";
import { getBlobSafeMimeType } from "./blobs";
export class DownloadError extends Error {
@ -40,14 +40,14 @@ export class DecryptError extends Error {
/**
* Decrypt a file attached to a matrix event.
* @param {IEncryptedFile} file The encrypted file information taken from the matrix event.
* @param {EncryptedFile} file The encrypted file information taken from the matrix event.
* This passed to [link]{@link https://github.com/matrix-org/matrix-encrypt-attachment}
* as the encryption info object, so will also have the those keys in addition to
* the keys below.
* @param {IMediaEventInfo} info The info parameter taken from the matrix event.
* @returns {Promise<Blob>} Resolves to a Blob of the file.
*/
export async function decryptFile(file?: IEncryptedFile, info?: IMediaEventInfo): Promise<Blob> {
export async function decryptFile(file?: EncryptedFile, info?: IMediaEventInfo): Promise<Blob> {
// throws if file is falsy
const media = mediaFromContent({ file });

6
src/utils/MediaEventHelper.ts
View file

@ -21,7 +21,7 @@ import { logger } from "matrix-js-sdk/src/logger";
import { LazyValue } from "./LazyValue";
import { Media, mediaFromContent } from "../customisations/Media";
import { decryptFile } from "./DecryptFile";
import { IMediaEventContent } from "../customisations/models/IMediaEventContent";
import { FileContent, ImageContent, IMediaEventContent } from "../customisations/models/IMediaEventContent";
import { IDestroyable } from "./IDestroyable";
// TODO: We should consider caching the blobs. https://github.com/vector-im/element-web/issues/17192
@ -48,7 +48,7 @@ export class MediaEventHelper implements IDestroyable {
public get fileName(): string {
return (
this.event.getContent<IMediaEventContent>().filename ||
this.event.getContent<FileContent>().filename ||
this.event.getContent<IMediaEventContent>().body ||
"download"
);
@ -92,7 +92,7 @@ export class MediaEventHelper implements IDestroyable {
if (!this.media.hasThumbnail) return Promise.resolve(null);
if (this.media.isEncrypted) {
const content = this.event.getContent<IMediaEventContent>();
const content = this.event.getContent<ImageContent>();
if (content.info?.thumbnail_file) {
return decryptFile(content.info.thumbnail_file, content.info.thumbnail_info);
} else {

4
src/utils/image-media.ts
View file

@ -15,7 +15,7 @@ limitations under the License.
*/
import { BlurhashEncoder } from "../BlurhashEncoder";
import { IEncryptedFile } from "../customisations/models/IMediaEventContent";
import { EncryptedFile } from "../customisations/models/IMediaEventContent";
type ThumbnailableElement = HTMLImageElement | HTMLVideoElement;
@ -33,7 +33,7 @@ interface IThumbnail {
h: number;
[BLURHASH_FIELD]?: string;
thumbnail_url?: string;
thumbnail_file?: IEncryptedFile;
thumbnail_file?: EncryptedFile;
};
thumbnail: Blob;
}

4
src/voice-broadcast/models/VoiceBroadcastRecording.ts
View file

@ -38,7 +38,7 @@ import {
VoiceBroadcastRecorderEvent,
} from "..";
import { uploadFile } from "../../ContentMessages";
import { IEncryptedFile } from "../../customisations/models/IMediaEventContent";
import { EncryptedFile } from "../../customisations/models/IMediaEventContent";
import { createVoiceMessageContent } from "../../utils/createVoiceMessageContent";
import { IDestroyable } from "../../utils/IDestroyable";
import dis from "../../dispatcher/dispatcher";
@ -367,7 +367,7 @@ export class VoiceBroadcastRecording
);
}
private async sendVoiceMessage(chunk: ChunkRecordedPayload, url?: string, file?: IEncryptedFile): Promise<void> {
private async sendVoiceMessage(chunk: ChunkRecordedPayload, url?: string, file?: EncryptedFile): Promise<void> {
/**
* Increment the last sequence number and use it for this message.
* Done outside of the sendMessageFn to get a scoped value.