diff --git a/package.json b/package.json index 4ad585ba7d..d6f94c15f8 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "dependencies": { "@babel/runtime": "^7.12.5", "await-lock": "^2.1.0", + "blurhash": "^1.1.3", "browser-encrypt-attachment": "^0.3.0", "browser-request": "^0.3.3", "cheerio": "^1.0.0-rc.9", diff --git a/res/css/views/messages/_MImageBody.scss b/res/css/views/messages/_MImageBody.scss index 515d867da5..878a4154cd 100644 --- a/res/css/views/messages/_MImageBody.scss +++ b/res/css/views/messages/_MImageBody.scss @@ -14,6 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +$timelineImageBorderRadius: 4px; + .mx_MImageBody { display: block; margin-right: 34px; @@ -25,7 +27,11 @@ limitations under the License. height: 100%; left: 0; top: 0; - border-radius: 4px; + border-radius: $timelineImageBorderRadius; + + > canvas { + border-radius: $timelineImageBorderRadius; + } } .mx_MImageBody_thumbnail_container { diff --git a/src/ContentMessages.tsx b/src/ContentMessages.tsx index ef0a89a690..1b8a3be37e 100644 --- a/src/ContentMessages.tsx +++ b/src/ContentMessages.tsx @@ -17,9 +17,10 @@ limitations under the License. */ import React from "react"; -import dis from './dispatcher/dispatcher'; -import { MatrixClientPeg } from './MatrixClientPeg'; +import { encode } from "blurhash"; import { MatrixClient } from "matrix-js-sdk/src/client"; + +import dis from './dispatcher/dispatcher'; import * as sdk from './index'; import { _t } from './languageHandler'; import Modal from './Modal'; @@ -47,6 +48,10 @@ const MAX_HEIGHT = 600; // 5669 px (x-axis) , 5669 px (y-axis) , per metre const PHYS_HIDPI = [0x00, 0x00, 0x16, 0x25, 0x00, 0x00, 0x16, 0x25, 0x01]; +export const BLURHASH_FIELD = "xyz.amorgan.blurhash"; // MSC2448 +const BLURHASH_X_COMPONENTS = 6; +const BLURHASH_Y_COMPONENTS = 6; + export class UploadCanceledError extends Error {} type ThumbnailableElement = HTMLImageElement | HTMLVideoElement; @@ -77,6 +82,7 @@ interface IThumbnail { }; w: number; h: number; + [BLURHASH_FIELD]: string; }; thumbnail: Blob; } @@ -124,7 +130,16 @@ function createThumbnail( const canvas = document.createElement("canvas"); canvas.width = targetWidth; canvas.height = targetHeight; - canvas.getContext("2d").drawImage(element, 0, 0, targetWidth, targetHeight); + const context = canvas.getContext("2d"); + context.drawImage(element, 0, 0, targetWidth, targetHeight); + const imageData = context.getImageData(0, 0, targetWidth, targetHeight); + const blurhash = encode( + imageData.data, + imageData.width, + imageData.height, + BLURHASH_X_COMPONENTS, + BLURHASH_Y_COMPONENTS, + ); canvas.toBlob(function(thumbnail) { resolve({ info: { @@ -136,8 +151,9 @@ function createThumbnail( }, w: inputWidth, h: inputHeight, + [BLURHASH_FIELD]: blurhash, }, - thumbnail: thumbnail, + thumbnail, }); }, mimeType); }); @@ -220,7 +236,8 @@ function infoForImageFile(matrixClient, roomId, imageFile) { } /** - * Load a file into a newly created video element. + * Load a file into a newly created video element and pull some strings + * in an attempt to guarantee the first frame will be showing. * * @param {File} videoFile The file to load in an video element. * @return {Promise} A promise that resolves with the video image element. @@ -229,20 +246,25 @@ function loadVideoElement(videoFile): Promise { return new Promise((resolve, reject) => { // Load the file into an html element const video = document.createElement("video"); + video.preload = "metadata"; + video.playsInline = true; + video.muted = true; const reader = new FileReader(); reader.onload = function(ev) { - video.src = ev.target.result as string; - - // Once ready, returns its size // Wait until we have enough data to thumbnail the first frame. - video.onloadeddata = function() { + video.onloadeddata = async function() { resolve(video); + video.pause(); }; video.onerror = function(e) { reject(e); }; + + video.src = ev.target.result as string; + video.load(); + video.play(); }; reader.onerror = function(e) { reject(e); @@ -347,7 +369,7 @@ export function uploadFile( }); (prom as IAbortablePromise).abort = () => { canceled = true; - if (uploadPromise) MatrixClientPeg.get().cancelUpload(uploadPromise); + if (uploadPromise) matrixClient.cancelUpload(uploadPromise); }; return prom; } else { @@ -357,11 +379,11 @@ export function uploadFile( const promise1 = basePromise.then(function(url) { if (canceled) throw new UploadCanceledError(); // If the attachment isn't encrypted then include the URL directly. - return { "url": url }; + return { url }; }); (promise1 as any).abort = () => { canceled = true; - MatrixClientPeg.get().cancelUpload(basePromise); + matrixClient.cancelUpload(basePromise); }; return promise1; } @@ -373,7 +395,7 @@ export default class ContentMessages { sendStickerContentToRoom(url: string, roomId: string, info: IImageInfo, text: string, matrixClient: MatrixClient) { const startTime = CountlyAnalytics.getTimestamp(); - const prom = MatrixClientPeg.get().sendStickerMessage(roomId, url, info, text).catch((e) => { + const prom = matrixClient.sendStickerMessage(roomId, url, info, text).catch((e) => { console.warn(`Failed to send content with URL ${url} to room ${roomId}`, e); throw e; }); @@ -415,7 +437,7 @@ export default class ContentMessages { if (!this.mediaConfig) { // hot-path optimization to not flash a spinner if we don't need to const modal = Modal.createDialog(Spinner, null, 'mx_Dialog_spinner'); - await this.ensureMediaConfigFetched(); + await this.ensureMediaConfigFetched(matrixClient); modal.close(); } @@ -470,7 +492,7 @@ export default class ContentMessages { return this.inprogress.filter(u => !u.canceled); } - cancelUpload(promise: Promise) { + cancelUpload(promise: Promise, matrixClient: MatrixClient) { let upload: IUpload; for (let i = 0; i < this.inprogress.length; ++i) { if (this.inprogress[i].promise === promise) { @@ -480,7 +502,7 @@ export default class ContentMessages { } if (upload) { upload.canceled = true; - MatrixClientPeg.get().cancelUpload(upload.promise); + matrixClient.cancelUpload(upload.promise); dis.dispatch({ action: Action.UploadCanceled, upload }); } } @@ -621,11 +643,11 @@ export default class ContentMessages { return true; } - private ensureMediaConfigFetched() { + private ensureMediaConfigFetched(matrixClient: MatrixClient) { if (this.mediaConfig !== null) return; console.log("[Media Config] Fetching"); - return MatrixClientPeg.get().getMediaConfig().then((config) => { + return matrixClient.getMediaConfig().then((config) => { console.log("[Media Config] Fetched config:", config); return config; }).catch(() => { diff --git a/src/components/structures/RoomView.tsx b/src/components/structures/RoomView.tsx index d08eaa2ecd..82a44de7a5 100644 --- a/src/components/structures/RoomView.tsx +++ b/src/components/structures/RoomView.tsx @@ -1261,7 +1261,7 @@ export default class RoomView extends React.Component { }); }; - private injectSticker(url, info, text) { + private injectSticker(url: string, info: object, text: string) { if (this.context.isGuest()) { dis.dispatch({ action: 'require_registration' }); return; diff --git a/src/components/structures/UploadBar.tsx b/src/components/structures/UploadBar.tsx index b8dce48235..c8e90a1c0a 100644 --- a/src/components/structures/UploadBar.tsx +++ b/src/components/structures/UploadBar.tsx @@ -26,6 +26,7 @@ import ProgressBar from "../views/elements/ProgressBar"; import AccessibleButton from "../views/elements/AccessibleButton"; import { IUpload } from "../../models/IUpload"; import { replaceableComponent } from "../../utils/replaceableComponent"; +import MatrixClientContext from "../../contexts/MatrixClientContext"; interface IProps { room: Room; @@ -38,6 +39,8 @@ interface IState { @replaceableComponent("structures.UploadBar") export default class UploadBar extends React.Component { + static contextType = MatrixClientContext; + private dispatcherRef: string; private mounted: boolean; @@ -82,7 +85,7 @@ export default class UploadBar extends React.Component { private onCancelClick = (ev) => { ev.preventDefault(); - ContentMessages.sharedInstance().cancelUpload(this.state.currentUpload.promise); + ContentMessages.sharedInstance().cancelUpload(this.state.currentUpload.promise, this.context); }; render() { diff --git a/src/components/views/elements/BlurhashPlaceholder.tsx b/src/components/views/elements/BlurhashPlaceholder.tsx new file mode 100644 index 0000000000..0e59253fe8 --- /dev/null +++ b/src/components/views/elements/BlurhashPlaceholder.tsx @@ -0,0 +1,56 @@ +/* + Copyright 2020 The Matrix.org Foundation C.I.C. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +import React from 'react'; +import { decode } from "blurhash"; + +interface IProps { + blurhash: string; + width: number; + height: number; +} + +export default class BlurhashPlaceholder extends React.PureComponent { + private canvas: React.RefObject = React.createRef(); + + public componentDidMount() { + this.draw(); + } + + public componentDidUpdate() { + this.draw(); + } + + private draw() { + if (!this.canvas.current) return; + + try { + const { width, height } = this.props; + + const pixels = decode(this.props.blurhash, Math.ceil(width), Math.ceil(height)); + const ctx = this.canvas.current.getContext("2d"); + const imgData = ctx.createImageData(width, height); + imgData.data.set(pixels); + ctx.putImageData(imgData, 0, 0); + } catch (e) { + console.error("Error rendering blurhash: ", e); + } + } + + public render() { + return ; + } +} diff --git a/src/components/views/messages/MImageBody.js b/src/components/views/messages/MImageBody.js index 7e85f15898..5566f5aec0 100644 --- a/src/components/views/messages/MImageBody.js +++ b/src/components/views/messages/MImageBody.js @@ -29,6 +29,8 @@ import MatrixClientContext from "../../../contexts/MatrixClientContext"; import InlineSpinner from '../elements/InlineSpinner'; import { replaceableComponent } from "../../../utils/replaceableComponent"; import { mediaFromContent } from "../../../customisations/Media"; +import BlurhashPlaceholder from "../elements/BlurhashPlaceholder"; +import { BLURHASH_FIELD } from "../../../ContentMessages"; @replaceableComponent("views.messages.MImageBody") export default class MImageBody extends React.Component { @@ -333,7 +335,8 @@ export default class MImageBody extends React.Component { infoWidth = content.info.w; infoHeight = content.info.h; } else { - // Whilst the image loads, display nothing. + // Whilst the image loads, display nothing. We also don't display a blurhash image + // because we don't really know what size of image we'll end up with. // // Once loaded, use the loaded image dimensions stored in `loadedImageDimensions`. // @@ -368,12 +371,8 @@ export default class MImageBody extends React.Component { let placeholder = null; let gifLabel = null; - // e2e image hasn't been decrypted yet - if (content.file !== undefined && this.state.decryptedUrl === null) { - placeholder = ; - } else if (!this.state.imgLoaded) { - // Deliberately, getSpinner is left unimplemented here, MStickerBody overides - placeholder = this.getPlaceholder(); + if (!this.state.imgLoaded) { + placeholder = this.getPlaceholder(maxWidth, maxHeight); } let showPlaceholder = Boolean(placeholder); @@ -395,7 +394,7 @@ export default class MImageBody extends React.Component { if (!this.state.showImage) { img = ; - showPlaceholder = false; // because we're hiding the image, so don't show the sticker icon. + showPlaceholder = false; // because we're hiding the image, so don't show the placeholder. } if (this._isGif() && !SettingsStore.getValue("autoplayGifsAndVideos") && !this.state.hover) { @@ -411,9 +410,7 @@ export default class MImageBody extends React.Component { // Constrain width here so that spinner appears central to the loaded thumbnail maxWidth: infoWidth + "px", }}> -
- { placeholder } -
+ { placeholder } } @@ -437,9 +434,12 @@ export default class MImageBody extends React.Component { } // Overidden by MStickerBody - getPlaceholder() { - // MImageBody doesn't show a placeholder whilst the image loads, (but it could do) - return null; + getPlaceholder(width, height) { + const blurhash = this.props.mxEvent.getContent().info[BLURHASH_FIELD]; + if (blurhash) return ; + return
+ +
; } // Overidden by MStickerBody diff --git a/src/components/views/messages/MStickerBody.js b/src/components/views/messages/MStickerBody.js index eb3635b0c0..31af66baf5 100644 --- a/src/components/views/messages/MStickerBody.js +++ b/src/components/views/messages/MStickerBody.js @@ -18,6 +18,7 @@ import React from 'react'; import MImageBody from './MImageBody'; import * as sdk from '../../../index'; import { replaceableComponent } from "../../../utils/replaceableComponent"; +import { BLURHASH_FIELD } from "../../../ContentMessages"; @replaceableComponent("views.messages.MStickerBody") export default class MStickerBody extends MImageBody { @@ -41,7 +42,8 @@ export default class MStickerBody extends MImageBody { // Placeholder to show in place of the sticker image if // img onLoad hasn't fired yet. - getPlaceholder() { + getPlaceholder(width, height) { + if (this.props.mxEvent.getContent().info[BLURHASH_FIELD]) return super.getPlaceholder(width, height); return ; } diff --git a/src/components/views/messages/MVideoBody.tsx b/src/components/views/messages/MVideoBody.tsx index ef79e96370..d882bb1eb0 100644 --- a/src/components/views/messages/MVideoBody.tsx +++ b/src/components/views/messages/MVideoBody.tsx @@ -16,6 +16,8 @@ limitations under the License. */ import React from 'react'; +import { decode } from "blurhash"; + import MFileBody from './MFileBody'; import { decryptFile } from '../../../utils/DecryptFile'; import { _t } from '../../../languageHandler'; @@ -23,6 +25,7 @@ import SettingsStore from "../../../settings/SettingsStore"; import InlineSpinner from '../elements/InlineSpinner'; import { replaceableComponent } from "../../../utils/replaceableComponent"; import { mediaFromContent } from "../../../customisations/Media"; +import { BLURHASH_FIELD } from "../../../ContentMessages"; interface IProps { /* the MatrixEvent to show */ @@ -32,11 +35,13 @@ interface IProps { } interface IState { - decryptedUrl: string|null, - decryptedThumbnailUrl: string|null, - decryptedBlob: Blob|null, - error: any|null, - fetchingData: boolean, + decryptedUrl?: string; + decryptedThumbnailUrl?: string; + decryptedBlob?: Blob; + error?: any; + fetchingData: boolean; + posterLoading: boolean; + blurhashUrl: string; } @replaceableComponent("views.messages.MVideoBody") @@ -51,10 +56,12 @@ export default class MVideoBody extends React.PureComponent { decryptedThumbnailUrl: null, decryptedBlob: null, error: null, + posterLoading: false, + blurhashUrl: null, }; } - thumbScale(fullWidth: number, fullHeight: number, thumbWidth: number, thumbHeight: number) { + thumbScale(fullWidth: number, fullHeight: number, thumbWidth = 480, thumbHeight = 360) { if (!fullWidth || !fullHeight) { // Cannot calculate thumbnail height for image: missing w/h in metadata. We can't even // log this because it's spammy @@ -92,8 +99,11 @@ export default class MVideoBody extends React.PureComponent { private getThumbUrl(): string|null { const content = this.props.mxEvent.getContent(); const media = mediaFromContent(content); - if (media.isEncrypted) { + + if (media.isEncrypted && this.state.decryptedThumbnailUrl) { return this.state.decryptedThumbnailUrl; + } else if (this.state.posterLoading) { + return this.state.blurhashUrl; } else if (media.hasThumbnail) { return media.thumbnailHttp; } else { @@ -101,18 +111,57 @@ export default class MVideoBody extends React.PureComponent { } } + private loadBlurhash() { + const info = this.props.mxEvent.getContent()?.info; + if (!info[BLURHASH_FIELD]) return; + + const canvas = document.createElement("canvas"); + + let width = info.w; + let height = info.h; + const scale = this.thumbScale(info.w, info.h); + if (scale) { + width = Math.floor(info.w * scale); + height = Math.floor(info.h * scale); + } + + canvas.width = width; + canvas.height = height; + + const pixels = decode(info[BLURHASH_FIELD], width, height); + const ctx = canvas.getContext("2d"); + const imgData = ctx.createImageData(width, height); + imgData.data.set(pixels); + ctx.putImageData(imgData, 0, 0); + + this.setState({ + blurhashUrl: canvas.toDataURL(), + posterLoading: true, + }); + + const content = this.props.mxEvent.getContent(); + const media = mediaFromContent(content); + if (media.hasThumbnail) { + const image = new Image(); + image.onload = () => { + this.setState({ posterLoading: false }); + }; + image.src = media.thumbnailHttp; + } + } + async componentDidMount() { const autoplay = SettingsStore.getValue("autoplayGifsAndVideos") as boolean; const content = this.props.mxEvent.getContent(); + this.loadBlurhash(); + if (content.file !== undefined && this.state.decryptedUrl === null) { let thumbnailPromise = Promise.resolve(null); - if (content.info && content.info.thumbnail_file) { - thumbnailPromise = decryptFile( - content.info.thumbnail_file, - ).then(function(blob) { - return URL.createObjectURL(blob); - }); + if (content?.info?.thumbnail_file) { + thumbnailPromise = decryptFile(content.info.thumbnail_file) + .then(blob => URL.createObjectURL(blob)); } + try { const thumbnailUrl = await thumbnailPromise; if (autoplay) { @@ -218,7 +267,7 @@ export default class MVideoBody extends React.PureComponent { let poster = null; let preload = "metadata"; if (content.info) { - const scale = this.thumbScale(content.info.w, content.info.h, 480, 360); + const scale = this.thumbScale(content.info.w, content.info.h); if (scale) { width = Math.floor(content.info.w * scale); height = Math.floor(content.info.h * scale); diff --git a/yarn.lock b/yarn.lock index 6f08372e18..ea92cd1b59 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2190,6 +2190,11 @@ bluebird@^3.5.0: resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== +blurhash@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/blurhash/-/blurhash-1.1.3.tgz#dc325af7da836d07a0861d830bdd63694382483e" + integrity sha512-yUhPJvXexbqbyijCIE/T2NCXcj9iNPhWmOKbPTuR/cm7Q5snXYIfnVnz6m7MWOXxODMz/Cr3UcVkRdHiuDVRDw== + boolbase@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e"