Generate thumbnails when sending m.image and m.video messages. (#555)

* Send a thumbnail when sending a m.image

* Use the 'thumbnail_file' when displaying encrypted images

* Whitespace

* Generate thumbnails for m.video

* Fix docstring, remove unused vars, use const

* Don't change the upload promise behaviour

* Polyfill for Canvas.toBlob to support older browsers

* Lowercase for integer types in jsdoc

package.json

@@ -42,6 +42,7 @@
   },
   "dependencies": {
     "babel-runtime": "^6.11.6",
+    "blueimp-canvas-to-blob": "^3.5.0",
     "browser-encrypt-attachment": "^0.1.0",
     "browser-request": "^0.3.3",
     "classnames": "^2.1.2",

src/ContentMessages.js

@@ -25,22 +25,87 @@ var Modal = require('./Modal');
 
 var encrypt = require("browser-encrypt-attachment");
 
-function infoForImageFile(imageFile) {
+// Polyfill for Canvas.toBlob API using Canvas.toDataURL
-    var deferred = q.defer();
+require("blueimp-canvas-to-blob");
+
+const MAX_WIDTH = 800;
+const MAX_HEIGHT = 600;
+
+
+/**
+ * Create a thumbnail for a image DOM element.
+ * The image will be smaller than MAX_WIDTH and MAX_HEIGHT.
+ * The thumbnail will have the same aspect ratio as the original.
+ * Draws the element into a canvas using CanvasRenderingContext2D.drawImage
+ * Then calls Canvas.toBlob to get a blob object for the image data.
+ *
+ * Since it needs to calculate the dimensions of the source image and the
+ * thumbnailed image it returns an info object filled out with information
+ * about the original image and the thumbnail.
+ *
+ * @param {HTMLElement} element The element to thumbnail.
+ * @param {integer} inputWidth The width of the image in the input element.
+ * @param {integer} inputHeight the width of the image in the input element.
+ * @param {String} mimeType The mimeType to save the blob as.
+ * @return {Promise} A promise that resolves with an object with an info key
+ *  and a thumbnail key.
+ */
+function createThumbnail(element, inputWidth, inputHeight, mimeType) {
+    const deferred = q.defer();
+
+    var targetWidth = inputWidth;
+    var targetHeight = inputHeight;
+    if (targetHeight > MAX_HEIGHT) {
+        targetWidth = Math.floor(targetWidth * (MAX_HEIGHT / targetHeight));
+        targetHeight = MAX_HEIGHT;
+    }
+    if (targetWidth > MAX_WIDTH) {
+        targetHeight = Math.floor(targetHeight * (MAX_WIDTH / targetWidth));
+        targetWidth = MAX_WIDTH;
+    }
+
+    const canvas = document.createElement("canvas");
+    canvas.width = targetWidth;
+    canvas.height = targetHeight;
+    canvas.getContext("2d").drawImage(element, 0, 0, targetWidth, targetHeight);
+    canvas.toBlob(function(thumbnail) {
+        deferred.resolve({
+            info: {
+                thumbnail_info: {
+                    w: targetWidth,
+                    h: targetHeight,
+                    mimetype: thumbnail.type,
+                    size: thumbnail.size,
+                },
+                w: inputWidth,
+                h: inputHeight,
+            },
+            thumbnail: thumbnail
+        });
+    }, mimeType);
+
+    return deferred.promise;
+}
+
+/**
+ * Load a file into a newly created image element.
+ *
+ * @param {File} file The file to load in an image element.
+ * @return {Promise} A promise that resolves with the html image element.
+ */
+function loadImageElement(imageFile) {
+    const deferred = q.defer();
 
     // Load the file into an html element
-    var img = document.createElement("img");
+    const img = document.createElement("img");
 
-    var reader = new FileReader();
+    const reader = new FileReader();
     reader.onload = function(e) {
         img.src = e.target.result;
 
-        // Once ready, returns its size
+        // Once ready, create a thumbnail
         img.onload = function() {
-            deferred.resolve({
+            deferred.resolve(img);
-                w: img.width,
-                h: img.height
-            });
         };
         img.onerror = function(e) {
             deferred.reject(e);
@@ -54,22 +119,53 @@ function infoForImageFile(imageFile) {
     return deferred.promise;
 }
 
-function infoForVideoFile(videoFile) {
+/**
-    var deferred = q.defer();
+ * Read the metadata for an image file and create and upload a thumbnail of the image.
+ *
+ * @param {MatrixClient} matrixClient A matrixClient to upload the thumbnail with.
+ * @param {String} roomId The ID of the room the image will be uploaded in.
+ * @param {File} The image to read and thumbnail.
+ * @return {Promise} A promise that resolves with the attachment info.
+ */
+function infoForImageFile(matrixClient, roomId, imageFile) {
+    var thumbnailType = "image/png";
+    if (imageFile.type == "image/jpeg") {
+        thumbnailType = "image/jpeg";
+    }
+
+    var imageInfo;
+    return loadImageElement(imageFile).then(function(img) {
+        return createThumbnail(img, img.width, img.height, thumbnailType);
+    }).then(function(result) {
+        imageInfo = result.info;
+        return uploadFile(matrixClient, roomId, result.thumbnail);
+    }).then(function(result) {
+        imageInfo.thumbnail_url = result.url;
+        imageInfo.thumbnail_file = result.file;
+        return imageInfo;
+    });
+}
+
+/**
+ * Load a file into a newly created video element.
+ *
+ * @param {File} file The file to load in an video element.
+ * @return {Promise} A promise that resolves with the video image element.
+ */
+function loadVideoElement(videoFile) {
+    const deferred = q.defer();
 
     // Load the file into an html element
-    var video = document.createElement("video");
+    const video = document.createElement("video");
 
-    var reader = new FileReader();
+    const reader = new FileReader();
     reader.onload = function(e) {
         video.src = e.target.result;
 
         // Once ready, returns its size
-        video.onloadedmetadata = function() {
+        // Wait until we have enough data to thumbnail the first frame.
-            deferred.resolve({
+        video.onloadeddata = function() {
-                w: video.videoWidth,
+            deferred.resolve(video);
-                h: video.videoHeight
-            });
         };
         video.onerror = function(e) {
             deferred.reject(e);
@@ -83,6 +179,30 @@ function infoForVideoFile(videoFile) {
     return deferred.promise;
 }
 
+/**
+ * Read the metadata for a video file and create and upload a thumbnail of the video.
+ *
+ * @param {MatrixClient} matrixClient A matrixClient to upload the thumbnail with.
+ * @param {String} roomId The ID of the room the video will be uploaded to.
+ * @param {File} The video to read and thumbnail.
+ * @return {Promise} A promise that resolves with the attachment info.
+ */
+function infoForVideoFile(matrixClient, roomId, videoFile) {
+    const thumbnailType = "image/jpeg";
+
+    var videoInfo;
+    return loadVideoElement(videoFile).then(function(video) {
+        return createThumbnail(video, video.videoWidth, video.videoHeight, thumbnailType);
+    }).then(function(result) {
+        videoInfo = result.info;
+        return uploadFile(matrixClient, roomId, result.thumbnail);
+    }).then(function(result) {
+        videoInfo.thumbnail_url = result.url;
+        videoInfo.thumbnail_file = result.file;
+        return videoInfo;
+    });
+}
+
 /**
  * Read the file as an ArrayBuffer.
  * @return {Promise} A promise that resolves with an ArrayBuffer when the file
@@ -101,6 +221,48 @@ function readFileAsArrayBuffer(file) {
     return deferred.promise;
 }
 
+/**
+ * Upload the file to the content repository.
+ * If the room is encrypted then encrypt the file before uploading.
+ *
+ * @param {MatrixClient} matrixClient The matrix client to upload the file with.
+ * @param {String} roomId The ID of the room being uploaded to.
+ * @param {File} file The file to upload.
+ * @return {Promise} A promise that resolves with an object.
+ *  If the file is unencrypted then the object will have a "url" key.
+ *  If the file is encrypted then the object will have a "file" key.
+ */
+function uploadFile(matrixClient, roomId, file) {
+    if (matrixClient.isRoomEncrypted(roomId)) {
+        // If the room is encrypted then encrypt the file before uploading it.
+        // First read the file into memory.
+        return readFileAsArrayBuffer(file).then(function(data) {
+            // Then encrypt the file.
+            return encrypt.encryptAttachment(data);
+        }).then(function(encryptResult) {
+            // Record the information needed to decrypt the attachment.
+            const encryptInfo = encryptResult.info;
+            // Pass the encrypted data as a Blob to the uploader.
+            const blob = new Blob([encryptResult.data]);
+            return matrixClient.uploadContent(blob).then(function(url) {
+                // If the attachment is encrypted then bundle the URL along
+                // with the information needed to decrypt the attachment and
+                // add it under a file key.
+                encryptInfo.url = url;
+                if (file.type) {
+                    encryptInfo.mimetype = file.type;
+                }
+                return {"file": encryptInfo};
+            });
+        });
+    } else {
+        return matrixClient.uploadContent(file).then(function(url) {
+            // If the attachment isn't encrypted then include the URL directly.
+            return {"url": url};
+        });
+    }
+}
+
 
 class ContentMessages {
     constructor() {
@@ -109,7 +271,7 @@ class ContentMessages {
     }
 
     sendContentToRoom(file, roomId, matrixClient) {
-        var content = {
+        const content = {
             body: file.name,
             info: {
                 size: file.size,
@@ -121,13 +283,14 @@ class ContentMessages {
             content.info.mimetype = file.type;
         }
 
-        var def = q.defer();
+        const def = q.defer();
         if (file.type.indexOf('image/') == 0) {
             content.msgtype = 'm.image';
-            infoForImageFile(file).then(imageInfo=>{
+            infoForImageFile(matrixClient, roomId, file).then(imageInfo=>{
                 extend(content.info, imageInfo);
                 def.resolve();
             }, error=>{
+                console.error(error);
                 content.msgtype = 'm.file';
                 def.resolve();
             });
@@ -136,7 +299,7 @@ class ContentMessages {
             def.resolve();
         } else if (file.type.indexOf('video/') == 0) {
             content.msgtype = 'm.video';
-            infoForVideoFile(file).then(videoInfo=>{
+            infoForVideoFile(matrixClient, roomId, file).then(videoInfo=>{
                 extend(content.info, videoInfo);
                 def.resolve();
             }, error=>{
@@ -148,35 +311,23 @@ class ContentMessages {
             def.resolve();
         }
 
-        var upload = {
+        const upload = {
             fileName: file.name,
             roomId: roomId,
             total: 0,
-            loaded: 0
+            loaded: 0,
         };
         this.inprogress.push(upload);
         dis.dispatch({action: 'upload_started'});
 
-        var encryptInfo = null;
         var error;
-        var self = this;
         return def.promise.then(function() {
-            if (matrixClient.isRoomEncrypted(roomId)) {
+            upload.promise = uploadFile(
-                // If the room is encrypted then encrypt the file before uploading it.
+                matrixClient, roomId, file
-                // First read the file into memory.
+            ).then(function(result) {
-                upload.promise = readFileAsArrayBuffer(file).then(function(data) {
+                content.file = result.file;
-                    // Then encrypt the file.
+                content.url = result.url;
-                    return encrypt.encryptAttachment(data);
+            });
-                }).then(function(encryptResult) {
-                    // Record the information needed to decrypt the attachment.
-                    encryptInfo = encryptResult.info;
-                    // Pass the encrypted data as a Blob to the uploader.
-                    var blob = new Blob([encryptResult.data]);
-                    return matrixClient.uploadContent(blob);
-                });
-            } else {
-                upload.promise = matrixClient.uploadContent(file);
-            }
             return upload.promise;
         }).progress(function(ev) {
             if (ev) {
@@ -185,19 +336,6 @@ class ContentMessages {
                 dis.dispatch({action: 'upload_progress', upload: upload});
             }
         }).then(function(url) {
-            if (encryptInfo === null) {
-                // If the attachment isn't encrypted then include the URL directly.
-                content.url = url;
-            } else {
-                // If the attachment is encrypted then bundle the URL along
-                // with the information needed to decrypt the attachment and
-                // add it under a file key.
-                encryptInfo.url = url;
-                if (file.type) {
-                    encryptInfo.mimetype = file.type;
-                }
-                content.file = encryptInfo;
-            }
             return matrixClient.sendMessage(roomId, content);
         }, function(err) {
             error = err;
@@ -212,12 +350,12 @@ class ContentMessages {
                     description: desc
                 });
             }
-        }).finally(function() {
+        }).finally(() => {
-            var inprogressKeys = Object.keys(self.inprogress);
+            const inprogressKeys = Object.keys(this.inprogress);
-            for (var i = 0; i < self.inprogress.length; ++i) {
+            for (var i = 0; i < this.inprogress.length; ++i) {
                 var k = inprogressKeys[i];
-                if (self.inprogress[k].promise === upload.promise) {
+                if (this.inprogress[k].promise === upload.promise) {
-                    self.inprogress.splice(k, 1);
+                    this.inprogress.splice(k, 1);
                     break;
                 }
             }
@@ -235,7 +373,7 @@ class ContentMessages {
     }
 
     cancelUpload(promise) {
-        var inprogressKeys = Object.keys(this.inprogress);
+        const inprogressKeys = Object.keys(this.inprogress);
         var upload;
         for (var i = 0; i < this.inprogress.length; ++i) {
             var k = inprogressKeys[i];

src/components/views/messages/MImageBody.js

@@ -24,6 +24,7 @@ import Modal from '../../../Modal';
 import sdk from '../../../index';
 import dis from '../../../dispatcher';
 import {decryptFile} from '../../../utils/DecryptFile';
+import q from 'q';
 
 module.exports = React.createClass({
     displayName: 'MImageBody',
@@ -36,6 +37,7 @@ module.exports = React.createClass({
     getInitialState: function() {
         return {
             decryptedUrl: null,
+            decryptedThumbnailUrl: null,
         };
     },
 
@@ -94,7 +96,9 @@ module.exports = React.createClass({
     _getThumbUrl: function() {
         const content = this.props.mxEvent.getContent();
         if (content.file !== undefined) {
-            // TODO: Decrypt and use the thumbnail file if one is present.
+            if (this.state.decryptedThumbnailUrl) {
+                return this.state.decryptedThumbnailUrl;
+            }
             return this.state.decryptedUrl;
         } else {
             return MatrixClientPeg.get().mxcUrlToHttp(content.url, 800, 600);
@@ -106,15 +110,24 @@ module.exports = React.createClass({
         this.fixupHeight();
         const content = this.props.mxEvent.getContent();
         if (content.file !== undefined && this.state.decryptedUrl === null) {
-            decryptFile(content.file).done((url) => {
+            var thumbnailPromise = q(null);
-                this.setState({
+            if (content.info.thumbnail_file) {
-                    decryptedUrl: url,
+                thumbnailPromise = decryptFile(
+                    content.info.thumbnail_file
+                );
+            }
+            thumbnailPromise.then((thumbnailUrl) => {
+                decryptFile(content.file).then((contentUrl) => {
+                    this.setState({
+                        decryptedUrl: contentUrl,
+                        decryptedThumbnailUrl: thumbnailUrl,
+                    });
                 });
-            }, (err) => {
+            }).catch((err) => {
                 console.warn("Unable to decrypt attachment: ", err)
                 // Set a placeholder image when we can't decrypt the image.
                 this.refs.image.src = "img/warning.svg";
-            });
+            }).done();
         }
     },