Merge pull request #3745 from matrix-org/t3chguy/get_rid_of_emoji_stripped_data

Get rid of stripped-emoji.json in favour of an in-memory single source of truth

src/components/views/emojipicker/EmojiPicker.js

@@ -16,54 +16,12 @@ limitations under the License.
 
 import React from 'react';
 import PropTypes from 'prop-types';
-import EMOJIBASE from 'emojibase-data/en/compact.json';
 
 import sdk from '../../../index';
 import { _t } from '../../../languageHandler';
 
 import * as recent from './recent';
-
-const EMOJIBASE_CATEGORY_IDS = [
-    "people", // smileys
-    "people", // actually people
-    "control", // modifiers and such, not displayed in picker
-    "nature",
-    "foods",
-    "places",
-    "activity",
-    "objects",
-    "symbols",
-    "flags",
-];
-
-const DATA_BY_CATEGORY = {
-    "people": [],
-    "nature": [],
-    "foods": [],
-    "places": [],
-    "activity": [],
-    "objects": [],
-    "symbols": [],
-    "flags": [],
-};
-const DATA_BY_EMOJI = {};
-
-const VARIATION_SELECTOR = String.fromCharCode(0xFE0F);
-EMOJIBASE.forEach(emoji => {
-    if (emoji.unicode.includes(VARIATION_SELECTOR)) {
-        // Clone data into variation-less version
-        emoji = Object.assign({}, emoji, {
-            unicode: emoji.unicode.replace(VARIATION_SELECTOR, ""),
-        });
-    }
-    DATA_BY_EMOJI[emoji.unicode] = emoji;
-    const categoryId = EMOJIBASE_CATEGORY_IDS[emoji.group];
-    if (DATA_BY_CATEGORY.hasOwnProperty(categoryId)) {
-        DATA_BY_CATEGORY[categoryId].push(emoji);
-    }
-    // This is used as the string to match the query against when filtering emojis.
-    emoji.filterString = `${emoji.annotation}\n${emoji.shortcodes.join('\n')}}\n${emoji.emoticon || ''}`.toLowerCase();
-});
+import {DATA_BY_CATEGORY, getEmojiFromUnicode} from "../../../emoji";
 
 export const CATEGORY_HEADER_HEIGHT = 22;
 export const EMOJI_HEIGHT = 37;
@@ -91,7 +49,7 @@ class EmojiPicker extends React.Component {
 
         // Convert recent emoji characters to emoji data, removing unknowns.
         this.recentlyUsed = recent.get()
-            .map(unicode => DATA_BY_EMOJI[unicode])
+            .map(unicode => getEmojiFromUnicode(unicode))
             .filter(data => !!data);
         this.memoizedDataByCategory = {
             recent: this.recentlyUsed,

src/components/views/emojipicker/QuickReactions.js

@@ -19,15 +19,15 @@ import PropTypes from 'prop-types';
 
 import sdk from '../../../index';
 import { _t } from '../../../languageHandler';
-import { findEmojiData } from '../../../HtmlUtils';
+import {getEmojiFromUnicode} from "../../../emoji";
 
+// We use the variation-selector Heart in Quick Reactions for some reason
 const QUICK_REACTIONS = ["👍", "👎", "😄", "🎉", "😕", "❤️", "🚀", "👀"].map(emoji => {
-    const data = findEmojiData(emoji);
+    const data = getEmojiFromUnicode(emoji);
     if (!data) {
         throw new Error(`Emoji ${emoji} doesn't exist in emojibase`);
     }
-    // Prefer our unicode value for quick reactions (which does not have
-    // variation selectors).
+    // Prefer our unicode value for quick reactions as we sometimes use variation selectors.
     return Object.assign({}, data, { unicode: emoji });
 });
 

src/components/views/rooms/BasicMessageComposer.js

@@ -34,11 +34,11 @@ import {parsePlainTextMessage} from '../../../editor/deserialize';
 import {renderModel} from '../../../editor/render';
 import {Room} from 'matrix-js-sdk';
 import TypingStore from "../../../stores/TypingStore";
-import EMOJIBASE from 'emojibase-data/en/compact.json';
 import SettingsStore from "../../../settings/SettingsStore";
 import EMOTICON_REGEX from 'emojibase-regex/emoticon';
 import sdk from '../../../index';
 import {Key} from "../../../Keyboard";
+import {EMOTICON_TO_EMOJI} from "../../../emoji";
 
 const REGEX_EMOTICON_WHITESPACE = new RegExp('(?:^|\\s)(' + EMOTICON_REGEX.source + ')\\s$');
 
@@ -108,7 +108,8 @@ export default class BasicMessageEditor extends React.Component {
         const emoticonMatch = REGEX_EMOTICON_WHITESPACE.exec(range.text);
         if (emoticonMatch) {
             const query = emoticonMatch[1].toLowerCase().replace("-", "");
-            const data = EMOJIBASE.find(e => e.emoticon ? e.emoticon.toLowerCase() === query : false);
+            const data = EMOTICON_TO_EMOJI.get(query);
+
             if (data) {
                 const {partCreator} = model;
                 const hasPrecedingSpace = emoticonMatch[0][0] === " ";

src/components/views/rooms/MessageComposerInput.js

@@ -48,7 +48,6 @@ import Markdown from '../../../Markdown';
 import MessageComposerStore from '../../../stores/MessageComposerStore';
 import ContentMessages from '../../../ContentMessages';
 
-import EMOJIBASE from 'emojibase-data/en/compact.json';
 import EMOTICON_REGEX from 'emojibase-regex/emoticon';
 
 import SettingsStore, {SettingLevel} from "../../../settings/SettingsStore";
@@ -61,6 +60,7 @@ import AccessibleButton from '../elements/AccessibleButton';
 import {findEditableEvent} from '../../../utils/EventUtils';
 import SlateComposerHistoryManager from "../../../SlateComposerHistoryManager";
 import TypingStore from "../../../stores/TypingStore";
+import {EMOTICON_TO_EMOJI} from "../../../emoji";
 
 const REGEX_EMOTICON_WHITESPACE = new RegExp('(?:^|\\s)(' + EMOTICON_REGEX.source + ')\\s$');
 
@@ -464,7 +464,7 @@ export default class MessageComposerInput extends React.Component {
                 const emoticonMatch = REGEX_EMOTICON_WHITESPACE.exec(text.slice(0, currentStartOffset));
                 if (emoticonMatch) {
                     const query = emoticonMatch[1].toLowerCase().replace("-", "");
-                    const data = EMOJIBASE.find(e => e.emoticon ? e.emoticon.toLowerCase() === query : false);
+                    const data = EMOTICON_TO_EMOJI.get(query);
 
                     // only perform replacement if we found a match, otherwise we would be not letting user type
                     if (data) {