Use grapheme-splitter instead of lodash for saving emoji from being ripped apart (#10976)

* Use grapheme-splitter instead of lodash for saving emoji from being ripped apart

* Move to a more appropriate place

* Add tests and improve types

src/Avatar.ts

@@ -18,11 +18,11 @@ import { RoomMember } from "matrix-js-sdk/src/models/room-member";
 import { User } from "matrix-js-sdk/src/models/user";
 import { Room } from "matrix-js-sdk/src/models/room";
 import { ResizeMethod } from "matrix-js-sdk/src/@types/partials";
-import { split } from "lodash";
 
 import DMRoomMap from "./utils/DMRoomMap";
 import { mediaFromMxc } from "./customisations/Media";
 import { isLocalRoom } from "./utils/localRoom/isLocalRoom";
+import { getFirstGrapheme } from "./utils/strings";
 
 // Not to be used for BaseAvatar urls as that has similar default avatar fallback already
 export function avatarUrlForMember(
@@ -133,8 +133,7 @@ export function getInitialLetter(name: string): string | undefined {
         name = name.substring(1);
     }
 
-    // rely on the grapheme cluster splitter in lodash so that we don't break apart compound emojis
-    return split(name, "", 1)[0].toUpperCase();
+    return getFirstGrapheme(name).toUpperCase();
 }
 
 export function avatarUrlForRoom(

src/HtmlUtils.tsx

@@ -21,13 +21,14 @@ import React, { LegacyRef, ReactElement, ReactNode } from "react";
 import sanitizeHtml from "sanitize-html";
 import classNames from "classnames";
 import EMOJIBASE_REGEX from "emojibase-regex";
-import { merge, split } from "lodash";
+import { merge } from "lodash";
 import katex from "katex";
 import { decode } from "html-entities";
 import { IContent } from "matrix-js-sdk/src/models/event";
 import { Optional } from "matrix-events-sdk";
 import _Linkify from "linkify-react";
 import escapeHtml from "escape-html";
+import GraphemeSplitter from "grapheme-splitter";
 
 import {
     _linkifyElement,
@@ -463,14 +464,18 @@ const emojiToJsxSpan = (emoji: string, key: number): JSX.Element => (
  * @returns if isHtmlMessage is true, returns an array of strings, otherwise return an array of React Elements for emojis
  * and plain text for everything else
  */
-function formatEmojis(message: string | undefined, isHtmlMessage: boolean): (JSX.Element | string)[] {
+export function formatEmojis(message: string | undefined, isHtmlMessage?: false): JSX.Element[];
+export function formatEmojis(message: string | undefined, isHtmlMessage: true): string[];
+export function formatEmojis(message: string | undefined, isHtmlMessage: boolean): (JSX.Element | string)[] {
     const emojiToSpan = isHtmlMessage ? emojiToHtmlSpan : emojiToJsxSpan;
     const result: (JSX.Element | string)[] = [];
+    if (!message) return result;
+
     let text = "";
     let key = 0;
 
-    // We use lodash's grapheme splitter to avoid breaking apart compound emojis
-    for (const char of split(message, "")) {
+    const splitter = new GraphemeSplitter();
+    for (const char of splitter.iterateGraphemes(message)) {
         if (EMOJIBASE_REGEX.test(char)) {
             if (text) {
                 result.push(text);
@@ -661,7 +666,7 @@ export function topicToHtml(
         isFormattedTopic = false; // Fall back to plain-text topic
     }
 
-    let emojiBodyElements: ReturnType<typeof formatEmojis> | undefined;
+    let emojiBodyElements: JSX.Element[] | undefined;
     if (!isFormattedTopic && topicHasEmoji) {
         emojiBodyElements = formatEmojis(topic, false);
     }

src/editor/parts.ts

@@ -15,11 +15,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { split } from "lodash";
 import EMOJIBASE_REGEX from "emojibase-regex";
 import { MatrixClient } from "matrix-js-sdk/src/client";
 import { RoomMember } from "matrix-js-sdk/src/models/room-member";
 import { Room } from "matrix-js-sdk/src/models/room";
+import GraphemeSplitter from "grapheme-splitter";
 
 import AutocompleteWrapperModel, { GetAutocompleterComponent, UpdateCallback, UpdateQuery } from "./autocomplete";
 import { unicodeToShortcode } from "../HtmlUtils";
@@ -27,6 +27,7 @@ import * as Avatar from "../Avatar";
 import defaultDispatcher from "../dispatcher/dispatcher";
 import { Action } from "../dispatcher/actions";
 import SettingsStore from "../settings/SettingsStore";
+import { getFirstGrapheme } from "../utils/strings";
 
 const REGIONAL_EMOJI_SEPARATOR = String.fromCodePoint(0x200b);
 
@@ -133,8 +134,7 @@ abstract class BasePart {
         // To only need to grapheme split the bits of the string we're working on.
         let buffer = str;
         while (buffer) {
-            // We use lodash's grapheme splitter to avoid breaking apart compound emojis
-            const [char] = split(buffer, "", 2);
+            const char = getFirstGrapheme(buffer);
             if (!this.acceptsInsertion(char, offset + str.length - buffer.length, inputType)) {
                 break;
             }
@@ -562,8 +562,7 @@ export class PartCreator {
             case "\n":
                 return new NewlinePart();
             default:
-                // We use lodash's grapheme splitter to avoid breaking apart compound emojis
-                if (EMOJIBASE_REGEX.test(split(input, "", 2)[0])) {
+                if (EMOJIBASE_REGEX.test(getFirstGrapheme(input))) {
                     return new EmojiPart();
                 }
                 return new PlainPart();
@@ -639,8 +638,8 @@ export class PartCreator {
         const parts: (PlainPart | EmojiPart)[] = [];
         let plainText = "";
 
-        // We use lodash's grapheme splitter to avoid breaking apart compound emojis
-        for (const char of split(text, "")) {
+        const splitter = new GraphemeSplitter();
+        for (const char of splitter.iterateGraphemes(text)) {
             if (EMOJIBASE_REGEX.test(char)) {
                 if (plainText) {
                     parts.push(this.plain(plainText));

src/utils/strings.ts

@@ -21,6 +21,7 @@ limitations under the License.
  * @param text the plaintext to put in the user's clipboard
  */
 import { logger } from "matrix-js-sdk/src/logger";
+import GraphemeSplitter from "grapheme-splitter";
 
 export async function copyPlaintext(text: string): Promise<boolean> {
     try {
@@ -83,3 +84,15 @@ export function copyNode(ref?: Element | null): boolean {
 export function getSelectedText(): string {
     return window.getSelection()!.toString();
 }
+
+/**
+ * Returns the first grapheme in the given string,
+ * especially useful for strings containing emoji, will not break compound emoji up.
+ * @param str string to parse
+ * @returns the first grapheme or an empty string if given an empty string
+ */
+export function getFirstGrapheme(str: string): string {
+    const splitter = new GraphemeSplitter();
+    const result = splitter.iterateGraphemes(str).next();
+    return result.done ? "" : result.value;
+}