fix regional emojis converted to flags (#9294)

Co-authored-by: grimhilt <grimhilt@users.noreply.github.com>
Co-authored-by: Michael Telatynski <7t3chguy@gmail.com>
Co-authored-by: Faye Duxovni <duxovni@duxovni.org>
Co-authored-by: Faye Duxovni <fayed@element.io>
Fixes https://github.com/vector-im/element-web/issues/19000

cypress/e2e/timeline/timeline.spec.ts

@@ -384,5 +384,24 @@ describe("Timeline", () => {
                 1,
             );
         });
+
+        it("should not be possible to send flag with regional emojis", () => {
+            cy.visit("/#/room/" + roomId);
+
+            // Send a message
+            cy.getComposer().type(":regional_indicator_a");
+            cy.contains(".mx_Autocomplete_Completion_title", ":regional_indicator_a:").click();
+            cy.getComposer().type(":regional_indicator_r");
+            cy.contains(".mx_Autocomplete_Completion_title", ":regional_indicator_r:").click();
+            cy.getComposer().type(" :regional_indicator_z");
+            cy.contains(".mx_Autocomplete_Completion_title", ":regional_indicator_z:").click();
+            cy.getComposer().type(":regional_indicator_a");
+            cy.contains(".mx_Autocomplete_Completion_title", ":regional_indicator_a:").click();
+            cy.getComposer().type("{enter}");
+
+            cy.get(".mx_RoomView_body .mx_EventTile .mx_EventTile_line .mx_MTextBody .mx_EventTile_bigEmoji")
+                .children()
+                .should("have.length", 4);
+        });
     });
 });

src/HtmlUtils.tsx

@@ -49,11 +49,8 @@ const SURROGATE_PAIR_PATTERN = /([\ud800-\udbff])([\udc00-\udfff])/;
 // (with plenty of false positives, but that's OK)
 const SYMBOL_PATTERN = /([\u2100-\u2bff])/;
 
-// Regex pattern for Zero-Width joiner unicode characters
+// Regex pattern for non-emoji characters that can appear in an "all-emoji" message (Zero-Width Joiner, Zero-Width Space, other whitespace)
-const ZWJ_REGEX = /[\u200D\u2003]/g;
+const EMOJI_SEPARATOR_REGEX = /[\u200D\u200B\s]/g;
-
-// Regex pattern for whitespace characters
-const WHITESPACE_REGEX = /\s/g;
 
 const BIGEMOJI_REGEX = new RegExp(`^(${EMOJIBASE_REGEX.source})+$`, "i");
 
@@ -591,14 +588,11 @@ export function bodyToHtml(content: IContent, highlights: Optional<string[]>, op
     if (!opts.disableBigEmoji && bodyHasEmoji) {
         let contentBodyTrimmed = contentBody !== undefined ? contentBody.trim() : "";
 
-        // Ignore spaces in body text. Emojis with spaces in between should
+        // Remove zero width joiner, zero width spaces and other spaces in body
-        // still be counted as purely emoji messages.
+        // text. This ensures that emojis with spaces in between or that are made
-        contentBodyTrimmed = contentBodyTrimmed.replace(WHITESPACE_REGEX, "");
+        // up of multiple unicode characters are still counted as purely emoji
-
+        // messages.
-        // Remove zero width joiner characters from emoji messages. This ensures
+        contentBodyTrimmed = contentBodyTrimmed.replace(EMOJI_SEPARATOR_REGEX, "");
-        // that emojis that are made up of multiple unicode characters are still
-        // presented as large.
-        contentBodyTrimmed = contentBodyTrimmed.replace(ZWJ_REGEX, "");
 
         const match = BIGEMOJI_REGEX.exec(contentBodyTrimmed);
         emojiBody =

src/editor/parts.ts

@@ -27,6 +27,8 @@ import defaultDispatcher from "../dispatcher/dispatcher";
 import { Action } from "../dispatcher/actions";
 import SettingsStore from "../settings/SettingsStore";
 
+const REGIONAL_EMOJI_SEPARATOR = String.fromCodePoint(0x200b);
+
 interface ISerializedPart {
     type: Type.Plain | Type.Newline | Type.Emoji | Type.Command | Type.PillCandidate;
     text: string;
@@ -209,9 +211,13 @@ abstract class PlainBasePart extends BasePart {
                 return false;
             }
 
-            // or split if the previous character is a space
+            // or split if the previous character is a space or regional emoji separator
             // or if it is a + and this is a :
-            return this._text[offset - 1] !== " " && (this._text[offset - 1] !== "+" || chr !== ":");
+            return (
+                this._text[offset - 1] !== " " &&
+                this._text[offset - 1] !== REGIONAL_EMOJI_SEPARATOR &&
+                (this._text[offset - 1] !== "+" || chr !== ":")
+            );
         }
         return true;
     }
@@ -626,8 +632,13 @@ export class PartCreator {
         return new UserPillPart(userId, displayName, member);
     }
 
+    private static isRegionalIndicator(c: string): boolean {
+        const codePoint = c.codePointAt(0) ?? 0;
+        return codePoint != 0 && c.length == 2 && 0x1f1e6 <= codePoint && codePoint <= 0x1f1ff;
+    }
+
     public plainWithEmoji(text: string): (PlainPart | EmojiPart)[] {
-        const parts = [];
+        const parts: (PlainPart | EmojiPart)[] = [];
         let plainText = "";
 
         // We use lodash's grapheme splitter to avoid breaking apart compound emojis
@@ -638,6 +649,9 @@ export class PartCreator {
                     plainText = "";
                 }
                 parts.push(this.emoji(char));
+                if (PartCreator.isRegionalIndicator(text)) {
+                    parts.push(this.plain(REGIONAL_EMOJI_SEPARATOR));
+                }
             } else {
                 plainText += char;
             }

test/editor/model-test.ts

@@ -348,4 +348,32 @@ describe("editor/model", function () {
             expect(model.parts[0].text).toBe("foo@a");
         });
     });
+    describe("emojis", function () {
+        it("regional emojis should be separated to prevent them to be converted to flag", () => {
+            const renderer = createRenderer();
+            const pc = createPartCreator();
+            const model = new EditorModel([], pc, renderer);
+            const regionalEmojiA = String.fromCodePoint(127462);
+            const regionalEmojiZ = String.fromCodePoint(127487);
+            const caret = new DocumentOffset(0, true);
+
+            const regionalEmojis: string[] = [];
+            regionalEmojis.push(regionalEmojiA);
+            regionalEmojis.push(regionalEmojiZ);
+            for (let i = 0; i < 2; i++) {
+                const position = model.positionForOffset(caret.offset, caret.atNodeEnd);
+                model.transform(() => {
+                    const addedLen = model.insert(pc.plainWithEmoji(regionalEmojis[i]), position);
+                    caret.offset += addedLen;
+                    return model.positionForOffset(caret.offset, true);
+                });
+            }
+
+            expect(model.parts.length).toBeGreaterThanOrEqual(4);
+            expect(model.parts[0].type).toBe("emoji");
+            expect(model.parts[1].type).not.toBe("emoji");
+            expect(model.parts[2].type).toBe("emoji");
+            expect(model.parts[3].type).not.toBe("emoji");
+        });
+    });
 });