Implement MSC3575: Sliding Sync (#8328)

* Add labs flag for sliding sync; add sliding_sync_proxy_url to config.json * Disable the labs toggle if sliding_sync_proxy_url is not set * Do validation checks on the sliding sync proxy URL before enabling it in Labs * Enable sliding sync and add SlidingSyncManager * Get room subscriptions working * Hijack renderSublists in sliding sync mode * Add support for sorting alphabetically/recency and room name filters * Filter out tombstoned rooms; start adding show more logic list ranges update but the UI doesn't * update the UI when the list is updated * bugfix: make sure the list sorts numerically * Get invites transitioning correctly * Force enable sliding sync and labs for now * Linting * Disable spotlight search * Initial cypress plugins for Sliding Sync Proxy * Use --rm when running Synapse in Docker for Cypress tests * Update src/MatrixClientPeg.ts Co-authored-by: Travis Ralston <travisr@matrix.org> * Update src/components/views/rooms/RoomSublist.tsx Co-authored-by: Travis Ralston <travisr@matrix.org> * Update src/settings/controllers/SlidingSyncController.ts Co-authored-by: Travis Ralston <travisr@matrix.org> * Update src/components/views/rooms/RoomSublist.tsx Co-authored-by: Travis Ralston <travisr@matrix.org> * WIP add room searching to spotlight search * Only read sliding sync results when there is a result, else use the local cache * Use feature_sliding_sync not slidingSync * Some review comments * More review comments * Use RoomViewStore to set room subscriptions * Comment why any * Update src/components/views/rooms/RoomSublist.tsx Co-authored-by: Michael Telatynski <7t3chguy@gmail.com> * Fix cypress docker abstraction * Iterate sliding sync proxy support * Stash mostly functional test * Update sliding sync proxy image * i18n * Add support for spaces; use list ID -> index mappings - Mappings are more reusable and easier to understand than racing for index positions. - Register for all spaces immediately on startup. * When the active space is updated, update the list registration * Set spaces filter in the correct place * Skeleton placeholder whilst loading the space * Filter out spaces from the room list * Use the new txn_id promises * Ensure we actually resolve list registrations * Fix matrix-org/sliding-sync#30: don't show tombstoned search results * Remove unused imports * Add SYNCV3_SECRET to proxy to ensure it starts up; correct aliases for SS test * Add another basic sliding sync e2e test * Unbreak netlify * Add more logging for debugging duplicate rooms * If sliding sync is enabled, always use the rooms result even if it's empty * Drop-in copy of RoomListStore for sliding sync * Remove conditionals from RoomListStore - we have SlidingRoomListStore now * WIP SlidingRoomListStore * Add most sliding sync logic to SlidingRoomListStore Still lots of logic in RoomSublist. Broken things: - Join count is wrong completely. - No skeleton placeholder when switching spaces. * Migrate joined count to SS RLS * Reinstate the skeleton UI when the list is loading * linting * Add support for sticky rooms based on the currently active room * Add a bunch of passing SS E2E tests; some WIP * Unbreak build from git merge * Suppress unread indicators in sliding sync mode * Add regression test for https://github.com/matrix-org/sliding-sync/issues/28 * Add invite test flows; show the invite list The refactor to SS RLS removed the invite list entirely. * Remove show more click as it wasn't the bug * Linting and i18n * only enable SS by default on netlify * Jest fixes; merge conflict fixes; remove debug logging; use right sort enum values * Actually fix jest tests * Add support for favourites and low priority * Bump sliding sync version * Update sliding sync labs to be user configurable * delint * To disable SS or change proxy URL the user has to log out * Review comments * Linting * Apply suggestions from code review Co-authored-by: Travis Ralston <travisr@matrix.org> * Update src/stores/room-list/SlidingRoomListStore.ts Co-authored-by: Travis Ralston <travisr@matrix.org> * Review comments * Add issue link for TODO markers * Linting * Apply suggestions from code review Co-authored-by: Michael Telatynski <7t3chguy@gmail.com> * More review comments * More review comments * stricter types Co-authored-by: Michael Telatynski <7t3chguy@gmail.com> Co-authored-by: Travis Ralston <travisr@matrix.org>
2022-09-07 16:42:39 +01:00 · 2022-09-07 16:42:39 +01:00 · a215027c6b
commit a215027c6b
parent 5bdae150fa
25 changed files with 1632 additions and 51 deletions
--- a/src/stores/room-list/SlidingRoomListStore.ts
+++ b/src/stores/room-list/SlidingRoomListStore.ts
@ -0,0 +1,386 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { Room } from "matrix-js-sdk/src/models/room";
+import { logger } from "matrix-js-sdk/src/logger";
+import { MSC3575Filter, SlidingSyncEvent } from "matrix-js-sdk/src/sliding-sync";
+
+import { RoomUpdateCause, TagID, OrderedDefaultTagIDs, DefaultTagID } from "./models";
+import { ITagMap, ListAlgorithm, SortAlgorithm } from "./algorithms/models";
+import { ActionPayload } from "../../dispatcher/payloads";
+import defaultDispatcher from "../../dispatcher/dispatcher";
+import { IFilterCondition } from "./filters/IFilterCondition";
+import { AsyncStoreWithClient } from "../AsyncStoreWithClient";
+import { RoomListStore as Interface, RoomListStoreEvent } from "./Interface";
+import { SlidingSyncManager } from "../../SlidingSyncManager";
+import SpaceStore from "../spaces/SpaceStore";
+import { MetaSpace, SpaceKey, UPDATE_SELECTED_SPACE } from "../spaces";
+import { LISTS_LOADING_EVENT } from "./RoomListStore";
+import { RoomViewStore } from "../RoomViewStore";
+
+interface IState {
+    // state is tracked in underlying classes
+}
+
+export const SlidingSyncSortToFilter: Record<SortAlgorithm, string[]> = {
+    [SortAlgorithm.Alphabetic]: ["by_name", "by_recency"],
+    [SortAlgorithm.Recent]: ["by_highlight_count", "by_notification_count", "by_recency"],
+    [SortAlgorithm.Manual]: ["by_recency"],
+};
+
+const filterConditions: Record<TagID, MSC3575Filter> = {
+    [DefaultTagID.Invite]: {
+        is_invite: true,
+    },
+    [DefaultTagID.Favourite]: {
+        tags: ["m.favourite"],
+        is_tombstoned: false,
+    },
+    // TODO https://github.com/vector-im/element-web/issues/23207
+    // DefaultTagID.SavedItems,
+    [DefaultTagID.DM]: {
+        is_dm: true,
+        is_invite: false,
+        is_tombstoned: false,
+    },
+    [DefaultTagID.Untagged]: {
+        is_dm: false,
+        is_invite: false,
+        is_tombstoned: false,
+        not_room_types: ["m.space"],
+        not_tags: ["m.favourite", "m.lowpriority"],
+        // spaces filter added dynamically
+    },
+    [DefaultTagID.LowPriority]: {
+        tags: ["m.lowpriority"],
+        is_tombstoned: false,
+    },
+    // TODO https://github.com/vector-im/element-web/issues/23207
+    // DefaultTagID.ServerNotice,
+    // DefaultTagID.Suggested,
+    // DefaultTagID.Archived,
+};
+
+export const LISTS_UPDATE_EVENT = RoomListStoreEvent.ListsUpdate;
+
+export class SlidingRoomListStoreClass extends AsyncStoreWithClient<IState> implements Interface {
+    private tagIdToSortAlgo: Record<TagID, SortAlgorithm> = {};
+    private tagMap: ITagMap = {};
+    private counts: Record<TagID, number> = {};
+    private stickyRoomId: string | null;
+
+    public constructor() {
+        super(defaultDispatcher);
+        this.setMaxListeners(20); // RoomList + LeftPanel + 8xRoomSubList + spares
+    }
+
+    public async setTagSorting(tagId: TagID, sort: SortAlgorithm) {
+        logger.info("SlidingRoomListStore.setTagSorting ", tagId, sort);
+        this.tagIdToSortAlgo[tagId] = sort;
+        const slidingSyncIndex = SlidingSyncManager.instance.getOrAllocateListIndex(tagId);
+        switch (sort) {
+            case SortAlgorithm.Alphabetic:
+                await SlidingSyncManager.instance.ensureListRegistered(
+                    slidingSyncIndex, {
+                        sort: SlidingSyncSortToFilter[SortAlgorithm.Alphabetic],
+                    },
+                );
+                break;
+            case SortAlgorithm.Recent:
+                await SlidingSyncManager.instance.ensureListRegistered(
+                    slidingSyncIndex, {
+                        sort: SlidingSyncSortToFilter[SortAlgorithm.Recent],
+                    },
+                );
+                break;
+            case SortAlgorithm.Manual:
+                logger.error("cannot enable manual sort in sliding sync mode");
+                break;
+            default:
+                logger.error("unknown sort mode: ", sort);
+        }
+    }
+
+    public getTagSorting(tagId: TagID): SortAlgorithm {
+        let algo = this.tagIdToSortAlgo[tagId];
+        if (!algo) {
+            logger.warn("SlidingRoomListStore.getTagSorting: no sort algorithm for tag ", tagId);
+            algo = SortAlgorithm.Recent; // why not, we have to do something..
+        }
+        return algo;
+    }
+
+    public getCount(tagId: TagID): number {
+        return this.counts[tagId] || 0;
+    }
+
+    public setListOrder(tagId: TagID, order: ListAlgorithm) {
+        // TODO: https://github.com/vector-im/element-web/issues/23207
+    }
+
+    public getListOrder(tagId: TagID): ListAlgorithm {
+        // TODO: handle unread msgs first? https://github.com/vector-im/element-web/issues/23207
+        return ListAlgorithm.Natural;
+    }
+
+    /**
+     * Adds a filter condition to the room list store. Filters may be applied async,
+     * and thus might not cause an update to the store immediately.
+     * @param {IFilterCondition} filter The filter condition to add.
+     */
+    public async addFilter(filter: IFilterCondition): Promise<void> {
+        // Do nothing, the filters are only used by SpaceWatcher to see if a room should appear
+        // in the room list. We do not support arbitrary code for filters in sliding sync.
+    }
+
+    /**
+     * Removes a filter condition from the room list store. If the filter was
+     * not previously added to the room list store, this will no-op. The effects
+     * of removing a filter may be applied async and therefore might not cause
+     * an update right away.
+     * @param {IFilterCondition} filter The filter condition to remove.
+     */
+    public removeFilter(filter: IFilterCondition): void {
+        // Do nothing, the filters are only used by SpaceWatcher to see if a room should appear
+        // in the room list. We do not support arbitrary code for filters in sliding sync.
+    }
+
+    /**
+     * Gets the tags for a room identified by the store. The returned set
+     * should never be empty, and will contain DefaultTagID.Untagged if
+     * the store is not aware of any tags.
+     * @param room The room to get the tags for.
+     * @returns The tags for the room.
+     */
+    public getTagsForRoom(room: Room): TagID[] {
+        // check all lists for each tag we know about and see if the room is there
+        const tags: TagID[] = [];
+        for (const tagId in this.tagIdToSortAlgo) {
+            const index = SlidingSyncManager.instance.getOrAllocateListIndex(tagId);
+            const { roomIndexToRoomId } = SlidingSyncManager.instance.slidingSync.getListData(index);
+            for (const roomIndex in roomIndexToRoomId) {
+                const roomId = roomIndexToRoomId[roomIndex];
+                if (roomId === room.roomId) {
+                    tags.push(tagId);
+                    break;
+                }
+            }
+        }
+        return tags;
+    }
+
+    /**
+     * Manually update a room with a given cause. This should only be used if the
+     * room list store would otherwise be incapable of doing the update itself. Note
+     * that this may race with the room list's regular operation.
+     * @param {Room} room The room to update.
+     * @param {RoomUpdateCause} cause The cause to update for.
+     */
+    public async manualRoomUpdate(room: Room, cause: RoomUpdateCause) {
+        // TODO: this is only used when you forget a room, not that important for now.
+    }
+
+    public get orderedLists(): ITagMap {
+        return this.tagMap;
+    }
+
+    private refreshOrderedLists(tagId: string, roomIndexToRoomId: Record<number, string>): void {
+        const tagMap = this.tagMap;
+
+        // this room will not move due to it being viewed: it is sticky. This can be null to indicate
+        // no sticky room if you aren't viewing a room.
+        this.stickyRoomId = RoomViewStore.instance.getRoomId();
+        let stickyRoomNewIndex = -1;
+        const stickyRoomOldIndex = (tagMap[tagId] || []).findIndex((room) => {
+            return room.roomId === this.stickyRoomId;
+        });
+
+        // order from low to high
+        const orderedRoomIndexes = Object.keys(roomIndexToRoomId).map((numStr) => {
+            return Number(numStr);
+        }).sort((a, b) => {
+            return a-b;
+        });
+        const seenRoomIds = new Set<string>();
+        const orderedRoomIds = orderedRoomIndexes.map((i) => {
+            const rid = roomIndexToRoomId[i];
+            if (seenRoomIds.has(rid)) {
+                logger.error("room " + rid + " already has an index position: duplicate room!");
+            }
+            seenRoomIds.add(rid);
+            if (!rid) {
+                throw new Error("index " + i + " has no room ID: Map => " + JSON.stringify(roomIndexToRoomId));
+            }
+            if (rid === this.stickyRoomId) {
+                stickyRoomNewIndex = i;
+            }
+            return rid;
+        });
+        logger.debug(
+            `SlidingRoomListStore.refreshOrderedLists ${tagId} sticky: ${this.stickyRoomId}`,
+            `${stickyRoomOldIndex} -> ${stickyRoomNewIndex}`,
+            "rooms:",
+            orderedRoomIds.length < 30 ? orderedRoomIds : orderedRoomIds.length,
+        );
+
+        if (this.stickyRoomId && stickyRoomOldIndex >= 0 && stickyRoomNewIndex >= 0) {
+            // this update will move this sticky room from old to new, which we do not want.
+            // Instead, keep the sticky room ID index position as it is, swap it with
+            // whatever was in its place.
+            // Some scenarios with sticky room S and bump room B (other letters unimportant):
+            // A, S, C, B                                  S, A, B
+            // B, A, S, C  <---- without sticky rooms ---> B, S, A
+            // B, S, A, C  <- with sticky rooms applied -> S, B, A
+            // In other words, we need to swap positions to keep it locked in place.
+            const inWayRoomId = orderedRoomIds[stickyRoomOldIndex];
+            orderedRoomIds[stickyRoomOldIndex] = this.stickyRoomId;
+            orderedRoomIds[stickyRoomNewIndex] = inWayRoomId;
+        }
+
+        // now set the rooms
+        const rooms = orderedRoomIds.map((roomId) => {
+            return this.matrixClient.getRoom(roomId);
+        });
+        tagMap[tagId] = rooms;
+        this.tagMap = tagMap;
+    }
+
+    private onSlidingSyncListUpdate(listIndex: number, joinCount: number, roomIndexToRoomId: Record<number, string>) {
+        const tagId = SlidingSyncManager.instance.listIdForIndex(listIndex);
+        this.counts[tagId]= joinCount;
+        this.refreshOrderedLists(tagId, roomIndexToRoomId);
+        // let the UI update
+        this.emit(LISTS_UPDATE_EVENT);
+    }
+
+    private onRoomViewStoreUpdated() {
+        // we only care about this to know when the user has clicked on a room to set the stickiness value
+        if (RoomViewStore.instance.getRoomId() === this.stickyRoomId) {
+            return;
+        }
+
+        let hasUpdatedAnyList = false;
+
+        // every list with the OLD sticky room ID needs to be resorted because it now needs to take
+        // its proper place as it is no longer sticky. The newly sticky room can remain the same though,
+        // as we only actually care about its sticky status when we get list updates.
+        const oldStickyRoom = this.stickyRoomId;
+        // it's not safe to check the data in slidingSync as it is tracking the server's view of the
+        // room list. There's an edge case whereby the sticky room has gone outside the window and so
+        // would not be present in the roomIndexToRoomId map anymore, and hence clicking away from it
+        // will make it disappear eventually. We need to check orderedLists as that is the actual
+        // sorted renderable list of rooms which sticky rooms apply to.
+        for (const tagId in this.orderedLists) {
+            const list = this.orderedLists[tagId];
+            const room = list.find((room) => {
+                return room.roomId === oldStickyRoom;
+            });
+            if (room) {
+                // resort it based on the slidingSync view of the list. This may cause this old sticky
+                // room to cease to exist.
+                const index = SlidingSyncManager.instance.getOrAllocateListIndex(tagId);
+                const { roomIndexToRoomId } = SlidingSyncManager.instance.slidingSync.getListData(index);
+                this.refreshOrderedLists(tagId, roomIndexToRoomId);
+                hasUpdatedAnyList = true;
+            }
+        }
+        // in the event we didn't call refreshOrderedLists, it helps to still remember the sticky room ID.
+        this.stickyRoomId = RoomViewStore.instance.getRoomId();
+
+        if (hasUpdatedAnyList) {
+            this.emit(LISTS_UPDATE_EVENT);
+        }
+    }
+
+    protected async onReady(): Promise<any> {
+        logger.info("SlidingRoomListStore.onReady");
+        // permanent listeners: never get destroyed. Could be an issue if we want to test this in isolation.
+        SlidingSyncManager.instance.slidingSync.on(SlidingSyncEvent.List, this.onSlidingSyncListUpdate.bind(this));
+        RoomViewStore.instance.addListener(this.onRoomViewStoreUpdated.bind(this));
+        SpaceStore.instance.on(UPDATE_SELECTED_SPACE, this.onSelectedSpaceUpdated.bind(this));
+        if (SpaceStore.instance.activeSpace) {
+            this.onSelectedSpaceUpdated(SpaceStore.instance.activeSpace, false);
+        }
+
+        // sliding sync has an initial response for spaces. Now request all the lists.
+        // We do the spaces list _first_ to avoid potential flickering on DefaultTagID.Untagged list
+        // which would be caused by initially having no `spaces` filter set, and then suddenly setting one.
+        OrderedDefaultTagIDs.forEach((tagId) => {
+            const filter = filterConditions[tagId];
+            if (!filter) {
+                logger.info("SlidingRoomListStore.onReady unsupported list ", tagId);
+                return; // we do not support this list yet.
+            }
+            const sort = SortAlgorithm.Recent; // default to recency sort, TODO: read from config
+            this.tagIdToSortAlgo[tagId] = sort;
+            this.emit(LISTS_LOADING_EVENT, tagId, true);
+            const index = SlidingSyncManager.instance.getOrAllocateListIndex(tagId);
+            SlidingSyncManager.instance.ensureListRegistered(index, {
+                filters: filter,
+                sort: SlidingSyncSortToFilter[sort],
+            }).then(() => {
+                this.emit(LISTS_LOADING_EVENT, tagId, false);
+            });
+        });
+    }
+
+    private onSelectedSpaceUpdated = (activeSpace: SpaceKey, allRoomsInHome: boolean) => {
+        logger.info("SlidingRoomListStore.onSelectedSpaceUpdated", activeSpace);
+        // update the untagged filter
+        const tagId = DefaultTagID.Untagged;
+        const filters = filterConditions[tagId];
+        const oldSpace = filters.spaces?.[0];
+        filters.spaces = (activeSpace && activeSpace != MetaSpace.Home) ? [activeSpace] : undefined;
+        if (oldSpace !== activeSpace) {
+            this.emit(LISTS_LOADING_EVENT, tagId, true);
+            SlidingSyncManager.instance.ensureListRegistered(
+                SlidingSyncManager.instance.getOrAllocateListIndex(tagId),
+                {
+                    filters: filters,
+                },
+            ).then(() => {
+                this.emit(LISTS_LOADING_EVENT, tagId, false);
+            });
+        }
+    };
+
+    // Intended for test usage
+    public async resetStore() {
+        // Test function
+    }
+
+    /**
+     * Regenerates the room whole room list, discarding any previous results.
+     *
+     * Note: This is only exposed externally for the tests. Do not call this from within
+     * the app.
+     * @param trigger Set to false to prevent a list update from being sent. Should only
+     * be used if the calling code will manually trigger the update.
+     */
+    public regenerateAllLists({ trigger = true }) {
+        // Test function
+    }
+
+    protected async onNotReady(): Promise<any> {
+        await this.resetStore();
+    }
+
+    protected async onAction(payload: ActionPayload) {
+    }
+
+    protected async onDispatchAsync(payload: ActionPayload) {
+    }
+}