411 lines
18 KiB
TypeScript
411 lines
18 KiB
TypeScript
/*
|
|
Copyright 2024 New Vector Ltd.
|
|
Copyright 2022 The Matrix.org Foundation C.I.C.
|
|
|
|
SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only
|
|
Please see LICENSE files in the repository root for full details.
|
|
*/
|
|
|
|
/*
|
|
* Sliding Sync Architecture - MSC https://github.com/matrix-org/matrix-spec-proposals/pull/3575
|
|
*
|
|
* This is a holistic summary of the changes made to Element-Web / React SDK / JS SDK to enable sliding sync.
|
|
* This summary will hopefully signpost where developers need to look if they want to make changes to this code.
|
|
*
|
|
* At the lowest level, the JS SDK contains an HTTP API wrapper function in client.ts. This is used by
|
|
* a SlidingSync class in JS SDK, which contains code to handle list operations (INSERT/DELETE/SYNC/etc)
|
|
* and contains the main request API bodies, but has no code to control updating JS SDK structures: it just
|
|
* exposes an EventEmitter to listen for updates. When MatrixClient.startClient is called, callers need to
|
|
* provide a SlidingSync instance as this contains the main request API params (timeline limit, required state,
|
|
* how many lists, etc).
|
|
*
|
|
* The SlidingSyncSdk INTERNAL class in JS SDK attaches listeners to SlidingSync to update JS SDK Room objects,
|
|
* and it conveniently exposes an identical public API to SyncApi (to allow it to be a drop-in replacement).
|
|
*
|
|
* At the highest level, SlidingSyncManager contains mechanisms to tell UI lists which rooms to show,
|
|
* and contains the core request API params used in Element-Web. It does this by listening for events
|
|
* emitted by the SlidingSync class and by modifying the request API params on the SlidingSync class.
|
|
*
|
|
* (entry point) (updates JS SDK)
|
|
* SlidingSyncManager SlidingSyncSdk
|
|
* | |
|
|
* +------------------.------------------+
|
|
* listens | listens
|
|
* SlidingSync
|
|
* (sync loop,
|
|
* list ops)
|
|
*/
|
|
|
|
import { MatrixClient, EventType, ClientEvent, Room } from "matrix-js-sdk/src/matrix";
|
|
import {
|
|
MSC3575Filter,
|
|
MSC3575List,
|
|
MSC3575SlidingSyncResponse,
|
|
MSC3575_STATE_KEY_LAZY,
|
|
MSC3575_STATE_KEY_ME,
|
|
MSC3575_WILDCARD,
|
|
SlidingSync,
|
|
SlidingSyncEvent,
|
|
SlidingSyncState,
|
|
} from "matrix-js-sdk/src/sliding-sync";
|
|
import { logger } from "matrix-js-sdk/src/logger";
|
|
import { defer, sleep } from "matrix-js-sdk/src/utils";
|
|
|
|
import SlidingSyncController from "./settings/controllers/SlidingSyncController";
|
|
|
|
// how long to long poll for
|
|
const SLIDING_SYNC_TIMEOUT_MS = 20 * 1000;
|
|
|
|
// The state events we will get for every single room/space/old room/etc
|
|
// This list is only augmented when a direct room subscription is made. (e.g you view a room)
|
|
const REQUIRED_STATE_LIST = [
|
|
[EventType.RoomJoinRules, ""], // the public icon on the room list
|
|
[EventType.RoomAvatar, ""], // any room avatar
|
|
[EventType.RoomCanonicalAlias, ""], // for room name calculations
|
|
[EventType.RoomTombstone, ""], // lets JS SDK hide rooms which are dead
|
|
[EventType.RoomEncryption, ""], // lets rooms be configured for E2EE correctly
|
|
[EventType.RoomCreate, ""], // for isSpaceRoom checks
|
|
[EventType.SpaceChild, MSC3575_WILDCARD], // all space children
|
|
[EventType.SpaceParent, MSC3575_WILDCARD], // all space parents
|
|
[EventType.RoomMember, MSC3575_STATE_KEY_ME], // lets the client calculate that we are in fact in the room
|
|
];
|
|
|
|
// the things to fetch when a user clicks on a room
|
|
const DEFAULT_ROOM_SUBSCRIPTION_INFO = {
|
|
timeline_limit: 50,
|
|
// missing required_state which will change depending on the kind of room
|
|
include_old_rooms: {
|
|
timeline_limit: 0,
|
|
required_state: REQUIRED_STATE_LIST,
|
|
},
|
|
};
|
|
// lazy load room members so rooms like Matrix HQ don't take forever to load
|
|
const UNENCRYPTED_SUBSCRIPTION_NAME = "unencrypted";
|
|
const UNENCRYPTED_SUBSCRIPTION = {
|
|
required_state: [
|
|
[EventType.RoomMember, MSC3575_STATE_KEY_ME], // except for m.room.members, get our own membership
|
|
[EventType.RoomMember, MSC3575_STATE_KEY_LAZY], // ...and lazy load the rest.
|
|
],
|
|
...DEFAULT_ROOM_SUBSCRIPTION_INFO,
|
|
};
|
|
|
|
// we need all the room members in encrypted rooms because we need to know which users to encrypt
|
|
// messages for.
|
|
const ENCRYPTED_SUBSCRIPTION = {
|
|
required_state: [
|
|
[MSC3575_WILDCARD, MSC3575_WILDCARD], // all events
|
|
],
|
|
...DEFAULT_ROOM_SUBSCRIPTION_INFO,
|
|
};
|
|
|
|
// the complete set of lists made in SSS. The manager will spider all of these lists depending
|
|
// on the count for each one.
|
|
const sssLists: Record<string, MSC3575List> = {
|
|
spaces: {
|
|
ranges: [[0, 10]],
|
|
timeline_limit: 0, // we don't care about the most recent message for spaces
|
|
required_state: REQUIRED_STATE_LIST,
|
|
include_old_rooms: {
|
|
timeline_limit: 0,
|
|
required_state: REQUIRED_STATE_LIST,
|
|
},
|
|
filters: {
|
|
room_types: ["m.space"],
|
|
},
|
|
},
|
|
invites: {
|
|
ranges: [[0, 10]],
|
|
timeline_limit: 1, // most recent message display
|
|
required_state: REQUIRED_STATE_LIST,
|
|
include_old_rooms: {
|
|
timeline_limit: 0,
|
|
required_state: REQUIRED_STATE_LIST,
|
|
},
|
|
filters: {
|
|
is_invite: true,
|
|
},
|
|
},
|
|
favourites: {
|
|
ranges: [[0, 10]],
|
|
timeline_limit: 1, // most recent message display
|
|
required_state: REQUIRED_STATE_LIST,
|
|
include_old_rooms: {
|
|
timeline_limit: 0,
|
|
required_state: REQUIRED_STATE_LIST,
|
|
},
|
|
filters: {
|
|
tags: ["m.favourite"],
|
|
},
|
|
},
|
|
dms: {
|
|
ranges: [[0, 10]],
|
|
timeline_limit: 1, // most recent message display
|
|
required_state: REQUIRED_STATE_LIST,
|
|
include_old_rooms: {
|
|
timeline_limit: 0,
|
|
required_state: REQUIRED_STATE_LIST,
|
|
},
|
|
filters: {
|
|
is_dm: true,
|
|
is_invite: false,
|
|
// If a DM has a Favourite & Low Prio tag then it'll be shown in those lists instead
|
|
not_tags: ["m.favourite", "m.lowpriority"],
|
|
},
|
|
},
|
|
untagged: {
|
|
// SSS will dupe suppress invites/dms from here, so we don't need "not dms, not invites"
|
|
ranges: [[0, 10]],
|
|
timeline_limit: 1, // most recent message display
|
|
required_state: REQUIRED_STATE_LIST,
|
|
include_old_rooms: {
|
|
timeline_limit: 0,
|
|
required_state: REQUIRED_STATE_LIST,
|
|
},
|
|
},
|
|
};
|
|
|
|
export type PartialSlidingSyncRequest = {
|
|
filters?: MSC3575Filter;
|
|
sort?: string[];
|
|
ranges?: [startIndex: number, endIndex: number][];
|
|
};
|
|
|
|
/**
|
|
* This class manages the entirety of sliding sync at a high UI/UX level. It controls the placement
|
|
* of placeholders in lists, controls updating sliding window ranges, and controls which events
|
|
* are pulled down when. The intention behind this manager is be the single place to look for sliding
|
|
* sync options and code.
|
|
*/
|
|
export class SlidingSyncManager {
|
|
public static readonly ListSpaces = "space_list";
|
|
public static readonly ListSearch = "search_list";
|
|
private static readonly internalInstance = new SlidingSyncManager();
|
|
|
|
public slidingSync?: SlidingSync;
|
|
private client?: MatrixClient;
|
|
|
|
private configureDefer = defer<void>();
|
|
|
|
public static get instance(): SlidingSyncManager {
|
|
return SlidingSyncManager.internalInstance;
|
|
}
|
|
|
|
private configure(client: MatrixClient, proxyUrl: string): SlidingSync {
|
|
this.client = client;
|
|
// create the set of lists we will use.
|
|
const lists = new Map();
|
|
for (const listName in sssLists) {
|
|
lists.set(listName, sssLists[listName]);
|
|
}
|
|
// by default use the encrypted subscription as that gets everything, which is a safer
|
|
// default than potentially missing member events.
|
|
this.slidingSync = new SlidingSync(
|
|
proxyUrl,
|
|
lists,
|
|
ENCRYPTED_SUBSCRIPTION,
|
|
client,
|
|
SLIDING_SYNC_TIMEOUT_MS,
|
|
);
|
|
this.slidingSync.addCustomSubscription(UNENCRYPTED_SUBSCRIPTION_NAME, UNENCRYPTED_SUBSCRIPTION);
|
|
this.configureDefer.resolve();
|
|
return this.slidingSync;
|
|
}
|
|
|
|
/**
|
|
* Ensure that this list is registered.
|
|
* @param listKey The list key to register
|
|
* @param updateArgs The fields to update on the list.
|
|
* @returns The complete list request params
|
|
*/
|
|
public async ensureListRegistered(listKey: string, updateArgs: PartialSlidingSyncRequest): Promise<MSC3575List> {
|
|
logger.debug("ensureListRegistered:::", listKey, updateArgs);
|
|
await this.configureDefer.promise;
|
|
let list = this.slidingSync!.getListParams(listKey);
|
|
if (!list) {
|
|
list = {
|
|
ranges: [[0, 20]],
|
|
sort: ["by_notification_level", "by_recency"],
|
|
timeline_limit: 1, // most recent message display: though this seems to only be needed for favourites?
|
|
required_state: [
|
|
[EventType.RoomJoinRules, ""], // the public icon on the room list
|
|
[EventType.RoomAvatar, ""], // any room avatar
|
|
[EventType.RoomTombstone, ""], // lets JS SDK hide rooms which are dead
|
|
[EventType.RoomEncryption, ""], // lets rooms be configured for E2EE correctly
|
|
[EventType.RoomCreate, ""], // for isSpaceRoom checks
|
|
[EventType.RoomMember, MSC3575_STATE_KEY_ME], // lets the client calculate that we are in fact in the room
|
|
],
|
|
include_old_rooms: {
|
|
timeline_limit: 0,
|
|
required_state: [
|
|
[EventType.RoomCreate, ""],
|
|
[EventType.RoomTombstone, ""], // lets JS SDK hide rooms which are dead
|
|
[EventType.SpaceChild, MSC3575_WILDCARD], // all space children
|
|
[EventType.SpaceParent, MSC3575_WILDCARD], // all space parents
|
|
[EventType.RoomMember, MSC3575_STATE_KEY_ME], // lets the client calculate that we are in fact in the room
|
|
],
|
|
},
|
|
...updateArgs,
|
|
};
|
|
} else {
|
|
const updatedList = { ...list, ...updateArgs };
|
|
// cannot use objectHasDiff as we need to do deep diff checking
|
|
if (JSON.stringify(list) === JSON.stringify(updatedList)) {
|
|
logger.debug("list matches, not sending, update => ", updateArgs);
|
|
return list;
|
|
}
|
|
list = updatedList;
|
|
}
|
|
|
|
try {
|
|
// if we only have range changes then call a different function so we don't nuke the list from before
|
|
if (updateArgs.ranges && Object.keys(updateArgs).length === 1) {
|
|
await this.slidingSync!.setListRanges(listKey, updateArgs.ranges);
|
|
} else {
|
|
await this.slidingSync!.setList(listKey, list);
|
|
}
|
|
} catch (err) {
|
|
logger.debug("ensureListRegistered: update failed txn_id=", err);
|
|
}
|
|
return this.slidingSync!.getListParams(listKey)!;
|
|
}
|
|
|
|
public async setRoomVisible(roomId: string, visible: boolean): Promise<string> {
|
|
await this.configureDefer.promise;
|
|
const subscriptions = this.slidingSync!.getRoomSubscriptions();
|
|
if (visible) {
|
|
subscriptions.add(roomId);
|
|
} else {
|
|
subscriptions.delete(roomId);
|
|
}
|
|
const room = this.client?.getRoom(roomId);
|
|
// default to safety: request all state if we can't work it out. This can happen if you
|
|
// refresh the app whilst viewing a room: we call setRoomVisible before we know anything
|
|
// about the room.
|
|
let shouldLazyLoad = false;
|
|
if (room) {
|
|
// do not lazy load encrypted rooms as we need the entire member list.
|
|
shouldLazyLoad = !(await this.client?.getCrypto()?.isEncryptionEnabledInRoom(roomId));
|
|
}
|
|
logger.log("SlidingSync setRoomVisible:", roomId, visible, "shouldLazyLoad:", shouldLazyLoad);
|
|
if (shouldLazyLoad) {
|
|
// lazy load this room
|
|
this.slidingSync!.useCustomSubscription(roomId, UNENCRYPTED_SUBSCRIPTION_NAME);
|
|
}
|
|
this.slidingSync!.modifyRoomSubscriptions(subscriptions);
|
|
if (room) {
|
|
return roomId; // we have data already for this room, show immediately e.g it's in a list
|
|
}
|
|
// wait until we know about this room. This may take a little while.
|
|
return new Promise((resolve) => {
|
|
logger.log(`SlidingSync setRoomVisible room ${roomId} not found, waiting for ClientEvent.Room`);
|
|
const waitForRoom = (r: Room): void => {
|
|
if (r.roomId === roomId) {
|
|
this.client?.off(ClientEvent.Room, waitForRoom);
|
|
logger.log(`SlidingSync room ${roomId} found, resolving setRoomVisible`);
|
|
resolve(roomId);
|
|
}
|
|
};
|
|
this.client?.on(ClientEvent.Room, waitForRoom);
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Retrieve all rooms on the user's account. Retrieval is gradual over time.
|
|
* This function MUST be called BEFORE the first sync request goes out.
|
|
* @param batchSize The number of rooms to return in each request.
|
|
* @param gapBetweenRequestsMs The number of milliseconds to wait between requests.
|
|
*/
|
|
private async startSpidering(slidingSync: SlidingSync, batchSize: number, gapBetweenRequestsMs: number): Promise<void> {
|
|
// The manager has created several lists (see `sssLists` in this file), all of which will be spidered simultaneously.
|
|
// There are multiple lists to ensure that we can populate invites/favourites/DMs sections immediately, rather than
|
|
// potentially waiting minutes if they are all very old rooms (and hence are returned last by the server). In this
|
|
// way, the lists are effectively priority requests. We don't actually care which room goes into which list at this
|
|
// point, as the RoomListStore will calculate this based on the returned data.
|
|
|
|
// copy the initial set of list names and ranges, we'll keep this map updated.
|
|
const listToUpperBound = new Map(Object.keys(sssLists).map((listName) => {
|
|
return [listName, sssLists[listName].ranges[0][1]];
|
|
}));
|
|
console.log("startSpidering:",listToUpperBound);
|
|
|
|
// listen for a response from the server. ANY 200 OK will do here, as we assume that it is ACKing
|
|
// the request change we have sent out. TODO: this may not be true if you concurrently subscribe to a room :/
|
|
// but in that case, for spidering at least, it isn't the end of the world as request N+1 includes all indexes
|
|
// from request N.
|
|
const lifecycle = async (state: SlidingSyncState, _: MSC3575SlidingSyncResponse | null, err?: Error): Promise<void> => {
|
|
if (state !== SlidingSyncState.Complete) {
|
|
return;
|
|
}
|
|
await sleep(gapBetweenRequestsMs); // don't tightloop; even on errors
|
|
if (err) {
|
|
return;
|
|
}
|
|
|
|
// for all lists with total counts > range => increase the range
|
|
let hasSetRanges = false;
|
|
listToUpperBound.forEach((currentUpperBound, listName) => {
|
|
const totalCount = slidingSync.getListData(listName)?.joinedCount || 0;
|
|
if (currentUpperBound < totalCount) {
|
|
// increment the upper bound
|
|
const newUpperBound = currentUpperBound + batchSize;
|
|
console.log(`startSpidering: ${listName} ${currentUpperBound} => ${newUpperBound}`);
|
|
listToUpperBound.set(listName, newUpperBound);
|
|
// make the next request. This will only send the request when this callback has finished, so if
|
|
// we set all the list ranges at once we will only send 1 new request.
|
|
slidingSync.setListRanges(listName, [[0,newUpperBound]]);
|
|
hasSetRanges = true;
|
|
}
|
|
});
|
|
if (!hasSetRanges) { // finish spidering
|
|
slidingSync.off(SlidingSyncEvent.Lifecycle, lifecycle);
|
|
}
|
|
};
|
|
slidingSync.on(SlidingSyncEvent.Lifecycle, lifecycle);
|
|
}
|
|
|
|
/**
|
|
* Set up the Sliding Sync instance; configures the end point and starts spidering.
|
|
* The sliding sync endpoint is derived the following way:
|
|
* 1. The user-defined sliding sync proxy URL (legacy, for backwards compatibility)
|
|
* 2. The client `well-known` sliding sync proxy URL [declared at the unstable prefix](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/sync-v3/proposals/3575-sync.md#unstable-prefix)
|
|
* 3. The homeserver base url (for native server support)
|
|
* @param client The MatrixClient to use
|
|
* @returns A working Sliding Sync or undefined
|
|
*/
|
|
public async setup(client: MatrixClient): Promise<SlidingSync | undefined> {
|
|
const slidingSync = this.configure(client, client.baseUrl);
|
|
logger.info("Simplified Sliding Sync activated at", client.baseUrl);
|
|
this.startSpidering(slidingSync, 50, 50); // 50 rooms at a time, 50ms apart
|
|
return slidingSync;
|
|
}
|
|
|
|
/**
|
|
* Check if the server "natively" supports sliding sync (with an unstable endpoint).
|
|
* @param client The MatrixClient to use
|
|
* @return Whether the "native" (unstable) endpoint is supported
|
|
*/
|
|
public async nativeSlidingSyncSupport(client: MatrixClient): Promise<boolean> {
|
|
// Per https://github.com/matrix-org/matrix-spec-proposals/pull/3575/files#r1589542561
|
|
// `client` can be undefined/null in tests for some reason.
|
|
const support = await client?.doesServerSupportUnstableFeature("org.matrix.simplified_msc3575");
|
|
if (support) {
|
|
logger.log("nativeSlidingSyncSupport: org.matrix.simplified_msc3575 sliding sync advertised as unstable");
|
|
}
|
|
return support;
|
|
}
|
|
|
|
/**
|
|
* Check whether our homeserver has sliding sync support, that the endpoint is up, and
|
|
* is a sliding sync endpoint.
|
|
*
|
|
* Sets static member `SlidingSyncController.serverSupportsSlidingSync`
|
|
* @param client The MatrixClient to use
|
|
*/
|
|
public async checkSupport(client: MatrixClient): Promise<void> {
|
|
if (await this.nativeSlidingSyncSupport(client)) {
|
|
SlidingSyncController.serverSupportsSlidingSync = true;
|
|
return;
|
|
}
|
|
SlidingSyncController.serverSupportsSlidingSync = false;
|
|
}
|
|
}
|