imterah/element-portable
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use native js-sdk group call support (#9625)

Browse source 
* Use native js-sdk group call support

Now that the js-sdk supports group calls natively, our group call implementation can be simplified a bit. Switching to the js-sdk implementation also brings the react-sdk up to date with recent MSC3401 changes, and adds support for joining calls from multiple devices. (So, the previous logic which sent to-device messages to prevent multi-device sessions is no longer necessary.)

* Fix strings

* Fix strict type errors
This commit is contained in:
Robin 2022-11-28 16:37:32 -05:00 • committed by GitHub
parent 3c7781a561
commit 2c612d5aa1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 383 additions and 567 deletions
Download patch file Download diff file Expand all files Collapse all files

432
src/models/Call.ts
View file

@ -17,13 +17,20 @@ limitations under the License.
import { TypedEventEmitter } from "matrix-js-sdk/src/models/typed-event-emitter";
import { logger } from "matrix-js-sdk/src/logger";
import { randomString } from "matrix-js-sdk/src/randomstring";
import { ClientEvent, MatrixClient } from "matrix-js-sdk/src/client";
import { MatrixClient } from "matrix-js-sdk/src/client";
import { RoomEvent } from "matrix-js-sdk/src/models/room";
import { RoomStateEvent } from "matrix-js-sdk/src/models/room-state";
import { CallType } from "matrix-js-sdk/src/webrtc/call";
import { NamespacedValue } from "matrix-js-sdk/src/NamespacedValue";
import { IWidgetApiRequest, MatrixWidgetType } from "matrix-widget-api";
import { MatrixEvent, MatrixEventEvent } from "matrix-js-sdk/src/models/event";
import {
GroupCall,
GroupCallEvent,
GroupCallIntent,
GroupCallState,
GroupCallType,
} from "matrix-js-sdk/src/webrtc/groupCall";
import { EventType } from "matrix-js-sdk/src/@types/event";
import type EventEmitter from "events";
import type { IMyDevice } from "matrix-js-sdk/src/client";
@ -89,7 +96,10 @@ export enum CallEvent {
interface CallEventHandlerMap {
[CallEvent.ConnectionState]: (state: ConnectionState, prevState: ConnectionState) => void;
[CallEvent.Participants]: (participants: Set<RoomMember>, prevParticipants: Set<RoomMember>) => void;
[CallEvent.Participants]: (
participants: Map<RoomMember, Set<string>>,
prevParticipants: Map<RoomMember, Set<string>>,
) => void;
[CallEvent.Layout]: (layout: Layout) => void;
[CallEvent.Destroy]: () => void;
}
@ -135,11 +145,14 @@ export abstract class Call extends TypedEventEmitter<CallEvent, CallEventHandler
return isConnected(this.connectionState);
}
private _participants = new Set<RoomMember>();
public get participants(): Set<RoomMember> {
private _participants = new Map<RoomMember, Set<string>>();
/**
* The participants in the call, as a map from members to device IDs.
*/
public get participants(): Map<RoomMember, Set<string>> {
return this._participants;
}
protected set participants(value: Set<RoomMember>) {
protected set participants(value: Map<RoomMember, Set<string>>) {
const prevValue = this._participants;
this._participants = value;
this.emit(CallEvent.Participants, value, prevValue);
@ -164,68 +177,11 @@ export abstract class Call extends TypedEventEmitter<CallEvent, CallEventHandler
return ElementCall.get(room) ?? JitsiCall.get(room);
}
/**
* Gets the connected devices associated with the given user in room state.
* @param userId The user's ID.
* @returns The IDs of the user's connected devices.
*/
protected abstract getDevices(userId: string): string[];
/**
* Sets the connected devices associated with ourselves in room state.
* @param devices The devices with which we're connected.
*/
protected abstract setDevices(devices: string[]): Promise<void>;
/**
* Updates our member state with the devices returned by the given function.
* @param fn A function from the current devices to the new devices. If it
* returns null, the update is skipped.
*/
protected async updateDevices(fn: (devices: string[]) => (string[] | null)): Promise<void> {
if (this.room.getMyMembership() !== "join") return;
const devices = fn(this.getDevices(this.client.getUserId()!));
if (devices) {
await this.setDevices(devices);
}
}
/**
* Performs a routine check of the call's associated room state, cleaning up
* any data left over from an unclean disconnection.
*/
public async clean(): Promise<void> {
const now = Date.now();
const { devices: myDevices } = await this.client.getDevices();
const deviceMap = new Map<string, IMyDevice>(myDevices.map(d => [d.device_id, d]));
// Clean up our member state by filtering out logged out devices,
// inactive devices, and our own device (if we're disconnected)
await this.updateDevices(devices => {
const newDevices = devices.filter(d => {
const device = deviceMap.get(d);
return device?.last_seen_ts !== undefined
&& !(d === this.client.getDeviceId() && !this.connected)
&& (now - device.last_seen_ts) < this.STUCK_DEVICE_TIMEOUT_MS;
});
// Skip the update if the devices are unchanged
return newDevices.length === devices.length ? null : newDevices;
});
}
protected async addOurDevice(): Promise<void> {
await this.updateDevices(devices => Array.from(new Set(devices).add(this.client.getDeviceId())));
}
protected async removeOurDevice(): Promise<void> {
await this.updateDevices(devices => {
const devicesSet = new Set(devices);
devicesSet.delete(this.client.getDeviceId());
return Array.from(devicesSet);
});
}
public abstract clean(): Promise<void>;
/**
* Contacts the widget to connect to the call.
@ -384,7 +340,7 @@ export class JitsiCall extends Call {
this.participantsExpirationTimer = null;
}
const members = new Set<RoomMember>();
const participants = new Map<RoomMember, Set<string>>();
const now = Date.now();
let allExpireAt = Infinity;
@ -392,44 +348,95 @@ export class JitsiCall extends Call {
const member = this.room.getMember(e.getStateKey()!);
const content = e.getContent<JitsiCallMemberContent>();
const expiresAt = typeof content.expires_ts === "number" ? content.expires_ts : -Infinity;
let devices = expiresAt > now && Array.isArray(content.devices) ? content.devices : [];
let devices = expiresAt > now && Array.isArray(content.devices)
? content.devices.filter(d => typeof d === "string")
: [];
// Apply local echo for the disconnected case
if (!this.connected && member?.userId === this.client.getUserId()) {
devices = devices.filter(d => d !== this.client.getDeviceId());
}
// Must have a connected device and still be joined to the room
if (devices.length && member?.membership === "join") {
members.add(member);
if (devices.length > 0 && member?.membership === "join") {
participants.set(member, new Set(devices));
if (expiresAt < allExpireAt) allExpireAt = expiresAt;
}
}
// Apply local echo for the connected case
if (this.connected) members.add(this.room.getMember(this.client.getUserId()!)!);
if (this.connected) {
const localMember = this.room.getMember(this.client.getUserId()!)!;
let devices = participants.get(localMember);
if (devices === undefined) {
devices = new Set();
participants.set(localMember, devices);
}
this.participants = members;
devices.add(this.client.getDeviceId()!);
}
this.participants = participants;
if (allExpireAt < Infinity) {
this.participantsExpirationTimer = setTimeout(() => this.updateParticipants(), allExpireAt - now);
}
}
protected getDevices(userId: string): string[] {
const event = this.room.currentState.getStateEvents(JitsiCall.MEMBER_EVENT_TYPE, userId);
/**
* Updates our member state with the devices returned by the given function.
* @param fn A function from the current devices to the new devices. If it
* returns null, the update is skipped.
*/
private async updateDevices(fn: (devices: string[]) => (string[] | null)): Promise<void> {
if (this.room.getMyMembership() !== "join") return;
const event = this.room.currentState.getStateEvents(JitsiCall.MEMBER_EVENT_TYPE, this.client.getUserId()!);
const content = event?.getContent<JitsiCallMemberContent>();
const expiresAt = typeof content?.expires_ts === "number" ? content.expires_ts : -Infinity;
return expiresAt > Date.now() && Array.isArray(content?.devices) ? content.devices : [];
const devices = expiresAt > Date.now() && Array.isArray(content?.devices) ? content!.devices : [];
const newDevices = fn(devices);
if (newDevices !== null) {
const newContent: JitsiCallMemberContent = {
devices: newDevices,
expires_ts: Date.now() + this.STUCK_DEVICE_TIMEOUT_MS,
};
await this.client.sendStateEvent(
this.roomId, JitsiCall.MEMBER_EVENT_TYPE, newContent, this.client.getUserId()!,
);
}
}
protected async setDevices(devices: string[]): Promise<void> {
const content: JitsiCallMemberContent = {
devices,
expires_ts: Date.now() + this.STUCK_DEVICE_TIMEOUT_MS,
};
public async clean(): Promise<void> {
const now = Date.now();
const { devices: myDevices } = await this.client.getDevices();
const deviceMap = new Map<string, IMyDevice>(myDevices.map(d => [d.device_id, d]));
await this.client.sendStateEvent(
this.roomId, JitsiCall.MEMBER_EVENT_TYPE, content, this.client.getUserId()!,
);
// Clean up our member state by filtering out logged out devices,
// inactive devices, and our own device (if we're disconnected)
await this.updateDevices(devices => {
const newDevices = devices.filter(d => {
const device = deviceMap.get(d);
return device?.last_seen_ts !== undefined
&& !(d === this.client.getDeviceId() && !this.connected)
&& (now - device.last_seen_ts) < this.STUCK_DEVICE_TIMEOUT_MS;
});
// Skip the update if the devices are unchanged
return newDevices.length === devices.length ? null : newDevices;
});
}
private async addOurDevice(): Promise<void> {
await this.updateDevices(devices => Array.from(new Set(devices).add(this.client.getDeviceId()!)));
}
private async removeOurDevice(): Promise<void> {
await this.updateDevices(devices => {
const devicesSet = new Set(devices);
devicesSet.delete(this.client.getDeviceId()!);
return Array.from(devicesSet);
});
}
protected async performConnection(
@ -591,31 +598,15 @@ export class JitsiCall extends Call {
};
}
export interface ElementCallMemberContent {
"m.expires_ts": number;
"m.calls": {
"m.call_id": string;
"m.devices": {
device_id: string;
session_id: string;
feeds: unknown[]; // We don't care about what these are
}[];
}[];
}
/**
* A group call using MSC3401 and Element Call as a backend.
* (somewhat cheekily named)
*/
export class ElementCall extends Call {
public static readonly CALL_EVENT_TYPE = new NamespacedValue(null, "org.matrix.msc3401.call");
public static readonly MEMBER_EVENT_TYPE = new NamespacedValue(null, "org.matrix.msc3401.call.member");
public static readonly DUPLICATE_CALL_DEVICE_EVENT_TYPE = "io.element.duplicate_call_device";
public static readonly CALL_EVENT_TYPE = new NamespacedValue(null, EventType.GroupCallPrefix);
public static readonly MEMBER_EVENT_TYPE = new NamespacedValue(null, EventType.GroupCallMemberPrefix);
public readonly STUCK_DEVICE_TIMEOUT_MS = 1000 * 60 * 60; // 1 hour
private kickedOutByAnotherDevice = false;
private connectionTime: number | null = null;
private participantsExpirationTimer: number | null = null;
private terminationTimer: number | null = null;
private _layout = Layout.Tile;
@ -627,7 +618,7 @@ export class ElementCall extends Call {
this.emit(CallEvent.Layout, value);
}
private constructor(public readonly groupCall: MatrixEvent, client: MatrixClient) {
private constructor(public readonly groupCall: GroupCall, client: MatrixClient) {
// Splice together the Element Call URL for this call
const url = new URL(SdkConfig.get("element_call").url ?? DEFAULTS.element_call.url!);
url.pathname = "/room";
@ -636,8 +627,8 @@ export class ElementCall extends Call {
preload: "",
hideHeader: "",
userId: client.getUserId()!,
deviceId: client.getDeviceId(),
roomId: groupCall.getRoomId()!,
deviceId: client.getDeviceId()!,
roomId: groupCall.room.roomId,
baseUrl: client.baseUrl,
lang: getCurrentLanguage().replace("_", "-"),
});
@ -652,14 +643,15 @@ export class ElementCall extends Call {
name: "Element Call",
type: MatrixWidgetType.Custom,
url: url.toString(),
}, groupCall.getRoomId()!),
}, groupCall.room.roomId),
client,
);
this.groupCall.on(MatrixEventEvent.BeforeRedaction, this.onBeforeRedaction);
this.room.on(RoomStateEvent.Update, this.onRoomState);
this.on(CallEvent.ConnectionState, this.onConnectionState);
this.on(CallEvent.Participants, this.onParticipants);
groupCall.on(GroupCallEvent.ParticipantsChanged, this.onGroupCallParticipants);
groupCall.on(GroupCallEvent.GroupCallStateChanged, this.onGroupCallState);
this.updateParticipants();
}
@ -672,22 +664,8 @@ export class ElementCall extends Call {
&& room.isCallRoom()
)
) {
const groupCalls = ElementCall.CALL_EVENT_TYPE.names.flatMap(eventType =>
room.currentState.getStateEvents(eventType),
);
// Find the newest unterminated call
let groupCall: MatrixEvent | null = null;
for (const event of groupCalls) {
if (
!("m.terminated" in event.getContent())
&& (groupCall === null || event.getTs() > groupCall.getTs())
) {
groupCall = event;
}
}
if (groupCall !== null) return new ElementCall(groupCall, room.client);
const groupCall = room.client.groupCallEventHandler!.groupCalls.get(room.roomId);
if (groupCall !== undefined) return new ElementCall(groupCall, room.client);
}
return null;
@ -698,113 +676,25 @@ export class ElementCall extends Call {
&& SettingsStore.getValue("feature_element_call_video_rooms")
&& room.isCallRoom();
await room.client.sendStateEvent(room.roomId, ElementCall.CALL_EVENT_TYPE.name, {
"m.intent": isVideoRoom ? "m.room" : "m.prompt",
"m.type": "m.video",
}, randomString(24));
}
private updateParticipants() {
if (this.participantsExpirationTimer !== null) {
clearTimeout(this.participantsExpirationTimer);
this.participantsExpirationTimer = null;
}
const members = new Set<RoomMember>();
const now = Date.now();
let allExpireAt = Infinity;
const memberEvents = ElementCall.MEMBER_EVENT_TYPE.names.flatMap(eventType =>
this.room.currentState.getStateEvents(eventType),
const groupCall = new GroupCall(
room.client,
room,
GroupCallType.Video,
false,
isVideoRoom ? GroupCallIntent.Room : GroupCallIntent.Prompt,
);
for (const e of memberEvents) {
const member = this.room.getMember(e.getStateKey()!);
const content = e.getContent<ElementCallMemberContent>();
const expiresAt = typeof content["m.expires_ts"] === "number" ? content["m.expires_ts"] : -Infinity;
const calls = expiresAt > now && Array.isArray(content["m.calls"]) ? content["m.calls"] : [];
const call = calls.find(call => call["m.call_id"] === this.groupCall.getStateKey());
let devices = Array.isArray(call?.["m.devices"]) ? call!["m.devices"] : [];
// Apply local echo for the disconnected case
if (!this.connected && member?.userId === this.client.getUserId()) {
devices = devices.filter(d => d.device_id !== this.client.getDeviceId());
}
// Must have a connected device and still be joined to the room
if (devices.length && member?.membership === "join") {
members.add(member);
if (expiresAt < allExpireAt) allExpireAt = expiresAt;
}
}
// Apply local echo for the connected case
if (this.connected) members.add(this.room.getMember(this.client.getUserId()!)!);
this.participants = members;
if (allExpireAt < Infinity) {
this.participantsExpirationTimer = setTimeout(() => this.updateParticipants(), allExpireAt - now);
}
await groupCall.create();
}
private getCallsState(userId: string): ElementCallMemberContent["m.calls"] {
const event = (() => {
for (const eventType of ElementCall.MEMBER_EVENT_TYPE.names) {
const e = this.room.currentState.getStateEvents(eventType, userId);
if (e) return e;
}
return null;
})();
const content = event?.getContent<ElementCallMemberContent>();
const expiresAt = typeof content?.["m.expires_ts"] === "number" ? content["m.expires_ts"] : -Infinity;
return expiresAt > Date.now() && Array.isArray(content?.["m.calls"]) ? content!["m.calls"] : [];
}
protected getDevices(userId: string): string[] {
const calls = this.getCallsState(userId);
const call = calls.find(call => call["m.call_id"] === this.groupCall.getStateKey());
const devices = Array.isArray(call?.["m.devices"]) ? call!["m.devices"] : [];
return devices.map(d => d.device_id);
}
protected async setDevices(devices: string[]): Promise<void> {
const calls = this.getCallsState(this.client.getUserId()!);
const call = calls.find(c => c["m.call_id"] === this.groupCall.getStateKey())!;
const prevDevices = Array.isArray(call?.["m.devices"]) ? call!["m.devices"] : [];
const prevDevicesMap = new Map(prevDevices.map(d => [d.device_id, d]));
const newContent: ElementCallMemberContent = {
"m.expires_ts": Date.now() + this.STUCK_DEVICE_TIMEOUT_MS,
"m.calls": [
{
"m.call_id": this.groupCall.getStateKey()!,
// This method will only ever be used to remove devices, so
// it's safe to assume that all requested devices are
// present in the map
"m.devices": devices.map(d => prevDevicesMap.get(d)!),
},
...calls.filter(c => c !== call),
],
};
await this.client.sendStateEvent(
this.roomId, ElementCall.MEMBER_EVENT_TYPE.name, newContent, this.client.getUserId()!,
);
public clean(): Promise<void> {
return this.groupCall.cleanMemberState();
}
protected async performConnection(
audioInput: MediaDeviceInfo | null,
videoInput: MediaDeviceInfo | null,
): Promise<void> {
this.kickedOutByAnotherDevice = false;
this.client.on(ClientEvent.ToDeviceEvent, this.onToDeviceEvent);
this.connectionTime = Date.now();
await this.client.sendToDevice(ElementCall.DUPLICATE_CALL_DEVICE_EVENT_TYPE, {
[this.client.getUserId()]: {
"*": { device_id: this.client.getDeviceId(), timestamp: this.connectionTime },
},
});
try {
await this.messaging!.transport.send(ElementWidgetActions.JoinCall, {
audioInput: audioInput?.label ?? null,
@ -829,7 +719,6 @@ export class ElementCall extends Call {
}
public setDisconnected() {
this.client.off(ClientEvent.ToDeviceEvent, this.onToDeviceEvent);
this.messaging!.off(`action:${ElementWidgetActions.HangupCall}`, this.onHangup);
this.messaging!.off(`action:${ElementWidgetActions.TileLayout}`, this.onTileLayout);
this.messaging!.off(`action:${ElementWidgetActions.SpotlightLayout}`, this.onSpotlightLayout);
@ -838,16 +727,12 @@ export class ElementCall extends Call {
}
public destroy() {
this.groupCall.off(MatrixEventEvent.BeforeRedaction, this.onBeforeRedaction);
WidgetStore.instance.removeVirtualWidget(this.widget.id, this.groupCall.getRoomId()!);
this.room.off(RoomStateEvent.Update, this.onRoomState);
WidgetStore.instance.removeVirtualWidget(this.widget.id, this.groupCall.room.roomId);
this.off(CallEvent.ConnectionState, this.onConnectionState);
this.off(CallEvent.Participants, this.onParticipants);
this.groupCall.off(GroupCallEvent.ParticipantsChanged, this.onGroupCallParticipants);
this.groupCall.off(GroupCallEvent.GroupCallStateChanged, this.onGroupCallState);
if (this.participantsExpirationTimer !== null) {
clearTimeout(this.participantsExpirationTimer);
this.participantsExpirationTimer = null;
}
if (this.terminationTimer !== null) {
clearTimeout(this.terminationTimer);
this.terminationTimer = null;
@ -868,50 +753,35 @@ export class ElementCall extends Call {
await this.messaging!.transport.send(action, {});
}
private updateParticipants() {
const participants = new Map<RoomMember, Set<string>>();
for (const [member, deviceMap] of this.groupCall.participants) {
participants.set(member, new Set(deviceMap.keys()));
}
// We never enter group calls natively, so the GroupCall will think it's
// disconnected regardless of what our call member state says. Thus we
// have to insert our own device manually when connected via the widget.
if (this.connected) {
const localMember = this.room.getMember(this.client.getUserId()!)!;
let devices = participants.get(localMember);
if (devices === undefined) {
devices = new Set();
participants.set(localMember, devices);
}
devices.add(this.client.getDeviceId()!);
}
this.participants = participants;
}
private get mayTerminate(): boolean {
if (this.kickedOutByAnotherDevice) return false;
if (this.groupCall.getContent()["m.intent"] === "m.room") return false;
if (
!this.room.currentState.mayClientSendStateEvent(ElementCall.CALL_EVENT_TYPE.name, this.client)
) return false;
return true;
return this.groupCall.intent !== GroupCallIntent.Room
&& this.room.currentState.mayClientSendStateEvent(ElementCall.CALL_EVENT_TYPE.name, this.client);
}
private async terminate(): Promise<void> {
await this.client.sendStateEvent(
this.roomId,
ElementCall.CALL_EVENT_TYPE.name,
{ ...this.groupCall.getContent(), "m.terminated": "Call ended" },
this.groupCall.getStateKey(),
);
}
private onBeforeRedaction = (): void => {
this.disconnect();
};
private onRoomState = () => {
this.updateParticipants();
// Destroy the call if it's been terminated
const newGroupCall = this.room.currentState.getStateEvents(
this.groupCall.getType(), this.groupCall.getStateKey()!,
);
if ("m.terminated" in newGroupCall.getContent()) this.destroy();
};
private onToDeviceEvent = (event: MatrixEvent): void => {
const content = event.getContent();
if (event.getType() !== ElementCall.DUPLICATE_CALL_DEVICE_EVENT_TYPE) return;
if (event.getSender() !== this.client.getUserId()) return;
if (content.device_id === this.client.getDeviceId()) return;
if (content.timestamp <= this.connectionTime) return;
this.kickedOutByAnotherDevice = true;
this.disconnect();
};
private onConnectionState = (state: ConnectionState, prevState: ConnectionState) => {
if (
(state === ConnectionState.Connected && !isConnected(prevState))
@ -921,12 +791,21 @@ export class ElementCall extends Call {
}
};
private onParticipants = async (participants: Set<RoomMember>, prevParticipants: Set<RoomMember>) => {
private onParticipants = async (
participants: Map<RoomMember, Set<string>>,
prevParticipants: Map<RoomMember, Set<string>>,
) => {
let participantCount = 0;
for (const devices of participants.values()) participantCount += devices.size;
let prevParticipantCount = 0;
for (const devices of prevParticipants.values()) prevParticipantCount += devices.size;
// If the last participant disconnected, terminate the call
if (participants.size === 0 && prevParticipants.size > 0 && this.mayTerminate) {
if (prevParticipants.has(this.room.getMember(this.client.getUserId()!)!)) {
if (participantCount === 0 && prevParticipantCount > 0 && this.mayTerminate) {
if (prevParticipants.get(this.room.getMember(this.client.getUserId()!)!)?.has(this.client.getDeviceId()!)) {
// If we were that last participant, do the termination ourselves
await this.terminate();
await this.groupCall.terminate();
} else {
// We don't appear to have been the last participant, but because of
// the potential for races, users lacking permission, and a myriad of
@ -935,11 +814,20 @@ export class ElementCall extends Call {
// randomly between 2 and 8 seconds before terminating the call, to
// probabilistically reduce event spam. If someone else beats us to it,
// this timer will be automatically cleared upon the call's destruction.
this.terminationTimer = setTimeout(() => this.terminate(), Math.random() * 6000 + 2000);
this.terminationTimer = setTimeout(
() => this.groupCall.terminate(),
Math.random() * 6000 + 2000,
);
}
}
};
private onGroupCallParticipants = () => this.updateParticipants();
private onGroupCallState = (state: GroupCallState) => {
if (state === GroupCallState.Ended) this.destroy();
};
private onHangup = async (ev: CustomEvent<IWidgetApiRequest>) => {
ev.preventDefault();
await this.messaging!.transport.reply(ev.detail, {}); // ack