Use new matrixRTC calling (#11792)

* initial

Signed-off-by: Timo K <toger5@hotmail.de>

* cleanup1

Signed-off-by: Timo K <toger5@hotmail.de>

* bring back call timer

Signed-off-by: Timo K <toger5@hotmail.de>

* more cleanup and test removals

Signed-off-by: Timo K <toger5@hotmail.de>

* remove event

Signed-off-by: Timo K <toger5@hotmail.de>

* cleanups and minor fixes

Signed-off-by: Timo K <toger5@hotmail.de>

* add matrixRTC to stubClient

Signed-off-by: Timo K <toger5@hotmail.de>

* update tests (some got removed)
The removal is a consequence of EW now doing less call logic.
More logic is done by the js sdk (MatrixRTCSession)
And therefore in EC itself.

Signed-off-by: Timo K <toger5@hotmail.de>

* cleanups

Signed-off-by: Timo K <toger5@hotmail.de>

* mock the session

Signed-off-by: Timo K <toger5@hotmail.de>

* lint

Signed-off-by: Timo K <toger5@hotmail.de>

* remove GroupCallDuration

Signed-off-by: Timo K <toger5@hotmail.de>

* review and fixing tests

Signed-off-by: Timo K <toger5@hotmail.de>

---------

Signed-off-by: Timo K <toger5@hotmail.de>
This commit is contained in:
Timo 2023-10-30 16:14:27 +01:00 committed by GitHub
parent c4852dd216
commit 860764c057
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 176 additions and 255 deletions

View file

@ -15,7 +15,7 @@ limitations under the License.
*/
import React, { useCallback } from "react";
import { EventType, Room } from "matrix-js-sdk/src/matrix";
import { Room } from "matrix-js-sdk/src/matrix";
import { logger } from "matrix-js-sdk/src/logger";
import { _t } from "../../../languageHandler";
@ -27,7 +27,7 @@ import { ConnectionState, ElementCall } from "../../../models/Call";
import { useCall } from "../../../hooks/useCall";
import { useEventEmitterState } from "../../../hooks/useEventEmitter";
import { OwnBeaconStore, OwnBeaconStoreEvent } from "../../../stores/OwnBeaconStore";
import { GroupCallDuration } from "../voip/CallDuration";
import { SessionDuration } from "../voip/CallDuration";
import { SdkContextClass } from "../../../contexts/SDKContext";
interface RoomCallBannerProps {
@ -49,12 +49,13 @@ const RoomCallBannerInner: React.FC<RoomCallBannerProps> = ({ roomId, call }) =>
[roomId],
);
// TODO matrix rtc
const onClick = useCallback(() => {
const event = call.groupCall.room.currentState.getStateEvents(
EventType.GroupCallPrefix,
call.groupCall.groupCallId,
);
if (event === null) {
logger.log("clicking on the call banner is not supported anymore - there are no timeline events anymore.");
let messageLikeEventId: string | undefined;
if (!messageLikeEventId) {
// Until we have a timeline event for calls this will always be true.
// We will never jump to the non existing timeline event.
logger.error("Couldn't find a group call event to jump to");
return;
}
@ -63,17 +64,17 @@ const RoomCallBannerInner: React.FC<RoomCallBannerProps> = ({ roomId, call }) =>
action: Action.ViewRoom,
room_id: roomId,
metricsTrigger: undefined,
event_id: event.getId(),
event_id: messageLikeEventId,
scroll_into_view: true,
highlighted: true,
});
}, [call, roomId]);
}, [roomId]);
return (
<div className="mx_RoomCallBanner" onClick={onClick}>
<div className="mx_RoomCallBanner_text">
<span className="mx_RoomCallBanner_label">{_t("voip|video_call")}</span>
<GroupCallDuration groupCall={call.groupCall} />
<SessionDuration session={call.session} />
</div>
<AccessibleButton onClick={connect} kind="primary" element="button" disabled={false}>

View file

@ -33,7 +33,7 @@ import MemberAvatar from "../avatars/MemberAvatar";
import { LiveContentSummary, LiveContentType } from "../rooms/LiveContentSummary";
import FacePile from "../elements/FacePile";
import MatrixClientContext from "../../../contexts/MatrixClientContext";
import { CallDuration, GroupCallDuration } from "../voip/CallDuration";
import { CallDuration, SessionDuration } from "../voip/CallDuration";
import AccessibleTooltipButton from "../elements/AccessibleTooltipButton";
const MAX_FACES = 8;
@ -77,7 +77,7 @@ const ActiveCallEvent = forwardRef<any, ActiveCallEventProps>(
/>
<FacePile members={facePileMembers} size="24px" overflow={facePileOverflow} />
</div>
{call && <GroupCallDuration groupCall={call.groupCall} />}
{call && <SessionDuration session={call.session} />}
<AccessibleTooltipButton
className="mx_CallEvent_button"
kind={buttonKind}

View file

@ -65,7 +65,7 @@ import IconizedContextMenu, {
IconizedContextMenuRadio,
} from "../context_menus/IconizedContextMenu";
import { ViewRoomPayload } from "../../../dispatcher/payloads/ViewRoomPayload";
import { GroupCallDuration } from "../voip/CallDuration";
import { SessionDuration } from "../voip/CallDuration";
import { Alignment } from "../elements/Tooltip";
import RoomCallBanner from "../beacon/RoomCallBanner";
import { shouldShowComponent } from "../../../customisations/helpers/UIComponents";
@ -787,7 +787,7 @@ export default class RoomHeader extends React.Component<IProps, IState> {
{icon}
{name}
{this.props.activeCall instanceof ElementCall && (
<GroupCallDuration groupCall={this.props.activeCall.groupCall} />
<SessionDuration session={this.props.activeCall?.session} />
)}
{/* Empty topic element to fill out space */}
<div className="mx_LegacyRoomHeader_topic" />

View file

@ -15,7 +15,8 @@ limitations under the License.
*/
import React, { FC, useState, useEffect, memo } from "react";
import { GroupCall } from "matrix-js-sdk/src/matrix";
// eslint-disable-next-line no-restricted-imports
import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { formatPreciseDuration } from "../../../DateUtils";
@ -32,20 +33,25 @@ export const CallDuration: FC<CallDurationProps> = memo(({ delta }) => {
return <div className="mx_CallDuration">{formatPreciseDuration(delta)}</div>;
});
interface GroupCallDurationProps {
groupCall: GroupCall;
interface SessionDurationProps {
session: MatrixRTCSession | undefined;
}
/**
* A call duration counter that automatically counts up, given a live GroupCall
* A call duration counter that automatically counts up, given a matrixRTC session
* object.
*/
export const GroupCallDuration: FC<GroupCallDurationProps> = ({ groupCall }) => {
export const SessionDuration: FC<SessionDurationProps> = ({ session }) => {
const [now, setNow] = useState(() => Date.now());
useEffect(() => {
const timer = window.setInterval(() => setNow(Date.now()), 1000);
return () => clearInterval(timer);
}, []);
return groupCall.creationTs === null ? null : <CallDuration delta={now - groupCall.creationTs} />;
// This is a temporal solution.
// Using the oldest membership will update when this user leaves.
// This implies that the displayed call duration will also update consequently.
const createdTs = session?.getOldestMembership()?.createdTs();
return createdTs ? <CallDuration delta={now - createdTs} /> : <CallDuration delta={0} />;
};

View file

@ -20,20 +20,21 @@ import {
RoomStateEvent,
EventType,
MatrixClient,
GroupCall,
GroupCallEvent,
GroupCallIntent,
GroupCallState,
GroupCallType,
IMyDevice,
Room,
RoomMember,
} from "matrix-js-sdk/src/matrix";
import { logger } from "matrix-js-sdk/src/logger";
import { randomString } from "matrix-js-sdk/src/randomstring";
import { CallType } from "matrix-js-sdk/src/webrtc/call";
import { NamespacedValue } from "matrix-js-sdk/src/NamespacedValue";
import { IWidgetApiRequest, MatrixWidgetType } from "matrix-widget-api";
import { IWidgetApiRequest } from "matrix-widget-api";
// eslint-disable-next-line no-restricted-imports
import { MatrixRTCSession, MatrixRTCSessionEvent } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
// eslint-disable-next-line no-restricted-imports
import { MatrixRTCSessionManagerEvents } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSessionManager";
import type EventEmitter from "events";
import type { IMyDevice, Room, RoomMember } from "matrix-js-sdk/src/matrix";
import type { ClientWidgetApi } from "matrix-widget-api";
import type { IApp } from "../stores/WidgetStore";
import SdkConfig, { DEFAULTS } from "../SdkConfig";
@ -615,12 +616,13 @@ export class JitsiCall extends Call {
* (somewhat cheekily named)
*/
export class ElementCall extends Call {
// TODO this is only there to support backwards compatiblity in timeline rendering
// this should not be part of this class since it has nothing to do with it.
public static readonly CALL_EVENT_TYPE = new NamespacedValue(null, EventType.GroupCallPrefix);
public static readonly MEMBER_EVENT_TYPE = new NamespacedValue(null, EventType.GroupCallMemberPrefix);
public readonly STUCK_DEVICE_TIMEOUT_MS = 1000 * 60 * 60; // 1 hour
private terminationTimer: number | null = null;
private _layout = Layout.Tile;
public get layout(): Layout {
return this._layout;
@ -630,7 +632,13 @@ export class ElementCall extends Call {
this.emit(CallEvent.Layout, value);
}
private constructor(public readonly groupCall: GroupCall, client: MatrixClient) {
private static createCallWidget(roomId: string, client: MatrixClient): IApp {
const ecWidget = WidgetStore.instance.getApps(roomId).find((app) => WidgetType.CALL.matches(app.type));
if (ecWidget) {
logger.log("There is already a widget in this room, so we recreate it");
ActiveWidgetStore.instance.destroyPersistentWidget(ecWidget.id, ecWidget.roomId);
WidgetStore.instance.removeVirtualWidget(ecWidget.id, ecWidget.roomId);
}
const accountAnalyticsData = client.getAccountData(PosthogAnalytics.ANALYTICS_EVENT_TYPE);
// The analyticsID is passed directly to element call (EC) since this codepath is only for EC and no other widget.
// We really don't want the same analyticID's for the EC and EW posthog instances (Data on posthog should be limited/anonymized as much as possible).
@ -639,7 +647,6 @@ export class ElementCall extends Call {
const analyticsID: string = accountAnalyticsData?.getContent().pseudonymousAnalyticsOptIn
? accountAnalyticsData?.getContent().id
: "";
// Splice together the Element Call URL for this call
const params = new URLSearchParams({
embed: "true", // We're embedding EC within another application
@ -648,14 +655,14 @@ export class ElementCall extends Call {
hideHeader: "true", // Hide the header since our room header is enough
userId: client.getUserId()!,
deviceId: client.getDeviceId()!,
roomId: groupCall.room.roomId,
roomId: roomId,
baseUrl: client.baseUrl,
lang: getCurrentLanguage().replace("_", "-"),
fontScale: `${(SettingsStore.getValue("baseFontSizeV2") ?? 16) / FontWatcher.DEFAULT_SIZE}`,
analyticsID,
});
if (client.isRoomEncrypted(groupCall.room.roomId)) params.append("perParticipantE2EE", "");
if (client.isRoomEncrypted(roomId)) params.append("perParticipantE2EE", "");
if (SettingsStore.getValue("fallbackICEServerAllowed")) params.append("allowIceFallback", "");
if (SettingsStore.getValue("feature_allow_screen_share_only_mode")) params.append("allowVoipWithNoMedia", "");
@ -678,30 +685,23 @@ export class ElementCall extends Call {
// To use Element Call without touching room state, we create a virtual
// widget (one that doesn't have a corresponding state event)
super(
WidgetStore.instance.addVirtualWidget(
{
id: randomString(24), // So that it's globally unique
creatorUserId: client.getUserId()!,
name: "Element Call",
type: MatrixWidgetType.Custom,
url: url.toString(),
// This option makes the widget API wait for the 'contentLoaded' event instead
// of waiting for a 'load' event from the iframe, which means the widget code isn't
// racing to set up its listener before the 'load' event is fired. EC sends this event
// of of https://github.com/matrix-org/matrix-js-sdk/pull/3556 so we should uncomment
// the line below once we've made both livekit and full-mesh releases that include that
// PR, and everything will be less racy.
//waitForIframeLoad: false,
},
groupCall.room.roomId,
),
client,
return WidgetStore.instance.addVirtualWidget(
{
id: randomString(24), // So that it's globally unique
creatorUserId: client.getUserId()!,
name: "Element Call",
type: WidgetType.CALL.preferred,
url: url.toString(),
// waitForIframeLoad: false,
},
roomId,
);
}
private constructor(public session: MatrixRTCSession, widget: IApp, client: MatrixClient) {
super(widget, client);
this.on(CallEvent.Participants, this.onParticipants);
groupCall.on(GroupCallEvent.ParticipantsChanged, this.onGroupCallParticipants);
groupCall.on(GroupCallEvent.GroupCallStateChanged, this.onGroupCallState);
this.session.on(MatrixRTCSessionEvent.MembershipsChanged, this.onMembershipChanged);
this.client.matrixRTC.on(MatrixRTCSessionManagerEvents.SessionEnded, this.onRTCSessionEnded);
this.updateParticipants();
}
@ -714,8 +714,18 @@ export class ElementCall extends Call {
SettingsStore.getValue("feature_element_call_video_rooms") &&
room.isCallRoom())
) {
const groupCall = room.client.groupCallEventHandler!.groupCalls.get(room.roomId);
if (groupCall !== undefined) return new ElementCall(groupCall, room.client);
const apps = WidgetStore.instance.getApps(room.roomId);
const ecWidget = apps.find((app) => WidgetType.CALL.matches(app.type));
const session = room.client.matrixRTC.getRoomSession(room);
// A call is present if we
// - have a widget: This means the create function was called
// - or there is a running session where we have not yet created a widget for.
if (ecWidget || session.memberships.length !== 0) {
// create a widget for the case we are joining a running call and don't have on yet.
const availableOrCreatedWidget = ecWidget ?? ElementCall.createCallWidget(room.roomId, room.client);
return new ElementCall(session, availableOrCreatedWidget, room.client);
}
}
return null;
@ -727,19 +737,8 @@ export class ElementCall extends Call {
SettingsStore.getValue("feature_element_call_video_rooms") &&
room.isCallRoom();
const groupCall = new GroupCall(
room.client,
room,
GroupCallType.Video,
false,
isVideoRoom ? GroupCallIntent.Room : GroupCallIntent.Prompt,
);
await groupCall.create();
}
public clean(): Promise<void> {
return this.groupCall.cleanMemberState();
console.log("Intend is ", isVideoRoom ? "VideoRoom" : "Prompt", " TODO, handle intent appropriately");
ElementCall.createCallWidget(room.roomId, room.client);
}
protected async performConnection(
@ -755,7 +754,6 @@ export class ElementCall extends Call {
throw new Error(`Failed to join call in room ${this.roomId}: ${e}`);
}
this.groupCall.enteredViaAnotherSession = true;
this.messaging!.on(`action:${ElementWidgetActions.HangupCall}`, this.onHangup);
this.messaging!.on(`action:${ElementWidgetActions.TileLayout}`, this.onTileLayout);
this.messaging!.on(`action:${ElementWidgetActions.SpotlightLayout}`, this.onSpotlightLayout);
@ -774,15 +772,14 @@ export class ElementCall extends Call {
this.messaging!.off(`action:${ElementWidgetActions.TileLayout}`, this.onTileLayout);
this.messaging!.off(`action:${ElementWidgetActions.SpotlightLayout}`, this.onSpotlightLayout);
super.setDisconnected();
this.groupCall.enteredViaAnotherSession = false;
}
public destroy(): void {
ActiveWidgetStore.instance.destroyPersistentWidget(this.widget.id, this.groupCall.room.roomId);
WidgetStore.instance.removeVirtualWidget(this.widget.id, this.groupCall.room.roomId);
this.off(CallEvent.Participants, this.onParticipants);
this.groupCall.off(GroupCallEvent.ParticipantsChanged, this.onGroupCallParticipants);
this.groupCall.off(GroupCallEvent.GroupCallStateChanged, this.onGroupCallState);
ActiveWidgetStore.instance.destroyPersistentWidget(this.widget.id, this.widget.roomId);
WidgetStore.instance.removeVirtualWidget(this.widget.id, this.widget.roomId);
this.session.off(MatrixRTCSessionEvent.MembershipsChanged, this.onMembershipChanged);
this.client.matrixRTC.off(MatrixRTCSessionManagerEvents.SessionEnded, this.onRTCSessionEnded);
if (this.terminationTimer !== null) {
clearTimeout(this.terminationTimer);
@ -792,70 +789,41 @@ export class ElementCall extends Call {
super.destroy();
}
private onRTCSessionEnded = (roomId: string, session: MatrixRTCSession): void => {
if (roomId == this.roomId) {
this.destroy();
}
};
/**
* Sets the call's layout.
* @param layout The layout to switch to.
*/
public async setLayout(layout: Layout): Promise<void> {
const action = layout === Layout.Tile ? ElementWidgetActions.TileLayout : ElementWidgetActions.SpotlightLayout;
await this.messaging!.transport.send(action, {});
}
private onMembershipChanged = (): void => this.updateParticipants();
private updateParticipants(): void {
const participants = new Map<RoomMember, Set<string>>();
for (const [member, deviceMap] of this.groupCall.participants) {
participants.set(member, new Set(deviceMap.keys()));
for (const m of this.session.memberships) {
if (!m.sender) continue;
const member = this.room.getMember(m.sender);
if (member) {
if (participants.has(member)) {
participants.get(member)?.add(m.deviceId);
} else {
participants.set(member, new Set([m.deviceId]));
}
}
}
this.participants = participants;
}
private get mayTerminate(): boolean {
return (
this.groupCall.intent !== GroupCallIntent.Room &&
this.room.currentState.mayClientSendStateEvent(ElementCall.CALL_EVENT_TYPE.name, this.client)
);
}
private onParticipants = async (
participants: Map<RoomMember, Set<string>>,
prevParticipants: Map<RoomMember, Set<string>>,
): Promise<void> => {
let participantCount = 0;
for (const devices of participants.values()) participantCount += devices.size;
let prevParticipantCount = 0;
for (const devices of prevParticipants.values()) prevParticipantCount += devices.size;
// If the last participant disconnected, terminate the call
if (participantCount === 0 && prevParticipantCount > 0 && this.mayTerminate) {
if (prevParticipants.get(this.room.getMember(this.client.getUserId()!)!)?.has(this.client.getDeviceId()!)) {
// If we were that last participant, do the termination ourselves
await this.groupCall.terminate();
} else {
// We don't appear to have been the last participant, but because of
// the potential for races, users lacking permission, and a myriad of
// other reasons, we can't rely on other clients to terminate the call.
// Since it's likely that other clients are using this same logic, we wait
// randomly between 2 and 8 seconds before terminating the call, to
// probabilistically reduce event spam. If someone else beats us to it,
// this timer will be automatically cleared upon the call's destruction.
this.terminationTimer = window.setTimeout(
() => this.groupCall.terminate(),
Math.random() * 6000 + 2000,
);
}
}
};
private onGroupCallParticipants = (): void => this.updateParticipants();
private onGroupCallState = (state: GroupCallState): void => {
if (state === GroupCallState.Ended) this.destroy();
};
private onHangup = async (ev: CustomEvent<IWidgetApiRequest>): Promise<void> => {
ev.preventDefault();
await this.messaging!.transport.reply(ev.detail, {}); // ack
@ -873,4 +841,8 @@ export class ElementCall extends Call {
this.layout = Layout.Spotlight;
await this.messaging!.transport.reply(ev.detail, {}); // ack
};
public clean(): Promise<void> {
return Promise.resolve();
}
}

View file

@ -16,6 +16,10 @@ limitations under the License.
import { logger } from "matrix-js-sdk/src/logger";
import { GroupCallEventHandlerEvent } from "matrix-js-sdk/src/webrtc/groupCallEventHandler";
// eslint-disable-next-line no-restricted-imports
import { MatrixRTCSessionManagerEvents } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSessionManager";
// eslint-disable-next-line no-restricted-imports
import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import type { GroupCall, Room } from "matrix-js-sdk/src/matrix";
import defaultDispatcher from "../dispatcher/dispatcher";
@ -61,6 +65,8 @@ export class CallStore extends AsyncStoreWithClient<{}> {
}
this.matrixClient.on(GroupCallEventHandlerEvent.Incoming, this.onGroupCall);
this.matrixClient.on(GroupCallEventHandlerEvent.Outgoing, this.onGroupCall);
this.matrixClient.matrixRTC.on(MatrixRTCSessionManagerEvents.SessionStarted, this.onRTCSession);
this.matrixClient.matrixRTC.on(MatrixRTCSessionManagerEvents.SessionEnded, this.onRTCSession);
WidgetStore.instance.on(UPDATE_EVENT, this.onWidgets);
// If the room ID of a previously connected call is still in settings at
@ -94,6 +100,8 @@ export class CallStore extends AsyncStoreWithClient<{}> {
this.matrixClient.off(GroupCallEventHandlerEvent.Incoming, this.onGroupCall);
this.matrixClient.off(GroupCallEventHandlerEvent.Outgoing, this.onGroupCall);
this.matrixClient.off(GroupCallEventHandlerEvent.Ended, this.onGroupCall);
this.matrixClient.matrixRTC.off(MatrixRTCSessionManagerEvents.SessionStarted, this.onRTCSession);
this.matrixClient.matrixRTC.off(MatrixRTCSessionManagerEvents.SessionEnded, this.onRTCSession);
}
WidgetStore.instance.off(UPDATE_EVENT, this.onWidgets);
}
@ -191,4 +199,7 @@ export class CallStore extends AsyncStoreWithClient<{}> {
};
private onGroupCall = (groupCall: GroupCall): void => this.updateRoom(groupCall.room);
private onRTCSession = (roomId: string, session: MatrixRTCSession): void => {
this.updateRoom(session.room);
};
}

View file

@ -202,6 +202,7 @@ export default class WidgetStore extends AsyncStoreWithClient<IState> {
const app = WidgetUtils.makeAppConfig(widget.id, widget, widget.creatorUserId, roomId, undefined);
this.widgetMap.set(WidgetUtils.getWidgetUid(app), app);
this.roomMap.get(roomId)!.widgets.push(app);
this.emit(UPDATE_EVENT, roomId);
return app;
}

View file

@ -20,6 +20,7 @@ export class WidgetType {
public static readonly STICKERPICKER = new WidgetType("m.stickerpicker", "m.stickerpicker");
public static readonly INTEGRATION_MANAGER = new WidgetType("m.integration_manager", "m.integration_manager");
public static readonly CUSTOM = new WidgetType("m.custom", "m.custom");
public static readonly CALL = new WidgetType("m.call", "m.call");
public constructor(public readonly preferred: string, public readonly legacy: string) {}