Merge pull request #5833 from SimonBrandner/feed

Support for multiple streams (not MSC3077)

res/css/views/voip/_CallView.scss

@@ -65,14 +65,17 @@ limitations under the License.
     }
 }
 
-.mx_CallView_voice {
+.mx_CallView_content {
     position: relative;
     display: flex;
-    flex-direction: column;
+    border-radius: 8px;
+}
+
+.mx_CallView_voice {
     align-items: center;
     justify-content: center;
+    flex-direction: column;
     background-color: $inverted-bg-color;
-    border-radius: 8px;
 }
 
 .mx_CallView_voice_avatarsContainer {
@@ -109,9 +112,7 @@ limitations under the License.
 .mx_CallView_video {
     width: 100%;
     height: 100%;
-    position: relative;
     z-index: 30;
-    border-radius: 8px;
     overflow: hidden;
 }
 

res/css/views/voip/_VideoFeed.scss

@@ -14,21 +14,37 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+.mx_VideoFeed_voice {
+    // We don't want to collide with the call controls that have 52px of height
+    padding-bottom: 52px;
+    background-color: $inverted-bg-color;
+}
+
+
 .mx_VideoFeed_remote {
     width: 100%;
     height: 100%;
-    background-color: #000;
+    display: flex;
-    z-index: 50;
+    justify-content: center;
+    align-items: center;
+
+    &.mx_VideoFeed_video {
+        background-color: #000;
+    }
 }
 
 .mx_VideoFeed_local {
-    width: 25%;
+    max-width: 25%;
-    height: 25%;
+    max-height: 25%;
     position: absolute;
     right: 10px;
     top: 10px;
     z-index: 100;
     border-radius: 4px;
+
+    &.mx_VideoFeed_video {
+        background-color: transparent;
+    }
 }
 
 .mx_VideoFeed_mirror {

src/@types/global.d.ts

@@ -118,6 +118,16 @@ declare global {
 
     interface HTMLAudioElement {
         type?: string;
+        // sinkId & setSinkId are experimental and typescript doesn't know about them
+        sinkId: string;
+        setSinkId(outputId: string);
+    }
+
+    interface HTMLVideoElement {
+        type?: string;
+        // sinkId & setSinkId are experimental and typescript doesn't know about them
+        sinkId: string;
+        setSinkId(outputId: string);
     }
 
     interface Element {

src/CallHandler.tsx

@@ -85,6 +85,7 @@ import { Action } from './dispatcher/actions';
 import VoipUserMapper from './VoipUserMapper';
 import { addManagedHybridWidget, isManagedHybridWidgetEnabled } from './widgets/ManagedHybrid';
 import { randomUppercaseString, randomLowercaseString } from "matrix-js-sdk/src/randomstring";
+import EventEmitter from 'events';
 import SdkConfig from './SdkConfig';
 import { ensureDMExists, findDMForUser } from './createRoom';
 
@@ -138,22 +139,12 @@ export enum PlaceCallType {
     ScreenSharing = 'screensharing',
 }
 
-function getRemoteAudioElement(): HTMLAudioElement {
+export enum CallHandlerEvent {
-    // this needs to be somewhere at the top of the DOM which
+    CallsChanged = "calls_changed",
-    // always exists to avoid audio interruptions.
+    CallChangeRoom = "call_change_room",
-    // Might as well just use DOM.
-    const remoteAudioElement = document.getElementById("remoteAudio") as HTMLAudioElement;
-    if (!remoteAudioElement) {
-        console.error(
-            "Failed to find remoteAudio element - cannot play audio!" +
-            "You need to add an <audio/> to the DOM.",
-        );
-        return null;
-    }
-    return remoteAudioElement;
 }
 
-export default class CallHandler {
+export default class CallHandler extends EventEmitter {
     private calls = new Map<string, MatrixCall>(); // roomId -> call
     // Calls started as an attended transfer, ie. with the intention of transferring another
     // call with a different party to this one.
@@ -514,6 +505,7 @@ export default class CallHandler {
             }
 
             this.calls.set(mappedRoomId, newCall);
+            this.emit(CallHandlerEvent.CallsChanged, this.calls);
             this.setCallListeners(newCall);
             this.setCallState(newCall, newCall.state);
         });
@@ -546,10 +538,7 @@ export default class CallHandler {
                     this.removeCallForRoom(mappedRoomId);
                     mappedRoomId = newMappedRoomId;
                     this.calls.set(mappedRoomId, call);
-                    dis.dispatch({
+                    this.emit(CallHandlerEvent.CallChangeRoom, call);
-                        action: Action.CallChangeRoom,
-                        call,
-                    });
                 }
             }
         });
@@ -598,11 +587,6 @@ export default class CallHandler {
         }
     }
 
-    private setCallAudioElement(call: MatrixCall) {
-        const audioElement = getRemoteAudioElement();
-        if (audioElement) call.setRemoteAudioElement(audioElement);
-    }
-
     private setCallState(call: MatrixCall, status: CallState) {
         const mappedRoomId = CallHandler.sharedInstance().roomIdForCall(call);
 
@@ -619,6 +603,7 @@ export default class CallHandler {
 
     private removeCallForRoom(roomId: string) {
         this.calls.delete(roomId);
+        this.emit(CallHandlerEvent.CallsChanged, this.calls);
     }
 
     private showICEFallbackPrompt() {
@@ -679,11 +664,7 @@ export default class CallHandler {
         }, null, true);
     }
 
-    private async placeCall(
+    private async placeCall(roomId: string, type: PlaceCallType, transferee: MatrixCall) {
-        roomId: string, type: PlaceCallType,
-        localElement: HTMLVideoElement, remoteElement: HTMLVideoElement,
-        transferee: MatrixCall,
-    ) {
         Analytics.trackEvent('voip', 'placeCall', 'type', type);
         CountlyAnalytics.instance.trackStartCall(roomId, type === PlaceCallType.Video, false);
 
@@ -695,22 +676,19 @@ export default class CallHandler {
         const call = MatrixClientPeg.get().createCall(mappedRoomId);
 
         this.calls.set(roomId, call);
+        this.emit(CallHandlerEvent.CallsChanged, this.calls);
         if (transferee) {
             this.transferees[call.callId] = transferee;
         }
 
         this.setCallListeners(call);
-        this.setCallAudioElement(call);
 
         this.setActiveCallRoomId(roomId);
 
         if (type === PlaceCallType.Voice) {
             call.placeVoiceCall();
         } else if (type === 'video') {
-            call.placeVideoCall(
+            call.placeVideoCall();
-                remoteElement,
-                localElement,
-            );
         } else if (type === PlaceCallType.ScreenSharing) {
             const screenCapErrorString = PlatformPeg.get().screenCaptureErrorString();
             if (screenCapErrorString) {
@@ -724,13 +702,12 @@ export default class CallHandler {
             }
 
             call.placeScreenSharingCall(
-                remoteElement,
-                localElement,
                 async (): Promise<DesktopCapturerSource> => {
                     const {finished} = Modal.createDialog(DesktopCapturerSourcePicker);
                     const [source] = await finished;
                     return source;
-                });
+                },
+            );
         } else {
             console.error("Unknown conf call type: " + type);
         }
@@ -787,17 +764,12 @@ export default class CallHandler {
                     } else if (members.length === 2) {
                         console.info(`Place ${payload.type} call in ${payload.room_id}`);
 
-                        this.placeCall(
+                        this.placeCall(payload.room_id, payload.type, payload.transferee);
-                            payload.room_id, payload.type, payload.local_element, payload.remote_element,
-                            payload.transferee,
-                        );
                     } else { // > 2
                         dis.dispatch({
                             action: "place_conference_call",
                             room_id: payload.room_id,
                             type: payload.type,
-                            remote_element: payload.remote_element,
-                            local_element: payload.local_element,
                         });
                     }
                 }
@@ -833,6 +805,7 @@ export default class CallHandler {
 
                     Analytics.trackEvent('voip', 'receiveCall', 'type', call.type);
                     this.calls.set(mappedRoomId, call)
+                    this.emit(CallHandlerEvent.CallsChanged, this.calls);
                     this.setCallListeners(call);
 
                     // get ready to send encrypted events in the room, so if the user does answer
@@ -875,7 +848,6 @@ export default class CallHandler {
 
                 const call = this.calls.get(payload.room_id);
                 call.answer();
-                this.setCallAudioElement(call);
                 this.setActiveCallRoomId(payload.room_id);
                 CountlyAnalytics.instance.trackJoinCall(payload.room_id, call.type === CallType.Video, false);
                 dis.dispatch({

src/CallMediaHandler.js

@@ -16,7 +16,7 @@
 
 import SettingsStore from "./settings/SettingsStore";
 import {SettingLevel} from "./settings/SettingLevel";
-import {setMatrixCallAudioInput, setMatrixCallAudioOutput, setMatrixCallVideoInput} from "matrix-js-sdk/src/matrix";
+import {setMatrixCallAudioInput, setMatrixCallVideoInput} from "matrix-js-sdk/src/matrix";
 
 export default {
     hasAnyLabeledDevices: async function() {
@@ -50,18 +50,15 @@ export default {
     },
 
     loadDevices: function() {
-        const audioOutDeviceId = SettingsStore.getValue("webrtc_audiooutput");
         const audioDeviceId = SettingsStore.getValue("webrtc_audioinput");
         const videoDeviceId = SettingsStore.getValue("webrtc_videoinput");
 
-        setMatrixCallAudioOutput(audioOutDeviceId);
         setMatrixCallAudioInput(audioDeviceId);
         setMatrixCallVideoInput(videoDeviceId);
     },
 
     setAudioOutput: function(deviceId) {
         SettingsStore.setValue("webrtc_audiooutput", null, SettingLevel.DEVICE, deviceId);
-        setMatrixCallAudioOutput(deviceId);
     },
 
     setAudioInput: function(deviceId) {

src/components/structures/LoggedInView.tsx

@@ -59,6 +59,9 @@ import { getKeyBindingsManager, NavigationAction, RoomAction } from '../../KeyBi
 import { IOpts } from "../../createRoom";
 import SpacePanel from "../views/spaces/SpacePanel";
 import {replaceableComponent} from "../../utils/replaceableComponent";
+import CallHandler, { CallHandlerEvent } from '../../CallHandler';
+import { MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
+import AudioFeedArrayForCall from '../views/voip/AudioFeedArrayForCall';
 
 // We need to fetch each pinned message individually (if we don't already have it)
 // so each pinned message may trigger a request. Limit the number per room for sanity.
@@ -119,6 +122,7 @@ interface IState {
     usageLimitEventContent?: IUsageLimit;
     usageLimitEventTs?: number;
     useCompactLayout: boolean;
+    activeCalls: Array<MatrixCall>;
 }
 
 /**
@@ -160,6 +164,7 @@ class LoggedInView extends React.Component<IProps, IState> {
             // use compact timeline view
             useCompactLayout: SettingsStore.getValue('useCompactLayout'),
             usageLimitDismissed: false,
+            activeCalls: [],
         };
 
         // stash the MatrixClient in case we log out before we are unmounted
@@ -175,6 +180,7 @@ class LoggedInView extends React.Component<IProps, IState> {
 
     componentDidMount() {
         document.addEventListener('keydown', this._onNativeKeyDown, false);
+        CallHandler.sharedInstance().addListener(CallHandlerEvent.CallsChanged, this.onCallsChanged);
 
         this._updateServerNoticeEvents();
 
@@ -199,6 +205,7 @@ class LoggedInView extends React.Component<IProps, IState> {
 
     componentWillUnmount() {
         document.removeEventListener('keydown', this._onNativeKeyDown, false);
+        CallHandler.sharedInstance().removeListener(CallHandlerEvent.CallsChanged, this.onCallsChanged);
         this._matrixClient.removeListener("accountData", this.onAccountData);
         this._matrixClient.removeListener("sync", this.onSync);
         this._matrixClient.removeListener("RoomState.events", this.onRoomStateEvents);
@@ -206,6 +213,12 @@ class LoggedInView extends React.Component<IProps, IState> {
         this.resizer.detach();
     }
 
+    private onCallsChanged = () => {
+        this.setState({
+            activeCalls: CallHandler.sharedInstance().getAllActiveCalls(),
+        });
+    };
+
     // Child components assume that the client peg will not be null, so give them some
     // sort of assurance here by only allowing a re-render if the client is truthy.
     //
@@ -661,6 +674,12 @@ class LoggedInView extends React.Component<IProps, IState> {
             bodyClasses += ' mx_MatrixChat_useCompactLayout';
         }
 
+        const audioFeedArraysForCalls = this.state.activeCalls.map((call) => {
+            return (
+                <AudioFeedArrayForCall call={call} key={call.callId} />
+            );
+        });
+
         return (
             <MatrixClientContext.Provider value={this._matrixClient}>
                 <div
@@ -685,6 +704,7 @@ class LoggedInView extends React.Component<IProps, IState> {
                 <CallContainer />
                 <NonUrgentToastContainer />
                 <HostSignupContainer />
+                {audioFeedArraysForCalls}
             </MatrixClientContext.Provider>
         );
     }

src/components/views/voip/AudioFeed.tsx

@@ -0,0 +1,97 @@
+/*
+Copyright 2021 Šimon Brandner <simon.bra.ag@gmail.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React, {createRef} from 'react';
+import { CallFeed, CallFeedEvent } from 'matrix-js-sdk/src/webrtc/callFeed';
+import { logger } from 'matrix-js-sdk/src/logger';
+import CallMediaHandler from "../../../CallMediaHandler";
+
+interface IProps {
+    feed: CallFeed,
+}
+
+export default class AudioFeed extends React.Component<IProps> {
+    private element = createRef<HTMLAudioElement>();
+
+    componentDidMount() {
+        this.props.feed.addListener(CallFeedEvent.NewStream, this.onNewStream);
+        this.playMedia();
+    }
+
+    componentWillUnmount() {
+        this.props.feed.removeListener(CallFeedEvent.NewStream, this.onNewStream);
+        this.stopMedia();
+    }
+
+    private playMedia() {
+        const element = this.element.current;
+        const audioOutput = CallMediaHandler.getAudioOutput();
+
+        if (audioOutput) {
+            try {
+                // This seems quite unreliable in Chrome, although I haven't yet managed to make a jsfiddle where
+                // it fails.
+                // It seems reliable if you set the sink ID after setting the srcObject and then set the sink ID
+                // back to the default after the call is over - Dave
+                element.setSinkId(audioOutput);
+            } catch (e) {
+                console.error("Couldn't set requested audio output device: using default", e);
+                logger.warn("Couldn't set requested audio output device: using default", e);
+            }
+        }
+
+        element.muted = false;
+        element.srcObject = this.props.feed.stream;
+        element.autoplay = true;
+
+        try {
+            // A note on calling methods on media elements:
+            // We used to have queues per media element to serialise all calls on those elements.
+            // The reason given for this was that load() and play() were racing. However, we now
+            // never call load() explicitly so this seems unnecessary. However, serialising every
+            // operation was causing bugs where video would not resume because some play command
+            // had got stuck and all media operations were queued up behind it. If necessary, we
+            // should serialise the ones that need to be serialised but then be able to interrupt
+            // them with another load() which will cancel the pending one, but since we don't call
+            // load() explicitly, it shouldn't be a problem. - Dave
+            element.play()
+        } catch (e) {
+            logger.info("Failed to play media element with feed", this.props.feed, e);
+        }
+    }
+
+    private stopMedia() {
+        const element = this.element.current;
+
+        element.pause();
+        element.src = null;
+
+        // As per comment in componentDidMount, setting the sink ID back to the
+        // default once the call is over makes setSinkId work reliably. - Dave
+        // Since we are not using the same element anymore, the above doesn't
+        // seem to be necessary - Šimon
+    }
+
+    private onNewStream = () => {
+        this.playMedia();
+    };
+
+    render() {
+        return (
+            <audio ref={this.element} />
+        );
+    }
+}

src/components/views/voip/AudioFeedArrayForCall.tsx

@@ -0,0 +1,60 @@
+/*
+Copyright 2021 Šimon Brandner <simon.bra.ag@gmail.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from "react";
+import AudioFeed from "./AudioFeed"
+import { CallEvent, MatrixCall } from "matrix-js-sdk/src/webrtc/call";
+import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed";
+
+interface IProps {
+    call: MatrixCall;
+}
+
+interface IState {
+    feeds: Array<CallFeed>;
+}
+
+export default class AudioFeedArrayForCall extends React.Component<IProps, IState> {
+    constructor(props: IProps) {
+        super(props);
+
+        this.state = {
+            feeds: [],
+        };
+    }
+
+    componentDidMount() {
+        this.props.call.addListener(CallEvent.FeedsChanged, this.onFeedsChanged);
+    }
+
+    componentWillUnmount() {
+        this.props.call.removeListener(CallEvent.FeedsChanged, this.onFeedsChanged);
+    }
+
+    onFeedsChanged = () => {
+        this.setState({
+            feeds: this.props.call.getRemoteFeeds(),
+        });
+    }
+
+    render() {
+        return this.state.feeds.map((feed, i) => {
+            return (
+                <AudioFeed feed={feed} key={i} />
+            );
+        });
+    }
+}

src/components/views/voip/CallPreview.tsx

@@ -19,7 +19,7 @@ import React from 'react';
 
 import CallView from "./CallView";
 import RoomViewStore from '../../../stores/RoomViewStore';
-import CallHandler from '../../../CallHandler';
+import CallHandler, { CallHandlerEvent } from '../../../CallHandler';
 import dis from '../../../dispatcher/dispatcher';
 import { ActionPayload } from '../../../dispatcher/payloads';
 import PersistentApp from "../elements/PersistentApp";
@@ -27,7 +27,6 @@ import SettingsStore from "../../../settings/SettingsStore";
 import { CallEvent, CallState, MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
 import { MatrixClientPeg } from '../../../MatrixClientPeg';
 import {replaceableComponent} from "../../../utils/replaceableComponent";
-import { Action } from '../../../dispatcher/actions';
 
 const SHOW_CALL_IN_STATES = [
     CallState.Connected,
@@ -110,12 +109,14 @@ export default class CallPreview extends React.Component<IProps, IState> {
     }
 
     public componentDidMount() {
+        CallHandler.sharedInstance().addListener(CallHandlerEvent.CallChangeRoom, this.updateCalls);
         this.roomStoreToken = RoomViewStore.addListener(this.onRoomViewStoreUpdate);
         this.dispatcherRef = dis.register(this.onAction);
         MatrixClientPeg.get().on(CallEvent.RemoteHoldUnhold, this.onCallRemoteHold);
     }
 
     public componentWillUnmount() {
+        CallHandler.sharedInstance().removeListener(CallHandlerEvent.CallChangeRoom, this.updateCalls);
         MatrixClientPeg.get().removeListener(CallEvent.RemoteHoldUnhold, this.onCallRemoteHold);
         if (this.roomStoreToken) {
             this.roomStoreToken.remove();
@@ -143,21 +144,24 @@ export default class CallPreview extends React.Component<IProps, IState> {
         switch (payload.action) {
             // listen for call state changes to prod the render method, which
             // may hide the global CallView if the call it is tracking is dead
-            case Action.CallChangeRoom:
             case 'call_state': {
-                const [primaryCall, secondaryCalls] = getPrimarySecondaryCalls(
+                this.updateCalls();
-                    CallHandler.sharedInstance().getAllActiveCallsNotInRoom(this.state.roomId),
-                );
-
-                this.setState({
-                    primaryCall: primaryCall,
-                    secondaryCall: secondaryCalls[0],
-                });
                 break;
             }
         }
     };
 
+    private updateCalls = () => {
+        const [primaryCall, secondaryCalls] = getPrimarySecondaryCalls(
+            CallHandler.sharedInstance().getAllActiveCallsNotInRoom(this.state.roomId),
+        );
+
+        this.setState({
+            primaryCall: primaryCall,
+            secondaryCall: secondaryCalls[0],
+        });
+    };
+
     private onCallRemoteHold = () => {
         const [primaryCall, secondaryCalls] = getPrimarySecondaryCalls(
             CallHandler.sharedInstance().getAllActiveCallsNotInRoom(this.state.roomId),

src/components/views/voip/CallView.tsx

@@ -20,10 +20,9 @@ import dis from '../../../dispatcher/dispatcher';
 import CallHandler from '../../../CallHandler';
 import {MatrixClientPeg} from '../../../MatrixClientPeg';
 import { _t, _td } from '../../../languageHandler';
-import VideoFeed, { VideoFeedType } from "./VideoFeed";
+import VideoFeed from './VideoFeed';
 import RoomAvatar from "../avatars/RoomAvatar";
-import { CallState, CallType, MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
+import { CallState, CallType, MatrixCall, CallEvent } from 'matrix-js-sdk/src/webrtc/call';
-import { CallEvent } from 'matrix-js-sdk/src/webrtc/call';
 import classNames from 'classnames';
 import AccessibleButton from '../elements/AccessibleButton';
 import {isOnlyCtrlOrCmdKeyEvent, Key} from '../../../Keyboard';
@@ -31,6 +30,7 @@ import {alwaysAboveLeftOf, alwaysAboveRightOf, ChevronFace, ContextMenuButton} f
 import CallContextMenu from '../context_menus/CallContextMenu';
 import { avatarUrlForMember } from '../../../Avatar';
 import DialpadContextMenu from '../context_menus/DialpadContextMenu';
+import { CallFeed } from 'matrix-js-sdk/src/webrtc/callFeed';
 import {replaceableComponent} from "../../../utils/replaceableComponent";
 
 interface IProps {
@@ -40,11 +40,11 @@ interface IProps {
         // Another ongoing call to display information about
         secondaryCall?: MatrixCall,
 
-        // a callback which is called when the content in the callview changes
+        // a callback which is called when the content in the CallView changes
         // in a way that is likely to cause a resize.
         onResize?: any;
 
-        // Whether this call view is for picture-in-pictue mode
+        // Whether this call view is for picture-in-picture mode
         // otherwise, it's the larger call view when viewing the room the call is in.
         // This is sort of a proxy for a number of things but we currently have no
         // need to control those things separately, so this is simpler.
@@ -60,6 +60,7 @@ interface IState {
     controlsVisible: boolean,
     showMoreMenu: boolean,
     showDialpad: boolean,
+    feeds: CallFeed[],
 }
 
 function getFullScreenElement() {
@@ -115,6 +116,7 @@ export default class CallView extends React.Component<IProps, IState> {
             controlsVisible: true,
             showMoreMenu: false,
             showDialpad: false,
+            feeds: this.props.call.getFeeds(),
         }
 
         this.updateCallListeners(null, this.props.call);
@@ -172,11 +174,13 @@ export default class CallView extends React.Component<IProps, IState> {
             oldCall.removeListener(CallEvent.State, this.onCallState);
             oldCall.removeListener(CallEvent.LocalHoldUnhold, this.onCallLocalHoldUnhold);
             oldCall.removeListener(CallEvent.RemoteHoldUnhold, this.onCallRemoteHoldUnhold);
+            oldCall.removeListener(CallEvent.FeedsChanged, this.onFeedsChanged);
         }
         if (newCall) {
             newCall.on(CallEvent.State, this.onCallState);
             newCall.on(CallEvent.LocalHoldUnhold, this.onCallLocalHoldUnhold);
             newCall.on(CallEvent.RemoteHoldUnhold, this.onCallRemoteHoldUnhold);
+            newCall.on(CallEvent.FeedsChanged, this.onFeedsChanged);
         }
     }
 
@@ -186,6 +190,10 @@ export default class CallView extends React.Component<IProps, IState> {
         });
     };
 
+    private onFeedsChanged = (newFeeds: Array<CallFeed>) => {
+        this.setState({feeds: newFeeds});
+    };
+
     private onCallLocalHoldUnhold = () => {
         this.setState({
             isLocalOnHold: this.props.call.isLocalOnHold(),
@@ -304,7 +312,7 @@ export default class CallView extends React.Component<IProps, IState> {
     }
 
     // we register global shortcuts here, they *must not conflict* with local shortcuts elsewhere or both will fire
-    // Note that this assumes we always have a callview on screen at any given time
+    // Note that this assumes we always have a CallView on screen at any given time
     // CallHandler would probably be a better place for this
     private onNativeKeyDown = ev => {
         let handled = false;
@@ -474,6 +482,8 @@ export default class CallView extends React.Component<IProps, IState> {
             {contextMenuButton}
         </div>;
 
+        const avatarSize = this.props.pipMode ? 76 : 160;
+
         // The 'content' for the call, ie. the videos for a video call and profile picture
         // for voice calls (fills the bg)
         let contentView: React.ReactNode;
@@ -524,41 +534,85 @@ export default class CallView extends React.Component<IProps, IState> {
             </div>;
         }
 
-        if (this.props.call.type === CallType.Video) {
+        // This is a bit messy. I can't see a reason to have two onHold/transfer screens
-            let localVideoFeed = null;
+        if (isOnHold || transfereeCall) {
-            let onHoldBackground = null;
+            if (this.props.call.type === CallType.Video) {
-            const backgroundStyle: CSSProperties = {};
+                const containerClasses = classNames({
-            const containerClasses = classNames({
+                    mx_CallView_content: true,
-                mx_CallView_video: true,
+                    mx_CallView_video: true,
-                mx_CallView_video_hold: isOnHold,
+                    mx_CallView_video_hold: isOnHold,
-            });
+                });
-            if (isOnHold) {
+                let onHoldBackground = null;
+                const backgroundStyle: CSSProperties = {};
                 const backgroundAvatarUrl = avatarUrlForMember(
-                    // is it worth getting the size of the div to pass here?
+                // is it worth getting the size of the div to pass here?
                     this.props.call.getOpponentMember(), 1024, 1024, 'crop',
                 );
                 backgroundStyle.backgroundImage = 'url(' + backgroundAvatarUrl + ')';
                 onHoldBackground = <div className="mx_CallView_video_holdBackground" style={backgroundStyle} />;
-            }
-            if (!this.state.vidMuted) {
-                localVideoFeed = <VideoFeed type={VideoFeedType.Local} call={this.props.call} />;
-            }
 
-            contentView = <div className={containerClasses} ref={this.contentRef} onMouseMove={this.onMouseMove}>
+                contentView = (
-                {onHoldBackground}
+                    <div className={containerClasses} ref={this.contentRef} onMouseMove={this.onMouseMove}>
-                <VideoFeed type={VideoFeedType.Remote} call={this.props.call} onResize={this.props.onResize} />
+                        {onHoldBackground}
-                {localVideoFeed}
+                        {holdTransferContent}
-                {holdTransferContent}
+                        {callControls}
-                {callControls}
+                    </div>
-            </div>;
+                );
-        } else {
+            } else {
-            const avatarSize = this.props.pipMode ? 76 : 160;
+                const classes = classNames({
+                    mx_CallView_content: true,
+                    mx_CallView_voice: true,
+                    mx_CallView_voice_hold: isOnHold,
+                });
+
+                contentView =(
+                    <div className={classes} onMouseMove={this.onMouseMove}>
+                        <div className="mx_CallView_voice_avatarsContainer">
+                            <div
+                                className="mx_CallView_voice_avatarContainer"
+                                style={{width: avatarSize, height: avatarSize}}
+                            >
+                                <RoomAvatar
+                                    room={callRoom}
+                                    height={avatarSize}
+                                    width={avatarSize}
+                                />
+                            </div>
+                        </div>
+                        {holdTransferContent}
+                        {callControls}
+                    </div>
+                );
+            }
+        } else if (this.props.call.noIncomingFeeds()) {
+            // Here we're reusing the css classes from voice on hold, because
+            // I am lazy. If this gets merged, the CallView might be subject
+            // to change anyway - I might take an axe to this file in order to
+            // try to get other things working
             const classes = classNames({
+                mx_CallView_content: true,
                 mx_CallView_voice: true,
-                mx_CallView_voice_hold: isOnHold,
             });
 
+            const feeds = this.props.call.getLocalFeeds().map((feed, i) => {
+                // Here we check to hide local audio feeds to achieve the same UI/UX
+                // as before. But once again this might be subject to change
+                if (feed.isVideoMuted()) return;
+                return (
+                    <VideoFeed
+                        key={i}
+                        feed={feed}
+                        call={this.props.call}
+                        pipMode={this.props.pipMode}
+                        onResize={this.props.onResize}
+                    />
+                );
+            });
+
+            // Saying "Connecting" here isn't really true, but the best thing
+            // I can come up with, but this might be subject to change as well
             contentView = <div className={classes} onMouseMove={this.onMouseMove}>
+                {feeds}
                 <div className="mx_CallView_voice_avatarsContainer">
                     <div className="mx_CallView_voice_avatarContainer" style={{width: avatarSize, height: avatarSize}}>
                         <RoomAvatar
@@ -568,7 +622,35 @@ export default class CallView extends React.Component<IProps, IState> {
                         />
                     </div>
                 </div>
-                {holdTransferContent}
+                <div className="mx_CallView_holdTransferContent">{_t("Connecting")}</div>
+                {callControls}
+            </div>;
+        } else {
+            const containerClasses = classNames({
+                mx_CallView_content: true,
+                mx_CallView_video: true,
+            });
+
+            // TODO: Later the CallView should probably be reworked to support
+            // any number of feeds but now we can always expect there to be two
+            // feeds. This is because the js-sdk ignores any new incoming streams
+            const feeds = this.state.feeds.map((feed, i) => {
+                // Here we check to hide local audio feeds to achieve the same UI/UX
+                // as before. But once again this might be subject to change
+                if (feed.isVideoMuted() && feed.isLocal()) return;
+                return (
+                    <VideoFeed
+                        key={i}
+                        feed={feed}
+                        call={this.props.call}
+                        pipMode={this.props.pipMode}
+                        onResize={this.props.onResize}
+                    />
+                );
+            });
+
+            contentView = <div className={containerClasses} ref={this.contentRef} onMouseMove={this.onMouseMove}>
+                {feeds}
                 {callControls}
             </div>;
         }

src/components/views/voip/CallViewForRoom.tsx

@@ -16,13 +16,12 @@ limitations under the License.
 
 import { CallState, MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
 import React from 'react';
-import CallHandler from '../../../CallHandler';
+import CallHandler, { CallHandlerEvent } from '../../../CallHandler';
 import CallView from './CallView';
 import dis from '../../../dispatcher/dispatcher';
 import {Resizable} from "re-resizable";
 import ResizeNotifier from "../../../utils/ResizeNotifier";
 import {replaceableComponent} from "../../../utils/replaceableComponent";
-import { Action } from '../../../dispatcher/actions';
 
 interface IProps {
     // What room we should display the call for
@@ -55,25 +54,30 @@ export default class CallViewForRoom extends React.Component<IProps, IState> {
 
     public componentDidMount() {
         this.dispatcherRef = dis.register(this.onAction);
+        CallHandler.sharedInstance().addListener(CallHandlerEvent.CallChangeRoom, this.updateCall);
     }
 
     public componentWillUnmount() {
         dis.unregister(this.dispatcherRef);
+        CallHandler.sharedInstance().removeListener(CallHandlerEvent.CallChangeRoom, this.updateCall);
     }
 
     private onAction = (payload) => {
         switch (payload.action) {
-            case Action.CallChangeRoom:
             case 'call_state': {
-                const newCall = this.getCall();
+                this.updateCall();
-                if (newCall !== this.state.call) {
-                    this.setState({call: newCall});
-                }
                 break;
             }
         }
     };
 
+    private updateCall = () => {
+        const newCall = this.getCall();
+        if (newCall !== this.state.call) {
+            this.setState({call: newCall});
+        }
+    };
+
     private getCall(): MatrixCall {
         const call = CallHandler.sharedInstance().getCallForRoom(this.props.roomId);
 

src/components/views/voip/VideoFeed.tsx

@@ -18,52 +18,102 @@ import classnames from 'classnames';
 import { MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
 import React, {createRef} from 'react';
 import SettingsStore from "../../../settings/SettingsStore";
+import { CallFeed, CallFeedEvent } from 'matrix-js-sdk/src/webrtc/callFeed';
+import { logger } from 'matrix-js-sdk/src/logger';
+import MemberAvatar from "../avatars/MemberAvatar"
 import {replaceableComponent} from "../../../utils/replaceableComponent";
 
-export enum VideoFeedType {
-    Local,
-    Remote,
-}
-
 interface IProps {
     call: MatrixCall,
 
-    type: VideoFeedType,
+    feed: CallFeed,
+
+    // Whether this call view is for picture-in-picture mode
+    // otherwise, it's the larger call view when viewing the room the call is in.
+    // This is sort of a proxy for a number of things but we currently have no
+    // need to control those things separately, so this is simpler.
+    pipMode?: boolean;
 
     // a callback which is called when the video element is resized
     // due to a change in video metadata
     onResize?: (e: Event) => void,
 }
 
-@replaceableComponent("views.voip.VideoFeed")
+interface IState {
-export default class VideoFeed extends React.Component<IProps> {
+    audioMuted: boolean;
-    private vid = createRef<HTMLVideoElement>();
+    videoMuted: boolean;
+}
 
-    componentDidMount() {
+@replaceableComponent("views.voip.VideoFeed")
-        this.vid.current.addEventListener('resize', this.onResize);
+export default class VideoFeed extends React.Component<IProps, IState> {
-        this.setVideoElement();
+    private element = createRef<HTMLVideoElement>();
+
+    constructor(props: IProps) {
+        super(props);
+
+        this.state = {
+            audioMuted: this.props.feed.isAudioMuted(),
+            videoMuted: this.props.feed.isVideoMuted(),
+        };
     }
 
-    componentDidUpdate(prevProps) {
+    componentDidMount() {
-        if (this.props.call !== prevProps.call) {
+        this.props.feed.addListener(CallFeedEvent.NewStream, this.onNewStream);
-            this.setVideoElement();
+        this.playMedia();
-        }
     }
 
     componentWillUnmount() {
-        this.vid.current.removeEventListener('resize', this.onResize);
+        this.props.feed.removeListener(CallFeedEvent.NewStream, this.onNewStream);
+        this.element.current?.removeEventListener('resize', this.onResize);
+        this.stopMedia();
     }
 
-    private setVideoElement() {
+    private playMedia() {
-        if (this.props.type === VideoFeedType.Local) {
+        const element = this.element.current;
-            this.props.call.setLocalVideoElement(this.vid.current);
+        if (!element) return;
-        } else {
+        // We play audio in AudioFeed, not here
-            this.props.call.setRemoteVideoElement(this.vid.current);
+        element.muted = true;
+        element.srcObject = this.props.feed.stream;
+        element.autoplay = true;
+        try {
+            // A note on calling methods on media elements:
+            // We used to have queues per media element to serialise all calls on those elements.
+            // The reason given for this was that load() and play() were racing. However, we now
+            // never call load() explicitly so this seems unnecessary. However, serialising every
+            // operation was causing bugs where video would not resume because some play command
+            // had got stuck and all media operations were queued up behind it. If necessary, we
+            // should serialise the ones that need to be serialised but then be able to interrupt
+            // them with another load() which will cancel the pending one, but since we don't call
+            // load() explicitly, it shouldn't be a problem. - Dave
+            element.play()
+        } catch (e) {
+            logger.info("Failed to play media element with feed", this.props.feed, e);
         }
     }
 
-    onResize = (e) => {
+    private stopMedia() {
-        if (this.props.onResize) {
+        const element = this.element.current;
+        if (!element) return;
+
+        element.pause();
+        element.src = null;
+
+        // As per comment in componentDidMount, setting the sink ID back to the
+        // default once the call is over makes setSinkId work reliably. - Dave
+        // Since we are not using the same element anymore, the above doesn't
+        // seem to be necessary - Šimon
+    }
+
+    private onNewStream = () => {
+        this.setState({
+            audioMuted: this.props.feed.isAudioMuted(),
+            videoMuted: this.props.feed.isVideoMuted(),
+        });
+        this.playMedia();
+    };
+
+    private onResize = (e) => {
+        if (this.props.onResize && !this.props.feed.isLocal()) {
             this.props.onResize(e);
         }
     };
@@ -71,14 +121,33 @@ export default class VideoFeed extends React.Component<IProps> {
     render() {
         const videoClasses = {
             mx_VideoFeed: true,
-            mx_VideoFeed_local: this.props.type === VideoFeedType.Local,
+            mx_VideoFeed_local: this.props.feed.isLocal(),
-            mx_VideoFeed_remote: this.props.type === VideoFeedType.Remote,
+            mx_VideoFeed_remote: !this.props.feed.isLocal(),
+            mx_VideoFeed_voice: this.state.videoMuted,
+            mx_VideoFeed_video: !this.state.videoMuted,
             mx_VideoFeed_mirror: (
-                this.props.type === VideoFeedType.Local &&
+                this.props.feed.isLocal() &&
                 SettingsStore.getValue('VideoView.flipVideoHorizontally')
             ),
         };
 
-        return <video className={classnames(videoClasses)} ref={this.vid} />;
+        if (this.state.videoMuted) {
+            const member = this.props.feed.getMember();
+            const avatarSize = this.props.pipMode ? 76 : 160;
+
+            return (
+                <div className={classnames(videoClasses)} >
+                    <MemberAvatar
+                        member={member}
+                        height={avatarSize}
+                        width={avatarSize}
+                    />
+                </div>
+            );
+        } else {
+            return (
+                <video className={classnames(videoClasses)} ref={this.element} />
+            );
+        }
     }
 }

src/dispatcher/actions.ts

@@ -114,9 +114,6 @@ export enum Action {
      */
     VirtualRoomSupportUpdated = "virtual_room_support_updated",
 
-    // Probably would be better to have a VoIP states in a store and have the store emit changes
-    CallChangeRoom = "call_change_room",
-
     /**
      * Fired when an upload has started. Should be used with UploadStartedPayload.
      */

src/i18n/strings/en_EN.json

@@ -885,6 +885,7 @@
     "You held the call <a>Switch</a>": "You held the call <a>Switch</a>",
     "You held the call <a>Resume</a>": "You held the call <a>Resume</a>",
     "%(peerName)s held the call": "%(peerName)s held the call",
+    "Connecting": "Connecting",
     "Video Call": "Video Call",
     "Voice Call": "Voice Call",
     "Fill Screen": "Fill Screen",

test/CallHandler-test.ts

@@ -16,7 +16,7 @@ limitations under the License.
 
 import './skinned-sdk';
 
-import CallHandler, { PlaceCallType } from '../src/CallHandler';
+import CallHandler, { PlaceCallType, CallHandlerEvent } from '../src/CallHandler';
 import { stubClient, mkStubRoom } from './test-utils';
 import { MatrixClientPeg } from '../src/MatrixClientPeg';
 import dis from '../src/dispatcher/dispatcher';
@@ -172,11 +172,9 @@ describe('CallHandler', () => {
 
         let callRoomChangeEventCount = 0;
         const roomChangePromise = new Promise<void>(resolve => {
-            dispatchHandle = dis.register(payload => {
+            callHandler.addListener(CallHandlerEvent.CallChangeRoom, () => {
-                if (payload.action === Action.CallChangeRoom) {
+                ++callRoomChangeEventCount;
-                    ++callRoomChangeEventCount;
+                resolve();
-                    resolve();
-                }
             });
         });
 
@@ -201,7 +199,7 @@ describe('CallHandler', () => {
         fakeCall.emit(CallEvent.AssertedIdentityChanged);
 
         await roomChangePromise;
-        dis.unregister(dispatchHandle);
+        callHandler.removeAllListeners();
 
         // If everything's gone well, we should have seen only one room change
         // event and the call should now be in user 3's room.