Merge pull request #5992 from SimonBrandner/fix/12652/screen-share

Add support for screen sharing in 1:1 calls

src/components/views/elements/DesktopCapturerSourcePicker.tsx

@@ -17,9 +17,12 @@ limitations under the License.
 import React from 'react';
 import { _t } from '../../../languageHandler';
 import BaseDialog from "..//dialogs/BaseDialog";
+import DialogButtons from "./DialogButtons";
+import classNames from 'classnames';
 import AccessibleButton from './AccessibleButton';
 import { getDesktopCapturerSources } from "matrix-js-sdk/src/webrtc/call";
 import { replaceableComponent } from "../../../utils/replaceableComponent";
+import TabbedView, { Tab, TabLocation } from '../../structures/TabbedView';
 
 export interface DesktopCapturerSource {
     id: string;
@@ -28,62 +31,70 @@ export interface DesktopCapturerSource {
 }
 
 export enum Tabs {
-    Screens = "screens",
-    Windows = "windows",
+    Screens = "screen",
+    Windows = "window",
 }
 
-export interface DesktopCapturerSourceIProps {
+export interface ExistingSourceIProps {
     source: DesktopCapturerSource;
     onSelect(source: DesktopCapturerSource): void;
+    selected: boolean;
 }
 
-export class ExistingSource extends React.Component<DesktopCapturerSourceIProps> {
-    constructor(props) {
+export class ExistingSource extends React.Component<ExistingSourceIProps> {
+    constructor(props: ExistingSourceIProps) {
         super(props);
     }
 
-    onClick = (ev) => {
+    private onClick = (): void => {
         this.props.onSelect(this.props.source);
     };
 
     render() {
+        const thumbnailClasses = classNames({
+            mx_desktopCapturerSourcePicker_source_thumbnail: true,
+            mx_desktopCapturerSourcePicker_source_thumbnail_selected: this.props.selected,
+        });
+
         return (
             <AccessibleButton
-                className="mx_desktopCapturerSourcePicker_stream_button"
+                className="mx_desktopCapturerSourcePicker_source"
                 title={this.props.source.name}
                 onClick={this.onClick}
             >
                 <img
-                    className="mx_desktopCapturerSourcePicker_stream_thumbnail"
+                    className={thumbnailClasses}
                     src={this.props.source.thumbnailURL}
                 />
-                <span className="mx_desktopCapturerSourcePicker_stream_name">{ this.props.source.name }</span>
+                <span className="mx_desktopCapturerSourcePicker_source_name">{ this.props.source.name }</span>
             </AccessibleButton>
         );
     }
 }
 
-export interface DesktopCapturerSourcePickerIState {
+export interface PickerIState {
     selectedTab: Tabs;
     sources: Array<DesktopCapturerSource>;
+    selectedSource: DesktopCapturerSource | null;
 }
-export interface DesktopCapturerSourcePickerIProps {
+export interface PickerIProps {
     onFinished(source: DesktopCapturerSource): void;
 }
 
 @replaceableComponent("views.elements.DesktopCapturerSourcePicker")
 export default class DesktopCapturerSourcePicker extends React.Component<
-    DesktopCapturerSourcePickerIProps,
-    DesktopCapturerSourcePickerIState
-    > {
-    interval;
+    PickerIProps,
+    PickerIState
+> {
+    interval: number;
 
-    constructor(props) {
+    constructor(props: PickerIProps) {
         super(props);
 
         this.state = {
             selectedTab: Tabs.Screens,
             sources: [],
+            selectedSource: null,
         };
     }
 
@@ -107,69 +118,61 @@ export default class DesktopCapturerSourcePicker extends React.Component<
         clearInterval(this.interval);
     }
 
-    onSelect = (source) => {
-        this.props.onFinished(source);
+    private onSelect = (source: DesktopCapturerSource): void => {
+        this.setState({ selectedSource: source });
     };
 
-    onScreensClick = (ev) => {
-        this.setState({ selectedTab: Tabs.Screens });
+    private onShare = (): void => {
+        this.props.onFinished(this.state.selectedSource);
     };
 
-    onWindowsClick = (ev) => {
-        this.setState({ selectedTab: Tabs.Windows });
+    private onTabChange = (): void => {
+        this.setState({ selectedSource: null });
     };
 
-    onCloseClick = (ev) => {
+    private onCloseClick = (): void => {
         this.props.onFinished(null);
     };
 
-    render() {
-        let sources;
-        if (this.state.selectedTab === Tabs.Screens) {
-            sources = this.state.sources
-                .filter((source) => {
-                    return source.id.startsWith("screen");
-                })
-                .map((source) => {
-                    return <ExistingSource source={source} onSelect={this.onSelect} key={source.id} />;
-                });
-        } else {
-            sources = this.state.sources
-                .filter((source) => {
-                    return source.id.startsWith("window");
-                })
-                .map((source) => {
-                    return <ExistingSource source={source} onSelect={this.onSelect} key={source.id} />;
-                });
-        }
+    private getTab(type: "screen" | "window", label: string): Tab {
+        const sources = this.state.sources.filter((source) => source.id.startsWith(type)).map((source) => {
+            return (
+                <ExistingSource
+                    selected={this.state.selectedSource?.id === source.id}
+                    source={source}
+                    onSelect={this.onSelect}
+                    key={source.id}
+                />
+            );
+        });
 
-        const buttonStyle = "mx_desktopCapturerSourcePicker_tabLabel";
-        const screensButtonStyle = buttonStyle + ((this.state.selectedTab === Tabs.Screens) ? "_selected" : "");
-        const windowsButtonStyle = buttonStyle + ((this.state.selectedTab === Tabs.Windows) ? "_selected" : "");
+        return new Tab(type, label, null, (
+            <div className="mx_desktopCapturerSourcePicker_tab">
+                { sources }
+            </div>
+        ));
+    }
+
+    render() {
+        const tabs = [
+            this.getTab("screen", _t("Share entire screen")),
+            this.getTab("window", _t("Application window")),
+        ];
 
         return (
             <BaseDialog
                 className="mx_desktopCapturerSourcePicker"
                 onFinished={this.onCloseClick}
-                title={_t("Share your screen")}
+                title={_t("Share content")}
             >
-                <div className="mx_desktopCapturerSourcePicker_tabLabels">
-                    <AccessibleButton
-                        className={screensButtonStyle}
-                        onClick={this.onScreensClick}
-                    >
-                        { _t("Screens") }
-                    </AccessibleButton>
-                    <AccessibleButton
-                        className={windowsButtonStyle}
-                        onClick={this.onWindowsClick}
-                    >
-                        { _t("Windows") }
-                    </AccessibleButton>
-                </div>
-                <div className="mx_desktopCapturerSourcePicker_panel">
-                    { sources }
-                </div>
+                <TabbedView tabs={tabs} tabLocation={TabLocation.TOP} onChange={this.onTabChange} />
+                <DialogButtons
+                    primaryButton={_t("Share")}
+                    hasCancel={true}
+                    onCancel={this.onCloseClick}
+                    onPrimaryButtonClick={this.onShare}
+                    primaryDisabled={!this.state.selectedSource}
+                />
             </BaseDialog>
         );
     }

src/components/views/rooms/RoomHeader.tsx

@@ -29,6 +29,8 @@ import RoomTopic from "../elements/RoomTopic";
 import RoomName from "../elements/RoomName";
 import { PlaceCallType } from "../../../CallHandler";
 import { replaceableComponent } from "../../../utils/replaceableComponent";
+import Modal from '../../../Modal';
+import InfoDialog from "../dialogs/InfoDialog";
 import { throttle } from 'lodash';
 import { MatrixEvent, Room, RoomState } from 'matrix-js-sdk/src';
 import { E2EStatus } from '../../../utils/ShieldUtils';
@@ -87,6 +89,14 @@ export default class RoomHeader extends React.Component<IProps> {
         this.forceUpdate();
     }, 500, { leading: true, trailing: true });
 
+    private displayInfoDialogAboutScreensharing() {
+        Modal.createDialog(InfoDialog, {
+            title: _t("Screen sharing is here!"),
+            description: _t("You can now share your screen by pressing the \"screen share\" " +
+            "button during a call. You can even do this in audio calls if both sides support it!"),
+        });
+    }
+
     public render() {
         let searchStatus = null;
 
@@ -185,8 +195,8 @@ export default class RoomHeader extends React.Component<IProps> {
             videoCallButton =
                 <AccessibleTooltipButton
                     className="mx_RoomHeader_button mx_RoomHeader_videoCallButton"
-                    onClick={(ev) => this.props.onCallPlaced(
-                        ev.shiftKey ? PlaceCallType.ScreenSharing : PlaceCallType.Video)}
+                    onClick={(ev) => ev.shiftKey ?
+                        this.displayInfoDialogAboutScreensharing() : this.props.onCallPlaced(PlaceCallType.Video)}
                     title={_t("Video call")} />;
         }
 

src/components/views/voip/AudioFeed.tsx

@@ -23,9 +23,21 @@ interface IProps {
     feed: CallFeed;
 }
 
-export default class AudioFeed extends React.Component<IProps> {
+interface IState {
+    audioMuted: boolean;
+}
+
+export default class AudioFeed extends React.Component<IProps, IState> {
     private element = createRef<HTMLAudioElement>();
 
+    constructor(props: IProps) {
+        super(props);
+
+        this.state = {
+            audioMuted: this.props.feed.isAudioMuted(),
+        };
+    }
+
     componentDidMount() {
         MediaDeviceHandler.instance.addListener(
             MediaDeviceHandlerEvent.AudioOutputChanged,
@@ -62,6 +74,7 @@ export default class AudioFeed extends React.Component<IProps> {
 
     private playMedia() {
         const element = this.element.current;
+        if (!element) return;
         this.onAudioOutputChanged(MediaDeviceHandler.getAudioOutput());
         element.muted = false;
         element.srcObject = this.props.feed.stream;
@@ -85,6 +98,7 @@ export default class AudioFeed extends React.Component<IProps> {
 
     private stopMedia() {
         const element = this.element.current;
+        if (!element) return;
 
         element.pause();
         element.src = null;
@@ -96,10 +110,16 @@ export default class AudioFeed extends React.Component<IProps> {
     }
 
     private onNewStream = () => {
+        this.setState({
+            audioMuted: this.props.feed.isAudioMuted(),
+        });
         this.playMedia();
     };
 
     render() {
+        // Do not render the audio element if there is no audio track
+        if (this.state.audioMuted) return null;
+
         return (
             <audio ref={this.element} />
         );

src/components/views/voip/CallView.tsx

@@ -1,6 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
 Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
+Copyright 2021 Šimon Brandner <simon.bra.ag@gmail.com>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -32,6 +33,10 @@ import { avatarUrlForMember } from '../../../Avatar';
 import DialpadContextMenu from '../context_menus/DialpadContextMenu';
 import { CallFeed } from 'matrix-js-sdk/src/webrtc/callFeed';
 import { replaceableComponent } from "../../../utils/replaceableComponent";
+import DesktopCapturerSourcePicker from "../elements/DesktopCapturerSourcePicker";
+import Modal from '../../../Modal';
+import { SDPStreamMetadataPurpose } from 'matrix-js-sdk/src/webrtc/callEventTypes';
+import CallViewSidebar from './CallViewSidebar';
 
 interface IProps {
         // The call for us to display
@@ -59,11 +64,14 @@ interface IState {
     isRemoteOnHold: boolean;
     micMuted: boolean;
     vidMuted: boolean;
+    screensharing: boolean;
     callState: CallState;
     controlsVisible: boolean;
     showMoreMenu: boolean;
     showDialpad: boolean;
-    feeds: CallFeed[];
+    primaryFeed: CallFeed;
+    secondaryFeeds: Array<CallFeed>;
+    sidebarShown: boolean;
 }
 
 function getFullScreenElement() {
@@ -110,16 +118,21 @@ export default class CallView extends React.Component<IProps, IState> {
     constructor(props: IProps) {
         super(props);
 
+        const { primary, secondary } = this.getOrderedFeeds(this.props.call.getFeeds());
+
         this.state = {
             isLocalOnHold: this.props.call.isLocalOnHold(),
             isRemoteOnHold: this.props.call.isRemoteOnHold(),
             micMuted: this.props.call.isMicrophoneMuted(),
             vidMuted: this.props.call.isLocalVideoMuted(),
+            screensharing: this.props.call.isScreensharing(),
             callState: this.props.call.state,
             controlsVisible: true,
             showMoreMenu: false,
             showDialpad: false,
-            feeds: this.props.call.getFeeds(),
+            primaryFeed: primary,
+            secondaryFeeds: secondary,
+            sidebarShown: true,
         };
 
         this.updateCallListeners(null, this.props.call);
@@ -194,7 +207,11 @@ export default class CallView extends React.Component<IProps, IState> {
     };
 
     private onFeedsChanged = (newFeeds: Array<CallFeed>) => {
-        this.setState({ feeds: newFeeds });
+        const { primary, secondary } = this.getOrderedFeeds(newFeeds);
+        this.setState({
+            primaryFeed: primary,
+            secondaryFeeds: secondary,
+        });
     };
 
     private onCallLocalHoldUnhold = () => {
@@ -237,7 +254,30 @@ export default class CallView extends React.Component<IProps, IState> {
         this.showControls();
     };
 
-    private showControls() {
+    private getOrderedFeeds(feeds: Array<CallFeed>): { primary: CallFeed, secondary: Array<CallFeed> } {
+        let primary;
+
+        // Try to use a screensharing as primary, a remote one if possible
+        const screensharingFeeds = feeds.filter((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare);
+        primary = screensharingFeeds.find((feed) => !feed.isLocal()) || screensharingFeeds[0];
+        // If we didn't find remote screen-sharing stream, try to find any remote stream
+        if (!primary) {
+            primary = feeds.find((feed) => !feed.isLocal());
+        }
+
+        const secondary = [...feeds];
+        // Remove the primary feed from the array
+        if (primary) secondary.splice(secondary.indexOf(primary), 1);
+        secondary.sort((a, b) => {
+            if (a.isLocal() && !b.isLocal()) return -1;
+            if (!a.isLocal() && b.isLocal()) return 1;
+            return 0;
+        });
+
+        return { primary, secondary };
+    }
+
+    private showControls(): void {
         if (this.state.showMoreMenu || this.state.showDialpad) return;
 
         if (!this.state.controlsVisible) {
@@ -251,7 +291,7 @@ export default class CallView extends React.Component<IProps, IState> {
         this.controlsHideTimer = window.setTimeout(this.onControlsHideTimer, CONTROLS_HIDE_DELAY);
     }
 
-    private onDialpadClick = () => {
+    private onDialpadClick = (): void => {
         if (!this.state.showDialpad) {
             if (this.controlsHideTimer) {
                 clearTimeout(this.controlsHideTimer);
@@ -274,21 +314,37 @@ export default class CallView extends React.Component<IProps, IState> {
         }
     };
 
-    private onMicMuteClick = () => {
+    private onMicMuteClick = (): void => {
         const newVal = !this.state.micMuted;
 
         this.props.call.setMicrophoneMuted(newVal);
         this.setState({ micMuted: newVal });
     };
 
-    private onVidMuteClick = () => {
+    private onVidMuteClick = (): void => {
         const newVal = !this.state.vidMuted;
 
         this.props.call.setLocalVideoMuted(newVal);
         this.setState({ vidMuted: newVal });
     };
 
-    private onMoreClick = () => {
+    private onScreenshareClick = async (): Promise<void> => {
+        const isScreensharing = await this.props.call.setScreensharingEnabled(
+            !this.state.screensharing,
+            async (): Promise<DesktopCapturerSource> => {
+                const { finished } = Modal.createDialog(DesktopCapturerSourcePicker);
+                const [source] = await finished;
+                return source;
+            },
+        );
+
+        this.setState({
+            sidebarShown: true,
+            screensharing: isScreensharing,
+        });
+    };
+
+    private onMoreClick = (): void => {
         if (this.controlsHideTimer) {
             clearTimeout(this.controlsHideTimer);
             this.controlsHideTimer = null;
@@ -300,14 +356,14 @@ export default class CallView extends React.Component<IProps, IState> {
         });
     };
 
-    private closeDialpad = () => {
+    private closeDialpad = (): void => {
         this.setState({
             showDialpad: false,
         });
         this.controlsHideTimer = window.setTimeout(this.onControlsHideTimer, CONTROLS_HIDE_DELAY);
     };
 
-    private closeContextMenu = () => {
+    private closeContextMenu = (): void => {
         this.setState({
             showMoreMenu: false,
         });
@@ -317,7 +373,7 @@ export default class CallView extends React.Component<IProps, IState> {
     // we register global shortcuts here, they *must not conflict* with local shortcuts elsewhere or both will fire
     // Note that this assumes we always have a CallView on screen at any given time
     // CallHandler would probably be a better place for this
-    private onNativeKeyDown = ev => {
+    private onNativeKeyDown = (ev): void => {
         let handled = false;
         const ctrlCmdOnly = isOnlyCtrlOrCmdKeyEvent(ev);
 
@@ -347,7 +403,7 @@ export default class CallView extends React.Component<IProps, IState> {
         }
     };
 
-    private onRoomAvatarClick = () => {
+    private onRoomAvatarClick = (): void => {
         const userFacingRoomId = CallHandler.sharedInstance().roomIdForCall(this.props.call);
         dis.dispatch({
             action: 'view_room',
@@ -355,7 +411,7 @@ export default class CallView extends React.Component<IProps, IState> {
         });
     };
 
-    private onSecondaryRoomAvatarClick = () => {
+    private onSecondaryRoomAvatarClick = (): void => {
         const userFacingRoomId = CallHandler.sharedInstance().roomIdForCall(this.props.secondaryCall);
 
         dis.dispatch({
@@ -364,50 +420,30 @@ export default class CallView extends React.Component<IProps, IState> {
         });
     };
 
-    private onCallResumeClick = () => {
+    private onCallResumeClick = (): void => {
         const userFacingRoomId = CallHandler.sharedInstance().roomIdForCall(this.props.call);
         CallHandler.sharedInstance().setActiveCallRoomId(userFacingRoomId);
     };
 
-    private onTransferClick = () => {
+    private onTransferClick = (): void => {
         const transfereeCall = CallHandler.sharedInstance().getTransfereeForCallId(this.props.call.callId);
         this.props.call.transferToCall(transfereeCall);
     };
 
-    public render() {
-        const client = MatrixClientPeg.get();
-        const callRoomId = CallHandler.sharedInstance().roomIdForCall(this.props.call);
-        const secondaryCallRoomId = CallHandler.sharedInstance().roomIdForCall(this.props.secondaryCall);
-        const callRoom = client.getRoom(callRoomId);
-        const secCallRoom = this.props.secondaryCall ? client.getRoom(secondaryCallRoomId) : null;
+    private onHangupClick = (): void => {
+        dis.dispatch({
+            action: 'hangup',
+            room_id: CallHandler.sharedInstance().roomIdForCall(this.props.call),
+        });
+    };
 
-        let dialPad;
-        let contextMenu;
-
-        if (this.state.showDialpad) {
-            dialPad = <DialpadContextMenu
-                {...alwaysAboveRightOf(
-                    this.dialpadButton.current.getBoundingClientRect(),
-                    ChevronFace.None,
-                    CONTEXT_MENU_VPADDING,
-                )}
-                onFinished={this.closeDialpad}
-                call={this.props.call}
-            />;
-        }
-
-        if (this.state.showMoreMenu) {
-            contextMenu = <CallContextMenu
-                {...alwaysAboveLeftOf(
-                    this.contextMenuButton.current.getBoundingClientRect(),
-                    ChevronFace.None,
-                    CONTEXT_MENU_VPADDING,
-                )}
-                onFinished={this.closeContextMenu}
-                call={this.props.call}
-            />;
-        }
+    private onToggleSidebar = (): void => {
+        this.setState({
+            sidebarShown: !this.state.sidebarShown,
+        });
+    };
 
+    private renderCallControls(): JSX.Element {
         const micClasses = classNames({
             mx_CallView_callControls_button: true,
             mx_CallView_callControls_button_micOn: !this.state.micMuted,
@@ -420,6 +456,18 @@ export default class CallView extends React.Component<IProps, IState> {
             mx_CallView_callControls_button_vidOff: this.state.vidMuted,
         });
 
+        const screensharingClasses = classNames({
+            mx_CallView_callControls_button: true,
+            mx_CallView_callControls_button_screensharingOn: this.state.screensharing,
+            mx_CallView_callControls_button_screensharingOff: !this.state.screensharing,
+        });
+
+        const sidebarButtonClasses = classNames({
+            mx_CallView_callControls_button: true,
+            mx_CallView_callControls_button_sidebarOn: this.state.sidebarShown,
+            mx_CallView_callControls_button_sidebarOff: !this.state.sidebarShown,
+        });
+
         // Put the other states of the mic/video icons in the document to make sure they're cached
         // (otherwise the icon disappears briefly when toggled)
         const micCacheClasses = classNames({
@@ -441,59 +489,116 @@ export default class CallView extends React.Component<IProps, IState> {
             mx_CallView_callControls_hidden: !this.state.controlsVisible,
         });
 
-        const vidMuteButton = this.props.call.type === CallType.Video ? <AccessibleButton
-            className={vidClasses}
-            onClick={this.onVidMuteClick}
-        /> : null;
+        // We don't support call upgrades (yet) so hide the video mute button in voice calls
+        let vidMuteButton;
+        if (this.props.call.type === CallType.Video) {
+            vidMuteButton = (
+                <AccessibleButton
+                    className={vidClasses}
+                    onClick={this.onVidMuteClick}
+                />
+            );
+        }
+
+        // Screensharing is possible, if we can send a second stream and
+        // identify it using SDPStreamMetadata or if we can replace the already
+        // existing usermedia track by a screensharing track. We also need to be
+        // connected to know the state of the other side
+        let screensharingButton;
+        if (
+            (this.props.call.opponentSupportsSDPStreamMetadata() || this.props.call.type === CallType.Video) &&
+            this.props.call.state === CallState.Connected
+        ) {
+            screensharingButton = (
+                <AccessibleButton
+                    className={screensharingClasses}
+                    onClick={this.onScreenshareClick}
+                />
+            );
+        }
+
+        // To show the sidebar we need secondary feeds, if we don't have them,
+        // we can hide this button. If we are in PiP, sidebar is also hidden, so
+        // we can hide the button too
+        let sidebarButton;
+        if (
+            !this.props.pipMode &&
+            (
+                this.state.primaryFeed?.purpose === SDPStreamMetadataPurpose.Screenshare ||
+                this.props.call.isScreensharing()
+            )
+        ) {
+            sidebarButton = (
+                <AccessibleButton
+                    className={sidebarButtonClasses}
+                    onClick={this.onToggleSidebar}
+                />
+            );
+        }
 
         // The dial pad & 'more' button actions are only relevant in a connected call
         // When not connected, we have to put something there to make the flexbox alignment correct
-        const dialpadButton = this.state.callState === CallState.Connected ? <ContextMenuButton
-            className="mx_CallView_callControls_button mx_CallView_callControls_dialpad"
-            inputRef={this.dialpadButton}
-            onClick={this.onDialpadClick}
-            isExpanded={this.state.showDialpad}
-        /> : <div className="mx_CallView_callControls_button mx_CallView_callControls_button_dialpad_hidden" />;
+        let dialpadButton;
+        let contextMenuButton;
+        if (this.state.callState === CallState.Connected) {
+            contextMenuButton = (
+                <ContextMenuButton
+                    className="mx_CallView_callControls_button mx_CallView_callControls_button_more"
+                    onClick={this.onMoreClick}
+                    inputRef={this.contextMenuButton}
+                    isExpanded={this.state.showMoreMenu}
+                />
+            );
+            dialpadButton = (
+                <ContextMenuButton
+                    className="mx_CallView_callControls_button mx_CallView_callControls_dialpad"
+                    inputRef={this.dialpadButton}
+                    onClick={this.onDialpadClick}
+                    isExpanded={this.state.showDialpad}
+                />
+            );
+        }
 
-        const contextMenuButton = this.state.callState === CallState.Connected ? <ContextMenuButton
-            className="mx_CallView_callControls_button mx_CallView_callControls_button_more"
-            onClick={this.onMoreClick}
-            inputRef={this.contextMenuButton}
-            isExpanded={this.state.showMoreMenu}
-        /> : <div className="mx_CallView_callControls_button mx_CallView_callControls_button_more_hidden" />;
-
-        // in the near future, the dial pad button will go on the left. For now, it's the nothing button
-        // because something needs to have margin-right: auto to make the alignment correct.
-        const callControls = <div className={callControlsClasses}>
-            { dialpadButton }
-            <AccessibleButton
-                className={micClasses}
-                onClick={this.onMicMuteClick}
-            />
-            <AccessibleButton
-                className="mx_CallView_callControls_button mx_CallView_callControls_button_hangup"
-                onClick={() => {
-                    dis.dispatch({
-                        action: 'hangup',
-                        room_id: callRoomId,
-                    });
-                }}
-            />
-            { vidMuteButton }
-            <div className={micCacheClasses} />
-            <div className={vidCacheClasses} />
-            { contextMenuButton }
-        </div>;
+        return (
+            <div className={callControlsClasses}>
+                { dialpadButton }
+                <AccessibleButton
+                    className={micClasses}
+                    onClick={this.onMicMuteClick}
+                />
+                { vidMuteButton }
+                <div className={micCacheClasses} />
+                <div className={vidCacheClasses} />
+                { screensharingButton }
+                { sidebarButton }
+                { contextMenuButton }
+                <AccessibleButton
+                    className="mx_CallView_callControls_button mx_CallView_callControls_button_hangup"
+                    onClick={this.onHangupClick}
+                />
+            </div>
+        );
+    }
 
+    public render() {
+        const client = MatrixClientPeg.get();
+        const callRoomId = CallHandler.sharedInstance().roomIdForCall(this.props.call);
+        const secondaryCallRoomId = CallHandler.sharedInstance().roomIdForCall(this.props.secondaryCall);
+        const callRoom = client.getRoom(callRoomId);
+        const secCallRoom = this.props.secondaryCall ? client.getRoom(secondaryCallRoomId) : null;
         const avatarSize = this.props.pipMode ? 76 : 160;
-
-        // The 'content' for the call, ie. the videos for a video call and profile picture
-        // for voice calls (fills the bg)
-        let contentView: React.ReactNode;
-
         const transfereeCall = CallHandler.sharedInstance().getTransfereeForCallId(this.props.call.callId);
         const isOnHold = this.state.isLocalOnHold || this.state.isRemoteOnHold;
+        const isScreensharing = this.props.call.isScreensharing();
+        const sidebarShown = this.state.sidebarShown;
+        const someoneIsScreensharing = this.props.call.getFeeds().some((feed) => {
+            return feed.purpose === SDPStreamMetadataPurpose.Screenshare;
+        });
+        const isVideoCall = this.props.call.type === CallType.Video;
+
+        let contentView: React.ReactNode;
         let holdTransferContent;
+
         if (transfereeCall) {
             const transferTargetRoom = MatrixClientPeg.get().getRoom(
                 CallHandler.sharedInstance().roomIdForCall(this.props.call),
@@ -539,9 +644,25 @@ export default class CallView extends React.Component<IProps, IState> {
             </div>;
         }
 
+        let sidebar;
+        if (
+            !isOnHold &&
+            !transfereeCall &&
+            sidebarShown &&
+            (isVideoCall || someoneIsScreensharing)
+        ) {
+            sidebar = (
+                <CallViewSidebar
+                    feeds={this.state.secondaryFeeds}
+                    call={this.props.call}
+                    pipMode={this.props.pipMode}
+                />
+            );
+        }
+
         // This is a bit messy. I can't see a reason to have two onHold/transfer screens
         if (isOnHold || transfereeCall) {
-            if (this.props.call.type === CallType.Video) {
+            if (isVideoCall) {
                 const containerClasses = classNames({
                     mx_CallView_content: true,
                     mx_CallView_video: true,
@@ -560,7 +681,7 @@ export default class CallView extends React.Component<IProps, IState> {
                     <div className={containerClasses} ref={this.contentRef} onMouseMove={this.onMouseMove}>
                         { onHoldBackground }
                         { holdTransferContent }
-                        { callControls }
+                        { this.renderCallControls() }
                     </div>
                 );
             } else {
@@ -585,7 +706,7 @@ export default class CallView extends React.Component<IProps, IState> {
                             </div>
                         </div>
                         { holdTransferContent }
-                        { callControls }
+                        { this.renderCallControls() }
                     </div>
                 );
             }
@@ -599,77 +720,91 @@ export default class CallView extends React.Component<IProps, IState> {
                 mx_CallView_voice: true,
             });
 
-            const feeds = this.props.call.getLocalFeeds().map((feed, i) => {
-                // Here we check to hide local audio feeds to achieve the same UI/UX
-                // as before. But once again this might be subject to change
-                if (feed.isVideoMuted()) return;
-                return (
-                    <VideoFeed
-                        key={i}
-                        feed={feed}
-                        call={this.props.call}
-                        pipMode={this.props.pipMode}
-                        onResize={this.props.onResize}
-                    />
-                );
-            });
-
             // Saying "Connecting" here isn't really true, but the best thing
             // I can come up with, but this might be subject to change as well
-            contentView = <div className={classes} onMouseMove={this.onMouseMove}>
-                { feeds }
-                <div className="mx_CallView_voice_avatarsContainer">
-                    <div className="mx_CallView_voice_avatarContainer" style={{ width: avatarSize, height: avatarSize }}>
-                        <RoomAvatar
-                            room={callRoom}
-                            height={avatarSize}
-                            width={avatarSize}
-                        />
+            contentView = (
+                <div
+                    className={classes}
+                    onMouseMove={this.onMouseMove}
+                >
+                    { sidebar }
+                    <div className="mx_CallView_voice_avatarsContainer">
+                        <div
+                            className="mx_CallView_voice_avatarContainer"
+                            style={{ width: avatarSize, height: avatarSize }}
+                        >
+                            <RoomAvatar
+                                room={callRoom}
+                                height={avatarSize}
+                                width={avatarSize}
+                            />
+                        </div>
                     </div>
+                    <div className="mx_CallView_holdTransferContent">{ _t("Connecting") }</div>
+                    { this.renderCallControls() }
                 </div>
-                <div className="mx_CallView_holdTransferContent">{ _t("Connecting") }</div>
-                { callControls }
-            </div>;
+            );
         } else {
             const containerClasses = classNames({
                 mx_CallView_content: true,
                 mx_CallView_video: true,
             });
 
-            // TODO: Later the CallView should probably be reworked to support
-            // any number of feeds but now we can always expect there to be two
-            // feeds. This is because the js-sdk ignores any new incoming streams
-            const feeds = this.state.feeds.map((feed, i) => {
-                // Here we check to hide local audio feeds to achieve the same UI/UX
-                // as before. But once again this might be subject to change
-                if (feed.isVideoMuted() && feed.isLocal()) return;
-                return (
+            let toast;
+            if (someoneIsScreensharing) {
+                const presentingClasses = classNames({
+                    mx_CallView_presenting: true,
+                    mx_CallView_presenting_hidden: !this.state.controlsVisible,
+                });
+                const sharerName = this.state.primaryFeed.getMember().name;
+                let text = isScreensharing
+                    ? _t("You are presenting")
+                    : _t('%(sharerName)s is presenting', { sharerName });
+                if (!this.state.sidebarShown && isVideoCall) {
+                    text += " • " + (this.props.call.isLocalVideoMuted()
+                        ? _t("Your camera is turned off")
+                        : _t("Your camera is still enabled"));
+                }
+
+                toast = (
+                    <div className={presentingClasses}>
+                        { text }
+                    </div>
+                );
+            }
+
+            contentView = (
+                <div
+                    className={containerClasses}
+                    ref={this.contentRef}
+                    onMouseMove={this.onMouseMove}
+                >
+                    { toast }
+                    { sidebar }
                     <VideoFeed
-                        key={i}
-                        feed={feed}
+                        feed={this.state.primaryFeed}
                         call={this.props.call}
                         pipMode={this.props.pipMode}
                         onResize={this.props.onResize}
+                        primary={true}
                     />
-                );
-            });
-
-            contentView = <div className={containerClasses} ref={this.contentRef} onMouseMove={this.onMouseMove}>
-                { feeds }
-                { callControls }
-            </div>;
+                    { this.renderCallControls() }
+                </div>
+            );
         }
 
-        const callTypeText = this.props.call.type === CallType.Video ? _t("Video Call") : _t("Voice Call");
+        const callTypeText = isVideoCall ? _t("Video Call") : _t("Voice Call");
         let myClassName;
 
         let fullScreenButton;
-        if (this.props.call.type === CallType.Video && !this.props.pipMode) {
-            fullScreenButton = <div
-                className="mx_CallView_header_button mx_CallView_header_button_fullscreen"
-                onClick={this.onFullscreenClick}
-                title={_t("Fill Screen")}
-            />;
+        if (!this.props.pipMode) {
+            fullScreenButton = (
+                <div
+                    className="mx_CallView_header_button mx_CallView_header_button_fullscreen"
+                    onClick={this.onFullscreenClick}
+                    title={_t("Fill Screen")}
+                />
+            );
         }
 
         let expandButton;
@@ -728,6 +863,32 @@ export default class CallView extends React.Component<IProps, IState> {
             myClassName = 'mx_CallView_pip';
         }
 
+        let dialPad;
+        if (this.state.showDialpad) {
+            dialPad = <DialpadContextMenu
+                {...alwaysAboveRightOf(
+                    this.dialpadButton.current.getBoundingClientRect(),
+                    ChevronFace.None,
+                    CONTEXT_MENU_VPADDING,
+                )}
+                onFinished={this.closeDialpad}
+                call={this.props.call}
+            />;
+        }
+
+        let contextMenu;
+        if (this.state.showMoreMenu) {
+            contextMenu = <CallContextMenu
+                {...alwaysAboveLeftOf(
+                    this.contextMenuButton.current.getBoundingClientRect(),
+                    ChevronFace.None,
+                    CONTEXT_MENU_VPADDING,
+                )}
+                onFinished={this.closeContextMenu}
+                call={this.props.call}
+            />;
+        }
+
         return <div className={"mx_CallView " + myClassName}>
             { header }
             { contentView }

src/components/views/voip/CallViewSidebar.tsx

@@ -0,0 +1,53 @@
+/*
+Copyright 2021 Šimon Brandner <simon.bra.ag@gmail.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from "react";
+import { MatrixCall } from "matrix-js-sdk/src/webrtc/call";
+import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed";
+import VideoFeed from "./VideoFeed";
+import classNames from "classnames";
+
+interface IProps {
+    feeds: Array<CallFeed>;
+    call: MatrixCall;
+    pipMode: boolean;
+}
+
+export default class CallViewSidebar extends React.Component<IProps> {
+    render() {
+        const feeds = this.props.feeds.map((feed) => {
+            return (
+                <VideoFeed
+                    key={feed.stream.id}
+                    feed={feed}
+                    call={this.props.call}
+                    primary={false}
+                    pipMode={this.props.pipMode}
+                />
+            );
+        });
+
+        const className = classNames("mx_CallViewSidebar", {
+            mx_CallViewSidebar_pipMode: this.props.pipMode,
+        });
+
+        return (
+            <div className={className}>
+                { feeds }
+            </div>
+        );
+    }
+}

src/components/views/voip/VideoFeed.tsx

@@ -16,7 +16,7 @@ limitations under the License.
 
 import classnames from 'classnames';
 import { MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
-import React, { createRef } from 'react';
+import React from 'react';
 import SettingsStore from "../../../settings/SettingsStore";
 import { CallFeed, CallFeedEvent } from 'matrix-js-sdk/src/webrtc/callFeed';
 import { logger } from 'matrix-js-sdk/src/logger';
@@ -37,6 +37,8 @@ interface IProps {
     // a callback which is called when the video element is resized
     // due to a change in video metadata
     onResize?: (e: Event) => void;
+
+    primary: boolean;
 }
 
 interface IState {
@@ -46,7 +48,7 @@ interface IState {
 
 @replaceableComponent("views.voip.VideoFeed")
 export default class VideoFeed extends React.Component<IProps, IState> {
-    private element = createRef<HTMLVideoElement>();
+    private element: HTMLVideoElement;
 
     constructor(props: IProps) {
         super(props);
@@ -58,19 +60,47 @@ export default class VideoFeed extends React.Component<IProps, IState> {
     }
 
     componentDidMount() {
-        this.props.feed.addListener(CallFeedEvent.NewStream, this.onNewStream);
-        this.element.current?.addEventListener('resize', this.onResize);
+        this.updateFeed(null, this.props.feed);
         this.playMedia();
     }
 
     componentWillUnmount() {
-        this.props.feed.removeListener(CallFeedEvent.NewStream, this.onNewStream);
-        this.element.current?.removeEventListener('resize', this.onResize);
-        this.stopMedia();
+        this.updateFeed(this.props.feed, null);
+    }
+
+    componentDidUpdate(prevProps: IProps) {
+        this.updateFeed(prevProps.feed, this.props.feed);
+    }
+
+    static getDerivedStateFromProps(props: IProps) {
+        return {
+            audioMuted: props.feed.isAudioMuted(),
+            videoMuted: props.feed.isVideoMuted(),
+        };
+    }
+
+    private setElementRef = (element: HTMLVideoElement): void => {
+        if (!element) element.removeEventListener('resize', this.onResize);
+
+        this.element = element;
+        element.addEventListener('resize', this.onResize);
+    };
+
+    private updateFeed(oldFeed: CallFeed, newFeed: CallFeed) {
+        if (oldFeed === newFeed) return;
+
+        if (oldFeed) {
+            this.props.feed.removeListener(CallFeedEvent.NewStream, this.onNewStream);
+            this.stopMedia();
+        }
+        if (newFeed) {
+            this.props.feed.addListener(CallFeedEvent.NewStream, this.onNewStream);
+            this.playMedia();
+        }
     }
 
     private playMedia() {
-        const element = this.element.current;
+        const element = this.element;
         if (!element) return;
         // We play audio in AudioFeed, not here
         element.muted = true;
@@ -93,7 +123,7 @@ export default class VideoFeed extends React.Component<IProps, IState> {
     }
 
     private stopMedia() {
-        const element = this.element.current;
+        const element = this.element;
         if (!element) return;
 
         element.pause();
@@ -122,8 +152,6 @@ export default class VideoFeed extends React.Component<IProps, IState> {
     render() {
         const videoClasses = {
             mx_VideoFeed: true,
-            mx_VideoFeed_local: this.props.feed.isLocal(),
-            mx_VideoFeed_remote: !this.props.feed.isLocal(),
             mx_VideoFeed_voice: this.state.videoMuted,
             mx_VideoFeed_video: !this.state.videoMuted,
             mx_VideoFeed_mirror: (
@@ -132,9 +160,15 @@ export default class VideoFeed extends React.Component<IProps, IState> {
             ),
         };
 
+        const { pipMode, primary } = this.props;
+
         if (this.state.videoMuted) {
             const member = this.props.feed.getMember();
-            const avatarSize = this.props.pipMode ? 76 : 160;
+            let avatarSize;
+            if (pipMode && primary) avatarSize = 76;
+            else if (pipMode && !primary) avatarSize = 16;
+            else if (!pipMode && primary) avatarSize = 160;
+            else; // TBD
 
             return (
                 <div className={classnames(videoClasses)}>
@@ -147,7 +181,7 @@ export default class VideoFeed extends React.Component<IProps, IState> {
             );
         } else {
             return (
-                <video className={classnames(videoClasses)} ref={this.element} />
+                <video className={classnames(videoClasses)} ref={this.setElementRef} />
             );
         }
     }