Merge remote-tracking branch 'upstream/develop' into fix/17130/draggable-pip

This commit is contained in:
Šimon Brandner 2021-06-17 15:41:19 +02:00
commit d89392a1d9
No known key found for this signature in database
GPG key ID: 9760693FDD98A790
418 changed files with 17369 additions and 6592 deletions

View file

@ -0,0 +1,97 @@
/*
Copyright 2021 Šimon Brandner <simon.bra.ag@gmail.com>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React, {createRef} from 'react';
import { CallFeed, CallFeedEvent } from 'matrix-js-sdk/src/webrtc/callFeed';
import { logger } from 'matrix-js-sdk/src/logger';
import CallMediaHandler from "../../../CallMediaHandler";
interface IProps {
feed: CallFeed,
}
export default class AudioFeed extends React.Component<IProps> {
private element = createRef<HTMLAudioElement>();
componentDidMount() {
this.props.feed.addListener(CallFeedEvent.NewStream, this.onNewStream);
this.playMedia();
}
componentWillUnmount() {
this.props.feed.removeListener(CallFeedEvent.NewStream, this.onNewStream);
this.stopMedia();
}
private playMedia() {
const element = this.element.current;
const audioOutput = CallMediaHandler.getAudioOutput();
if (audioOutput) {
try {
// This seems quite unreliable in Chrome, although I haven't yet managed to make a jsfiddle where
// it fails.
// It seems reliable if you set the sink ID after setting the srcObject and then set the sink ID
// back to the default after the call is over - Dave
element.setSinkId(audioOutput);
} catch (e) {
console.error("Couldn't set requested audio output device: using default", e);
logger.warn("Couldn't set requested audio output device: using default", e);
}
}
element.muted = false;
element.srcObject = this.props.feed.stream;
element.autoplay = true;
try {
// A note on calling methods on media elements:
// We used to have queues per media element to serialise all calls on those elements.
// The reason given for this was that load() and play() were racing. However, we now
// never call load() explicitly so this seems unnecessary. However, serialising every
// operation was causing bugs where video would not resume because some play command
// had got stuck and all media operations were queued up behind it. If necessary, we
// should serialise the ones that need to be serialised but then be able to interrupt
// them with another load() which will cancel the pending one, but since we don't call
// load() explicitly, it shouldn't be a problem. - Dave
element.play()
} catch (e) {
logger.info("Failed to play media element with feed", this.props.feed, e);
}
}
private stopMedia() {
const element = this.element.current;
element.pause();
element.src = null;
// As per comment in componentDidMount, setting the sink ID back to the
// default once the call is over makes setSinkId work reliably. - Dave
// Since we are not using the same element anymore, the above doesn't
// seem to be necessary - Šimon
}
private onNewStream = () => {
this.playMedia();
};
render() {
return (
<audio ref={this.element} />
);
}
}

View file

@ -0,0 +1,60 @@
/*
Copyright 2021 Šimon Brandner <simon.bra.ag@gmail.com>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React from "react";
import AudioFeed from "./AudioFeed"
import { CallEvent, MatrixCall } from "matrix-js-sdk/src/webrtc/call";
import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed";
interface IProps {
call: MatrixCall;
}
interface IState {
feeds: Array<CallFeed>;
}
export default class AudioFeedArrayForCall extends React.Component<IProps, IState> {
constructor(props: IProps) {
super(props);
this.state = {
feeds: [],
};
}
componentDidMount() {
this.props.call.addListener(CallEvent.FeedsChanged, this.onFeedsChanged);
}
componentWillUnmount() {
this.props.call.removeListener(CallEvent.FeedsChanged, this.onFeedsChanged);
}
onFeedsChanged = () => {
this.setState({
feeds: this.props.call.getRemoteFeeds(),
});
}
render() {
return this.state.feeds.map((feed, i) => {
return (
<AudioFeed feed={feed} key={i} />
);
});
}
}

View file

@ -19,7 +19,7 @@ import React, { createRef } from 'react';
import CallView from "./CallView";
import RoomViewStore from '../../../stores/RoomViewStore';
import CallHandler from '../../../CallHandler';
import CallHandler, { CallHandlerEvent } from '../../../CallHandler';
import dis from '../../../dispatcher/dispatcher';
import { ActionPayload } from '../../../dispatcher/payloads';
import PersistentApp from "../elements/PersistentApp";
@ -27,7 +27,6 @@ import SettingsStore from "../../../settings/SettingsStore";
import { CallEvent, CallState, MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
import { MatrixClientPeg } from '../../../MatrixClientPeg';
import {replaceableComponent} from "../../../utils/replaceableComponent";
import { Action } from '../../../dispatcher/actions';
const PIP_VIEW_WIDTH = 336;
const PIP_VIEW_HEIGHT = 232;
@ -131,6 +130,7 @@ export default class CallPreview extends React.Component<IProps, IState> {
private initY = 0;
public componentDidMount() {
CallHandler.sharedInstance().addListener(CallHandlerEvent.CallChangeRoom, this.updateCalls);
this.roomStoreToken = RoomViewStore.addListener(this.onRoomViewStoreUpdate);
document.addEventListener("mousemove", this.onMoving);
document.addEventListener("mouseup", this.onEndMoving);
@ -140,6 +140,7 @@ export default class CallPreview extends React.Component<IProps, IState> {
}
public componentWillUnmount() {
CallHandler.sharedInstance().removeListener(CallHandlerEvent.CallChangeRoom, this.updateCalls);
MatrixClientPeg.get().removeListener(CallEvent.RemoteHoldUnhold, this.onCallRemoteHold);
document.removeEventListener("mousemove", this.onMoving);
document.removeEventListener("mouseup", this.onEndMoving);
@ -205,21 +206,24 @@ export default class CallPreview extends React.Component<IProps, IState> {
switch (payload.action) {
// listen for call state changes to prod the render method, which
// may hide the global CallView if the call it is tracking is dead
case Action.CallChangeRoom:
case 'call_state': {
const [primaryCall, secondaryCalls] = getPrimarySecondaryCalls(
CallHandler.sharedInstance().getAllActiveCallsNotInRoom(this.state.roomId),
);
this.setState({
primaryCall: primaryCall,
secondaryCall: secondaryCalls[0],
});
this.updateCalls();
break;
}
}
};
private updateCalls = () => {
const [primaryCall, secondaryCalls] = getPrimarySecondaryCalls(
CallHandler.sharedInstance().getAllActiveCallsNotInRoom(this.state.roomId),
);
this.setState({
primaryCall: primaryCall,
secondaryCall: secondaryCalls[0],
});
};
private onCallRemoteHold = () => {
const [primaryCall, secondaryCalls] = getPrimarySecondaryCalls(
CallHandler.sharedInstance().getAllActiveCallsNotInRoom(this.state.roomId),

View file

@ -20,10 +20,9 @@ import dis from '../../../dispatcher/dispatcher';
import CallHandler from '../../../CallHandler';
import {MatrixClientPeg} from '../../../MatrixClientPeg';
import { _t, _td } from '../../../languageHandler';
import VideoFeed, { VideoFeedType } from "./VideoFeed";
import VideoFeed from './VideoFeed';
import RoomAvatar from "../avatars/RoomAvatar";
import { CallState, CallType, MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
import { CallEvent } from 'matrix-js-sdk/src/webrtc/call';
import { CallState, CallType, MatrixCall, CallEvent } from 'matrix-js-sdk/src/webrtc/call';
import classNames from 'classnames';
import AccessibleButton from '../elements/AccessibleButton';
import {isOnlyCtrlOrCmdKeyEvent, Key} from '../../../Keyboard';
@ -31,6 +30,7 @@ import {alwaysAboveLeftOf, alwaysAboveRightOf, ChevronFace, ContextMenuButton} f
import CallContextMenu from '../context_menus/CallContextMenu';
import { avatarUrlForMember } from '../../../Avatar';
import DialpadContextMenu from '../context_menus/DialpadContextMenu';
import { CallFeed } from 'matrix-js-sdk/src/webrtc/callFeed';
import {replaceableComponent} from "../../../utils/replaceableComponent";
interface IProps {
@ -40,11 +40,11 @@ interface IProps {
// Another ongoing call to display information about
secondaryCall?: MatrixCall,
// a callback which is called when the content in the callview changes
// a callback which is called when the content in the CallView changes
// in a way that is likely to cause a resize.
onResize?: any;
// Whether this call view is for picture-in-pictue mode
// Whether this call view is for picture-in-picture mode
// otherwise, it's the larger call view when viewing the room the call is in.
// This is sort of a proxy for a number of things but we currently have no
// need to control those things separately, so this is simpler.
@ -63,6 +63,7 @@ interface IState {
controlsVisible: boolean,
showMoreMenu: boolean,
showDialpad: boolean,
feeds: CallFeed[],
}
function getFullScreenElement() {
@ -118,6 +119,7 @@ export default class CallView extends React.Component<IProps, IState> {
controlsVisible: true,
showMoreMenu: false,
showDialpad: false,
feeds: this.props.call.getFeeds(),
}
this.updateCallListeners(null, this.props.call);
@ -175,11 +177,13 @@ export default class CallView extends React.Component<IProps, IState> {
oldCall.removeListener(CallEvent.State, this.onCallState);
oldCall.removeListener(CallEvent.LocalHoldUnhold, this.onCallLocalHoldUnhold);
oldCall.removeListener(CallEvent.RemoteHoldUnhold, this.onCallRemoteHoldUnhold);
oldCall.removeListener(CallEvent.FeedsChanged, this.onFeedsChanged);
}
if (newCall) {
newCall.on(CallEvent.State, this.onCallState);
newCall.on(CallEvent.LocalHoldUnhold, this.onCallLocalHoldUnhold);
newCall.on(CallEvent.RemoteHoldUnhold, this.onCallRemoteHoldUnhold);
newCall.on(CallEvent.FeedsChanged, this.onFeedsChanged);
}
}
@ -189,6 +193,10 @@ export default class CallView extends React.Component<IProps, IState> {
});
};
private onFeedsChanged = (newFeeds: Array<CallFeed>) => {
this.setState({feeds: newFeeds});
};
private onCallLocalHoldUnhold = () => {
this.setState({
isLocalOnHold: this.props.call.isLocalOnHold(),
@ -307,7 +315,7 @@ export default class CallView extends React.Component<IProps, IState> {
}
// we register global shortcuts here, they *must not conflict* with local shortcuts elsewhere or both will fire
// Note that this assumes we always have a callview on screen at any given time
// Note that this assumes we always have a CallView on screen at any given time
// CallHandler would probably be a better place for this
private onNativeKeyDown = ev => {
let handled = false;
@ -477,6 +485,8 @@ export default class CallView extends React.Component<IProps, IState> {
{contextMenuButton}
</div>;
const avatarSize = this.props.pipMode ? 76 : 160;
// The 'content' for the call, ie. the videos for a video call and profile picture
// for voice calls (fills the bg)
let contentView: React.ReactNode;
@ -527,41 +537,85 @@ export default class CallView extends React.Component<IProps, IState> {
</div>;
}
if (this.props.call.type === CallType.Video) {
let localVideoFeed = null;
let onHoldBackground = null;
const backgroundStyle: CSSProperties = {};
const containerClasses = classNames({
mx_CallView_video: true,
mx_CallView_video_hold: isOnHold,
});
if (isOnHold) {
// This is a bit messy. I can't see a reason to have two onHold/transfer screens
if (isOnHold || transfereeCall) {
if (this.props.call.type === CallType.Video) {
const containerClasses = classNames({
mx_CallView_content: true,
mx_CallView_video: true,
mx_CallView_video_hold: isOnHold,
});
let onHoldBackground = null;
const backgroundStyle: CSSProperties = {};
const backgroundAvatarUrl = avatarUrlForMember(
// is it worth getting the size of the div to pass here?
// is it worth getting the size of the div to pass here?
this.props.call.getOpponentMember(), 1024, 1024, 'crop',
);
backgroundStyle.backgroundImage = 'url(' + backgroundAvatarUrl + ')';
onHoldBackground = <div className="mx_CallView_video_holdBackground" style={backgroundStyle} />;
}
if (!this.state.vidMuted) {
localVideoFeed = <VideoFeed type={VideoFeedType.Local} call={this.props.call} />;
}
contentView = <div className={containerClasses} ref={this.contentRef} onMouseMove={this.onMouseMove}>
{onHoldBackground}
<VideoFeed type={VideoFeedType.Remote} call={this.props.call} onResize={this.props.onResize} />
{localVideoFeed}
{holdTransferContent}
{callControls}
</div>;
} else {
const avatarSize = this.props.pipMode ? 76 : 160;
contentView = (
<div className={containerClasses} ref={this.contentRef} onMouseMove={this.onMouseMove}>
{onHoldBackground}
{holdTransferContent}
{callControls}
</div>
);
} else {
const classes = classNames({
mx_CallView_content: true,
mx_CallView_voice: true,
mx_CallView_voice_hold: isOnHold,
});
contentView =(
<div className={classes} onMouseMove={this.onMouseMove}>
<div className="mx_CallView_voice_avatarsContainer">
<div
className="mx_CallView_voice_avatarContainer"
style={{width: avatarSize, height: avatarSize}}
>
<RoomAvatar
room={callRoom}
height={avatarSize}
width={avatarSize}
/>
</div>
</div>
{holdTransferContent}
{callControls}
</div>
);
}
} else if (this.props.call.noIncomingFeeds()) {
// Here we're reusing the css classes from voice on hold, because
// I am lazy. If this gets merged, the CallView might be subject
// to change anyway - I might take an axe to this file in order to
// try to get other things working
const classes = classNames({
mx_CallView_content: true,
mx_CallView_voice: true,
mx_CallView_voice_hold: isOnHold,
});
const feeds = this.props.call.getLocalFeeds().map((feed, i) => {
// Here we check to hide local audio feeds to achieve the same UI/UX
// as before. But once again this might be subject to change
if (feed.isVideoMuted()) return;
return (
<VideoFeed
key={i}
feed={feed}
call={this.props.call}
pipMode={this.props.pipMode}
onResize={this.props.onResize}
/>
);
});
// Saying "Connecting" here isn't really true, but the best thing
// I can come up with, but this might be subject to change as well
contentView = <div className={classes} onMouseMove={this.onMouseMove}>
{feeds}
<div className="mx_CallView_voice_avatarsContainer">
<div className="mx_CallView_voice_avatarContainer" style={{width: avatarSize, height: avatarSize}}>
<RoomAvatar
@ -571,7 +625,35 @@ export default class CallView extends React.Component<IProps, IState> {
/>
</div>
</div>
{holdTransferContent}
<div className="mx_CallView_holdTransferContent">{_t("Connecting")}</div>
{callControls}
</div>;
} else {
const containerClasses = classNames({
mx_CallView_content: true,
mx_CallView_video: true,
});
// TODO: Later the CallView should probably be reworked to support
// any number of feeds but now we can always expect there to be two
// feeds. This is because the js-sdk ignores any new incoming streams
const feeds = this.state.feeds.map((feed, i) => {
// Here we check to hide local audio feeds to achieve the same UI/UX
// as before. But once again this might be subject to change
if (feed.isVideoMuted() && feed.isLocal()) return;
return (
<VideoFeed
key={i}
feed={feed}
call={this.props.call}
pipMode={this.props.pipMode}
onResize={this.props.onResize}
/>
);
});
contentView = <div className={containerClasses} ref={this.contentRef} onMouseMove={this.onMouseMove}>
{feeds}
{callControls}
</div>;
}

View file

@ -16,13 +16,12 @@ limitations under the License.
import { CallState, MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
import React from 'react';
import CallHandler from '../../../CallHandler';
import CallHandler, { CallHandlerEvent } from '../../../CallHandler';
import CallView from './CallView';
import dis from '../../../dispatcher/dispatcher';
import {Resizable} from "re-resizable";
import ResizeNotifier from "../../../utils/ResizeNotifier";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import { Action } from '../../../dispatcher/actions';
interface IProps {
// What room we should display the call for
@ -55,25 +54,30 @@ export default class CallViewForRoom extends React.Component<IProps, IState> {
public componentDidMount() {
this.dispatcherRef = dis.register(this.onAction);
CallHandler.sharedInstance().addListener(CallHandlerEvent.CallChangeRoom, this.updateCall);
}
public componentWillUnmount() {
dis.unregister(this.dispatcherRef);
CallHandler.sharedInstance().removeListener(CallHandlerEvent.CallChangeRoom, this.updateCall);
}
private onAction = (payload) => {
switch (payload.action) {
case Action.CallChangeRoom:
case 'call_state': {
const newCall = this.getCall();
if (newCall !== this.state.call) {
this.setState({call: newCall});
}
this.updateCall();
break;
}
}
};
private updateCall = () => {
const newCall = this.getCall();
if (newCall !== this.state.call) {
this.setState({call: newCall});
}
};
private getCall(): MatrixCall {
const call = CallHandler.sharedInstance().getCallForRoom(this.props.roomId);

View file

@ -15,17 +15,14 @@ limitations under the License.
*/
import * as React from "react";
import { ensureDMExists } from "../../../createRoom";
import { _t } from "../../../languageHandler";
import { MatrixClientPeg } from "../../../MatrixClientPeg";
import AccessibleButton from "../elements/AccessibleButton";
import Field from "../elements/Field";
import DialPad from './DialPad';
import dis from '../../../dispatcher/dispatcher';
import Modal from "../../../Modal";
import ErrorDialog from "../../views/dialogs/ErrorDialog";
import CallHandler from "../../../CallHandler";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import { DialNumberPayload } from "../../../dispatcher/payloads/DialNumberPayload";
import { Action } from "../../../dispatcher/actions";
interface IProps {
onFinished: (boolean) => void;
@ -67,21 +64,11 @@ export default class DialpadModal extends React.PureComponent<IProps, IState> {
}
onDialPress = async () => {
const results = await CallHandler.sharedInstance().pstnLookup(this.state.value);
if (!results || results.length === 0 || !results[0].userid) {
Modal.createTrackedDialog('', '', ErrorDialog, {
title: _t("Unable to look up phone number"),
description: _t("There was an error looking up the phone number"),
});
}
const userId = results[0].userid;
const roomId = await ensureDMExists(MatrixClientPeg.get(), userId);
dis.dispatch({
action: 'view_room',
room_id: roomId,
});
const payload: DialNumberPayload = {
action: Action.DialNumber,
number: this.state.value,
};
dis.dispatch(payload);
this.props.onFinished(true);
}

View file

@ -18,52 +18,102 @@ import classnames from 'classnames';
import { MatrixCall } from 'matrix-js-sdk/src/webrtc/call';
import React, {createRef} from 'react';
import SettingsStore from "../../../settings/SettingsStore";
import { CallFeed, CallFeedEvent } from 'matrix-js-sdk/src/webrtc/callFeed';
import { logger } from 'matrix-js-sdk/src/logger';
import MemberAvatar from "../avatars/MemberAvatar"
import {replaceableComponent} from "../../../utils/replaceableComponent";
export enum VideoFeedType {
Local,
Remote,
}
interface IProps {
call: MatrixCall,
type: VideoFeedType,
feed: CallFeed,
// Whether this call view is for picture-in-picture mode
// otherwise, it's the larger call view when viewing the room the call is in.
// This is sort of a proxy for a number of things but we currently have no
// need to control those things separately, so this is simpler.
pipMode?: boolean;
// a callback which is called when the video element is resized
// due to a change in video metadata
onResize?: (e: Event) => void,
}
@replaceableComponent("views.voip.VideoFeed")
export default class VideoFeed extends React.Component<IProps> {
private vid = createRef<HTMLVideoElement>();
interface IState {
audioMuted: boolean;
videoMuted: boolean;
}
componentDidMount() {
this.vid.current.addEventListener('resize', this.onResize);
this.setVideoElement();
@replaceableComponent("views.voip.VideoFeed")
export default class VideoFeed extends React.Component<IProps, IState> {
private element = createRef<HTMLVideoElement>();
constructor(props: IProps) {
super(props);
this.state = {
audioMuted: this.props.feed.isAudioMuted(),
videoMuted: this.props.feed.isVideoMuted(),
};
}
componentDidUpdate(prevProps) {
if (this.props.call !== prevProps.call) {
this.setVideoElement();
}
componentDidMount() {
this.props.feed.addListener(CallFeedEvent.NewStream, this.onNewStream);
this.playMedia();
}
componentWillUnmount() {
this.vid.current.removeEventListener('resize', this.onResize);
this.props.feed.removeListener(CallFeedEvent.NewStream, this.onNewStream);
this.element.current?.removeEventListener('resize', this.onResize);
this.stopMedia();
}
private setVideoElement() {
if (this.props.type === VideoFeedType.Local) {
this.props.call.setLocalVideoElement(this.vid.current);
} else {
this.props.call.setRemoteVideoElement(this.vid.current);
private playMedia() {
const element = this.element.current;
if (!element) return;
// We play audio in AudioFeed, not here
element.muted = true;
element.srcObject = this.props.feed.stream;
element.autoplay = true;
try {
// A note on calling methods on media elements:
// We used to have queues per media element to serialise all calls on those elements.
// The reason given for this was that load() and play() were racing. However, we now
// never call load() explicitly so this seems unnecessary. However, serialising every
// operation was causing bugs where video would not resume because some play command
// had got stuck and all media operations were queued up behind it. If necessary, we
// should serialise the ones that need to be serialised but then be able to interrupt
// them with another load() which will cancel the pending one, but since we don't call
// load() explicitly, it shouldn't be a problem. - Dave
element.play()
} catch (e) {
logger.info("Failed to play media element with feed", this.props.feed, e);
}
}
onResize = (e) => {
if (this.props.onResize) {
private stopMedia() {
const element = this.element.current;
if (!element) return;
element.pause();
element.src = null;
// As per comment in componentDidMount, setting the sink ID back to the
// default once the call is over makes setSinkId work reliably. - Dave
// Since we are not using the same element anymore, the above doesn't
// seem to be necessary - Šimon
}
private onNewStream = () => {
this.setState({
audioMuted: this.props.feed.isAudioMuted(),
videoMuted: this.props.feed.isVideoMuted(),
});
this.playMedia();
};
private onResize = (e) => {
if (this.props.onResize && !this.props.feed.isLocal()) {
this.props.onResize(e);
}
};
@ -71,14 +121,33 @@ export default class VideoFeed extends React.Component<IProps> {
render() {
const videoClasses = {
mx_VideoFeed: true,
mx_VideoFeed_local: this.props.type === VideoFeedType.Local,
mx_VideoFeed_remote: this.props.type === VideoFeedType.Remote,
mx_VideoFeed_local: this.props.feed.isLocal(),
mx_VideoFeed_remote: !this.props.feed.isLocal(),
mx_VideoFeed_voice: this.state.videoMuted,
mx_VideoFeed_video: !this.state.videoMuted,
mx_VideoFeed_mirror: (
this.props.type === VideoFeedType.Local &&
this.props.feed.isLocal() &&
SettingsStore.getValue('VideoView.flipVideoHorizontally')
),
};
return <video className={classnames(videoClasses)} ref={this.vid} />;
if (this.state.videoMuted) {
const member = this.props.feed.getMember();
const avatarSize = this.props.pipMode ? 76 : 160;
return (
<div className={classnames(videoClasses)} >
<MemberAvatar
member={member}
height={avatarSize}
width={avatarSize}
/>
</div>
);
} else {
return (
<video className={classnames(videoClasses)} ref={this.element} />
);
}
}
}