From 5dd34de5fe89504ea1354d3e81c3a23cefcc3805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 27 Jul 2021 14:31:42 +0200 Subject: [PATCH 1/9] Handle mute state changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/components/views/voip/VideoFeed.tsx | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/components/views/voip/VideoFeed.tsx b/src/components/views/voip/VideoFeed.tsx index 95cc5ee3e3..3d873cef0a 100644 --- a/src/components/views/voip/VideoFeed.tsx +++ b/src/components/views/voip/VideoFeed.tsx @@ -85,10 +85,12 @@ export default class VideoFeed extends React.Component { if (oldFeed) { this.props.feed.removeListener(CallFeedEvent.NewStream, this.onNewStream); + this.props.feed.removeListener(CallFeedEvent.MuteStateChanged, this.onMuteStateChanged); this.stopMedia(); } if (newFeed) { this.props.feed.addListener(CallFeedEvent.NewStream, this.onNewStream); + this.props.feed.addListener(CallFeedEvent.MuteStateChanged, this.onMuteStateChanged); this.playMedia(); } } @@ -137,6 +139,14 @@ export default class VideoFeed extends React.Component { this.playMedia(); }; + private onMuteStateChanged = () => { + this.setState({ + audioMuted: this.props.feed.isAudioMuted(), + videoMuted: this.props.feed.isVideoMuted(), + }); + this.playMedia(); + }; + private onResize = (e) => { if (this.props.onResize && !this.props.feed.isLocal()) { this.props.onResize(e); From 91e65534fa270fc22f62e30f77585ac1f67689c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 28 Jul 2021 15:04:33 +0200 Subject: [PATCH 2/9] await setState to avoid races where we would try to play media without an HTMLVideoElement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/components/views/voip/VideoFeed.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/components/views/voip/VideoFeed.tsx b/src/components/views/voip/VideoFeed.tsx index fef3aa0691..ad5b6f42fd 100644 --- a/src/components/views/voip/VideoFeed.tsx +++ b/src/components/views/voip/VideoFeed.tsx @@ -140,16 +140,16 @@ export default class VideoFeed extends React.Component { // seem to be necessary - Šimon } - private onNewStream = () => { - this.setState({ + private onNewStream = async () => { + await this.setState({ audioMuted: this.props.feed.isAudioMuted(), videoMuted: this.props.feed.isVideoMuted(), }); this.playMedia(); }; - private onMuteStateChanged = () => { - this.setState({ + private onMuteStateChanged = async () => { + await this.setState({ audioMuted: this.props.feed.isAudioMuted(), videoMuted: this.props.feed.isVideoMuted(), }); From 7c4e3efbff953c100efcd30e813c2447f9527775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 28 Jul 2021 15:11:31 +0200 Subject: [PATCH 3/9] Extend PureComponent to avoid unnecessary renders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/components/views/voip/VideoFeed.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/views/voip/VideoFeed.tsx b/src/components/views/voip/VideoFeed.tsx index ad5b6f42fd..41c6b5185c 100644 --- a/src/components/views/voip/VideoFeed.tsx +++ b/src/components/views/voip/VideoFeed.tsx @@ -47,7 +47,7 @@ interface IState { } @replaceableComponent("views.voip.VideoFeed") -export default class VideoFeed extends React.Component { +export default class VideoFeed extends React.PureComponent { private element: HTMLVideoElement; constructor(props: IProps) { From 537ce40f429c0284c7ba837f6e7912238242b9cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 28 Jul 2021 16:32:55 +0200 Subject: [PATCH 4/9] Add a TODO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/components/views/voip/VideoFeed.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/components/views/voip/VideoFeed.tsx b/src/components/views/voip/VideoFeed.tsx index 41c6b5185c..9975f70d62 100644 --- a/src/components/views/voip/VideoFeed.tsx +++ b/src/components/views/voip/VideoFeed.tsx @@ -46,6 +46,7 @@ interface IState { videoMuted: boolean; } +// TODO: We shouldn't be calling playMedia() all the time @replaceableComponent("views.voip.VideoFeed") export default class VideoFeed extends React.PureComponent { private element: HTMLVideoElement; From ae647658706a87c352b7e120423ebc87f33d8dba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 29 Jul 2021 08:45:32 +0200 Subject: [PATCH 5/9] playMedia only if necessary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/components/views/voip/VideoFeed.tsx | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/components/views/voip/VideoFeed.tsx b/src/components/views/voip/VideoFeed.tsx index 9975f70d62..af2fd92016 100644 --- a/src/components/views/voip/VideoFeed.tsx +++ b/src/components/views/voip/VideoFeed.tsx @@ -22,6 +22,7 @@ import { CallFeed, CallFeedEvent } from 'matrix-js-sdk/src/webrtc/callFeed'; import { logger } from 'matrix-js-sdk/src/logger'; import MemberAvatar from "../avatars/MemberAvatar"; import { replaceableComponent } from "../../../utils/replaceableComponent"; +import { objectHasDiff } from '../../../utils/objects'; interface IProps { call: MatrixCall; @@ -46,7 +47,6 @@ interface IState { videoMuted: boolean; } -// TODO: We shouldn't be calling playMedia() all the time @replaceableComponent("views.voip.VideoFeed") export default class VideoFeed extends React.PureComponent { private element: HTMLVideoElement; @@ -69,8 +69,10 @@ export default class VideoFeed extends React.PureComponent { this.updateFeed(this.props.feed, null); } - componentDidUpdate(prevProps: IProps) { + componentDidUpdate(prevProps: IProps, prevState: IState) { this.updateFeed(prevProps.feed, this.props.feed); + // If the mutes state has changed, we try to playMedia() + if (prevState.videoMuted !== this.state.videoMuted) this.playMedia(); } static getDerivedStateFromProps(props: IProps) { @@ -142,7 +144,7 @@ export default class VideoFeed extends React.PureComponent { } private onNewStream = async () => { - await this.setState({ + this.setState({ audioMuted: this.props.feed.isAudioMuted(), videoMuted: this.props.feed.isVideoMuted(), }); @@ -150,11 +152,10 @@ export default class VideoFeed extends React.PureComponent { }; private onMuteStateChanged = async () => { - await this.setState({ + this.setState({ audioMuted: this.props.feed.isAudioMuted(), videoMuted: this.props.feed.isVideoMuted(), }); - this.playMedia(); }; private onResize = (e) => { From 152168ef2ddc99a598fcebbbd517560e053fa850 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 29 Jul 2021 10:20:59 +0200 Subject: [PATCH 6/9] Add mic mute icons MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- res/img/voip/mic-muted.svg | 5 +++++ res/img/voip/mic-unmuted.svg | 4 ++++ 2 files changed, 9 insertions(+) create mode 100644 res/img/voip/mic-muted.svg create mode 100644 res/img/voip/mic-unmuted.svg diff --git a/res/img/voip/mic-muted.svg b/res/img/voip/mic-muted.svg new file mode 100644 index 0000000000..0cb7ad1c9e --- /dev/null +++ b/res/img/voip/mic-muted.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/res/img/voip/mic-unmuted.svg b/res/img/voip/mic-unmuted.svg new file mode 100644 index 0000000000..8334cafa0a --- /dev/null +++ b/res/img/voip/mic-unmuted.svg @@ -0,0 +1,4 @@ + + + + From cb89dd408c260a14bd9726fd65fe3f6f5acf2ecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 29 Jul 2021 15:05:26 +0200 Subject: [PATCH 7/9] Use mic mute icons MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- res/css/views/voip/_CallView.scss | 10 +++- res/css/views/voip/_CallViewSidebar.scss | 11 +++++ res/css/views/voip/_VideoFeed.scss | 48 +++++++++++++++--- src/components/views/voip/VideoFeed.tsx | 63 ++++++++++++++++-------- 4 files changed, 103 insertions(+), 29 deletions(-) diff --git a/res/css/views/voip/_CallView.scss b/res/css/views/voip/_CallView.scss index 104e2993d8..eff865f20c 100644 --- a/res/css/views/voip/_CallView.scss +++ b/res/css/views/voip/_CallView.scss @@ -76,16 +76,22 @@ limitations under the License. &.mx_VideoFeed_voice { // We don't want to collide with the call controls that have 52px of height - padding-bottom: 52px; + margin-bottom: 52px; background-color: $inverted-bg-color; display: flex; justify-content: center; align-items: center; } - &.mx_VideoFeed_video { + .mx_VideoFeed_video { + height: 100%; background-color: #000; } + + .mx_VideoFeed_mic { + left: 10px; + bottom: 10px; + } } } diff --git a/res/css/views/voip/_CallViewSidebar.scss b/res/css/views/voip/_CallViewSidebar.scss index 79bf3cbf09..892a137a32 100644 --- a/res/css/views/voip/_CallViewSidebar.scss +++ b/res/css/views/voip/_CallViewSidebar.scss @@ -35,12 +35,23 @@ limitations under the License. width: 100%; &.mx_VideoFeed_voice { + border-radius: 4px; + display: flex; align-items: center; justify-content: center; aspect-ratio: 16 / 9; } + + .mx_VideoFeed_video { + border-radius: 4px; + } + + .mx_VideoFeed_mic { + left: 6px; + bottom: 6px; + } } &.mx_CallViewSidebar_pipMode { diff --git a/res/css/views/voip/_VideoFeed.scss b/res/css/views/voip/_VideoFeed.scss index 07a4a0e530..3a0f62636e 100644 --- a/res/css/views/voip/_VideoFeed.scss +++ b/res/css/views/voip/_VideoFeed.scss @@ -15,18 +15,52 @@ limitations under the License. */ .mx_VideoFeed { - border-radius: 4px; - + overflow: hidden; + position: relative; &.mx_VideoFeed_voice { background-color: $inverted-bg-color; } - &.mx_VideoFeed_video { + .mx_VideoFeed_video { + width: 100%; background-color: transparent; + + &.mx_VideoFeed_video_mirror { + transform: scale(-1, 1); + } + } + + .mx_VideoFeed_mic { + position: absolute; + display: flex; + align-items: center; + justify-content: center; + + width: 24px; + height: 24px; + + background-color: rgba(0, 0, 0, 0.5); // Same on both themes + border-radius: 100%; + + &::before { + position: absolute; + content: ""; + width: 16px; + height: 16px; + mask-repeat: no-repeat; + mask-size: contain; + mask-position: center; + background-color: white; // Same on both themes + border-radius: 7px; + } + + &.mx_VideoFeed_mic_muted::before { + mask-image: url('$(res)/img/voip/mic-muted.svg'); + } + + &.mx_VideoFeed_mic_unmuted::before { + mask-image: url('$(res)/img/voip/mic-unmuted.svg'); + } } } - -.mx_VideoFeed_mirror { - transform: scale(-1, 1); -} diff --git a/src/components/views/voip/VideoFeed.tsx b/src/components/views/voip/VideoFeed.tsx index af2fd92016..09d0c97a0d 100644 --- a/src/components/views/voip/VideoFeed.tsx +++ b/src/components/views/voip/VideoFeed.tsx @@ -22,7 +22,7 @@ import { CallFeed, CallFeedEvent } from 'matrix-js-sdk/src/webrtc/callFeed'; import { logger } from 'matrix-js-sdk/src/logger'; import MemberAvatar from "../avatars/MemberAvatar"; import { replaceableComponent } from "../../../utils/replaceableComponent"; -import { objectHasDiff } from '../../../utils/objects'; +import { SDPStreamMetadataPurpose } from 'matrix-js-sdk/src/webrtc/callEventTypes'; interface IProps { call: MatrixCall; @@ -165,39 +165,62 @@ export default class VideoFeed extends React.PureComponent { }; render() { - const videoClasses = { - mx_VideoFeed: true, + const { pipMode, primary, feed } = this.props; + + const wrapperClasses = classnames("mx_VideoFeed", { mx_VideoFeed_voice: this.state.videoMuted, - mx_VideoFeed_video: !this.state.videoMuted, - mx_VideoFeed_mirror: ( - this.props.feed.isLocal() && - SettingsStore.getValue('VideoView.flipVideoHorizontally') - ), - }; + }); + const micIconClasses = classnames("mx_VideoFeed_mic", { + mx_VideoFeed_mic_muted: this.state.audioMuted, + mx_VideoFeed_mic_unmuted: !this.state.audioMuted, + }); - const { pipMode, primary } = this.props; + let micIcon; + if ( + feed.purpose !== SDPStreamMetadataPurpose.Screenshare && + !pipMode && + !feed.isLocal() + ) { + micIcon = ( +
+ ); + } + let content; if (this.state.videoMuted) { const member = this.props.feed.getMember(); + let avatarSize; if (pipMode && primary) avatarSize = 76; else if (pipMode && !primary) avatarSize = 16; else if (!pipMode && primary) avatarSize = 160; else; // TBD - return ( -
- -
+ content =( + ); } else { - return ( -