Redesign the picture-in-picture window (#9800)

* Remove unnecessary PipContainer component

* Redesign the picture-in-picture window

* Add a hover effect to the controls

* Clarify that WidgetPip has call-specific behavior
This commit is contained in:
Robin 2023-01-03 23:44:38 -05:00 committed by GitHub
parent 6b155620e4
commit cb1af0d3de
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 491 additions and 328 deletions

View file

@ -41,7 +41,6 @@ import { DefaultTagID } from "../../stores/room-list/models";
import { hideToast as hideServerLimitToast, showToast as showServerLimitToast } from "../../toasts/ServerLimitToast";
import { Action } from "../../dispatcher/actions";
import LeftPanel from "./LeftPanel";
import PipContainer from "../views/voip/PipContainer";
import { ViewRoomDeltaPayload } from "../../dispatcher/payloads/ViewRoomDeltaPayload";
import RoomListStore from "../../stores/room-list/RoomListStore";
import NonUrgentToastContainer from "./NonUrgentToastContainer";
@ -71,6 +70,7 @@ import { SwitchSpacePayload } from "../../dispatcher/payloads/SwitchSpacePayload
import { IConfigOptions } from "../../IConfigOptions";
import LeftPanelLiveShareWarning from "../views/beacon/LeftPanelLiveShareWarning";
import { UserOnboardingPage } from "../views/user-onboarding/UserOnboardingPage";
import { PipContainer } from "./PipContainer";
// We need to fetch each pinned message individually (if we don't already have it)
// so each pinned message may trigger a request. Limit the number per room for sanity.

View file

@ -0,0 +1,259 @@
/*
Copyright 2021-2022 New Vector Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React, { createRef } from "react";
import UIStore, { UI_EVENTS } from "../../stores/UIStore";
import { lerp } from "../../utils/AnimationUtils";
import { MarkedExecution } from "../../utils/MarkedExecution";
const PIP_VIEW_WIDTH = 336;
const PIP_VIEW_HEIGHT = 232;
const MOVING_AMT = 0.2;
const SNAPPING_AMT = 0.1;
const PADDING = {
top: 58,
bottom: 58,
left: 76,
right: 8,
};
/**
* The type of a callback which will create the pip content children.
*/
export type CreatePipChildren = (options: IChildrenOptions) => JSX.Element;
interface IChildrenOptions {
// a callback which is called when a mouse event (most likely mouse down) occurs at start of moving the pip around
onStartMoving: (event: React.MouseEvent<Element, MouseEvent>) => void;
// a callback which is called when the content fo the pip changes in a way that is likely to cause a resize
onResize: (event: Event) => void;
}
interface IProps {
className?: string;
children: Array<CreatePipChildren>;
draggable: boolean;
onDoubleClick?: () => void;
onMove?: () => void;
}
/**
* PictureInPictureDragger shows a small version of CallView hovering over the UI in 'picture-in-picture'
* (PiP mode). It displays the call(s) which is *not* in the room the user is currently viewing.
*/
export default class PictureInPictureDragger extends React.Component<IProps> {
private callViewWrapper = createRef<HTMLDivElement>();
private initX = 0;
private initY = 0;
private desiredTranslationX = UIStore.instance.windowWidth - PADDING.right - PIP_VIEW_WIDTH;
private desiredTranslationY = UIStore.instance.windowHeight - PADDING.bottom - PIP_VIEW_HEIGHT;
private translationX = this.desiredTranslationX;
private translationY = this.desiredTranslationY;
private mouseHeld = false;
private scheduledUpdate: MarkedExecution = new MarkedExecution(
() => this.animationCallback(),
() => requestAnimationFrame(() => this.scheduledUpdate.trigger()),
);
private _moving = false;
public get moving(): boolean {
return this._moving;
}
private set moving(value: boolean) {
this._moving = value;
}
public componentDidMount() {
document.addEventListener("mousemove", this.onMoving);
document.addEventListener("mouseup", this.onEndMoving);
UIStore.instance.on(UI_EVENTS.Resize, this.onResize);
// correctly position the PiP
this.snap();
}
public componentWillUnmount() {
document.removeEventListener("mousemove", this.onMoving);
document.removeEventListener("mouseup", this.onEndMoving);
UIStore.instance.off(UI_EVENTS.Resize, this.onResize);
}
public componentDidUpdate(prevProps: Readonly<IProps>): void {
if (prevProps.children !== this.props.children) this.snap(true);
}
private animationCallback = () => {
if (
!this.moving &&
Math.abs(this.translationX - this.desiredTranslationX) <= 1 &&
Math.abs(this.translationY - this.desiredTranslationY) <= 1
) {
// Break the loop by settling the element into its final position
this.translationX = this.desiredTranslationX;
this.translationY = this.desiredTranslationY;
this.setStyle();
} else {
const amt = this.moving ? MOVING_AMT : SNAPPING_AMT;
this.translationX = lerp(this.translationX, this.desiredTranslationX, amt);
this.translationY = lerp(this.translationY, this.desiredTranslationY, amt);
this.setStyle();
this.scheduledUpdate.mark();
}
this.props.onMove?.();
};
private setStyle = () => {
if (!this.callViewWrapper.current) return;
// Set the element's style directly, bypassing React for efficiency
this.callViewWrapper.current.style.transform = `translateX(${this.translationX}px) translateY(${this.translationY}px)`;
};
private setTranslation(inTranslationX: number, inTranslationY: number) {
const width = this.callViewWrapper.current?.clientWidth || PIP_VIEW_WIDTH;
const height = this.callViewWrapper.current?.clientHeight || PIP_VIEW_HEIGHT;
// Avoid overflow on the x axis
if (inTranslationX + width >= UIStore.instance.windowWidth) {
this.desiredTranslationX = UIStore.instance.windowWidth - width;
} else if (inTranslationX <= 0) {
this.desiredTranslationX = 0;
} else {
this.desiredTranslationX = inTranslationX;
}
// Avoid overflow on the y axis
if (inTranslationY + height >= UIStore.instance.windowHeight) {
this.desiredTranslationY = UIStore.instance.windowHeight - height;
} else if (inTranslationY <= 0) {
this.desiredTranslationY = 0;
} else {
this.desiredTranslationY = inTranslationY;
}
}
private onResize = (): void => {
this.snap(false);
};
private snap = (animate = false) => {
const translationX = this.desiredTranslationX;
const translationY = this.desiredTranslationY;
// We subtract the PiP size from the window size in order to calculate
// the position to snap to from the PiP center and not its top-left
// corner
const windowWidth =
UIStore.instance.windowWidth - (this.callViewWrapper.current?.clientWidth || PIP_VIEW_WIDTH);
const windowHeight =
UIStore.instance.windowHeight - (this.callViewWrapper.current?.clientHeight || PIP_VIEW_HEIGHT);
if (translationX >= windowWidth / 2 && translationY >= windowHeight / 2) {
this.desiredTranslationX = windowWidth - PADDING.right;
this.desiredTranslationY = windowHeight - PADDING.bottom;
} else if (translationX >= windowWidth / 2 && translationY <= windowHeight / 2) {
this.desiredTranslationX = windowWidth - PADDING.right;
this.desiredTranslationY = PADDING.top;
} else if (translationX <= windowWidth / 2 && translationY >= windowHeight / 2) {
this.desiredTranslationX = PADDING.left;
this.desiredTranslationY = windowHeight - PADDING.bottom;
} else {
this.desiredTranslationX = PADDING.left;
this.desiredTranslationY = PADDING.top;
}
if (!animate) {
this.translationX = this.desiredTranslationX;
this.translationY = this.desiredTranslationY;
}
// We start animating here because we want the PiP to move when we're
// resizing the window
this.scheduledUpdate.mark();
};
private onStartMoving = (event: React.MouseEvent | MouseEvent) => {
event.preventDefault();
event.stopPropagation();
this.mouseHeld = true;
};
private onMoving = (event: MouseEvent) => {
if (!this.mouseHeld) return;
event.preventDefault();
event.stopPropagation();
if (!this.moving) {
this.moving = true;
this.initX = event.pageX - this.desiredTranslationX;
this.initY = event.pageY - this.desiredTranslationY;
this.scheduledUpdate.mark();
}
this.setTranslation(event.pageX - this.initX, event.pageY - this.initY);
};
private onEndMoving = (event: MouseEvent) => {
if (!this.mouseHeld) return;
event.preventDefault();
event.stopPropagation();
this.mouseHeld = false;
// Delaying this to the next event loop tick is necessary for click
// event cancellation to work
setImmediate(() => (this.moving = false));
this.snap(true);
};
private onClickCapture = (event: React.MouseEvent) => {
// To prevent mouse up events during dragging from being double-counted
// as clicks, we cancel clicks before they ever reach the target
if (this.moving) {
event.preventDefault();
event.stopPropagation();
}
};
public render() {
const style = {
transform: `translateX(${this.translationX}px) translateY(${this.translationY}px)`,
};
const children = this.props.children.map((create: CreatePipChildren) => {
return create({
onStartMoving: this.onStartMoving,
onResize: this.onResize,
});
});
return (
<aside
className={this.props.className}
style={style}
ref={this.callViewWrapper}
onClickCapture={this.onClickCapture}
onDoubleClick={this.props.onDoubleClick}
>
{children}
</aside>
);
}
}

View file

@ -0,0 +1,372 @@
/*
Copyright 2017 - 2022 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React, { MutableRefObject, useContext, useRef } from "react";
import { CallEvent, CallState, MatrixCall } from "matrix-js-sdk/src/webrtc/call";
import { logger } from "matrix-js-sdk/src/logger";
import { Optional } from "matrix-events-sdk";
import LegacyCallView from "../views/voip/LegacyCallView";
import LegacyCallHandler, { LegacyCallHandlerEvent } from "../../LegacyCallHandler";
import { MatrixClientPeg } from "../../MatrixClientPeg";
import PictureInPictureDragger, { CreatePipChildren } from "./PictureInPictureDragger";
import dis from "../../dispatcher/dispatcher";
import { Action } from "../../dispatcher/actions";
import { WidgetLayoutStore } from "../../stores/widgets/WidgetLayoutStore";
import ActiveWidgetStore, { ActiveWidgetStoreEvent } from "../../stores/ActiveWidgetStore";
import { ViewRoomPayload } from "../../dispatcher/payloads/ViewRoomPayload";
import { UPDATE_EVENT } from "../../stores/AsyncStore";
import { SDKContext, SdkContextClass } from "../../contexts/SDKContext";
import {
useCurrentVoiceBroadcastPreRecording,
useCurrentVoiceBroadcastRecording,
VoiceBroadcastPlayback,
VoiceBroadcastPlaybackBody,
VoiceBroadcastPreRecording,
VoiceBroadcastPreRecordingPip,
VoiceBroadcastRecording,
VoiceBroadcastRecordingPip,
VoiceBroadcastSmallPlaybackBody,
} from "../../voice-broadcast";
import { useCurrentVoiceBroadcastPlayback } from "../../voice-broadcast/hooks/useCurrentVoiceBroadcastPlayback";
import { WidgetPip } from "../views/pips/WidgetPip";
const SHOW_CALL_IN_STATES = [
CallState.Connected,
CallState.InviteSent,
CallState.Connecting,
CallState.CreateAnswer,
CallState.CreateOffer,
CallState.WaitLocalMedia,
];
interface IProps {
voiceBroadcastRecording: Optional<VoiceBroadcastRecording>;
voiceBroadcastPreRecording: Optional<VoiceBroadcastPreRecording>;
voiceBroadcastPlayback: Optional<VoiceBroadcastPlayback>;
movePersistedElement: MutableRefObject<(() => void) | undefined>;
}
interface IState {
viewedRoomId?: string;
// The main call that we are displaying (ie. not including the call in the room being viewed, if any)
primaryCall: MatrixCall | null;
// Any other call we're displaying: only if the user is on two calls and not viewing either of the rooms
// they belong to
secondaryCall: MatrixCall;
// widget candidate to be displayed in the pip view.
persistentWidgetId: string;
persistentRoomId: string;
showWidgetInPip: boolean;
}
// Splits a list of calls into one 'primary' one and a list
// (which should be a single element) of other calls.
// The primary will be the one not on hold, or an arbitrary one
// if they're all on hold)
function getPrimarySecondaryCallsForPip(roomId: Optional<string>): [MatrixCall | null, MatrixCall[]] {
if (!roomId) return [null, []];
const calls = LegacyCallHandler.instance.getAllActiveCallsForPip(roomId);
let primary: MatrixCall | null = null;
let secondaries: MatrixCall[] = [];
for (const call of calls) {
if (!SHOW_CALL_IN_STATES.includes(call.state)) continue;
if (!call.isRemoteOnHold() && primary === null) {
primary = call;
} else {
secondaries.push(call);
}
}
if (primary === null && secondaries.length > 0) {
primary = secondaries[0];
secondaries = secondaries.slice(1);
}
if (secondaries.length > 1) {
// We should never be in more than two calls so this shouldn't happen
logger.log("Found more than 1 secondary call! Other calls will not be shown.");
}
return [primary, secondaries];
}
/**
* PipContainer shows a small version of the LegacyCallView or a sticky widget hovering over the UI in
* 'picture-in-picture' (PiP mode). It displays the call(s) which is *not* in the room the user is currently viewing
* and all widgets that are active but not shown in any other possible container.
*/
class PipContainerInner extends React.Component<IProps, IState> {
public constructor(props: IProps) {
super(props);
const roomId = SdkContextClass.instance.roomViewStore.getRoomId();
const [primaryCall, secondaryCalls] = getPrimarySecondaryCallsForPip(roomId);
this.state = {
viewedRoomId: roomId || undefined,
primaryCall: primaryCall || null,
secondaryCall: secondaryCalls[0],
persistentWidgetId: ActiveWidgetStore.instance.getPersistentWidgetId(),
persistentRoomId: ActiveWidgetStore.instance.getPersistentRoomId(),
showWidgetInPip: false,
};
}
public componentDidMount() {
LegacyCallHandler.instance.addListener(LegacyCallHandlerEvent.CallChangeRoom, this.updateCalls);
LegacyCallHandler.instance.addListener(LegacyCallHandlerEvent.CallState, this.updateCalls);
SdkContextClass.instance.roomViewStore.addListener(UPDATE_EVENT, this.onRoomViewStoreUpdate);
MatrixClientPeg.get().on(CallEvent.RemoteHoldUnhold, this.onCallRemoteHold);
const room = MatrixClientPeg.get()?.getRoom(this.state.viewedRoomId);
if (room) {
WidgetLayoutStore.instance.on(WidgetLayoutStore.emissionForRoom(room), this.updateCalls);
}
ActiveWidgetStore.instance.on(ActiveWidgetStoreEvent.Persistence, this.onWidgetPersistence);
ActiveWidgetStore.instance.on(ActiveWidgetStoreEvent.Dock, this.onWidgetDockChanges);
ActiveWidgetStore.instance.on(ActiveWidgetStoreEvent.Undock, this.onWidgetDockChanges);
}
public componentWillUnmount() {
LegacyCallHandler.instance.removeListener(LegacyCallHandlerEvent.CallChangeRoom, this.updateCalls);
LegacyCallHandler.instance.removeListener(LegacyCallHandlerEvent.CallState, this.updateCalls);
const cli = MatrixClientPeg.get();
cli?.removeListener(CallEvent.RemoteHoldUnhold, this.onCallRemoteHold);
SdkContextClass.instance.roomViewStore.removeListener(UPDATE_EVENT, this.onRoomViewStoreUpdate);
const room = cli?.getRoom(this.state.viewedRoomId);
if (room) {
WidgetLayoutStore.instance.off(WidgetLayoutStore.emissionForRoom(room), this.updateCalls);
}
ActiveWidgetStore.instance.off(ActiveWidgetStoreEvent.Persistence, this.onWidgetPersistence);
ActiveWidgetStore.instance.off(ActiveWidgetStoreEvent.Dock, this.onWidgetDockChanges);
ActiveWidgetStore.instance.off(ActiveWidgetStoreEvent.Undock, this.onWidgetDockChanges);
}
private onMove = () => this.props.movePersistedElement.current?.();
private onRoomViewStoreUpdate = () => {
const newRoomId = SdkContextClass.instance.roomViewStore.getRoomId();
const oldRoomId = this.state.viewedRoomId;
if (newRoomId === oldRoomId) return;
// The WidgetLayoutStore observer always tracks the currently viewed Room,
// so we don't end up with multiple observers and know what observer to remove on unmount
const oldRoom = MatrixClientPeg.get()?.getRoom(oldRoomId);
if (oldRoom) {
WidgetLayoutStore.instance.off(WidgetLayoutStore.emissionForRoom(oldRoom), this.updateCalls);
}
const newRoom = MatrixClientPeg.get()?.getRoom(newRoomId || undefined);
if (newRoom) {
WidgetLayoutStore.instance.on(WidgetLayoutStore.emissionForRoom(newRoom), this.updateCalls);
}
if (!newRoomId) return;
const [primaryCall, secondaryCalls] = getPrimarySecondaryCallsForPip(newRoomId);
this.setState({
viewedRoomId: newRoomId,
primaryCall: primaryCall,
secondaryCall: secondaryCalls[0],
});
this.updateShowWidgetInPip();
};
private onWidgetPersistence = (): void => {
this.updateShowWidgetInPip(
ActiveWidgetStore.instance.getPersistentWidgetId(),
ActiveWidgetStore.instance.getPersistentRoomId(),
);
};
private onWidgetDockChanges = (): void => {
this.updateShowWidgetInPip();
};
private updateCalls = (): void => {
if (!this.state.viewedRoomId) return;
const [primaryCall, secondaryCalls] = getPrimarySecondaryCallsForPip(this.state.viewedRoomId);
this.setState({
primaryCall: primaryCall,
secondaryCall: secondaryCalls[0],
});
this.updateShowWidgetInPip();
};
private onCallRemoteHold = () => {
if (!this.state.viewedRoomId) return;
const [primaryCall, secondaryCalls] = getPrimarySecondaryCallsForPip(this.state.viewedRoomId);
this.setState({
primaryCall: primaryCall,
secondaryCall: secondaryCalls[0],
});
};
private onDoubleClick = (): void => {
const callRoomId = this.state.primaryCall?.roomId;
if (callRoomId ?? this.state.persistentRoomId) {
dis.dispatch<ViewRoomPayload>({
action: Action.ViewRoom,
room_id: callRoomId ?? this.state.persistentRoomId,
metricsTrigger: "WebFloatingCallWindow",
});
}
};
// Accepts a persistentWidgetId to be able to skip awaiting the setState for persistentWidgetId
public updateShowWidgetInPip(
persistentWidgetId = this.state.persistentWidgetId,
persistentRoomId = this.state.persistentRoomId,
) {
let fromAnotherRoom = false;
let notDocked = false;
// Sanity check the room - the widget may have been destroyed between render cycles, and
// thus no room is associated anymore.
if (persistentWidgetId && MatrixClientPeg.get().getRoom(persistentRoomId)) {
notDocked = !ActiveWidgetStore.instance.isDocked(persistentWidgetId, persistentRoomId);
fromAnotherRoom = this.state.viewedRoomId !== persistentRoomId;
}
// The widget should only be shown as a persistent app (in a floating
// pip container) if it is not visible on screen: either because we are
// viewing a different room OR because it is in none of the possible
// containers of the room view.
const showWidgetInPip = fromAnotherRoom || notDocked;
this.setState({ showWidgetInPip, persistentWidgetId, persistentRoomId });
}
private createVoiceBroadcastPlaybackPipContent(voiceBroadcastPlayback: VoiceBroadcastPlayback): CreatePipChildren {
if (this.state.viewedRoomId === voiceBroadcastPlayback.infoEvent.getRoomId()) {
return ({ onStartMoving }) => (
<div onMouseDown={onStartMoving}>
<VoiceBroadcastPlaybackBody playback={voiceBroadcastPlayback} pip={true} />
</div>
);
}
return ({ onStartMoving }) => (
<div onMouseDown={onStartMoving}>
<VoiceBroadcastSmallPlaybackBody playback={voiceBroadcastPlayback} />
</div>
);
}
private createVoiceBroadcastPreRecordingPipContent(
voiceBroadcastPreRecording: VoiceBroadcastPreRecording,
): CreatePipChildren {
return ({ onStartMoving }) => (
<div onMouseDown={onStartMoving}>
<VoiceBroadcastPreRecordingPip voiceBroadcastPreRecording={voiceBroadcastPreRecording} />
</div>
);
}
private createVoiceBroadcastRecordingPipContent(
voiceBroadcastRecording: VoiceBroadcastRecording,
): CreatePipChildren {
return ({ onStartMoving }) => (
<div onMouseDown={onStartMoving}>
<VoiceBroadcastRecordingPip recording={voiceBroadcastRecording} />
</div>
);
}
public render() {
const pipMode = true;
let pipContent: Array<CreatePipChildren> = [];
if (this.props.voiceBroadcastRecording) {
pipContent = [this.createVoiceBroadcastRecordingPipContent(this.props.voiceBroadcastRecording)];
} else if (this.props.voiceBroadcastPreRecording) {
pipContent = [this.createVoiceBroadcastPreRecordingPipContent(this.props.voiceBroadcastPreRecording)];
} else if (this.props.voiceBroadcastPlayback) {
pipContent = [this.createVoiceBroadcastPlaybackPipContent(this.props.voiceBroadcastPlayback)];
}
if (this.state.primaryCall) {
// get a ref to call inside the current scope
const call = this.state.primaryCall;
pipContent.push(({ onStartMoving, onResize }) => (
<LegacyCallView
onMouseDownOnHeader={onStartMoving}
call={call}
secondaryCall={this.state.secondaryCall}
pipMode={pipMode}
onResize={onResize}
/>
));
}
if (this.state.showWidgetInPip) {
pipContent.push(({ onStartMoving }) => (
<WidgetPip
widgetId={this.state.persistentWidgetId}
room={MatrixClientPeg.get().getRoom(this.state.persistentRoomId)!}
viewingRoom={this.state.viewedRoomId === this.state.persistentRoomId}
onStartMoving={onStartMoving}
movePersistedElement={this.props.movePersistedElement}
/>
));
}
if (pipContent.length) {
return (
<PictureInPictureDragger
className="mx_LegacyCallPreview"
draggable={pipMode}
onDoubleClick={this.onDoubleClick}
onMove={this.onMove}
>
{pipContent}
</PictureInPictureDragger>
);
}
return null;
}
}
export const PipContainer: React.FC = () => {
const sdkContext = useContext(SDKContext);
const voiceBroadcastPreRecordingStore = sdkContext.voiceBroadcastPreRecordingStore;
const { currentVoiceBroadcastPreRecording } = useCurrentVoiceBroadcastPreRecording(voiceBroadcastPreRecordingStore);
const voiceBroadcastRecordingsStore = sdkContext.voiceBroadcastRecordingsStore;
const { currentVoiceBroadcastRecording } = useCurrentVoiceBroadcastRecording(voiceBroadcastRecordingsStore);
const voiceBroadcastPlaybacksStore = sdkContext.voiceBroadcastPlaybacksStore;
const { currentVoiceBroadcastPlayback } = useCurrentVoiceBroadcastPlayback(voiceBroadcastPlaybacksStore);
const movePersistedElement = useRef<() => void>();
return (
<PipContainerInner
voiceBroadcastPlayback={currentVoiceBroadcastPlayback}
voiceBroadcastPreRecording={currentVoiceBroadcastPreRecording}
voiceBroadcastRecording={currentVoiceBroadcastRecording}
movePersistedElement={movePersistedElement}
/>
);
};