Prevent Element appearing in system media controls (#10995)
* Use WebAudio API to play notification sound So that it won't appear in system media control. * Run prettier * Chosse from mp3 and ogg * Run prettier * Use WebAudioAPI everywhere There's still one remoteAudio. I'm not sure what it does. It seems it's only used in tests... * Run prettier * Eliminate a stupid error * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Update setupManualMocks.ts * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * delint Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * mocks Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * mocks Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Simplify Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * covg Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --------- Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Co-authored-by: Michael Telatynski <7t3chguy@gmail.com>
This commit is contained in:
SuperKenVery
GitHub
parent
c61eca8c24
commit
e288f61f0a
12 changed files with 255 additions and 164 deletions
|
|
@ -66,6 +66,7 @@ import { localNotificationsAreSilenced } from "./utils/notifications";
|
|||
import { SdkContextClass } from "./contexts/SDKContext";
|
||||
import { showCantStartACallDialog } from "./voice-broadcast/utils/showCantStartACallDialog";
|
||||
import { isNotNull } from "./Typeguards";
|
||||
import { BackgroundAudio } from "./audio/BackgroundAudio";
|
||||
|
||||
export const PROTOCOL_PSTN = "m.protocol.pstn";
|
||||
export const PROTOCOL_PSTN_PREFIXED = "im.vector.protocol.pstn";
|
||||
|
|
@ -157,8 +158,6 @@ export default class LegacyCallHandler extends EventEmitter {
|
|||
// Calls started as an attended transfer, ie. with the intention of transferring another
|
||||
// call with a different party to this one.
|
||||
private transferees = new Map<string, MatrixCall>(); // callId (target) -> call (transferee)
|
||||
private audioPromises = new Map<AudioID, Promise<void>>();
|
||||
private audioElementsWithListeners = new Map<HTMLMediaElement, boolean>();
|
||||
private supportsPstnProtocol: boolean | null = null;
|
||||
private pstnSupportPrefixed: boolean | null = null; // True if the server only support the prefixed pstn protocol
|
||||
private supportsSipNativeVirtual: boolean | null = null; // im.vector.protocol.sip_virtual and im.vector.protocol.sip_native
|
||||
|
|
@ -170,6 +169,9 @@ export default class LegacyCallHandler extends EventEmitter {
|
|||
|
||||
private silencedCalls = new Set<string>(); // callIds
|
||||
|
||||
private backgroundAudio = new BackgroundAudio();
|
||||
private playingSources: Record<string, AudioBufferSourceNode> = {}; // Record them for stopping
|
||||
|
||||
public static get instance(): LegacyCallHandler {
|
||||
if (!window.mxLegacyCallHandler) {
|
||||
window.mxLegacyCallHandler = new LegacyCallHandler();
|
||||
|
|
@ -199,33 +201,11 @@ export default class LegacyCallHandler extends EventEmitter {
|
|||
}
|
||||
|
||||
public start(): void {
|
||||
// add empty handlers for media actions, otherwise the media keys
|
||||
// end up causing the audio elements with our ring/ringback etc
|
||||
// audio clips in to play.
|
||||
if (navigator.mediaSession) {
|
||||
navigator.mediaSession.setActionHandler("play", function () {});
|
||||
navigator.mediaSession.setActionHandler("pause", function () {});
|
||||
navigator.mediaSession.setActionHandler("seekbackward", function () {});
|
||||
navigator.mediaSession.setActionHandler("seekforward", function () {});
|
||||
navigator.mediaSession.setActionHandler("previoustrack", function () {});
|
||||
navigator.mediaSession.setActionHandler("nexttrack", function () {});
|
||||
}
|
||||
|
||||
if (SettingsStore.getValue(UIFeature.Voip)) {
|
||||
MatrixClientPeg.safeGet().on(CallEventHandlerEvent.Incoming, this.onCallIncoming);
|
||||
}
|
||||
|
||||
this.checkProtocols(CHECK_PROTOCOLS_ATTEMPTS);
|
||||
|
||||
// Add event listeners for the <audio> elements
|
||||
Object.values(AudioID).forEach((audioId) => {
|
||||
const audioElement = document.getElementById(audioId) as HTMLMediaElement;
|
||||
if (audioElement) {
|
||||
this.addEventListenersForAudioElement(audioElement);
|
||||
} else {
|
||||
logger.warn(`LegacyCallHandler: missing <audio id="${audioId}"> from page`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public stop(): void {
|
||||
|
|
@ -233,27 +213,6 @@ export default class LegacyCallHandler extends EventEmitter {
|
|||
if (cli) {
|
||||
cli.removeListener(CallEventHandlerEvent.Incoming, this.onCallIncoming);
|
||||
}
|
||||
|
||||
// Remove event listeners for the <audio> elements
|
||||
Array.from(this.audioElementsWithListeners.keys()).forEach((audioElement) => {
|
||||
this.removeEventListenersForAudioElement(audioElement);
|
||||
});
|
||||
}
|
||||
|
||||
private addEventListenersForAudioElement(audioElement: HTMLMediaElement): void {
|
||||
// Only need to setup the listeners once
|
||||
if (!this.audioElementsWithListeners.get(audioElement)) {
|
||||
MEDIA_EVENT_TYPES.forEach((errorEventType) => {
|
||||
audioElement.addEventListener(errorEventType, this);
|
||||
this.audioElementsWithListeners.set(audioElement, true);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private removeEventListenersForAudioElement(audioElement: HTMLMediaElement): void {
|
||||
MEDIA_EVENT_TYPES.forEach((errorEventType) => {
|
||||
audioElement.removeEventListener(errorEventType, this);
|
||||
});
|
||||
}
|
||||
|
||||
/* istanbul ignore next (remove if we start using this function for things other than debug logging) */
|
||||
|
|
@ -465,74 +424,37 @@ export default class LegacyCallHandler extends EventEmitter {
|
|||
return this.transferees.get(callId);
|
||||
}
|
||||
|
||||
public play(audioId: AudioID): void {
|
||||
public async play(audioId: AudioID): Promise<void> {
|
||||
const logPrefix = `LegacyCallHandler.play(${audioId}):`;
|
||||
logger.debug(`${logPrefix} beginning of function`);
|
||||
// TODO: Attach an invisible element for this instead
|
||||
// which listens?
|
||||
const audio = document.getElementById(audioId) as HTMLMediaElement;
|
||||
if (audio) {
|
||||
this.addEventListenersForAudioElement(audio);
|
||||
const playAudio = async (): Promise<void> => {
|
||||
try {
|
||||
if (audio.muted) {
|
||||
logger.error(
|
||||
`${logPrefix} <audio> element was unexpectedly muted but we recovered ` +
|
||||
`gracefully by unmuting it`,
|
||||
);
|
||||
// Recover gracefully
|
||||
audio.muted = false;
|
||||
}
|
||||
|
||||
// This still causes the chrome debugger to break on promise rejection if
|
||||
// the promise is rejected, even though we're catching the exception.
|
||||
logger.debug(`${logPrefix} attempting to play audio at volume=${audio.volume}`);
|
||||
await audio.play();
|
||||
logger.debug(`${logPrefix} playing audio successfully`);
|
||||
} catch (e) {
|
||||
// This is usually because the user hasn't interacted with the document,
|
||||
// or chrome doesn't think so and is denying the request. Not sure what
|
||||
// we can really do here...
|
||||
// https://github.com/vector-im/element-web/issues/7657
|
||||
logger.warn(`${logPrefix} unable to play audio clip`, e);
|
||||
}
|
||||
};
|
||||
if (this.audioPromises.has(audioId)) {
|
||||
this.audioPromises.set(
|
||||
audioId,
|
||||
this.audioPromises.get(audioId)!.then(() => {
|
||||
audio.load();
|
||||
return playAudio();
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
this.audioPromises.set(audioId, playAudio());
|
||||
}
|
||||
} else {
|
||||
logger.warn(`${logPrefix} unable to find <audio> element for ${audioId}`);
|
||||
}
|
||||
const audioInfo: Record<AudioID, [prefix: string, loop: boolean]> = {
|
||||
[AudioID.Ring]: [`./media/ring`, true],
|
||||
[AudioID.Ringback]: [`./media/ringback`, true],
|
||||
[AudioID.CallEnd]: [`./media/callend`, false],
|
||||
[AudioID.Busy]: [`./media/busy`, false],
|
||||
};
|
||||
|
||||
const [urlPrefix, loop] = audioInfo[audioId];
|
||||
const source = await this.backgroundAudio.pickFormatAndPlay(urlPrefix, ["mp3", "ogg"], loop);
|
||||
this.playingSources[audioId] = source;
|
||||
logger.debug(`${logPrefix} playing audio successfully`);
|
||||
}
|
||||
|
||||
public pause(audioId: AudioID): void {
|
||||
const logPrefix = `LegacyCallHandler.pause(${audioId}):`;
|
||||
logger.debug(`${logPrefix} beginning of function`);
|
||||
// TODO: Attach an invisible element for this instead
|
||||
// which listens?
|
||||
const audio = document.getElementById(audioId) as HTMLMediaElement;
|
||||
const pauseAudio = (): void => {
|
||||
logger.debug(`${logPrefix} pausing audio`);
|
||||
// pause doesn't return a promise, so just do it
|
||||
audio.pause();
|
||||
};
|
||||
if (audio) {
|
||||
if (this.audioPromises.has(audioId)) {
|
||||
this.audioPromises.set(audioId, this.audioPromises.get(audioId)!.then(pauseAudio));
|
||||
} else {
|
||||
pauseAudio();
|
||||
}
|
||||
} else {
|
||||
logger.warn(`${logPrefix} unable to find <audio> element for ${audioId}`);
|
||||
|
||||
const source = this.playingSources[audioId];
|
||||
if (!source) {
|
||||
logger.debug(`${logPrefix} audio not playing`);
|
||||
return;
|
||||
}
|
||||
|
||||
source.stop();
|
||||
delete this.playingSources[audioId];
|
||||
|
||||
logger.debug(`${logPrefix} paused audio`);
|
||||
}
|
||||
|
||||
private matchesCallForThisRoom(call: MatrixCall): boolean {
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ import ToastStore from "./stores/ToastStore";
|
|||
import { VoiceBroadcastChunkEventType, VoiceBroadcastInfoEventType } from "./voice-broadcast";
|
||||
import { getSenderName } from "./utils/event/getSenderName";
|
||||
import { stripPlainReply } from "./utils/Reply";
|
||||
import { BackgroundAudio } from "./audio/BackgroundAudio";
|
||||
|
||||
/*
|
||||
* Dispatches:
|
||||
|
|
@ -112,6 +113,8 @@ class NotifierClass {
|
|||
private toolbarHidden?: boolean;
|
||||
private isSyncing?: boolean;
|
||||
|
||||
private backgroundAudio = new BackgroundAudio();
|
||||
|
||||
public notificationMessageForEvent(ev: MatrixEvent): string | null {
|
||||
const msgType = ev.getContent().msgtype;
|
||||
if (msgType && msgTypeHandlers.hasOwnProperty(msgType)) {
|
||||
|
|
@ -226,28 +229,14 @@ class NotifierClass {
|
|||
return;
|
||||
}
|
||||
|
||||
// Play notification sound here
|
||||
const sound = this.getSoundForRoom(room.roomId);
|
||||
logger.log(`Got sound ${(sound && sound.name) || "default"} for ${room.roomId}`);
|
||||
|
||||
try {
|
||||
const selector = document.querySelector<HTMLAudioElement>(
|
||||
sound ? `audio[src='${sound.url}']` : "#messageAudio",
|
||||
);
|
||||
let audioElement = selector;
|
||||
if (!audioElement) {
|
||||
if (!sound) {
|
||||
logger.error("No audio element or sound to play for notification");
|
||||
return;
|
||||
}
|
||||
audioElement = new Audio(sound.url);
|
||||
if (sound.type) {
|
||||
audioElement.type = sound.type;
|
||||
}
|
||||
document.body.appendChild(audioElement);
|
||||
}
|
||||
await audioElement.play();
|
||||
} catch (ex) {
|
||||
logger.warn("Caught error when trying to fetch room notification sound:", ex);
|
||||
if (sound) {
|
||||
await this.backgroundAudio.play(sound.url);
|
||||
} else {
|
||||
await this.backgroundAudio.pickFormatAndPlay("media/message", ["mp3", "ogg"]);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
74
src/audio/BackgroundAudio.ts
Normal file
74
src/audio/BackgroundAudio.ts
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
/*
|
||||
Copyright 2024 The Matrix.org Foundation C.I.C.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
import { logger } from "matrix-js-sdk/src/logger";
|
||||
|
||||
import { createAudioContext } from "./compat";
|
||||
|
||||
const formatMap = {
|
||||
mp3: "audio/mpeg",
|
||||
ogg: "audio/ogg",
|
||||
};
|
||||
|
||||
export class BackgroundAudio {
|
||||
private audioContext = createAudioContext();
|
||||
private sounds: Record<string, AudioBuffer> = {};
|
||||
|
||||
public async pickFormatAndPlay<F extends Array<keyof typeof formatMap>>(
|
||||
urlPrefix: string,
|
||||
formats: F,
|
||||
loop = false,
|
||||
): Promise<AudioBufferSourceNode> {
|
||||
const format = this.pickFormat(...formats);
|
||||
if (!format) {
|
||||
console.log("Browser doesn't support any of the formats", formats);
|
||||
// Will probably never happen. If happened, format="" and will fail to load audio. Who cares...
|
||||
}
|
||||
|
||||
return this.play(`${urlPrefix}.${format}`, loop);
|
||||
}
|
||||
|
||||
public async play(url: string, loop = false): Promise<AudioBufferSourceNode> {
|
||||
if (!this.sounds.hasOwnProperty(url)) {
|
||||
// No cache, fetch it
|
||||
const response = await fetch(url);
|
||||
if (response.status != 200) {
|
||||
logger.warn("Failed to fetch error audio");
|
||||
}
|
||||
const buffer = await response.arrayBuffer();
|
||||
const sound = await this.audioContext.decodeAudioData(buffer);
|
||||
this.sounds[url] = sound;
|
||||
}
|
||||
const source = this.audioContext.createBufferSource();
|
||||
source.buffer = this.sounds[url];
|
||||
source.loop = loop;
|
||||
source.connect(this.audioContext.destination);
|
||||
source.start();
|
||||
return source;
|
||||
}
|
||||
|
||||
private pickFormat<F extends Array<keyof typeof formatMap>>(...formats: F): F[number] | null {
|
||||
// Detect supported formats
|
||||
const audioElement = document.createElement("audio");
|
||||
|
||||
for (const format of formats) {
|
||||
if (audioElement.canPlayType(formatMap[format])) {
|
||||
return format;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -47,6 +47,7 @@ import { VoiceBroadcastChunkEvents } from "../utils/VoiceBroadcastChunkEvents";
|
|||
import { RelationsHelper, RelationsHelperEvent } from "../../events/RelationsHelper";
|
||||
import { createReconnectedListener } from "../../utils/connection";
|
||||
import { localNotificationsAreSilenced } from "../../utils/notifications";
|
||||
import { BackgroundAudio } from "../../audio/BackgroundAudio";
|
||||
|
||||
export enum VoiceBroadcastRecordingEvent {
|
||||
StateChanged = "liveness_changed",
|
||||
|
|
@ -75,6 +76,7 @@ export class VoiceBroadcastRecording
|
|||
private reconnectedListener: ClientEventHandlerMap[ClientEvent.Sync];
|
||||
private roomId: string;
|
||||
private infoEventId: string;
|
||||
private backgroundAudio = new BackgroundAudio();
|
||||
|
||||
/**
|
||||
* Broadcast chunks have a sequence number to bring them in the correct order and to know if a message is missing.
|
||||
|
|
@ -346,15 +348,7 @@ export class VoiceBroadcastRecording
|
|||
return;
|
||||
}
|
||||
|
||||
// Audio files are added to the document in Element Web.
|
||||
// See <audio> elements in https://github.com/vector-im/element-web/blob/develop/src/vector/index.html
|
||||
const audioElement = document.querySelector<HTMLAudioElement>("audio#errorAudio");
|
||||
|
||||
try {
|
||||
await audioElement?.play();
|
||||
} catch (e) {
|
||||
logger.warn("error playing 'errorAudio'", e);
|
||||
}
|
||||
await this.backgroundAudio.pickFormatAndPlay("./media/error", ["mp3", "ogg"]);
|
||||
}
|
||||
|
||||
private async uploadFile(chunk: ChunkRecordedPayload): ReturnType<typeof uploadFile> {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue