import { derived, get, Readable, readable, writable, Writable } from "svelte/store"; import { localUserStore } from "../Connexion/LocalUserStore"; import { userMovingStore } from "./GameStore"; import { HtmlUtils } from "../WebRtc/HtmlUtils"; import { BrowserTooOldError } from "./Errors/BrowserTooOldError"; import { errorStore } from "./ErrorStore"; import { isIOS } from "../WebRtc/DeviceUtils"; import { WebviewOnOldIOS } from "./Errors/WebviewOnOldIOS"; import { gameOverlayVisibilityStore } from "./GameOverlayStoreVisibility"; import { peerStore } from "./PeerStore"; import { privacyShutdownStore } from "./PrivacyShutdownStore"; /** * A store that contains the camera state requested by the user (on or off). */ function createRequestedCameraState() { const { subscribe, set, update } = writable(true); return { subscribe, enableWebcam: () => set(true), disableWebcam: () => set(false), }; } /** * A store that contains the microphone state requested by the user (on or off). */ function createRequestedMicrophoneState() { const { subscribe, set, update } = writable(true); return { subscribe, enableMicrophone: () => set(true), disableMicrophone: () => set(false), }; } /** * A store that contains whether the EnableCameraScene is shown or not. */ function createEnableCameraSceneVisibilityStore() { const { subscribe, set, update } = writable(false); return { subscribe, showEnableCameraScene: () => set(true), hideEnableCameraScene: () => set(false), }; } export const requestedCameraState = createRequestedCameraState(); export const requestedMicrophoneState = createRequestedMicrophoneState(); export const enableCameraSceneVisibilityStore = createEnableCameraSceneVisibilityStore(); /** * A store containing whether the webcam was enabled in the last 10 seconds */ const enabledWebCam10secondsAgoStore = readable(false, function start(set) { let timeout: NodeJS.Timeout | null = null; const unsubscribe = requestedCameraState.subscribe((enabled) => { if (enabled === true) { if (timeout) { clearTimeout(timeout); } timeout = setTimeout(() => { set(false); }, 10000); set(true); } else { set(false); } }); return function stop() { unsubscribe(); }; }); /** * A store containing whether the webcam was enabled in the last 5 seconds */ const userMoved5SecondsAgoStore = readable(false, function start(set) { let timeout: NodeJS.Timeout | null = null; const unsubscribe = userMovingStore.subscribe((moving) => { if (moving === true) { if (timeout) { clearTimeout(timeout); } set(true); } else { timeout = setTimeout(() => { set(false); }, 5000); } }); return function stop() { unsubscribe(); }; }); /** * A store containing whether the mouse is getting close the bottom right corner. */ const mouseInBottomRight = readable(false, function start(set) { let lastInBottomRight = false; const gameDiv = HtmlUtils.getElementByIdOrFail("game"); const detectInBottomRight = (event: MouseEvent) => { const rect = gameDiv.getBoundingClientRect(); const inBottomRight = event.x - rect.left > (rect.width * 3) / 4 && event.y - rect.top > (rect.height * 3) / 4; if (inBottomRight !== lastInBottomRight) { lastInBottomRight = inBottomRight; set(inBottomRight); } }; document.addEventListener("mousemove", detectInBottomRight); return function stop() { document.removeEventListener("mousemove", detectInBottomRight); }; }); /** * A store that contains "true" if the webcam should be stopped for energy efficiency reason - i.e. we are not moving and not in a conversation. */ export const cameraEnergySavingStore = derived( [userMoved5SecondsAgoStore, peerStore, enabledWebCam10secondsAgoStore, mouseInBottomRight], ([$userMoved5SecondsAgoStore, $peerStore, $enabledWebCam10secondsAgoStore, $mouseInBottomRight]) => { return ( !$mouseInBottomRight && !$userMoved5SecondsAgoStore && $peerStore.size === 0 && !$enabledWebCam10secondsAgoStore ); } ); /** * A store that contains video constraints. */ function createVideoConstraintStore() { const { subscribe, set, update } = writable({ width: { min: 640, ideal: 1280, max: 1920 }, height: { min: 400, ideal: 720 }, frameRate: { ideal: localUserStore.getVideoQualityValue() }, facingMode: "user", resizeMode: "crop-and-scale", aspectRatio: 1.777777778, } as MediaTrackConstraints); return { subscribe, setDeviceId: (deviceId: string | undefined) => update((constraints) => { if (deviceId !== undefined) { constraints.deviceId = { exact: deviceId, }; } else { delete constraints.deviceId; } return constraints; }), setFrameRate: (frameRate: number) => update((constraints) => { constraints.frameRate = { ideal: frameRate }; return constraints; }), }; } export const videoConstraintStore = createVideoConstraintStore(); /** * A store that contains video constraints. */ function createAudioConstraintStore() { const { subscribe, set, update } = writable({ //TODO: make these values configurable in the game settings menu and store them in localstorage autoGainControl: false, echoCancellation: true, noiseSuppression: true, } as boolean | MediaTrackConstraints); let selectedDeviceId = null; return { subscribe, setDeviceId: (deviceId: string | undefined) => update((constraints) => { selectedDeviceId = deviceId; if (typeof constraints === "boolean") { constraints = {}; } if (deviceId !== undefined) { constraints.deviceId = { exact: selectedDeviceId, }; } else { delete constraints.deviceId; } return constraints; }), }; } export const audioConstraintStore = createAudioConstraintStore(); let timeout: NodeJS.Timeout; let previousComputedVideoConstraint: boolean | MediaTrackConstraints = false; let previousComputedAudioConstraint: boolean | MediaTrackConstraints = false; /** * A store containing the media constraints we want to apply. */ export const mediaStreamConstraintsStore = derived( [ requestedCameraState, requestedMicrophoneState, gameOverlayVisibilityStore, enableCameraSceneVisibilityStore, videoConstraintStore, audioConstraintStore, privacyShutdownStore, cameraEnergySavingStore, ], ( [ $requestedCameraState, $requestedMicrophoneState, $gameOverlayVisibilityStore, $enableCameraSceneVisibilityStore, $videoConstraintStore, $audioConstraintStore, $privacyShutdownStore, $cameraEnergySavingStore, ], set ) => { let currentVideoConstraint: boolean | MediaTrackConstraints = $videoConstraintStore; let currentAudioConstraint: boolean | MediaTrackConstraints = $audioConstraintStore; if ($enableCameraSceneVisibilityStore) { set({ video: currentVideoConstraint, audio: currentAudioConstraint, }); return; } // Disable webcam if the user requested so if ($requestedCameraState === false) { currentVideoConstraint = false; } // Disable microphone if the user requested so if ($requestedMicrophoneState === false) { currentAudioConstraint = false; } // Disable webcam and microphone when in a Jitsi if ($gameOverlayVisibilityStore === false) { currentVideoConstraint = false; currentAudioConstraint = false; } // Disable webcam for privacy reasons (the game is not visible and we were talking to no one) if ($privacyShutdownStore === true) { currentVideoConstraint = false; } // Disable webcam for energy reasons (the user is not moving and we are talking to no one) if ($cameraEnergySavingStore === true) { currentVideoConstraint = false; currentAudioConstraint = false; } // Let's make the changes only if the new value is different from the old one. if ( previousComputedVideoConstraint != currentVideoConstraint || previousComputedAudioConstraint != currentAudioConstraint ) { previousComputedVideoConstraint = currentVideoConstraint; previousComputedAudioConstraint = currentAudioConstraint; // Let's copy the objects. if (typeof previousComputedVideoConstraint !== "boolean") { previousComputedVideoConstraint = { ...previousComputedVideoConstraint }; } if (typeof previousComputedAudioConstraint !== "boolean") { previousComputedAudioConstraint = { ...previousComputedAudioConstraint }; } if (timeout) { clearTimeout(timeout); } // Let's wait a little bit to avoid sending too many constraint changes. timeout = setTimeout(() => { set({ video: currentVideoConstraint, audio: currentAudioConstraint, }); }, 100); } }, { video: false, audio: false, } as MediaStreamConstraints ); export type LocalStreamStoreValue = StreamSuccessValue | StreamErrorValue; interface StreamSuccessValue { type: "success"; stream: MediaStream | null; // The constraints that we got (and not the one that have been requested) constraints: MediaStreamConstraints; } interface StreamErrorValue { type: "error"; error: Error; constraints: MediaStreamConstraints; } let currentStream: MediaStream | null = null; /** * Stops the camera from filming */ function stopCamera(): void { if (currentStream) { for (const track of currentStream.getVideoTracks()) { track.stop(); } } } /** * Stops the microphone from listening */ function stopMicrophone(): void { if (currentStream) { for (const track of currentStream.getAudioTracks()) { track.stop(); } } } /** * A store containing the MediaStream object (or null if nothing requested, or Error if an error occurred) */ export const localStreamStore = derived, LocalStreamStoreValue>( mediaStreamConstraintsStore, ($mediaStreamConstraintsStore, set) => { const constraints = { ...$mediaStreamConstraintsStore }; if (navigator.mediaDevices === undefined) { if (window.location.protocol === "http:") { //throw new Error('Unable to access your camera or microphone. You need to use a HTTPS connection.'); set({ type: "error", error: new Error("Unable to access your camera or microphone. You need to use a HTTPS connection."), constraints, }); return; } else if (isIOS()) { set({ type: "error", error: new WebviewOnOldIOS(), constraints, }); return; } else { set({ type: "error", error: new BrowserTooOldError(), constraints, }); return; } } if (constraints.audio === false) { stopMicrophone(); } if (constraints.video === false) { stopCamera(); } if (constraints.audio === false && constraints.video === false) { currentStream = null; set({ type: "success", stream: null, constraints, }); return; } (async () => { try { stopMicrophone(); stopCamera(); currentStream = await navigator.mediaDevices.getUserMedia(constraints); set({ type: "success", stream: currentStream, constraints, }); return; } catch (e) { if (constraints.video !== false) { console.info( "Error. Unable to get microphone and/or camera access. Trying audio only.", $mediaStreamConstraintsStore, e ); // TODO: does it make sense to pop this error when retrying? set({ type: "error", error: e, constraints, }); // Let's try without video constraints requestedCameraState.disableWebcam(); } else { console.info( "Error. Unable to get microphone and/or camera access.", $mediaStreamConstraintsStore, e ); set({ type: "error", error: e, constraints, }); } /*constraints.video = false; if (constraints.audio === false) { console.info("Error. Unable to get microphone and/or camera access.", $mediaStreamConstraintsStore, e); set({ type: 'error', error: e, constraints }); // Let's make as if the user did not ask. requestedCameraState.disableWebcam(); } else { console.info("Error. Unable to get microphone and/or camera access. Trying audio only.", $mediaStreamConstraintsStore, e); try { currentStream = await navigator.mediaDevices.getUserMedia(constraints); set({ type: 'success', stream: currentStream, constraints }); return; } catch (e2) { console.info("Error. Unable to get microphone fallback access.", $mediaStreamConstraintsStore, e2); set({ type: 'error', error: e, constraints }); } }*/ } })(); } ); /** * A store containing the real active media constrained (not the one requested by the user, but the one we got from the system) */ export const obtainedMediaConstraintStore = derived(localStreamStore, ($localStreamStore) => { return $localStreamStore.constraints; }); /** * Device list */ export const deviceListStore = readable([], function start(set) { let deviceListCanBeQueried = false; const queryDeviceList = () => { // Note: so far, we are ignoring any failures. navigator.mediaDevices .enumerateDevices() .then((mediaDeviceInfos) => { set(mediaDeviceInfos); }) .catch((e) => { console.error(e); throw e; }); }; const unsubscribe = localStreamStore.subscribe((streamResult) => { if (streamResult.type === "success" && streamResult.stream !== null) { if (deviceListCanBeQueried === false) { queryDeviceList(); deviceListCanBeQueried = true; } } }); if (navigator.mediaDevices) { navigator.mediaDevices.addEventListener("devicechange", queryDeviceList); } return function stop() { unsubscribe(); if (navigator.mediaDevices) { navigator.mediaDevices.removeEventListener("devicechange", queryDeviceList); } }; }); export const cameraListStore = derived(deviceListStore, ($deviceListStore) => { return $deviceListStore.filter((device) => device.kind === "videoinput"); }); export const microphoneListStore = derived(deviceListStore, ($deviceListStore) => { return $deviceListStore.filter((device) => device.kind === "audioinput"); }); // TODO: detect the new webcam and automatically switch on it. cameraListStore.subscribe((devices) => { // If the selected camera is unplugged, let's remove the constraint on deviceId const constraints = get(videoConstraintStore); if (!constraints.deviceId) { return; } // If we cannot find the device ID, let's remove it. // @ts-ignore if (!devices.find((device) => device.deviceId === constraints.deviceId.exact)) { videoConstraintStore.setDeviceId(undefined); } }); microphoneListStore.subscribe((devices) => { // If the selected camera is unplugged, let's remove the constraint on deviceId const constraints = get(audioConstraintStore); if (typeof constraints === "boolean") { return; } if (!constraints.deviceId) { return; } // If we cannot find the device ID, let's remove it. // @ts-ignore if (!devices.find((device) => device.deviceId === constraints.deviceId.exact)) { audioConstraintStore.setDeviceId(undefined); } }); localStreamStore.subscribe((streamResult) => { if (streamResult.type === "error") { if (streamResult.error.name === BrowserTooOldError.NAME || streamResult.error.name === WebviewOnOldIOS.NAME) { errorStore.addErrorMessage(streamResult.error); } } });