lbehm/workadventure
1
0
Fork 0
Code Issues Pull requests Releases Activity

Merge pull request #1402 from thecodingmachine/betterMEdiaStore

Browse source 
FEATURE: improved the mediaStore code to disable tracks instead of deleting them
This commit is contained in:
Kharhamel 2021-09-01 17:30:59 +02:00 committed by GitHub
parent 49c2ba3c4b 52fe79df47
commit 095a486621
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 112 additions and 239 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
front/src/Components/MyCamera.svelte
View file

@ -1,27 +1,11 @@
<script lang="typescript">
import {obtainedMediaConstraintStore} from "../Stores/MediaStore";
import {localStreamStore} from "../Stores/MediaStore";
import SoundMeterWidget from "./SoundMeterWidget.svelte";
import {onDestroy} from "svelte";
function srcObject(node: HTMLVideoElement, stream: MediaStream) {
node.srcObject = stream;
return {
update(newStream: MediaStream) {
if (node.srcObject != newStream) {
node.srcObject = newStream
}
}
}
}
import {srcObject} from "./Video/utils";
let stream : MediaStream|null;
/*$: {
if ($localStreamStore.type === 'success') {
stream = $localStreamStore.stream;
} else {
stream = null;
}
}*/
const unsubscribe = localStreamStore.subscribe(value => {
if (value.type === 'success') {
@ -37,9 +21,9 @@
<div>
<div class="video-container div-myCamVideo" class:hide={!$localStreamStore.constraints.video}>
{#if $localStreamStore.type === "success" && $localStreamStore.stream }
<video class="myCamVideo" use:srcObject={$localStreamStore.stream} autoplay muted playsinline></video>
<div class="video-container div-myCamVideo" class:hide={!$obtainedMediaConstraintStore.video}>
{#if $localStreamStore.type === "success" && $localStreamStore.stream}
<video class="myCamVideo" use:srcObject={stream} autoplay muted playsinline></video>
<SoundMeterWidget stream={stream}></SoundMeterWidget>
{/if}
</div>

2
front/src/Phaser/Game/GameScene.ts
View file

@ -712,7 +712,7 @@ export class GameScene extends DirtyScene {
});
// When connection is performed, let's connect SimplePeer
this.simplePeer = new SimplePeer(this.connection, !this.room.isPublic, this.playerName);
this.simplePeer = new SimplePeer(this.connection);
peerStore.connectToSimplePeer(this.simplePeer);
screenSharingPeerStore.connectToSimplePeer(this.simplePeer);
videoFocusStore.connectToSimplePeer(this.simplePeer);

158
front/src/Stores/MediaStore.ts
View file

@ -324,14 +324,11 @@ export type LocalStreamStoreValue = StreamSuccessValue | StreamErrorValue;
interface StreamSuccessValue {
type: "success";
stream: MediaStream | null;
// The constraints that we got (and not the one that have been requested)
constraints: MediaStreamConstraints;
}
interface StreamErrorValue {
type: "error";
error: Error;
constraints: MediaStreamConstraints;
}
let currentStream: MediaStream | null = null;
@ -339,10 +336,13 @@ let currentStream: MediaStream | null = null;
/**
* Stops the camera from filming
*/
function stopCamera(): void {
function applyCameraConstraints(currentStream: MediaStream | null, constraints: MediaTrackConstraints | boolean): void {
if (currentStream) {
for (const track of currentStream.getVideoTracks()) {
track.stop();
track.enabled = constraints !== false;
if (constraints && constraints !== true) {
track.applyConstraints(constraints);
}
}
}
}
@ -350,10 +350,16 @@ function stopCamera(): void {
/**
* Stops the microphone from listening
*/
function stopMicrophone(): void {
function applyMicrophoneConstraints(
currentStream: MediaStream | null,
constraints: MediaTrackConstraints | boolean
): void {
if (currentStream) {
for (const track of currentStream.getAudioTracks()) {
track.stop();
track.enabled = constraints !== false;
if (constraints && constraints !== true) {
track.applyConstraints(constraints);
}
}
}
}
@ -372,122 +378,96 @@ export const localStreamStore = derived<Readable<MediaStreamConstraints>, LocalS
set({
type: "error",
error: new Error("Unable to access your camera or microphone. You need to use a HTTPS connection."),
constraints,
});
return;
} else if (isIOS()) {
set({
type: "error",
error: new WebviewOnOldIOS(),
constraints,
});
return;
} else {
set({
type: "error",
error: new BrowserTooOldError(),
constraints,
});
return;
}
}
if (constraints.audio === false) {
stopMicrophone();
}
if (constraints.video === false) {
stopCamera();
}
applyMicrophoneConstraints(currentStream, constraints.audio || false);
applyCameraConstraints(currentStream, constraints.video || false);
if (constraints.audio === false && constraints.video === false) {
currentStream = null;
if (currentStream === null) {
// we need to assign a first value to the stream because getUserMedia is async
set({
type: "success",
stream: null,
constraints,
});
return;
}
(async () => {
try {
stopMicrophone();
stopCamera();
currentStream = await navigator.mediaDevices.getUserMedia(constraints);
set({
type: "success",
stream: currentStream,
constraints,
});
return;
} catch (e) {
if (constraints.video !== false) {
console.info(
"Error. Unable to get microphone and/or camera access. Trying audio only.",
$mediaStreamConstraintsStore,
e
);
// TODO: does it make sense to pop this error when retrying?
set({
type: "error",
error: e,
constraints,
});
// Let's try without video constraints
requestedCameraState.disableWebcam();
} else {
console.info(
"Error. Unable to get microphone and/or camera access.",
$mediaStreamConstraintsStore,
e
);
set({
type: "error",
error: e,
constraints,
});
}
/*constraints.video = false;
if (constraints.audio === false) {
console.info("Error. Unable to get microphone and/or camera access.", $mediaStreamConstraintsStore, e);
set({
type: 'error',
error: e,
constraints
});
// Let's make as if the user did not ask.
requestedCameraState.disableWebcam();
} else {
console.info("Error. Unable to get microphone and/or camera access. Trying audio only.", $mediaStreamConstraintsStore, e);
(async () => {
try {
currentStream = await navigator.mediaDevices.getUserMedia(constraints);
set({
type: 'success',
type: "success",
stream: currentStream,
constraints
});
return;
} catch (e2) {
console.info("Error. Unable to get microphone fallback access.", $mediaStreamConstraintsStore, e2);
set({
type: 'error',
error: e,
constraints
});
} catch (e) {
if (constraints.video !== false) {
console.info(
"Error. Unable to get microphone and/or camera access. Trying audio only.",
$mediaStreamConstraintsStore,
e
);
// TODO: does it make sense to pop this error when retrying?
set({
type: "error",
error: e,
});
// Let's try without video constraints
requestedCameraState.disableWebcam();
} else {
console.info(
"Error. Unable to get microphone and/or camera access.",
$mediaStreamConstraintsStore,
e
);
set({
type: "error",
error: e,
});
}
}
}*/
}
})();
})();
}
}
);
export interface ObtainedMediaStreamConstraints {
video: boolean;
audio: boolean;
}
let obtainedMediaConstraint: ObtainedMediaStreamConstraints = {
audio: false,
video: false,
};
/**
* A store containing the real active media constrained (not the one requested by the user, but the one we got from the system)
* A store containing the actual states of audio and video (activated or deactivated)
*/
export const obtainedMediaConstraintStore = derived(localStreamStore, ($localStreamStore) => {
return $localStreamStore.constraints;
});
export const obtainedMediaConstraintStore = derived<Readable<MediaStreamConstraints>, ObtainedMediaStreamConstraints>(
mediaStreamConstraintsStore,
($mediaStreamConstraintsStore, set) => {
const newObtainedMediaConstraint = {
video: !!$mediaStreamConstraintsStore.video,
audio: !!$mediaStreamConstraintsStore.audio,
};
if (newObtainedMediaConstraint !== obtainedMediaConstraint) {
obtainedMediaConstraint = newObtainedMediaConstraint;
set(obtainedMediaConstraint);
}
}
);
/**
* Device list

12
front/src/Stores/ScreenSharingStore.ts
View file

@ -1,7 +1,6 @@
import { derived, get, Readable, readable, writable, Writable } from "svelte/store";
import { derived, Readable, readable, writable } from "svelte/store";
import { peerStore } from "./PeerStore";
import type { LocalStreamStoreValue } from "./MediaStore";
import { DivImportance } from "../WebRtc/LayoutManager";
import { gameOverlayVisibilityStore } from "./GameOverlayStoreVisibility";
declare const navigator: any; // eslint-disable-line @typescript-eslint/no-explicit-any
@ -106,7 +105,6 @@ export const screenSharingLocalStreamStore = derived<Readable<MediaStreamConstra
set({
type: "success",
stream: null,
constraints,
});
return;
}
@ -121,7 +119,6 @@ export const screenSharingLocalStreamStore = derived<Readable<MediaStreamConstra
set({
type: "error",
error: new Error("Your browser does not support sharing screen"),
constraints,
});
return;
}
@ -141,10 +138,6 @@ export const screenSharingLocalStreamStore = derived<Readable<MediaStreamConstra
set({
type: "success",
stream: null,
constraints: {
video: false,
audio: false,
},
});
};
}
@ -152,7 +145,6 @@ export const screenSharingLocalStreamStore = derived<Readable<MediaStreamConstra
set({
type: "success",
stream: currentStream,
constraints,
});
return;
} catch (e) {
@ -162,7 +154,6 @@ export const screenSharingLocalStreamStore = derived<Readable<MediaStreamConstra
set({
type: "error",
error: e,
constraints,
});
}
})();
@ -184,7 +175,6 @@ export const screenSharingAvailableStore = derived(peerStore, ($peerStore, set)
export interface ScreenSharingLocalMedia {
uniqueId: string;
stream: MediaStream | null;
//subscribe(this: void, run: Subscriber<ScreenSharingLocalMedia>, invalidate?: (value?: ScreenSharingLocalMedia) => void): Unsubscriber;
}
/**

83
front/src/WebRtc/SimplePeer.ts
View file

@ -4,16 +4,12 @@ import type {
} from "../Connexion/ConnexionModels";
import { mediaManager, StartScreenSharingCallback, StopScreenSharingCallback } from "./MediaManager";
import { ScreenSharingPeer } from "./ScreenSharingPeer";
import { MESSAGE_TYPE_BLOCKED, MESSAGE_TYPE_CONSTRAINT, MESSAGE_TYPE_MESSAGE, VideoPeer } from "./VideoPeer";
import { VideoPeer } from "./VideoPeer";
import type { RoomConnection } from "../Connexion/RoomConnection";
import { blackListManager } from "./BlackListManager";
import { get } from "svelte/store";
import { localStreamStore, LocalStreamStoreValue, obtainedMediaConstraintStore } from "../Stores/MediaStore";
import { screenSharingLocalStreamStore } from "../Stores/ScreenSharingStore";
import { discussionManager } from "./DiscussionManager";
import { playersStore } from "../Stores/PlayersStore";
import { newChatMessageStore } from "../Stores/ChatStore";
import { isMobile } from "../Enum/EnvironmentVariable";
export interface UserSimplePeerInterface {
userId: number;
@ -46,19 +42,14 @@ export class SimplePeer {
private lastWebrtcUserName: string | undefined;
private lastWebrtcPassword: string | undefined;
constructor(private Connection: RoomConnection, private enableReporting: boolean, private myName: string) {
constructor(private Connection: RoomConnection) {
// We need to go through this weird bound function pointer in order to be able to "free" this reference later.
this.sendLocalScreenSharingStreamCallback = this.sendLocalScreenSharingStream.bind(this);
this.stopLocalScreenSharingStreamCallback = this.stopLocalScreenSharingStream.bind(this);
this.unsubscribers.push(
localStreamStore.subscribe((streamResult) => {
this.sendLocalVideoStream(streamResult);
})
);
let localScreenCapture: MediaStream | null = null;
//todo
this.unsubscribers.push(
screenSharingLocalStreamStore.subscribe((streamResult) => {
if (streamResult.type === "error") {
@ -126,19 +117,14 @@ export class SimplePeer {
if (!user.initiator) {
return;
}
const streamResult = get(localStreamStore);
let stream: MediaStream | null = null;
if (streamResult.type === "success" && streamResult.stream) {
stream = streamResult.stream;
}
this.createPeerConnection(user, stream);
this.createPeerConnection(user);
}
/**
* create peer connection to bind users
*/
private createPeerConnection(user: UserSimplePeerInterface, localStream: MediaStream | null): VideoPeer | null {
private createPeerConnection(user: UserSimplePeerInterface): VideoPeer | null {
const peerConnection = this.PeerConnectionArray.get(user.userId);
if (peerConnection) {
if (peerConnection.destroyed) {
@ -160,7 +146,7 @@ export class SimplePeer {
this.lastWebrtcUserName = user.webRtcUser;
this.lastWebrtcPassword = user.webRtcPassword;
const peer = new VideoPeer(user, user.initiator ? user.initiator : false, name, this.Connection, localStream);
const peer = new VideoPeer(user, user.initiator ? user.initiator : false, name, this.Connection);
peer.toClose = false;
// When a connection is established to a video stream, and if a screen sharing is taking place,
@ -204,7 +190,7 @@ export class SimplePeer {
if (!peerConnexionDeleted) {
throw "Error to delete peer connection";
}
this.createPeerConnection(user, stream);
this.createPeerConnection(user);
} else {
peerConnection.toClose = false;
}
@ -282,7 +268,6 @@ export class SimplePeer {
*/
private closeScreenSharingConnection(userId: number) {
try {
//mediaManager.removeActiveScreenSharingVideo("" + userId);
const peer = this.PeerScreenSharingConnectionArray.get(userId);
if (peer === undefined) {
console.warn(
@ -295,12 +280,6 @@ export class SimplePeer {
// FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray"
// I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel.
peer.destroy();
//Comment this peer connection because if we delete and try to reshare screen, the RTCPeerConnection send renegotiate event. This array will be remove when user left circle discussion
/*if(!this.PeerScreenSharingConnectionArray.delete(userId)){
throw 'Couln\'t delete peer screen sharing connexion';
}*/
//console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size);
} catch (err) {
console.error("closeConnection", err);
}
@ -330,13 +309,7 @@ export class SimplePeer {
try {
//if offer type, create peer connection
if (data.signal.type === "offer") {
const streamResult = get(localStreamStore);
let stream: MediaStream | null = null;
if (streamResult.type === "success" && streamResult.stream) {
stream = streamResult.stream;
}
this.createPeerConnection(data, stream);
this.createPeerConnection(data);
}
const peer = this.PeerConnectionArray.get(data.userId);
if (peer !== undefined) {
@ -379,42 +352,10 @@ export class SimplePeer {
} catch (e) {
console.error(`receiveWebrtcSignal => ${data.userId}`, e);
//Comment this peer connection because if we delete and try to reshare screen, the RTCPeerConnection send renegotiate event. This array will be remove when user left circle discussion
//this.PeerScreenSharingConnectionArray.delete(data.userId);
this.receiveWebrtcScreenSharingSignal(data);
}
}
private pushVideoToRemoteUser(userId: number, streamResult: LocalStreamStoreValue) {
try {
const PeerConnection = this.PeerConnectionArray.get(userId);
if (!PeerConnection) {
throw new Error("While adding media, cannot find user with ID " + userId);
}
PeerConnection.write(
new Buffer(
JSON.stringify({
type: MESSAGE_TYPE_CONSTRAINT,
...streamResult.constraints,
isMobile: isMobile(),
})
)
);
if (streamResult.type === "error") {
return;
}
const localStream: MediaStream | null = streamResult.stream;
if (!localStream) {
return;
}
PeerConnection.addStream(localStream);
} catch (e) {
console.error(`pushVideoToRemoteUser => ${userId}`, e);
}
}
private pushScreenSharingToRemoteUser(userId: number, localScreenCapture: MediaStream) {
const PeerConnection = this.PeerScreenSharingConnectionArray.get(userId);
if (!PeerConnection) {
@ -427,12 +368,6 @@ export class SimplePeer {
return;
}
public sendLocalVideoStream(streamResult: LocalStreamStoreValue) {
for (const user of this.Users) {
this.pushVideoToRemoteUser(user.userId, streamResult);
}
}
/**
* Triggered locally when clicking on the screen sharing button
*/
@ -486,8 +421,6 @@ export class SimplePeer {
if (!PeerConnectionScreenSharing.isReceivingScreenSharingStream()) {
PeerConnectionScreenSharing.destroy();
//Comment this peer connection because if we delete and try to reshare screen, the RTCPeerConnection send renegotiate event. This array will be remove when user left circle discussion
//this.PeerScreenSharingConnectionArray.delete(userId);
}
}
}

70
front/src/WebRtc/VideoPeer.ts
View file

@ -5,7 +5,12 @@ import { blackListManager } from "./BlackListManager";
import type { Subscription } from "rxjs";
import type { UserSimplePeerInterface } from "./SimplePeer";
import { get, readable, Readable, Unsubscriber } from "svelte/store";
import { obtainedMediaConstraintIsMobileStore, obtainedMediaConstraintStore } from "../Stores/MediaStore";
import {
localStreamStore,
obtainedMediaConstraintIsMobileStore,
obtainedMediaConstraintStore,
ObtainedMediaStreamConstraints,
} from "../Stores/MediaStore";
import { playersStore } from "../Stores/PlayersStore";
import { chatMessagesStore, chatVisibilityStore, newChatMessageStore } from "../Stores/ChatStore";
import { getIceServersConfig } from "../Components/Video/utils";
@ -34,16 +39,17 @@ export class VideoPeer extends Peer {
private onUnBlockSubscribe: Subscription;
public readonly streamStore: Readable<MediaStream | null>;
public readonly statusStore: Readable<PeerStatus>;
public readonly constraintsStore: Readable<MediaStreamConstraints | null>;
public readonly constraintsStore: Readable<ObtainedMediaStreamConstraints | null>;
private newMessageunsubscriber: Unsubscriber | null = null;
private closing: Boolean = false; //this is used to prevent destroy() from being called twice
private localStreamStoreSubscribe: Unsubscriber;
private obtainedMediaConstraintStoreSubscribe: Unsubscriber;
constructor(
public user: UserSimplePeerInterface,
initiator: boolean,
public readonly userName: string,
private connection: RoomConnection,
localStream: MediaStream | null
private connection: RoomConnection
) {
super({
initiator,
@ -60,27 +66,15 @@ export class VideoPeer extends Peer {
const onStream = (stream: MediaStream | null) => {
set(stream);
};
const onData = (chunk: Buffer) => {
this.on("data", (chunk: Buffer) => {
const message = JSON.parse(chunk.toString("utf8"));
if (message.type === MESSAGE_TYPE_CONSTRAINT) {
if (!message.video) {
set(null);
}
}
});
};
this.on("stream", onStream);
this.on("data", onData);
return () => {
this.off("stream", onStream);
this.off("data", onData);
};
});
this.constraintsStore = readable<MediaStreamConstraints | null>(null, (set) => {
this.constraintsStore = readable<ObtainedMediaStreamConstraints | null>(null, (set) => {
const onData = (chunk: Buffer) => {
const message = JSON.parse(chunk.toString("utf8"));
if (message.type === MESSAGE_TYPE_CONSTRAINT) {
@ -191,7 +185,6 @@ export class VideoPeer extends Peer {
this._onFinish();
});
this.pushVideoToRemoteUser(localStream);
this.onBlockSubscribe = blackListManager.onBlockStream.subscribe((userUuid) => {
if (userUuid === this.userUuid) {
this.toggleRemoteStream(false);
@ -208,6 +201,21 @@ export class VideoPeer extends Peer {
if (blackListManager.isBlackListed(this.userUuid)) {
this.sendBlockMessage(true);
}
this.localStreamStoreSubscribe = localStreamStore.subscribe((streamValue) => {
if (streamValue.type === "success" && streamValue.stream) this.addStream(streamValue.stream);
});
this.obtainedMediaConstraintStoreSubscribe = obtainedMediaConstraintStore.subscribe((constraints) => {
this.write(
new Buffer(
JSON.stringify({
type: MESSAGE_TYPE_CONSTRAINT,
...constraints,
isMobile: isMobile(),
})
)
);
});
}
private sendBlockMessage(blocking: boolean) {
@ -264,6 +272,8 @@ export class VideoPeer extends Peer {
this.onUnBlockSubscribe.unsubscribe();
if (this.newMessageunsubscriber) this.newMessageunsubscriber();
chatMessagesStore.addOutcomingUser(this.userId);
if (this.localStreamStoreSubscribe) this.localStreamStoreSubscribe();
if (this.obtainedMediaConstraintStoreSubscribe) this.obtainedMediaConstraintStoreSubscribe();
super.destroy();
} catch (err) {
console.error("VideoPeer::destroy", err);
@ -281,28 +291,4 @@ export class VideoPeer extends Peer {
this.once("connect", destroySoon);
}
}
private pushVideoToRemoteUser(localStream: MediaStream | null) {
try {
this.write(
new Buffer(
JSON.stringify({
type: MESSAGE_TYPE_CONSTRAINT,
...get(obtainedMediaConstraintStore),
isMobile: isMobile(),
})
)
);
if (!localStream) {
return;
}
for (const track of localStream.getTracks()) {
this.addTrack(track, localStream);
}
} catch (e) {
console.error(`pushVideoToRemoteUser => ${this.userId}`, e);
}
}
}