From 208b91e52a476869e6dca7b917ae976f6152f4d6 Mon Sep 17 00:00:00 2001 From: Gregoire Parant Date: Sat, 6 Jun 2020 17:03:10 +0200 Subject: [PATCH 01/23] Feature screen sharing - Send stream of screen sharing in peer connexion - Add button for share your screen --- front/dist/index.html | 11 +++ front/dist/resources/logos/monitor-close.svg | 44 ++++++++++++ front/dist/resources/logos/monitor.svg | 15 ++++ front/dist/resources/style/style.css | 4 ++ front/src/WebRtc/MediaManager.ts | 73 ++++++++++++++++++++ front/src/WebRtc/SimplePeer.ts | 58 ++++++++-------- 6 files changed, 174 insertions(+), 31 deletions(-) create mode 100644 front/dist/resources/logos/monitor-close.svg create mode 100644 front/dist/resources/logos/monitor.svg diff --git a/front/dist/index.html b/front/dist/index.html index 92a7bf3c..360d5a9a 100644 --- a/front/dist/index.html +++ b/front/dist/index.html @@ -77,6 +77,10 @@ +
+ + +
@@ -100,6 +104,13 @@ +
+ + +
+ --> diff --git a/front/dist/resources/logos/monitor-close.svg b/front/dist/resources/logos/monitor-close.svg new file mode 100644 index 00000000..80056e2d --- /dev/null +++ b/front/dist/resources/logos/monitor-close.svg @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/front/dist/resources/logos/monitor.svg b/front/dist/resources/logos/monitor.svg new file mode 100644 index 00000000..d4b586c6 --- /dev/null +++ b/front/dist/resources/logos/monitor.svg @@ -0,0 +1,15 @@ + + + + + + diff --git a/front/dist/resources/style/style.css b/front/dist/resources/style/style.css index 30e099ef..413bce71 100644 --- a/front/dist/resources/style/style.css +++ b/front/dist/resources/style/style.css @@ -109,6 +109,10 @@ video#myCamVideo{ transition: all .2s; right: 134px; } +.btn-monitor{ + transition: all .2s; + right: 224px; +} /*.btn-call{ transition: all .1s; left: 0px; diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 39a61738..a11532ac 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -5,6 +5,9 @@ const videoConstraint: boolean|MediaTrackConstraints = { height: { ideal: 720 }, facingMode: "user" }; +interface MediaServiceInterface extends MediaDevices{ + getDisplayMedia(constrain: any) : Promise; +} type UpdatedLocalStreamCallback = (media: MediaStream) => void; @@ -12,10 +15,13 @@ type UpdatedLocalStreamCallback = (media: MediaStream) => void; // TODO: verify that microphone event listeners are not triggered plenty of time NOW (since MediaManager is created many times!!!!) export class MediaManager { localStream: MediaStream|null = null; + localScreenCapture: MediaStream|null = null; private remoteVideo: Map = new Map(); myCamVideo: HTMLVideoElement; cinemaClose: HTMLImageElement; cinema: HTMLImageElement; + monitorClose: HTMLImageElement; + monitor: HTMLImageElement; microphoneClose: HTMLImageElement; microphone: HTMLImageElement; webrtcInAudio: HTMLAudioElement; @@ -57,6 +63,21 @@ export class MediaManager { this.disabledCamera(); //update tracking }); + + this.monitorClose = document.getElementById('monitor-close'); + this.monitorClose.style.display = "block"; + this.monitorClose.addEventListener('click', (e: any) => { + e.preventDefault(); + this.enabledMonitor(); + //update tracking + }); + this.monitor = document.getElementById('monitor'); + this.monitor.style.display = "none"; + this.monitor.addEventListener('click', (e: any) => { + e.preventDefault(); + this.disabledMonitor(); + //update tracking + }); } onUpdateLocalStream(callback: UpdatedLocalStreamCallback): void { @@ -126,6 +147,58 @@ export class MediaManager { }); } + enabledMonitor() { + this.monitorClose.style.display = "none"; + this.monitor.style.display = "block"; + this.getScreenMedia().then((stream) => { + this.updatedLocalStreamCallBack(stream); + }); + } + + disabledMonitor() { + this.monitorClose.style.display = "block"; + this.monitor.style.display = "none"; + this.localScreenCapture?.getTracks().forEach((track: MediaStreamTrack) => { + track.stop(); + }); + this.localScreenCapture = null; + this.getCamera().then((stream) => { + this.updatedLocalStreamCallBack(stream); + }); + } + + //get screen + getScreenMedia() : Promise{ + try { + return this._startScreenCapture() + .then((stream: MediaStream) => { + this.localScreenCapture = stream; + return stream; + }) + .catch((err: any) => { + console.error("Error => getScreenMedia => " + err); + throw err; + }); + }catch (err) { + return new Promise((resolve, reject) => { // eslint-disable-line no-unused-vars + reject(err); + }); + } + } + + private _startScreenCapture() { + if ((navigator as any).getDisplayMedia) { + return (navigator as any).getDisplayMedia({video: true}); + } else if ((navigator.mediaDevices as any).getDisplayMedia) { + return (navigator.mediaDevices as any).getDisplayMedia({video: true}); + } else { + //return navigator.mediaDevices.getUserMedia(({video: {mediaSource: 'screen'}} as any)); + return new Promise((resolve, reject) => { // eslint-disable-line no-unused-vars + reject("error sharing screen"); + }); + } + } + //get camera async getCamera(): Promise { if (navigator.mediaDevices === undefined) { diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index fdc2d0c2..96f047b7 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -156,22 +156,11 @@ export class SimplePeer { videoActive = true; } }); - if(microphoneActive){ - mediaManager.enabledMicrophoneByUserId(user.userId); - }else{ - mediaManager.disabledMicrophoneByUserId(user.userId); - } - if(videoActive){ - mediaManager.enabledVideoByUserId(user.userId); - }else{ - mediaManager.disabledVideoByUserId(user.userId); - } this.stream(user.userId, stream); }); /*peer.on('track', (track: MediaStreamTrack, stream: MediaStream) => { - this.stream(user.userId, stream); });*/ peer.on('close', () => { @@ -190,9 +179,19 @@ export class SimplePeer { }); peer.on('data', (chunk: Buffer) => { - const data = JSON.parse(chunk.toString('utf8')); - if(data.type === "stream"){ - this.stream(user.userId, data.stream); + let constraint = JSON.parse(chunk.toString('utf8')); + + if (constraint.audio) { + mediaManager.enabledMicrophoneByUserId(user.userId); + } else { + mediaManager.disabledMicrophoneByUserId(user.userId); + } + + if (constraint.video) { + mediaManager.enabledVideoByUserId(user.userId); + } else { + this.stream(user.userId); + mediaManager.disabledVideoByUserId(user.userId); } }); @@ -279,7 +278,7 @@ export class SimplePeer { * @param userId * @param stream */ - private stream(userId : string, stream: MediaStream) { + private stream(userId : string, stream?: MediaStream) { if(!stream){ mediaManager.disabledVideoByUserId(userId); mediaManager.disabledMicrophoneByUserId(userId); @@ -294,24 +293,21 @@ export class SimplePeer { */ private addMedia (userId : string) { try { - const localStream: MediaStream|null = mediaManager.localStream; - const peer = this.PeerConnectionArray.get(userId); - if(localStream === null) { - //send fake signal - if(peer === undefined){ - return; - } - peer.write(new Buffer(JSON.stringify({ - type: "stream", - stream: null - }))); - return; - } + let localStream: MediaStream | null = mediaManager.localStream; + let localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; + let peer = this.PeerConnectionArray.get(userId); if (peer === undefined) { - throw new Error('While adding media, cannot find user with ID '+userId); + throw new Error('While adding media, cannot find user with ID ' + userId); } - for (const track of localStream.getTracks()) { - peer.addTrack(track, localStream); + peer.write(new Buffer(JSON.stringify(mediaManager.constraintsMedia))); + if (localScreenCapture !== null) { + for (const track of localScreenCapture.getTracks()) { + peer.addTrack(track, localScreenCapture); + } + } else if (localStream) { + for (const track of localStream.getTracks()) { + peer.addTrack(track, localStream); + } } }catch (e) { console.error(`addMedia => addMedia => ${userId}`, e); From eed5333d693d57183f0895d81109b1c21fdeba1f Mon Sep 17 00:00:00 2001 From: Gregoire Parant Date: Sat, 6 Jun 2020 19:52:34 +0200 Subject: [PATCH 02/23] Stability simple peer --- front/src/WebRtc/SimplePeer.ts | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 96f047b7..78dc41f9 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -180,14 +180,13 @@ export class SimplePeer { peer.on('data', (chunk: Buffer) => { let constraint = JSON.parse(chunk.toString('utf8')); - if (constraint.audio) { mediaManager.enabledMicrophoneByUserId(user.userId); } else { mediaManager.disabledMicrophoneByUserId(user.userId); } - if (constraint.video) { + if (constraint.video || constraint.screen) { mediaManager.enabledVideoByUserId(user.userId); } else { this.stream(user.userId); @@ -295,18 +294,30 @@ export class SimplePeer { try { let localStream: MediaStream | null = mediaManager.localStream; let localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; - let peer = this.PeerConnectionArray.get(userId); - if (peer === undefined) { + let PeerConnection : any = this.PeerConnectionArray.get(userId); + if (PeerConnection === undefined) { throw new Error('While adding media, cannot find user with ID ' + userId); } - peer.write(new Buffer(JSON.stringify(mediaManager.constraintsMedia))); + PeerConnection.write(new Buffer(JSON.stringify(Object.assign(mediaManager.constraintsMedia, {screen: localScreenCapture !== null})))); + + //remove current stream + try { + if (PeerConnection._pc) { + PeerConnection._pc.getRemoteStreams().forEach((stream: MediaStream) => { + stream.getTracks().forEach((track: MediaStreamTrack) => { + PeerConnection.removeTrack(track, stream); + }); + }); + } + }catch (e) {} + if (localScreenCapture !== null) { for (const track of localScreenCapture.getTracks()) { - peer.addTrack(track, localScreenCapture); + PeerConnection.addTrack(track, localScreenCapture); } } else if (localStream) { for (const track of localStream.getTracks()) { - peer.addTrack(track, localStream); + PeerConnection.addTrack(track, localStream); } } }catch (e) { From 6c1b8122ef739b5be04deaf773331ae932e13264 Mon Sep 17 00:00:00 2001 From: Gregoire Parant Date: Sat, 6 Jun 2020 20:13:30 +0200 Subject: [PATCH 03/23] Fix CI --- front/src/WebRtc/SimplePeer.ts | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 78dc41f9..825bc612 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -294,23 +294,13 @@ export class SimplePeer { try { let localStream: MediaStream | null = mediaManager.localStream; let localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; - let PeerConnection : any = this.PeerConnectionArray.get(userId); - if (PeerConnection === undefined) { + let PeerConnection = this.PeerConnectionArray.get(userId); + + if (!PeerConnection || PeerConnection === undefined) { throw new Error('While adding media, cannot find user with ID ' + userId); } PeerConnection.write(new Buffer(JSON.stringify(Object.assign(mediaManager.constraintsMedia, {screen: localScreenCapture !== null})))); - //remove current stream - try { - if (PeerConnection._pc) { - PeerConnection._pc.getRemoteStreams().forEach((stream: MediaStream) => { - stream.getTracks().forEach((track: MediaStreamTrack) => { - PeerConnection.removeTrack(track, stream); - }); - }); - } - }catch (e) {} - if (localScreenCapture !== null) { for (const track of localScreenCapture.getTracks()) { PeerConnection.addTrack(track, localScreenCapture); From 209057e3fc1a5823f9db68e9b7e9e05a2e96fdf4 Mon Sep 17 00:00:00 2001 From: Gregoire Parant Date: Mon, 8 Jun 2020 09:20:36 +0200 Subject: [PATCH 04/23] New fictive user screen sharing - Create new fictive user - Add new fictive user in WebRtc group - Add screen sharing video on left side --- front/src/Connection.ts | 4 +- front/src/WebRtc/MediaManager.ts | 29 ++++++++++++-- front/src/WebRtc/SimplePeer.ts | 67 ++++++++++++++++++++++++-------- 3 files changed, 79 insertions(+), 21 deletions(-) diff --git a/front/src/Connection.ts b/front/src/Connection.ts index 04715df6..69121837 100644 --- a/front/src/Connection.ts +++ b/front/src/Connection.ts @@ -6,7 +6,7 @@ import {SetPlayerDetailsMessage} from "./Messages/SetPlayerDetailsMessage"; const SocketIo = require('socket.io-client'); import Socket = SocketIOClient.Socket; import {PlayerAnimationNames} from "./Phaser/Player/Animation"; -import {UserSimplePeer} from "./WebRtc/SimplePeer"; +import {UserSimplePeerInterface} from "./WebRtc/SimplePeer"; import {SignalData} from "simple-peer"; @@ -72,7 +72,7 @@ export interface GroupCreatedUpdatedMessageInterface { export interface WebRtcStartMessageInterface { roomId: string, - clients: UserSimplePeer[] + clients: UserSimplePeerInterface[] } export interface WebRtcDisconnectMessageInterface { diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index a11532ac..cfe4843a 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -1,3 +1,4 @@ +import * as SimplePeerNamespace from "simple-peer"; import {DivImportance, layoutManager} from "./LayoutManager"; const videoConstraint: boolean|MediaTrackConstraints = { @@ -30,8 +31,12 @@ export class MediaManager { video: videoConstraint }; updatedLocalStreamCallBacks : Set = new Set(); + // TODO: updatedScreenSharingCallBack should have same signature as updatedLocalStreamCallBacks + updatedScreenSharingCallBack : Function; + + constructor(updatedScreenSharingCallBack : Function) { + this.updatedScreenSharingCallBack = updatedScreenSharingCallBack; - constructor() { this.myCamVideo = this.getElementByIdOrFail('myCamVideo'); this.webrtcInAudio = this.getElementByIdOrFail('audio-webrtc-in'); this.webrtcInAudio.volume = 0.2; @@ -151,7 +156,7 @@ export class MediaManager { this.monitorClose.style.display = "none"; this.monitor.style.display = "block"; this.getScreenMedia().then((stream) => { - this.updatedLocalStreamCallBack(stream); + this.updatedScreenSharingCallBack(stream); }); } @@ -163,7 +168,7 @@ export class MediaManager { }); this.localScreenCapture = null; this.getCamera().then((stream) => { - this.updatedLocalStreamCallBack(stream); + this.updatedScreenSharingCallBack(stream); }); } @@ -278,6 +283,24 @@ export class MediaManager { this.remoteVideo.set(userId, this.getElementByIdOrFail(userId)); } + /** + * + * @param userId + */ + addScreenSharingActiveVideo(userId : string, userName: string = ""){ + this.webrtcInAudio.play(); + // FIXME: switch to DisplayManager! + let elementRemoteVideo = this.getElementByIdOrFail("activeScreenSharing"); + userName = userName.toUpperCase(); + let color = this.getColorByString(userName); + elementRemoteVideo.insertAdjacentHTML('beforeend', ` +
+ +
+ `); + this.remoteVideo.set(userId, document.getElementById(userId)); + } + /** * * @param userId diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 825bc612..6d39444e 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -8,7 +8,7 @@ import { mediaManager } from "./MediaManager"; import * as SimplePeerNamespace from "simple-peer"; const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer'); -export interface UserSimplePeer{ +export interface UserSimplePeerInterface{ userId: string; name?: string; initiator?: boolean; @@ -26,10 +26,11 @@ export interface PeerConnectionListener { export class SimplePeer { private Connection: Connection; private WebRtcRoomId: string; - private Users: Array = new Array(); + private Users: Array = new Array(); private PeerConnectionArray: Map = new Map(); private readonly updateLocalStreamCallback: (media: MediaStream) => void; + private readonly updateScreenSharingCallback: (media: MediaStream) => void; private readonly peerConnectionListeners: Array = new Array(); constructor(Connection: Connection, WebRtcRoomId: string = "test-webrtc") { @@ -37,7 +38,9 @@ export class SimplePeer { this.WebRtcRoomId = WebRtcRoomId; // We need to go through this weird bound function pointer in order to be able to "free" this reference later. this.updateLocalStreamCallback = this.updatedLocalStream.bind(this); + this.updateScreenSharingCallback = this.updatedScreenSharing.bind(this); mediaManager.onUpdateLocalStream(this.updateLocalStreamCallback); + mediaManager.onUpdateScreenSharing(this.updateScreenSharingCallback); this.initialise(); } @@ -93,7 +96,7 @@ export class SimplePeer { * server has two people connected, start the meet */ private startWebRtc() { - this.Users.forEach((user: UserSimplePeer) => { + this.Users.forEach((user: UserSimplePeerInterface) => { //if it's not an initiator, peer connection will be created when gamer will receive offer signal if(!user.initiator){ return; @@ -105,22 +108,24 @@ export class SimplePeer { /** * create peer connection to bind users */ - private createPeerConnection(user : UserSimplePeer) { + private createPeerConnection(user : UserSimplePeerInterface) : SimplePeerNamespace.Instance | null{ if(this.PeerConnectionArray.has(user.userId)) { - return; + return null; } - //console.log("Creating connection with peer "+user.userId); - let name = user.name; if(!name){ - const userSearch = this.Users.find((userSearch: UserSimplePeer) => userSearch.userId === user.userId); + const userSearch = this.Users.find((userSearch: UserSimplePeerInterface) => userSearch.userId === user.userId); if(userSearch) { name = userSearch.name; } } - mediaManager.removeActiveVideo(user.userId); - mediaManager.addActiveVideo(user.userId, name); + + let screenSharing : boolean = name !== undefined && name.indexOf("screenSharing") > -1; + if(!screenSharing) { + mediaManager.removeActiveVideo(user.userId); + mediaManager.addActiveVideo(user.userId, name); + } const peer : SimplePeerNamespace.Instance = new Peer({ initiator: user.initiator ? user.initiator : false, @@ -146,6 +151,11 @@ export class SimplePeer { }); peer.on('stream', (stream: MediaStream) => { + if(screenSharing){ + //add stream video on + return; + } + let videoActive = false; let microphoneActive = false; stream.getTracks().forEach((track : MediaStreamTrack) => { @@ -199,6 +209,7 @@ export class SimplePeer { for (const peerConnectionListener of this.peerConnectionListeners) { peerConnectionListener.onConnect(user); } + return peer; } /** @@ -301,11 +312,10 @@ export class SimplePeer { } PeerConnection.write(new Buffer(JSON.stringify(Object.assign(mediaManager.constraintsMedia, {screen: localScreenCapture !== null})))); - if (localScreenCapture !== null) { - for (const track of localScreenCapture.getTracks()) { - PeerConnection.addTrack(track, localScreenCapture); - } - } else if (localStream) { + if(!localStream){ + return; + } + if (localStream) { for (const track of localStream.getTracks()) { PeerConnection.addTrack(track, localStream); } @@ -316,8 +326,33 @@ export class SimplePeer { } updatedLocalStream(){ - this.Users.forEach((user: UserSimplePeer) => { + this.Users.forEach((user: UserSimplePeerInterface) => { this.addMedia(user.userId); }) } + + updatedScreenSharing() { + if (this.MediaManager.localScreenCapture) { + let screenSharingUser: UserSimplePeerInterface = { + userId: `screenSharing-${this.Connection.userId}`, + initiator: true + }; + let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser); + if (!PeerConnectionScreenSharing) { + return; + } + for (const track of this.MediaManager.localScreenCapture.getTracks()) { + PeerConnectionScreenSharing.addTrack(track, this.MediaManager.localScreenCapture); + } + } else { + if (!this.PeerConnectionArray.has(`screenSharing-${this.Connection.userId}`)) { + return; + } + let PeerConnectionScreenSharing = this.PeerConnectionArray.get(`screenSharing-${this.Connection.userId}`); + if (!PeerConnectionScreenSharing) { + return; + } + PeerConnectionScreenSharing.destroy(); + } + } } From 3e2c5049f2e633ce6357da46df629d379e9b3da6 Mon Sep 17 00:00:00 2001 From: Gregoire Parant Date: Mon, 8 Jun 2020 09:36:07 +0200 Subject: [PATCH 05/23] Fix to add special screen sharing --- front/src/WebRtc/MediaManager.ts | 6 +++++- front/src/WebRtc/SimplePeer.ts | 20 +++----------------- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index cfe4843a..4341c52e 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -298,7 +298,11 @@ export class MediaManager { `); - this.remoteVideo.set(userId, document.getElementById(userId)); + let activeHTMLVideoElement : HTMLElement|null = document.getElementById(userId); + if(!activeHTMLVideoElement){ + return; + } + this.remoteVideo.set(userId, (activeHTMLVideoElement as HTMLVideoElement)); } /** diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 6d39444e..f24cc31d 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -122,9 +122,11 @@ export class SimplePeer { } let screenSharing : boolean = name !== undefined && name.indexOf("screenSharing") > -1; + mediaManager.removeActiveVideo(user.userId); if(!screenSharing) { - mediaManager.removeActiveVideo(user.userId); mediaManager.addActiveVideo(user.userId, name); + }else{ + mediaManager.addScreenSharingActiveVideo(user.userId, name); } const peer : SimplePeerNamespace.Instance = new Peer({ @@ -151,22 +153,6 @@ export class SimplePeer { }); peer.on('stream', (stream: MediaStream) => { - if(screenSharing){ - //add stream video on - return; - } - - let videoActive = false; - let microphoneActive = false; - stream.getTracks().forEach((track : MediaStreamTrack) => { - if(track.kind === "audio"){ - microphoneActive = true; - } - if(track.kind === "video"){ - videoActive = true; - } - }); - this.stream(user.userId, stream); }); From 0bbed7717a57f317a1f9efde5173f8bada37034f Mon Sep 17 00:00:00 2001 From: Gregoire Parant Date: Mon, 8 Jun 2020 22:52:25 +0200 Subject: [PATCH 06/23] Continue screen sharing --- front/dist/index.html | 5 ++--- front/dist/resources/style/style.css | 30 ++++++++++++++++++++++++++++ front/src/WebRtc/SimplePeer.ts | 28 +++++++++++++++++++------- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/front/dist/index.html b/front/dist/index.html index 360d5a9a..02ec0205 100644 --- a/front/dist/index.html +++ b/front/dist/index.html @@ -108,12 +108,11 @@ - --> +
+
diff --git a/front/dist/resources/style/style.css b/front/dist/resources/style/style.css index 413bce71..d0dc2cd4 100644 --- a/front/dist/resources/style/style.css +++ b/front/dist/resources/style/style.css @@ -365,3 +365,33 @@ body { .chat-mode > div:last-child { flex-grow: 5; } + +/*SCREEN SHARING*/ +.active-screen-sharing video{ + transform: scaleX(1); +} +.active-screen-sharing .screen-sharing-video-container video:hover{ + width: 50%; +} +.active-screen-sharing .screen-sharing-video-container video{ + position: absolute; + width: 25%; + height: auto; + left: 0; + top: 0; + transition: all 0.2s ease; +} + +.active-screen-sharing .screen-sharing-video-container video:nth-child(1){ + /*this is for camera of user*/ + top: 0%; +} +.active-screen-sharing .screen-sharing-video-container video:nth-child(2){ + top: 25%; +} +.active-screen-sharing .screen-sharing-video-container video:nth-child(3){ + top: 50%; +} +.active-screen-sharing .screen-sharing-video-container video:nth-child(4) { + top: 75%; +} diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index f24cc31d..6f5fd69a 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -289,14 +289,22 @@ export class SimplePeer { */ private addMedia (userId : string) { try { - let localStream: MediaStream | null = mediaManager.localStream; - let localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; let PeerConnection = this.PeerConnectionArray.get(userId); - - if (!PeerConnection || PeerConnection === undefined) { + if (!PeerConnection) { throw new Error('While adding media, cannot find user with ID ' + userId); } - PeerConnection.write(new Buffer(JSON.stringify(Object.assign(mediaManager.constraintsMedia, {screen: localScreenCapture !== null})))); + + if(userId.indexOf("screenSharing") > -1 && mediaManager.localScreenCapture){ + for (const track of mediaManager.localScreenCapture.getTracks()) { + PeerConnection.addTrack(track, mediaManager.localScreenCapture); + } + return; + } + + let localStream: MediaStream | null = mediaManager.localStream; + let localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; + + PeerConnection.write(new Buffer(JSON.stringify(Object.assign(this.MediaManager.constraintsMedia, {screen: localScreenCapture !== null})))); if(!localStream){ return; @@ -321,15 +329,21 @@ export class SimplePeer { if (this.MediaManager.localScreenCapture) { let screenSharingUser: UserSimplePeerInterface = { userId: `screenSharing-${this.Connection.userId}`, + name: 'screenSharing', initiator: true }; let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser); if (!PeerConnectionScreenSharing) { return; } - for (const track of this.MediaManager.localScreenCapture.getTracks()) { - PeerConnectionScreenSharing.addTrack(track, this.MediaManager.localScreenCapture); + try { + for (const track of this.MediaManager.localScreenCapture.getTracks()) { + PeerConnectionScreenSharing.addTrack(track, this.MediaManager.localScreenCapture); + } + }catch (e) { + console.error("updatedScreenSharing => ", e); } + this.MediaManager.addStreamRemoteVideo(screenSharingUser.userId, this.MediaManager.localScreenCapture); } else { if (!this.PeerConnectionArray.has(`screenSharing-${this.Connection.userId}`)) { return; From a4f42111d7543a0bb87bcdc18ca0c4126b62ba48 Mon Sep 17 00:00:00 2001 From: Gregoire Parant Date: Thu, 11 Jun 2020 23:18:06 +0200 Subject: [PATCH 07/23] Update screen sharing feature --- back/src/Controller/IoSocketController.ts | 33 +++-- front/src/Connection.ts | 28 ++++- front/src/WebRtc/MediaManager.ts | 14 ++- front/src/WebRtc/SimplePeer.ts | 140 +++++++++++++++++----- 4 files changed, 171 insertions(+), 44 deletions(-) diff --git a/back/src/Controller/IoSocketController.ts b/back/src/Controller/IoSocketController.ts index 28dd2da2..2f99f1e6 100644 --- a/back/src/Controller/IoSocketController.ts +++ b/back/src/Controller/IoSocketController.ts @@ -28,6 +28,7 @@ enum SockerIoEvent { USER_MOVED = "user-moved", // From server to client USER_LEFT = "user-left", // From server to client WEBRTC_SIGNAL = "webrtc-signal", + WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal", WEBRTC_START = "webrtc-start", WEBRTC_DISCONNECT = "webrtc-disconect", MESSAGE_ERROR = "message-error", @@ -226,18 +227,11 @@ export class IoSocketController { }); socket.on(SockerIoEvent.WEBRTC_SIGNAL, (data: unknown) => { - if (!isWebRtcSignalMessageInterface(data)) { - socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'}); - console.warn('Invalid WEBRTC_SIGNAL message received: ', data); - return; - } - //send only at user - const client = this.sockets.get(data.receiverId); - if (client === undefined) { - console.warn("While exchanging a WebRTC signal: client with id ", data.receiverId, " does not exist. This might be a race condition."); - return; - } - return client.emit(SockerIoEvent.WEBRTC_SIGNAL, data); + this.emit((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SIGNAL); + }); + + socket.on(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, (data: unknown) => { + this.emit((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL); }); socket.on(SockerIoEvent.DISCONNECT, () => { @@ -284,6 +278,21 @@ export class IoSocketController { }); } + emit(socket: ExSocketInterface, data: unknown, event: SockerIoEvent){ + if (!isWebRtcSignalMessageInterface(data)) { + socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'}); + console.warn('Invalid WEBRTC_SIGNAL message received: ', data); + return; + } + //send only at user + const client = this.sockets.get(data.receiverId); + if (client === undefined) { + console.warn("While exchanging a WebRTC signal: client with id ", data.receiverId, " does not exist. This might be a race condition."); + return; + } + return client.emit(event, data); + } + searchClientByIdOrFail(userId: string): ExSocketInterface { const client: ExSocketInterface|undefined = this.sockets.get(userId); if (client === undefined) { diff --git a/front/src/Connection.ts b/front/src/Connection.ts index 69121837..bceef68a 100644 --- a/front/src/Connection.ts +++ b/front/src/Connection.ts @@ -9,9 +9,9 @@ import {PlayerAnimationNames} from "./Phaser/Player/Animation"; import {UserSimplePeerInterface} from "./WebRtc/SimplePeer"; import {SignalData} from "simple-peer"; - enum EventMessage{ WEBRTC_SIGNAL = "webrtc-signal", + WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal", WEBRTC_START = "webrtc-start", JOIN_ROOM = "join-room", // bi-directional USER_POSITION = "user-position", // bi-directional @@ -197,6 +197,15 @@ export class Connection implements Connection { }); } + sendWebrtcScreenSharingSignal(signal: any, roomId: string, userId? : string|null, receiverId? : string) { + return this.getSocket().emit(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, { + userId: userId ? userId : this.userId, + receiverId: receiverId ? receiverId : this.userId, + roomId: roomId, + signal: signal + }); + } + public receiveWebrtcStart(callback: (message: WebRtcStartMessageInterface) => void) { this.socket.on(EventMessage.WEBRTC_START, callback); } @@ -205,6 +214,23 @@ export class Connection implements Connection { return this.socket.on(EventMessage.WEBRTC_SIGNAL, callback); } + receiveWebrtcScreenSharingSignal(callback: Function) { + return this.getSocket().on(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, callback); + } + + private errorMessage(): void { + this.getSocket().on(EventMessage.MESSAGE_ERROR, (message: string) => { + console.error(EventMessage.MESSAGE_ERROR, message); + }) + } + + private disconnectServer(): void { + this.getSocket().on(EventMessage.CONNECT_ERROR, () => { + this.GameManager.switchToDisconnectedScene(); + }); + + } + public onServerDisconnected(callback: (reason: string) => void): void { this.socket.on('disconnect', (reason: string) => { if (reason === 'io client disconnect') { diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 4341c52e..706b9f49 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -287,14 +287,13 @@ export class MediaManager { * * @param userId */ - addScreenSharingActiveVideo(userId : string, userName: string = ""){ + addScreenSharingActiveVideo(userId : string){ + userId = `screen-sharing-${userId}`; this.webrtcInAudio.play(); // FIXME: switch to DisplayManager! let elementRemoteVideo = this.getElementByIdOrFail("activeScreenSharing"); - userName = userName.toUpperCase(); - let color = this.getColorByString(userName); elementRemoteVideo.insertAdjacentHTML('beforeend', ` -
+
`); @@ -302,6 +301,7 @@ export class MediaManager { if(!activeHTMLVideoElement){ return; } + console.log(userId, (activeHTMLVideoElement as HTMLVideoElement)); this.remoteVideo.set(userId, (activeHTMLVideoElement as HTMLVideoElement)); } @@ -372,6 +372,9 @@ export class MediaManager { } remoteVideo.srcObject = stream; } + addStreamRemoteScreenSharing(userId : string, stream : MediaStream){ + this.addStreamRemoteVideo(`screen-sharing-${userId}`, stream); + } /** * @@ -381,6 +384,9 @@ export class MediaManager { layoutManager.remove(userId); this.remoteVideo.delete(userId); } + removeActiveScreenSharingVideo(userId : string) { + this.removeActiveVideo(`screen-sharing-${userId}`) + } isConnecting(userId : string): void { const connectingSpinnerDiv = this.getSpinner(userId); diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 6f5fd69a..81bffd6d 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -28,6 +28,7 @@ export class SimplePeer { private WebRtcRoomId: string; private Users: Array = new Array(); + private PeerScreenSharingConnectionArray: Map = new Map(); private PeerConnectionArray: Map = new Map(); private readonly updateLocalStreamCallback: (media: MediaStream) => void; private readonly updateScreenSharingCallback: (media: MediaStream) => void; @@ -62,6 +63,11 @@ export class SimplePeer { this.receiveWebrtcSignal(message); }); + //receive signal by gemer + this.Connection.receiveWebrtcScreenSharingSignal((message: any) => { + this.receiveWebrtcScreenSharingSignal(message); + }); + mediaManager.activeVisio(); mediaManager.getCamera().then(() => { @@ -108,7 +114,8 @@ export class SimplePeer { /** * create peer connection to bind users */ - private createPeerConnection(user : UserSimplePeerInterface) : SimplePeerNamespace.Instance | null{ + private createPeerConnection(user : UserSimplePeerInterface, screenSharing: boolean = false) : SimplePeerNamespace.Instance | null{ + console.log("createPeerConnection => screenSharing", screenSharing) if(this.PeerConnectionArray.has(user.userId)) { return null; } @@ -121,12 +128,11 @@ export class SimplePeer { } } - let screenSharing : boolean = name !== undefined && name.indexOf("screenSharing") > -1; mediaManager.removeActiveVideo(user.userId); - if(!screenSharing) { - mediaManager.addActiveVideo(user.userId, name); + if(screenSharing) { + mediaManager.addScreenSharingActiveVideo(user.userId); }else{ - mediaManager.addScreenSharingActiveVideo(user.userId, name); + mediaManager.addActiveVideo(user.userId, name); } const peer : SimplePeerNamespace.Instance = new Peer({ @@ -145,10 +151,19 @@ export class SimplePeer { ] }, }); - this.PeerConnectionArray.set(user.userId, peer); + if(screenSharing){ + this.PeerScreenSharingConnectionArray.set(user.userId, peer); + }else { + this.PeerConnectionArray.set(user.userId, peer); + } //start listen signal for the peer connection peer.on('signal', (data: unknown) => { + console.log("screenSharing", screenSharing); + if(screenSharing){ + this.sendWebrtcScreenSharingSignal(data, user.userId); + return; + } this.sendWebrtcSignal(data, user.userId); }); @@ -160,6 +175,9 @@ export class SimplePeer { });*/ peer.on('close', () => { + if(screenSharing){ + this.closeScreenSharingConnection(user.userId); + } this.closeConnection(user.userId); }); @@ -190,7 +208,11 @@ export class SimplePeer { } }); - this.addMedia(user.userId); + if(screenSharing){ + this.addMediaScreenSharing(user.userId); + }else { + this.addMedia(user.userId); + } for (const peerConnectionListener of this.peerConnectionListeners) { peerConnectionListener.onConnect(user); @@ -225,6 +247,30 @@ export class SimplePeer { } } + /** + * This is triggered twice. Once by the server, and once by a remote client disconnecting + * + * @param userId + */ + private closeScreenSharingConnection(userId : string) { + try { + mediaManager.removeActiveScreenSharingVideo(userId); + let peer = this.PeerScreenSharingConnectionArray.get(userId); + if (peer === undefined) { + console.warn("Tried to close connection for user "+userId+" but could not find user") + return; + } + // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" + // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. + //console.log('Closing connection with '+userId); + peer.destroy(); + this.PeerScreenSharingConnectionArray.delete(userId) + //console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size); + } catch (err) { + console.error("closeConnection", err) + } + } + public closeAllConnections() { for (const userId of this.PeerConnectionArray.keys()) { this.closeConnection(userId); @@ -244,6 +290,7 @@ export class SimplePeer { * @param data */ private sendWebrtcSignal(data: unknown, userId : string) { + console.log("sendWebrtcSignal", data); try { this.Connection.sendWebrtcSignal(data, this.WebRtcRoomId, null, userId); }catch (e) { @@ -251,6 +298,20 @@ export class SimplePeer { } } + /** + * + * @param userId + * @param data + */ + private sendWebrtcScreenSharingSignal(data: any, userId : string) { + console.log("sendWebrtcScreenSharingSignal", data); + try { + this.Connection.sendWebrtcScreenSharingSignal(data, this.WebRtcRoomId, null, userId); + }catch (e) { + console.error(`sendWebrtcSignal => ${userId}`, e); + } + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any private receiveWebrtcSignal(data: WebRtcSignalMessageInterface) { try { @@ -269,6 +330,24 @@ export class SimplePeer { } } + private receiveWebrtcScreenSharingSignal(data: any) { + console.log("receiveWebrtcScreenSharingSignal", data); + try { + //if offer type, create peer connection + if(data.signal.type === "offer"){ + this.createPeerConnection(data, true); + } + let peer = this.PeerConnectionArray.get(data.userId); + if (peer !== undefined) { + peer.signal(data.signal); + } else { + console.error('Could not find peer whose ID is "'+data.userId+'" in PeerConnectionArray'); + } + } catch (e) { + console.error(`receiveWebrtcSignal => ${data.userId}`, e); + } + } + /** * * @param userId @@ -293,18 +372,8 @@ export class SimplePeer { if (!PeerConnection) { throw new Error('While adding media, cannot find user with ID ' + userId); } - - if(userId.indexOf("screenSharing") > -1 && mediaManager.localScreenCapture){ - for (const track of mediaManager.localScreenCapture.getTracks()) { - PeerConnection.addTrack(track, mediaManager.localScreenCapture); - } - return; - } - let localStream: MediaStream | null = mediaManager.localStream; - let localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; - - PeerConnection.write(new Buffer(JSON.stringify(Object.assign(this.MediaManager.constraintsMedia, {screen: localScreenCapture !== null})))); + PeerConnection.write(new Buffer(JSON.stringify(mediaManager.constraintsMedia))); if(!localStream){ return; @@ -319,6 +388,21 @@ export class SimplePeer { } } + private addMediaScreenSharing (userId : any = null) { + let PeerConnection = this.PeerScreenSharingConnectionArray.get(userId); + if (!PeerConnection) { + throw new Error('While adding media, cannot find user with ID ' + userId); + } + let localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; + if(!localScreenCapture){ + return; + } + /*for (const track of localScreenCapture.getTracks()) { + PeerConnection.addTrack(track, localScreenCapture); + }*/ + return; + } + updatedLocalStream(){ this.Users.forEach((user: UserSimplePeerInterface) => { this.addMedia(user.userId); @@ -326,29 +410,31 @@ export class SimplePeer { } updatedScreenSharing() { - if (this.MediaManager.localScreenCapture) { + if (mediaManager.localScreenCapture) { + if(!this.Connection.userId){ + return; + } let screenSharingUser: UserSimplePeerInterface = { - userId: `screenSharing-${this.Connection.userId}`, - name: 'screenSharing', + userId: this.Connection.userId, initiator: true }; - let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser); + let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true); if (!PeerConnectionScreenSharing) { return; } try { - for (const track of this.MediaManager.localScreenCapture.getTracks()) { - PeerConnectionScreenSharing.addTrack(track, this.MediaManager.localScreenCapture); + for (const track of mediaManager.localScreenCapture.getTracks()) { + PeerConnectionScreenSharing.addTrack(track, mediaManager.localScreenCapture); } }catch (e) { console.error("updatedScreenSharing => ", e); } - this.MediaManager.addStreamRemoteVideo(screenSharingUser.userId, this.MediaManager.localScreenCapture); + mediaManager.addStreamRemoteScreenSharing(screenSharingUser.userId, mediaManager.localScreenCapture); } else { - if (!this.PeerConnectionArray.has(`screenSharing-${this.Connection.userId}`)) { + if (!this.Connection.userId || !this.PeerScreenSharingConnectionArray.has(this.Connection.userId)) { return; } - let PeerConnectionScreenSharing = this.PeerConnectionArray.get(`screenSharing-${this.Connection.userId}`); + let PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(this.Connection.userId); if (!PeerConnectionScreenSharing) { return; } From a8f27e60844fd80257434a0725fb0c3a1e0da2bf Mon Sep 17 00:00:00 2001 From: Gregoire Parant Date: Sun, 14 Jun 2020 14:47:16 +0200 Subject: [PATCH 08/23] Create event to start webrtc screen charing --- back/src/Controller/IoSocketController.ts | 25 ++++- .../Model/Websocket/WebRtcSignalMessage.ts | 11 ++ front/src/Connection.ts | 17 ++- front/src/WebRtc/MediaManager.ts | 5 + front/src/WebRtc/SimplePeer.ts | 103 ++++++++++++------ 5 files changed, 123 insertions(+), 38 deletions(-) diff --git a/back/src/Controller/IoSocketController.ts b/back/src/Controller/IoSocketController.ts index 2f99f1e6..81a7b16b 100644 --- a/back/src/Controller/IoSocketController.ts +++ b/back/src/Controller/IoSocketController.ts @@ -17,7 +17,7 @@ import os from 'os'; import {TokenInterface} from "../Controller/AuthenticateController"; import {isJoinRoomMessageInterface} from "../Model/Websocket/JoinRoomMessage"; import {isPointInterface, PointInterface} from "../Model/Websocket/PointInterface"; -import {isWebRtcSignalMessageInterface} from "../Model/Websocket/WebRtcSignalMessage"; +import {isWebRtcSignalMessageInterface, isWebRtcScreenSharingSignalMessageInterface, isWebRtcScreenSharingStartMessageInterface} from "../Model/Websocket/WebRtcSignalMessage"; import {UserInGroupInterface} from "../Model/Websocket/UserInGroupInterface"; enum SockerIoEvent { @@ -30,6 +30,7 @@ enum SockerIoEvent { WEBRTC_SIGNAL = "webrtc-signal", WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal", WEBRTC_START = "webrtc-start", + WEBRTC_SCREEN_SHARING_START = "webrtc-screen-sharing-start", WEBRTC_DISCONNECT = "webrtc-disconect", MESSAGE_ERROR = "message-error", GROUP_CREATE_UPDATE = "group-create-update", @@ -231,7 +232,17 @@ export class IoSocketController { }); socket.on(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, (data: unknown) => { - this.emit((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL); + this.emitScreenSharing((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL); + }); + + socket.on(SockerIoEvent.WEBRTC_SCREEN_SHARING_START, (data: unknown) => { + console.log(SockerIoEvent.WEBRTC_SCREEN_SHARING_START, data); + if (!isWebRtcScreenSharingStartMessageInterface(data)) { + socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'}); + console.warn('Invalid WEBRTC_SIGNAL message received: ', data); + return; + } + this.Io.in(data.roomId).emit(SockerIoEvent.WEBRTC_SCREEN_SHARING_START, data); }); socket.on(SockerIoEvent.DISCONNECT, () => { @@ -293,6 +304,16 @@ export class IoSocketController { return client.emit(event, data); } + emitScreenSharing(socket: ExSocketInterface, data: unknown, event: SockerIoEvent){ + if (!isWebRtcScreenSharingSignalMessageInterface(data)) { + socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'}); + console.warn('Invalid WEBRTC_SIGNAL message received: ', data); + return; + } + //share at all others clients send only at user + return socket.broadcast.emit(event, data); + } + searchClientByIdOrFail(userId: string): ExSocketInterface { const client: ExSocketInterface|undefined = this.sockets.get(userId); if (client === undefined) { diff --git a/back/src/Model/Websocket/WebRtcSignalMessage.ts b/back/src/Model/Websocket/WebRtcSignalMessage.ts index 7edffdfa..8236d338 100644 --- a/back/src/Model/Websocket/WebRtcSignalMessage.ts +++ b/back/src/Model/Websocket/WebRtcSignalMessage.ts @@ -7,4 +7,15 @@ export const isWebRtcSignalMessageInterface = roomId: tg.isString, signal: tg.isUnknown }).get(); +export const isWebRtcScreenSharingSignalMessageInterface = + new tg.IsInterface().withProperties({ + userId: tg.isString, + roomId: tg.isString, + signal: tg.isUnknown + }).get(); +export const isWebRtcScreenSharingStartMessageInterface = + new tg.IsInterface().withProperties({ + userId: tg.isString, + roomId: tg.isString + }).get(); export type WebRtcSignalMessageInterface = tg.GuardedType; diff --git a/front/src/Connection.ts b/front/src/Connection.ts index bceef68a..50750c59 100644 --- a/front/src/Connection.ts +++ b/front/src/Connection.ts @@ -13,6 +13,7 @@ enum EventMessage{ WEBRTC_SIGNAL = "webrtc-signal", WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal", WEBRTC_START = "webrtc-start", + WEBRTC_SCREEN_SHARING_START = "webrtc-screen-sharing-start", JOIN_ROOM = "join-room", // bi-directional USER_POSITION = "user-position", // bi-directional USER_MOVED = "user-moved", // From server to client @@ -197,10 +198,16 @@ export class Connection implements Connection { }); } - sendWebrtcScreenSharingSignal(signal: any, roomId: string, userId? : string|null, receiverId? : string) { + sendWebrtcScreenSharingStart(roomId: string) { + return this.getSocket().emit(EventMessage.WEBRTC_SCREEN_SHARING_START, { + userId: this.userId, + roomId: roomId + }); + } + + sendWebrtcScreenSharingSignal(signal: any, roomId: string, userId? : string|null) { return this.getSocket().emit(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, { - userId: userId ? userId : this.userId, - receiverId: receiverId ? receiverId : this.userId, + userId: userId, roomId: roomId, signal: signal }); @@ -210,6 +217,10 @@ export class Connection implements Connection { this.socket.on(EventMessage.WEBRTC_START, callback); } + public receiveWebrtcScreenSharingStart(callback: (message: WebRtcDisconnectMessageInterface) => void) { + this.socket.on(EventMessage.WEBRTC_SCREEN_SHARING_START, callback); + } + public receiveWebrtcSignal(callback: (message: WebRtcSignalMessageInterface) => void) { return this.socket.on(EventMessage.WEBRTC_SIGNAL, callback); } diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 706b9f49..167faded 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -405,6 +405,7 @@ export class MediaManager { } isError(userId : string): void { + console.log("isError", `div-${userId}`); const element = document.getElementById(`div-${userId}`); if(!element){ return; @@ -415,6 +416,10 @@ export class MediaManager { } errorDiv.style.display = 'block'; } + isErrorScreenSharing(userId : string): void { + this.isError(`screen-sharing-${userId}`); + } + private getSpinner(userId : string): HTMLDivElement|null { const element = document.getElementById(`div-${userId}`); diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 81bffd6d..0377ea1a 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -63,8 +63,16 @@ export class SimplePeer { this.receiveWebrtcSignal(message); }); + this.Connection.receiveWebrtcScreenSharingStart((message: WebRtcDisconnectMessageInterface) => { + console.log("receiveWebrtcScreenSharingStart => initiator", message.userId === this.Connection.userId); + if(message.userId === this.Connection.userId) { + console.log("receiveWebrtcScreenSharingStart => initiator => create peer connexion"); + this.receiveWebrtcScreenSharingStart(message); + } + }); + //receive signal by gemer - this.Connection.receiveWebrtcScreenSharingSignal((message: any) => { + this.Connection.receiveWebrtcScreenSharingSignal((message: WebRtcDisconnectMessageInterface) => { this.receiveWebrtcScreenSharingSignal(message); }); @@ -98,6 +106,31 @@ export class SimplePeer { this.startWebRtc(); } + private receiveWebrtcScreenSharingStart(data: WebRtcDisconnectMessageInterface) { + console.log("receiveWebrtcScreenSharingStart", data); + let screenSharingUser: UserSimplePeerInterface = { + userId: data.userId, + initiator: this.Connection.userId === data.userId + }; + let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true); + if (!PeerConnectionScreenSharing) { + console.error("receiveWebrtcScreenSharingStart => cannot create peer connexion", PeerConnectionScreenSharing); + return; + } + console.log(`receiveWebrtcScreenSharingStart => ${screenSharingUser.initiator}`, mediaManager.localScreenCapture) + if (!mediaManager.localScreenCapture) { + return; + } + try { + for (const track of mediaManager.localScreenCapture.getTracks()) { + PeerConnectionScreenSharing.addTrack(track, mediaManager.localScreenCapture); + } + } catch (e) { + console.error("updatedScreenSharing => ", e); + } + mediaManager.addStreamRemoteScreenSharing(screenSharingUser.userId, mediaManager.localScreenCapture); + } + /** * server has two people connected, start the meet */ @@ -115,8 +148,10 @@ export class SimplePeer { * create peer connection to bind users */ private createPeerConnection(user : UserSimplePeerInterface, screenSharing: boolean = false) : SimplePeerNamespace.Instance | null{ - console.log("createPeerConnection => screenSharing", screenSharing) - if(this.PeerConnectionArray.has(user.userId)) { + if( + (screenSharing && this.PeerScreenSharingConnectionArray.has(user.userId)) + || (!screenSharing && this.PeerConnectionArray.has(user.userId)) + ){ return null; } @@ -128,14 +163,15 @@ export class SimplePeer { } } - mediaManager.removeActiveVideo(user.userId); if(screenSharing) { + mediaManager.removeActiveScreenSharingVideo(user.userId); mediaManager.addScreenSharingActiveVideo(user.userId); }else{ + mediaManager.removeActiveVideo(user.userId); mediaManager.addActiveVideo(user.userId, name); } - const peer : SimplePeerNamespace.Instance = new Peer({ + const peerOption : SimplePeerNamespace.Instance = new Peer({ initiator: user.initiator ? user.initiator : false, reconnectTimer: 10000, config: { @@ -149,8 +185,10 @@ export class SimplePeer { credential: 'itcugcOHxle9Acqi$' }, ] - }, - }); + } + }; + console.log("peerOption", peerOption); + let peer : SimplePeerNamespace.Instance = new Peer(peerOption); if(screenSharing){ this.PeerScreenSharingConnectionArray.set(user.userId, peer); }else { @@ -159,7 +197,6 @@ export class SimplePeer { //start listen signal for the peer connection peer.on('signal', (data: unknown) => { - console.log("screenSharing", screenSharing); if(screenSharing){ this.sendWebrtcScreenSharingSignal(data, user.userId); return; @@ -168,7 +205,7 @@ export class SimplePeer { }); peer.on('stream', (stream: MediaStream) => { - this.stream(user.userId, stream); + this.stream(user.userId, stream, screenSharing); }); /*peer.on('track', (track: MediaStreamTrack, stream: MediaStream) => { @@ -177,6 +214,7 @@ export class SimplePeer { peer.on('close', () => { if(screenSharing){ this.closeScreenSharingConnection(user.userId); + return; } this.closeConnection(user.userId); }); @@ -184,6 +222,10 @@ export class SimplePeer { // eslint-disable-next-line @typescript-eslint/no-explicit-any peer.on('error', (err: any) => { console.error(`error => ${user.userId} => ${err.code}`, err); + if(screenSharing){ + //mediaManager.isErrorScreenSharing(user.userId); + return; + } mediaManager.isError(user.userId); }); @@ -194,6 +236,7 @@ export class SimplePeer { peer.on('data', (chunk: Buffer) => { let constraint = JSON.parse(chunk.toString('utf8')); + console.log("data", constraint); if (constraint.audio) { mediaManager.enabledMicrophoneByUserId(user.userId); } else { @@ -237,7 +280,8 @@ export class SimplePeer { // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. //console.log('Closing connection with '+userId); peer.destroy(); - this.PeerConnectionArray.delete(userId) + this.PeerConnectionArray.delete(userId); + this.closeScreenSharingConnection(userId); //console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size); for (const peerConnectionListener of this.peerConnectionListeners) { peerConnectionListener.onDisconnect(userId); @@ -306,7 +350,7 @@ export class SimplePeer { private sendWebrtcScreenSharingSignal(data: any, userId : string) { console.log("sendWebrtcScreenSharingSignal", data); try { - this.Connection.sendWebrtcScreenSharingSignal(data, this.WebRtcRoomId, null, userId); + this.Connection.sendWebrtcScreenSharingSignal(data, this.WebRtcRoomId, userId); }catch (e) { console.error(`sendWebrtcSignal => ${userId}`, e); } @@ -337,11 +381,11 @@ export class SimplePeer { if(data.signal.type === "offer"){ this.createPeerConnection(data, true); } - let peer = this.PeerConnectionArray.get(data.userId); + let peer = this.PeerScreenSharingConnectionArray.get(data.userId); if (peer !== undefined) { peer.signal(data.signal); } else { - console.error('Could not find peer whose ID is "'+data.userId+'" in PeerConnectionArray'); + console.error('Could not find peer whose ID is "'+data.userId+'" in receiveWebrtcScreenSharingSignal'); } } catch (e) { console.error(`receiveWebrtcSignal => ${data.userId}`, e); @@ -353,7 +397,16 @@ export class SimplePeer { * @param userId * @param stream */ - private stream(userId : string, stream?: MediaStream) { + private stream(userId : string, stream?: MediaStream, screenSharing?: boolean) { + console.log(`stream => ${userId} => screenSharing => ${screenSharing}`, stream); + if(screenSharing){ + if(!stream){ + mediaManager.removeActiveScreenSharingVideo(userId); + return; + } + mediaManager.addStreamRemoteScreenSharing(userId, stream); + return; + } if(!stream){ mediaManager.disabledVideoByUserId(userId); mediaManager.disabledMicrophoneByUserId(userId); @@ -411,34 +464,18 @@ export class SimplePeer { updatedScreenSharing() { if (mediaManager.localScreenCapture) { - if(!this.Connection.userId){ - return; - } - let screenSharingUser: UserSimplePeerInterface = { - userId: this.Connection.userId, - initiator: true - }; - let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true); - if (!PeerConnectionScreenSharing) { - return; - } - try { - for (const track of mediaManager.localScreenCapture.getTracks()) { - PeerConnectionScreenSharing.addTrack(track, mediaManager.localScreenCapture); - } - }catch (e) { - console.error("updatedScreenSharing => ", e); - } - mediaManager.addStreamRemoteScreenSharing(screenSharingUser.userId, mediaManager.localScreenCapture); + this.Connection.sendWebrtcScreenSharingStart(this.WebRtcRoomId); } else { if (!this.Connection.userId || !this.PeerScreenSharingConnectionArray.has(this.Connection.userId)) { return; } let PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(this.Connection.userId); + console.log("updatedScreenSharing => destroy", PeerConnectionScreenSharing); if (!PeerConnectionScreenSharing) { return; } PeerConnectionScreenSharing.destroy(); + this.PeerScreenSharingConnectionArray.delete(this.Connection.userId); } } } From 4b729581938dbf417cebe6c1589acdcbc9c083f6 Mon Sep 17 00:00:00 2001 From: Gregoire Parant Date: Sun, 14 Jun 2020 20:53:18 +0200 Subject: [PATCH 09/23] Fix peer connexion for two player with screen sharing --- back/src/Controller/IoSocketController.ts | 23 +++++----- front/dist/resources/style/style.css | 18 +++++--- front/src/Connection.ts | 12 ----- front/src/WebRtc/SimplePeer.ts | 56 +++++++++-------------- 4 files changed, 45 insertions(+), 64 deletions(-) diff --git a/back/src/Controller/IoSocketController.ts b/back/src/Controller/IoSocketController.ts index 81a7b16b..0c1956f3 100644 --- a/back/src/Controller/IoSocketController.ts +++ b/back/src/Controller/IoSocketController.ts @@ -30,7 +30,6 @@ enum SockerIoEvent { WEBRTC_SIGNAL = "webrtc-signal", WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal", WEBRTC_START = "webrtc-start", - WEBRTC_SCREEN_SHARING_START = "webrtc-screen-sharing-start", WEBRTC_DISCONNECT = "webrtc-disconect", MESSAGE_ERROR = "message-error", GROUP_CREATE_UPDATE = "group-create-update", @@ -45,6 +44,8 @@ export class IoSocketController { private nbClientsGauge: Gauge; private nbClientsPerRoomGauge: Gauge; + private offerScreenSharingByClient: Map> = new Map>(); + constructor(server: http.Server) { this.Io = socketIO(server); this.nbClientsGauge = new Gauge({ @@ -235,16 +236,6 @@ export class IoSocketController { this.emitScreenSharing((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL); }); - socket.on(SockerIoEvent.WEBRTC_SCREEN_SHARING_START, (data: unknown) => { - console.log(SockerIoEvent.WEBRTC_SCREEN_SHARING_START, data); - if (!isWebRtcScreenSharingStartMessageInterface(data)) { - socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'}); - console.warn('Invalid WEBRTC_SIGNAL message received: ', data); - return; - } - this.Io.in(data.roomId).emit(SockerIoEvent.WEBRTC_SCREEN_SHARING_START, data); - }); - socket.on(SockerIoEvent.DISCONNECT, () => { const Client = (socket as ExSocketInterface); try { @@ -310,8 +301,16 @@ export class IoSocketController { console.warn('Invalid WEBRTC_SIGNAL message received: ', data); return; } + if(data && data.signal && (data.signal as any).type === "offer"){ + let roomOffer = this.offerScreenSharingByClient.get(data.roomId); + if(!roomOffer){ + roomOffer = new Map(); + } + roomOffer.set(data.userId, data.signal); + this.offerScreenSharingByClient.set(data.roomId, roomOffer); + } //share at all others clients send only at user - return socket.broadcast.emit(event, data); + return socket.in(data.roomId).emit(event, data); } searchClientByIdOrFail(userId: string): ExSocketInterface { diff --git a/front/dist/resources/style/style.css b/front/dist/resources/style/style.css index d0dc2cd4..ac1b1527 100644 --- a/front/dist/resources/style/style.css +++ b/front/dist/resources/style/style.css @@ -370,28 +370,34 @@ body { .active-screen-sharing video{ transform: scaleX(1); } +.screen-sharing-video-container { + width: 25%; + position: absolute; +} .active-screen-sharing .screen-sharing-video-container video:hover{ - width: 50%; + width: 200%; + z-index: 11; } .active-screen-sharing .screen-sharing-video-container video{ position: absolute; - width: 25%; + width: 100%; height: auto; left: 0; top: 0; transition: all 0.2s ease; + z-index: 1; } -.active-screen-sharing .screen-sharing-video-container video:nth-child(1){ +.active-screen-sharing .screen-sharing-video-container:nth-child(1){ /*this is for camera of user*/ top: 0%; } -.active-screen-sharing .screen-sharing-video-container video:nth-child(2){ +.active-screen-sharing .screen-sharing-video-container:nth-child(2){ top: 25%; } -.active-screen-sharing .screen-sharing-video-container video:nth-child(3){ +.active-screen-sharing .screen-sharing-video-container:nth-child(3){ top: 50%; } -.active-screen-sharing .screen-sharing-video-container video:nth-child(4) { +.active-screen-sharing .screen-sharing-video-container:nth-child(4) { top: 75%; } diff --git a/front/src/Connection.ts b/front/src/Connection.ts index 50750c59..2ac81f30 100644 --- a/front/src/Connection.ts +++ b/front/src/Connection.ts @@ -13,7 +13,6 @@ enum EventMessage{ WEBRTC_SIGNAL = "webrtc-signal", WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal", WEBRTC_START = "webrtc-start", - WEBRTC_SCREEN_SHARING_START = "webrtc-screen-sharing-start", JOIN_ROOM = "join-room", // bi-directional USER_POSITION = "user-position", // bi-directional USER_MOVED = "user-moved", // From server to client @@ -198,13 +197,6 @@ export class Connection implements Connection { }); } - sendWebrtcScreenSharingStart(roomId: string) { - return this.getSocket().emit(EventMessage.WEBRTC_SCREEN_SHARING_START, { - userId: this.userId, - roomId: roomId - }); - } - sendWebrtcScreenSharingSignal(signal: any, roomId: string, userId? : string|null) { return this.getSocket().emit(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, { userId: userId, @@ -217,10 +209,6 @@ export class Connection implements Connection { this.socket.on(EventMessage.WEBRTC_START, callback); } - public receiveWebrtcScreenSharingStart(callback: (message: WebRtcDisconnectMessageInterface) => void) { - this.socket.on(EventMessage.WEBRTC_SCREEN_SHARING_START, callback); - } - public receiveWebrtcSignal(callback: (message: WebRtcSignalMessageInterface) => void) { return this.socket.on(EventMessage.WEBRTC_SIGNAL, callback); } diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 0377ea1a..e0dbe4a6 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -63,14 +63,6 @@ export class SimplePeer { this.receiveWebrtcSignal(message); }); - this.Connection.receiveWebrtcScreenSharingStart((message: WebRtcDisconnectMessageInterface) => { - console.log("receiveWebrtcScreenSharingStart => initiator", message.userId === this.Connection.userId); - if(message.userId === this.Connection.userId) { - console.log("receiveWebrtcScreenSharingStart => initiator => create peer connexion"); - this.receiveWebrtcScreenSharingStart(message); - } - }); - //receive signal by gemer this.Connection.receiveWebrtcScreenSharingSignal((message: WebRtcDisconnectMessageInterface) => { this.receiveWebrtcScreenSharingSignal(message); @@ -106,31 +98,6 @@ export class SimplePeer { this.startWebRtc(); } - private receiveWebrtcScreenSharingStart(data: WebRtcDisconnectMessageInterface) { - console.log("receiveWebrtcScreenSharingStart", data); - let screenSharingUser: UserSimplePeerInterface = { - userId: data.userId, - initiator: this.Connection.userId === data.userId - }; - let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true); - if (!PeerConnectionScreenSharing) { - console.error("receiveWebrtcScreenSharingStart => cannot create peer connexion", PeerConnectionScreenSharing); - return; - } - console.log(`receiveWebrtcScreenSharingStart => ${screenSharingUser.initiator}`, mediaManager.localScreenCapture) - if (!mediaManager.localScreenCapture) { - return; - } - try { - for (const track of mediaManager.localScreenCapture.getTracks()) { - PeerConnectionScreenSharing.addTrack(track, mediaManager.localScreenCapture); - } - } catch (e) { - console.error("updatedScreenSharing => ", e); - } - mediaManager.addStreamRemoteScreenSharing(screenSharingUser.userId, mediaManager.localScreenCapture); - } - /** * server has two people connected, start the meet */ @@ -464,7 +431,28 @@ export class SimplePeer { updatedScreenSharing() { if (mediaManager.localScreenCapture) { - this.Connection.sendWebrtcScreenSharingStart(this.WebRtcRoomId); + + //this.Connection.sendWebrtcScreenSharingStart(this.WebRtcRoomId); + + if(!this.Connection.userId){ + return; + } + let screenSharingUser: UserSimplePeerInterface = { + userId: this.Connection.userId, + initiator: true + }; + let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true); + if (!PeerConnectionScreenSharing) { + return; + } + try { + for (const track of mediaManager.localScreenCapture.getTracks()) { + PeerConnectionScreenSharing.addTrack(track, mediaManager.localScreenCapture); + } + }catch (e) { + console.error("updatedScreenSharing => ", e); + } + mediaManager.addStreamRemoteScreenSharing(screenSharingUser.userId, mediaManager.localScreenCapture); } else { if (!this.Connection.userId || !this.PeerScreenSharingConnectionArray.has(this.Connection.userId)) { return; From 2e61c2ef6296bd526e397f29587059ee87901ec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Tue, 18 Aug 2020 00:12:38 +0200 Subject: [PATCH 10/23] Getting back code in compilable fashion after huge rebase --- front/src/Connection.ts | 21 ++++----------------- front/src/Phaser/Game/GameScene.ts | 4 ++-- front/src/WebRtc/MediaManager.ts | 27 +++++++++++++++++++-------- front/src/WebRtc/SimplePeer.ts | 20 ++++++++++---------- 4 files changed, 35 insertions(+), 37 deletions(-) diff --git a/front/src/Connection.ts b/front/src/Connection.ts index 2ac81f30..f2c72d64 100644 --- a/front/src/Connection.ts +++ b/front/src/Connection.ts @@ -197,8 +197,8 @@ export class Connection implements Connection { }); } - sendWebrtcScreenSharingSignal(signal: any, roomId: string, userId? : string|null) { - return this.getSocket().emit(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, { + public sendWebrtcScreenSharingSignal(signal: unknown, roomId: string, userId? : string|null) { + return this.socket.emit(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, { userId: userId, roomId: roomId, signal: signal @@ -213,21 +213,8 @@ export class Connection implements Connection { return this.socket.on(EventMessage.WEBRTC_SIGNAL, callback); } - receiveWebrtcScreenSharingSignal(callback: Function) { - return this.getSocket().on(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, callback); - } - - private errorMessage(): void { - this.getSocket().on(EventMessage.MESSAGE_ERROR, (message: string) => { - console.error(EventMessage.MESSAGE_ERROR, message); - }) - } - - private disconnectServer(): void { - this.getSocket().on(EventMessage.CONNECT_ERROR, () => { - this.GameManager.switchToDisconnectedScene(); - }); - + public receiveWebrtcScreenSharingSignal(callback: Function) { + return this.socket.on(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, callback); } public onServerDisconnected(callback: (reason: string) => void): void { diff --git a/front/src/Phaser/Game/GameScene.ts b/front/src/Phaser/Game/GameScene.ts index c4517545..3c3a6536 100644 --- a/front/src/Phaser/Game/GameScene.ts +++ b/front/src/Phaser/Game/GameScene.ts @@ -18,7 +18,7 @@ import {PlayerMovement} from "./PlayerMovement"; import {PlayersPositionInterpolator} from "./PlayersPositionInterpolator"; import {RemotePlayer} from "../Entity/RemotePlayer"; import {Queue} from 'queue-typescript'; -import {SimplePeer, UserSimplePeer} from "../../WebRtc/SimplePeer"; +import {SimplePeer, UserSimplePeerInterface} from "../../WebRtc/SimplePeer"; import {ReconnectingSceneName} from "../Reconnecting/ReconnectingScene"; import {FourOFourSceneName} from "../Reconnecting/FourOFourScene"; import {loadAllLayers} from "../Entity/body_character"; @@ -229,7 +229,7 @@ export class GameScene extends Phaser.Scene { this.simplePeer = new SimplePeer(this.connection); const self = this; this.simplePeer.registerPeerConnectionListener({ - onConnect(user: UserSimplePeer) { + onConnect(user: UserSimplePeerInterface) { self.presentationModeSprite.setVisible(true); self.chatModeSprite.setVisible(true); }, diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 167faded..d7b40c39 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -11,6 +11,7 @@ interface MediaServiceInterface extends MediaDevices{ } type UpdatedLocalStreamCallback = (media: MediaStream) => void; +type UpdatedScreenSharingCallback = (media: MediaStream) => void; // TODO: Split MediaManager in 2 classes: MediaManagerUI (in charge of HTML) and MediaManager (singleton in charge of the camera only) // TODO: verify that microphone event listeners are not triggered plenty of time NOW (since MediaManager is created many times!!!!) @@ -31,11 +32,10 @@ export class MediaManager { video: videoConstraint }; updatedLocalStreamCallBacks : Set = new Set(); - // TODO: updatedScreenSharingCallBack should have same signature as updatedLocalStreamCallBacks - updatedScreenSharingCallBack : Function; + updatedScreenSharingCallBacks : Set = new Set(); - constructor(updatedScreenSharingCallBack : Function) { - this.updatedScreenSharingCallBack = updatedScreenSharingCallBack; + + constructor() { this.myCamVideo = this.getElementByIdOrFail('myCamVideo'); this.webrtcInAudio = this.getElementByIdOrFail('audio-webrtc-in'); @@ -69,14 +69,14 @@ export class MediaManager { //update tracking }); - this.monitorClose = document.getElementById('monitor-close'); + this.monitorClose = this.getElementByIdOrFail('monitor-close'); this.monitorClose.style.display = "block"; this.monitorClose.addEventListener('click', (e: any) => { e.preventDefault(); this.enabledMonitor(); //update tracking }); - this.monitor = document.getElementById('monitor'); + this.monitor = this.getElementByIdOrFail('monitor'); this.monitor.style.display = "none"; this.monitor.addEventListener('click', (e: any) => { e.preventDefault(); @@ -90,6 +90,11 @@ export class MediaManager { this.updatedLocalStreamCallBacks.add(callback); } + onUpdateScreenSharing(callback: UpdatedScreenSharingCallback): void { + + this.updatedScreenSharingCallBacks.add(callback); + } + removeUpdateLocalStreamEventListener(callback: UpdatedLocalStreamCallback): void { this.updatedLocalStreamCallBacks.delete(callback); } @@ -100,6 +105,12 @@ export class MediaManager { } } + private triggerUpdatedScreenSharingCallbacks(stream: MediaStream): void { + for (const callback of this.updatedScreenSharingCallBacks) { + callback(stream); + } + } + activeVisio(){ const gameOverlay = this.getElementByIdOrFail('game-overlay'); gameOverlay.classList.add('active'); @@ -156,7 +167,7 @@ export class MediaManager { this.monitorClose.style.display = "none"; this.monitor.style.display = "block"; this.getScreenMedia().then((stream) => { - this.updatedScreenSharingCallBack(stream); + this.triggerUpdatedScreenSharingCallbacks(stream); }); } @@ -168,7 +179,7 @@ export class MediaManager { }); this.localScreenCapture = null; this.getCamera().then((stream) => { - this.updatedScreenSharingCallBack(stream); + this.triggerUpdatedScreenSharingCallbacks(stream); }); } diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index e0dbe4a6..582b8eb2 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -15,7 +15,7 @@ export interface UserSimplePeerInterface{ } export interface PeerConnectionListener { - onConnect(user: UserSimplePeer): void; + onConnect(user: UserSimplePeerInterface): void; onDisconnect(userId: string): void; } @@ -138,7 +138,7 @@ export class SimplePeer { mediaManager.addActiveVideo(user.userId, name); } - const peerOption : SimplePeerNamespace.Instance = new Peer({ + const peer : SimplePeerNamespace.Instance = new Peer({ initiator: user.initiator ? user.initiator : false, reconnectTimer: 10000, config: { @@ -153,9 +153,7 @@ export class SimplePeer { }, ] } - }; - console.log("peerOption", peerOption); - let peer : SimplePeerNamespace.Instance = new Peer(peerOption); + }); if(screenSharing){ this.PeerScreenSharingConnectionArray.set(user.userId, peer); }else { @@ -434,11 +432,12 @@ export class SimplePeer { //this.Connection.sendWebrtcScreenSharingStart(this.WebRtcRoomId); - if(!this.Connection.userId){ + const userId = this.Connection.getUserId(); + if(!userId){ return; } let screenSharingUser: UserSimplePeerInterface = { - userId: this.Connection.userId, + userId, initiator: true }; let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true); @@ -454,16 +453,17 @@ export class SimplePeer { } mediaManager.addStreamRemoteScreenSharing(screenSharingUser.userId, mediaManager.localScreenCapture); } else { - if (!this.Connection.userId || !this.PeerScreenSharingConnectionArray.has(this.Connection.userId)) { + const userId = this.Connection.getUserId(); + if (!userId || !this.PeerScreenSharingConnectionArray.has(userId)) { return; } - let PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(this.Connection.userId); + let PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(userId); console.log("updatedScreenSharing => destroy", PeerConnectionScreenSharing); if (!PeerConnectionScreenSharing) { return; } PeerConnectionScreenSharing.destroy(); - this.PeerScreenSharingConnectionArray.delete(this.Connection.userId); + this.PeerScreenSharingConnectionArray.delete(userId); } } } From cc1cb2f671c42574e115fe1c8b7b28bbe84ef2dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Tue, 18 Aug 2020 14:59:50 +0200 Subject: [PATCH 11/23] Fixing linting --- front/src/Connection.ts | 10 ++++++++-- front/src/WebRtc/MediaManager.ts | 26 +++++++++++++------------- front/src/WebRtc/SimplePeer.ts | 32 ++++++++++++++++---------------- 3 files changed, 37 insertions(+), 31 deletions(-) diff --git a/front/src/Connection.ts b/front/src/Connection.ts index f2c72d64..b7926328 100644 --- a/front/src/Connection.ts +++ b/front/src/Connection.ts @@ -81,7 +81,13 @@ export interface WebRtcDisconnectMessageInterface { export interface WebRtcSignalMessageInterface { userId: string, - receiverId: string, + receiverId: string, // TODO: is this needed? (can we merge this with WebRtcScreenSharingMessageInterface?) + roomId: string, + signal: SignalData +} + +export interface WebRtcScreenSharingMessageInterface { + userId: string, roomId: string, signal: SignalData } @@ -213,7 +219,7 @@ export class Connection implements Connection { return this.socket.on(EventMessage.WEBRTC_SIGNAL, callback); } - public receiveWebrtcScreenSharingSignal(callback: Function) { + public receiveWebrtcScreenSharingSignal(callback: (message: WebRtcScreenSharingMessageInterface) => void) { return this.socket.on(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, callback); } diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index d7b40c39..b63dc515 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -6,9 +6,6 @@ const videoConstraint: boolean|MediaTrackConstraints = { height: { ideal: 720 }, facingMode: "user" }; -interface MediaServiceInterface extends MediaDevices{ - getDisplayMedia(constrain: any) : Promise; -} type UpdatedLocalStreamCallback = (media: MediaStream) => void; type UpdatedScreenSharingCallback = (media: MediaStream) => void; @@ -71,14 +68,14 @@ export class MediaManager { this.monitorClose = this.getElementByIdOrFail('monitor-close'); this.monitorClose.style.display = "block"; - this.monitorClose.addEventListener('click', (e: any) => { + this.monitorClose.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); this.enabledMonitor(); //update tracking }); this.monitor = this.getElementByIdOrFail('monitor'); this.monitor.style.display = "none"; - this.monitor.addEventListener('click', (e: any) => { + this.monitor.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); this.disabledMonitor(); //update tracking @@ -191,8 +188,8 @@ export class MediaManager { this.localScreenCapture = stream; return stream; }) - .catch((err: any) => { - console.error("Error => getScreenMedia => " + err); + .catch((err: unknown) => { + console.error("Error => getScreenMedia => ", err); throw err; }); }catch (err) { @@ -203,10 +200,13 @@ export class MediaManager { } private _startScreenCapture() { - if ((navigator as any).getDisplayMedia) { - return (navigator as any).getDisplayMedia({video: true}); - } else if ((navigator.mediaDevices as any).getDisplayMedia) { - return (navigator.mediaDevices as any).getDisplayMedia({video: true}); + // getDisplayMedia was moved to mediaDevices in 2018. Typescript definitions are not up to date yet. + // See: https://github.com/w3c/mediacapture-screen-share/pull/86 + // https://github.com/microsoft/TypeScript/issues/31821 + if ((navigator as any).getDisplayMedia) { // eslint-disable-line @typescript-eslint/no-explicit-any + return (navigator as any).getDisplayMedia({video: true}); // eslint-disable-line @typescript-eslint/no-explicit-any + } else if ((navigator.mediaDevices as any).getDisplayMedia) { // eslint-disable-line @typescript-eslint/no-explicit-any + return (navigator.mediaDevices as any).getDisplayMedia({video: true}); // eslint-disable-line @typescript-eslint/no-explicit-any } else { //return navigator.mediaDevices.getUserMedia(({video: {mediaSource: 'screen'}} as any)); return new Promise((resolve, reject) => { // eslint-disable-line no-unused-vars @@ -302,13 +302,13 @@ export class MediaManager { userId = `screen-sharing-${userId}`; this.webrtcInAudio.play(); // FIXME: switch to DisplayManager! - let elementRemoteVideo = this.getElementByIdOrFail("activeScreenSharing"); + const elementRemoteVideo = this.getElementByIdOrFail("activeScreenSharing"); elementRemoteVideo.insertAdjacentHTML('beforeend', `
`); - let activeHTMLVideoElement : HTMLElement|null = document.getElementById(userId); + const activeHTMLVideoElement : HTMLElement|null = document.getElementById(userId); if(!activeHTMLVideoElement){ return; } diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 582b8eb2..72786d63 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -1,6 +1,6 @@ import { Connection, - WebRtcDisconnectMessageInterface, + WebRtcDisconnectMessageInterface, WebRtcScreenSharingMessageInterface, WebRtcSignalMessageInterface, WebRtcStartMessageInterface } from "../Connection"; @@ -64,7 +64,7 @@ export class SimplePeer { }); //receive signal by gemer - this.Connection.receiveWebrtcScreenSharingSignal((message: WebRtcDisconnectMessageInterface) => { + this.Connection.receiveWebrtcScreenSharingSignal((message: WebRtcScreenSharingMessageInterface) => { this.receiveWebrtcScreenSharingSignal(message); }); @@ -200,7 +200,7 @@ export class SimplePeer { }); peer.on('data', (chunk: Buffer) => { - let constraint = JSON.parse(chunk.toString('utf8')); + const constraint = JSON.parse(chunk.toString('utf8')); console.log("data", constraint); if (constraint.audio) { mediaManager.enabledMicrophoneByUserId(user.userId); @@ -264,7 +264,7 @@ export class SimplePeer { private closeScreenSharingConnection(userId : string) { try { mediaManager.removeActiveScreenSharingVideo(userId); - let peer = this.PeerScreenSharingConnectionArray.get(userId); + const peer = this.PeerScreenSharingConnectionArray.get(userId); if (peer === undefined) { console.warn("Tried to close connection for user "+userId+" but could not find user") return; @@ -312,12 +312,12 @@ export class SimplePeer { * @param userId * @param data */ - private sendWebrtcScreenSharingSignal(data: any, userId : string) { + private sendWebrtcScreenSharingSignal(data: unknown, userId : string) { console.log("sendWebrtcScreenSharingSignal", data); try { this.Connection.sendWebrtcScreenSharingSignal(data, this.WebRtcRoomId, userId); }catch (e) { - console.error(`sendWebrtcSignal => ${userId}`, e); + console.error(`sendWebrtcScreenSharingSignal => ${userId}`, e); } } @@ -339,14 +339,14 @@ export class SimplePeer { } } - private receiveWebrtcScreenSharingSignal(data: any) { + private receiveWebrtcScreenSharingSignal(data: WebRtcScreenSharingMessageInterface) { console.log("receiveWebrtcScreenSharingSignal", data); try { //if offer type, create peer connection if(data.signal.type === "offer"){ this.createPeerConnection(data, true); } - let peer = this.PeerScreenSharingConnectionArray.get(data.userId); + const peer = this.PeerScreenSharingConnectionArray.get(data.userId); if (peer !== undefined) { peer.signal(data.signal); } else { @@ -386,11 +386,11 @@ export class SimplePeer { */ private addMedia (userId : string) { try { - let PeerConnection = this.PeerConnectionArray.get(userId); + const PeerConnection = this.PeerConnectionArray.get(userId); if (!PeerConnection) { throw new Error('While adding media, cannot find user with ID ' + userId); } - let localStream: MediaStream | null = mediaManager.localStream; + const localStream: MediaStream | null = mediaManager.localStream; PeerConnection.write(new Buffer(JSON.stringify(mediaManager.constraintsMedia))); if(!localStream){ @@ -406,12 +406,12 @@ export class SimplePeer { } } - private addMediaScreenSharing (userId : any = null) { - let PeerConnection = this.PeerScreenSharingConnectionArray.get(userId); + private addMediaScreenSharing(userId : string) { + const PeerConnection = this.PeerScreenSharingConnectionArray.get(userId); if (!PeerConnection) { throw new Error('While adding media, cannot find user with ID ' + userId); } - let localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; + const localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; if(!localScreenCapture){ return; } @@ -436,11 +436,11 @@ export class SimplePeer { if(!userId){ return; } - let screenSharingUser: UserSimplePeerInterface = { + const screenSharingUser: UserSimplePeerInterface = { userId, initiator: true }; - let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true); + const PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true); if (!PeerConnectionScreenSharing) { return; } @@ -457,7 +457,7 @@ export class SimplePeer { if (!userId || !this.PeerScreenSharingConnectionArray.has(userId)) { return; } - let PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(userId); + const PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(userId); console.log("updatedScreenSharing => destroy", PeerConnectionScreenSharing); if (!PeerConnectionScreenSharing) { return; From 6c5772e84986187b568d39a4e6a709696e99a033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Tue, 18 Aug 2020 15:31:42 +0200 Subject: [PATCH 12/23] Fixing typipng in back --- back/src/Controller/IoSocketController.ts | 2 +- back/src/Model/Websocket/WebRtcSignalMessage.ts | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/back/src/Controller/IoSocketController.ts b/back/src/Controller/IoSocketController.ts index 0c1956f3..923f36a9 100644 --- a/back/src/Controller/IoSocketController.ts +++ b/back/src/Controller/IoSocketController.ts @@ -301,7 +301,7 @@ export class IoSocketController { console.warn('Invalid WEBRTC_SIGNAL message received: ', data); return; } - if(data && data.signal && (data.signal as any).type === "offer"){ + if(data.signal.type === "offer"){ let roomOffer = this.offerScreenSharingByClient.get(data.roomId); if(!roomOffer){ roomOffer = new Map(); diff --git a/back/src/Model/Websocket/WebRtcSignalMessage.ts b/back/src/Model/Websocket/WebRtcSignalMessage.ts index 8236d338..56a19060 100644 --- a/back/src/Model/Websocket/WebRtcSignalMessage.ts +++ b/back/src/Model/Websocket/WebRtcSignalMessage.ts @@ -1,17 +1,22 @@ import * as tg from "generic-type-guard"; +export const isSignalData = + new tg.IsInterface().withProperties({ + type: tg.isOptional(tg.isString) + }).get(); + export const isWebRtcSignalMessageInterface = new tg.IsInterface().withProperties({ userId: tg.isString, receiverId: tg.isString, roomId: tg.isString, - signal: tg.isUnknown + signal: isSignalData }).get(); export const isWebRtcScreenSharingSignalMessageInterface = new tg.IsInterface().withProperties({ userId: tg.isString, roomId: tg.isString, - signal: tg.isUnknown + signal: isSignalData }).get(); export const isWebRtcScreenSharingStartMessageInterface = new tg.IsInterface().withProperties({ @@ -19,3 +24,4 @@ export const isWebRtcScreenSharingStartMessageInterface = roomId: tg.isString }).get(); export type WebRtcSignalMessageInterface = tg.GuardedType; +export type WebRtcScreenSharingMessageInterface = tg.GuardedType; From 011953428359e28eb55320172023b65cb895579d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 20 Aug 2020 00:05:00 +0200 Subject: [PATCH 13/23] First version of screen-sharing that works when a user is joining a group after screen sharing begun. --- back/src/Controller/IoSocketController.ts | 39 +++-- .../Model/Websocket/WebRtcSignalMessage.ts | 7 - front/dist/resources/style/style.css | 36 ----- front/src/Connection.ts | 17 +- front/src/WebRtc/MediaManager.ts | 51 +++--- front/src/WebRtc/SimplePeer.ts | 148 ++++++++++-------- 6 files changed, 130 insertions(+), 168 deletions(-) diff --git a/back/src/Controller/IoSocketController.ts b/back/src/Controller/IoSocketController.ts index 923f36a9..501c6145 100644 --- a/back/src/Controller/IoSocketController.ts +++ b/back/src/Controller/IoSocketController.ts @@ -44,8 +44,6 @@ export class IoSocketController { private nbClientsGauge: Gauge; private nbClientsPerRoomGauge: Gauge; - private offerScreenSharingByClient: Map> = new Map>(); - constructor(server: http.Server) { this.Io = socketIO(server); this.nbClientsGauge = new Gauge({ @@ -229,11 +227,11 @@ export class IoSocketController { }); socket.on(SockerIoEvent.WEBRTC_SIGNAL, (data: unknown) => { - this.emit((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SIGNAL); + this.emitVideo((socket as ExSocketInterface), data); }); socket.on(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, (data: unknown) => { - this.emitScreenSharing((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL); + this.emitScreenSharing((socket as ExSocketInterface), data); }); socket.on(SockerIoEvent.DISCONNECT, () => { @@ -280,7 +278,7 @@ export class IoSocketController { }); } - emit(socket: ExSocketInterface, data: unknown, event: SockerIoEvent){ + emitVideo(socket: ExSocketInterface, data: unknown){ if (!isWebRtcSignalMessageInterface(data)) { socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'}); console.warn('Invalid WEBRTC_SIGNAL message received: ', data); @@ -292,25 +290,22 @@ export class IoSocketController { console.warn("While exchanging a WebRTC signal: client with id ", data.receiverId, " does not exist. This might be a race condition."); return; } - return client.emit(event, data); + return client.emit(SockerIoEvent.WEBRTC_SIGNAL, data); } - emitScreenSharing(socket: ExSocketInterface, data: unknown, event: SockerIoEvent){ - if (!isWebRtcScreenSharingSignalMessageInterface(data)) { - socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'}); - console.warn('Invalid WEBRTC_SIGNAL message received: ', data); + emitScreenSharing(socket: ExSocketInterface, data: unknown){ + if (!isWebRtcSignalMessageInterface(data)) { + socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SCREEN_SHARING message.'}); + console.warn('Invalid WEBRTC_SCREEN_SHARING message received: ', data); return; } - if(data.signal.type === "offer"){ - let roomOffer = this.offerScreenSharingByClient.get(data.roomId); - if(!roomOffer){ - roomOffer = new Map(); - } - roomOffer.set(data.userId, data.signal); - this.offerScreenSharingByClient.set(data.roomId, roomOffer); + //send only at user + const client = this.sockets.get(data.receiverId); + if (client === undefined) { + console.warn("While exchanging a WEBRTC_SCREEN_SHARING signal: client with id ", data.receiverId, " does not exist. This might be a race condition."); + return; } - //share at all others clients send only at user - return socket.in(data.roomId).emit(event, data); + return client.emit(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, data); } searchClientByIdOrFail(userId: string): ExSocketInterface { @@ -393,13 +388,15 @@ export class IoSocketController { if (this.Io.sockets.adapter.rooms[roomId].length < 2 /*|| this.Io.sockets.adapter.rooms[roomId].length >= 4*/) { return; } + + // TODO: scanning all sockets is maybe not the most efficient const clients: Array = (Object.values(this.Io.sockets.sockets) as Array) .filter((client: ExSocketInterface) => client.webRtcRoomId && client.webRtcRoomId === roomId); //send start at one client to initialise offer webrtc //send all users in room to create PeerConnection in front clients.forEach((client: ExSocketInterface, index: number) => { - const clientsId = clients.reduce((tabs: Array, clientId: ExSocketInterface, indexClientId: number) => { + const peerClients = clients.reduce((tabs: Array, clientId: ExSocketInterface, indexClientId: number) => { if (!clientId.userId || clientId.userId === client.userId) { return tabs; } @@ -411,7 +408,7 @@ export class IoSocketController { return tabs; }, []); - client.emit(SockerIoEvent.WEBRTC_START, {clients: clientsId, roomId: roomId}); + client.emit(SockerIoEvent.WEBRTC_START, {clients: peerClients, roomId: roomId}); }); } diff --git a/back/src/Model/Websocket/WebRtcSignalMessage.ts b/back/src/Model/Websocket/WebRtcSignalMessage.ts index 56a19060..4f59f617 100644 --- a/back/src/Model/Websocket/WebRtcSignalMessage.ts +++ b/back/src/Model/Websocket/WebRtcSignalMessage.ts @@ -12,16 +12,9 @@ export const isWebRtcSignalMessageInterface = roomId: tg.isString, signal: isSignalData }).get(); -export const isWebRtcScreenSharingSignalMessageInterface = - new tg.IsInterface().withProperties({ - userId: tg.isString, - roomId: tg.isString, - signal: isSignalData - }).get(); export const isWebRtcScreenSharingStartMessageInterface = new tg.IsInterface().withProperties({ userId: tg.isString, roomId: tg.isString }).get(); export type WebRtcSignalMessageInterface = tg.GuardedType; -export type WebRtcScreenSharingMessageInterface = tg.GuardedType; diff --git a/front/dist/resources/style/style.css b/front/dist/resources/style/style.css index ac1b1527..413bce71 100644 --- a/front/dist/resources/style/style.css +++ b/front/dist/resources/style/style.css @@ -365,39 +365,3 @@ body { .chat-mode > div:last-child { flex-grow: 5; } - -/*SCREEN SHARING*/ -.active-screen-sharing video{ - transform: scaleX(1); -} -.screen-sharing-video-container { - width: 25%; - position: absolute; -} -.active-screen-sharing .screen-sharing-video-container video:hover{ - width: 200%; - z-index: 11; -} -.active-screen-sharing .screen-sharing-video-container video{ - position: absolute; - width: 100%; - height: auto; - left: 0; - top: 0; - transition: all 0.2s ease; - z-index: 1; -} - -.active-screen-sharing .screen-sharing-video-container:nth-child(1){ - /*this is for camera of user*/ - top: 0%; -} -.active-screen-sharing .screen-sharing-video-container:nth-child(2){ - top: 25%; -} -.active-screen-sharing .screen-sharing-video-container:nth-child(3){ - top: 50%; -} -.active-screen-sharing .screen-sharing-video-container:nth-child(4) { - top: 75%; -} diff --git a/front/src/Connection.ts b/front/src/Connection.ts index b7926328..ec1db6b1 100644 --- a/front/src/Connection.ts +++ b/front/src/Connection.ts @@ -80,14 +80,8 @@ export interface WebRtcDisconnectMessageInterface { } export interface WebRtcSignalMessageInterface { - userId: string, - receiverId: string, // TODO: is this needed? (can we merge this with WebRtcScreenSharingMessageInterface?) - roomId: string, - signal: SignalData -} - -export interface WebRtcScreenSharingMessageInterface { - userId: string, + userId: string, // TODO: is this needed? + receiverId: string, roomId: string, signal: SignalData } @@ -203,9 +197,10 @@ export class Connection implements Connection { }); } - public sendWebrtcScreenSharingSignal(signal: unknown, roomId: string, userId? : string|null) { + public sendWebrtcScreenSharingSignal(signal: unknown, roomId: string, userId? : string|null, receiverId? : string) { return this.socket.emit(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, { - userId: userId, + userId: userId ? userId : this.userId, + receiverId: receiverId ? receiverId : this.userId, roomId: roomId, signal: signal }); @@ -219,7 +214,7 @@ export class Connection implements Connection { return this.socket.on(EventMessage.WEBRTC_SIGNAL, callback); } - public receiveWebrtcScreenSharingSignal(callback: (message: WebRtcScreenSharingMessageInterface) => void) { + public receiveWebrtcScreenSharingSignal(callback: (message: WebRtcSignalMessageInterface) => void) { return this.socket.on(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, callback); } diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index b63dc515..635174be 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -42,13 +42,13 @@ export class MediaManager { this.microphoneClose.style.display = "none"; this.microphoneClose.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.enabledMicrophone(); + this.enableMicrophone(); //update tracking }); this.microphone = this.getElementByIdOrFail('microphone'); this.microphone.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.disabledMicrophone(); + this.disableMicrophone(); //update tracking }); @@ -56,13 +56,13 @@ export class MediaManager { this.cinemaClose.style.display = "none"; this.cinemaClose.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.enabledCamera(); + this.enableCamera(); //update tracking }); this.cinema = this.getElementByIdOrFail('cinema'); this.cinema.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.disabledCamera(); + this.disableCamera(); //update tracking }); @@ -70,24 +70,24 @@ export class MediaManager { this.monitorClose.style.display = "block"; this.monitorClose.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.enabledMonitor(); + this.enableScreenSharing(); //update tracking }); this.monitor = this.getElementByIdOrFail('monitor'); this.monitor.style.display = "none"; this.monitor.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.disabledMonitor(); + this.disableScreenSharing(); //update tracking }); } - onUpdateLocalStream(callback: UpdatedLocalStreamCallback): void { + public onUpdateLocalStream(callback: UpdatedLocalStreamCallback): void { this.updatedLocalStreamCallBacks.add(callback); } - onUpdateScreenSharing(callback: UpdatedScreenSharingCallback): void { + public onUpdateScreenSharing(callback: UpdatedScreenSharingCallback): void { this.updatedScreenSharingCallBacks.add(callback); } @@ -108,12 +108,12 @@ export class MediaManager { } } - activeVisio(){ + showGameOverlay(){ const gameOverlay = this.getElementByIdOrFail('game-overlay'); gameOverlay.classList.add('active'); } - enabledCamera() { + private enableCamera() { this.cinemaClose.style.display = "none"; this.cinema.style.display = "block"; this.constraintsMedia.video = videoConstraint; @@ -122,7 +122,7 @@ export class MediaManager { }); } - disabledCamera() { + private disableCamera() { this.cinemaClose.style.display = "block"; this.cinema.style.display = "none"; this.constraintsMedia.video = false; @@ -137,7 +137,7 @@ export class MediaManager { }); } - enabledMicrophone() { + private enableMicrophone() { this.microphoneClose.style.display = "none"; this.microphone.style.display = "block"; this.constraintsMedia.audio = true; @@ -146,7 +146,7 @@ export class MediaManager { }); } - disabledMicrophone() { + private disableMicrophone() { this.microphoneClose.style.display = "block"; this.microphone.style.display = "none"; this.constraintsMedia.audio = false; @@ -160,7 +160,7 @@ export class MediaManager { }); } - enabledMonitor() { + private enableScreenSharing() { this.monitorClose.style.display = "none"; this.monitor.style.display = "block"; this.getScreenMedia().then((stream) => { @@ -168,7 +168,7 @@ export class MediaManager { }); } - disabledMonitor() { + private disableScreenSharing() { this.monitorClose.style.display = "block"; this.monitor.style.display = "none"; this.localScreenCapture?.getTracks().forEach((track: MediaStreamTrack) => { @@ -299,21 +299,18 @@ export class MediaManager { * @param userId */ addScreenSharingActiveVideo(userId : string){ - userId = `screen-sharing-${userId}`; this.webrtcInAudio.play(); - // FIXME: switch to DisplayManager! - const elementRemoteVideo = this.getElementByIdOrFail("activeScreenSharing"); - elementRemoteVideo.insertAdjacentHTML('beforeend', ` -
+ + userId = `screen-sharing-${userId}`; + const html = ` +
- `); - const activeHTMLVideoElement : HTMLElement|null = document.getElementById(userId); - if(!activeHTMLVideoElement){ - return; - } - console.log(userId, (activeHTMLVideoElement as HTMLVideoElement)); - this.remoteVideo.set(userId, (activeHTMLVideoElement as HTMLVideoElement)); + `; + + layoutManager.add(DivImportance.Important, userId, html); + + this.remoteVideo.set(userId, this.getElementByIdOrFail(userId)); } /** diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 72786d63..0d2dd068 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -1,6 +1,6 @@ import { Connection, - WebRtcDisconnectMessageInterface, WebRtcScreenSharingMessageInterface, + WebRtcDisconnectMessageInterface, WebRtcSignalMessageInterface, WebRtcStartMessageInterface } from "../Connection"; @@ -30,18 +30,18 @@ export class SimplePeer { private PeerScreenSharingConnectionArray: Map = new Map(); private PeerConnectionArray: Map = new Map(); - private readonly updateLocalStreamCallback: (media: MediaStream) => void; - private readonly updateScreenSharingCallback: (media: MediaStream) => void; + private readonly sendLocalVideoStreamCallback: (media: MediaStream) => void; + private readonly sendLocalScreenSharingStreamCallback: (media: MediaStream) => void; private readonly peerConnectionListeners: Array = new Array(); constructor(Connection: Connection, WebRtcRoomId: string = "test-webrtc") { this.Connection = Connection; this.WebRtcRoomId = WebRtcRoomId; // We need to go through this weird bound function pointer in order to be able to "free" this reference later. - this.updateLocalStreamCallback = this.updatedLocalStream.bind(this); - this.updateScreenSharingCallback = this.updatedScreenSharing.bind(this); - mediaManager.onUpdateLocalStream(this.updateLocalStreamCallback); - mediaManager.onUpdateScreenSharing(this.updateScreenSharingCallback); + this.sendLocalVideoStreamCallback = this.sendLocalVideoStream.bind(this); + this.sendLocalScreenSharingStreamCallback = this.sendLocalScreenSharingStream.bind(this); + mediaManager.onUpdateLocalStream(this.sendLocalVideoStreamCallback); + mediaManager.onUpdateScreenSharing(this.sendLocalScreenSharingStreamCallback); this.initialise(); } @@ -64,11 +64,11 @@ export class SimplePeer { }); //receive signal by gemer - this.Connection.receiveWebrtcScreenSharingSignal((message: WebRtcScreenSharingMessageInterface) => { + this.Connection.receiveWebrtcScreenSharingSignal((message: WebRtcSignalMessageInterface) => { this.receiveWebrtcScreenSharingSignal(message); }); - mediaManager.activeVisio(); + mediaManager.showGameOverlay(); mediaManager.getCamera().then(() => { //receive message start @@ -88,7 +88,7 @@ export class SimplePeer { private receiveWebrtcStart(data: WebRtcStartMessageInterface) { this.WebRtcRoomId = data.roomId; this.Users = data.clients; - // Note: the clients array contain the list of all clients (event the ones we are already connected to in case a user joints a group) + // Note: the clients array contain the list of all clients (even the ones we are already connected to in case a user joints a group) // So we can receive a request we already had before. (which will abort at the first line of createPeerConnection) // TODO: refactor this to only send a message to connect to one user (rather than several users). // This would be symmetrical to the way we handle disconnection. @@ -102,6 +102,7 @@ export class SimplePeer { * server has two people connected, start the meet */ private startWebRtc() { + console.warn('startWebRtc startWebRtc'); this.Users.forEach((user: UserSimplePeerInterface) => { //if it's not an initiator, peer connection will be created when gamer will receive offer signal if(!user.initiator){ @@ -131,8 +132,11 @@ export class SimplePeer { } if(screenSharing) { - mediaManager.removeActiveScreenSharingVideo(user.userId); - mediaManager.addScreenSharingActiveVideo(user.userId); + // We should display the screen sharing ONLY if we are not initiator + if (!user.initiator) { + mediaManager.removeActiveScreenSharingVideo(user.userId); + mediaManager.addScreenSharingActiveVideo(user.userId); + } }else{ mediaManager.removeActiveVideo(user.userId); mediaManager.addActiveVideo(user.userId, name); @@ -156,17 +160,18 @@ export class SimplePeer { }); if(screenSharing){ this.PeerScreenSharingConnectionArray.set(user.userId, peer); - }else { + } else { this.PeerConnectionArray.set(user.userId, peer); } //start listen signal for the peer connection peer.on('signal', (data: unknown) => { if(screenSharing){ + //console.log('Sending WebRTC offer for screen sharing ', data, ' to ', user.userId); this.sendWebrtcScreenSharingSignal(data, user.userId); - return; + } else { + this.sendWebrtcSignal(data, user.userId); } - this.sendWebrtcSignal(data, user.userId); }); peer.on('stream', (stream: MediaStream) => { @@ -197,6 +202,12 @@ export class SimplePeer { peer.on('connect', () => { mediaManager.isConnected(user.userId); console.info(`connect => ${user.userId}`); + + // When a connection is established to a video stream, and if a screen sharing is taking place, + // the user sharing screen should also initiate a connection to the remote user! + if (screenSharing === false && mediaManager.localScreenCapture) { + this.sendLocalScreenSharingStreamToUser(user.userId); + } }); peer.on('data', (chunk: Buffer) => { @@ -217,9 +228,9 @@ export class SimplePeer { }); if(screenSharing){ - this.addMediaScreenSharing(user.userId); + this.pushScreenSharingToRemoteUser(user.userId); }else { - this.addMedia(user.userId); + this.pushVideoToRemoteUser(user.userId); } for (const peerConnectionListener of this.peerConnectionListeners) { @@ -290,7 +301,7 @@ export class SimplePeer { * Unregisters any held event handler. */ public unregister() { - mediaManager.removeUpdateLocalStreamEventListener(this.updateLocalStreamCallback); + mediaManager.removeUpdateLocalStreamEventListener(this.sendLocalVideoStreamCallback); } /** @@ -299,7 +310,6 @@ export class SimplePeer { * @param data */ private sendWebrtcSignal(data: unknown, userId : string) { - console.log("sendWebrtcSignal", data); try { this.Connection.sendWebrtcSignal(data, this.WebRtcRoomId, null, userId); }catch (e) { @@ -315,7 +325,7 @@ export class SimplePeer { private sendWebrtcScreenSharingSignal(data: unknown, userId : string) { console.log("sendWebrtcScreenSharingSignal", data); try { - this.Connection.sendWebrtcScreenSharingSignal(data, this.WebRtcRoomId, userId); + this.Connection.sendWebrtcScreenSharingSignal(data, this.WebRtcRoomId, null, userId); }catch (e) { console.error(`sendWebrtcScreenSharingSignal => ${userId}`, e); } @@ -339,7 +349,7 @@ export class SimplePeer { } } - private receiveWebrtcScreenSharingSignal(data: WebRtcScreenSharingMessageInterface) { + private receiveWebrtcScreenSharingSignal(data: WebRtcSignalMessageInterface) { console.log("receiveWebrtcScreenSharingSignal", data); try { //if offer type, create peer connection @@ -384,7 +394,7 @@ export class SimplePeer { * * @param userId */ - private addMedia (userId : string) { + private pushVideoToRemoteUser(userId : string) { try { const PeerConnection = this.PeerConnectionArray.get(userId); if (!PeerConnection) { @@ -396,74 +406,80 @@ export class SimplePeer { if(!localStream){ return; } - if (localStream) { - for (const track of localStream.getTracks()) { - PeerConnection.addTrack(track, localStream); - } + + for (const track of localStream.getTracks()) { + PeerConnection.addTrack(track, localStream); } }catch (e) { - console.error(`addMedia => addMedia => ${userId}`, e); + console.error(`pushVideoToRemoteUser => ${userId}`, e); } } - private addMediaScreenSharing(userId : string) { + private pushScreenSharingToRemoteUser(userId : string) { const PeerConnection = this.PeerScreenSharingConnectionArray.get(userId); if (!PeerConnection) { - throw new Error('While adding media, cannot find user with ID ' + userId); + throw new Error('While pushing screen sharing, cannot find user with ID ' + userId); } const localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; if(!localScreenCapture){ return; } - /*for (const track of localScreenCapture.getTracks()) { + + for (const track of localScreenCapture.getTracks()) { PeerConnection.addTrack(track, localScreenCapture); - }*/ + } return; } - updatedLocalStream(){ + public sendLocalVideoStream(){ this.Users.forEach((user: UserSimplePeerInterface) => { - this.addMedia(user.userId); + this.pushVideoToRemoteUser(user.userId); }) } - updatedScreenSharing() { + /** + * Triggered locally when clicking on the screen sharing button + */ + public sendLocalScreenSharingStream() { if (mediaManager.localScreenCapture) { - - //this.Connection.sendWebrtcScreenSharingStart(this.WebRtcRoomId); - - const userId = this.Connection.getUserId(); - if(!userId){ - return; + for (const user of this.Users) { + this.sendLocalScreenSharingStreamToUser(user.userId); } - const screenSharingUser: UserSimplePeerInterface = { - userId, - initiator: true - }; - const PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true); - if (!PeerConnectionScreenSharing) { - return; - } - try { - for (const track of mediaManager.localScreenCapture.getTracks()) { - PeerConnectionScreenSharing.addTrack(track, mediaManager.localScreenCapture); - } - }catch (e) { - console.error("updatedScreenSharing => ", e); - } - mediaManager.addStreamRemoteScreenSharing(screenSharingUser.userId, mediaManager.localScreenCapture); } else { - const userId = this.Connection.getUserId(); - if (!userId || !this.PeerScreenSharingConnectionArray.has(userId)) { - return; + for (const user of this.Users) { + this.stopLocalScreenSharingStreamToUser(user.userId); } - const PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(userId); - console.log("updatedScreenSharing => destroy", PeerConnectionScreenSharing); - if (!PeerConnectionScreenSharing) { - return; - } - PeerConnectionScreenSharing.destroy(); - this.PeerScreenSharingConnectionArray.delete(userId); } } + + private sendLocalScreenSharingStreamToUser(userId: string): void { + // If a connection already exists with user (because it is already sharing a screen with us... let's use this connection) + if (this.PeerScreenSharingConnectionArray.has(userId)) { + this.pushScreenSharingToRemoteUser(userId); + return; + } + + const screenSharingUser: UserSimplePeerInterface = { + userId, + initiator: true + }; + const PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true); + if (!PeerConnectionScreenSharing) { + return; + } + } + + private stopLocalScreenSharingStreamToUser(userId: string): void { + const PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(userId); + if (!PeerConnectionScreenSharing) { + throw new Error('Weird, screen sharing connection to user ' + userId + 'not found') + } + + console.log("updatedScreenSharing => destroy", PeerConnectionScreenSharing); + // FIXME: maybe we don't want to destroy the connexion if it is used in the other way around! + // FIXME: maybe we don't want to destroy the connexion if it is used in the other way around! + // FIXME: maybe we don't want to destroy the connexion if it is used in the other way around! + PeerConnectionScreenSharing.destroy(); + this.PeerScreenSharingConnectionArray.delete(userId); + } } From 894f7c8009dda668763a5af4962dceb8a3b44779 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 20 Aug 2020 15:21:07 +0200 Subject: [PATCH 14/23] Removing useless roomID parameter in WebRtcSignal message --- back/src/Model/Websocket/WebRtcSignalMessage.ts | 1 - front/src/Connection.ts | 11 ++++------- front/src/WebRtc/SimplePeer.ts | 4 ++-- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/back/src/Model/Websocket/WebRtcSignalMessage.ts b/back/src/Model/Websocket/WebRtcSignalMessage.ts index 4f59f617..865319be 100644 --- a/back/src/Model/Websocket/WebRtcSignalMessage.ts +++ b/back/src/Model/Websocket/WebRtcSignalMessage.ts @@ -9,7 +9,6 @@ export const isWebRtcSignalMessageInterface = new tg.IsInterface().withProperties({ userId: tg.isString, receiverId: tg.isString, - roomId: tg.isString, signal: isSignalData }).get(); export const isWebRtcScreenSharingStartMessageInterface = diff --git a/front/src/Connection.ts b/front/src/Connection.ts index ec1db6b1..4cb95d01 100644 --- a/front/src/Connection.ts +++ b/front/src/Connection.ts @@ -82,7 +82,6 @@ export interface WebRtcDisconnectMessageInterface { export interface WebRtcSignalMessageInterface { userId: string, // TODO: is this needed? receiverId: string, - roomId: string, signal: SignalData } @@ -188,22 +187,20 @@ export class Connection implements Connection { this.socket.on(EventMessage.CONNECT_ERROR, callback) } - public sendWebrtcSignal(signal: unknown, roomId: string, userId? : string|null, receiverId? : string) { + public sendWebrtcSignal(signal: unknown, userId? : string|null, receiverId? : string) { return this.socket.emit(EventMessage.WEBRTC_SIGNAL, { userId: userId ? userId : this.userId, receiverId: receiverId ? receiverId : this.userId, - roomId: roomId, signal: signal - }); + } as WebRtcSignalMessageInterface); } - public sendWebrtcScreenSharingSignal(signal: unknown, roomId: string, userId? : string|null, receiverId? : string) { + public sendWebrtcScreenSharingSignal(signal: unknown, userId? : string|null, receiverId? : string) { return this.socket.emit(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, { userId: userId ? userId : this.userId, receiverId: receiverId ? receiverId : this.userId, - roomId: roomId, signal: signal - }); + } as WebRtcSignalMessageInterface); } public receiveWebrtcStart(callback: (message: WebRtcStartMessageInterface) => void) { diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 0d2dd068..17a92b5b 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -311,7 +311,7 @@ export class SimplePeer { */ private sendWebrtcSignal(data: unknown, userId : string) { try { - this.Connection.sendWebrtcSignal(data, this.WebRtcRoomId, null, userId); + this.Connection.sendWebrtcSignal(data, null, userId); }catch (e) { console.error(`sendWebrtcSignal => ${userId}`, e); } @@ -325,7 +325,7 @@ export class SimplePeer { private sendWebrtcScreenSharingSignal(data: unknown, userId : string) { console.log("sendWebrtcScreenSharingSignal", data); try { - this.Connection.sendWebrtcScreenSharingSignal(data, this.WebRtcRoomId, null, userId); + this.Connection.sendWebrtcScreenSharingSignal(data, null, userId); }catch (e) { console.error(`sendWebrtcScreenSharingSignal => ${userId}`, e); } From 27ffb6b13d77972de5dc29616e8629ebacae063f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 20 Aug 2020 16:56:10 +0200 Subject: [PATCH 15/23] Refactoring SimplePeer code: splitting Peer instantiation into 2 subclasses (VideoPeer and ScreenSharingPeer). This leads to way leaner code. --- back/src/Controller/IoSocketController.ts | 10 +- .../Model/Websocket/WebRtcSignalMessage.ts | 1 - front/src/Connection.ts | 26 ++- front/src/WebRtc/ScreenSharingPeer.ts | 106 +++++++++ front/src/WebRtc/SimplePeer.ts | 209 +++++------------- front/src/WebRtc/VideoPeer.ts | 128 +++++++++++ front/tsconfig.json | 3 +- 7 files changed, 307 insertions(+), 176 deletions(-) create mode 100644 front/src/WebRtc/ScreenSharingPeer.ts create mode 100644 front/src/WebRtc/VideoPeer.ts diff --git a/back/src/Controller/IoSocketController.ts b/back/src/Controller/IoSocketController.ts index 501c6145..2eca7e44 100644 --- a/back/src/Controller/IoSocketController.ts +++ b/back/src/Controller/IoSocketController.ts @@ -290,7 +290,10 @@ export class IoSocketController { console.warn("While exchanging a WebRTC signal: client with id ", data.receiverId, " does not exist. This might be a race condition."); return; } - return client.emit(SockerIoEvent.WEBRTC_SIGNAL, data); + return client.emit(SockerIoEvent.WEBRTC_SIGNAL, { + userId: socket.userId, + signal: data.signal + }); } emitScreenSharing(socket: ExSocketInterface, data: unknown){ @@ -305,7 +308,10 @@ export class IoSocketController { console.warn("While exchanging a WEBRTC_SCREEN_SHARING signal: client with id ", data.receiverId, " does not exist. This might be a race condition."); return; } - return client.emit(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, data); + return client.emit(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, { + userId: socket.userId, + signal: data.signal + }); } searchClientByIdOrFail(userId: string): ExSocketInterface { diff --git a/back/src/Model/Websocket/WebRtcSignalMessage.ts b/back/src/Model/Websocket/WebRtcSignalMessage.ts index 865319be..5a0dd1af 100644 --- a/back/src/Model/Websocket/WebRtcSignalMessage.ts +++ b/back/src/Model/Websocket/WebRtcSignalMessage.ts @@ -7,7 +7,6 @@ export const isSignalData = export const isWebRtcSignalMessageInterface = new tg.IsInterface().withProperties({ - userId: tg.isString, receiverId: tg.isString, signal: isSignalData }).get(); diff --git a/front/src/Connection.ts b/front/src/Connection.ts index 4cb95d01..783b5d41 100644 --- a/front/src/Connection.ts +++ b/front/src/Connection.ts @@ -79,12 +79,16 @@ export interface WebRtcDisconnectMessageInterface { userId: string } -export interface WebRtcSignalMessageInterface { - userId: string, // TODO: is this needed? +export interface WebRtcSignalSentMessageInterface { receiverId: string, signal: SignalData } +export interface WebRtcSignalReceivedMessageInterface { + userId: string, + signal: SignalData +} + export interface StartMapInterface { mapUrlStart: string, startInstance: string @@ -187,31 +191,29 @@ export class Connection implements Connection { this.socket.on(EventMessage.CONNECT_ERROR, callback) } - public sendWebrtcSignal(signal: unknown, userId? : string|null, receiverId? : string) { + public sendWebrtcSignal(signal: unknown, receiverId : string) { return this.socket.emit(EventMessage.WEBRTC_SIGNAL, { - userId: userId ? userId : this.userId, - receiverId: receiverId ? receiverId : this.userId, + receiverId: receiverId, signal: signal - } as WebRtcSignalMessageInterface); + } as WebRtcSignalSentMessageInterface); } - public sendWebrtcScreenSharingSignal(signal: unknown, userId? : string|null, receiverId? : string) { + public sendWebrtcScreenSharingSignal(signal: unknown, receiverId : string) { return this.socket.emit(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, { - userId: userId ? userId : this.userId, - receiverId: receiverId ? receiverId : this.userId, + receiverId: receiverId, signal: signal - } as WebRtcSignalMessageInterface); + } as WebRtcSignalSentMessageInterface); } public receiveWebrtcStart(callback: (message: WebRtcStartMessageInterface) => void) { this.socket.on(EventMessage.WEBRTC_START, callback); } - public receiveWebrtcSignal(callback: (message: WebRtcSignalMessageInterface) => void) { + public receiveWebrtcSignal(callback: (message: WebRtcSignalReceivedMessageInterface) => void) { return this.socket.on(EventMessage.WEBRTC_SIGNAL, callback); } - public receiveWebrtcScreenSharingSignal(callback: (message: WebRtcSignalMessageInterface) => void) { + public receiveWebrtcScreenSharingSignal(callback: (message: WebRtcSignalReceivedMessageInterface) => void) { return this.socket.on(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, callback); } diff --git a/front/src/WebRtc/ScreenSharingPeer.ts b/front/src/WebRtc/ScreenSharingPeer.ts new file mode 100644 index 00000000..4b03940c --- /dev/null +++ b/front/src/WebRtc/ScreenSharingPeer.ts @@ -0,0 +1,106 @@ +import * as SimplePeerNamespace from "simple-peer"; +import {mediaManager} from "./MediaManager"; +import {Connection} from "../Connection"; + +const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer'); + +/** + * A peer connection used to transmit video / audio signals between 2 peers. + */ +export class ScreenSharingPeer extends Peer { + constructor(private userId: string, initiator: boolean, private connection: Connection) { + super({ + initiator: initiator ? initiator : false, + reconnectTimer: 10000, + config: { + iceServers: [ + { + urls: 'stun:stun.l.google.com:19302' + }, + { + urls: 'turn:numb.viagenie.ca', + username: 'g.parant@thecodingmachine.com', + credential: 'itcugcOHxle9Acqi$' + }, + ] + } + }); + + //start listen signal for the peer connection + this.on('signal', (data: unknown) => { + this.sendWebrtcScreenSharingSignal(data); + }); + + this.on('stream', (stream: MediaStream) => { + this.stream(stream); + }); + + /*this.on('track', (track: MediaStreamTrack, stream: MediaStream) => { + });*/ + + this.on('close', () => { + this.closeScreenSharingConnection(); + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + this.on('error', (err: any) => { + console.error(`screen sharing error => ${this.userId} => ${err.code}`, err); + //mediaManager.isErrorScreenSharing(this.userId); + }); + + this.on('connect', () => { + // FIXME: we need to put the loader on the screen sharing connection + mediaManager.isConnected(this.userId); + console.info(`connect => ${this.userId}`); + }); + + this.pushScreenSharingToRemoteUser(); + } + + private sendWebrtcScreenSharingSignal(data: unknown) { + console.log("sendWebrtcScreenSharingSignal", data); + try { + this.connection.sendWebrtcScreenSharingSignal(data, this.userId); + }catch (e) { + console.error(`sendWebrtcScreenSharingSignal => ${this.userId}`, e); + } + } + + /** + * Sends received stream to screen. + */ + private stream(stream?: MediaStream) { + console.log(`ScreenSharingPeer::stream => ${this.userId}`, stream); + console.log(`stream => ${this.userId} => `, stream); + if(!stream){ + mediaManager.removeActiveScreenSharingVideo(this.userId); + } else { + mediaManager.addStreamRemoteScreenSharing(this.userId, stream); + } + } + + public closeScreenSharingConnection() { + try { + mediaManager.removeActiveScreenSharingVideo(this.userId); + // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" + // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. + //console.log('Closing connection with '+userId); + this.destroy(); + //console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size); + } catch (err) { + console.error("closeConnection", err) + } + } + + private pushScreenSharingToRemoteUser() { + const localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; + if(!localScreenCapture){ + return; + } + + for (const track of localScreenCapture.getTracks()) { + this.addTrack(track, localScreenCapture); + } + return; + } +} diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 17a92b5b..489f07a7 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -1,11 +1,13 @@ import { Connection, WebRtcDisconnectMessageInterface, - WebRtcSignalMessageInterface, + WebRtcSignalReceivedMessageInterface, WebRtcStartMessageInterface } from "../Connection"; import { mediaManager } from "./MediaManager"; import * as SimplePeerNamespace from "simple-peer"; +import {ScreenSharingPeer} from "./ScreenSharingPeer"; +import {VideoPeer} from "./VideoPeer"; const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer'); export interface UserSimplePeerInterface{ @@ -28,8 +30,8 @@ export class SimplePeer { private WebRtcRoomId: string; private Users: Array = new Array(); - private PeerScreenSharingConnectionArray: Map = new Map(); - private PeerConnectionArray: Map = new Map(); + private PeerScreenSharingConnectionArray: Map = new Map(); + private PeerConnectionArray: Map = new Map(); private readonly sendLocalVideoStreamCallback: (media: MediaStream) => void; private readonly sendLocalScreenSharingStreamCallback: (media: MediaStream) => void; private readonly peerConnectionListeners: Array = new Array(); @@ -59,12 +61,12 @@ export class SimplePeer { private initialise() { //receive signal by gemer - this.Connection.receiveWebrtcSignal((message: WebRtcSignalMessageInterface) => { + this.Connection.receiveWebrtcSignal((message: WebRtcSignalReceivedMessageInterface) => { this.receiveWebrtcSignal(message); }); //receive signal by gemer - this.Connection.receiveWebrtcScreenSharingSignal((message: WebRtcSignalMessageInterface) => { + this.Connection.receiveWebrtcScreenSharingSignal((message: WebRtcSignalReceivedMessageInterface) => { this.receiveWebrtcScreenSharingSignal(message); }); @@ -115,10 +117,9 @@ export class SimplePeer { /** * create peer connection to bind users */ - private createPeerConnection(user : UserSimplePeerInterface, screenSharing: boolean = false) : SimplePeerNamespace.Instance | null{ + private createPeerConnection(user : UserSimplePeerInterface) : VideoPeer | null{ if( - (screenSharing && this.PeerScreenSharingConnectionArray.has(user.userId)) - || (!screenSharing && this.PeerConnectionArray.has(user.userId)) + this.PeerConnectionArray.has(user.userId) ){ return null; } @@ -131,107 +132,43 @@ export class SimplePeer { } } - if(screenSharing) { - // We should display the screen sharing ONLY if we are not initiator - if (!user.initiator) { - mediaManager.removeActiveScreenSharingVideo(user.userId); - mediaManager.addScreenSharingActiveVideo(user.userId); - } - }else{ - mediaManager.removeActiveVideo(user.userId); - mediaManager.addActiveVideo(user.userId, name); - } - - const peer : SimplePeerNamespace.Instance = new Peer({ - initiator: user.initiator ? user.initiator : false, - reconnectTimer: 10000, - config: { - iceServers: [ - { - urls: 'stun:stun.l.google.com:19302' - }, - { - urls: 'turn:numb.viagenie.ca', - username: 'g.parant@thecodingmachine.com', - credential: 'itcugcOHxle9Acqi$' - }, - ] - } - }); - if(screenSharing){ - this.PeerScreenSharingConnectionArray.set(user.userId, peer); - } else { - this.PeerConnectionArray.set(user.userId, peer); - } - - //start listen signal for the peer connection - peer.on('signal', (data: unknown) => { - if(screenSharing){ - //console.log('Sending WebRTC offer for screen sharing ', data, ' to ', user.userId); - this.sendWebrtcScreenSharingSignal(data, user.userId); - } else { - this.sendWebrtcSignal(data, user.userId); - } - }); - - peer.on('stream', (stream: MediaStream) => { - this.stream(user.userId, stream, screenSharing); - }); - - /*peer.on('track', (track: MediaStreamTrack, stream: MediaStream) => { - });*/ - - peer.on('close', () => { - if(screenSharing){ - this.closeScreenSharingConnection(user.userId); - return; - } - this.closeConnection(user.userId); - }); - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - peer.on('error', (err: any) => { - console.error(`error => ${user.userId} => ${err.code}`, err); - if(screenSharing){ - //mediaManager.isErrorScreenSharing(user.userId); - return; - } - mediaManager.isError(user.userId); - }); + mediaManager.removeActiveVideo(user.userId); + mediaManager.addActiveVideo(user.userId, name); + let peer = new VideoPeer(user.userId, user.initiator ? user.initiator : false, this.Connection); + // When a connection is established to a video stream, and if a screen sharing is taking place, + // the user sharing screen should also initiate a connection to the remote user! peer.on('connect', () => { - mediaManager.isConnected(user.userId); - console.info(`connect => ${user.userId}`); - - // When a connection is established to a video stream, and if a screen sharing is taking place, - // the user sharing screen should also initiate a connection to the remote user! - if (screenSharing === false && mediaManager.localScreenCapture) { + if (mediaManager.localScreenCapture) { this.sendLocalScreenSharingStreamToUser(user.userId); } }); + this.PeerConnectionArray.set(user.userId, peer); - peer.on('data', (chunk: Buffer) => { - const constraint = JSON.parse(chunk.toString('utf8')); - console.log("data", constraint); - if (constraint.audio) { - mediaManager.enabledMicrophoneByUserId(user.userId); - } else { - mediaManager.disabledMicrophoneByUserId(user.userId); - } - - if (constraint.video || constraint.screen) { - mediaManager.enabledVideoByUserId(user.userId); - } else { - this.stream(user.userId); - mediaManager.disabledVideoByUserId(user.userId); - } - }); - - if(screenSharing){ - this.pushScreenSharingToRemoteUser(user.userId); - }else { - this.pushVideoToRemoteUser(user.userId); + for (const peerConnectionListener of this.peerConnectionListeners) { + peerConnectionListener.onConnect(user); } + return peer; + } + + /** + * create peer connection to bind users + */ + private createPeerScreenSharingConnection(user : UserSimplePeerInterface) : ScreenSharingPeer | null{ + if( + this.PeerScreenSharingConnectionArray.has(user.userId) + ){ + return null; + } + + // We should display the screen sharing ONLY if we are not initiator + if (!user.initiator) { + mediaManager.removeActiveScreenSharingVideo(user.userId); + mediaManager.addScreenSharingActiveVideo(user.userId); + } + + let peer = new ScreenSharingPeer(user.userId, user.initiator ? user.initiator : false, this.Connection); + this.PeerScreenSharingConnectionArray.set(user.userId, peer); for (const peerConnectionListener of this.peerConnectionListeners) { peerConnectionListener.onConnect(user); @@ -246,16 +183,16 @@ export class SimplePeer { */ private closeConnection(userId : string) { try { - mediaManager.removeActiveVideo(userId); + //mediaManager.removeActiveVideo(userId); const peer = this.PeerConnectionArray.get(userId); if (peer === undefined) { console.warn("Tried to close connection for user "+userId+" but could not find user") return; } + peer.closeConnection(); // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. //console.log('Closing connection with '+userId); - peer.destroy(); this.PeerConnectionArray.delete(userId); this.closeScreenSharingConnection(userId); //console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size); @@ -283,7 +220,7 @@ export class SimplePeer { // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. //console.log('Closing connection with '+userId); - peer.destroy(); + peer.closeScreenSharingConnection(); this.PeerScreenSharingConnectionArray.delete(userId) //console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size); } catch (err) { @@ -295,6 +232,10 @@ export class SimplePeer { for (const userId of this.PeerConnectionArray.keys()) { this.closeConnection(userId); } + + for (const userId of this.PeerScreenSharingConnectionArray.keys()) { + this.closeScreenSharingConnection(userId); + } } /** @@ -304,35 +245,8 @@ export class SimplePeer { mediaManager.removeUpdateLocalStreamEventListener(this.sendLocalVideoStreamCallback); } - /** - * - * @param userId - * @param data - */ - private sendWebrtcSignal(data: unknown, userId : string) { - try { - this.Connection.sendWebrtcSignal(data, null, userId); - }catch (e) { - console.error(`sendWebrtcSignal => ${userId}`, e); - } - } - - /** - * - * @param userId - * @param data - */ - private sendWebrtcScreenSharingSignal(data: unknown, userId : string) { - console.log("sendWebrtcScreenSharingSignal", data); - try { - this.Connection.sendWebrtcScreenSharingSignal(data, null, userId); - }catch (e) { - console.error(`sendWebrtcScreenSharingSignal => ${userId}`, e); - } - } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private receiveWebrtcSignal(data: WebRtcSignalMessageInterface) { + private receiveWebrtcSignal(data: WebRtcSignalReceivedMessageInterface) { try { //if offer type, create peer connection if(data.signal.type === "offer"){ @@ -349,12 +263,12 @@ export class SimplePeer { } } - private receiveWebrtcScreenSharingSignal(data: WebRtcSignalMessageInterface) { + private receiveWebrtcScreenSharingSignal(data: WebRtcSignalReceivedMessageInterface) { console.log("receiveWebrtcScreenSharingSignal", data); try { //if offer type, create peer connection if(data.signal.type === "offer"){ - this.createPeerConnection(data, true); + this.createPeerScreenSharingConnection(data); } const peer = this.PeerScreenSharingConnectionArray.get(data.userId); if (peer !== undefined) { @@ -367,29 +281,6 @@ export class SimplePeer { } } - /** - * - * @param userId - * @param stream - */ - private stream(userId : string, stream?: MediaStream, screenSharing?: boolean) { - console.log(`stream => ${userId} => screenSharing => ${screenSharing}`, stream); - if(screenSharing){ - if(!stream){ - mediaManager.removeActiveScreenSharingVideo(userId); - return; - } - mediaManager.addStreamRemoteScreenSharing(userId, stream); - return; - } - if(!stream){ - mediaManager.disabledVideoByUserId(userId); - mediaManager.disabledMicrophoneByUserId(userId); - return; - } - mediaManager.addStreamRemoteVideo(userId, stream); - } - /** * * @param userId @@ -463,7 +354,7 @@ export class SimplePeer { userId, initiator: true }; - const PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true); + const PeerConnectionScreenSharing = this.createPeerScreenSharingConnection(screenSharingUser); if (!PeerConnectionScreenSharing) { return; } diff --git a/front/src/WebRtc/VideoPeer.ts b/front/src/WebRtc/VideoPeer.ts new file mode 100644 index 00000000..bb624250 --- /dev/null +++ b/front/src/WebRtc/VideoPeer.ts @@ -0,0 +1,128 @@ +import * as SimplePeerNamespace from "simple-peer"; +import {mediaManager} from "./MediaManager"; +import {Connection} from "../Connection"; + +const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer'); + +/** + * A peer connection used to transmit video / audio signals between 2 peers. + */ +export class VideoPeer extends Peer { + constructor(private userId: string, initiator: boolean, private connection: Connection) { + super({ + initiator: initiator ? initiator : false, + reconnectTimer: 10000, + config: { + iceServers: [ + { + urls: 'stun:stun.l.google.com:19302' + }, + { + urls: 'turn:numb.viagenie.ca', + username: 'g.parant@thecodingmachine.com', + credential: 'itcugcOHxle9Acqi$' + }, + ] + } + }); + + //start listen signal for the peer connection + this.on('signal', (data: unknown) => { + this.sendWebrtcSignal(data); + }); + + this.on('stream', (stream: MediaStream) => { + this.stream(stream); + }); + + /*peer.on('track', (track: MediaStreamTrack, stream: MediaStream) => { + });*/ + + this.on('close', () => { + this.closeConnection(); + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + this.on('error', (err: any) => { + console.error(`error => ${this.userId} => ${err.code}`, err); + mediaManager.isError(userId); + }); + + this.on('connect', () => { + mediaManager.isConnected(this.userId); + console.info(`connect => ${this.userId}`); + }); + + this.on('data', (chunk: Buffer) => { + const constraint = JSON.parse(chunk.toString('utf8')); + console.log("data", constraint); + if (constraint.audio) { + mediaManager.enabledMicrophoneByUserId(this.userId); + } else { + mediaManager.disabledMicrophoneByUserId(this.userId); + } + + if (constraint.video || constraint.screen) { + mediaManager.enabledVideoByUserId(this.userId); + } else { + this.stream(undefined); + mediaManager.disabledVideoByUserId(this.userId); + } + }); + + this.pushVideoToRemoteUser(); + } + + private sendWebrtcSignal(data: unknown) { + try { + this.connection.sendWebrtcSignal(data, this.userId); + }catch (e) { + console.error(`sendWebrtcSignal => ${this.userId}`, e); + } + } + + /** + * Sends received stream to screen. + */ + private stream(stream?: MediaStream) { + console.log(`VideoPeer::stream => ${this.userId}`, stream); + if(!stream){ + mediaManager.disabledVideoByUserId(this.userId); + mediaManager.disabledMicrophoneByUserId(this.userId); + } else { + mediaManager.addStreamRemoteVideo(this.userId, stream); + } + } + + /** + * This is triggered twice. Once by the server, and once by a remote client disconnecting + */ + public closeConnection() { + try { + mediaManager.removeActiveVideo(this.userId); + // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" + // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. + //console.log('Closing connection with '+userId); + this.destroy(); + } catch (err) { + console.error("closeConnection", err) + } + } + + private pushVideoToRemoteUser() { + try { + const localStream: MediaStream | null = mediaManager.localStream; + this.write(new Buffer(JSON.stringify(mediaManager.constraintsMedia))); + + if(!localStream){ + return; + } + + for (const track of localStream.getTracks()) { + this.addTrack(track, localStream); + } + }catch (e) { + console.error(`pushVideoToRemoteUser => ${this.userId}`, e); + } + } +} diff --git a/front/tsconfig.json b/front/tsconfig.json index e56a6ee7..64d71e42 100644 --- a/front/tsconfig.json +++ b/front/tsconfig.json @@ -3,9 +3,8 @@ "outDir": "./dist/", "sourceMap": true, "moduleResolution": "node", - "noImplicitAny": true, "module": "CommonJS", - "target": "es5", + "target": "es6", "downlevelIteration": true, "jsx": "react", "allowJs": true, From 11624394792adc9e7ae3776e524d35093572c7d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 20 Aug 2020 22:23:22 +0200 Subject: [PATCH 16/23] Overloading destroy method instead of having a separate method to remove video. --- front/src/WebRtc/ScreenSharingPeer.ts | 8 ++++---- front/src/WebRtc/SimplePeer.ts | 4 ++-- front/src/WebRtc/VideoPeer.ts | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/front/src/WebRtc/ScreenSharingPeer.ts b/front/src/WebRtc/ScreenSharingPeer.ts index 4b03940c..3ce3c409 100644 --- a/front/src/WebRtc/ScreenSharingPeer.ts +++ b/front/src/WebRtc/ScreenSharingPeer.ts @@ -39,7 +39,7 @@ export class ScreenSharingPeer extends Peer { });*/ this.on('close', () => { - this.closeScreenSharingConnection(); + this.destroy(); }); // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -79,16 +79,16 @@ export class ScreenSharingPeer extends Peer { } } - public closeScreenSharingConnection() { + public destroy(error?: Error): void { try { mediaManager.removeActiveScreenSharingVideo(this.userId); // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. //console.log('Closing connection with '+userId); - this.destroy(); + super.destroy(error); //console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size); } catch (err) { - console.error("closeConnection", err) + console.error("ScreenSharingPeer::destroy", err) } } diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 489f07a7..498a8608 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -189,7 +189,7 @@ export class SimplePeer { console.warn("Tried to close connection for user "+userId+" but could not find user") return; } - peer.closeConnection(); + peer.destroy(); // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. //console.log('Closing connection with '+userId); @@ -220,7 +220,7 @@ export class SimplePeer { // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. //console.log('Closing connection with '+userId); - peer.closeScreenSharingConnection(); + peer.destroy(); this.PeerScreenSharingConnectionArray.delete(userId) //console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size); } catch (err) { diff --git a/front/src/WebRtc/VideoPeer.ts b/front/src/WebRtc/VideoPeer.ts index bb624250..ec7f2576 100644 --- a/front/src/WebRtc/VideoPeer.ts +++ b/front/src/WebRtc/VideoPeer.ts @@ -39,7 +39,7 @@ export class VideoPeer extends Peer { });*/ this.on('close', () => { - this.closeConnection(); + this.destroy(); }); // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -97,15 +97,15 @@ export class VideoPeer extends Peer { /** * This is triggered twice. Once by the server, and once by a remote client disconnecting */ - public closeConnection() { + public destroy(error?: Error): void { try { mediaManager.removeActiveVideo(this.userId); // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. //console.log('Closing connection with '+userId); - this.destroy(); + super.destroy(error); } catch (err) { - console.error("closeConnection", err) + console.error("VideoPeer::destroy", err) } } From dc36af19bc221813f66433a5f0922a56de579679 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 20 Aug 2020 22:23:38 +0200 Subject: [PATCH 17/23] Detecting press on "stop screen sharing" --- front/src/WebRtc/MediaManager.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 635174be..63814ee4 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -186,6 +186,14 @@ export class MediaManager { return this._startScreenCapture() .then((stream: MediaStream) => { this.localScreenCapture = stream; + + // If stream ends (for instance if user clicks the stop screen sharing button in the browser), let's close the view + for (const track of stream.getTracks()) { + track.onended = () => { + this.disableScreenSharing(); + }; + } + return stream; }) .catch((err: unknown) => { From c442d6ce670582d3d19397c5fdedd6c420109d25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 20 Aug 2020 22:29:14 +0200 Subject: [PATCH 18/23] Lint --- front/src/WebRtc/SimplePeer.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 498a8608..02573273 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -135,7 +135,7 @@ export class SimplePeer { mediaManager.removeActiveVideo(user.userId); mediaManager.addActiveVideo(user.userId, name); - let peer = new VideoPeer(user.userId, user.initiator ? user.initiator : false, this.Connection); + const peer = new VideoPeer(user.userId, user.initiator ? user.initiator : false, this.Connection); // When a connection is established to a video stream, and if a screen sharing is taking place, // the user sharing screen should also initiate a connection to the remote user! peer.on('connect', () => { @@ -167,7 +167,7 @@ export class SimplePeer { mediaManager.addScreenSharingActiveVideo(user.userId); } - let peer = new ScreenSharingPeer(user.userId, user.initiator ? user.initiator : false, this.Connection); + const peer = new ScreenSharingPeer(user.userId, user.initiator ? user.initiator : false, this.Connection); this.PeerScreenSharingConnectionArray.set(user.userId, peer); for (const peerConnectionListener of this.peerConnectionListeners) { From 2ae19b9f30a1a44bf50c09d71805d6906135da5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 20 Aug 2020 22:34:50 +0200 Subject: [PATCH 19/23] Fixing build --- back/src/Controller/IoSocketController.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/back/src/Controller/IoSocketController.ts b/back/src/Controller/IoSocketController.ts index 2eca7e44..1d5bbe06 100644 --- a/back/src/Controller/IoSocketController.ts +++ b/back/src/Controller/IoSocketController.ts @@ -13,11 +13,10 @@ import {MessageUserJoined} from "../Model/Websocket/MessageUserJoined"; import {MessageUserMoved} from "../Model/Websocket/MessageUserMoved"; import si from "systeminformation"; import {Gauge} from "prom-client"; -import os from 'os'; import {TokenInterface} from "../Controller/AuthenticateController"; import {isJoinRoomMessageInterface} from "../Model/Websocket/JoinRoomMessage"; import {isPointInterface, PointInterface} from "../Model/Websocket/PointInterface"; -import {isWebRtcSignalMessageInterface, isWebRtcScreenSharingSignalMessageInterface, isWebRtcScreenSharingStartMessageInterface} from "../Model/Websocket/WebRtcSignalMessage"; +import {isWebRtcSignalMessageInterface} from "../Model/Websocket/WebRtcSignalMessage"; import {UserInGroupInterface} from "../Model/Websocket/UserInGroupInterface"; enum SockerIoEvent { From f60b02f1dcfbb25172aca831ce4bfa5afa56e77e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 20 Aug 2020 22:57:34 +0200 Subject: [PATCH 20/23] Putting a wider onhover surface when clicking on one of the buttons to manage screen sharing or video/mic --- front/dist/index.html | 26 +++++++++++++------------- front/dist/resources/style/style.css | 11 +++++++++-- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/front/dist/index.html b/front/dist/index.html index 02ec0205..68940afe 100644 --- a/front/dist/index.html +++ b/front/dist/index.html @@ -68,19 +68,19 @@
-
-
- - -
-
- - -
-
- - -
+
+
+
+ + +
+
+ + +
+
+ +
diff --git a/front/dist/resources/style/style.css b/front/dist/resources/style/style.css index 413bce71..382a4444 100644 --- a/front/dist/resources/style/style.css +++ b/front/dist/resources/style/style.css @@ -79,6 +79,13 @@ video#myCamVideo{ } +.btn-cam-action { + position: absolute; + bottom: 0px; + right: 0px; + width: 450px; + height: 150px; +} /*btn animation*/ .btn-cam-action div{ cursor: pointer; @@ -93,7 +100,7 @@ video#myCamVideo{ transition-timing-function: ease-in-out; bottom: 20px; } -#activeCam:hover .btn-cam-action div{ +.btn-cam-action:hover div{ transform: translateY(0); } .btn-cam-action div:hover{ @@ -106,7 +113,7 @@ video#myCamVideo{ right: 44px; } .btn-video{ - transition: all .2s; + transition: all .25s; right: 134px; } .btn-monitor{ From 91f422d0c34283fe42add59564ed2172adcc2c09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Fri, 21 Aug 2020 22:53:17 +0200 Subject: [PATCH 21/23] Fixing stop of stream in bi-directional screen sharing. --- front/src/WebRtc/MediaManager.ts | 54 ++++++++++++++++++++------- front/src/WebRtc/ScreenSharingPeer.ts | 33 +++++++++++++--- front/src/WebRtc/SimplePeer.ts | 45 ++++++++++++++-------- 3 files changed, 98 insertions(+), 34 deletions(-) diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 63814ee4..314fea14 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -1,5 +1,5 @@ -import * as SimplePeerNamespace from "simple-peer"; import {DivImportance, layoutManager} from "./LayoutManager"; +import {HtmlUtils} from "./HtmlUtils"; const videoConstraint: boolean|MediaTrackConstraints = { width: { ideal: 1280 }, @@ -8,7 +8,8 @@ const videoConstraint: boolean|MediaTrackConstraints = { }; type UpdatedLocalStreamCallback = (media: MediaStream) => void; -type UpdatedScreenSharingCallback = (media: MediaStream) => void; +type StartScreenSharingCallback = (media: MediaStream) => void; +type StopScreenSharingCallback = (media: MediaStream) => void; // TODO: Split MediaManager in 2 classes: MediaManagerUI (in charge of HTML) and MediaManager (singleton in charge of the camera only) // TODO: verify that microphone event listeners are not triggered plenty of time NOW (since MediaManager is created many times!!!!) @@ -29,7 +30,8 @@ export class MediaManager { video: videoConstraint }; updatedLocalStreamCallBacks : Set = new Set(); - updatedScreenSharingCallBacks : Set = new Set(); + startScreenSharingCallBacks : Set = new Set(); + stopScreenSharingCallBacks : Set = new Set(); constructor() { @@ -87,9 +89,14 @@ export class MediaManager { this.updatedLocalStreamCallBacks.add(callback); } - public onUpdateScreenSharing(callback: UpdatedScreenSharingCallback): void { + public onStartScreenSharing(callback: StartScreenSharingCallback): void { - this.updatedScreenSharingCallBacks.add(callback); + this.startScreenSharingCallBacks.add(callback); + } + + public onStopScreenSharing(callback: StopScreenSharingCallback): void { + + this.stopScreenSharingCallBacks.add(callback); } removeUpdateLocalStreamEventListener(callback: UpdatedLocalStreamCallback): void { @@ -102,8 +109,14 @@ export class MediaManager { } } - private triggerUpdatedScreenSharingCallbacks(stream: MediaStream): void { - for (const callback of this.updatedScreenSharingCallBacks) { + private triggerStartedScreenSharingCallbacks(stream: MediaStream): void { + for (const callback of this.startScreenSharingCallBacks) { + callback(stream); + } + } + + private triggerStoppedScreenSharingCallbacks(stream: MediaStream): void { + for (const callback of this.stopScreenSharingCallBacks) { callback(stream); } } @@ -164,20 +177,26 @@ export class MediaManager { this.monitorClose.style.display = "none"; this.monitor.style.display = "block"; this.getScreenMedia().then((stream) => { - this.triggerUpdatedScreenSharingCallbacks(stream); + this.triggerStartedScreenSharingCallbacks(stream); }); } private disableScreenSharing() { this.monitorClose.style.display = "block"; this.monitor.style.display = "none"; + this.removeActiveScreenSharingVideo('me'); this.localScreenCapture?.getTracks().forEach((track: MediaStreamTrack) => { track.stop(); }); - this.localScreenCapture = null; + if (this.localScreenCapture === null) { + console.warn('Weird: trying to remove a screen sharing that is not enabled'); + return; + } + const localScreenCapture = this.localScreenCapture; this.getCamera().then((stream) => { - this.triggerUpdatedScreenSharingCallbacks(stream); + this.triggerStoppedScreenSharingCallbacks(localScreenCapture); }); + this.localScreenCapture = null; } //get screen @@ -194,6 +213,9 @@ export class MediaManager { }; } + this.addScreenSharingActiveVideo('me', DivImportance.Normal); + HtmlUtils.getElementByIdOrFail('screen-sharing-me').srcObject = stream; + return stream; }) .catch((err: unknown) => { @@ -306,8 +328,8 @@ export class MediaManager { * * @param userId */ - addScreenSharingActiveVideo(userId : string){ - this.webrtcInAudio.play(); + addScreenSharingActiveVideo(userId : string, divImportance: DivImportance = DivImportance.Important){ + //this.webrtcInAudio.play(); userId = `screen-sharing-${userId}`; const html = ` @@ -316,7 +338,7 @@ export class MediaManager {
`; - layoutManager.add(DivImportance.Important, userId, html); + layoutManager.add(divImportance, userId, html); this.remoteVideo.set(userId, this.getElementByIdOrFail(userId)); } @@ -389,6 +411,12 @@ export class MediaManager { remoteVideo.srcObject = stream; } addStreamRemoteScreenSharing(userId : string, stream : MediaStream){ + // In the case of screen sharing (going both ways), we may need to create the HTML element if it does not exist yet + const remoteVideo = this.remoteVideo.get(`screen-sharing-${userId}`); + if (remoteVideo === undefined) { + this.addScreenSharingActiveVideo(userId); + } + this.addStreamRemoteVideo(`screen-sharing-${userId}`, stream); } diff --git a/front/src/WebRtc/ScreenSharingPeer.ts b/front/src/WebRtc/ScreenSharingPeer.ts index 3ce3c409..35f43201 100644 --- a/front/src/WebRtc/ScreenSharingPeer.ts +++ b/front/src/WebRtc/ScreenSharingPeer.ts @@ -8,6 +8,11 @@ const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer'); * A peer connection used to transmit video / audio signals between 2 peers. */ export class ScreenSharingPeer extends Peer { + /** + * Whether this connection is currently receiving a video stream from a remote user. + */ + private isReceivingStream:boolean = false; + constructor(private userId: string, initiator: boolean, private connection: Connection) { super({ initiator: initiator ? initiator : false, @@ -35,13 +40,20 @@ export class ScreenSharingPeer extends Peer { this.stream(stream); }); - /*this.on('track', (track: MediaStreamTrack, stream: MediaStream) => { - });*/ - this.on('close', () => { this.destroy(); }); + this.on('data', (chunk: Buffer) => { + // We unfortunately need to rely on an event to let the other party know a stream has stopped. + // It seems there is no native way to detect that. + const message = JSON.parse(chunk.toString('utf8')); + if (message.streamEnded !== true) { + console.error('Unexpected message on screen sharing peer connection'); + } + mediaManager.removeActiveScreenSharingVideo(this.userId); + }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any this.on('error', (err: any) => { console.error(`screen sharing error => ${this.userId} => ${err.code}`, err); @@ -74,11 +86,17 @@ export class ScreenSharingPeer extends Peer { console.log(`stream => ${this.userId} => `, stream); if(!stream){ mediaManager.removeActiveScreenSharingVideo(this.userId); + this.isReceivingStream = false; } else { mediaManager.addStreamRemoteScreenSharing(this.userId, stream); + this.isReceivingStream = true; } } + public isReceivingScreenSharingStream(): boolean { + return this.isReceivingStream; + } + public destroy(error?: Error): void { try { mediaManager.removeActiveScreenSharingVideo(this.userId); @@ -98,9 +116,12 @@ export class ScreenSharingPeer extends Peer { return; } - for (const track of localScreenCapture.getTracks()) { - this.addTrack(track, localScreenCapture); - } + this.addStream(localScreenCapture); return; } + + public stopPushingScreenSharingToRemoteUser(stream: MediaStream) { + this.removeStream(stream); + this.write(new Buffer(JSON.stringify({streamEnded: true}))); + } } diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 02573273..3acd65c5 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -34,6 +34,7 @@ export class SimplePeer { private PeerConnectionArray: Map = new Map(); private readonly sendLocalVideoStreamCallback: (media: MediaStream) => void; private readonly sendLocalScreenSharingStreamCallback: (media: MediaStream) => void; + private readonly stopLocalScreenSharingStreamCallback: (media: MediaStream) => void; private readonly peerConnectionListeners: Array = new Array(); constructor(Connection: Connection, WebRtcRoomId: string = "test-webrtc") { @@ -42,8 +43,10 @@ export class SimplePeer { // We need to go through this weird bound function pointer in order to be able to "free" this reference later. this.sendLocalVideoStreamCallback = this.sendLocalVideoStream.bind(this); this.sendLocalScreenSharingStreamCallback = this.sendLocalScreenSharingStream.bind(this); + this.stopLocalScreenSharingStreamCallback = this.stopLocalScreenSharingStream.bind(this); mediaManager.onUpdateLocalStream(this.sendLocalVideoStreamCallback); - mediaManager.onUpdateScreenSharing(this.sendLocalScreenSharingStreamCallback); + mediaManager.onStartScreenSharing(this.sendLocalScreenSharingStreamCallback); + mediaManager.onStopScreenSharing(this.stopLocalScreenSharingStreamCallback); this.initialise(); } @@ -332,14 +335,22 @@ export class SimplePeer { * Triggered locally when clicking on the screen sharing button */ public sendLocalScreenSharingStream() { - if (mediaManager.localScreenCapture) { - for (const user of this.Users) { - this.sendLocalScreenSharingStreamToUser(user.userId); - } - } else { - for (const user of this.Users) { - this.stopLocalScreenSharingStreamToUser(user.userId); - } + if (!mediaManager.localScreenCapture) { + console.error('Could not find localScreenCapture to share') + return; + } + + for (const user of this.Users) { + this.sendLocalScreenSharingStreamToUser(user.userId); + } + } + + /** + * Triggered locally when clicking on the screen sharing button + */ + public stopLocalScreenSharingStream(stream: MediaStream) { + for (const user of this.Users) { + this.stopLocalScreenSharingStreamToUser(user.userId, stream); } } @@ -360,17 +371,21 @@ export class SimplePeer { } } - private stopLocalScreenSharingStreamToUser(userId: string): void { + private stopLocalScreenSharingStreamToUser(userId: string, stream: MediaStream): void { const PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(userId); if (!PeerConnectionScreenSharing) { throw new Error('Weird, screen sharing connection to user ' + userId + 'not found') } console.log("updatedScreenSharing => destroy", PeerConnectionScreenSharing); - // FIXME: maybe we don't want to destroy the connexion if it is used in the other way around! - // FIXME: maybe we don't want to destroy the connexion if it is used in the other way around! - // FIXME: maybe we don't want to destroy the connexion if it is used in the other way around! - PeerConnectionScreenSharing.destroy(); - this.PeerScreenSharingConnectionArray.delete(userId); + + // Stop sending stream and close peer connection if peer is not sending stream too + PeerConnectionScreenSharing.stopPushingScreenSharingToRemoteUser(stream); + + if (!PeerConnectionScreenSharing.isReceivingScreenSharingStream()) { + PeerConnectionScreenSharing.destroy(); + + this.PeerScreenSharingConnectionArray.delete(userId); + } } } From b7c2f8be7bfae6c8a41979c2ea6df7c082b4ab67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Sat, 22 Aug 2020 15:26:40 +0200 Subject: [PATCH 22/23] Adding colors for cam/mic/screen share button Microphone and camera are now red when disabled. Screen-share is green when enabled. Also, they are now always partially visible (they were completely hidden previously until hovering in the lower right corner) --- front/dist/index.html | 6 +++--- front/dist/resources/style/style.css | 8 +++++++- front/src/WebRtc/MediaManager.ts | 12 ++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/front/dist/index.html b/front/dist/index.html index 68940afe..5de00b3b 100644 --- a/front/dist/index.html +++ b/front/dist/index.html @@ -70,15 +70,15 @@
-
+
-
+
-
+
diff --git a/front/dist/resources/style/style.css b/front/dist/resources/style/style.css index 382a4444..c5a3cc67 100644 --- a/front/dist/resources/style/style.css +++ b/front/dist/resources/style/style.css @@ -96,10 +96,16 @@ video#myCamVideo{ background: #666; box-shadow: 2px 2px 24px #444; border-radius: 48px; - transform: translateY(12vh); + transform: translateY(40px); transition-timing-function: ease-in-out; bottom: 20px; } +.btn-cam-action div.disabled { + background: #d75555; +} +.btn-cam-action div.enabled { + background: #73c973; +} .btn-cam-action:hover div{ transform: translateY(0); } diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 314fea14..a043e51e 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -32,6 +32,9 @@ export class MediaManager { updatedLocalStreamCallBacks : Set = new Set(); startScreenSharingCallBacks : Set = new Set(); stopScreenSharingCallBacks : Set = new Set(); + private microphoneBtn: HTMLDivElement; + private cinemaBtn: HTMLDivElement; + private monitorBtn: HTMLDivElement; constructor() { @@ -40,6 +43,7 @@ export class MediaManager { this.webrtcInAudio = this.getElementByIdOrFail('audio-webrtc-in'); this.webrtcInAudio.volume = 0.2; + this.microphoneBtn = this.getElementByIdOrFail('btn-micro'); this.microphoneClose = this.getElementByIdOrFail('microphone-close'); this.microphoneClose.style.display = "none"; this.microphoneClose.addEventListener('click', (e: MouseEvent) => { @@ -54,6 +58,7 @@ export class MediaManager { //update tracking }); + this.cinemaBtn = this.getElementByIdOrFail('btn-video'); this.cinemaClose = this.getElementByIdOrFail('cinema-close'); this.cinemaClose.style.display = "none"; this.cinemaClose.addEventListener('click', (e: MouseEvent) => { @@ -68,6 +73,7 @@ export class MediaManager { //update tracking }); + this.monitorBtn = this.getElementByIdOrFail('btn-monitor'); this.monitorClose = this.getElementByIdOrFail('monitor-close'); this.monitorClose.style.display = "block"; this.monitorClose.addEventListener('click', (e: MouseEvent) => { @@ -128,6 +134,7 @@ export class MediaManager { private enableCamera() { this.cinemaClose.style.display = "none"; + this.cinemaBtn.classList.remove("disabled"); this.cinema.style.display = "block"; this.constraintsMedia.video = videoConstraint; this.getCamera().then((stream: MediaStream) => { @@ -138,6 +145,7 @@ export class MediaManager { private disableCamera() { this.cinemaClose.style.display = "block"; this.cinema.style.display = "none"; + this.cinemaBtn.classList.add("disabled"); this.constraintsMedia.video = false; this.myCamVideo.srcObject = null; if (this.localStream) { @@ -153,6 +161,7 @@ export class MediaManager { private enableMicrophone() { this.microphoneClose.style.display = "none"; this.microphone.style.display = "block"; + this.microphoneBtn.classList.remove("disabled"); this.constraintsMedia.audio = true; this.getCamera().then((stream) => { this.triggerUpdatedLocalStreamCallbacks(stream); @@ -162,6 +171,7 @@ export class MediaManager { private disableMicrophone() { this.microphoneClose.style.display = "block"; this.microphone.style.display = "none"; + this.microphoneBtn.classList.add("disabled"); this.constraintsMedia.audio = false; if(this.localStream) { this.localStream.getAudioTracks().forEach((MediaStreamTrack: MediaStreamTrack) => { @@ -176,6 +186,7 @@ export class MediaManager { private enableScreenSharing() { this.monitorClose.style.display = "none"; this.monitor.style.display = "block"; + this.monitorBtn.classList.add("enabled"); this.getScreenMedia().then((stream) => { this.triggerStartedScreenSharingCallbacks(stream); }); @@ -184,6 +195,7 @@ export class MediaManager { private disableScreenSharing() { this.monitorClose.style.display = "block"; this.monitor.style.display = "none"; + this.monitorBtn.classList.remove("enabled"); this.removeActiveScreenSharingVideo('me'); this.localScreenCapture?.getTracks().forEach((track: MediaStreamTrack) => { track.stop(); From 044495cf05bbacb3e403597e6608129d96c01393 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Mon, 24 Aug 2020 14:19:36 +0200 Subject: [PATCH 23/23] Centering character in free space This commit adds the ability to put the character where there is free space when a discussion is hapening (either in presentation or chat mode) --- front/dist/resources/style/style.css | 6 +- front/src/Phaser/Game/GameScene.ts | 36 ++++++- front/src/WebRtc/LayoutManager.ts | 138 +++++++++++++++++++++++++++ 3 files changed, 173 insertions(+), 7 deletions(-) diff --git a/front/dist/resources/style/style.css b/front/dist/resources/style/style.css index c5a3cc67..8d232fb5 100644 --- a/front/dist/resources/style/style.css +++ b/front/dist/resources/style/style.css @@ -317,13 +317,13 @@ body { flex: 0 0 75%; display: flex; justify-content: start; - /*align-items: flex-start;*/ + align-items: flex-start; flex-wrap: wrap; } .main-section > div { - margin: 5%; - flex-basis: 90%; + margin: 2%; + flex-basis: 96%; /*flex-shrink: 2;*/ } diff --git a/front/src/Phaser/Game/GameScene.ts b/front/src/Phaser/Game/GameScene.ts index 3c3a6536..431bd3db 100644 --- a/front/src/Phaser/Game/GameScene.ts +++ b/front/src/Phaser/Game/GameScene.ts @@ -9,7 +9,7 @@ import { PositionInterface } from "../../Connection"; import {CurrentGamerInterface, hasMovedEventName, Player} from "../Player/Player"; -import {DEBUG_MODE, POSITION_DELAY, ZOOM_LEVEL} from "../../Enum/EnvironmentVariable"; +import {DEBUG_MODE, POSITION_DELAY, RESOLUTION, ZOOM_LEVEL} from "../../Enum/EnvironmentVariable"; import {ITiledMap, ITiledMapLayer, ITiledMapLayerProperty, ITiledTileSet} from "../Map/ITiledMap"; import {PLAYER_RESOURCES, PlayerResourceDescriptionInterface} from "../Entity/Character"; import {AddPlayerInterface} from "./AddPlayerInterface"; @@ -22,7 +22,7 @@ import {SimplePeer, UserSimplePeerInterface} from "../../WebRtc/SimplePeer"; import {ReconnectingSceneName} from "../Reconnecting/ReconnectingScene"; import {FourOFourSceneName} from "../Reconnecting/FourOFourScene"; import {loadAllLayers} from "../Entity/body_character"; -import {layoutManager, LayoutMode} from "../../WebRtc/LayoutManager"; +import {CenterListener, layoutManager, LayoutMode} from "../../WebRtc/LayoutManager"; import Texture = Phaser.Textures.Texture; import Sprite = Phaser.GameObjects.Sprite; import CanvasTexture = Phaser.Textures.CanvasTexture; @@ -69,7 +69,7 @@ interface DeleteGroupEventInterface { groupId: string } -export class GameScene extends Phaser.Scene { +export class GameScene extends Phaser.Scene implements CenterListener { GameManager : GameManager; Terrains : Array; CurrentPlayer!: CurrentGamerInterface; @@ -408,6 +408,9 @@ export class GameScene extends Phaser.Scene { this.repositionCallback = this.reposition.bind(this); window.addEventListener('resize', this.repositionCallback); this.reposition(); + + // From now, this game scene will be notified of reposition events + layoutManager.setListener(this); } private switchLayoutMode(): void { @@ -527,7 +530,7 @@ export class GameScene extends Phaser.Scene { //todo: in a dedicated class/function? initCamera() { this.cameras.main.setBounds(0,0, this.Map.widthInPixels, this.Map.heightInPixels); - this.cameras.main.startFollow(this.CurrentPlayer); + this.updateCameraOffset(); this.cameras.main.setZoom(ZOOM_LEVEL); } @@ -874,5 +877,30 @@ export class GameScene extends Phaser.Scene { private reposition(): void { this.presentationModeSprite.setY(this.game.renderer.height - 2); this.chatModeSprite.setY(this.game.renderer.height - 2); + + // Recompute camera offset if needed + this.updateCameraOffset(); + } + + /** + * Updates the offset of the character compared to the center of the screen according to the layout mananger + * (tries to put the character in the center of the reamining space if there is a discussion going on. + */ + private updateCameraOffset(): void { + const array = layoutManager.findBiggestAvailableArray(); + let xCenter = (array.xEnd - array.xStart) / 2 + array.xStart; + let yCenter = (array.yEnd - array.yStart) / 2 + array.yStart; + + // Let's put this in Game coordinates by applying the zoom level: + xCenter /= ZOOM_LEVEL * RESOLUTION; + yCenter /= ZOOM_LEVEL * RESOLUTION; + + //console.log("updateCameraOffset", array, xCenter, yCenter, this.game.renderer.width, this.game.renderer.height); + + this.cameras.main.startFollow(this.CurrentPlayer, true, 1, 1, xCenter - this.game.renderer.width / 2, yCenter - this.game.renderer.height / 2); + } + + public onCenterChange(): void { + this.updateCameraOffset(); } } diff --git a/front/src/WebRtc/LayoutManager.ts b/front/src/WebRtc/LayoutManager.ts index 6695fe7f..c2bb683e 100644 --- a/front/src/WebRtc/LayoutManager.ts +++ b/front/src/WebRtc/LayoutManager.ts @@ -14,6 +14,14 @@ export enum DivImportance { Normal = "Normal", } +/** + * Classes implementing this interface can be notified when the center of the screen (the player position) should be + * changed. + */ +export interface CenterListener { + onCenterChange(): void; +} + /** * This class is in charge of the video-conference layout. * It receives positioning requests for videos and does its best to place them on the screen depending on the active layout mode. @@ -23,6 +31,11 @@ class LayoutManager { private importantDivs: Map = new Map(); private normalDivs: Map = new Map(); + private listener: CenterListener|null = null; + + public setListener(centerListener: CenterListener|null) { + this.listener = centerListener; + } public add(importance: DivImportance, userId: string, html: string): void { const div = document.createElement('div'); @@ -45,6 +58,7 @@ class LayoutManager { this.positionDiv(div, importance); this.adjustVideoChatClass(); + this.listener?.onCenterChange(); } private positionDiv(elem: HTMLDivElement, importance: DivImportance): void { @@ -72,6 +86,7 @@ class LayoutManager { div.remove(); this.importantDivs.delete(userId); this.adjustVideoChatClass(); + this.listener?.onCenterChange(); return; } @@ -80,6 +95,7 @@ class LayoutManager { div.remove(); this.normalDivs.delete(userId); this.adjustVideoChatClass(); + this.listener?.onCenterChange(); return; } @@ -123,11 +139,133 @@ class LayoutManager { for (const div of this.normalDivs.values()) { this.positionDiv(div, DivImportance.Normal); } + this.listener?.onCenterChange(); } public getLayoutMode(): LayoutMode { return this.mode; } + + /*public getGameCenter(): {x: number, y: number} { + + }*/ + + /** + * Tries to find the biggest available box of remaining space (this is a space where we can center the character) + */ + public findBiggestAvailableArray(): {xStart: number, yStart: number, xEnd: number, yEnd: number} { + if (this.mode === LayoutMode.VideoChat) { + const children = document.querySelectorAll('div.chat-mode > div'); + const htmlChildren = Array.from(children.values()); + + // No chat? Let's go full center + if (htmlChildren.length === 0) { + return { + xStart: 0, + yStart: 0, + xEnd: window.innerWidth, + yEnd: window.innerHeight + } + } + + const lastDiv = htmlChildren[htmlChildren.length - 1]; + // Compute area between top right of the last div and bottom right of window + const area1 = (window.innerWidth - (lastDiv.offsetLeft + lastDiv.offsetWidth)) + * (window.innerHeight - lastDiv.offsetTop); + + // Compute area between bottom of last div and bottom of the screen on whole width + const area2 = window.innerWidth + * (window.innerHeight - (lastDiv.offsetTop + lastDiv.offsetHeight)); + + if (area1 < 0 && area2 < 0) { + // If screen is full, let's not attempt something foolish and simply center character in the middle. + return { + xStart: 0, + yStart: 0, + xEnd: window.innerWidth, + yEnd: window.innerHeight + } + } + if (area1 <= area2) { + console.log('lastDiv', lastDiv.offsetTop, lastDiv.offsetHeight); + return { + xStart: 0, + yStart: lastDiv.offsetTop + lastDiv.offsetHeight, + xEnd: window.innerWidth, + yEnd: window.innerHeight + } + } else { + console.log('lastDiv', lastDiv.offsetTop); + return { + xStart: lastDiv.offsetLeft + lastDiv.offsetWidth, + yStart: lastDiv.offsetTop, + xEnd: window.innerWidth, + yEnd: window.innerHeight + } + } + } else { + // Possible destinations: at the center bottom or at the right bottom. + const mainSectionChildren = Array.from(document.querySelectorAll('div.main-section > div').values()); + const sidebarChildren = Array.from(document.querySelectorAll('aside.sidebar > div').values()); + + // Nothing? Let's center + if (mainSectionChildren.length === 0 && sidebarChildren.length === 0) { + return { + xStart: 0, + yStart: 0, + xEnd: window.innerWidth, + yEnd: window.innerHeight + } + } + + if (mainSectionChildren.length === 0) { + const lastSidebarDiv = sidebarChildren[sidebarChildren.length-1]; + + // No presentation? Let's center on the main-section space + return { + xStart: 0, + yStart: 0, + xEnd: lastSidebarDiv.offsetLeft, + yEnd: window.innerHeight + } + } + + // At this point, we know we have at least one element in the main section. + const lastPresentationDiv = mainSectionChildren[mainSectionChildren.length-1]; + + const presentationArea = (window.innerHeight - (lastPresentationDiv.offsetTop + lastPresentationDiv.offsetHeight)) + * (lastPresentationDiv.offsetLeft + lastPresentationDiv.offsetWidth); + + let leftSideBar: number; + let bottomSideBar: number; + if (sidebarChildren.length === 0) { + leftSideBar = HtmlUtils.getElementByIdOrFail('sidebar').offsetLeft; + bottomSideBar = 0; + } else { + const lastSideBarChildren = sidebarChildren[sidebarChildren.length - 1]; + leftSideBar = lastSideBarChildren.offsetLeft; + bottomSideBar = lastSideBarChildren.offsetTop + lastSideBarChildren.offsetHeight; + } + const sideBarArea = (window.innerWidth - leftSideBar) + * (window.innerHeight - bottomSideBar); + + if (presentationArea <= sideBarArea) { + return { + xStart: leftSideBar, + yStart: bottomSideBar, + xEnd: window.innerWidth, + yEnd: window.innerHeight + } + } else { + return { + xStart: 0, + yStart: lastPresentationDiv.offsetTop + lastPresentationDiv.offsetHeight, + xEnd: /*lastPresentationDiv.offsetLeft + lastPresentationDiv.offsetWidth*/ window.innerWidth , // To avoid flickering when a chat start, we center on the center of the screen, not the center of the main content area + yEnd: window.innerHeight + } + } + } + } } const layoutManager = new LayoutManager();