diff --git a/back/src/Controller/IoSocketController.ts b/back/src/Controller/IoSocketController.ts index 28dd2da2..2f99f1e6 100644 --- a/back/src/Controller/IoSocketController.ts +++ b/back/src/Controller/IoSocketController.ts @@ -28,6 +28,7 @@ enum SockerIoEvent { USER_MOVED = "user-moved", // From server to client USER_LEFT = "user-left", // From server to client WEBRTC_SIGNAL = "webrtc-signal", + WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal", WEBRTC_START = "webrtc-start", WEBRTC_DISCONNECT = "webrtc-disconect", MESSAGE_ERROR = "message-error", @@ -226,18 +227,11 @@ export class IoSocketController { }); socket.on(SockerIoEvent.WEBRTC_SIGNAL, (data: unknown) => { - if (!isWebRtcSignalMessageInterface(data)) { - socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'}); - console.warn('Invalid WEBRTC_SIGNAL message received: ', data); - return; - } - //send only at user - const client = this.sockets.get(data.receiverId); - if (client === undefined) { - console.warn("While exchanging a WebRTC signal: client with id ", data.receiverId, " does not exist. This might be a race condition."); - return; - } - return client.emit(SockerIoEvent.WEBRTC_SIGNAL, data); + this.emit((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SIGNAL); + }); + + socket.on(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, (data: unknown) => { + this.emit((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL); }); socket.on(SockerIoEvent.DISCONNECT, () => { @@ -284,6 +278,21 @@ export class IoSocketController { }); } + emit(socket: ExSocketInterface, data: unknown, event: SockerIoEvent){ + if (!isWebRtcSignalMessageInterface(data)) { + socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'}); + console.warn('Invalid WEBRTC_SIGNAL message received: ', data); + return; + } + //send only at user + const client = this.sockets.get(data.receiverId); + if (client === undefined) { + console.warn("While exchanging a WebRTC signal: client with id ", data.receiverId, " does not exist. This might be a race condition."); + return; + } + return client.emit(event, data); + } + searchClientByIdOrFail(userId: string): ExSocketInterface { const client: ExSocketInterface|undefined = this.sockets.get(userId); if (client === undefined) { diff --git a/front/src/Connection.ts b/front/src/Connection.ts index 69121837..bceef68a 100644 --- a/front/src/Connection.ts +++ b/front/src/Connection.ts @@ -9,9 +9,9 @@ import {PlayerAnimationNames} from "./Phaser/Player/Animation"; import {UserSimplePeerInterface} from "./WebRtc/SimplePeer"; import {SignalData} from "simple-peer"; - enum EventMessage{ WEBRTC_SIGNAL = "webrtc-signal", + WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal", WEBRTC_START = "webrtc-start", JOIN_ROOM = "join-room", // bi-directional USER_POSITION = "user-position", // bi-directional @@ -197,6 +197,15 @@ export class Connection implements Connection { }); } + sendWebrtcScreenSharingSignal(signal: any, roomId: string, userId? : string|null, receiverId? : string) { + return this.getSocket().emit(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, { + userId: userId ? userId : this.userId, + receiverId: receiverId ? receiverId : this.userId, + roomId: roomId, + signal: signal + }); + } + public receiveWebrtcStart(callback: (message: WebRtcStartMessageInterface) => void) { this.socket.on(EventMessage.WEBRTC_START, callback); } @@ -205,6 +214,23 @@ export class Connection implements Connection { return this.socket.on(EventMessage.WEBRTC_SIGNAL, callback); } + receiveWebrtcScreenSharingSignal(callback: Function) { + return this.getSocket().on(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, callback); + } + + private errorMessage(): void { + this.getSocket().on(EventMessage.MESSAGE_ERROR, (message: string) => { + console.error(EventMessage.MESSAGE_ERROR, message); + }) + } + + private disconnectServer(): void { + this.getSocket().on(EventMessage.CONNECT_ERROR, () => { + this.GameManager.switchToDisconnectedScene(); + }); + + } + public onServerDisconnected(callback: (reason: string) => void): void { this.socket.on('disconnect', (reason: string) => { if (reason === 'io client disconnect') { diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 4341c52e..706b9f49 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -287,14 +287,13 @@ export class MediaManager { * * @param userId */ - addScreenSharingActiveVideo(userId : string, userName: string = ""){ + addScreenSharingActiveVideo(userId : string){ + userId = `screen-sharing-${userId}`; this.webrtcInAudio.play(); // FIXME: switch to DisplayManager! let elementRemoteVideo = this.getElementByIdOrFail("activeScreenSharing"); - userName = userName.toUpperCase(); - let color = this.getColorByString(userName); elementRemoteVideo.insertAdjacentHTML('beforeend', ` -
+
`); @@ -302,6 +301,7 @@ export class MediaManager { if(!activeHTMLVideoElement){ return; } + console.log(userId, (activeHTMLVideoElement as HTMLVideoElement)); this.remoteVideo.set(userId, (activeHTMLVideoElement as HTMLVideoElement)); } @@ -372,6 +372,9 @@ export class MediaManager { } remoteVideo.srcObject = stream; } + addStreamRemoteScreenSharing(userId : string, stream : MediaStream){ + this.addStreamRemoteVideo(`screen-sharing-${userId}`, stream); + } /** * @@ -381,6 +384,9 @@ export class MediaManager { layoutManager.remove(userId); this.remoteVideo.delete(userId); } + removeActiveScreenSharingVideo(userId : string) { + this.removeActiveVideo(`screen-sharing-${userId}`) + } isConnecting(userId : string): void { const connectingSpinnerDiv = this.getSpinner(userId); diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 6f5fd69a..81bffd6d 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -28,6 +28,7 @@ export class SimplePeer { private WebRtcRoomId: string; private Users: Array = new Array(); + private PeerScreenSharingConnectionArray: Map = new Map(); private PeerConnectionArray: Map = new Map(); private readonly updateLocalStreamCallback: (media: MediaStream) => void; private readonly updateScreenSharingCallback: (media: MediaStream) => void; @@ -62,6 +63,11 @@ export class SimplePeer { this.receiveWebrtcSignal(message); }); + //receive signal by gemer + this.Connection.receiveWebrtcScreenSharingSignal((message: any) => { + this.receiveWebrtcScreenSharingSignal(message); + }); + mediaManager.activeVisio(); mediaManager.getCamera().then(() => { @@ -108,7 +114,8 @@ export class SimplePeer { /** * create peer connection to bind users */ - private createPeerConnection(user : UserSimplePeerInterface) : SimplePeerNamespace.Instance | null{ + private createPeerConnection(user : UserSimplePeerInterface, screenSharing: boolean = false) : SimplePeerNamespace.Instance | null{ + console.log("createPeerConnection => screenSharing", screenSharing) if(this.PeerConnectionArray.has(user.userId)) { return null; } @@ -121,12 +128,11 @@ export class SimplePeer { } } - let screenSharing : boolean = name !== undefined && name.indexOf("screenSharing") > -1; mediaManager.removeActiveVideo(user.userId); - if(!screenSharing) { - mediaManager.addActiveVideo(user.userId, name); + if(screenSharing) { + mediaManager.addScreenSharingActiveVideo(user.userId); }else{ - mediaManager.addScreenSharingActiveVideo(user.userId, name); + mediaManager.addActiveVideo(user.userId, name); } const peer : SimplePeerNamespace.Instance = new Peer({ @@ -145,10 +151,19 @@ export class SimplePeer { ] }, }); - this.PeerConnectionArray.set(user.userId, peer); + if(screenSharing){ + this.PeerScreenSharingConnectionArray.set(user.userId, peer); + }else { + this.PeerConnectionArray.set(user.userId, peer); + } //start listen signal for the peer connection peer.on('signal', (data: unknown) => { + console.log("screenSharing", screenSharing); + if(screenSharing){ + this.sendWebrtcScreenSharingSignal(data, user.userId); + return; + } this.sendWebrtcSignal(data, user.userId); }); @@ -160,6 +175,9 @@ export class SimplePeer { });*/ peer.on('close', () => { + if(screenSharing){ + this.closeScreenSharingConnection(user.userId); + } this.closeConnection(user.userId); }); @@ -190,7 +208,11 @@ export class SimplePeer { } }); - this.addMedia(user.userId); + if(screenSharing){ + this.addMediaScreenSharing(user.userId); + }else { + this.addMedia(user.userId); + } for (const peerConnectionListener of this.peerConnectionListeners) { peerConnectionListener.onConnect(user); @@ -225,6 +247,30 @@ export class SimplePeer { } } + /** + * This is triggered twice. Once by the server, and once by a remote client disconnecting + * + * @param userId + */ + private closeScreenSharingConnection(userId : string) { + try { + mediaManager.removeActiveScreenSharingVideo(userId); + let peer = this.PeerScreenSharingConnectionArray.get(userId); + if (peer === undefined) { + console.warn("Tried to close connection for user "+userId+" but could not find user") + return; + } + // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" + // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. + //console.log('Closing connection with '+userId); + peer.destroy(); + this.PeerScreenSharingConnectionArray.delete(userId) + //console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size); + } catch (err) { + console.error("closeConnection", err) + } + } + public closeAllConnections() { for (const userId of this.PeerConnectionArray.keys()) { this.closeConnection(userId); @@ -244,6 +290,7 @@ export class SimplePeer { * @param data */ private sendWebrtcSignal(data: unknown, userId : string) { + console.log("sendWebrtcSignal", data); try { this.Connection.sendWebrtcSignal(data, this.WebRtcRoomId, null, userId); }catch (e) { @@ -251,6 +298,20 @@ export class SimplePeer { } } + /** + * + * @param userId + * @param data + */ + private sendWebrtcScreenSharingSignal(data: any, userId : string) { + console.log("sendWebrtcScreenSharingSignal", data); + try { + this.Connection.sendWebrtcScreenSharingSignal(data, this.WebRtcRoomId, null, userId); + }catch (e) { + console.error(`sendWebrtcSignal => ${userId}`, e); + } + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any private receiveWebrtcSignal(data: WebRtcSignalMessageInterface) { try { @@ -269,6 +330,24 @@ export class SimplePeer { } } + private receiveWebrtcScreenSharingSignal(data: any) { + console.log("receiveWebrtcScreenSharingSignal", data); + try { + //if offer type, create peer connection + if(data.signal.type === "offer"){ + this.createPeerConnection(data, true); + } + let peer = this.PeerConnectionArray.get(data.userId); + if (peer !== undefined) { + peer.signal(data.signal); + } else { + console.error('Could not find peer whose ID is "'+data.userId+'" in PeerConnectionArray'); + } + } catch (e) { + console.error(`receiveWebrtcSignal => ${data.userId}`, e); + } + } + /** * * @param userId @@ -293,18 +372,8 @@ export class SimplePeer { if (!PeerConnection) { throw new Error('While adding media, cannot find user with ID ' + userId); } - - if(userId.indexOf("screenSharing") > -1 && mediaManager.localScreenCapture){ - for (const track of mediaManager.localScreenCapture.getTracks()) { - PeerConnection.addTrack(track, mediaManager.localScreenCapture); - } - return; - } - let localStream: MediaStream | null = mediaManager.localStream; - let localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; - - PeerConnection.write(new Buffer(JSON.stringify(Object.assign(this.MediaManager.constraintsMedia, {screen: localScreenCapture !== null})))); + PeerConnection.write(new Buffer(JSON.stringify(mediaManager.constraintsMedia))); if(!localStream){ return; @@ -319,6 +388,21 @@ export class SimplePeer { } } + private addMediaScreenSharing (userId : any = null) { + let PeerConnection = this.PeerScreenSharingConnectionArray.get(userId); + if (!PeerConnection) { + throw new Error('While adding media, cannot find user with ID ' + userId); + } + let localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; + if(!localScreenCapture){ + return; + } + /*for (const track of localScreenCapture.getTracks()) { + PeerConnection.addTrack(track, localScreenCapture); + }*/ + return; + } + updatedLocalStream(){ this.Users.forEach((user: UserSimplePeerInterface) => { this.addMedia(user.userId); @@ -326,29 +410,31 @@ export class SimplePeer { } updatedScreenSharing() { - if (this.MediaManager.localScreenCapture) { + if (mediaManager.localScreenCapture) { + if(!this.Connection.userId){ + return; + } let screenSharingUser: UserSimplePeerInterface = { - userId: `screenSharing-${this.Connection.userId}`, - name: 'screenSharing', + userId: this.Connection.userId, initiator: true }; - let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser); + let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true); if (!PeerConnectionScreenSharing) { return; } try { - for (const track of this.MediaManager.localScreenCapture.getTracks()) { - PeerConnectionScreenSharing.addTrack(track, this.MediaManager.localScreenCapture); + for (const track of mediaManager.localScreenCapture.getTracks()) { + PeerConnectionScreenSharing.addTrack(track, mediaManager.localScreenCapture); } }catch (e) { console.error("updatedScreenSharing => ", e); } - this.MediaManager.addStreamRemoteVideo(screenSharingUser.userId, this.MediaManager.localScreenCapture); + mediaManager.addStreamRemoteScreenSharing(screenSharingUser.userId, mediaManager.localScreenCapture); } else { - if (!this.PeerConnectionArray.has(`screenSharing-${this.Connection.userId}`)) { + if (!this.Connection.userId || !this.PeerScreenSharingConnectionArray.has(this.Connection.userId)) { return; } - let PeerConnectionScreenSharing = this.PeerConnectionArray.get(`screenSharing-${this.Connection.userId}`); + let PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(this.Connection.userId); if (!PeerConnectionScreenSharing) { return; }