diff --git a/back/src/Controller/IoSocketController.ts b/back/src/Controller/IoSocketController.ts index 923f36a9..501c6145 100644 --- a/back/src/Controller/IoSocketController.ts +++ b/back/src/Controller/IoSocketController.ts @@ -44,8 +44,6 @@ export class IoSocketController { private nbClientsGauge: Gauge; private nbClientsPerRoomGauge: Gauge; - private offerScreenSharingByClient: Map> = new Map>(); - constructor(server: http.Server) { this.Io = socketIO(server); this.nbClientsGauge = new Gauge({ @@ -229,11 +227,11 @@ export class IoSocketController { }); socket.on(SockerIoEvent.WEBRTC_SIGNAL, (data: unknown) => { - this.emit((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SIGNAL); + this.emitVideo((socket as ExSocketInterface), data); }); socket.on(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, (data: unknown) => { - this.emitScreenSharing((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL); + this.emitScreenSharing((socket as ExSocketInterface), data); }); socket.on(SockerIoEvent.DISCONNECT, () => { @@ -280,7 +278,7 @@ export class IoSocketController { }); } - emit(socket: ExSocketInterface, data: unknown, event: SockerIoEvent){ + emitVideo(socket: ExSocketInterface, data: unknown){ if (!isWebRtcSignalMessageInterface(data)) { socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'}); console.warn('Invalid WEBRTC_SIGNAL message received: ', data); @@ -292,25 +290,22 @@ export class IoSocketController { console.warn("While exchanging a WebRTC signal: client with id ", data.receiverId, " does not exist. This might be a race condition."); return; } - return client.emit(event, data); + return client.emit(SockerIoEvent.WEBRTC_SIGNAL, data); } - emitScreenSharing(socket: ExSocketInterface, data: unknown, event: SockerIoEvent){ - if (!isWebRtcScreenSharingSignalMessageInterface(data)) { - socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'}); - console.warn('Invalid WEBRTC_SIGNAL message received: ', data); + emitScreenSharing(socket: ExSocketInterface, data: unknown){ + if (!isWebRtcSignalMessageInterface(data)) { + socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SCREEN_SHARING message.'}); + console.warn('Invalid WEBRTC_SCREEN_SHARING message received: ', data); return; } - if(data.signal.type === "offer"){ - let roomOffer = this.offerScreenSharingByClient.get(data.roomId); - if(!roomOffer){ - roomOffer = new Map(); - } - roomOffer.set(data.userId, data.signal); - this.offerScreenSharingByClient.set(data.roomId, roomOffer); + //send only at user + const client = this.sockets.get(data.receiverId); + if (client === undefined) { + console.warn("While exchanging a WEBRTC_SCREEN_SHARING signal: client with id ", data.receiverId, " does not exist. This might be a race condition."); + return; } - //share at all others clients send only at user - return socket.in(data.roomId).emit(event, data); + return client.emit(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, data); } searchClientByIdOrFail(userId: string): ExSocketInterface { @@ -393,13 +388,15 @@ export class IoSocketController { if (this.Io.sockets.adapter.rooms[roomId].length < 2 /*|| this.Io.sockets.adapter.rooms[roomId].length >= 4*/) { return; } + + // TODO: scanning all sockets is maybe not the most efficient const clients: Array = (Object.values(this.Io.sockets.sockets) as Array) .filter((client: ExSocketInterface) => client.webRtcRoomId && client.webRtcRoomId === roomId); //send start at one client to initialise offer webrtc //send all users in room to create PeerConnection in front clients.forEach((client: ExSocketInterface, index: number) => { - const clientsId = clients.reduce((tabs: Array, clientId: ExSocketInterface, indexClientId: number) => { + const peerClients = clients.reduce((tabs: Array, clientId: ExSocketInterface, indexClientId: number) => { if (!clientId.userId || clientId.userId === client.userId) { return tabs; } @@ -411,7 +408,7 @@ export class IoSocketController { return tabs; }, []); - client.emit(SockerIoEvent.WEBRTC_START, {clients: clientsId, roomId: roomId}); + client.emit(SockerIoEvent.WEBRTC_START, {clients: peerClients, roomId: roomId}); }); } diff --git a/back/src/Model/Websocket/WebRtcSignalMessage.ts b/back/src/Model/Websocket/WebRtcSignalMessage.ts index 56a19060..4f59f617 100644 --- a/back/src/Model/Websocket/WebRtcSignalMessage.ts +++ b/back/src/Model/Websocket/WebRtcSignalMessage.ts @@ -12,16 +12,9 @@ export const isWebRtcSignalMessageInterface = roomId: tg.isString, signal: isSignalData }).get(); -export const isWebRtcScreenSharingSignalMessageInterface = - new tg.IsInterface().withProperties({ - userId: tg.isString, - roomId: tg.isString, - signal: isSignalData - }).get(); export const isWebRtcScreenSharingStartMessageInterface = new tg.IsInterface().withProperties({ userId: tg.isString, roomId: tg.isString }).get(); export type WebRtcSignalMessageInterface = tg.GuardedType; -export type WebRtcScreenSharingMessageInterface = tg.GuardedType; diff --git a/front/dist/resources/style/style.css b/front/dist/resources/style/style.css index ac1b1527..413bce71 100644 --- a/front/dist/resources/style/style.css +++ b/front/dist/resources/style/style.css @@ -365,39 +365,3 @@ body { .chat-mode > div:last-child { flex-grow: 5; } - -/*SCREEN SHARING*/ -.active-screen-sharing video{ - transform: scaleX(1); -} -.screen-sharing-video-container { - width: 25%; - position: absolute; -} -.active-screen-sharing .screen-sharing-video-container video:hover{ - width: 200%; - z-index: 11; -} -.active-screen-sharing .screen-sharing-video-container video{ - position: absolute; - width: 100%; - height: auto; - left: 0; - top: 0; - transition: all 0.2s ease; - z-index: 1; -} - -.active-screen-sharing .screen-sharing-video-container:nth-child(1){ - /*this is for camera of user*/ - top: 0%; -} -.active-screen-sharing .screen-sharing-video-container:nth-child(2){ - top: 25%; -} -.active-screen-sharing .screen-sharing-video-container:nth-child(3){ - top: 50%; -} -.active-screen-sharing .screen-sharing-video-container:nth-child(4) { - top: 75%; -} diff --git a/front/src/Connection.ts b/front/src/Connection.ts index b7926328..ec1db6b1 100644 --- a/front/src/Connection.ts +++ b/front/src/Connection.ts @@ -80,14 +80,8 @@ export interface WebRtcDisconnectMessageInterface { } export interface WebRtcSignalMessageInterface { - userId: string, - receiverId: string, // TODO: is this needed? (can we merge this with WebRtcScreenSharingMessageInterface?) - roomId: string, - signal: SignalData -} - -export interface WebRtcScreenSharingMessageInterface { - userId: string, + userId: string, // TODO: is this needed? + receiverId: string, roomId: string, signal: SignalData } @@ -203,9 +197,10 @@ export class Connection implements Connection { }); } - public sendWebrtcScreenSharingSignal(signal: unknown, roomId: string, userId? : string|null) { + public sendWebrtcScreenSharingSignal(signal: unknown, roomId: string, userId? : string|null, receiverId? : string) { return this.socket.emit(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, { - userId: userId, + userId: userId ? userId : this.userId, + receiverId: receiverId ? receiverId : this.userId, roomId: roomId, signal: signal }); @@ -219,7 +214,7 @@ export class Connection implements Connection { return this.socket.on(EventMessage.WEBRTC_SIGNAL, callback); } - public receiveWebrtcScreenSharingSignal(callback: (message: WebRtcScreenSharingMessageInterface) => void) { + public receiveWebrtcScreenSharingSignal(callback: (message: WebRtcSignalMessageInterface) => void) { return this.socket.on(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, callback); } diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index b63dc515..635174be 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -42,13 +42,13 @@ export class MediaManager { this.microphoneClose.style.display = "none"; this.microphoneClose.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.enabledMicrophone(); + this.enableMicrophone(); //update tracking }); this.microphone = this.getElementByIdOrFail('microphone'); this.microphone.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.disabledMicrophone(); + this.disableMicrophone(); //update tracking }); @@ -56,13 +56,13 @@ export class MediaManager { this.cinemaClose.style.display = "none"; this.cinemaClose.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.enabledCamera(); + this.enableCamera(); //update tracking }); this.cinema = this.getElementByIdOrFail('cinema'); this.cinema.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.disabledCamera(); + this.disableCamera(); //update tracking }); @@ -70,24 +70,24 @@ export class MediaManager { this.monitorClose.style.display = "block"; this.monitorClose.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.enabledMonitor(); + this.enableScreenSharing(); //update tracking }); this.monitor = this.getElementByIdOrFail('monitor'); this.monitor.style.display = "none"; this.monitor.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.disabledMonitor(); + this.disableScreenSharing(); //update tracking }); } - onUpdateLocalStream(callback: UpdatedLocalStreamCallback): void { + public onUpdateLocalStream(callback: UpdatedLocalStreamCallback): void { this.updatedLocalStreamCallBacks.add(callback); } - onUpdateScreenSharing(callback: UpdatedScreenSharingCallback): void { + public onUpdateScreenSharing(callback: UpdatedScreenSharingCallback): void { this.updatedScreenSharingCallBacks.add(callback); } @@ -108,12 +108,12 @@ export class MediaManager { } } - activeVisio(){ + showGameOverlay(){ const gameOverlay = this.getElementByIdOrFail('game-overlay'); gameOverlay.classList.add('active'); } - enabledCamera() { + private enableCamera() { this.cinemaClose.style.display = "none"; this.cinema.style.display = "block"; this.constraintsMedia.video = videoConstraint; @@ -122,7 +122,7 @@ export class MediaManager { }); } - disabledCamera() { + private disableCamera() { this.cinemaClose.style.display = "block"; this.cinema.style.display = "none"; this.constraintsMedia.video = false; @@ -137,7 +137,7 @@ export class MediaManager { }); } - enabledMicrophone() { + private enableMicrophone() { this.microphoneClose.style.display = "none"; this.microphone.style.display = "block"; this.constraintsMedia.audio = true; @@ -146,7 +146,7 @@ export class MediaManager { }); } - disabledMicrophone() { + private disableMicrophone() { this.microphoneClose.style.display = "block"; this.microphone.style.display = "none"; this.constraintsMedia.audio = false; @@ -160,7 +160,7 @@ export class MediaManager { }); } - enabledMonitor() { + private enableScreenSharing() { this.monitorClose.style.display = "none"; this.monitor.style.display = "block"; this.getScreenMedia().then((stream) => { @@ -168,7 +168,7 @@ export class MediaManager { }); } - disabledMonitor() { + private disableScreenSharing() { this.monitorClose.style.display = "block"; this.monitor.style.display = "none"; this.localScreenCapture?.getTracks().forEach((track: MediaStreamTrack) => { @@ -299,21 +299,18 @@ export class MediaManager { * @param userId */ addScreenSharingActiveVideo(userId : string){ - userId = `screen-sharing-${userId}`; this.webrtcInAudio.play(); - // FIXME: switch to DisplayManager! - const elementRemoteVideo = this.getElementByIdOrFail("activeScreenSharing"); - elementRemoteVideo.insertAdjacentHTML('beforeend', ` -
+ + userId = `screen-sharing-${userId}`; + const html = ` +
- `); - const activeHTMLVideoElement : HTMLElement|null = document.getElementById(userId); - if(!activeHTMLVideoElement){ - return; - } - console.log(userId, (activeHTMLVideoElement as HTMLVideoElement)); - this.remoteVideo.set(userId, (activeHTMLVideoElement as HTMLVideoElement)); + `; + + layoutManager.add(DivImportance.Important, userId, html); + + this.remoteVideo.set(userId, this.getElementByIdOrFail(userId)); } /** diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 72786d63..0d2dd068 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -1,6 +1,6 @@ import { Connection, - WebRtcDisconnectMessageInterface, WebRtcScreenSharingMessageInterface, + WebRtcDisconnectMessageInterface, WebRtcSignalMessageInterface, WebRtcStartMessageInterface } from "../Connection"; @@ -30,18 +30,18 @@ export class SimplePeer { private PeerScreenSharingConnectionArray: Map = new Map(); private PeerConnectionArray: Map = new Map(); - private readonly updateLocalStreamCallback: (media: MediaStream) => void; - private readonly updateScreenSharingCallback: (media: MediaStream) => void; + private readonly sendLocalVideoStreamCallback: (media: MediaStream) => void; + private readonly sendLocalScreenSharingStreamCallback: (media: MediaStream) => void; private readonly peerConnectionListeners: Array = new Array(); constructor(Connection: Connection, WebRtcRoomId: string = "test-webrtc") { this.Connection = Connection; this.WebRtcRoomId = WebRtcRoomId; // We need to go through this weird bound function pointer in order to be able to "free" this reference later. - this.updateLocalStreamCallback = this.updatedLocalStream.bind(this); - this.updateScreenSharingCallback = this.updatedScreenSharing.bind(this); - mediaManager.onUpdateLocalStream(this.updateLocalStreamCallback); - mediaManager.onUpdateScreenSharing(this.updateScreenSharingCallback); + this.sendLocalVideoStreamCallback = this.sendLocalVideoStream.bind(this); + this.sendLocalScreenSharingStreamCallback = this.sendLocalScreenSharingStream.bind(this); + mediaManager.onUpdateLocalStream(this.sendLocalVideoStreamCallback); + mediaManager.onUpdateScreenSharing(this.sendLocalScreenSharingStreamCallback); this.initialise(); } @@ -64,11 +64,11 @@ export class SimplePeer { }); //receive signal by gemer - this.Connection.receiveWebrtcScreenSharingSignal((message: WebRtcScreenSharingMessageInterface) => { + this.Connection.receiveWebrtcScreenSharingSignal((message: WebRtcSignalMessageInterface) => { this.receiveWebrtcScreenSharingSignal(message); }); - mediaManager.activeVisio(); + mediaManager.showGameOverlay(); mediaManager.getCamera().then(() => { //receive message start @@ -88,7 +88,7 @@ export class SimplePeer { private receiveWebrtcStart(data: WebRtcStartMessageInterface) { this.WebRtcRoomId = data.roomId; this.Users = data.clients; - // Note: the clients array contain the list of all clients (event the ones we are already connected to in case a user joints a group) + // Note: the clients array contain the list of all clients (even the ones we are already connected to in case a user joints a group) // So we can receive a request we already had before. (which will abort at the first line of createPeerConnection) // TODO: refactor this to only send a message to connect to one user (rather than several users). // This would be symmetrical to the way we handle disconnection. @@ -102,6 +102,7 @@ export class SimplePeer { * server has two people connected, start the meet */ private startWebRtc() { + console.warn('startWebRtc startWebRtc'); this.Users.forEach((user: UserSimplePeerInterface) => { //if it's not an initiator, peer connection will be created when gamer will receive offer signal if(!user.initiator){ @@ -131,8 +132,11 @@ export class SimplePeer { } if(screenSharing) { - mediaManager.removeActiveScreenSharingVideo(user.userId); - mediaManager.addScreenSharingActiveVideo(user.userId); + // We should display the screen sharing ONLY if we are not initiator + if (!user.initiator) { + mediaManager.removeActiveScreenSharingVideo(user.userId); + mediaManager.addScreenSharingActiveVideo(user.userId); + } }else{ mediaManager.removeActiveVideo(user.userId); mediaManager.addActiveVideo(user.userId, name); @@ -156,17 +160,18 @@ export class SimplePeer { }); if(screenSharing){ this.PeerScreenSharingConnectionArray.set(user.userId, peer); - }else { + } else { this.PeerConnectionArray.set(user.userId, peer); } //start listen signal for the peer connection peer.on('signal', (data: unknown) => { if(screenSharing){ + //console.log('Sending WebRTC offer for screen sharing ', data, ' to ', user.userId); this.sendWebrtcScreenSharingSignal(data, user.userId); - return; + } else { + this.sendWebrtcSignal(data, user.userId); } - this.sendWebrtcSignal(data, user.userId); }); peer.on('stream', (stream: MediaStream) => { @@ -197,6 +202,12 @@ export class SimplePeer { peer.on('connect', () => { mediaManager.isConnected(user.userId); console.info(`connect => ${user.userId}`); + + // When a connection is established to a video stream, and if a screen sharing is taking place, + // the user sharing screen should also initiate a connection to the remote user! + if (screenSharing === false && mediaManager.localScreenCapture) { + this.sendLocalScreenSharingStreamToUser(user.userId); + } }); peer.on('data', (chunk: Buffer) => { @@ -217,9 +228,9 @@ export class SimplePeer { }); if(screenSharing){ - this.addMediaScreenSharing(user.userId); + this.pushScreenSharingToRemoteUser(user.userId); }else { - this.addMedia(user.userId); + this.pushVideoToRemoteUser(user.userId); } for (const peerConnectionListener of this.peerConnectionListeners) { @@ -290,7 +301,7 @@ export class SimplePeer { * Unregisters any held event handler. */ public unregister() { - mediaManager.removeUpdateLocalStreamEventListener(this.updateLocalStreamCallback); + mediaManager.removeUpdateLocalStreamEventListener(this.sendLocalVideoStreamCallback); } /** @@ -299,7 +310,6 @@ export class SimplePeer { * @param data */ private sendWebrtcSignal(data: unknown, userId : string) { - console.log("sendWebrtcSignal", data); try { this.Connection.sendWebrtcSignal(data, this.WebRtcRoomId, null, userId); }catch (e) { @@ -315,7 +325,7 @@ export class SimplePeer { private sendWebrtcScreenSharingSignal(data: unknown, userId : string) { console.log("sendWebrtcScreenSharingSignal", data); try { - this.Connection.sendWebrtcScreenSharingSignal(data, this.WebRtcRoomId, userId); + this.Connection.sendWebrtcScreenSharingSignal(data, this.WebRtcRoomId, null, userId); }catch (e) { console.error(`sendWebrtcScreenSharingSignal => ${userId}`, e); } @@ -339,7 +349,7 @@ export class SimplePeer { } } - private receiveWebrtcScreenSharingSignal(data: WebRtcScreenSharingMessageInterface) { + private receiveWebrtcScreenSharingSignal(data: WebRtcSignalMessageInterface) { console.log("receiveWebrtcScreenSharingSignal", data); try { //if offer type, create peer connection @@ -384,7 +394,7 @@ export class SimplePeer { * * @param userId */ - private addMedia (userId : string) { + private pushVideoToRemoteUser(userId : string) { try { const PeerConnection = this.PeerConnectionArray.get(userId); if (!PeerConnection) { @@ -396,74 +406,80 @@ export class SimplePeer { if(!localStream){ return; } - if (localStream) { - for (const track of localStream.getTracks()) { - PeerConnection.addTrack(track, localStream); - } + + for (const track of localStream.getTracks()) { + PeerConnection.addTrack(track, localStream); } }catch (e) { - console.error(`addMedia => addMedia => ${userId}`, e); + console.error(`pushVideoToRemoteUser => ${userId}`, e); } } - private addMediaScreenSharing(userId : string) { + private pushScreenSharingToRemoteUser(userId : string) { const PeerConnection = this.PeerScreenSharingConnectionArray.get(userId); if (!PeerConnection) { - throw new Error('While adding media, cannot find user with ID ' + userId); + throw new Error('While pushing screen sharing, cannot find user with ID ' + userId); } const localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; if(!localScreenCapture){ return; } - /*for (const track of localScreenCapture.getTracks()) { + + for (const track of localScreenCapture.getTracks()) { PeerConnection.addTrack(track, localScreenCapture); - }*/ + } return; } - updatedLocalStream(){ + public sendLocalVideoStream(){ this.Users.forEach((user: UserSimplePeerInterface) => { - this.addMedia(user.userId); + this.pushVideoToRemoteUser(user.userId); }) } - updatedScreenSharing() { + /** + * Triggered locally when clicking on the screen sharing button + */ + public sendLocalScreenSharingStream() { if (mediaManager.localScreenCapture) { - - //this.Connection.sendWebrtcScreenSharingStart(this.WebRtcRoomId); - - const userId = this.Connection.getUserId(); - if(!userId){ - return; + for (const user of this.Users) { + this.sendLocalScreenSharingStreamToUser(user.userId); } - const screenSharingUser: UserSimplePeerInterface = { - userId, - initiator: true - }; - const PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true); - if (!PeerConnectionScreenSharing) { - return; - } - try { - for (const track of mediaManager.localScreenCapture.getTracks()) { - PeerConnectionScreenSharing.addTrack(track, mediaManager.localScreenCapture); - } - }catch (e) { - console.error("updatedScreenSharing => ", e); - } - mediaManager.addStreamRemoteScreenSharing(screenSharingUser.userId, mediaManager.localScreenCapture); } else { - const userId = this.Connection.getUserId(); - if (!userId || !this.PeerScreenSharingConnectionArray.has(userId)) { - return; + for (const user of this.Users) { + this.stopLocalScreenSharingStreamToUser(user.userId); } - const PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(userId); - console.log("updatedScreenSharing => destroy", PeerConnectionScreenSharing); - if (!PeerConnectionScreenSharing) { - return; - } - PeerConnectionScreenSharing.destroy(); - this.PeerScreenSharingConnectionArray.delete(userId); } } + + private sendLocalScreenSharingStreamToUser(userId: string): void { + // If a connection already exists with user (because it is already sharing a screen with us... let's use this connection) + if (this.PeerScreenSharingConnectionArray.has(userId)) { + this.pushScreenSharingToRemoteUser(userId); + return; + } + + const screenSharingUser: UserSimplePeerInterface = { + userId, + initiator: true + }; + const PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true); + if (!PeerConnectionScreenSharing) { + return; + } + } + + private stopLocalScreenSharingStreamToUser(userId: string): void { + const PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(userId); + if (!PeerConnectionScreenSharing) { + throw new Error('Weird, screen sharing connection to user ' + userId + 'not found') + } + + console.log("updatedScreenSharing => destroy", PeerConnectionScreenSharing); + // FIXME: maybe we don't want to destroy the connexion if it is used in the other way around! + // FIXME: maybe we don't want to destroy the connexion if it is used in the other way around! + // FIXME: maybe we don't want to destroy the connexion if it is used in the other way around! + PeerConnectionScreenSharing.destroy(); + this.PeerScreenSharingConnectionArray.delete(userId); + } }