From a8f27e60844fd80257434a0725fb0c3a1e0da2bf Mon Sep 17 00:00:00 2001 From: Gregoire Parant Date: Sun, 14 Jun 2020 14:47:16 +0200 Subject: [PATCH] Create event to start webrtc screen charing --- back/src/Controller/IoSocketController.ts | 25 ++++- .../Model/Websocket/WebRtcSignalMessage.ts | 11 ++ front/src/Connection.ts | 17 ++- front/src/WebRtc/MediaManager.ts | 5 + front/src/WebRtc/SimplePeer.ts | 103 ++++++++++++------ 5 files changed, 123 insertions(+), 38 deletions(-) diff --git a/back/src/Controller/IoSocketController.ts b/back/src/Controller/IoSocketController.ts index 2f99f1e6..81a7b16b 100644 --- a/back/src/Controller/IoSocketController.ts +++ b/back/src/Controller/IoSocketController.ts @@ -17,7 +17,7 @@ import os from 'os'; import {TokenInterface} from "../Controller/AuthenticateController"; import {isJoinRoomMessageInterface} from "../Model/Websocket/JoinRoomMessage"; import {isPointInterface, PointInterface} from "../Model/Websocket/PointInterface"; -import {isWebRtcSignalMessageInterface} from "../Model/Websocket/WebRtcSignalMessage"; +import {isWebRtcSignalMessageInterface, isWebRtcScreenSharingSignalMessageInterface, isWebRtcScreenSharingStartMessageInterface} from "../Model/Websocket/WebRtcSignalMessage"; import {UserInGroupInterface} from "../Model/Websocket/UserInGroupInterface"; enum SockerIoEvent { @@ -30,6 +30,7 @@ enum SockerIoEvent { WEBRTC_SIGNAL = "webrtc-signal", WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal", WEBRTC_START = "webrtc-start", + WEBRTC_SCREEN_SHARING_START = "webrtc-screen-sharing-start", WEBRTC_DISCONNECT = "webrtc-disconect", MESSAGE_ERROR = "message-error", GROUP_CREATE_UPDATE = "group-create-update", @@ -231,7 +232,17 @@ export class IoSocketController { }); socket.on(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, (data: unknown) => { - this.emit((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL); + this.emitScreenSharing((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL); + }); + + socket.on(SockerIoEvent.WEBRTC_SCREEN_SHARING_START, (data: unknown) => { + console.log(SockerIoEvent.WEBRTC_SCREEN_SHARING_START, data); + if (!isWebRtcScreenSharingStartMessageInterface(data)) { + socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'}); + console.warn('Invalid WEBRTC_SIGNAL message received: ', data); + return; + } + this.Io.in(data.roomId).emit(SockerIoEvent.WEBRTC_SCREEN_SHARING_START, data); }); socket.on(SockerIoEvent.DISCONNECT, () => { @@ -293,6 +304,16 @@ export class IoSocketController { return client.emit(event, data); } + emitScreenSharing(socket: ExSocketInterface, data: unknown, event: SockerIoEvent){ + if (!isWebRtcScreenSharingSignalMessageInterface(data)) { + socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'}); + console.warn('Invalid WEBRTC_SIGNAL message received: ', data); + return; + } + //share at all others clients send only at user + return socket.broadcast.emit(event, data); + } + searchClientByIdOrFail(userId: string): ExSocketInterface { const client: ExSocketInterface|undefined = this.sockets.get(userId); if (client === undefined) { diff --git a/back/src/Model/Websocket/WebRtcSignalMessage.ts b/back/src/Model/Websocket/WebRtcSignalMessage.ts index 7edffdfa..8236d338 100644 --- a/back/src/Model/Websocket/WebRtcSignalMessage.ts +++ b/back/src/Model/Websocket/WebRtcSignalMessage.ts @@ -7,4 +7,15 @@ export const isWebRtcSignalMessageInterface = roomId: tg.isString, signal: tg.isUnknown }).get(); +export const isWebRtcScreenSharingSignalMessageInterface = + new tg.IsInterface().withProperties({ + userId: tg.isString, + roomId: tg.isString, + signal: tg.isUnknown + }).get(); +export const isWebRtcScreenSharingStartMessageInterface = + new tg.IsInterface().withProperties({ + userId: tg.isString, + roomId: tg.isString + }).get(); export type WebRtcSignalMessageInterface = tg.GuardedType; diff --git a/front/src/Connection.ts b/front/src/Connection.ts index bceef68a..50750c59 100644 --- a/front/src/Connection.ts +++ b/front/src/Connection.ts @@ -13,6 +13,7 @@ enum EventMessage{ WEBRTC_SIGNAL = "webrtc-signal", WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal", WEBRTC_START = "webrtc-start", + WEBRTC_SCREEN_SHARING_START = "webrtc-screen-sharing-start", JOIN_ROOM = "join-room", // bi-directional USER_POSITION = "user-position", // bi-directional USER_MOVED = "user-moved", // From server to client @@ -197,10 +198,16 @@ export class Connection implements Connection { }); } - sendWebrtcScreenSharingSignal(signal: any, roomId: string, userId? : string|null, receiverId? : string) { + sendWebrtcScreenSharingStart(roomId: string) { + return this.getSocket().emit(EventMessage.WEBRTC_SCREEN_SHARING_START, { + userId: this.userId, + roomId: roomId + }); + } + + sendWebrtcScreenSharingSignal(signal: any, roomId: string, userId? : string|null) { return this.getSocket().emit(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, { - userId: userId ? userId : this.userId, - receiverId: receiverId ? receiverId : this.userId, + userId: userId, roomId: roomId, signal: signal }); @@ -210,6 +217,10 @@ export class Connection implements Connection { this.socket.on(EventMessage.WEBRTC_START, callback); } + public receiveWebrtcScreenSharingStart(callback: (message: WebRtcDisconnectMessageInterface) => void) { + this.socket.on(EventMessage.WEBRTC_SCREEN_SHARING_START, callback); + } + public receiveWebrtcSignal(callback: (message: WebRtcSignalMessageInterface) => void) { return this.socket.on(EventMessage.WEBRTC_SIGNAL, callback); } diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 706b9f49..167faded 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -405,6 +405,7 @@ export class MediaManager { } isError(userId : string): void { + console.log("isError", `div-${userId}`); const element = document.getElementById(`div-${userId}`); if(!element){ return; @@ -415,6 +416,10 @@ export class MediaManager { } errorDiv.style.display = 'block'; } + isErrorScreenSharing(userId : string): void { + this.isError(`screen-sharing-${userId}`); + } + private getSpinner(userId : string): HTMLDivElement|null { const element = document.getElementById(`div-${userId}`); diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 81bffd6d..0377ea1a 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -63,8 +63,16 @@ export class SimplePeer { this.receiveWebrtcSignal(message); }); + this.Connection.receiveWebrtcScreenSharingStart((message: WebRtcDisconnectMessageInterface) => { + console.log("receiveWebrtcScreenSharingStart => initiator", message.userId === this.Connection.userId); + if(message.userId === this.Connection.userId) { + console.log("receiveWebrtcScreenSharingStart => initiator => create peer connexion"); + this.receiveWebrtcScreenSharingStart(message); + } + }); + //receive signal by gemer - this.Connection.receiveWebrtcScreenSharingSignal((message: any) => { + this.Connection.receiveWebrtcScreenSharingSignal((message: WebRtcDisconnectMessageInterface) => { this.receiveWebrtcScreenSharingSignal(message); }); @@ -98,6 +106,31 @@ export class SimplePeer { this.startWebRtc(); } + private receiveWebrtcScreenSharingStart(data: WebRtcDisconnectMessageInterface) { + console.log("receiveWebrtcScreenSharingStart", data); + let screenSharingUser: UserSimplePeerInterface = { + userId: data.userId, + initiator: this.Connection.userId === data.userId + }; + let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true); + if (!PeerConnectionScreenSharing) { + console.error("receiveWebrtcScreenSharingStart => cannot create peer connexion", PeerConnectionScreenSharing); + return; + } + console.log(`receiveWebrtcScreenSharingStart => ${screenSharingUser.initiator}`, mediaManager.localScreenCapture) + if (!mediaManager.localScreenCapture) { + return; + } + try { + for (const track of mediaManager.localScreenCapture.getTracks()) { + PeerConnectionScreenSharing.addTrack(track, mediaManager.localScreenCapture); + } + } catch (e) { + console.error("updatedScreenSharing => ", e); + } + mediaManager.addStreamRemoteScreenSharing(screenSharingUser.userId, mediaManager.localScreenCapture); + } + /** * server has two people connected, start the meet */ @@ -115,8 +148,10 @@ export class SimplePeer { * create peer connection to bind users */ private createPeerConnection(user : UserSimplePeerInterface, screenSharing: boolean = false) : SimplePeerNamespace.Instance | null{ - console.log("createPeerConnection => screenSharing", screenSharing) - if(this.PeerConnectionArray.has(user.userId)) { + if( + (screenSharing && this.PeerScreenSharingConnectionArray.has(user.userId)) + || (!screenSharing && this.PeerConnectionArray.has(user.userId)) + ){ return null; } @@ -128,14 +163,15 @@ export class SimplePeer { } } - mediaManager.removeActiveVideo(user.userId); if(screenSharing) { + mediaManager.removeActiveScreenSharingVideo(user.userId); mediaManager.addScreenSharingActiveVideo(user.userId); }else{ + mediaManager.removeActiveVideo(user.userId); mediaManager.addActiveVideo(user.userId, name); } - const peer : SimplePeerNamespace.Instance = new Peer({ + const peerOption : SimplePeerNamespace.Instance = new Peer({ initiator: user.initiator ? user.initiator : false, reconnectTimer: 10000, config: { @@ -149,8 +185,10 @@ export class SimplePeer { credential: 'itcugcOHxle9Acqi$' }, ] - }, - }); + } + }; + console.log("peerOption", peerOption); + let peer : SimplePeerNamespace.Instance = new Peer(peerOption); if(screenSharing){ this.PeerScreenSharingConnectionArray.set(user.userId, peer); }else { @@ -159,7 +197,6 @@ export class SimplePeer { //start listen signal for the peer connection peer.on('signal', (data: unknown) => { - console.log("screenSharing", screenSharing); if(screenSharing){ this.sendWebrtcScreenSharingSignal(data, user.userId); return; @@ -168,7 +205,7 @@ export class SimplePeer { }); peer.on('stream', (stream: MediaStream) => { - this.stream(user.userId, stream); + this.stream(user.userId, stream, screenSharing); }); /*peer.on('track', (track: MediaStreamTrack, stream: MediaStream) => { @@ -177,6 +214,7 @@ export class SimplePeer { peer.on('close', () => { if(screenSharing){ this.closeScreenSharingConnection(user.userId); + return; } this.closeConnection(user.userId); }); @@ -184,6 +222,10 @@ export class SimplePeer { // eslint-disable-next-line @typescript-eslint/no-explicit-any peer.on('error', (err: any) => { console.error(`error => ${user.userId} => ${err.code}`, err); + if(screenSharing){ + //mediaManager.isErrorScreenSharing(user.userId); + return; + } mediaManager.isError(user.userId); }); @@ -194,6 +236,7 @@ export class SimplePeer { peer.on('data', (chunk: Buffer) => { let constraint = JSON.parse(chunk.toString('utf8')); + console.log("data", constraint); if (constraint.audio) { mediaManager.enabledMicrophoneByUserId(user.userId); } else { @@ -237,7 +280,8 @@ export class SimplePeer { // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. //console.log('Closing connection with '+userId); peer.destroy(); - this.PeerConnectionArray.delete(userId) + this.PeerConnectionArray.delete(userId); + this.closeScreenSharingConnection(userId); //console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size); for (const peerConnectionListener of this.peerConnectionListeners) { peerConnectionListener.onDisconnect(userId); @@ -306,7 +350,7 @@ export class SimplePeer { private sendWebrtcScreenSharingSignal(data: any, userId : string) { console.log("sendWebrtcScreenSharingSignal", data); try { - this.Connection.sendWebrtcScreenSharingSignal(data, this.WebRtcRoomId, null, userId); + this.Connection.sendWebrtcScreenSharingSignal(data, this.WebRtcRoomId, userId); }catch (e) { console.error(`sendWebrtcSignal => ${userId}`, e); } @@ -337,11 +381,11 @@ export class SimplePeer { if(data.signal.type === "offer"){ this.createPeerConnection(data, true); } - let peer = this.PeerConnectionArray.get(data.userId); + let peer = this.PeerScreenSharingConnectionArray.get(data.userId); if (peer !== undefined) { peer.signal(data.signal); } else { - console.error('Could not find peer whose ID is "'+data.userId+'" in PeerConnectionArray'); + console.error('Could not find peer whose ID is "'+data.userId+'" in receiveWebrtcScreenSharingSignal'); } } catch (e) { console.error(`receiveWebrtcSignal => ${data.userId}`, e); @@ -353,7 +397,16 @@ export class SimplePeer { * @param userId * @param stream */ - private stream(userId : string, stream?: MediaStream) { + private stream(userId : string, stream?: MediaStream, screenSharing?: boolean) { + console.log(`stream => ${userId} => screenSharing => ${screenSharing}`, stream); + if(screenSharing){ + if(!stream){ + mediaManager.removeActiveScreenSharingVideo(userId); + return; + } + mediaManager.addStreamRemoteScreenSharing(userId, stream); + return; + } if(!stream){ mediaManager.disabledVideoByUserId(userId); mediaManager.disabledMicrophoneByUserId(userId); @@ -411,34 +464,18 @@ export class SimplePeer { updatedScreenSharing() { if (mediaManager.localScreenCapture) { - if(!this.Connection.userId){ - return; - } - let screenSharingUser: UserSimplePeerInterface = { - userId: this.Connection.userId, - initiator: true - }; - let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true); - if (!PeerConnectionScreenSharing) { - return; - } - try { - for (const track of mediaManager.localScreenCapture.getTracks()) { - PeerConnectionScreenSharing.addTrack(track, mediaManager.localScreenCapture); - } - }catch (e) { - console.error("updatedScreenSharing => ", e); - } - mediaManager.addStreamRemoteScreenSharing(screenSharingUser.userId, mediaManager.localScreenCapture); + this.Connection.sendWebrtcScreenSharingStart(this.WebRtcRoomId); } else { if (!this.Connection.userId || !this.PeerScreenSharingConnectionArray.has(this.Connection.userId)) { return; } let PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(this.Connection.userId); + console.log("updatedScreenSharing => destroy", PeerConnectionScreenSharing); if (!PeerConnectionScreenSharing) { return; } PeerConnectionScreenSharing.destroy(); + this.PeerScreenSharingConnectionArray.delete(this.Connection.userId); } } }