From 27ffb6b13d77972de5dc29616e8629ebacae063f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Thu, 20 Aug 2020 16:56:10 +0200 Subject: [PATCH] Refactoring SimplePeer code: splitting Peer instantiation into 2 subclasses (VideoPeer and ScreenSharingPeer). This leads to way leaner code. --- back/src/Controller/IoSocketController.ts | 10 +- .../Model/Websocket/WebRtcSignalMessage.ts | 1 - front/src/Connection.ts | 26 ++- front/src/WebRtc/ScreenSharingPeer.ts | 106 +++++++++ front/src/WebRtc/SimplePeer.ts | 209 +++++------------- front/src/WebRtc/VideoPeer.ts | 128 +++++++++++ front/tsconfig.json | 3 +- 7 files changed, 307 insertions(+), 176 deletions(-) create mode 100644 front/src/WebRtc/ScreenSharingPeer.ts create mode 100644 front/src/WebRtc/VideoPeer.ts diff --git a/back/src/Controller/IoSocketController.ts b/back/src/Controller/IoSocketController.ts index 501c6145..2eca7e44 100644 --- a/back/src/Controller/IoSocketController.ts +++ b/back/src/Controller/IoSocketController.ts @@ -290,7 +290,10 @@ export class IoSocketController { console.warn("While exchanging a WebRTC signal: client with id ", data.receiverId, " does not exist. This might be a race condition."); return; } - return client.emit(SockerIoEvent.WEBRTC_SIGNAL, data); + return client.emit(SockerIoEvent.WEBRTC_SIGNAL, { + userId: socket.userId, + signal: data.signal + }); } emitScreenSharing(socket: ExSocketInterface, data: unknown){ @@ -305,7 +308,10 @@ export class IoSocketController { console.warn("While exchanging a WEBRTC_SCREEN_SHARING signal: client with id ", data.receiverId, " does not exist. This might be a race condition."); return; } - return client.emit(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, data); + return client.emit(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, { + userId: socket.userId, + signal: data.signal + }); } searchClientByIdOrFail(userId: string): ExSocketInterface { diff --git a/back/src/Model/Websocket/WebRtcSignalMessage.ts b/back/src/Model/Websocket/WebRtcSignalMessage.ts index 865319be..5a0dd1af 100644 --- a/back/src/Model/Websocket/WebRtcSignalMessage.ts +++ b/back/src/Model/Websocket/WebRtcSignalMessage.ts @@ -7,7 +7,6 @@ export const isSignalData = export const isWebRtcSignalMessageInterface = new tg.IsInterface().withProperties({ - userId: tg.isString, receiverId: tg.isString, signal: isSignalData }).get(); diff --git a/front/src/Connection.ts b/front/src/Connection.ts index 4cb95d01..783b5d41 100644 --- a/front/src/Connection.ts +++ b/front/src/Connection.ts @@ -79,12 +79,16 @@ export interface WebRtcDisconnectMessageInterface { userId: string } -export interface WebRtcSignalMessageInterface { - userId: string, // TODO: is this needed? +export interface WebRtcSignalSentMessageInterface { receiverId: string, signal: SignalData } +export interface WebRtcSignalReceivedMessageInterface { + userId: string, + signal: SignalData +} + export interface StartMapInterface { mapUrlStart: string, startInstance: string @@ -187,31 +191,29 @@ export class Connection implements Connection { this.socket.on(EventMessage.CONNECT_ERROR, callback) } - public sendWebrtcSignal(signal: unknown, userId? : string|null, receiverId? : string) { + public sendWebrtcSignal(signal: unknown, receiverId : string) { return this.socket.emit(EventMessage.WEBRTC_SIGNAL, { - userId: userId ? userId : this.userId, - receiverId: receiverId ? receiverId : this.userId, + receiverId: receiverId, signal: signal - } as WebRtcSignalMessageInterface); + } as WebRtcSignalSentMessageInterface); } - public sendWebrtcScreenSharingSignal(signal: unknown, userId? : string|null, receiverId? : string) { + public sendWebrtcScreenSharingSignal(signal: unknown, receiverId : string) { return this.socket.emit(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, { - userId: userId ? userId : this.userId, - receiverId: receiverId ? receiverId : this.userId, + receiverId: receiverId, signal: signal - } as WebRtcSignalMessageInterface); + } as WebRtcSignalSentMessageInterface); } public receiveWebrtcStart(callback: (message: WebRtcStartMessageInterface) => void) { this.socket.on(EventMessage.WEBRTC_START, callback); } - public receiveWebrtcSignal(callback: (message: WebRtcSignalMessageInterface) => void) { + public receiveWebrtcSignal(callback: (message: WebRtcSignalReceivedMessageInterface) => void) { return this.socket.on(EventMessage.WEBRTC_SIGNAL, callback); } - public receiveWebrtcScreenSharingSignal(callback: (message: WebRtcSignalMessageInterface) => void) { + public receiveWebrtcScreenSharingSignal(callback: (message: WebRtcSignalReceivedMessageInterface) => void) { return this.socket.on(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, callback); } diff --git a/front/src/WebRtc/ScreenSharingPeer.ts b/front/src/WebRtc/ScreenSharingPeer.ts new file mode 100644 index 00000000..4b03940c --- /dev/null +++ b/front/src/WebRtc/ScreenSharingPeer.ts @@ -0,0 +1,106 @@ +import * as SimplePeerNamespace from "simple-peer"; +import {mediaManager} from "./MediaManager"; +import {Connection} from "../Connection"; + +const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer'); + +/** + * A peer connection used to transmit video / audio signals between 2 peers. + */ +export class ScreenSharingPeer extends Peer { + constructor(private userId: string, initiator: boolean, private connection: Connection) { + super({ + initiator: initiator ? initiator : false, + reconnectTimer: 10000, + config: { + iceServers: [ + { + urls: 'stun:stun.l.google.com:19302' + }, + { + urls: 'turn:numb.viagenie.ca', + username: 'g.parant@thecodingmachine.com', + credential: 'itcugcOHxle9Acqi$' + }, + ] + } + }); + + //start listen signal for the peer connection + this.on('signal', (data: unknown) => { + this.sendWebrtcScreenSharingSignal(data); + }); + + this.on('stream', (stream: MediaStream) => { + this.stream(stream); + }); + + /*this.on('track', (track: MediaStreamTrack, stream: MediaStream) => { + });*/ + + this.on('close', () => { + this.closeScreenSharingConnection(); + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + this.on('error', (err: any) => { + console.error(`screen sharing error => ${this.userId} => ${err.code}`, err); + //mediaManager.isErrorScreenSharing(this.userId); + }); + + this.on('connect', () => { + // FIXME: we need to put the loader on the screen sharing connection + mediaManager.isConnected(this.userId); + console.info(`connect => ${this.userId}`); + }); + + this.pushScreenSharingToRemoteUser(); + } + + private sendWebrtcScreenSharingSignal(data: unknown) { + console.log("sendWebrtcScreenSharingSignal", data); + try { + this.connection.sendWebrtcScreenSharingSignal(data, this.userId); + }catch (e) { + console.error(`sendWebrtcScreenSharingSignal => ${this.userId}`, e); + } + } + + /** + * Sends received stream to screen. + */ + private stream(stream?: MediaStream) { + console.log(`ScreenSharingPeer::stream => ${this.userId}`, stream); + console.log(`stream => ${this.userId} => `, stream); + if(!stream){ + mediaManager.removeActiveScreenSharingVideo(this.userId); + } else { + mediaManager.addStreamRemoteScreenSharing(this.userId, stream); + } + } + + public closeScreenSharingConnection() { + try { + mediaManager.removeActiveScreenSharingVideo(this.userId); + // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" + // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. + //console.log('Closing connection with '+userId); + this.destroy(); + //console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size); + } catch (err) { + console.error("closeConnection", err) + } + } + + private pushScreenSharingToRemoteUser() { + const localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; + if(!localScreenCapture){ + return; + } + + for (const track of localScreenCapture.getTracks()) { + this.addTrack(track, localScreenCapture); + } + return; + } +} diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 17a92b5b..489f07a7 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -1,11 +1,13 @@ import { Connection, WebRtcDisconnectMessageInterface, - WebRtcSignalMessageInterface, + WebRtcSignalReceivedMessageInterface, WebRtcStartMessageInterface } from "../Connection"; import { mediaManager } from "./MediaManager"; import * as SimplePeerNamespace from "simple-peer"; +import {ScreenSharingPeer} from "./ScreenSharingPeer"; +import {VideoPeer} from "./VideoPeer"; const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer'); export interface UserSimplePeerInterface{ @@ -28,8 +30,8 @@ export class SimplePeer { private WebRtcRoomId: string; private Users: Array = new Array(); - private PeerScreenSharingConnectionArray: Map = new Map(); - private PeerConnectionArray: Map = new Map(); + private PeerScreenSharingConnectionArray: Map = new Map(); + private PeerConnectionArray: Map = new Map(); private readonly sendLocalVideoStreamCallback: (media: MediaStream) => void; private readonly sendLocalScreenSharingStreamCallback: (media: MediaStream) => void; private readonly peerConnectionListeners: Array = new Array(); @@ -59,12 +61,12 @@ export class SimplePeer { private initialise() { //receive signal by gemer - this.Connection.receiveWebrtcSignal((message: WebRtcSignalMessageInterface) => { + this.Connection.receiveWebrtcSignal((message: WebRtcSignalReceivedMessageInterface) => { this.receiveWebrtcSignal(message); }); //receive signal by gemer - this.Connection.receiveWebrtcScreenSharingSignal((message: WebRtcSignalMessageInterface) => { + this.Connection.receiveWebrtcScreenSharingSignal((message: WebRtcSignalReceivedMessageInterface) => { this.receiveWebrtcScreenSharingSignal(message); }); @@ -115,10 +117,9 @@ export class SimplePeer { /** * create peer connection to bind users */ - private createPeerConnection(user : UserSimplePeerInterface, screenSharing: boolean = false) : SimplePeerNamespace.Instance | null{ + private createPeerConnection(user : UserSimplePeerInterface) : VideoPeer | null{ if( - (screenSharing && this.PeerScreenSharingConnectionArray.has(user.userId)) - || (!screenSharing && this.PeerConnectionArray.has(user.userId)) + this.PeerConnectionArray.has(user.userId) ){ return null; } @@ -131,107 +132,43 @@ export class SimplePeer { } } - if(screenSharing) { - // We should display the screen sharing ONLY if we are not initiator - if (!user.initiator) { - mediaManager.removeActiveScreenSharingVideo(user.userId); - mediaManager.addScreenSharingActiveVideo(user.userId); - } - }else{ - mediaManager.removeActiveVideo(user.userId); - mediaManager.addActiveVideo(user.userId, name); - } - - const peer : SimplePeerNamespace.Instance = new Peer({ - initiator: user.initiator ? user.initiator : false, - reconnectTimer: 10000, - config: { - iceServers: [ - { - urls: 'stun:stun.l.google.com:19302' - }, - { - urls: 'turn:numb.viagenie.ca', - username: 'g.parant@thecodingmachine.com', - credential: 'itcugcOHxle9Acqi$' - }, - ] - } - }); - if(screenSharing){ - this.PeerScreenSharingConnectionArray.set(user.userId, peer); - } else { - this.PeerConnectionArray.set(user.userId, peer); - } - - //start listen signal for the peer connection - peer.on('signal', (data: unknown) => { - if(screenSharing){ - //console.log('Sending WebRTC offer for screen sharing ', data, ' to ', user.userId); - this.sendWebrtcScreenSharingSignal(data, user.userId); - } else { - this.sendWebrtcSignal(data, user.userId); - } - }); - - peer.on('stream', (stream: MediaStream) => { - this.stream(user.userId, stream, screenSharing); - }); - - /*peer.on('track', (track: MediaStreamTrack, stream: MediaStream) => { - });*/ - - peer.on('close', () => { - if(screenSharing){ - this.closeScreenSharingConnection(user.userId); - return; - } - this.closeConnection(user.userId); - }); - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - peer.on('error', (err: any) => { - console.error(`error => ${user.userId} => ${err.code}`, err); - if(screenSharing){ - //mediaManager.isErrorScreenSharing(user.userId); - return; - } - mediaManager.isError(user.userId); - }); + mediaManager.removeActiveVideo(user.userId); + mediaManager.addActiveVideo(user.userId, name); + let peer = new VideoPeer(user.userId, user.initiator ? user.initiator : false, this.Connection); + // When a connection is established to a video stream, and if a screen sharing is taking place, + // the user sharing screen should also initiate a connection to the remote user! peer.on('connect', () => { - mediaManager.isConnected(user.userId); - console.info(`connect => ${user.userId}`); - - // When a connection is established to a video stream, and if a screen sharing is taking place, - // the user sharing screen should also initiate a connection to the remote user! - if (screenSharing === false && mediaManager.localScreenCapture) { + if (mediaManager.localScreenCapture) { this.sendLocalScreenSharingStreamToUser(user.userId); } }); + this.PeerConnectionArray.set(user.userId, peer); - peer.on('data', (chunk: Buffer) => { - const constraint = JSON.parse(chunk.toString('utf8')); - console.log("data", constraint); - if (constraint.audio) { - mediaManager.enabledMicrophoneByUserId(user.userId); - } else { - mediaManager.disabledMicrophoneByUserId(user.userId); - } - - if (constraint.video || constraint.screen) { - mediaManager.enabledVideoByUserId(user.userId); - } else { - this.stream(user.userId); - mediaManager.disabledVideoByUserId(user.userId); - } - }); - - if(screenSharing){ - this.pushScreenSharingToRemoteUser(user.userId); - }else { - this.pushVideoToRemoteUser(user.userId); + for (const peerConnectionListener of this.peerConnectionListeners) { + peerConnectionListener.onConnect(user); } + return peer; + } + + /** + * create peer connection to bind users + */ + private createPeerScreenSharingConnection(user : UserSimplePeerInterface) : ScreenSharingPeer | null{ + if( + this.PeerScreenSharingConnectionArray.has(user.userId) + ){ + return null; + } + + // We should display the screen sharing ONLY if we are not initiator + if (!user.initiator) { + mediaManager.removeActiveScreenSharingVideo(user.userId); + mediaManager.addScreenSharingActiveVideo(user.userId); + } + + let peer = new ScreenSharingPeer(user.userId, user.initiator ? user.initiator : false, this.Connection); + this.PeerScreenSharingConnectionArray.set(user.userId, peer); for (const peerConnectionListener of this.peerConnectionListeners) { peerConnectionListener.onConnect(user); @@ -246,16 +183,16 @@ export class SimplePeer { */ private closeConnection(userId : string) { try { - mediaManager.removeActiveVideo(userId); + //mediaManager.removeActiveVideo(userId); const peer = this.PeerConnectionArray.get(userId); if (peer === undefined) { console.warn("Tried to close connection for user "+userId+" but could not find user") return; } + peer.closeConnection(); // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. //console.log('Closing connection with '+userId); - peer.destroy(); this.PeerConnectionArray.delete(userId); this.closeScreenSharingConnection(userId); //console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size); @@ -283,7 +220,7 @@ export class SimplePeer { // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. //console.log('Closing connection with '+userId); - peer.destroy(); + peer.closeScreenSharingConnection(); this.PeerScreenSharingConnectionArray.delete(userId) //console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size); } catch (err) { @@ -295,6 +232,10 @@ export class SimplePeer { for (const userId of this.PeerConnectionArray.keys()) { this.closeConnection(userId); } + + for (const userId of this.PeerScreenSharingConnectionArray.keys()) { + this.closeScreenSharingConnection(userId); + } } /** @@ -304,35 +245,8 @@ export class SimplePeer { mediaManager.removeUpdateLocalStreamEventListener(this.sendLocalVideoStreamCallback); } - /** - * - * @param userId - * @param data - */ - private sendWebrtcSignal(data: unknown, userId : string) { - try { - this.Connection.sendWebrtcSignal(data, null, userId); - }catch (e) { - console.error(`sendWebrtcSignal => ${userId}`, e); - } - } - - /** - * - * @param userId - * @param data - */ - private sendWebrtcScreenSharingSignal(data: unknown, userId : string) { - console.log("sendWebrtcScreenSharingSignal", data); - try { - this.Connection.sendWebrtcScreenSharingSignal(data, null, userId); - }catch (e) { - console.error(`sendWebrtcScreenSharingSignal => ${userId}`, e); - } - } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private receiveWebrtcSignal(data: WebRtcSignalMessageInterface) { + private receiveWebrtcSignal(data: WebRtcSignalReceivedMessageInterface) { try { //if offer type, create peer connection if(data.signal.type === "offer"){ @@ -349,12 +263,12 @@ export class SimplePeer { } } - private receiveWebrtcScreenSharingSignal(data: WebRtcSignalMessageInterface) { + private receiveWebrtcScreenSharingSignal(data: WebRtcSignalReceivedMessageInterface) { console.log("receiveWebrtcScreenSharingSignal", data); try { //if offer type, create peer connection if(data.signal.type === "offer"){ - this.createPeerConnection(data, true); + this.createPeerScreenSharingConnection(data); } const peer = this.PeerScreenSharingConnectionArray.get(data.userId); if (peer !== undefined) { @@ -367,29 +281,6 @@ export class SimplePeer { } } - /** - * - * @param userId - * @param stream - */ - private stream(userId : string, stream?: MediaStream, screenSharing?: boolean) { - console.log(`stream => ${userId} => screenSharing => ${screenSharing}`, stream); - if(screenSharing){ - if(!stream){ - mediaManager.removeActiveScreenSharingVideo(userId); - return; - } - mediaManager.addStreamRemoteScreenSharing(userId, stream); - return; - } - if(!stream){ - mediaManager.disabledVideoByUserId(userId); - mediaManager.disabledMicrophoneByUserId(userId); - return; - } - mediaManager.addStreamRemoteVideo(userId, stream); - } - /** * * @param userId @@ -463,7 +354,7 @@ export class SimplePeer { userId, initiator: true }; - const PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true); + const PeerConnectionScreenSharing = this.createPeerScreenSharingConnection(screenSharingUser); if (!PeerConnectionScreenSharing) { return; } diff --git a/front/src/WebRtc/VideoPeer.ts b/front/src/WebRtc/VideoPeer.ts new file mode 100644 index 00000000..bb624250 --- /dev/null +++ b/front/src/WebRtc/VideoPeer.ts @@ -0,0 +1,128 @@ +import * as SimplePeerNamespace from "simple-peer"; +import {mediaManager} from "./MediaManager"; +import {Connection} from "../Connection"; + +const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer'); + +/** + * A peer connection used to transmit video / audio signals between 2 peers. + */ +export class VideoPeer extends Peer { + constructor(private userId: string, initiator: boolean, private connection: Connection) { + super({ + initiator: initiator ? initiator : false, + reconnectTimer: 10000, + config: { + iceServers: [ + { + urls: 'stun:stun.l.google.com:19302' + }, + { + urls: 'turn:numb.viagenie.ca', + username: 'g.parant@thecodingmachine.com', + credential: 'itcugcOHxle9Acqi$' + }, + ] + } + }); + + //start listen signal for the peer connection + this.on('signal', (data: unknown) => { + this.sendWebrtcSignal(data); + }); + + this.on('stream', (stream: MediaStream) => { + this.stream(stream); + }); + + /*peer.on('track', (track: MediaStreamTrack, stream: MediaStream) => { + });*/ + + this.on('close', () => { + this.closeConnection(); + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + this.on('error', (err: any) => { + console.error(`error => ${this.userId} => ${err.code}`, err); + mediaManager.isError(userId); + }); + + this.on('connect', () => { + mediaManager.isConnected(this.userId); + console.info(`connect => ${this.userId}`); + }); + + this.on('data', (chunk: Buffer) => { + const constraint = JSON.parse(chunk.toString('utf8')); + console.log("data", constraint); + if (constraint.audio) { + mediaManager.enabledMicrophoneByUserId(this.userId); + } else { + mediaManager.disabledMicrophoneByUserId(this.userId); + } + + if (constraint.video || constraint.screen) { + mediaManager.enabledVideoByUserId(this.userId); + } else { + this.stream(undefined); + mediaManager.disabledVideoByUserId(this.userId); + } + }); + + this.pushVideoToRemoteUser(); + } + + private sendWebrtcSignal(data: unknown) { + try { + this.connection.sendWebrtcSignal(data, this.userId); + }catch (e) { + console.error(`sendWebrtcSignal => ${this.userId}`, e); + } + } + + /** + * Sends received stream to screen. + */ + private stream(stream?: MediaStream) { + console.log(`VideoPeer::stream => ${this.userId}`, stream); + if(!stream){ + mediaManager.disabledVideoByUserId(this.userId); + mediaManager.disabledMicrophoneByUserId(this.userId); + } else { + mediaManager.addStreamRemoteVideo(this.userId, stream); + } + } + + /** + * This is triggered twice. Once by the server, and once by a remote client disconnecting + */ + public closeConnection() { + try { + mediaManager.removeActiveVideo(this.userId); + // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" + // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. + //console.log('Closing connection with '+userId); + this.destroy(); + } catch (err) { + console.error("closeConnection", err) + } + } + + private pushVideoToRemoteUser() { + try { + const localStream: MediaStream | null = mediaManager.localStream; + this.write(new Buffer(JSON.stringify(mediaManager.constraintsMedia))); + + if(!localStream){ + return; + } + + for (const track of localStream.getTracks()) { + this.addTrack(track, localStream); + } + }catch (e) { + console.error(`pushVideoToRemoteUser => ${this.userId}`, e); + } + } +} diff --git a/front/tsconfig.json b/front/tsconfig.json index e56a6ee7..64d71e42 100644 --- a/front/tsconfig.json +++ b/front/tsconfig.json @@ -3,9 +3,8 @@ "outDir": "./dist/", "sourceMap": true, "moduleResolution": "node", - "noImplicitAny": true, "module": "CommonJS", - "target": "es5", + "target": "es6", "downlevelIteration": true, "jsx": "react", "allowJs": true,