diff --git a/back/src/Controller/IoSocketController.ts b/back/src/Controller/IoSocketController.ts index 243290f9..1421a0ac 100644 --- a/back/src/Controller/IoSocketController.ts +++ b/back/src/Controller/IoSocketController.ts @@ -13,7 +13,6 @@ import {MessageUserJoined} from "../Model/Websocket/MessageUserJoined"; import {MessageUserMoved} from "../Model/Websocket/MessageUserMoved"; import si from "systeminformation"; import {Gauge} from "prom-client"; -import os from 'os'; import {TokenInterface} from "../Controller/AuthenticateController"; import {isJoinRoomMessageInterface} from "../Model/Websocket/JoinRoomMessage"; import {isPointInterface, PointInterface} from "../Model/Websocket/PointInterface"; @@ -29,6 +28,7 @@ enum SockerIoEvent { USER_MOVED = "user-moved", // From server to client USER_LEFT = "user-left", // From server to client WEBRTC_SIGNAL = "webrtc-signal", + WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal", WEBRTC_START = "webrtc-start", WEBRTC_DISCONNECT = "webrtc-disconect", MESSAGE_ERROR = "message-error", @@ -237,18 +237,11 @@ export class IoSocketController { }); socket.on(SockerIoEvent.WEBRTC_SIGNAL, (data: unknown) => { - if (!isWebRtcSignalMessageInterface(data)) { - socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'}); - console.warn('Invalid WEBRTC_SIGNAL message received: ', data); - return; - } - //send only at user - const client = this.sockets.get(data.receiverId); - if (client === undefined) { - console.warn("While exchanging a WebRTC signal: client with id ", data.receiverId, " does not exist. This might be a race condition."); - return; - } - return client.emit(SockerIoEvent.WEBRTC_SIGNAL, data); + this.emitVideo((socket as ExSocketInterface), data); + }); + + socket.on(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, (data: unknown) => { + this.emitScreenSharing((socket as ExSocketInterface), data); }); socket.on(SockerIoEvent.DISCONNECT, () => { @@ -318,6 +311,42 @@ export class IoSocketController { }); } + emitVideo(socket: ExSocketInterface, data: unknown){ + if (!isWebRtcSignalMessageInterface(data)) { + socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'}); + console.warn('Invalid WEBRTC_SIGNAL message received: ', data); + return; + } + //send only at user + const client = this.sockets.get(data.receiverId); + if (client === undefined) { + console.warn("While exchanging a WebRTC signal: client with id ", data.receiverId, " does not exist. This might be a race condition."); + return; + } + return client.emit(SockerIoEvent.WEBRTC_SIGNAL, { + userId: socket.userId, + signal: data.signal + }); + } + + emitScreenSharing(socket: ExSocketInterface, data: unknown){ + if (!isWebRtcSignalMessageInterface(data)) { + socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SCREEN_SHARING message.'}); + console.warn('Invalid WEBRTC_SCREEN_SHARING message received: ', data); + return; + } + //send only at user + const client = this.sockets.get(data.receiverId); + if (client === undefined) { + console.warn("While exchanging a WEBRTC_SCREEN_SHARING signal: client with id ", data.receiverId, " does not exist. This might be a race condition."); + return; + } + return client.emit(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, { + userId: socket.userId, + signal: data.signal + }); + } + searchClientByIdOrFail(userId: string): ExSocketInterface { const client: ExSocketInterface|undefined = this.sockets.get(userId); if (client === undefined) { @@ -398,13 +427,15 @@ export class IoSocketController { if (this.Io.sockets.adapter.rooms[roomId].length < 2 /*|| this.Io.sockets.adapter.rooms[roomId].length >= 4*/) { return; } + + // TODO: scanning all sockets is maybe not the most efficient const clients: Array = (Object.values(this.Io.sockets.sockets) as Array) .filter((client: ExSocketInterface) => client.webRtcRoomId && client.webRtcRoomId === roomId); //send start at one client to initialise offer webrtc //send all users in room to create PeerConnection in front clients.forEach((client: ExSocketInterface, index: number) => { - const clientsId = clients.reduce((tabs: Array, clientId: ExSocketInterface, indexClientId: number) => { + const peerClients = clients.reduce((tabs: Array, clientId: ExSocketInterface, indexClientId: number) => { if (!clientId.userId || clientId.userId === client.userId) { return tabs; } @@ -416,7 +447,7 @@ export class IoSocketController { return tabs; }, []); - client.emit(SockerIoEvent.WEBRTC_START, {clients: clientsId, roomId: roomId}); + client.emit(SockerIoEvent.WEBRTC_START, {clients: peerClients, roomId: roomId}); }); } diff --git a/back/src/Model/Websocket/WebRtcSignalMessage.ts b/back/src/Model/Websocket/WebRtcSignalMessage.ts index 7edffdfa..5a0dd1af 100644 --- a/back/src/Model/Websocket/WebRtcSignalMessage.ts +++ b/back/src/Model/Websocket/WebRtcSignalMessage.ts @@ -1,10 +1,18 @@ import * as tg from "generic-type-guard"; +export const isSignalData = + new tg.IsInterface().withProperties({ + type: tg.isOptional(tg.isString) + }).get(); + export const isWebRtcSignalMessageInterface = new tg.IsInterface().withProperties({ - userId: tg.isString, receiverId: tg.isString, - roomId: tg.isString, - signal: tg.isUnknown + signal: isSignalData + }).get(); +export const isWebRtcScreenSharingStartMessageInterface = + new tg.IsInterface().withProperties({ + userId: tg.isString, + roomId: tg.isString }).get(); export type WebRtcSignalMessageInterface = tg.GuardedType; diff --git a/front/dist/index.html b/front/dist/index.html index 92a7bf3c..5de00b3b 100644 --- a/front/dist/index.html +++ b/front/dist/index.html @@ -68,15 +68,19 @@
-
-
- - -
-
- - -
+
+
+
+ + +
+
+ + +
+
+ +
@@ -100,9 +104,15 @@ +
+ + +
--> +
+
diff --git a/front/dist/resources/logos/monitor-close.svg b/front/dist/resources/logos/monitor-close.svg new file mode 100644 index 00000000..80056e2d --- /dev/null +++ b/front/dist/resources/logos/monitor-close.svg @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/front/dist/resources/logos/monitor.svg b/front/dist/resources/logos/monitor.svg new file mode 100644 index 00000000..d4b586c6 --- /dev/null +++ b/front/dist/resources/logos/monitor.svg @@ -0,0 +1,15 @@ + + + + + + diff --git a/front/dist/resources/style/style.css b/front/dist/resources/style/style.css index 30e099ef..8d232fb5 100644 --- a/front/dist/resources/style/style.css +++ b/front/dist/resources/style/style.css @@ -79,6 +79,13 @@ video#myCamVideo{ } +.btn-cam-action { + position: absolute; + bottom: 0px; + right: 0px; + width: 450px; + height: 150px; +} /*btn animation*/ .btn-cam-action div{ cursor: pointer; @@ -89,11 +96,17 @@ video#myCamVideo{ background: #666; box-shadow: 2px 2px 24px #444; border-radius: 48px; - transform: translateY(12vh); + transform: translateY(40px); transition-timing-function: ease-in-out; bottom: 20px; } -#activeCam:hover .btn-cam-action div{ +.btn-cam-action div.disabled { + background: #d75555; +} +.btn-cam-action div.enabled { + background: #73c973; +} +.btn-cam-action:hover div{ transform: translateY(0); } .btn-cam-action div:hover{ @@ -106,9 +119,13 @@ video#myCamVideo{ right: 44px; } .btn-video{ - transition: all .2s; + transition: all .25s; right: 134px; } +.btn-monitor{ + transition: all .2s; + right: 224px; +} /*.btn-call{ transition: all .1s; left: 0px; @@ -300,13 +317,13 @@ body { flex: 0 0 75%; display: flex; justify-content: start; - /*align-items: flex-start;*/ + align-items: flex-start; flex-wrap: wrap; } .main-section > div { - margin: 5%; - flex-basis: 90%; + margin: 2%; + flex-basis: 96%; /*flex-shrink: 2;*/ } diff --git a/front/src/Connection.ts b/front/src/Connection.ts index f234bfd5..a83a3fb0 100644 --- a/front/src/Connection.ts +++ b/front/src/Connection.ts @@ -6,12 +6,12 @@ import {SetPlayerDetailsMessage} from "./Messages/SetPlayerDetailsMessage"; const SocketIo = require('socket.io-client'); import Socket = SocketIOClient.Socket; import {PlayerAnimationNames} from "./Phaser/Player/Animation"; -import {UserSimplePeer} from "./WebRtc/SimplePeer"; +import {UserSimplePeerInterface} from "./WebRtc/SimplePeer"; import {SignalData} from "simple-peer"; - enum EventMessage{ WEBRTC_SIGNAL = "webrtc-signal", + WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal", WEBRTC_START = "webrtc-start", JOIN_ROOM = "join-room", // bi-directional USER_POSITION = "user-position", // bi-directional @@ -73,17 +73,20 @@ export interface GroupCreatedUpdatedMessageInterface { export interface WebRtcStartMessageInterface { roomId: string, - clients: UserSimplePeer[] + clients: UserSimplePeerInterface[] } export interface WebRtcDisconnectMessageInterface { userId: string } -export interface WebRtcSignalMessageInterface { - userId: string, +export interface WebRtcSignalSentMessageInterface { receiverId: string, - roomId: string, + signal: SignalData +} + +export interface WebRtcSignalReceivedMessageInterface { + userId: string, signal: SignalData } @@ -201,23 +204,32 @@ export class Connection implements Connection { this.socket.on(EventMessage.CONNECT_ERROR, callback) } - public sendWebrtcSignal(signal: unknown, roomId: string, userId? : string|null, receiverId? : string) { + public sendWebrtcSignal(signal: unknown, receiverId : string) { return this.socket.emit(EventMessage.WEBRTC_SIGNAL, { - userId: userId ? userId : this.userId, - receiverId: receiverId ? receiverId : this.userId, - roomId: roomId, + receiverId: receiverId, signal: signal - }); + } as WebRtcSignalSentMessageInterface); + } + + public sendWebrtcScreenSharingSignal(signal: unknown, receiverId : string) { + return this.socket.emit(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, { + receiverId: receiverId, + signal: signal + } as WebRtcSignalSentMessageInterface); } public receiveWebrtcStart(callback: (message: WebRtcStartMessageInterface) => void) { this.socket.on(EventMessage.WEBRTC_START, callback); } - public receiveWebrtcSignal(callback: (message: WebRtcSignalMessageInterface) => void) { + public receiveWebrtcSignal(callback: (message: WebRtcSignalReceivedMessageInterface) => void) { return this.socket.on(EventMessage.WEBRTC_SIGNAL, callback); } + public receiveWebrtcScreenSharingSignal(callback: (message: WebRtcSignalReceivedMessageInterface) => void) { + return this.socket.on(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, callback); + } + public onServerDisconnected(callback: (reason: string) => void): void { this.socket.on('disconnect', (reason: string) => { if (reason === 'io client disconnect') { diff --git a/front/src/Phaser/Game/GameScene.ts b/front/src/Phaser/Game/GameScene.ts index 21775f54..b0154a0e 100644 --- a/front/src/Phaser/Game/GameScene.ts +++ b/front/src/Phaser/Game/GameScene.ts @@ -10,7 +10,7 @@ import { RoomJoinedMessageInterface } from "../../Connection"; import {CurrentGamerInterface, hasMovedEventName, Player} from "../Player/Player"; -import {DEBUG_MODE, POSITION_DELAY, ZOOM_LEVEL} from "../../Enum/EnvironmentVariable"; +import {DEBUG_MODE, POSITION_DELAY, RESOLUTION, ZOOM_LEVEL} from "../../Enum/EnvironmentVariable"; import { ITiledMap, ITiledMapLayer, @@ -24,10 +24,10 @@ import {PlayerMovement} from "./PlayerMovement"; import {PlayersPositionInterpolator} from "./PlayersPositionInterpolator"; import {RemotePlayer} from "../Entity/RemotePlayer"; import {Queue} from 'queue-typescript'; -import {SimplePeer, UserSimplePeer} from "../../WebRtc/SimplePeer"; +import {SimplePeer, UserSimplePeerInterface} from "../../WebRtc/SimplePeer"; import {ReconnectingSceneName} from "../Reconnecting/ReconnectingScene"; import {loadAllLayers} from "../Entity/body_character"; -import {layoutManager, LayoutMode} from "../../WebRtc/LayoutManager"; +import {CenterListener, layoutManager, LayoutMode} from "../../WebRtc/LayoutManager"; import Texture = Phaser.Textures.Texture; import Sprite = Phaser.GameObjects.Sprite; import CanvasTexture = Phaser.Textures.CanvasTexture; @@ -78,7 +78,7 @@ interface DeleteGroupEventInterface { groupId: string } -export class GameScene extends Phaser.Scene { +export class GameScene extends Phaser.Scene implements CenterListener { GameManager : GameManager; Terrains : Array; CurrentPlayer!: CurrentGamerInterface; @@ -263,7 +263,7 @@ export class GameScene extends Phaser.Scene { this.simplePeer = new SimplePeer(this.connection); const self = this; this.simplePeer.registerPeerConnectionListener({ - onConnect(user: UserSimplePeer) { + onConnect(user: UserSimplePeerInterface) { self.presentationModeSprite.setVisible(true); self.chatModeSprite.setVisible(true); }, @@ -537,6 +537,9 @@ export class GameScene extends Phaser.Scene { this.repositionCallback = this.reposition.bind(this); window.addEventListener('resize', this.repositionCallback); this.reposition(); + + // From now, this game scene will be notified of reposition events + layoutManager.setListener(this); } private switchLayoutMode(): void { @@ -656,7 +659,7 @@ export class GameScene extends Phaser.Scene { //todo: in a dedicated class/function? initCamera() { this.cameras.main.setBounds(0,0, this.Map.widthInPixels, this.Map.heightInPixels); - this.cameras.main.startFollow(this.CurrentPlayer); + this.updateCameraOffset(); this.cameras.main.setZoom(ZOOM_LEVEL); } @@ -1058,5 +1061,30 @@ export class GameScene extends Phaser.Scene { private reposition(): void { this.presentationModeSprite.setY(this.game.renderer.height - 2); this.chatModeSprite.setY(this.game.renderer.height - 2); + + // Recompute camera offset if needed + this.updateCameraOffset(); + } + + /** + * Updates the offset of the character compared to the center of the screen according to the layout mananger + * (tries to put the character in the center of the reamining space if there is a discussion going on. + */ + private updateCameraOffset(): void { + const array = layoutManager.findBiggestAvailableArray(); + let xCenter = (array.xEnd - array.xStart) / 2 + array.xStart; + let yCenter = (array.yEnd - array.yStart) / 2 + array.yStart; + + // Let's put this in Game coordinates by applying the zoom level: + xCenter /= ZOOM_LEVEL * RESOLUTION; + yCenter /= ZOOM_LEVEL * RESOLUTION; + + //console.log("updateCameraOffset", array, xCenter, yCenter, this.game.renderer.width, this.game.renderer.height); + + this.cameras.main.startFollow(this.CurrentPlayer, true, 1, 1, xCenter - this.game.renderer.width / 2, yCenter - this.game.renderer.height / 2); + } + + public onCenterChange(): void { + this.updateCameraOffset(); } } diff --git a/front/src/WebRtc/LayoutManager.ts b/front/src/WebRtc/LayoutManager.ts index 6695fe7f..c2bb683e 100644 --- a/front/src/WebRtc/LayoutManager.ts +++ b/front/src/WebRtc/LayoutManager.ts @@ -14,6 +14,14 @@ export enum DivImportance { Normal = "Normal", } +/** + * Classes implementing this interface can be notified when the center of the screen (the player position) should be + * changed. + */ +export interface CenterListener { + onCenterChange(): void; +} + /** * This class is in charge of the video-conference layout. * It receives positioning requests for videos and does its best to place them on the screen depending on the active layout mode. @@ -23,6 +31,11 @@ class LayoutManager { private importantDivs: Map = new Map(); private normalDivs: Map = new Map(); + private listener: CenterListener|null = null; + + public setListener(centerListener: CenterListener|null) { + this.listener = centerListener; + } public add(importance: DivImportance, userId: string, html: string): void { const div = document.createElement('div'); @@ -45,6 +58,7 @@ class LayoutManager { this.positionDiv(div, importance); this.adjustVideoChatClass(); + this.listener?.onCenterChange(); } private positionDiv(elem: HTMLDivElement, importance: DivImportance): void { @@ -72,6 +86,7 @@ class LayoutManager { div.remove(); this.importantDivs.delete(userId); this.adjustVideoChatClass(); + this.listener?.onCenterChange(); return; } @@ -80,6 +95,7 @@ class LayoutManager { div.remove(); this.normalDivs.delete(userId); this.adjustVideoChatClass(); + this.listener?.onCenterChange(); return; } @@ -123,11 +139,133 @@ class LayoutManager { for (const div of this.normalDivs.values()) { this.positionDiv(div, DivImportance.Normal); } + this.listener?.onCenterChange(); } public getLayoutMode(): LayoutMode { return this.mode; } + + /*public getGameCenter(): {x: number, y: number} { + + }*/ + + /** + * Tries to find the biggest available box of remaining space (this is a space where we can center the character) + */ + public findBiggestAvailableArray(): {xStart: number, yStart: number, xEnd: number, yEnd: number} { + if (this.mode === LayoutMode.VideoChat) { + const children = document.querySelectorAll('div.chat-mode > div'); + const htmlChildren = Array.from(children.values()); + + // No chat? Let's go full center + if (htmlChildren.length === 0) { + return { + xStart: 0, + yStart: 0, + xEnd: window.innerWidth, + yEnd: window.innerHeight + } + } + + const lastDiv = htmlChildren[htmlChildren.length - 1]; + // Compute area between top right of the last div and bottom right of window + const area1 = (window.innerWidth - (lastDiv.offsetLeft + lastDiv.offsetWidth)) + * (window.innerHeight - lastDiv.offsetTop); + + // Compute area between bottom of last div and bottom of the screen on whole width + const area2 = window.innerWidth + * (window.innerHeight - (lastDiv.offsetTop + lastDiv.offsetHeight)); + + if (area1 < 0 && area2 < 0) { + // If screen is full, let's not attempt something foolish and simply center character in the middle. + return { + xStart: 0, + yStart: 0, + xEnd: window.innerWidth, + yEnd: window.innerHeight + } + } + if (area1 <= area2) { + console.log('lastDiv', lastDiv.offsetTop, lastDiv.offsetHeight); + return { + xStart: 0, + yStart: lastDiv.offsetTop + lastDiv.offsetHeight, + xEnd: window.innerWidth, + yEnd: window.innerHeight + } + } else { + console.log('lastDiv', lastDiv.offsetTop); + return { + xStart: lastDiv.offsetLeft + lastDiv.offsetWidth, + yStart: lastDiv.offsetTop, + xEnd: window.innerWidth, + yEnd: window.innerHeight + } + } + } else { + // Possible destinations: at the center bottom or at the right bottom. + const mainSectionChildren = Array.from(document.querySelectorAll('div.main-section > div').values()); + const sidebarChildren = Array.from(document.querySelectorAll('aside.sidebar > div').values()); + + // Nothing? Let's center + if (mainSectionChildren.length === 0 && sidebarChildren.length === 0) { + return { + xStart: 0, + yStart: 0, + xEnd: window.innerWidth, + yEnd: window.innerHeight + } + } + + if (mainSectionChildren.length === 0) { + const lastSidebarDiv = sidebarChildren[sidebarChildren.length-1]; + + // No presentation? Let's center on the main-section space + return { + xStart: 0, + yStart: 0, + xEnd: lastSidebarDiv.offsetLeft, + yEnd: window.innerHeight + } + } + + // At this point, we know we have at least one element in the main section. + const lastPresentationDiv = mainSectionChildren[mainSectionChildren.length-1]; + + const presentationArea = (window.innerHeight - (lastPresentationDiv.offsetTop + lastPresentationDiv.offsetHeight)) + * (lastPresentationDiv.offsetLeft + lastPresentationDiv.offsetWidth); + + let leftSideBar: number; + let bottomSideBar: number; + if (sidebarChildren.length === 0) { + leftSideBar = HtmlUtils.getElementByIdOrFail('sidebar').offsetLeft; + bottomSideBar = 0; + } else { + const lastSideBarChildren = sidebarChildren[sidebarChildren.length - 1]; + leftSideBar = lastSideBarChildren.offsetLeft; + bottomSideBar = lastSideBarChildren.offsetTop + lastSideBarChildren.offsetHeight; + } + const sideBarArea = (window.innerWidth - leftSideBar) + * (window.innerHeight - bottomSideBar); + + if (presentationArea <= sideBarArea) { + return { + xStart: leftSideBar, + yStart: bottomSideBar, + xEnd: window.innerWidth, + yEnd: window.innerHeight + } + } else { + return { + xStart: 0, + yStart: lastPresentationDiv.offsetTop + lastPresentationDiv.offsetHeight, + xEnd: /*lastPresentationDiv.offsetLeft + lastPresentationDiv.offsetWidth*/ window.innerWidth , // To avoid flickering when a chat start, we center on the center of the screen, not the center of the main content area + yEnd: window.innerHeight + } + } + } + } } const layoutManager = new LayoutManager(); diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 39a61738..a043e51e 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -1,4 +1,5 @@ import {DivImportance, layoutManager} from "./LayoutManager"; +import {HtmlUtils} from "./HtmlUtils"; const videoConstraint: boolean|MediaTrackConstraints = { width: { ideal: 1280 }, @@ -7,15 +8,20 @@ const videoConstraint: boolean|MediaTrackConstraints = { }; type UpdatedLocalStreamCallback = (media: MediaStream) => void; +type StartScreenSharingCallback = (media: MediaStream) => void; +type StopScreenSharingCallback = (media: MediaStream) => void; // TODO: Split MediaManager in 2 classes: MediaManagerUI (in charge of HTML) and MediaManager (singleton in charge of the camera only) // TODO: verify that microphone event listeners are not triggered plenty of time NOW (since MediaManager is created many times!!!!) export class MediaManager { localStream: MediaStream|null = null; + localScreenCapture: MediaStream|null = null; private remoteVideo: Map = new Map(); myCamVideo: HTMLVideoElement; cinemaClose: HTMLImageElement; cinema: HTMLImageElement; + monitorClose: HTMLImageElement; + monitor: HTMLImageElement; microphoneClose: HTMLImageElement; microphone: HTMLImageElement; webrtcInAudio: HTMLAudioElement; @@ -24,46 +30,81 @@ export class MediaManager { video: videoConstraint }; updatedLocalStreamCallBacks : Set = new Set(); + startScreenSharingCallBacks : Set = new Set(); + stopScreenSharingCallBacks : Set = new Set(); + private microphoneBtn: HTMLDivElement; + private cinemaBtn: HTMLDivElement; + private monitorBtn: HTMLDivElement; + constructor() { + this.myCamVideo = this.getElementByIdOrFail('myCamVideo'); this.webrtcInAudio = this.getElementByIdOrFail('audio-webrtc-in'); this.webrtcInAudio.volume = 0.2; + this.microphoneBtn = this.getElementByIdOrFail('btn-micro'); this.microphoneClose = this.getElementByIdOrFail('microphone-close'); this.microphoneClose.style.display = "none"; this.microphoneClose.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.enabledMicrophone(); + this.enableMicrophone(); //update tracking }); this.microphone = this.getElementByIdOrFail('microphone'); this.microphone.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.disabledMicrophone(); + this.disableMicrophone(); //update tracking }); + this.cinemaBtn = this.getElementByIdOrFail('btn-video'); this.cinemaClose = this.getElementByIdOrFail('cinema-close'); this.cinemaClose.style.display = "none"; this.cinemaClose.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.enabledCamera(); + this.enableCamera(); //update tracking }); this.cinema = this.getElementByIdOrFail('cinema'); this.cinema.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.disabledCamera(); + this.disableCamera(); + //update tracking + }); + + this.monitorBtn = this.getElementByIdOrFail('btn-monitor'); + this.monitorClose = this.getElementByIdOrFail('monitor-close'); + this.monitorClose.style.display = "block"; + this.monitorClose.addEventListener('click', (e: MouseEvent) => { + e.preventDefault(); + this.enableScreenSharing(); + //update tracking + }); + this.monitor = this.getElementByIdOrFail('monitor'); + this.monitor.style.display = "none"; + this.monitor.addEventListener('click', (e: MouseEvent) => { + e.preventDefault(); + this.disableScreenSharing(); //update tracking }); } - onUpdateLocalStream(callback: UpdatedLocalStreamCallback): void { + public onUpdateLocalStream(callback: UpdatedLocalStreamCallback): void { this.updatedLocalStreamCallBacks.add(callback); } + public onStartScreenSharing(callback: StartScreenSharingCallback): void { + + this.startScreenSharingCallBacks.add(callback); + } + + public onStopScreenSharing(callback: StopScreenSharingCallback): void { + + this.stopScreenSharingCallBacks.add(callback); + } + removeUpdateLocalStreamEventListener(callback: UpdatedLocalStreamCallback): void { this.updatedLocalStreamCallBacks.delete(callback); } @@ -74,13 +115,26 @@ export class MediaManager { } } - activeVisio(){ + private triggerStartedScreenSharingCallbacks(stream: MediaStream): void { + for (const callback of this.startScreenSharingCallBacks) { + callback(stream); + } + } + + private triggerStoppedScreenSharingCallbacks(stream: MediaStream): void { + for (const callback of this.stopScreenSharingCallBacks) { + callback(stream); + } + } + + showGameOverlay(){ const gameOverlay = this.getElementByIdOrFail('game-overlay'); gameOverlay.classList.add('active'); } - enabledCamera() { + private enableCamera() { this.cinemaClose.style.display = "none"; + this.cinemaBtn.classList.remove("disabled"); this.cinema.style.display = "block"; this.constraintsMedia.video = videoConstraint; this.getCamera().then((stream: MediaStream) => { @@ -88,9 +142,10 @@ export class MediaManager { }); } - disabledCamera() { + private disableCamera() { this.cinemaClose.style.display = "block"; this.cinema.style.display = "none"; + this.cinemaBtn.classList.add("disabled"); this.constraintsMedia.video = false; this.myCamVideo.srcObject = null; if (this.localStream) { @@ -103,18 +158,20 @@ export class MediaManager { }); } - enabledMicrophone() { + private enableMicrophone() { this.microphoneClose.style.display = "none"; this.microphone.style.display = "block"; + this.microphoneBtn.classList.remove("disabled"); this.constraintsMedia.audio = true; this.getCamera().then((stream) => { this.triggerUpdatedLocalStreamCallbacks(stream); }); } - disabledMicrophone() { + private disableMicrophone() { this.microphoneClose.style.display = "block"; this.microphone.style.display = "none"; + this.microphoneBtn.classList.add("disabled"); this.constraintsMedia.audio = false; if(this.localStream) { this.localStream.getAudioTracks().forEach((MediaStreamTrack: MediaStreamTrack) => { @@ -126,6 +183,80 @@ export class MediaManager { }); } + private enableScreenSharing() { + this.monitorClose.style.display = "none"; + this.monitor.style.display = "block"; + this.monitorBtn.classList.add("enabled"); + this.getScreenMedia().then((stream) => { + this.triggerStartedScreenSharingCallbacks(stream); + }); + } + + private disableScreenSharing() { + this.monitorClose.style.display = "block"; + this.monitor.style.display = "none"; + this.monitorBtn.classList.remove("enabled"); + this.removeActiveScreenSharingVideo('me'); + this.localScreenCapture?.getTracks().forEach((track: MediaStreamTrack) => { + track.stop(); + }); + if (this.localScreenCapture === null) { + console.warn('Weird: trying to remove a screen sharing that is not enabled'); + return; + } + const localScreenCapture = this.localScreenCapture; + this.getCamera().then((stream) => { + this.triggerStoppedScreenSharingCallbacks(localScreenCapture); + }); + this.localScreenCapture = null; + } + + //get screen + getScreenMedia() : Promise{ + try { + return this._startScreenCapture() + .then((stream: MediaStream) => { + this.localScreenCapture = stream; + + // If stream ends (for instance if user clicks the stop screen sharing button in the browser), let's close the view + for (const track of stream.getTracks()) { + track.onended = () => { + this.disableScreenSharing(); + }; + } + + this.addScreenSharingActiveVideo('me', DivImportance.Normal); + HtmlUtils.getElementByIdOrFail('screen-sharing-me').srcObject = stream; + + return stream; + }) + .catch((err: unknown) => { + console.error("Error => getScreenMedia => ", err); + throw err; + }); + }catch (err) { + return new Promise((resolve, reject) => { // eslint-disable-line no-unused-vars + reject(err); + }); + } + } + + private _startScreenCapture() { + // getDisplayMedia was moved to mediaDevices in 2018. Typescript definitions are not up to date yet. + // See: https://github.com/w3c/mediacapture-screen-share/pull/86 + // https://github.com/microsoft/TypeScript/issues/31821 + if ((navigator as any).getDisplayMedia) { // eslint-disable-line @typescript-eslint/no-explicit-any + return (navigator as any).getDisplayMedia({video: true}); // eslint-disable-line @typescript-eslint/no-explicit-any + } else if ((navigator.mediaDevices as any).getDisplayMedia) { // eslint-disable-line @typescript-eslint/no-explicit-any + return (navigator.mediaDevices as any).getDisplayMedia({video: true}); // eslint-disable-line @typescript-eslint/no-explicit-any + } else { + //return navigator.mediaDevices.getUserMedia(({video: {mediaSource: 'screen'}} as any)); + return new Promise((resolve, reject) => { // eslint-disable-line no-unused-vars + reject("error sharing screen"); + }); + } + } + //get camera async getCamera(): Promise { if (navigator.mediaDevices === undefined) { @@ -205,6 +336,25 @@ export class MediaManager { this.remoteVideo.set(userId, this.getElementByIdOrFail(userId)); } + /** + * + * @param userId + */ + addScreenSharingActiveVideo(userId : string, divImportance: DivImportance = DivImportance.Important){ + //this.webrtcInAudio.play(); + + userId = `screen-sharing-${userId}`; + const html = ` +
+ +
+ `; + + layoutManager.add(divImportance, userId, html); + + this.remoteVideo.set(userId, this.getElementByIdOrFail(userId)); + } + /** * * @param userId @@ -272,6 +422,15 @@ export class MediaManager { } remoteVideo.srcObject = stream; } + addStreamRemoteScreenSharing(userId : string, stream : MediaStream){ + // In the case of screen sharing (going both ways), we may need to create the HTML element if it does not exist yet + const remoteVideo = this.remoteVideo.get(`screen-sharing-${userId}`); + if (remoteVideo === undefined) { + this.addScreenSharingActiveVideo(userId); + } + + this.addStreamRemoteVideo(`screen-sharing-${userId}`, stream); + } /** * @@ -281,6 +440,9 @@ export class MediaManager { layoutManager.remove(userId); this.remoteVideo.delete(userId); } + removeActiveScreenSharingVideo(userId : string) { + this.removeActiveVideo(`screen-sharing-${userId}`) + } isConnecting(userId : string): void { const connectingSpinnerDiv = this.getSpinner(userId); @@ -299,6 +461,7 @@ export class MediaManager { } isError(userId : string): void { + console.log("isError", `div-${userId}`); const element = document.getElementById(`div-${userId}`); if(!element){ return; @@ -309,6 +472,10 @@ export class MediaManager { } errorDiv.style.display = 'block'; } + isErrorScreenSharing(userId : string): void { + this.isError(`screen-sharing-${userId}`); + } + private getSpinner(userId : string): HTMLDivElement|null { const element = document.getElementById(`div-${userId}`); diff --git a/front/src/WebRtc/ScreenSharingPeer.ts b/front/src/WebRtc/ScreenSharingPeer.ts new file mode 100644 index 00000000..35f43201 --- /dev/null +++ b/front/src/WebRtc/ScreenSharingPeer.ts @@ -0,0 +1,127 @@ +import * as SimplePeerNamespace from "simple-peer"; +import {mediaManager} from "./MediaManager"; +import {Connection} from "../Connection"; + +const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer'); + +/** + * A peer connection used to transmit video / audio signals between 2 peers. + */ +export class ScreenSharingPeer extends Peer { + /** + * Whether this connection is currently receiving a video stream from a remote user. + */ + private isReceivingStream:boolean = false; + + constructor(private userId: string, initiator: boolean, private connection: Connection) { + super({ + initiator: initiator ? initiator : false, + reconnectTimer: 10000, + config: { + iceServers: [ + { + urls: 'stun:stun.l.google.com:19302' + }, + { + urls: 'turn:numb.viagenie.ca', + username: 'g.parant@thecodingmachine.com', + credential: 'itcugcOHxle9Acqi$' + }, + ] + } + }); + + //start listen signal for the peer connection + this.on('signal', (data: unknown) => { + this.sendWebrtcScreenSharingSignal(data); + }); + + this.on('stream', (stream: MediaStream) => { + this.stream(stream); + }); + + this.on('close', () => { + this.destroy(); + }); + + this.on('data', (chunk: Buffer) => { + // We unfortunately need to rely on an event to let the other party know a stream has stopped. + // It seems there is no native way to detect that. + const message = JSON.parse(chunk.toString('utf8')); + if (message.streamEnded !== true) { + console.error('Unexpected message on screen sharing peer connection'); + } + mediaManager.removeActiveScreenSharingVideo(this.userId); + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + this.on('error', (err: any) => { + console.error(`screen sharing error => ${this.userId} => ${err.code}`, err); + //mediaManager.isErrorScreenSharing(this.userId); + }); + + this.on('connect', () => { + // FIXME: we need to put the loader on the screen sharing connection + mediaManager.isConnected(this.userId); + console.info(`connect => ${this.userId}`); + }); + + this.pushScreenSharingToRemoteUser(); + } + + private sendWebrtcScreenSharingSignal(data: unknown) { + console.log("sendWebrtcScreenSharingSignal", data); + try { + this.connection.sendWebrtcScreenSharingSignal(data, this.userId); + }catch (e) { + console.error(`sendWebrtcScreenSharingSignal => ${this.userId}`, e); + } + } + + /** + * Sends received stream to screen. + */ + private stream(stream?: MediaStream) { + console.log(`ScreenSharingPeer::stream => ${this.userId}`, stream); + console.log(`stream => ${this.userId} => `, stream); + if(!stream){ + mediaManager.removeActiveScreenSharingVideo(this.userId); + this.isReceivingStream = false; + } else { + mediaManager.addStreamRemoteScreenSharing(this.userId, stream); + this.isReceivingStream = true; + } + } + + public isReceivingScreenSharingStream(): boolean { + return this.isReceivingStream; + } + + public destroy(error?: Error): void { + try { + mediaManager.removeActiveScreenSharingVideo(this.userId); + // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" + // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. + //console.log('Closing connection with '+userId); + super.destroy(error); + //console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size); + } catch (err) { + console.error("ScreenSharingPeer::destroy", err) + } + } + + private pushScreenSharingToRemoteUser() { + const localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; + if(!localScreenCapture){ + return; + } + + this.addStream(localScreenCapture); + return; + } + + public stopPushingScreenSharingToRemoteUser(stream: MediaStream) { + this.removeStream(stream); + this.write(new Buffer(JSON.stringify({streamEnded: true}))); + } +} diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index fdc2d0c2..3acd65c5 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -1,21 +1,23 @@ import { Connection, WebRtcDisconnectMessageInterface, - WebRtcSignalMessageInterface, + WebRtcSignalReceivedMessageInterface, WebRtcStartMessageInterface } from "../Connection"; import { mediaManager } from "./MediaManager"; import * as SimplePeerNamespace from "simple-peer"; +import {ScreenSharingPeer} from "./ScreenSharingPeer"; +import {VideoPeer} from "./VideoPeer"; const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer'); -export interface UserSimplePeer{ +export interface UserSimplePeerInterface{ userId: string; name?: string; initiator?: boolean; } export interface PeerConnectionListener { - onConnect(user: UserSimplePeer): void; + onConnect(user: UserSimplePeerInterface): void; onDisconnect(userId: string): void; } @@ -26,18 +28,25 @@ export interface PeerConnectionListener { export class SimplePeer { private Connection: Connection; private WebRtcRoomId: string; - private Users: Array = new Array(); + private Users: Array = new Array(); - private PeerConnectionArray: Map = new Map(); - private readonly updateLocalStreamCallback: (media: MediaStream) => void; + private PeerScreenSharingConnectionArray: Map = new Map(); + private PeerConnectionArray: Map = new Map(); + private readonly sendLocalVideoStreamCallback: (media: MediaStream) => void; + private readonly sendLocalScreenSharingStreamCallback: (media: MediaStream) => void; + private readonly stopLocalScreenSharingStreamCallback: (media: MediaStream) => void; private readonly peerConnectionListeners: Array = new Array(); constructor(Connection: Connection, WebRtcRoomId: string = "test-webrtc") { this.Connection = Connection; this.WebRtcRoomId = WebRtcRoomId; // We need to go through this weird bound function pointer in order to be able to "free" this reference later. - this.updateLocalStreamCallback = this.updatedLocalStream.bind(this); - mediaManager.onUpdateLocalStream(this.updateLocalStreamCallback); + this.sendLocalVideoStreamCallback = this.sendLocalVideoStream.bind(this); + this.sendLocalScreenSharingStreamCallback = this.sendLocalScreenSharingStream.bind(this); + this.stopLocalScreenSharingStreamCallback = this.stopLocalScreenSharingStream.bind(this); + mediaManager.onUpdateLocalStream(this.sendLocalVideoStreamCallback); + mediaManager.onStartScreenSharing(this.sendLocalScreenSharingStreamCallback); + mediaManager.onStopScreenSharing(this.stopLocalScreenSharingStreamCallback); this.initialise(); } @@ -55,11 +64,16 @@ export class SimplePeer { private initialise() { //receive signal by gemer - this.Connection.receiveWebrtcSignal((message: WebRtcSignalMessageInterface) => { + this.Connection.receiveWebrtcSignal((message: WebRtcSignalReceivedMessageInterface) => { this.receiveWebrtcSignal(message); }); - mediaManager.activeVisio(); + //receive signal by gemer + this.Connection.receiveWebrtcScreenSharingSignal((message: WebRtcSignalReceivedMessageInterface) => { + this.receiveWebrtcScreenSharingSignal(message); + }); + + mediaManager.showGameOverlay(); mediaManager.getCamera().then(() => { //receive message start @@ -79,7 +93,7 @@ export class SimplePeer { private receiveWebrtcStart(data: WebRtcStartMessageInterface) { this.WebRtcRoomId = data.roomId; this.Users = data.clients; - // Note: the clients array contain the list of all clients (event the ones we are already connected to in case a user joints a group) + // Note: the clients array contain the list of all clients (even the ones we are already connected to in case a user joints a group) // So we can receive a request we already had before. (which will abort at the first line of createPeerConnection) // TODO: refactor this to only send a message to connect to one user (rather than several users). // This would be symmetrical to the way we handle disconnection. @@ -93,7 +107,8 @@ export class SimplePeer { * server has two people connected, start the meet */ private startWebRtc() { - this.Users.forEach((user: UserSimplePeer) => { + console.warn('startWebRtc startWebRtc'); + this.Users.forEach((user: UserSimplePeerInterface) => { //if it's not an initiator, peer connection will be created when gamer will receive offer signal if(!user.initiator){ return; @@ -105,102 +120,63 @@ export class SimplePeer { /** * create peer connection to bind users */ - private createPeerConnection(user : UserSimplePeer) { - if(this.PeerConnectionArray.has(user.userId)) { - return; + private createPeerConnection(user : UserSimplePeerInterface) : VideoPeer | null{ + if( + this.PeerConnectionArray.has(user.userId) + ){ + return null; } - //console.log("Creating connection with peer "+user.userId); - let name = user.name; if(!name){ - const userSearch = this.Users.find((userSearch: UserSimplePeer) => userSearch.userId === user.userId); + const userSearch = this.Users.find((userSearch: UserSimplePeerInterface) => userSearch.userId === user.userId); if(userSearch) { name = userSearch.name; } } + mediaManager.removeActiveVideo(user.userId); mediaManager.addActiveVideo(user.userId, name); - const peer : SimplePeerNamespace.Instance = new Peer({ - initiator: user.initiator ? user.initiator : false, - reconnectTimer: 10000, - config: { - iceServers: [ - { - urls: 'stun:stun.l.google.com:19302' - }, - { - urls: 'turn:numb.viagenie.ca', - username: 'g.parant@thecodingmachine.com', - credential: 'itcugcOHxle9Acqi$' - }, - ] - }, + const peer = new VideoPeer(user.userId, user.initiator ? user.initiator : false, this.Connection); + // When a connection is established to a video stream, and if a screen sharing is taking place, + // the user sharing screen should also initiate a connection to the remote user! + peer.on('connect', () => { + if (mediaManager.localScreenCapture) { + this.sendLocalScreenSharingStreamToUser(user.userId); + } }); this.PeerConnectionArray.set(user.userId, peer); - //start listen signal for the peer connection - peer.on('signal', (data: unknown) => { - this.sendWebrtcSignal(data, user.userId); - }); - - peer.on('stream', (stream: MediaStream) => { - let videoActive = false; - let microphoneActive = false; - stream.getTracks().forEach((track : MediaStreamTrack) => { - if(track.kind === "audio"){ - microphoneActive = true; - } - if(track.kind === "video"){ - videoActive = true; - } - }); - if(microphoneActive){ - mediaManager.enabledMicrophoneByUserId(user.userId); - }else{ - mediaManager.disabledMicrophoneByUserId(user.userId); - } - - if(videoActive){ - mediaManager.enabledVideoByUserId(user.userId); - }else{ - mediaManager.disabledVideoByUserId(user.userId); - } - this.stream(user.userId, stream); - }); - - /*peer.on('track', (track: MediaStreamTrack, stream: MediaStream) => { - this.stream(user.userId, stream); - });*/ - - peer.on('close', () => { - this.closeConnection(user.userId); - }); - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - peer.on('error', (err: any) => { - console.error(`error => ${user.userId} => ${err.code}`, err); - mediaManager.isError(user.userId); - }); - - peer.on('connect', () => { - mediaManager.isConnected(user.userId); - console.info(`connect => ${user.userId}`); - }); - - peer.on('data', (chunk: Buffer) => { - const data = JSON.parse(chunk.toString('utf8')); - if(data.type === "stream"){ - this.stream(user.userId, data.stream); - } - }); - - this.addMedia(user.userId); - for (const peerConnectionListener of this.peerConnectionListeners) { peerConnectionListener.onConnect(user); } + return peer; + } + + /** + * create peer connection to bind users + */ + private createPeerScreenSharingConnection(user : UserSimplePeerInterface) : ScreenSharingPeer | null{ + if( + this.PeerScreenSharingConnectionArray.has(user.userId) + ){ + return null; + } + + // We should display the screen sharing ONLY if we are not initiator + if (!user.initiator) { + mediaManager.removeActiveScreenSharingVideo(user.userId); + mediaManager.addScreenSharingActiveVideo(user.userId); + } + + const peer = new ScreenSharingPeer(user.userId, user.initiator ? user.initiator : false, this.Connection); + this.PeerScreenSharingConnectionArray.set(user.userId, peer); + + for (const peerConnectionListener of this.peerConnectionListeners) { + peerConnectionListener.onConnect(user); + } + return peer; } /** @@ -210,17 +186,18 @@ export class SimplePeer { */ private closeConnection(userId : string) { try { - mediaManager.removeActiveVideo(userId); + //mediaManager.removeActiveVideo(userId); const peer = this.PeerConnectionArray.get(userId); if (peer === undefined) { console.warn("Tried to close connection for user "+userId+" but could not find user") return; } + peer.destroy(); // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. //console.log('Closing connection with '+userId); - peer.destroy(); - this.PeerConnectionArray.delete(userId) + this.PeerConnectionArray.delete(userId); + this.closeScreenSharingConnection(userId); //console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size); for (const peerConnectionListener of this.peerConnectionListeners) { peerConnectionListener.onDisconnect(userId); @@ -230,34 +207,49 @@ export class SimplePeer { } } + /** + * This is triggered twice. Once by the server, and once by a remote client disconnecting + * + * @param userId + */ + private closeScreenSharingConnection(userId : string) { + try { + mediaManager.removeActiveScreenSharingVideo(userId); + const peer = this.PeerScreenSharingConnectionArray.get(userId); + if (peer === undefined) { + console.warn("Tried to close connection for user "+userId+" but could not find user") + return; + } + // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" + // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. + //console.log('Closing connection with '+userId); + peer.destroy(); + this.PeerScreenSharingConnectionArray.delete(userId) + //console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size); + } catch (err) { + console.error("closeConnection", err) + } + } + public closeAllConnections() { for (const userId of this.PeerConnectionArray.keys()) { this.closeConnection(userId); } + + for (const userId of this.PeerScreenSharingConnectionArray.keys()) { + this.closeScreenSharingConnection(userId); + } } /** * Unregisters any held event handler. */ public unregister() { - mediaManager.removeUpdateLocalStreamEventListener(this.updateLocalStreamCallback); - } - - /** - * - * @param userId - * @param data - */ - private sendWebrtcSignal(data: unknown, userId : string) { - try { - this.Connection.sendWebrtcSignal(data, this.WebRtcRoomId, null, userId); - }catch (e) { - console.error(`sendWebrtcSignal => ${userId}`, e); - } + mediaManager.removeUpdateLocalStreamEventListener(this.sendLocalVideoStreamCallback); } // eslint-disable-next-line @typescript-eslint/no-explicit-any - private receiveWebrtcSignal(data: WebRtcSignalMessageInterface) { + private receiveWebrtcSignal(data: WebRtcSignalReceivedMessageInterface) { try { //if offer type, create peer connection if(data.signal.type === "offer"){ @@ -274,53 +266,126 @@ export class SimplePeer { } } - /** - * - * @param userId - * @param stream - */ - private stream(userId : string, stream: MediaStream) { - if(!stream){ - mediaManager.disabledVideoByUserId(userId); - mediaManager.disabledMicrophoneByUserId(userId); - return; + private receiveWebrtcScreenSharingSignal(data: WebRtcSignalReceivedMessageInterface) { + console.log("receiveWebrtcScreenSharingSignal", data); + try { + //if offer type, create peer connection + if(data.signal.type === "offer"){ + this.createPeerScreenSharingConnection(data); + } + const peer = this.PeerScreenSharingConnectionArray.get(data.userId); + if (peer !== undefined) { + peer.signal(data.signal); + } else { + console.error('Could not find peer whose ID is "'+data.userId+'" in receiveWebrtcScreenSharingSignal'); + } + } catch (e) { + console.error(`receiveWebrtcSignal => ${data.userId}`, e); } - mediaManager.addStreamRemoteVideo(userId, stream); } /** * * @param userId */ - private addMedia (userId : string) { + private pushVideoToRemoteUser(userId : string) { try { - const localStream: MediaStream|null = mediaManager.localStream; - const peer = this.PeerConnectionArray.get(userId); - if(localStream === null) { - //send fake signal - if(peer === undefined){ - return; - } - peer.write(new Buffer(JSON.stringify({ - type: "stream", - stream: null - }))); + const PeerConnection = this.PeerConnectionArray.get(userId); + if (!PeerConnection) { + throw new Error('While adding media, cannot find user with ID ' + userId); + } + const localStream: MediaStream | null = mediaManager.localStream; + PeerConnection.write(new Buffer(JSON.stringify(mediaManager.constraintsMedia))); + + if(!localStream){ return; } - if (peer === undefined) { - throw new Error('While adding media, cannot find user with ID '+userId); - } + for (const track of localStream.getTracks()) { - peer.addTrack(track, localStream); + PeerConnection.addTrack(track, localStream); } }catch (e) { - console.error(`addMedia => addMedia => ${userId}`, e); + console.error(`pushVideoToRemoteUser => ${userId}`, e); } } - updatedLocalStream(){ - this.Users.forEach((user: UserSimplePeer) => { - this.addMedia(user.userId); + private pushScreenSharingToRemoteUser(userId : string) { + const PeerConnection = this.PeerScreenSharingConnectionArray.get(userId); + if (!PeerConnection) { + throw new Error('While pushing screen sharing, cannot find user with ID ' + userId); + } + const localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; + if(!localScreenCapture){ + return; + } + + for (const track of localScreenCapture.getTracks()) { + PeerConnection.addTrack(track, localScreenCapture); + } + return; + } + + public sendLocalVideoStream(){ + this.Users.forEach((user: UserSimplePeerInterface) => { + this.pushVideoToRemoteUser(user.userId); }) } + + /** + * Triggered locally when clicking on the screen sharing button + */ + public sendLocalScreenSharingStream() { + if (!mediaManager.localScreenCapture) { + console.error('Could not find localScreenCapture to share') + return; + } + + for (const user of this.Users) { + this.sendLocalScreenSharingStreamToUser(user.userId); + } + } + + /** + * Triggered locally when clicking on the screen sharing button + */ + public stopLocalScreenSharingStream(stream: MediaStream) { + for (const user of this.Users) { + this.stopLocalScreenSharingStreamToUser(user.userId, stream); + } + } + + private sendLocalScreenSharingStreamToUser(userId: string): void { + // If a connection already exists with user (because it is already sharing a screen with us... let's use this connection) + if (this.PeerScreenSharingConnectionArray.has(userId)) { + this.pushScreenSharingToRemoteUser(userId); + return; + } + + const screenSharingUser: UserSimplePeerInterface = { + userId, + initiator: true + }; + const PeerConnectionScreenSharing = this.createPeerScreenSharingConnection(screenSharingUser); + if (!PeerConnectionScreenSharing) { + return; + } + } + + private stopLocalScreenSharingStreamToUser(userId: string, stream: MediaStream): void { + const PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(userId); + if (!PeerConnectionScreenSharing) { + throw new Error('Weird, screen sharing connection to user ' + userId + 'not found') + } + + console.log("updatedScreenSharing => destroy", PeerConnectionScreenSharing); + + // Stop sending stream and close peer connection if peer is not sending stream too + PeerConnectionScreenSharing.stopPushingScreenSharingToRemoteUser(stream); + + if (!PeerConnectionScreenSharing.isReceivingScreenSharingStream()) { + PeerConnectionScreenSharing.destroy(); + + this.PeerScreenSharingConnectionArray.delete(userId); + } + } } diff --git a/front/src/WebRtc/VideoPeer.ts b/front/src/WebRtc/VideoPeer.ts new file mode 100644 index 00000000..ec7f2576 --- /dev/null +++ b/front/src/WebRtc/VideoPeer.ts @@ -0,0 +1,128 @@ +import * as SimplePeerNamespace from "simple-peer"; +import {mediaManager} from "./MediaManager"; +import {Connection} from "../Connection"; + +const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer'); + +/** + * A peer connection used to transmit video / audio signals between 2 peers. + */ +export class VideoPeer extends Peer { + constructor(private userId: string, initiator: boolean, private connection: Connection) { + super({ + initiator: initiator ? initiator : false, + reconnectTimer: 10000, + config: { + iceServers: [ + { + urls: 'stun:stun.l.google.com:19302' + }, + { + urls: 'turn:numb.viagenie.ca', + username: 'g.parant@thecodingmachine.com', + credential: 'itcugcOHxle9Acqi$' + }, + ] + } + }); + + //start listen signal for the peer connection + this.on('signal', (data: unknown) => { + this.sendWebrtcSignal(data); + }); + + this.on('stream', (stream: MediaStream) => { + this.stream(stream); + }); + + /*peer.on('track', (track: MediaStreamTrack, stream: MediaStream) => { + });*/ + + this.on('close', () => { + this.destroy(); + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + this.on('error', (err: any) => { + console.error(`error => ${this.userId} => ${err.code}`, err); + mediaManager.isError(userId); + }); + + this.on('connect', () => { + mediaManager.isConnected(this.userId); + console.info(`connect => ${this.userId}`); + }); + + this.on('data', (chunk: Buffer) => { + const constraint = JSON.parse(chunk.toString('utf8')); + console.log("data", constraint); + if (constraint.audio) { + mediaManager.enabledMicrophoneByUserId(this.userId); + } else { + mediaManager.disabledMicrophoneByUserId(this.userId); + } + + if (constraint.video || constraint.screen) { + mediaManager.enabledVideoByUserId(this.userId); + } else { + this.stream(undefined); + mediaManager.disabledVideoByUserId(this.userId); + } + }); + + this.pushVideoToRemoteUser(); + } + + private sendWebrtcSignal(data: unknown) { + try { + this.connection.sendWebrtcSignal(data, this.userId); + }catch (e) { + console.error(`sendWebrtcSignal => ${this.userId}`, e); + } + } + + /** + * Sends received stream to screen. + */ + private stream(stream?: MediaStream) { + console.log(`VideoPeer::stream => ${this.userId}`, stream); + if(!stream){ + mediaManager.disabledVideoByUserId(this.userId); + mediaManager.disabledMicrophoneByUserId(this.userId); + } else { + mediaManager.addStreamRemoteVideo(this.userId, stream); + } + } + + /** + * This is triggered twice. Once by the server, and once by a remote client disconnecting + */ + public destroy(error?: Error): void { + try { + mediaManager.removeActiveVideo(this.userId); + // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" + // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. + //console.log('Closing connection with '+userId); + super.destroy(error); + } catch (err) { + console.error("VideoPeer::destroy", err) + } + } + + private pushVideoToRemoteUser() { + try { + const localStream: MediaStream | null = mediaManager.localStream; + this.write(new Buffer(JSON.stringify(mediaManager.constraintsMedia))); + + if(!localStream){ + return; + } + + for (const track of localStream.getTracks()) { + this.addTrack(track, localStream); + } + }catch (e) { + console.error(`pushVideoToRemoteUser => ${this.userId}`, e); + } + } +}