lbehm/workadventure
1
0
Fork 0
Code Issues Pull requests Releases Activity

Migrating the video overlay in Svelte (WIP)

Browse source
This commit is contained in:
David Négrier 2021-06-11 11:29:36 +02:00
parent e6264948b1
commit e7b0f859a5
20 changed files with 630 additions and 97 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
front/src/Components/App.svelte
View file

@ -1,5 +1,5 @@
<script lang="typescript">
import {enableCameraSceneVisibilityStore, gameOverlayVisibilityStore} from "../Stores/MediaStore";
import {enableCameraSceneVisibilityStore} from "../Stores/MediaStore";
import CameraControls from "./CameraControls.svelte";
import MyCamera from "./MyCamera.svelte";
import SelectCompanionScene from "./SelectCompanion/SelectCompanionScene.svelte";
@ -21,8 +21,11 @@
import AudioPlaying from "./UI/AudioPlaying.svelte";
import {soundPlayingStore} from "../Stores/SoundPlayingStore";
import ErrorDialog from "./UI/ErrorDialog.svelte";
import VideoOverlay from "./Video/VideoOverlay.svelte";
import {gameOverlayVisibilityStore} from "../Stores/GameOverlayStoreVisibility";
export let game: Game;
</script>
<div>
@ -66,6 +69,7 @@
-->
{#if $gameOverlayVisibilityStore}
<div>
<VideoOverlay></VideoOverlay>
<MyCamera></MyCamera>
<CameraControls></CameraControls>
</div>

2
front/src/Components/EnableCamera/HorizontalSoundMeterWidget.svelte
View file

@ -58,7 +58,7 @@
<div class="horizontal-sound-meter" class:active={display}>
{#each [...Array(NB_BARS).keys()] as i}
{#each [...Array(NB_BARS).keys()] as i (i)}
<div style={color(i, volume)}></div>
{/each}
</div>

10
front/src/Components/SoundMeterWidget.svelte
View file

@ -23,7 +23,7 @@
timeout = setInterval(() => {
try{
volume = parseInt((soundMeter.getVolume() / 100 * NB_BARS).toFixed(0));
volume = soundMeter.getVolume();
//console.log(volume);
}catch(err){
@ -45,9 +45,9 @@
<div class="sound-progress" class:active={display}>
<span class:active={volume > 1}></span>
<span class:active={volume > 2}></span>
<span class:active={volume > 3}></span>
<span class:active={volume > 4}></span>
<span class:active={volume > 5}></span>
<span class:active={volume > 10}></span>
<span class:active={volume > 15}></span>
<span class:active={volume > 40}></span>
<span class:active={volume > 70}></span>
</div>

20
front/src/Components/Video/LocalStreamMedia.svelte Normal file
View file

@ -0,0 +1,20 @@
<script lang="typescript">
function srcObject(node, stream) {
node.srcObject = stream;
return {
update(newStream) {
if (node.srcObject != newStream) {
node.srcObject = newStream
}
}
}
}
export let stream : MediaStream|undefined;
export let cssClass : string|undefined;
</script>
<div class="video-container {cssClass}" class:hide={!stream}>
<video class="myCamVideo" use:srcObject={stream} autoplay muted playsinline></video>
</div>

20
front/src/Components/Video/Peer.svelte Normal file
View file

@ -0,0 +1,20 @@
<script lang="ts">
import {VideoPeer} from "../../WebRtc/VideoPeer";
import VideoMedia from "./VideoMedia.svelte";
import ScreenSharingMedia from "./ScreenSharingMedia.svelte";
import {ScreenSharingPeer} from "../../WebRtc/ScreenSharingPeer";
import LocalStreamMedia from "./LocalStreamMedia.svelte";
import {DisplayableMedia} from "../../Stores/LayoutStore";
export let peer: DisplayableMedia;
</script>
<div class="media-container">
{#if peer instanceof VideoPeer}
<VideoMedia peer={peer}/>
{:else if peer instanceof ScreenSharingPeer}
<ScreenSharingMedia peer={peer}/>
{:else}
<LocalStreamMedia stream={peer.stream}/>
{/if}
</div>

57
front/src/Components/Video/ScreenSharingMedia.svelte Normal file
View file

@ -0,0 +1,57 @@
<script lang="ts">
import {ScreenSharingPeer} from "../../WebRtc/ScreenSharingPeer";
export let peer: ScreenSharingPeer;
let streamStore = peer.streamStore;
let name = peer.userName;
let statusStore = peer.statusStore;
function srcObject(node, stream) {
node.srcObject = stream;
return {
update(newStream) {
if (node.srcObject != newStream) {
node.srcObject = newStream
}
}
}
}
function getColorByString(str: string) : string|null {
let hash = 0;
if (str.length === 0) {
return null;
}
for (let i = 0; i < str.length; i++) {
hash = str.charCodeAt(i) + ((hash << 5) - hash);
hash = hash & hash;
}
let color = '#';
for (let i = 0; i < 3; i++) {
const value = (hash >> (i * 8)) & 255;
color += ('00' + value.toString(16)).substr(-2);
}
return color;
}
</script>
<div class="video-container">
{#if $statusStore === 'connecting'}
<div class="connecting-spinner"></div>
{/if}
{#if $statusStore === 'error'}
<div class="rtc-error"></div>
{/if}
{#if $streamStore === null}
<i style="background-color: {getColorByString(name)};">{name}</i>
{/if}
<video use:srcObject={$streamStore} autoplay playsinline></video>
</div>
<style lang="scss">
.video-container {
video {
width: 100%;
}
}
</style>

75
front/src/Components/Video/VideoMedia.svelte Normal file
View file

@ -0,0 +1,75 @@
<script lang="ts">
import {VideoPeer} from "../../WebRtc/VideoPeer";
import SoundMeterWidget from "../SoundMeterWidget.svelte";
import microphoneCloseImg from "../images/microphone-close.svg";
import reportImg from "./images/report.svg";
import blockSignImg from "./images/blockSign.svg";
export let peer: VideoPeer;
let streamStore = peer.streamStore;
let name = peer.userName;
let statusStore = peer.statusStore;
let constraintStore = peer.constraintsStore;
constraintStore.subscribe((vl) => console.log('CONS', vl));
function getColorByString(str: string) : string|null {
let hash = 0;
if (str.length === 0) {
return null;
}
for (let i = 0; i < str.length; i++) {
hash = str.charCodeAt(i) + ((hash << 5) - hash);
hash = hash & hash;
}
let color = '#';
for (let i = 0; i < 3; i++) {
const value = (hash >> (i * 8)) & 255;
color += ('00' + value.toString(16)).substr(-2);
}
return color;
}
function srcObject(node, stream) {
node.srcObject = stream;
return {
update(newStream) {
if (node.srcObject != newStream) {
node.srcObject = newStream
}
}
}
}
</script>
<div class="video-container">
{#if $statusStore === 'connecting'}
<div class="connecting-spinner"></div>
{/if}
{#if $statusStore === 'error'}
<div class="rtc-error"></div>
{/if}
{#if !$constraintStore || $constraintStore.video === false}
<i style="background-color: {getColorByString(name)};">{name}</i>
{/if}
{#if $constraintStore && $constraintStore.audio === false}
<img src={microphoneCloseImg} alt="Muted">
{/if}
<button class="report">
<img alt="Report this user" src={reportImg}>
<span>Report/Block</span>
</button>
<video use:srcObject={$streamStore} autoplay playsinline></video>
<img src={blockSignImg} class="block-logo" alt="Block">
{#if $constraintStore && $constraintStore.audio !== false}
<SoundMeterWidget stream={$streamStore}></SoundMeterWidget>
{/if}
</div>
<style lang="scss">
.video-container {
video {
width: 100%;
}
}
</style>

32
front/src/Components/Video/VideoOverlay.svelte Normal file
View file

@ -0,0 +1,32 @@
<script lang="ts">
import {screenSharingStreamStore} from "../../Stores/PeerStore";
import {DivImportance} from "../../WebRtc/LayoutManager";
import Peer from "./Peer.svelte";
import {layoutStore} from "../../Stores/LayoutStore";
</script>
<div class="video-overlay">
<div class="main-section">
{#each [...$layoutStore.get(DivImportance.Important).values()] as peer (peer.uniqueId)}
<Peer peer={peer}></Peer>
{/each}
</div>
<aside class="sidebar">
{#each [...$layoutStore.get(DivImportance.Normal).values()] as peer (peer.uniqueId)}
<Peer peer={peer}></Peer>
{/each}
</aside>
<div class="chat-mode three-col" style="display: none;">
</div>
</div>
<style lang="scss">
.video-overlay {
display: flex;
width: 100%;
height: 100%;
}
</style>

22
front/src/Components/Video/images/blockSign.svg Normal file
View file

@ -0,0 +1,22 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:svg="http://www.w3.org/2000/svg" xmlns="http://www.w3.org/2000/svg" xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" id="svg2985" version="1.1" inkscape:version="0.48.4 r9939" width="485.33627" height="485.33627" sodipodi:docname="600px-France_road_sign_B1j.svg[1].png">
<metadata id="metadata2991">
<rdf:RDF>
<cc:Work rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>
<dc:title/>
</cc:Work>
</rdf:RDF>
</metadata>
<defs id="defs2989"/>
<sodipodi:namedview pagecolor="#ffffff" bordercolor="#666666" borderopacity="1" objecttolerance="10" gridtolerance="10" guidetolerance="10" inkscape:pageopacity="0" inkscape:pageshadow="2" inkscape:window-width="1272" inkscape:window-height="745" id="namedview2987" showgrid="false" inkscape:snap-global="true" inkscape:snap-grids="true" inkscape:snap-bbox="true" inkscape:bbox-paths="true" inkscape:bbox-nodes="true" inkscape:snap-bbox-edge-midpoints="true" inkscape:snap-bbox-midpoints="true" inkscape:object-paths="true" inkscape:snap-intersection-paths="true" inkscape:object-nodes="true" inkscape:snap-smooth-nodes="true" inkscape:snap-midpoints="true" inkscape:snap-object-midpoints="true" inkscape:snap-center="false" fit-margin-top="0" fit-margin-left="0" fit-margin-right="0" fit-margin-bottom="0" inkscape:zoom="0.59970176" inkscape:cx="390.56499" inkscape:cy="244.34365" inkscape:window-x="86" inkscape:window-y="-8" inkscape:window-maximized="1" inkscape:current-layer="layer1">
<inkscape:grid type="xygrid" id="grid2995" empspacing="5" visible="true" enabled="true" snapvisiblegridlinesonly="true" originx="-57.33186px" originy="-57.33186px"/>
</sodipodi:namedview>
<g inkscape:groupmode="layer" id="layer1" inkscape:label="1" style="display:inline" transform="translate(-57.33186,-57.33186)">
<path sodipodi:type="arc" style="color:#000000;fill:#ffffff;fill-opacity:1;stroke:#000000;stroke-width:2.5;marker:none;visibility:visible;display:inline;overflow:visible;enable-background:accumulate" id="path2997" sodipodi:cx="300" sodipodi:cy="300" sodipodi:rx="240" sodipodi:ry="240" d="M 540,300 C 540,432.54834 432.54834,540 300,540 167.45166,540 60,432.54834 60,300 60,167.45166 167.45166,60 300,60 432.54834,60 540,167.45166 540,300 z" transform="matrix(1.0058783,0,0,1.0058783,-1.76349,-1.76349)"/>
<path sodipodi:type="arc" style="color:#000000;fill:#ff0000;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:2.5;marker:none;visibility:visible;display:inline;overflow:visible;enable-background:accumulate" id="path4005" sodipodi:cx="304.75" sodipodi:cy="214.75" sodipodi:rx="44.75" sodipodi:ry="44.75" d="m 349.5,214.75 c 0,24.71474 -20.03526,44.75 -44.75,44.75 -24.71474,0 -44.75,-20.03526 -44.75,-44.75 0,-24.71474 20.03526,-44.75 44.75,-44.75 24.71474,0 44.75,20.03526 44.75,44.75 z" transform="matrix(5.1364411,0,0,5.1364411,-1265.3304,-803.05073)"/>
<rect style="color:#000000;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:2.5;marker:none;visibility:visible;display:inline;overflow:visible;enable-background:accumulate" id="rect4001" width="345" height="80.599998" x="127.5" y="259.70001"/>
</g>
</svg>
Side by side

After

Width:  |  Height:  |  Size: 3.5 KiB

1
front/src/Components/Video/images/report.svg Normal file
View file

File diff suppressed because one or more lines are too long
Side by side

After

Width:  |  Height:  |  Size: 6.1 KiB

7
front/src/Phaser/Game/GameScene.ts
View file

@ -30,7 +30,7 @@ import {PlayerMovement} from "./PlayerMovement";
import {PlayersPositionInterpolator} from "./PlayersPositionInterpolator";
import {RemotePlayer} from "../Entity/RemotePlayer";
import {Queue} from 'queue-typescript';
import {SimplePeer, UserSimplePeerInterface} from "../../WebRtc/SimplePeer";
import {SimplePeer} from "../../WebRtc/SimplePeer";
import {ReconnectingSceneName} from "../Reconnecting/ReconnectingScene";
import {lazyLoadPlayerCharacterTextures, loadCustomTexture} from "../Entity/PlayerTexturesLoadingManager";
import {
@ -93,7 +93,7 @@ import {PinchManager} from "../UserInput/PinchManager";
import {joystickBaseImg, joystickBaseKey, joystickThumbImg, joystickThumbKey} from "../Components/MobileJoystick";
import {DEPTH_OVERLAY_INDEX} from "./DepthIndexes";
import {waScaleManager} from "../Services/WaScaleManager";
import {peerStore} from "../../Stores/PeerStore";
import {peerStore, screenSharingPeerStore} from "../../Stores/PeerStore";
import {EmoteManager} from "./EmoteManager";
export interface GameSceneInitInterface {
@ -646,12 +646,13 @@ export class GameScene extends DirtyScene implements CenterListener {
// When connection is performed, let's connect SimplePeer
this.simplePeer = new SimplePeer(this.connection, !this.room.isPublic, this.playerName);
peerStore.connectToSimplePeer(this.simplePeer);
screenSharingPeerStore.connectToSimplePeer(this.simplePeer);
this.GlobalMessageManager = new GlobalMessageManager(this.connection);
userMessageManager.setReceiveBanListener(this.bannedUser.bind(this));
const self = this;
this.simplePeer.registerPeerConnectionListener({
onConnect(user: UserSimplePeerInterface) {
onConnect(peer) {
self.presentationModeSprite.setVisible(true);
self.chatModeSprite.setVisible(true);
self.openChatIcon.setVisible(true);

17
front/src/Stores/GameOverlayStoreVisibility.ts Normal file
View file

@ -0,0 +1,17 @@
import {writable} from "svelte/store";
/**
* A store that contains whether the game overlay is shown or not.
* Typically, the overlay is hidden when entering Jitsi meet.
*/
function createGameOverlayVisibilityStore() {
const { subscribe, set, update } = writable(false);
return {
subscribe,
showGameOverlay: () => set(true),
hideGameOverlay: () => set(false),
};
}
export const gameOverlayVisibilityStore = createGameOverlayVisibilityStore();

58
front/src/Stores/LayoutStore.ts Normal file
View file

@ -0,0 +1,58 @@
import {derived, get} from "svelte/store";
import {ScreenSharingLocalMedia, screenSharingLocalMedia} from "./ScreenSharingStore";
import {DivImportance} from "../WebRtc/LayoutManager";
import { peerStore, screenSharingStreamStore} from "./PeerStore";
import type {RemotePeer} from "../WebRtc/SimplePeer";
export type DisplayableMedia = RemotePeer | ScreenSharingLocalMedia;
/**
* A store that contains the layout of the streams
*/
function createLayoutStore() {
let unsubscribes: (()=>void)[] = [];
return derived([
screenSharingStreamStore,
peerStore,
screenSharingLocalMedia,
], ([
$screenSharingStreamStore,
$peerStore,
$screenSharingLocalMedia,
], set) => {
for (const unsubscribe of unsubscribes) {
unsubscribe();
}
unsubscribes = [];
const peers = new Map<DivImportance, Map<string, DisplayableMedia>>();
peers.set(DivImportance.Normal, new Map<string, DisplayableMedia>());
peers.set(DivImportance.Important, new Map<string, DisplayableMedia>());
const addPeer = (peer: DisplayableMedia) => {
const importance = get(peer.importanceStore);
peers.get(importance)?.set(peer.uniqueId, peer);
unsubscribes.push(peer.importanceStore.subscribe((importance) => {
peers.forEach((category) => {
category.delete(peer.uniqueId);
});
peers.get(importance)?.set(peer.uniqueId, peer);
set(peers);
}));
};
$screenSharingStreamStore.forEach(addPeer);
$peerStore.forEach(addPeer);
if ($screenSharingLocalMedia?.stream) {
addPeer($screenSharingLocalMedia);
}
set(peers);
});
}
export const layoutStore = createLayoutStore();

17
front/src/Stores/MediaStore.ts
View file

@ -1,13 +1,13 @@
import {derived, get, Readable, readable, writable, Writable} from "svelte/store";
import {peerStore} from "./PeerStore";
import {localUserStore} from "../Connexion/LocalUserStore";
import {ITiledMapGroupLayer, ITiledMapObjectLayer, ITiledMapTileLayer} from "../Phaser/Map/ITiledMap";
import {userMovingStore} from "./GameStore";
import {HtmlUtils} from "../WebRtc/HtmlUtils";
import {BrowserTooOldError} from "./Errors/BrowserTooOldError";
import {errorStore} from "./ErrorStore";
import {isIOS} from "../WebRtc/DeviceUtils";
import {WebviewOnOldIOS} from "./Errors/WebviewOnOldIOS";
import {gameOverlayVisibilityStore} from "./GameOverlayStoreVisibility";
/**
* A store that contains the camera state requested by the user (on or off).
@ -50,20 +50,6 @@ export const visibilityStore = readable(document.visibilityState === 'visible',
};
});
/**
* A store that contains whether the game overlay is shown or not.
* Typically, the overlay is hidden when entering Jitsi meet.
*/
function createGameOverlayVisibilityStore() {
const { subscribe, set, update } = writable(false);
return {
subscribe,
showGameOverlay: () => set(true),
hideGameOverlay: () => set(false),
};
}
/**
* A store that contains whether the EnableCameraScene is shown or not.
*/
@ -79,7 +65,6 @@ function createEnableCameraSceneVisibilityStore() {
export const requestedCameraState = createRequestedCameraState();
export const requestedMicrophoneState = createRequestedMicrophoneState();
export const gameOverlayVisibilityStore = createGameOverlayVisibilityStore();
export const enableCameraSceneVisibilityStore = createEnableCameraSceneVisibilityStore();
/**

111
front/src/Stores/PeerStore.ts
View file

@ -1,26 +1,64 @@
import { derived, writable, Writable } from "svelte/store";
import type {UserSimplePeerInterface} from "../WebRtc/SimplePeer";
import type {SimplePeer} from "../WebRtc/SimplePeer";
import {derived, get, readable, writable} from "svelte/store";
import type {RemotePeer, SimplePeer} from "../WebRtc/SimplePeer";
import {VideoPeer} from "../WebRtc/VideoPeer";
import {ScreenSharingPeer} from "../WebRtc/ScreenSharingPeer";
/**
* A store that contains the camera state requested by the user (on or off).
* A store that contains the list of (video) peers we are connected to.
*/
function createPeerStore() {
let users = new Map<number, UserSimplePeerInterface>();
let peers = new Map<number, VideoPeer>();
const { subscribe, set, update } = writable(users);
const { subscribe, set, update } = writable(peers);
return {
subscribe,
connectToSimplePeer: (simplePeer: SimplePeer) => {
users = new Map<number, UserSimplePeerInterface>();
set(users);
peers = new Map<number, VideoPeer>();
set(peers);
simplePeer.registerPeerConnectionListener({
onConnect(user: UserSimplePeerInterface) {
onConnect(peer: RemotePeer) {
if (peer instanceof VideoPeer) {
update(users => {
users.set(peer.userId, peer);
return users;
});
}
console.log('CONNECT VIDEO', peers);
},
onDisconnect(userId: number) {
update(users => {
users.set(user.userId, user);
users.delete(userId);
return users;
});
console.log('DISCONNECT VIDEO', peers);
}
})
}
};
}
/**
* A store that contains the list of screen sharing peers we are connected to.
*/
function createScreenSharingPeerStore() {
let peers = new Map<number, ScreenSharingPeer>();
const { subscribe, set, update } = writable(peers);
return {
subscribe,
connectToSimplePeer: (simplePeer: SimplePeer) => {
peers = new Map<number, ScreenSharingPeer>();
set(peers);
simplePeer.registerPeerConnectionListener({
onConnect(peer: RemotePeer) {
if (peer instanceof ScreenSharingPeer) {
update(users => {
users.set(peer.userId, peer);
return users;
});
}
},
onDisconnect(userId: number) {
update(users => {
@ -34,3 +72,56 @@ function createPeerStore() {
}
export const peerStore = createPeerStore();
export const screenSharingPeerStore = createScreenSharingPeerStore();
/**
* A store that contains ScreenSharingPeer, ONLY if those ScreenSharingPeer are emitting a stream towards us!
*/
function createScreenSharingStreamStore() {
let peers = new Map<number, ScreenSharingPeer>();
return readable<Map<number, ScreenSharingPeer>>(peers, function start(set) {
let unsubscribes: (()=>void)[] = [];
const unsubscribe = screenSharingPeerStore.subscribe((screenSharingPeers) => {
for (const unsubscribe of unsubscribes) {
unsubscribe();
}
unsubscribes = [];
peers = new Map<number, ScreenSharingPeer>();
screenSharingPeers.forEach((screenSharingPeer: ScreenSharingPeer, key: number) => {
if (screenSharingPeer.isReceivingScreenSharingStream()) {
peers.set(key, screenSharingPeer);
}
unsubscribes.push(screenSharingPeer.streamStore.subscribe((stream) => {
if (stream) {
peers.set(key, screenSharingPeer);
} else {
peers.delete(key);
}
set(peers);
}));
});
set(peers);
});
return function stop() {
unsubscribe();
for (const unsubscribe of unsubscribes) {
unsubscribe();
}
};
})
}
export const screenSharingStreamStore = createScreenSharingStreamStore();

46
front/src/Stores/ScreenSharingStore.ts
View file

@ -1,16 +1,10 @@
import {derived, get, Readable, readable, writable, Writable} from "svelte/store";
import {peerStore} from "./PeerStore";
import {localUserStore} from "../Connexion/LocalUserStore";
import {ITiledMapGroupLayer, ITiledMapObjectLayer, ITiledMapTileLayer} from "../Phaser/Map/ITiledMap";
import {userMovingStore} from "./GameStore";
import {HtmlUtils} from "../WebRtc/HtmlUtils";
import {
audioConstraintStore, cameraEnergySavingStore,
enableCameraSceneVisibilityStore,
gameOverlayVisibilityStore, LocalStreamStoreValue, privacyShutdownStore,
requestedCameraState,
requestedMicrophoneState, videoConstraintStore
import type {
LocalStreamStoreValue,
} from "./MediaStore";
import {DivImportance} from "../WebRtc/LayoutManager";
import {gameOverlayVisibilityStore} from "./GameOverlayStoreVisibility";
declare const navigator:any; // eslint-disable-line @typescript-eslint/no-explicit-any
@ -191,3 +185,35 @@ export const screenSharingAvailableStore = derived(peerStore, ($peerStore, set)
set($peerStore.size !== 0);
});
export interface ScreenSharingLocalMedia {
uniqueId: string;
importanceStore: Writable<DivImportance>;
stream: MediaStream|null;
//subscribe(this: void, run: Subscriber<ScreenSharingLocalMedia>, invalidate?: (value?: ScreenSharingLocalMedia) => void): Unsubscriber;
}
/**
* The representation of the screen sharing stream.
*/
export const screenSharingLocalMedia = readable<ScreenSharingLocalMedia|null>(null, function start(set) {
const localMedia: ScreenSharingLocalMedia = {
uniqueId: "localScreenSharingStream",
importanceStore: writable(DivImportance.Normal),
stream: null
}
const unsubscribe = screenSharingLocalStreamStore.subscribe((screenSharingLocalStream) => {
if (screenSharingLocalStream.type === "success") {
localMedia.stream = screenSharingLocalStream.stream;
} else {
localMedia.stream = null;
}
set(localMedia);
});
return function stop() {
unsubscribe();
};
})

37
front/src/WebRtc/MediaManager.ts
View file

@ -7,7 +7,7 @@ import type {UserSimplePeerInterface} from "./SimplePeer";
import {SoundMeter} from "../Phaser/Components/SoundMeter";
import {DISABLE_NOTIFICATIONS} from "../Enum/EnvironmentVariable";
import {
gameOverlayVisibilityStore, localStreamStore,
localStreamStore,
} from "../Stores/MediaStore";
import {
screenSharingLocalStreamStore
@ -22,6 +22,7 @@ export type ShowReportCallBack = (userId: string, userName: string|undefined) =>
export type HelpCameraSettingsCallBack = () => void;
import {cowebsiteCloseButtonId} from "./CoWebsiteManager";
import {gameOverlayVisibilityStore} from "../Stores/GameOverlayStoreVisibility";
export class MediaManager {
private remoteVideo: Map<string, HTMLVideoElement> = new Map<string, HTMLVideoElement>();
@ -65,7 +66,7 @@ export class MediaManager {
}
});
let isScreenSharing = false;
//let isScreenSharing = false;
screenSharingLocalStreamStore.subscribe((result) => {
if (result.type === 'error') {
console.error(result.error);
@ -75,7 +76,7 @@ export class MediaManager {
return;
}
if (result.stream !== null) {
/*if (result.stream !== null) {
isScreenSharing = true;
this.addScreenSharingActiveVideo('me', DivImportance.Normal);
HtmlUtils.getElementByIdOrFail<HTMLVideoElement>('screen-sharing-me').srcObject = result.stream;
@ -84,7 +85,7 @@ export class MediaManager {
isScreenSharing = false;
this.removeActiveScreenSharingVideo('me');
}
}
}*/
});
@ -134,7 +135,7 @@ export class MediaManager {
gameOverlayVisibilityStore.hideGameOverlay();
}
addActiveVideo(user: UserSimplePeerInterface, userName: string = ""){
/*addActiveVideo(user: UserSimplePeerInterface, userName: string = ""){
const userId = ''+user.userId
userName = userName.toUpperCase();
@ -194,7 +195,7 @@ export class MediaManager {
layoutManager.add(divImportance, userId, html);
this.remoteVideo.set(userId, HtmlUtils.getElementByIdOrFail<HTMLVideoElement>(userId));
}
}*/
private getScreenSharingId(userId: string): string {
return `screen-sharing-${userId}`;
@ -242,19 +243,12 @@ export class MediaManager {
const blockLogoElement = HtmlUtils.getElementByIdOrFail<HTMLImageElement>('blocking-'+userId);
show ? blockLogoElement.classList.add('active') : blockLogoElement.classList.remove('active');
}
addStreamRemoteVideo(userId: string, stream : MediaStream): void {
/*addStreamRemoteVideo(userId: string, stream : MediaStream): void {
const remoteVideo = this.remoteVideo.get(userId);
if (remoteVideo === undefined) {
throw `Unable to find video for ${userId}`;
}
remoteVideo.srcObject = stream;
//FIX ME SOUNDMETER: check stalability of sound meter calculation
//sound metter
/*const soundMeter = new SoundMeter();
soundMeter.connectToSource(stream, new AudioContext());
this.soundMeters.set(userId, soundMeter);
this.soundMeterElements.set(userId, HtmlUtils.getElementByIdOrFail<HTMLImageElement>('soundMeter-'+userId));*/
}
addStreamRemoteScreenSharing(userId: string, stream : MediaStream){
// In the case of screen sharing (going both ways), we may need to create the HTML element if it does not exist yet
@ -264,23 +258,18 @@ export class MediaManager {
}
this.addStreamRemoteVideo(this.getScreenSharingId(userId), stream);
}
}*/
removeActiveVideo(userId: string){
layoutManager.remove(userId);
this.remoteVideo.delete(userId);
//FIX ME SOUNDMETER: check stalability of sound meter calculation
/*this.soundMeters.get(userId)?.stop();
this.soundMeters.delete(userId);
this.soundMeterElements.delete(userId);*/
//layoutManager.remove(userId);
//this.remoteVideo.delete(userId);
//permit to remove user in discussion part
this.removeParticipant(userId);
}
removeActiveScreenSharingVideo(userId: string) {
/*removeActiveScreenSharingVideo(userId: string) {
this.removeActiveVideo(this.getScreenSharingId(userId))
}
}*/
isConnecting(userId: string): void {
const connectingSpinnerDiv = this.getSpinner(userId);

73
front/src/WebRtc/ScreenSharingPeer.ts
View file

@ -1,9 +1,11 @@
import type * as SimplePeerNamespace from "simple-peer";
import {mediaManager} from "./MediaManager";
import {STUN_SERVER, TURN_SERVER, TURN_USER, TURN_PASSWORD} from "../Enum/EnvironmentVariable";
import {STUN_SERVER, TURN_PASSWORD, TURN_SERVER, TURN_USER} from "../Enum/EnvironmentVariable";
import type {RoomConnection} from "../Connexion/RoomConnection";
import {MESSAGE_TYPE_CONSTRAINT} from "./VideoPeer";
import type {UserSimplePeerInterface} from "./SimplePeer";
import {Readable, readable, writable, Writable} from "svelte/store";
import {DivImportance} from "./LayoutManager";
const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer');
@ -17,9 +19,13 @@ export class ScreenSharingPeer extends Peer {
private isReceivingStream:boolean = false;
public toClose: boolean = false;
public _connected: boolean = false;
private userId: number;
public readonly userId: number;
public readonly uniqueId: string;
public readonly streamStore: Readable<MediaStream | null>;
public readonly importanceStore: Writable<DivImportance>;
public readonly statusStore: Readable<"connecting" | "connected" | "error" | "closed">;
constructor(user: UserSimplePeerInterface, initiator: boolean, private connection: RoomConnection, stream: MediaStream | null) {
constructor(user: UserSimplePeerInterface, initiator: boolean, public readonly userName: string, private connection: RoomConnection, stream: MediaStream | null) {
super({
initiator: initiator ? initiator : false,
//reconnectTimer: 10000,
@ -38,6 +44,56 @@ export class ScreenSharingPeer extends Peer {
});
this.userId = user.userId;
this.uniqueId = 'screensharing_'+this.userId;
this.streamStore = readable<MediaStream|null>(null, (set) => {
const onStream = (stream: MediaStream|null) => {
set(stream);
};
const onData = (chunk: Buffer) => {
// We unfortunately need to rely on an event to let the other party know a stream has stopped.
// It seems there is no native way to detect that.
// TODO: we might rely on the "ended" event: https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/ended_event
const message = JSON.parse(chunk.toString('utf8'));
if (message.streamEnded !== true) {
console.error('Unexpected message on screen sharing peer connection');
return;
}
set(null);
}
this.on('stream', onStream);
this.on('data', onData);
return () => {
this.off('stream', onStream);
this.off('data', onData);
};
});
this.importanceStore = writable(DivImportance.Important);
this.statusStore = readable<"connecting" | "connected" | "error" | "closed">("connecting", (set) => {
const onConnect = () => {
set('connected');
};
const onError = () => {
set('error');
};
const onClose = () => {
set('closed');
};
this.on('connect', onConnect);
this.on('error', onError);
this.on('close', onClose);
return () => {
this.off('connect', onConnect);
this.off('error', onError);
this.off('close', onClose);
};
});
//start listen signal for the peer connection
this.on('signal', (data: unknown) => {
@ -54,16 +110,17 @@ export class ScreenSharingPeer extends Peer {
this.destroy();
});
this.on('data', (chunk: Buffer) => {
/*this.on('data', (chunk: Buffer) => {
// We unfortunately need to rely on an event to let the other party know a stream has stopped.
// It seems there is no native way to detect that.
// TODO: we might rely on the "ended" event: https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/ended_event
const message = JSON.parse(chunk.toString('utf8'));
if (message.streamEnded !== true) {
console.error('Unexpected message on screen sharing peer connection');
return;
}
mediaManager.removeActiveScreenSharingVideo("" + this.userId);
});
});*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
this.on('error', (err: any) => {
@ -103,10 +160,10 @@ export class ScreenSharingPeer extends Peer {
//console.log(`ScreenSharingPeer::stream => ${this.userId}`, stream);
//console.log(`stream => ${this.userId} => `, stream);
if(!stream){
mediaManager.removeActiveScreenSharingVideo("" + this.userId);
//mediaManager.removeActiveScreenSharingVideo("" + this.userId);
this.isReceivingStream = false;
} else {
mediaManager.addStreamRemoteScreenSharing("" + this.userId, stream);
//mediaManager.addStreamRemoteScreenSharing("" + this.userId, stream);
this.isReceivingStream = true;
}
}
@ -121,7 +178,7 @@ export class ScreenSharingPeer extends Peer {
if(!this.toClose){
return;
}
mediaManager.removeActiveScreenSharingVideo("" + this.userId);
//mediaManager.removeActiveScreenSharingVideo("" + this.userId);
// FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray"
// I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel.
//console.log('Closing connection with '+userId);

36
front/src/WebRtc/SimplePeer.ts
View file

@ -28,8 +28,10 @@ export interface UserSimplePeerInterface{
webRtcPassword?: string|undefined;
}
export type RemotePeer = VideoPeer | ScreenSharingPeer;
export interface PeerConnectionListener {
onConnect(user: UserSimplePeerInterface): void;
onConnect(user: RemotePeer): void;
onDisconnect(userId: number): void;
}
@ -159,20 +161,17 @@ export class SimplePeer {
let name = user.name;
if (!name) {
const userSearch = this.Users.find((userSearch: UserSimplePeerInterface) => userSearch.userId === user.userId);
if (userSearch) {
name = userSearch.name;
}
name = this.getName(user.userId);
}
mediaManager.removeActiveVideo("" + user.userId);
mediaManager.addActiveVideo(user, name);
//mediaManager.addActiveVideo(user, name);
this.lastWebrtcUserName = user.webRtcUser;
this.lastWebrtcPassword = user.webRtcPassword;
const peer = new VideoPeer(user, user.initiator ? user.initiator : false, this.Connection, localStream);
const peer = new VideoPeer(user, user.initiator ? user.initiator : false, name, this.Connection, localStream);
//permit to send message
mediaManager.addSendMessageCallback(user.userId,(message: string) => {
@ -196,11 +195,20 @@ export class SimplePeer {
this.PeerConnectionArray.set(user.userId, peer);
for (const peerConnectionListener of this.peerConnectionListeners) {
peerConnectionListener.onConnect(user);
peerConnectionListener.onConnect(peer);
}
return peer;
}
private getName(userId: number): string {
const userSearch = this.Users.find((userSearch: UserSimplePeerInterface) => userSearch.userId === userId);
if (userSearch) {
return userSearch.name || '';
} else {
return '';
}
}
/**
* create peer connection to bind users
*/
@ -222,10 +230,10 @@ export class SimplePeer {
}
// We should display the screen sharing ONLY if we are not initiator
if (!user.initiator) {
/* if (!user.initiator) {
mediaManager.removeActiveScreenSharingVideo("" + user.userId);
mediaManager.addScreenSharingActiveVideo("" + user.userId);
}
}*/
// Enrich the user with last known credentials (if they are not set in the user object, which happens when a user triggers the screen sharing)
if (user.webRtcUser === undefined) {
@ -233,11 +241,13 @@ export class SimplePeer {
user.webRtcPassword = this.lastWebrtcPassword;
}
const peer = new ScreenSharingPeer(user, user.initiator ? user.initiator : false, this.Connection, stream);
const name = this.getName(user.userId);
const peer = new ScreenSharingPeer(user, user.initiator ? user.initiator : false, name, this.Connection, stream);
this.PeerScreenSharingConnectionArray.set(user.userId, peer);
for (const peerConnectionListener of this.peerConnectionListeners) {
peerConnectionListener.onConnect(user);
peerConnectionListener.onConnect(peer);
}
return peer;
}
@ -288,7 +298,7 @@ export class SimplePeer {
*/
private closeScreenSharingConnection(userId : number) {
try {
mediaManager.removeActiveScreenSharingVideo("" + userId);
//mediaManager.removeActiveScreenSharingVideo("" + userId);
const peer = this.PeerScreenSharingConnectionArray.get(userId);
if (peer === undefined) {
console.warn("closeScreenSharingConnection => Tried to close connection for user "+userId+" but could not find user")

80
front/src/WebRtc/VideoPeer.ts
View file

@ -5,8 +5,9 @@ import type {RoomConnection} from "../Connexion/RoomConnection";
import {blackListManager} from "./BlackListManager";
import type {Subscription} from "rxjs";
import type {UserSimplePeerInterface} from "./SimplePeer";
import {get} from "svelte/store";
import {get, readable, Readable, writable, Writable} from "svelte/store";
import {obtainedMediaConstraintStore} from "../Stores/MediaStore";
import {DivImportance} from "./LayoutManager";
const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer');
@ -22,12 +23,16 @@ export class VideoPeer extends Peer {
public _connected: boolean = false;
private remoteStream!: MediaStream;
private blocked: boolean = false;
private userId: number;
private userName: string;
public readonly userId: number;
public readonly uniqueId: string;
private onBlockSubscribe: Subscription;
private onUnBlockSubscribe: Subscription;
public readonly streamStore: Readable<MediaStream | null>;
public readonly importanceStore: Writable<DivImportance>;
public readonly statusStore: Readable<"connecting" | "connected" | "error" | "closed">;
public readonly constraintsStore: Readable<MediaStreamConstraints|null>;
constructor(public user: UserSimplePeerInterface, initiator: boolean, private connection: RoomConnection, localStream: MediaStream | null) {
constructor(public user: UserSimplePeerInterface, initiator: boolean, public readonly userName: string, private connection: RoomConnection, localStream: MediaStream | null) {
super({
initiator: initiator ? initiator : false,
//reconnectTimer: 10000,
@ -46,7 +51,70 @@ export class VideoPeer extends Peer {
});
this.userId = user.userId;
this.userName = user.name || '';
this.uniqueId = 'video_'+this.userId;
this.streamStore = readable<MediaStream|null>(null, (set) => {
const onStream = (stream: MediaStream|null) => {
set(stream);
};
const onData = (chunk: Buffer) => {
this.on('data', (chunk: Buffer) => {
const message = JSON.parse(chunk.toString('utf8'));
if (message.type === MESSAGE_TYPE_CONSTRAINT) {
if (!message.video) {
set(null);
}
}
});
}
this.on('stream', onStream);
this.on('data', onData);
return () => {
this.off('stream', onStream);
this.off('data', onData);
};
});
this.constraintsStore = readable<MediaStreamConstraints|null>(null, (set) => {
const onData = (chunk: Buffer) => {
const message = JSON.parse(chunk.toString('utf8'));
if(message.type === MESSAGE_TYPE_CONSTRAINT) {
set(message);
}
}
this.on('data', onData);
return () => {
this.off('data', onData);
};
});
this.importanceStore = writable(DivImportance.Normal);
this.statusStore = readable<"connecting" | "connected" | "error" | "closed">("connecting", (set) => {
const onConnect = () => {
set('connected');
};
const onError = () => {
set('error');
};
const onClose = () => {
set('closed');
};
this.on('connect', onConnect);
this.on('error', onError);
this.on('close', onClose);
return () => {
this.off('connect', onConnect);
this.off('error', onError);
this.off('close', onClose);
};
});
//start listen signal for the peer connection
this.on('signal', (data: unknown) => {
@ -152,7 +220,7 @@ export class VideoPeer extends Peer {
if (blackListManager.isBlackListed(this.userId) || this.blocked) {
this.toggleRemoteStream(false);
}
mediaManager.addStreamRemoteVideo("" + this.userId, stream);
//mediaManager.addStreamRemoteVideo("" + this.userId, stream);
}catch (err){
console.error(err);
}