lbehm/workadventure
1
0
Fork 0
Code Issues Pull requests Releases Activity

Merge pull request #488 from thecodingmachine/apiscaling

Browse source 
Add scaling capability to the API servers with this one weird trick
This commit is contained in:
David Négrier 2020-12-16 10:54:12 +01:00 committed by GitHub
parent 16f7c3d204 49c7927e9f
commit 95a42197a6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 77 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
back/server.ts
View file

@ -3,11 +3,13 @@ import App from "./src/App";
import grpc from "grpc";
import {roomManager} from "./src/RoomManager";
import {IRoomManagerServer, RoomManagerService} from "./src/Messages/generated/messages_grpc_pb";
import {HTTP_PORT, GRPC_PORT} from "./src/Enum/EnvironmentVariable";
App.listen(8080, () => console.log(`WorkAdventure starting on port 8080!`))
App.listen(HTTP_PORT, () => console.log(`WorkAdventure HTTP API starting on port %d!`, HTTP_PORT))
const server = new grpc.Server();
server.addService<IRoomManagerServer>(RoomManagerService, roomManager);
server.bind('0.0.0.0:50051', grpc.ServerCredentials.createInsecure());
server.bind('0.0.0.0:'+GRPC_PORT, grpc.ServerCredentials.createInsecure());
server.start();
console.log('WorkAdventure HTTP/2 API starting on port %d!', GRPC_PORT);

6
back/src/Enum/EnvironmentVariable.ts
View file

@ -10,7 +10,9 @@ const CPU_OVERHEAT_THRESHOLD = Number(process.env.CPU_OVERHEAT_THRESHOLD) || 80;
const JITSI_URL : string|undefined = (process.env.JITSI_URL === '') ? undefined : process.env.JITSI_URL;
const JITSI_ISS = process.env.JITSI_ISS || '';
const SECRET_JITSI_KEY = process.env.SECRET_JITSI_KEY || '';
export const SOCKET_IDLE_TIMER = parseInt(process.env.SOCKET_IDLE_TIMER as string) || 30; // maximum time (in second) without activity before a socket is closed
const HTTP_PORT = parseInt(process.env.HTTP_PORT || '8080') || 8080;
const GRPC_PORT = parseInt(process.env.GRPC_PORT || '50051') || 50051;
export const SOCKET_IDLE_TIMER = parseInt(process.env.SOCKET_IDLE_TIMER as string) || 30; // maximum time (in second) without activity before a socket is closed
export {
SECRET_KEY,
@ -18,6 +20,8 @@ export {
MINIMUM_DISTANCE,
ADMIN_API_URL,
ADMIN_API_TOKEN,
HTTP_PORT,
GRPC_PORT,
MAX_USERS_PER_ROOM,
GROUP_RADIUS,
ALLOW_ARTILLERY,

54
deeployer.libsonnet
View file

@ -7,10 +7,10 @@
"$schema": "https://raw.githubusercontent.com/thecodingmachine/deeployer/master/deeployer.schema.json",
"version": "1.0",
"containers": {
"back": {
"back1": {
"image": "thecodingmachine/workadventure-back:"+tag,
"host": {
"url": "api."+url,
"url": "api1."+url,
"https": "enable",
"containerPort": 8080
},
@ -25,6 +25,24 @@
"ADMIN_API_URL": adminUrl,
} else {}
},
"back2": {
"image": "thecodingmachine/workadventure-back:"+tag,
"host": {
"url": "api2."+url,
"https": "enable",
"containerPort": 8080
},
"ports": [8080, 50051],
"env": {
"SECRET_KEY": "tempSecretKeyNeedsToChange",
"ADMIN_API_TOKEN": env.ADMIN_API_TOKEN,
"JITSI_ISS": env.JITSI_ISS,
"JITSI_URL": env.JITSI_URL,
"SECRET_JITSI_KEY": env.SECRET_JITSI_KEY,
} + if adminUrl != null then {
"ADMIN_API_URL": adminUrl,
} else {}
},
"pusher": {
"replicas": 2,
"image": "thecodingmachine/workadventure-pusher:"+tag,
@ -38,7 +56,7 @@
"ADMIN_API_TOKEN": env.ADMIN_API_TOKEN,
"JITSI_ISS": env.JITSI_ISS,
"JITSI_URL": env.JITSI_URL,
"API_URL": "back:50051",
"API_URL": "back1:50051,back2:50051",
"SECRET_JITSI_KEY": env.SECRET_JITSI_KEY,
} + if adminUrl != null then {
"ADMIN_API_URL": adminUrl,
@ -100,7 +118,35 @@
},
k8sextension(k8sConf)::
k8sConf + {
back+: {
back1+: {
deployment+: {
spec+: {
template+: {
metadata+: {
annotations+: {
"prometheus.io/port": "8080",
"prometheus.io/scrape": "true"
}
}
}
}
}
},
back2+: {
deployment+: {
spec+: {
template+: {
metadata+: {
annotations+: {
"prometheus.io/port": "8080",
"prometheus.io/scrape": "true"
}
}
}
}
}
},
pusher+: {
deployment+: {
spec+: {
template+: {

23
pusher/src/Services/ApiClientRepository.ts
View file

@ -3,17 +3,30 @@
*/
import {RoomManagerClient} from "../Messages/generated/messages_grpc_pb";
import grpc from 'grpc';
import crypto from 'crypto';
import {API_URL} from "../Enum/EnvironmentVariable";
import Debug from "debug";
const debug = Debug('apiClientRespository');
class ApiClientRepository {
private roomManagerClient: RoomManagerClient|null = null;
private roomManagerClients: RoomManagerClient[] = [];
public constructor(private apiUrls: string[]) {
}
public async getClient(roomId: string): Promise<RoomManagerClient> {
if (this.roomManagerClient === null) {
this.roomManagerClient = new RoomManagerClient(API_URL, grpc.credentials.createInsecure());
const array = new Uint32Array(crypto.createHash('md5').update(roomId).digest());
const index = array[0] % this.apiUrls.length;
let client = this.roomManagerClients[index];
if (client === undefined) {
this.roomManagerClients[index] = client = new RoomManagerClient(this.apiUrls[index], grpc.credentials.createInsecure());
debug('Mapping room %s to API server %s', roomId, this.apiUrls[index])
}
return Promise.resolve(this.roomManagerClient);
return Promise.resolve(client);
}
public async getAllClients(): Promise<RoomManagerClient[]> {
@ -21,6 +34,6 @@ class ApiClientRepository {
}
}
const apiClientRepository = new ApiClientRepository();
const apiClientRepository = new ApiClientRepository(API_URL.split(','));
export { apiClientRepository };