From a26f8b24d46450d13eab0580b08557b8ddbb7a07 Mon Sep 17 00:00:00 2001 From: Leonid Pospelov Date: Sat, 1 Jun 2024 06:30:29 +0500 Subject: [PATCH 1/2] fix(skymp5-server): fix additional-settings-sources being used not everywhere --- skymp5-server/ts/index.ts | 265 +--------------- skymp5-server/ts/settings.ts | 311 ++++++++++++++++--- skymp5-server/ts/systems/discordBanSystem.ts | 6 +- skymp5-server/ts/systems/login.ts | 5 +- skymp5-server/ts/systems/spawn.ts | 3 +- skymp5-server/ts/ui.ts | 3 +- 6 files changed, 301 insertions(+), 292 deletions(-) diff --git a/skymp5-server/ts/index.ts b/skymp5-server/ts/index.ts index 70667655e8..4ec7c8f2c0 100644 --- a/skymp5-server/ts/index.ts +++ b/skymp5-server/ts/index.ts @@ -26,23 +26,10 @@ import * as fs from "fs"; import * as chokidar from "chokidar"; import * as path from "path"; import * as os from "os"; -import * as crypto from "crypto"; import * as manifestGen from "./manifestGen"; import { DiscordBanSystem } from "./systems/discordBanSystem"; import { createScampServer } from "./scampNative"; -import { Octokit } from "@octokit/rest"; -import * as lodash from "lodash"; - -const { - master, - port, - maxPlayers, - name, - ip, - gamemodePath, - offlineMode, -} = Settings.get(); const gamemodeCache = new Map(); @@ -109,15 +96,6 @@ function requireUncached( } } -const log = console.log; -const systems = new Array(); -systems.push( - new MasterClient(log, port, master, maxPlayers, name, ip, 5000, offlineMode), - new Spawn(log), - new Login(log, maxPlayers, master, port, ip, offlineMode), - new DiscordBanSystem() -); - const setupStreams = (scampNative: any) => { class LogsStream { constructor(private logLevel: string) { @@ -145,239 +123,30 @@ const setupStreams = (scampNative: any) => { }; }; -/** - * Resolves a Git ref to a commit hash if it's not already a commit hash. - */ -async function resolveRefToCommitHash(octokit: Octokit, owner: string, repo: string, ref: string): Promise { - // Check if `ref` is already a 40-character hexadecimal string (commit hash). - if (/^[a-f0-9]{40}$/i.test(ref)) { - return ref; // It's already a commit hash. - } - - // Attempt to resolve the ref as both a branch and a tag. - try { - // First, try to resolve it as a branch. - return await getCommitHashFromRef(octokit, owner, repo, `heads/${ref}`); - } catch (error) { - try { - // If the branch resolution fails, try to resolve it as a tag. - return await getCommitHashFromRef(octokit, owner, repo, `tags/${ref}`); - } catch (tagError) { - throw new Error('Could not resolve ref to commit hash.'); - } - } -} - -async function getCommitHashFromRef(octokit: Octokit, owner: string, repo: string, ref: string): Promise { - const { data } = await octokit.git.getRef({ - owner, - repo, - ref, - }); - return data.object.sha; -} - -async function fetchServerSettings(): Promise { - // Load server-settings.json - const settingsPath = 'server-settings.json'; - const rawSettings = fs.readFileSync(settingsPath, 'utf8'); - let serverSettingsFile = JSON.parse(rawSettings); - - let serverSettings: Record = {}; - - const additionalServerSettings = serverSettingsFile.additionalServerSettings || []; - - let dumpFileNameSuffix = ''; - - for (let i = 0; i < additionalServerSettings.length; ++i) { - console.log(`Verifying additional server settings source ${i + 1} / ${additionalServerSettings.length}`); - - const { type, repo, ref, token, pathRegex } = serverSettingsFile.additionalServerSettings[i]; - - if (typeof type !== "string") { - throw new Error(`Expected additionalServerSettings[${i}].type to be string`); - } - - if (type !== "github") { - throw new Error(`Expected additionalServerSettings[${i}].type to be one of ["github"], but got ${type}`); - } - - if (typeof repo !== "string") { - throw new Error(`Expected additionalServerSettings[${i}].repo to be string`); - } - if (typeof ref !== "string") { - throw new Error(`Expected additionalServerSettings[${i}].ref to be string`); - } - if (typeof token !== "string") { - throw new Error(`Expected additionalServerSettings[${i}].token to be string`); - } - if (typeof pathRegex !== "string") { - throw new Error(`Expected additionalServerSettings[${i}].pathRegex to be string`); - } - - const octokit = new Octokit({ auth: token }); - - const [owner, repoName] = repo.split('/'); - - const commitHash = await resolveRefToCommitHash(octokit, owner, repoName, ref); - dumpFileNameSuffix += `-${commitHash}`; - } - - const dumpFileName = `server-settings-dump.json`; - - const readDump: Record | undefined = fs.existsSync(dumpFileName) ? JSON.parse(fs.readFileSync(dumpFileName, 'utf-8')) : undefined; - - let readDumpNoSha512 = structuredClone(readDump); - if (readDumpNoSha512) { - delete readDumpNoSha512['_sha512_']; - } - - const expectedSha512 = readDumpNoSha512 ? crypto.createHash('sha512').update(JSON.stringify(readDumpNoSha512)).digest('hex') : ''; - - if (readDump && readDump["_meta_"] === dumpFileNameSuffix && readDump["_sha512_"] === expectedSha512) { - console.log(`Loading settings dump from ${dumpFileName}`); - serverSettings = JSON.parse(fs.readFileSync(dumpFileName, 'utf-8')); - } - else { - for (let i = 0; i < additionalServerSettings.length; ++i) { - - const { repo, ref, token, pathRegex } = serverSettingsFile.additionalServerSettings[i]; - - console.log(`Fetching settings from "${repo}" at ref "${ref}" with path regex ${pathRegex}`); - - const regex = new RegExp(pathRegex); - - const octokit = new Octokit({ auth: token }); - - const [owner, repoName] = repo.split('/'); - - // List repository contents at specified ref - const rootContent = await octokit.repos.getContent({ - owner, - repo: repoName, - ref, - path: '', - }); - - const { data } = rootContent; - - const rateLimitRemainingInitial = parseInt(rootContent.headers["x-ratelimit-remaining"]) + 1; - let rateLimitRemaining = 0; - - const onFile = async (file: { path: string, name: string }) => { - if (file.name.endsWith('.json')) { - if (regex.test(file.path)) { - // Fetch individual file content if it matches the regex - const fileData = await octokit.repos.getContent({ - owner, - repo: repoName, - ref, - path: file.path, - }); - rateLimitRemaining = parseInt(fileData.headers["x-ratelimit-remaining"]); - - if ('content' in fileData.data && typeof fileData.data.content === 'string') { - // Decode Base64 content and parse JSON - const content = Buffer.from(fileData.data.content, 'base64').toString('utf-8'); - const jsonContent = JSON.parse(content); - // Merge or handle the JSON content as needed - console.log(`Merging "${file.path}"`); - - serverSettings = lodash.merge(serverSettings, jsonContent); - } - else { - throw new Error(`Expected content to be an array (${file.path})`); - } - } - else { - console.log(`Ignoring "${file.path}"`); - } - } - } - - const onDir = async (file: { path: string, name: string }) => { - const fileData = await octokit.repos.getContent({ - owner, - repo: repoName, - ref, - path: file.path, - }); - rateLimitRemaining = parseInt(fileData.headers["x-ratelimit-remaining"]); - - if (Array.isArray(fileData.data)) { - for (const item of fileData.data) { - if (item.type === "file") { - await onFile(item); - } - else if (item.type === "dir") { - await onDir(item); - } - else { - console.warn(`Skipping unsupported item type ${item.type} (${item.path})`); - } - } - } - else { - throw new Error(`Expected data to be an array (${file.path})`); - } - } - - if (Array.isArray(data)) { - for (const item of data) { - if (item.type === "file") { - await onFile(item); - } - else if (item.type === "dir") { - await onDir(item); - } - else { - console.warn(`Skipping unsupported item type ${item.type} (${item.path})`); - } - } - } - else { - throw new Error(`Expected data to be an array (root)`); - } - - console.log(`Rate limit spent: ${rateLimitRemainingInitial - rateLimitRemaining}, remaining: ${rateLimitRemaining}`); - - const xRateLimitReset = rootContent.headers["x-ratelimit-reset"]; - const resetDate = new Date(parseInt(xRateLimitReset, 10) * 1000); - const currentDate = new Date(); - if (resetDate > currentDate) { - console.log("The rate limit will reset in the future"); - const secondsUntilReset = (resetDate.getTime() - currentDate.getTime()) / 1000; - console.log(`Seconds until reset: ${secondsUntilReset}`); - } else { - console.log("The rate limit has already been reset"); - } - } - - if (JSON.stringify(serverSettings) !== JSON.stringify(JSON.parse(rawSettings))) { - console.log(`Dumping ${dumpFileName} for cache and debugging`); - serverSettings["_meta_"] = dumpFileNameSuffix; - serverSettings["_sha512_"] = crypto.createHash('sha512').update(JSON.stringify(serverSettings)).digest('hex'); - fs.writeFileSync(dumpFileName, JSON.stringify(serverSettings, null, 2)); - } - } - - console.log(`Merging "server-settings.json" (original settings file)`); - serverSettings = lodash.merge(serverSettings, serverSettingsFile); - - return serverSettings; -} - const main = async () => { + const settingsObject = await Settings.get(); + const { + port, master, maxPlayers, name, ip, offlineMode, gamemodePath + } = settingsObject; + + const log = console.log; + const systems = new Array(); + systems.push( + new MasterClient(log, port, master, maxPlayers, name, ip, 5000, offlineMode), + new Spawn(log), + new Login(log, maxPlayers, master, port, ip, offlineMode), + new DiscordBanSystem() + ); + setupStreams(scampNative.getScampNative()); - manifestGen.generateManifest(Settings.get()); - ui.main(); + manifestGen.generateManifest(settingsObject); + ui.main(settingsObject); let server: any; try { - const serverSettings = await fetchServerSettings(); - server = createScampServer(port, maxPlayers, serverSettings); + server = createScampServer(port, maxPlayers, settingsObject.allSettings); } catch (e) { console.error(e); diff --git a/skymp5-server/ts/settings.ts b/skymp5-server/ts/settings.ts index 0f0b2751aa..ef3c4cb4d3 100644 --- a/skymp5-server/ts/settings.ts +++ b/skymp5-server/ts/settings.ts @@ -1,5 +1,8 @@ -import { ArgumentParser } from 'argparse'; import * as fs from 'fs'; +import * as crypto from "crypto"; +import { Octokit } from '@octokit/rest'; +import { ArgumentParser } from 'argparse'; +import lodash from 'lodash'; export interface DiscordAuthSettings { botToken: string; @@ -28,50 +31,58 @@ export class Settings { ]; discordAuth: DiscordAuthSettings | null = null; - constructor() { + allSettings: Record | null = null; + + constructor() { } + + static async get(): Promise { + if (!Settings.cachedPromise) { + Settings.cachedPromise = (async () => { + const args = Settings.parseArgs(); + const res = new Settings(); + + await res.loadSettings(); // Load settings asynchronously + + // Override settings with command line arguments if available + res.port = +args['port'] || res.port; + res.maxPlayers = +args['maxPlayers'] || res.maxPlayers; + res.master = args['master'] || res.master; + res.name = args['name'] || res.name; + res.ip = args['ip'] || res.ip; + res.offlineMode = args['offlineMode'] || res.offlineMode; + + return res; + })(); + } + + return Settings.cachedPromise; + } + + private async loadSettings() { if (fs.existsSync('./skymp5-gamemode')) { this.gamemodePath = './skymp5-gamemode/gamemode.js'; } else { this.gamemodePath = './gamemode.js'; } - if (fs.existsSync('./server-settings.json')) { - const parsed = JSON.parse(fs.readFileSync('./server-settings.json', 'utf-8')); - [ - 'ip', - 'port', - 'maxPlayers', - 'master', - 'name', - 'gamemodePath', - 'loadOrder', - 'dataDir', - 'startPoints', - 'offlineMode', - 'discordAuth', - ].forEach((prop) => { - if (parsed[prop]) (this as Record)[prop] = parsed[prop]; - }); - } - } - - static cachedSettings: Settings | null = null; + const settings = await fetchServerSettings(); + [ + 'ip', + 'port', + 'maxPlayers', + 'master', + 'name', + 'gamemodePath', + 'loadOrder', + 'dataDir', + 'startPoints', + 'offlineMode', + 'discordAuth', + ].forEach((prop) => { + if (settings[prop]) (this as Record)[prop] = settings[prop]; + }); - static get(): Settings { - if (Settings.cachedSettings) { - return Settings.cachedSettings; - } - const args = Settings.parseArgs(); - const res = new Settings(); - - res.port = +args['port'] || res.port; - res.maxPlayers = +args['maxPlayers'] || res.maxPlayers; - res.master = args['master'] || res.master; - res.name = args['name'] || res.name; - res.ip = args['ip'] || res.ip; - res.offlineMode = args['offlineMode'] || res.offlineMode; - Settings.cachedSettings = res; - return res; + this.allSettings = settings; } private static parseArgs() { @@ -87,4 +98,228 @@ export class Settings { parser.add_argument('--offlineMode'); return parser.parse_args(); } + + private static cachedPromise: Promise | null = null; +} + +/** + * Resolves a Git ref to a commit hash if it's not already a commit hash. + */ +async function resolveRefToCommitHash(octokit: Octokit, owner: string, repo: string, ref: string): Promise { + // Check if `ref` is already a 40-character hexadecimal string (commit hash). + if (/^[a-f0-9]{40}$/i.test(ref)) { + return ref; // It's already a commit hash. + } + + // Attempt to resolve the ref as both a branch and a tag. + try { + // First, try to resolve it as a branch. + return await getCommitHashFromRef(octokit, owner, repo, `heads/${ref}`); + } catch (error) { + try { + // If the branch resolution fails, try to resolve it as a tag. + return await getCommitHashFromRef(octokit, owner, repo, `tags/${ref}`); + } catch (tagError) { + throw new Error('Could not resolve ref to commit hash.'); + } + } } + +async function getCommitHashFromRef(octokit: Octokit, owner: string, repo: string, ref: string): Promise { + const { data } = await octokit.git.getRef({ + owner, + repo, + ref, + }); + return data.object.sha; +} + +async function fetchServerSettings(): Promise { + // Load server-settings.json + const settingsPath = 'server-settings.json'; + const rawSettings = fs.readFileSync(settingsPath, 'utf8'); + let serverSettingsFile = JSON.parse(rawSettings); + + let serverSettings: Record = {}; + + const additionalServerSettings = serverSettingsFile.additionalServerSettings || []; + + let dumpFileNameSuffix = ''; + + for (let i = 0; i < additionalServerSettings.length; ++i) { + console.log(`Verifying additional server settings source ${i + 1} / ${additionalServerSettings.length}`); + + const { type, repo, ref, token, pathRegex } = serverSettingsFile.additionalServerSettings[i]; + + if (typeof type !== "string") { + throw new Error(`Expected additionalServerSettings[${i}].type to be string`); + } + + if (type !== "github") { + throw new Error(`Expected additionalServerSettings[${i}].type to be one of ["github"], but got ${type}`); + } + + if (typeof repo !== "string") { + throw new Error(`Expected additionalServerSettings[${i}].repo to be string`); + } + if (typeof ref !== "string") { + throw new Error(`Expected additionalServerSettings[${i}].ref to be string`); + } + if (typeof token !== "string") { + throw new Error(`Expected additionalServerSettings[${i}].token to be string`); + } + if (typeof pathRegex !== "string") { + throw new Error(`Expected additionalServerSettings[${i}].pathRegex to be string`); + } + + const octokit = new Octokit({ auth: token }); + + const [owner, repoName] = repo.split('/'); + + const commitHash = await resolveRefToCommitHash(octokit, owner, repoName, ref); + dumpFileNameSuffix += `-${commitHash}`; + } + + const dumpFileName = `server-settings-dump.json`; + + const readDump: Record | undefined = fs.existsSync(dumpFileName) ? JSON.parse(fs.readFileSync(dumpFileName, 'utf-8')) : undefined; + + let readDumpNoSha512 = structuredClone(readDump); + if (readDumpNoSha512) { + delete readDumpNoSha512['_sha512_']; + } + + const expectedSha512 = readDumpNoSha512 ? crypto.createHash('sha512').update(JSON.stringify(readDumpNoSha512)).digest('hex') : ''; + + if (readDump && readDump["_meta_"] === dumpFileNameSuffix && readDump["_sha512_"] === expectedSha512) { + console.log(`Loading settings dump from ${dumpFileName}`); + serverSettings = JSON.parse(fs.readFileSync(dumpFileName, 'utf-8')); + } + else { + for (let i = 0; i < additionalServerSettings.length; ++i) { + + const { repo, ref, token, pathRegex } = serverSettingsFile.additionalServerSettings[i]; + + console.log(`Fetching settings from "${repo}" at ref "${ref}" with path regex ${pathRegex}`); + + const regex = new RegExp(pathRegex); + + const octokit = new Octokit({ auth: token }); + + const [owner, repoName] = repo.split('/'); + + // List repository contents at specified ref + const rootContent = await octokit.repos.getContent({ + owner, + repo: repoName, + ref, + path: '', + }); + + const { data } = rootContent; + + const rateLimitRemainingInitial = parseInt(rootContent.headers["x-ratelimit-remaining"]) + 1; + let rateLimitRemaining = 0; + + const onFile = async (file: { path: string, name: string }) => { + if (file.name.endsWith('.json')) { + if (regex.test(file.path)) { + // Fetch individual file content if it matches the regex + const fileData = await octokit.repos.getContent({ + owner, + repo: repoName, + ref, + path: file.path, + }); + rateLimitRemaining = parseInt(fileData.headers["x-ratelimit-remaining"]); + + if ('content' in fileData.data && typeof fileData.data.content === 'string') { + // Decode Base64 content and parse JSON + const content = Buffer.from(fileData.data.content, 'base64').toString('utf-8'); + const jsonContent = JSON.parse(content); + // Merge or handle the JSON content as needed + console.log(`Merging "${file.path}"`); + + serverSettings = lodash.merge(serverSettings, jsonContent); + } + else { + throw new Error(`Expected content to be an array (${file.path})`); + } + } + else { + console.log(`Ignoring "${file.path}"`); + } + } + } + + const onDir = async (file: { path: string, name: string }) => { + const fileData = await octokit.repos.getContent({ + owner, + repo: repoName, + ref, + path: file.path, + }); + rateLimitRemaining = parseInt(fileData.headers["x-ratelimit-remaining"]); + + if (Array.isArray(fileData.data)) { + for (const item of fileData.data) { + if (item.type === "file") { + await onFile(item); + } + else if (item.type === "dir") { + await onDir(item); + } + else { + console.warn(`Skipping unsupported item type ${item.type} (${item.path})`); + } + } + } + else { + throw new Error(`Expected data to be an array (${file.path})`); + } + } + + if (Array.isArray(data)) { + for (const item of data) { + if (item.type === "file") { + await onFile(item); + } + else if (item.type === "dir") { + await onDir(item); + } + else { + console.warn(`Skipping unsupported item type ${item.type} (${item.path})`); + } + } + } + else { + throw new Error(`Expected data to be an array (root)`); + } + + console.log(`Rate limit spent: ${rateLimitRemainingInitial - rateLimitRemaining}, remaining: ${rateLimitRemaining}`); + + const xRateLimitReset = rootContent.headers["x-ratelimit-reset"]; + const resetDate = new Date(parseInt(xRateLimitReset, 10) * 1000); + const currentDate = new Date(); + if (resetDate > currentDate) { + console.log("The rate limit will reset in the future"); + const secondsUntilReset = (resetDate.getTime() - currentDate.getTime()) / 1000; + console.log(`Seconds until reset: ${secondsUntilReset}`); + } else { + console.log("The rate limit has already been reset"); + } + } + + if (JSON.stringify(serverSettings) !== JSON.stringify(JSON.parse(rawSettings))) { + console.log(`Dumping ${dumpFileName} for cache and debugging`); + serverSettings["_meta_"] = dumpFileNameSuffix; + serverSettings["_sha512_"] = crypto.createHash('sha512').update(JSON.stringify(serverSettings)).digest('hex'); + fs.writeFileSync(dumpFileName, JSON.stringify(serverSettings, null, 2)); + } + } + + console.log(`Merging "server-settings.json" (original settings file)`); + serverSettings = lodash.merge(serverSettings, serverSettingsFile); + + return serverSettings; +} \ No newline at end of file diff --git a/skymp5-server/ts/systems/discordBanSystem.ts b/skymp5-server/ts/systems/discordBanSystem.ts index 3d303fa92d..b49d753c40 100644 --- a/skymp5-server/ts/systems/discordBanSystem.ts +++ b/skymp5-server/ts/systems/discordBanSystem.ts @@ -11,9 +11,11 @@ export class DiscordBanSystem implements System { ) { } async initAsync(ctx: SystemContext): Promise { - let discordAuth = Settings.get().discordAuth; + const settingsObject = await Settings.get(); - if (Settings.get().offlineMode) { + let discordAuth = settingsObject.discordAuth; + + if (settingsObject.offlineMode) { return console.log("discord ban system is disabled due to offline mode"); } if (!discordAuth) { diff --git a/skymp5-server/ts/systems/login.ts b/skymp5-server/ts/systems/login.ts index af3a3203e3..1d4de08fc1 100644 --- a/skymp5-server/ts/systems/login.ts +++ b/skymp5-server/ts/systems/login.ts @@ -37,6 +37,8 @@ export class Login implements System { } async initAsync(ctx: SystemContext): Promise { + this.settingsObject = await Settings.get(); + if (this.ip && this.ip != "null") { this.myAddr = this.ip + ":" + this.serverPort; } else { @@ -61,7 +63,7 @@ export class Login implements System { const ip = ctx.svr.getUserIp(userId); console.log(`Connecting a user ${userId} with ip ${ip}`); - let discordAuth = Settings.get().discordAuth; + let discordAuth = this.settingsObject.discordAuth; const gameData = content["gameData"]; if (this.offlineMode === true && gameData && gameData.session) { @@ -162,4 +164,5 @@ export class Login implements System { } private myAddr: string; + private settingsObject: Settings; } diff --git a/skymp5-server/ts/systems/spawn.ts b/skymp5-server/ts/systems/spawn.ts index 06082bd761..c9d377abc7 100644 --- a/skymp5-server/ts/systems/spawn.ts +++ b/skymp5-server/ts/systems/spawn.ts @@ -13,8 +13,9 @@ export class Spawn implements System { constructor(private log: Log) {} async initAsync(ctx: SystemContext): Promise { + const settingsObject = await Settings.get(); ctx.gm.on("spawnAllowed", (userId: number, userProfileId: number, discordRoleIds: string[], discordId: string | undefined) => { - const { startPoints } = Settings.get(); + const { startPoints } = settingsObject; // TODO: Show race menu if character is not created after relogging let actorId = ctx.svr.getActorsByProfileId(userProfileId)[0]; if (actorId) { diff --git a/skymp5-server/ts/ui.ts b/skymp5-server/ts/ui.ts index 1e0327e184..ed718ff749 100644 --- a/skymp5-server/ts/ui.ts +++ b/skymp5-server/ts/ui.ts @@ -18,8 +18,7 @@ const createApp = (getOriginPort: () => number) => { return app; }; -export const main = (): void => { - const settings = Settings.get(); +export const main = (settings: Settings): void => { const devServerPort = 1234; From 6ae3a9dfe4d95c8f56e7b06242db4416520dbe57 Mon Sep 17 00:00:00 2001 From: Leonid Pospelov Date: Sat, 1 Jun 2024 06:32:13 +0500 Subject: [PATCH 2/2] format --- skymp5-server/ts/settings.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skymp5-server/ts/settings.ts b/skymp5-server/ts/settings.ts index ef3c4cb4d3..9c203cc915 100644 --- a/skymp5-server/ts/settings.ts +++ b/skymp5-server/ts/settings.ts @@ -322,4 +322,4 @@ async function fetchServerSettings(): Promise { serverSettings = lodash.merge(serverSettings, serverSettingsFile); return serverSettings; -} \ No newline at end of file +}