From 6a6fcbdc3fe2c6d25bc39cbb873141827384cf7d Mon Sep 17 00:00:00 2001 From: richardscull <106016833+richardscull@users.noreply.github.com> Date: Sat, 22 Nov 2025 23:36:58 +0200 Subject: [PATCH 1/7] feat: Apply DisableSafeRatelimitMode on config object level --- .../ratelimiter/rate-limiter.abstract.ts | 38 ++++++++++++------- server/tests/mirrors.manager.test.ts | 24 +++++++++++- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/server/src/core/abstracts/ratelimiter/rate-limiter.abstract.ts b/server/src/core/abstracts/ratelimiter/rate-limiter.abstract.ts index b2f2845..b7d8cd5 100644 --- a/server/src/core/abstracts/ratelimiter/rate-limiter.abstract.ts +++ b/server/src/core/abstracts/ratelimiter/rate-limiter.abstract.ts @@ -20,7 +20,23 @@ const DEFAULT_RATE_LIMIT = { export class ApiRateLimiter { protected api: BaseApi; - protected config: RateLimitOptions; + private readonly _config: RateLimitOptions; + + public get config(): RateLimitOptions { + return { + ...this._config, + rateLimits: this._config.rateLimits.map((limit) => ({ + ...limit, + limit: !config.DisableSafeRatelimitMode + ? Math.floor(limit.limit * 0.9) + : limit.limit, + })), + dailyRateLimit: + this._config.dailyRateLimit && !config.DisableSafeRatelimitMode + ? Math.floor(this._config.dailyRateLimit * 0.9) + : this._config.dailyRateLimit, + }; + } private readonly redis: Redis = RedisInstance; @@ -35,10 +51,10 @@ export class ApiRateLimiter { constructor(domainHash: string, api: BaseApi, config: RateLimitOptions) { this.api = api; - this.config = config; + this._config = config; this.redisDailyLimitsKey = `${RedisKeys.DAILY_RATE_LIMIT}${domainHash}`; - if (config.dailyRateLimit) { + if (this.config.dailyRateLimit) { this.dailyLimit = null; } @@ -145,6 +161,9 @@ export class ApiRateLimiter { }; } + /** + * @deprecated Use {@link config} instead + */ public get limiterConfig() { return this.config; } @@ -154,11 +173,7 @@ export class ApiRateLimiter { const dailyLimit = await this.getDailyRateLimitRemaining(); if (dailyLimit) { - const daiyLimitRequestsLeft = !config.DisableSafeRatelimitMode - ? Math.floor(dailyLimit.requestsLeft * 0.9) - : dailyLimit.requestsLeft; - - if (daiyLimitRequestsLeft <= 0) { + if (dailyLimit.requestsLeft <= 0) { this.log( `Tried to make request to ${route} while on daily cooldown. Ignored`, 'warn', @@ -276,12 +291,7 @@ export class ApiRateLimiter { ); } - return { - ...limit, - limit: !config.DisableSafeRatelimitMode - ? Math.floor(limit.limit * 0.9) - : limit.limit, - }; + return limit; } private async getDailyRateLimitRemaining(): Promise<{ diff --git a/server/tests/mirrors.manager.test.ts b/server/tests/mirrors.manager.test.ts index 64faff4..ed7e7ab 100644 --- a/server/tests/mirrors.manager.test.ts +++ b/server/tests/mirrors.manager.test.ts @@ -1098,7 +1098,18 @@ describe('MirrorsManager', () => { ClientAbilities.GetBeatmapById, ).limit; - const shouldStopAt = Math.floor(totalRequestsLimit * 0.9); + const shouldStopAt = totalRequestsLimit; + + expect(shouldStopAt).toBe( + Math.floor( + // @ts-expect-error skip type check due to protected property + minoClient.api._config.rateLimits.find((limit) => + limit.abilities.includes( + ClientAbilities.GetBeatmapById, + ), + )!.limit * 0.9, + ), + ); for (let i = 0; i < totalRequestsLimit; i++) { const mockMinoBeatmap = mockMinoBeatmapFunc({ @@ -1133,6 +1144,17 @@ describe('MirrorsManager', () => { ClientAbilities.GetBeatmapById, ).limit; + expect(totalRequestsLimit).toBe( + Math.floor( + // @ts-expect-error skip type check due to protected property + minoClient.api._config.rateLimits.find((limit) => + limit.abilities.includes( + ClientAbilities.GetBeatmapById, + ), + )!.limit, + ), + ); + for (let i = 0; i < totalRequestsLimit; i++) { const mockMinoBeatmap = mockMinoBeatmapFunc({ data: { From 6842f800d078b8aa8a29bd178774373b8998d09c Mon Sep 17 00:00:00 2001 From: richardscull <106016833+richardscull@users.noreply.github.com> Date: Sat, 22 Nov 2025 23:52:05 +0200 Subject: [PATCH 2/7] feat: Show more stats and ratelimits for capacities in API --- .../abstracts/client/base-client.abstract.ts | 19 +++- .../core/managers/mirrors/mirrors.manager.ts | 105 +++++++++++------- server/src/database/models/requests.ts | 52 +++++++++ server/src/types/stats.ts | 8 ++ server/src/utils/mirrors-stats.ts | 67 +++++++++++ 5 files changed, 208 insertions(+), 43 deletions(-) create mode 100644 server/src/types/stats.ts create mode 100644 server/src/utils/mirrors-stats.ts diff --git a/server/src/core/abstracts/client/base-client.abstract.ts b/server/src/core/abstracts/client/base-client.abstract.ts index 537a15c..46130f5 100644 --- a/server/src/core/abstracts/client/base-client.abstract.ts +++ b/server/src/core/abstracts/client/base-client.abstract.ts @@ -11,7 +11,7 @@ import { SearchBeatmapsetsOptions, } from './base-client.types'; import { BaseApi } from '../api/base-api.abstract'; -import { RateLimitOptions } from '../ratelimiter/rate-limiter.types'; +import { RateLimit, RateLimitOptions } from '../ratelimiter/rate-limiter.types'; import { ApiRateLimiter } from '../ratelimiter/rate-limiter.abstract'; import { Beatmap, Beatmapset } from '../../../types/general/beatmap'; import { ConvertService } from '../../services/convert.service'; @@ -110,6 +110,23 @@ export class BaseClient { return this.api.getCapacity(limit); } + getCapacities(): { + ability: string; + limit: number; + remaining: number; + }[] { + const rateLimits = this.api.limiterConfig.rateLimits; + const capacities = rateLimits.flatMap((rateLimit) => + rateLimit.abilities.map((ability) => ({ + ability: ClientAbilities[ability], + limit: this.getCapacity(ability).limit, + remaining: this.getCapacity(ability).remaining, + })), + ); + + return capacities; + } + get clientConfig(): ClientOptions { return this.config; } diff --git a/server/src/core/managers/mirrors/mirrors.manager.ts b/server/src/core/managers/mirrors/mirrors.manager.ts index a747393..a521974 100644 --- a/server/src/core/managers/mirrors/mirrors.manager.ts +++ b/server/src/core/managers/mirrors/mirrors.manager.ts @@ -17,10 +17,18 @@ import { Beatmap, Beatmapset } from '../../../types/general/beatmap'; import { MinoClient } from '../../domains/catboy.best/mino.client'; import { GatariClient } from '../../domains/gatari.pw/gatari.client'; import { NerinyanClient } from '../../domains/nerinyan.moe/nerinyan.client'; -import { getRequestsCount } from '../../../database/models/requests'; +import { + getMirrorsRequestsCountForStats, + getRequestsCount, +} from '../../../database/models/requests'; import { getUTCDate } from '../../../utils/date'; import { OsulabsClient } from '../../domains/beatmaps.download/osulabs.client'; import { StorageManager } from '../storage/storage.manager'; +import { TimeRange } from '../../../types/stats'; +import { + getMirrorsRequestsQueryData, + TIME_RANGES_FOR_MIRRORS_STATS, +} from '../../../utils/mirrors-stats'; const DEFAULT_CLIENT_PROPS = { weights: { @@ -191,57 +199,70 @@ export class MirrorsManager { } async getMirrorsStatistics() { - const applicationStartTime = - getUTCDate().getTime() - Bun.nanoseconds() / 1000000; + const data = await getMirrorsRequestsCountForStats( + getMirrorsRequestsQueryData(this.clients), + ); - const successfulStatusCodes = [200, 404]; - const failedStatusCodes = [500, 502, 503, 504, 429]; + const dataMap = new Map(); + for (const item of data) { + const statusKey = + item.statuscodes === null + ? 'null' + : item.statuscodes.includes(200) + ? 'success' + : item.statuscodes.includes(500) + ? 'fail' + : 'other'; + const key = `${item.name}|${item.createdafter}|${statusKey}`; + dataMap.set(key, item.count); + } return { activeMirrors: await Promise.all( this.clients.map(async (c) => { + const baseUrl = c.client.clientConfig.baseUrl; + const stats = Object.fromEntries( + TIME_RANGES_FOR_MIRRORS_STATS.map(({ name, time }) => [ + name, + { + total: Number( + dataMap.get(`${baseUrl}|${time}|null`) || 0, + ), + successful: Number( + dataMap.get(`${baseUrl}|${time}|success`) || + 0, + ), + failed: Number( + dataMap.get(`${baseUrl}|${time}|fail`) || 0, + ), + }, + ]), + ); + return { name: c.client.constructor.name, - url: c.client.clientConfig.baseUrl, - lifetime: { - total: await getRequestsCount( - c.client.clientConfig.baseUrl, - ), - successful: await getRequestsCount( - c.client.clientConfig.baseUrl, - undefined, - successfulStatusCodes, - ), - failed: await getRequestsCount( - c.client.clientConfig.baseUrl, - undefined, - failedStatusCodes, - ), - }, - session: { - total: await getRequestsCount( - c.client.clientConfig.baseUrl, - applicationStartTime, - ), - succsesful: await getRequestsCount( - c.client.clientConfig.baseUrl, - applicationStartTime, - successfulStatusCodes, - ), - failed: await getRequestsCount( - c.client.clientConfig.baseUrl, - applicationStartTime, - failedStatusCodes, - ), - }, + url: baseUrl, + rateLimit: c.client.getCapacities(), + requests: stats, }; }), ), - activeMethods: this.clients - .map((client) => client.client.clientConfig.abilities) - .flat() - .filter((value, index, self) => self.indexOf(value) === index) - .map((a) => ClientAbilities[a]), + rateLimitsTotal: this.clients.reduce( + (acc, c) => { + const clientCapacities = c.client.getCapacities(); + + for (const capacity of clientCapacities) { + if (!acc[capacity.ability]) { + acc[capacity.ability] = { total: 0, remaining: 0 }; + } + acc[capacity.ability].total += capacity.limit; + acc[capacity.ability].remaining += capacity.remaining; + } + + return acc; + }, + {} as Record, + ), }; } diff --git a/server/src/database/models/requests.ts b/server/src/database/models/requests.ts index 5558710..d374625 100644 --- a/server/src/database/models/requests.ts +++ b/server/src/database/models/requests.ts @@ -2,6 +2,8 @@ import { and, count, eq, gte, inArray, sql } from 'drizzle-orm'; import { db } from '../client'; import { NewRequest, Request, requests } from '../schema'; import { HttpStatusCode } from 'axios'; +import { unionAll } from 'drizzle-orm/pg-core'; +import { CasingCache } from 'drizzle-orm/casing'; export async function getRequestsCount( baseUrl: string, @@ -31,6 +33,56 @@ export async function getRequestsCount( return entities[0].count; } +export async function getMirrorsRequestsCountForStats( + dataRequests: { + baseUrl: string; + createdAfter: string | null; + statusCodes?: HttpStatusCode[]; + }[], +) { + const values = dataRequests + .map( + (_, i) => + `('${dataRequests[i].baseUrl}', ${dataRequests[i].createdAfter ? `'${dataRequests[i].createdAfter}'` : null}, ${dataRequests[i].statusCodes && dataRequests[i].statusCodes.length > 0 ? `ARRAY[${dataRequests[i].statusCodes}]` : null})`, + ) + .join(', '); + + const entities = await db + .execute<{ + name: string; + createdafter: string | null; + statuscodes: HttpStatusCode[] | null; + count: number; + }>( + ` + WITH request_params (baseUrl, createdAfter, statusCodes) AS ( + VALUES ${values} + ) + SELECT + rp.baseUrl AS name, + rp.createdAfter, + rp.statusCodes, + COUNT(r.*) AS count + FROM request_params rp + LEFT JOIN requests r + ON r.base_url = rp.baseUrl + AND ( + rp.createdAfter IS NULL + OR cast(r.created_at as timestamp) >= cast(rp.createdAfter as timestamp) + ) + AND ( + rp.statusCodes IS NULL + OR r.status = ANY(rp.statusCodes) + ) + GROUP BY rp.baseUrl, rp.createdAfter, rp.statusCodes + ORDER BY rp.baseUrl, rp.createdAfter + `, + ) + .then((result) => result.rows); + + return entities; +} + export async function getRequestsByBaseUrl( baseUrl: string, createdAfter: number, diff --git a/server/src/types/stats.ts b/server/src/types/stats.ts new file mode 100644 index 0000000..aae8362 --- /dev/null +++ b/server/src/types/stats.ts @@ -0,0 +1,8 @@ +export enum TimeRange { + Lifetime = 'lifetime', + Session = 'session', + Hour = 'hour', + Day = 'day', + Week = 'week', + Month = 'month', +} diff --git a/server/src/utils/mirrors-stats.ts b/server/src/utils/mirrors-stats.ts new file mode 100644 index 0000000..5ed57a0 --- /dev/null +++ b/server/src/utils/mirrors-stats.ts @@ -0,0 +1,67 @@ +import { TimeRange } from '../types/stats'; +import { MirrorClient } from '../core/abstracts/client/base-client.types'; +import { getUTCDate } from './date'; + +export const APPLICATION_START_TIME = + getUTCDate().getTime() - Bun.nanoseconds() / 1000000; + +const MINUTE = 1000 * 60; + +export const TIME_RANGES_FOR_MIRRORS_STATS = [ + { + time: null, + name: TimeRange.Lifetime, + }, + { + time: new Date(APPLICATION_START_TIME).toISOString(), + name: TimeRange.Session, + }, + { + time: new Date(APPLICATION_START_TIME - MINUTE * 60).toISOString(), + name: TimeRange.Hour, + }, + { + time: new Date(APPLICATION_START_TIME - MINUTE * 60 * 24).toISOString(), + name: TimeRange.Day, + }, + { + time: new Date( + APPLICATION_START_TIME - MINUTE * 60 * 24 * 7, + ).toISOString(), + name: TimeRange.Week, + }, + { + time: new Date( + APPLICATION_START_TIME - MINUTE * 60 * 24 * 30, + ).toISOString(), + name: TimeRange.Month, + }, +]; + +const successfulStatusCodes = [200, 404]; +const failedStatusCodes = [500, 502, 503, 504, 429]; + +export function getMirrorsRequestsQueryData(clients: MirrorClient[]) { + return clients + .flatMap((c) => { + return TIME_RANGES_FOR_MIRRORS_STATS.map(({ time }) => time).map( + (createdAfter) => [ + { + baseUrl: c.client.clientConfig.baseUrl, + createdAfter, + statusCodes: successfulStatusCodes, + }, + { + baseUrl: c.client.clientConfig.baseUrl, + createdAfter, + statusCodes: failedStatusCodes, + }, + { + baseUrl: c.client.clientConfig.baseUrl, + createdAfter, + }, + ], + ); + }) + .flat(); +} From f4d19eb619c32c1b7eb173b445e8ef928262b394 Mon Sep 17 00:00:00 2001 From: richardscull <106016833+richardscull@users.noreply.github.com> Date: Sat, 22 Nov 2025 23:57:44 +0200 Subject: [PATCH 3/7] feat: add timeout for all requests of 10 seconds --- server/src/core/abstracts/client/base-client.abstract.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/core/abstracts/client/base-client.abstract.ts b/server/src/core/abstracts/client/base-client.abstract.ts index 46130f5..028ab9a 100644 --- a/server/src/core/abstracts/client/base-client.abstract.ts +++ b/server/src/core/abstracts/client/base-client.abstract.ts @@ -41,6 +41,7 @@ export class BaseClient { Accept: 'application/json', 'Content-Type': 'application/json', }, + timeout: 10000, }); this.convertService = new ConvertService(this.config.baseUrl); From 47056ac26fc7e53f73fac50c186cab28b0f8750c Mon Sep 17 00:00:00 2001 From: richardscull <106016833+richardscull@users.noreply.github.com> Date: Sun, 23 Nov 2025 00:20:21 +0200 Subject: [PATCH 4/7] feat: Add DISABLE_DAILY_RATE_LIMIT and fix writing into get config for rate-limiter --- .env.example | 3 +- server/src/config.ts | 3 ++ .../ratelimiter/rate-limiter.abstract.ts | 30 ++++++++++--------- server/tests/mirrors.manager.test.ts | 22 ++++++++++++++ 4 files changed, 43 insertions(+), 15 deletions(-) diff --git a/.env.example b/.env.example index 9bb44ab..5db8b39 100644 --- a/.env.example +++ b/.env.example @@ -17,4 +17,5 @@ OSZ_FILES_LIFE_SPAN= # Optional: How long in hours should .osz files be valid; D MIRRORS_TO_IGNORE="someserver,otherserver,osulabs" # Optional: If you need to disable one of the mirrors from usage, enters it's short name here. # Options: 'mino' | 'bancho' | 'direct' | 'gatari' | 'nerinyan' | 'osulabs'; -DISABLE_SAFE_RATELIMIT_MODE= # Optional: By default, we use only 90% of the available APIs ratelimits to not get any hard restrictions. You can disable this behaviour for better performance. +DISABLE_SAFE_RATELIMIT_MODE= # Optional: By default, we use only 90% of the available APIs ratelimits to not get any hard restrictions. Add 'true' to disable this behaviour for better performance. +DISABLE_DAILY_RATE_LIMIT= # Optional: Add 'true' to ignore all daily ratelimits. \ No newline at end of file diff --git a/server/src/config.ts b/server/src/config.ts index 377cc17..d133119 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -31,6 +31,7 @@ const { OSZ_FILES_LIFE_SPAN, MIRRORS_TO_IGNORE, DISABLE_SAFE_RATELIMIT_MODE, + DISABLE_DAILY_RATE_LIMIT, } = process.env; if (!POSTGRES_USER || !POSTGRES_PASSWORD) { @@ -70,6 +71,7 @@ const config: { UseBancho: boolean; MirrorsToIgnore: string[]; DisableSafeRatelimitMode: boolean; + DisableDailyRateLimit: boolean; } = { PORT: PORT || '3000', POSTGRES_USER: POSTGRES_USER || 'admin', @@ -92,6 +94,7 @@ const config: { UseBancho: BANCHO_CLIENT_SECRET && BANCHO_CLIENT_ID ? true : false, MirrorsToIgnore: MIRRORS_TO_IGNORE?.split(',').map((v) => v.trim()) ?? [], DisableSafeRatelimitMode: DISABLE_SAFE_RATELIMIT_MODE === 'true', + DisableDailyRateLimit: DISABLE_DAILY_RATE_LIMIT === 'true', }; export default config; diff --git a/server/src/core/abstracts/ratelimiter/rate-limiter.abstract.ts b/server/src/core/abstracts/ratelimiter/rate-limiter.abstract.ts index b7d8cd5..cb7a7f2 100644 --- a/server/src/core/abstracts/ratelimiter/rate-limiter.abstract.ts +++ b/server/src/core/abstracts/ratelimiter/rate-limiter.abstract.ts @@ -20,21 +20,23 @@ const DEFAULT_RATE_LIMIT = { export class ApiRateLimiter { protected api: BaseApi; - private readonly _config: RateLimitOptions; + protected _config: RateLimitOptions; public get config(): RateLimitOptions { return { ...this._config, - rateLimits: this._config.rateLimits.map((limit) => ({ - ...limit, + rateLimits: this._config.rateLimits.map((data) => ({ + ...data, limit: !config.DisableSafeRatelimitMode - ? Math.floor(limit.limit * 0.9) - : limit.limit, + ? Math.floor(data.limit * 0.9) + : data.limit, })), - dailyRateLimit: - this._config.dailyRateLimit && !config.DisableSafeRatelimitMode - ? Math.floor(this._config.dailyRateLimit * 0.9) - : this._config.dailyRateLimit, + dailyRateLimit: config.DisableDailyRateLimit + ? undefined + : this._config.dailyRateLimit && + !config.DisableSafeRatelimitMode + ? Math.floor(this._config.dailyRateLimit * 0.9) + : this._config.dailyRateLimit, }; } @@ -61,7 +63,7 @@ export class ApiRateLimiter { if ( !this.config.rateLimits.find((limit) => limit.routes.includes('/')) ) { - this.config.rateLimits.push(DEFAULT_RATE_LIMIT); + this._config.rateLimits.push(DEFAULT_RATE_LIMIT); } this.config.rateLimits.forEach((limit) => { @@ -250,7 +252,7 @@ export class ApiRateLimiter { `Got axios error while making request to ${route}. Setting cooldown of 5 minutes`, 'warn', ); - this.config.onCooldownUntil = Date.now() + 5 * 60 * 1000; // 5 minutes + this._config.onCooldownUntil = Date.now() + 5 * 60 * 1000; // 5 minutes } if (response?.status === 429) { @@ -258,7 +260,7 @@ export class ApiRateLimiter { `Rate limit exceeded for ${route}. Setting cooldown`, 'warn', ); - this.config.onCooldownUntil = Date.now() + limit.reset * 1000; + this._config.onCooldownUntil = Date.now() + limit.reset * 1000; } if (response?.status === 403) { @@ -266,7 +268,7 @@ export class ApiRateLimiter { `Got forbidden status for ${route}. Setting cooldown of 1 hour`, 'warn', ); - this.config.onCooldownUntil = Date.now() + 60 * 60 * 1000; // 1 hour + this._config.onCooldownUntil = Date.now() + 60 * 60 * 1000; // 1 hour } if (response?.status && response.status >= 502) { @@ -274,7 +276,7 @@ export class ApiRateLimiter { `Server error (${response.status}) for ${route}. Setting cooldown of 5 minutes`, 'warn', ); - this.config.onCooldownUntil = Date.now() + 5 * 60 * 1000; // 5 minutes + this._config.onCooldownUntil = Date.now() + 5 * 60 * 1000; // 5 minutes } } diff --git a/server/tests/mirrors.manager.test.ts b/server/tests/mirrors.manager.test.ts index ed7e7ab..1922de6 100644 --- a/server/tests/mirrors.manager.test.ts +++ b/server/tests/mirrors.manager.test.ts @@ -1132,6 +1132,28 @@ describe('MirrorsManager', () => { } }); + test('DisableDailyRateLimit is set to true, daily rate limit should be undefined', async () => { + config.DisableDailyRateLimit = true; + + const minoClient = getMirrorClient(MinoClient); + + // @ts-expect-error skip type check due to protected property + const dailyRateLimit = minoClient.api.config.dailyRateLimit; + + expect(dailyRateLimit).toBeUndefined(); + }); + + test('DisableDailyRateLimit is set to false, daily rate limit should be defined', async () => { + config.DisableDailyRateLimit = false; + + const minoClient = getMirrorClient(MinoClient); + + // @ts-expect-error skip type check due to protected property + const dailyRateLimit = minoClient.api.config.dailyRateLimit; + + expect(dailyRateLimit).toBeDefined(); + }); + test('DisableSafeRatelimitMode is set to true, should complete 100% of the requests', async () => { config.DisableSafeRatelimitMode = true; From 2cf3d6aa209ecf4bd46260b58a31841928c99798 Mon Sep 17 00:00:00 2001 From: richardscull <106016833+richardscull@users.noreply.github.com> Date: Sun, 23 Nov 2025 00:47:51 +0200 Subject: [PATCH 5/7] feat: add tests --- server/src/setup.ts | 2 +- server/tests/stats.endpoint.test.ts | 305 ++++++++++++++++++++++++++++ 2 files changed, 306 insertions(+), 1 deletion(-) create mode 100644 server/tests/stats.endpoint.test.ts diff --git a/server/src/setup.ts b/server/src/setup.ts index 83c1e88..1157d5a 100644 --- a/server/src/setup.ts +++ b/server/src/setup.ts @@ -108,7 +108,7 @@ async function setup() { }, }), ) - .use(await autoload({ dir: 'controllers' })) + .use(await autoload({ dir: `${__dirname}/controllers` })) .use(swagger(swaggerOptions)) .get('/favicon.ico', () => Bun.file('./server/public/favicon.ico')); } diff --git a/server/tests/stats.endpoint.test.ts b/server/tests/stats.endpoint.test.ts new file mode 100644 index 0000000..d9b1181 --- /dev/null +++ b/server/tests/stats.endpoint.test.ts @@ -0,0 +1,305 @@ +import { beforeAll, beforeEach, describe, expect, jest, test } from 'bun:test'; +import { Elysia } from 'elysia'; +import { HttpStatusCode } from 'axios'; +import { Mocker } from './utils/mocker'; +import setup from '../src/setup'; + +describe('Stats Endpoint', () => { + let app: Elysia; + + beforeAll(async () => { + await Mocker.ensureDatabaseInitialized(); + }); + + beforeEach(async () => { + jest.restoreAllMocks(); + Mocker.mockMirrorsBenchmark(); + + app = await setup(); + }); + + test('Should return 200 status code', async () => { + const response = await app.handle( + new Request('http://localhost/stats', { + method: 'GET', + }), + ); + + expect(response.status).toBe(HttpStatusCode.Ok); + }); + + test('Should return JSON response with server and manager stats', async () => { + const response = await app.handle( + new Request('http://localhost/stats', { + method: 'GET', + }), + ); + + expect(response.status).toBe(HttpStatusCode.Ok); + + const data = await response.json(); + + expect(data).toHaveProperty('data'); + expect(data.data).toHaveProperty('server'); + expect(data.data).toHaveProperty('manager'); + }); + + test('Should include server statistics with correct structure', async () => { + const response = await app.handle( + new Request('http://localhost/stats', { + method: 'GET', + }), + ); + + const data = await response.json(); + const serverStats = data.data.server; + + expect(serverStats).toHaveProperty('uptime'); + expect(serverStats.uptime).toHaveProperty('nanoseconds'); + expect(serverStats.uptime).toHaveProperty('pretty'); + expect(typeof serverStats.uptime.nanoseconds).toBe('number'); + expect(typeof serverStats.uptime.pretty).toBe('string'); + + expect(serverStats).toHaveProperty('memory'); + expect(serverStats.memory).toHaveProperty('rss'); + expect(serverStats.memory).toHaveProperty('heapTotal'); + expect(serverStats.memory).toHaveProperty('heapUsed'); + expect(serverStats.memory).toHaveProperty('external'); + expect(serverStats.memory).toHaveProperty('arrayBuffers'); + + expect(serverStats).toHaveProperty('pid'); + expect(typeof serverStats.pid).toBe('number'); + + expect(serverStats).toHaveProperty('version'); + expect(typeof serverStats.version).toBe('string'); + + expect(serverStats).toHaveProperty('revision'); + expect(typeof serverStats.revision).toBe('string'); + }); + + test('Should include manager statistics with correct structure', async () => { + const response = await app.handle( + new Request('http://localhost/stats', { + method: 'GET', + }), + ); + + const data = await response.json(); + const managerStats = data.data.manager; + + expect(managerStats).toHaveProperty('storage'); + expect(managerStats).toHaveProperty('mirrors'); + }); + + test('Should include storage statistics with correct structure', async () => { + const response = await app.handle( + new Request('http://localhost/stats', { + method: 'GET', + }), + ); + + const data = await response.json(); + const storageStats = data.data.manager.storage; + + expect(storageStats).toHaveProperty('database'); + expect(storageStats.database).toHaveProperty('beatmaps'); + expect(storageStats.database).toHaveProperty('beatmapSets'); + expect(storageStats.database).toHaveProperty('beatmapSetFile'); + expect(storageStats.database).toHaveProperty('beatmapOsuFile'); + expect(typeof storageStats.database.beatmaps).toBe('number'); + expect(typeof storageStats.database.beatmapSets).toBe('number'); + expect(typeof storageStats.database.beatmapSetFile).toBe('number'); + expect(typeof storageStats.database.beatmapOsuFile).toBe('number'); + + expect(storageStats).toHaveProperty('files'); + expect(storageStats.files).toHaveProperty('totalFiles'); + expect(storageStats.files).toHaveProperty('totalBytes'); + expect(typeof storageStats.files.totalFiles).toBe('number'); + expect(typeof storageStats.files.totalBytes).toBe('string'); + + expect(storageStats).toHaveProperty('cache'); + expect(storageStats.cache).toHaveProperty('beatmaps'); + expect(storageStats.cache).toHaveProperty('beatmapsets'); + expect(storageStats.cache).toHaveProperty('beatmapsetFiles'); + expect(storageStats.cache).toHaveProperty('beatmapOsuFiles'); + }); + + test('Should include cache statistics with correct structure', async () => { + const response = await app.handle( + new Request('http://localhost/stats', { + method: 'GET', + }), + ); + + const data = await response.json(); + const cacheStats = data.data.manager.storage.cache; + + expect(cacheStats.beatmaps).toHaveProperty('byId'); + expect(cacheStats.beatmaps).toHaveProperty('ids'); + expect(cacheStats.beatmaps.ids).toHaveProperty('byHash'); + expect(typeof cacheStats.beatmaps.byId).toBe('number'); + expect(typeof cacheStats.beatmaps.ids.byHash).toBe('number'); + + expect(cacheStats.beatmapsets).toHaveProperty('byId'); + expect(typeof cacheStats.beatmapsets.byId).toBe('number'); + + expect(cacheStats.beatmapsetFiles).toHaveProperty('byId'); + expect(typeof cacheStats.beatmapsetFiles.byId).toBe('number'); + + expect(cacheStats.beatmapOsuFiles).toHaveProperty('byId'); + expect(typeof cacheStats.beatmapOsuFiles.byId).toBe('number'); + }); + + test('Should include mirrors statistics with correct structure', async () => { + const response = await app.handle( + new Request('http://localhost/stats', { + method: 'GET', + }), + ); + + const data = await response.json(); + const mirrorsStats = data.data.manager.mirrors; + + expect(mirrorsStats).toHaveProperty('activeMirrors'); + expect(Array.isArray(mirrorsStats.activeMirrors)).toBe(true); + + expect(mirrorsStats).toHaveProperty('rateLimitsTotal'); + expect(typeof mirrorsStats.rateLimitsTotal).toBe('object'); + }); + + test('Should include active mirrors with correct structure', async () => { + const response = await app.handle( + new Request('http://localhost/stats', { + method: 'GET', + }), + ); + + const data = await response.json(); + const activeMirrors = data.data.manager.mirrors.activeMirrors; + + if (activeMirrors.length > 0) { + const mirror = activeMirrors[0]; + + expect(mirror).toHaveProperty('name'); + expect(mirror).toHaveProperty('url'); + expect(mirror).toHaveProperty('rateLimit'); + expect(mirror).toHaveProperty('requests'); + + expect(typeof mirror.name).toBe('string'); + expect(typeof mirror.url).toBe('string'); + expect(Array.isArray(mirror.rateLimit)).toBe(true); + expect(typeof mirror.requests).toBe('object'); + + expect(mirror.requests).toHaveProperty('lifetime'); + expect(mirror.requests).toHaveProperty('session'); + expect(mirror.requests).toHaveProperty('hour'); + expect(mirror.requests).toHaveProperty('day'); + expect(mirror.requests).toHaveProperty('week'); + expect(mirror.requests).toHaveProperty('month'); + + const timeRange = mirror.requests.lifetime; + expect(timeRange).toHaveProperty('total'); + expect(timeRange).toHaveProperty('successful'); + expect(timeRange).toHaveProperty('failed'); + expect(typeof timeRange.total).toBe('number'); + expect(typeof timeRange.successful).toBe('number'); + expect(typeof timeRange.failed).toBe('number'); + } + }); + + test('Should include rate limit information in active mirrors', async () => { + const response = await app.handle( + new Request('http://localhost/stats', { + method: 'GET', + }), + ); + + const data = await response.json(); + const activeMirrors = data.data.manager.mirrors.activeMirrors; + + if (activeMirrors.length > 0) { + const mirror = activeMirrors[0]; + const rateLimit = mirror.rateLimit; + + if (rateLimit.length > 0) { + const capacity = rateLimit[0]; + expect(capacity).toHaveProperty('ability'); + expect(capacity).toHaveProperty('limit'); + expect(capacity).toHaveProperty('remaining'); + expect(typeof capacity.ability).toBe('string'); + expect(typeof capacity.limit).toBe('number'); + expect(typeof capacity.remaining).toBe('number'); + } + } + }); + + test('Should return consistent response structure on multiple requests', async () => { + const response1 = await app.handle( + new Request('http://localhost/stats', { + method: 'GET', + }), + ); + const data1 = await response1.json(); + + const response2 = await app.handle( + new Request('http://localhost/stats', { + method: 'GET', + }), + ); + const data2 = await response2.json(); + + // Both should have the same structure + expect(data1).toHaveProperty('data'); + expect(data2).toHaveProperty('data'); + expect(data1.data).toHaveProperty('server'); + expect(data2.data).toHaveProperty('server'); + expect(data1.data).toHaveProperty('manager'); + expect(data2.data).toHaveProperty('manager'); + + // Server stats should have same structure (values may differ) + expect(Object.keys(data1.data.server)).toEqual( + Object.keys(data2.data.server), + ); + expect(Object.keys(data1.data.manager)).toEqual( + Object.keys(data2.data.manager), + ); + }); + + test('Should have valid uptime format', async () => { + const response = await app.handle( + new Request('http://localhost/stats', { + method: 'GET', + }), + ); + + const data = await response.json(); + const uptime = data.data.server.uptime; + + // Uptime nanoseconds should be a positive number + expect(uptime.nanoseconds).toBeGreaterThan(0); + + // Uptime pretty should match format like "0d 0h 0m 0s" + expect(uptime.pretty).toMatch(/^\d+d \d+h \d+m \d+s$/); + }); + + test('Should have valid memory values', async () => { + const response = await app.handle( + new Request('http://localhost/stats', { + method: 'GET', + }), + ); + + const data = await response.json(); + const memory = data.data.server.memory; + + // Memory values should be strings (human-readable format) + expect(typeof memory.rss).toBe('string'); + expect(typeof memory.heapTotal).toBe('string'); + expect(typeof memory.heapUsed).toBe('string'); + expect(typeof memory.external).toBe('string'); + expect(typeof memory.arrayBuffers).toBe('string'); + + // Memory values should end with "MB" or similar + expect(memory.rss).toMatch(/MB$/); + }); +}); From 8697cafb863f6acee0a1b3410d1f6dff90ffe7d0 Mon Sep 17 00:00:00 2001 From: richardscull <106016833+richardscull@users.noreply.github.com> Date: Sun, 23 Nov 2025 00:56:11 +0200 Subject: [PATCH 6/7] feat: Add onCooldownUntil in stats --- server/src/core/abstracts/client/base-client.abstract.ts | 4 ++++ server/src/core/managers/mirrors/mirrors.manager.ts | 1 + server/tests/stats.endpoint.test.ts | 7 +++++++ 3 files changed, 12 insertions(+) diff --git a/server/src/core/abstracts/client/base-client.abstract.ts b/server/src/core/abstracts/client/base-client.abstract.ts index 028ab9a..f247516 100644 --- a/server/src/core/abstracts/client/base-client.abstract.ts +++ b/server/src/core/abstracts/client/base-client.abstract.ts @@ -128,6 +128,10 @@ export class BaseClient { return capacities; } + onCooldownUntil(): number | undefined { + return this.api.limiterConfig.onCooldownUntil; + } + get clientConfig(): ClientOptions { return this.config; } diff --git a/server/src/core/managers/mirrors/mirrors.manager.ts b/server/src/core/managers/mirrors/mirrors.manager.ts index a521974..b3b842e 100644 --- a/server/src/core/managers/mirrors/mirrors.manager.ts +++ b/server/src/core/managers/mirrors/mirrors.manager.ts @@ -242,6 +242,7 @@ export class MirrorsManager { return { name: c.client.constructor.name, url: baseUrl, + onCooldownUntil: c.client.onCooldownUntil(), rateLimit: c.client.getCapacities(), requests: stats, }; diff --git a/server/tests/stats.endpoint.test.ts b/server/tests/stats.endpoint.test.ts index d9b1181..4b2466f 100644 --- a/server/tests/stats.endpoint.test.ts +++ b/server/tests/stats.endpoint.test.ts @@ -187,6 +187,13 @@ describe('Stats Endpoint', () => { expect(typeof mirror.name).toBe('string'); expect(typeof mirror.url).toBe('string'); + + if ('onCooldownUntil' in mirror) { + expect( + mirror.onCooldownUntil === null || + typeof mirror.onCooldownUntil === 'number', + ).toBe(true); + } expect(Array.isArray(mirror.rateLimit)).toBe(true); expect(typeof mirror.requests).toBe('object'); From 40087e7cc7a45f7b5c880089956667a5d4d984e9 Mon Sep 17 00:00:00 2001 From: richardscull <106016833+richardscull@users.noreply.github.com> Date: Sun, 23 Nov 2025 01:00:11 +0200 Subject: [PATCH 7/7] feat: Show safe part of the config in the stats --- server/src/config.ts | 13 +++++++++++++ server/src/controllers/index.ts | 3 +++ 2 files changed, 16 insertions(+) diff --git a/server/src/config.ts b/server/src/config.ts index d133119..b43d09b 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -97,4 +97,17 @@ const config: { DisableDailyRateLimit: DISABLE_DAILY_RATE_LIMIT === 'true', }; +export const observationaryConfigPublic = { + RATELIMIT_CALLS_PER_WINDOW: Number(RATELIMIT_CALLS_PER_WINDOW) || 100, + RATELIMIT_TIME_WINDOW: Number(RATELIMIT_TIME_WINDOW) || 20 * 1000, + OSZ_FILES_LIFE_SPAN: Number(OSZ_FILES_LIFE_SPAN) || 24, + IsProduction: Bun.env.NODE_ENV === 'production', + IsAutomatedTesting: Bun.env.NODE_ENV === 'test', + IsDebug: DEBUG_MODE === 'true', + UseBancho: BANCHO_CLIENT_SECRET && BANCHO_CLIENT_ID ? true : false, + MirrorsToIgnore: MIRRORS_TO_IGNORE?.split(',').map((v) => v.trim()) ?? [], + DisableSafeRatelimitMode: DISABLE_SAFE_RATELIMIT_MODE === 'true', + DisableDailyRateLimit: DISABLE_DAILY_RATE_LIMIT === 'true', +}; + export default config; diff --git a/server/src/controllers/index.ts b/server/src/controllers/index.ts index 277630a..6c3000b 100644 --- a/server/src/controllers/index.ts +++ b/server/src/controllers/index.ts @@ -2,6 +2,7 @@ import type { App } from '../app'; import { StatsServicePlugin } from '../plugins/statsService'; import { HttpStatusCode } from 'axios'; import { BeatmapsManagerPlugin } from '../plugins/beatmapManager'; +import { observationaryConfigPublic } from '../config'; export default (app: App) => { app.get('/', ({ redirect }) => { @@ -16,10 +17,12 @@ export default (app: App) => { const serverStats = StatsServiceInstance.getServerStatistics(); const managerStats = await BeatmapsManagerInstance.getManagerStats(); + const serverConfig = observationaryConfigPublic; return { status: HttpStatusCode.Ok, data: { + config: serverConfig, server: serverStats, manager: managerStats, },