feat: Apply DisableSafeRatelimitMode on config object level

.env.example

@@ -17,4 +17,5 @@ OSZ_FILES_LIFE_SPAN= # Optional: How long in hours should .osz files be valid; D
 MIRRORS_TO_IGNORE="someserver,otherserver,osulabs" # Optional: If you need to disable one of the mirrors from usage, enters it's short name here.
 # Options: 'mino' | 'bancho' | 'direct' | 'gatari' | 'nerinyan' | 'osulabs';
 
-DISABLE_SAFE_RATELIMIT_MODE= # Optional: By default, we use only 90% of the available APIs ratelimits to not get any hard restrictions. You can disable this behaviour for better performance.
+DISABLE_SAFE_RATELIMIT_MODE= # Optional: By default, we use only 90% of the available APIs ratelimits to not get any hard restrictions. Add 'true' to disable this behaviour for better performance. 
+DISABLE_DAILY_RATE_LIMIT= # Optional: Add 'true' to ignore all daily ratelimits. 

server/src/config.ts

@@ -31,6 +31,7 @@ const {
     OSZ_FILES_LIFE_SPAN,
     MIRRORS_TO_IGNORE,
     DISABLE_SAFE_RATELIMIT_MODE,
+    DISABLE_DAILY_RATE_LIMIT,
 } = process.env;
 
 if (!POSTGRES_USER || !POSTGRES_PASSWORD) {
@@ -70,6 +71,7 @@ const config: {
     UseBancho: boolean;
     MirrorsToIgnore: string[];
     DisableSafeRatelimitMode: boolean;
+    DisableDailyRateLimit: boolean;
 } = {
     PORT: PORT || '3000',
     POSTGRES_USER: POSTGRES_USER || 'admin',
@@ -92,6 +94,20 @@ const config: {
     UseBancho: BANCHO_CLIENT_SECRET && BANCHO_CLIENT_ID ? true : false,
     MirrorsToIgnore: MIRRORS_TO_IGNORE?.split(',').map((v) => v.trim()) ?? [],
     DisableSafeRatelimitMode: DISABLE_SAFE_RATELIMIT_MODE === 'true',
+    DisableDailyRateLimit: DISABLE_DAILY_RATE_LIMIT === 'true',
+};
+
+export const observationaryConfigPublic = {
+    RATELIMIT_CALLS_PER_WINDOW: Number(RATELIMIT_CALLS_PER_WINDOW) || 100,
+    RATELIMIT_TIME_WINDOW: Number(RATELIMIT_TIME_WINDOW) || 20 * 1000,
+    OSZ_FILES_LIFE_SPAN: Number(OSZ_FILES_LIFE_SPAN) || 24,
+    IsProduction: Bun.env.NODE_ENV === 'production',
+    IsAutomatedTesting: Bun.env.NODE_ENV === 'test',
+    IsDebug: DEBUG_MODE === 'true',
+    UseBancho: BANCHO_CLIENT_SECRET && BANCHO_CLIENT_ID ? true : false,
+    MirrorsToIgnore: MIRRORS_TO_IGNORE?.split(',').map((v) => v.trim()) ?? [],
+    DisableSafeRatelimitMode: DISABLE_SAFE_RATELIMIT_MODE === 'true',
+    DisableDailyRateLimit: DISABLE_DAILY_RATE_LIMIT === 'true',
 };
 
 export default config;

server/src/controllers/index.ts

@@ -2,6 +2,7 @@ import type { App } from '../app';
 import { StatsServicePlugin } from '../plugins/statsService';
 import { HttpStatusCode } from 'axios';
 import { BeatmapsManagerPlugin } from '../plugins/beatmapManager';
+import { observationaryConfigPublic } from '../config';
 
 export default (app: App) => {
     app.get('/', ({ redirect }) => {
@@ -16,10 +17,12 @@ export default (app: App) => {
                 const serverStats = StatsServiceInstance.getServerStatistics();
                 const managerStats =
                     await BeatmapsManagerInstance.getManagerStats();
+                const serverConfig = observationaryConfigPublic;
 
                 return {
                     status: HttpStatusCode.Ok,
                     data: {
+                        config: serverConfig,
                         server: serverStats,
                         manager: managerStats,
                     },

server/src/core/abstracts/client/base-client.abstract.ts

@@ -11,7 +11,7 @@ import {
     SearchBeatmapsetsOptions,
 } from './base-client.types';
 import { BaseApi } from '../api/base-api.abstract';
-import { RateLimitOptions } from '../ratelimiter/rate-limiter.types';
+import { RateLimit, RateLimitOptions } from '../ratelimiter/rate-limiter.types';
 import { ApiRateLimiter } from '../ratelimiter/rate-limiter.abstract';
 import { Beatmap, Beatmapset } from '../../../types/general/beatmap';
 import { ConvertService } from '../../services/convert.service';
@@ -41,6 +41,7 @@ export class BaseClient {
                 Accept: 'application/json',
                 'Content-Type': 'application/json',
             },
+            timeout: 10000,
         });
 
         this.convertService = new ConvertService(this.config.baseUrl);
@@ -110,6 +111,27 @@ export class BaseClient {
         return this.api.getCapacity(limit);
     }
 
+    getCapacities(): {
+        ability: string;
+        limit: number;
+        remaining: number;
+    }[] {
+        const rateLimits = this.api.limiterConfig.rateLimits;
+        const capacities = rateLimits.flatMap((rateLimit) =>
+            rateLimit.abilities.map((ability) => ({
+                ability: ClientAbilities[ability],
+                limit: this.getCapacity(ability).limit,
+                remaining: this.getCapacity(ability).remaining,
+            })),
+        );
+
+        return capacities;
+    }
+
+    onCooldownUntil(): number | undefined {
+        return this.api.limiterConfig.onCooldownUntil;
+    }
+
     get clientConfig(): ClientOptions {
         return this.config;
     }

server/src/core/abstracts/ratelimiter/rate-limiter.abstract.ts

@@ -20,7 +20,25 @@ const DEFAULT_RATE_LIMIT = {
 
 export class ApiRateLimiter {
     protected api: BaseApi;
-    protected config: RateLimitOptions;
+    protected _config: RateLimitOptions;
+
+    public get config(): RateLimitOptions {
+        return {
+            ...this._config,
+            rateLimits: this._config.rateLimits.map((data) => ({
+                ...data,
+                limit: !config.DisableSafeRatelimitMode
+                    ? Math.floor(data.limit * 0.9)
+                    : data.limit,
+            })),
+            dailyRateLimit: config.DisableDailyRateLimit
+                ? undefined
+                : this._config.dailyRateLimit &&
+                    !config.DisableSafeRatelimitMode
+                  ? Math.floor(this._config.dailyRateLimit * 0.9)
+                  : this._config.dailyRateLimit,
+        };
+    }
 
     private readonly redis: Redis = RedisInstance;
 
@@ -35,17 +53,17 @@ export class ApiRateLimiter {
 
     constructor(domainHash: string, api: BaseApi, config: RateLimitOptions) {
         this.api = api;
-        this.config = config;
+        this._config = config;
         this.redisDailyLimitsKey = `${RedisKeys.DAILY_RATE_LIMIT}${domainHash}`;
 
-        if (config.dailyRateLimit) {
+        if (this.config.dailyRateLimit) {
             this.dailyLimit = null;
         }
 
         if (
             !this.config.rateLimits.find((limit) => limit.routes.includes('/'))
         ) {
-            this.config.rateLimits.push(DEFAULT_RATE_LIMIT);
+            this._config.rateLimits.push(DEFAULT_RATE_LIMIT);
         }
 
         this.config.rateLimits.forEach((limit) => {
@@ -145,6 +163,9 @@ export class ApiRateLimiter {
         };
     }
 
+    /**
+     * @deprecated Use {@link config} instead
+     */
     public get limiterConfig() {
         return this.config;
     }
@@ -154,11 +175,7 @@ export class ApiRateLimiter {
         const dailyLimit = await this.getDailyRateLimitRemaining();
 
         if (dailyLimit) {
-            const daiyLimitRequestsLeft = !config.DisableSafeRatelimitMode
-                ? Math.floor(dailyLimit.requestsLeft * 0.9)
-                : dailyLimit.requestsLeft;
-
-            if (daiyLimitRequestsLeft <= 0) {
+            if (dailyLimit.requestsLeft <= 0) {
                 this.log(
                     `Tried to make request to ${route} while on daily cooldown. Ignored`,
                     'warn',
@@ -235,31 +252,31 @@ export class ApiRateLimiter {
                 `Got axios error while making request to ${route}. Setting cooldown of 5 minutes`,
                 'warn',
             );
-            this.config.onCooldownUntil = Date.now() + 5 * 60 * 1000; // 5 minutes
+            this._config.onCooldownUntil = Date.now() + 5 * 60 * 1000; // 5 minutes
         }
 
         if (response?.status === 429) {
             this.log(
                 `Rate limit exceeded for ${route}. Setting cooldown`,
                 'warn',
             );
-            this.config.onCooldownUntil = Date.now() + limit.reset * 1000;
+            this._config.onCooldownUntil = Date.now() + limit.reset * 1000;
         }
 
         if (response?.status === 403) {
             this.log(
                 `Got forbidden status for ${route}. Setting cooldown of 1 hour`,
                 'warn',
             );
-            this.config.onCooldownUntil = Date.now() + 60 * 60 * 1000; // 1 hour
+            this._config.onCooldownUntil = Date.now() + 60 * 60 * 1000; // 1 hour
         }
 
         if (response?.status && response.status >= 502) {
             this.log(
                 `Server error (${response.status}) for ${route}. Setting cooldown of 5 minutes`,
                 'warn',
             );
-            this.config.onCooldownUntil = Date.now() + 5 * 60 * 1000; // 5 minutes
+            this._config.onCooldownUntil = Date.now() + 5 * 60 * 1000; // 5 minutes
         }
     }
 
@@ -276,12 +293,7 @@ export class ApiRateLimiter {
             );
         }
 
-        return {
-            ...limit,
-            limit: !config.DisableSafeRatelimitMode
-                ? Math.floor(limit.limit * 0.9)
-                : limit.limit,
-        };
+        return limit;
     }
 
     private async getDailyRateLimitRemaining(): Promise<{

server/src/core/managers/mirrors/mirrors.manager.ts

@@ -17,10 +17,18 @@ import { Beatmap, Beatmapset } from '../../../types/general/beatmap';
 import { MinoClient } from '../../domains/catboy.best/mino.client';
 import { GatariClient } from '../../domains/gatari.pw/gatari.client';
 import { NerinyanClient } from '../../domains/nerinyan.moe/nerinyan.client';
-import { getRequestsCount } from '../../../database/models/requests';
+import {
+    getMirrorsRequestsCountForStats,
+    getRequestsCount,
+} from '../../../database/models/requests';
 import { getUTCDate } from '../../../utils/date';
 import { OsulabsClient } from '../../domains/beatmaps.download/osulabs.client';
 import { StorageManager } from '../storage/storage.manager';
+import { TimeRange } from '../../../types/stats';
+import {
+    getMirrorsRequestsQueryData,
+    TIME_RANGES_FOR_MIRRORS_STATS,
+} from '../../../utils/mirrors-stats';
 
 const DEFAULT_CLIENT_PROPS = {
     weights: {
@@ -191,57 +199,71 @@ export class MirrorsManager {
     }
 
     async getMirrorsStatistics() {
-        const applicationStartTime =
-            getUTCDate().getTime() - Bun.nanoseconds() / 1000000;
+        const data = await getMirrorsRequestsCountForStats(
+            getMirrorsRequestsQueryData(this.clients),
+        );
 
-        const successfulStatusCodes = [200, 404];
-        const failedStatusCodes = [500, 502, 503, 504, 429];
+        const dataMap = new Map<string, number>();
+        for (const item of data) {
+            const statusKey =
+                item.statuscodes === null
+                    ? 'null'
+                    : item.statuscodes.includes(200)
+                      ? 'success'
+                      : item.statuscodes.includes(500)
+                        ? 'fail'
+                        : 'other';
+            const key = `${item.name}|${item.createdafter}|${statusKey}`;
+            dataMap.set(key, item.count);
+        }
 
         return {
             activeMirrors: await Promise.all(
                 this.clients.map(async (c) => {
+                    const baseUrl = c.client.clientConfig.baseUrl;
+                    const stats = Object.fromEntries(
+                        TIME_RANGES_FOR_MIRRORS_STATS.map(({ name, time }) => [
+                            name,
+                            {
+                                total: Number(
+                                    dataMap.get(`${baseUrl}|${time}|null`) || 0,
+                                ),
+                                successful: Number(
+                                    dataMap.get(`${baseUrl}|${time}|success`) ||
+                                        0,
+                                ),
+                                failed: Number(
+                                    dataMap.get(`${baseUrl}|${time}|fail`) || 0,
+                                ),
+                            },
+                        ]),
+                    );
+
                     return {
                         name: c.client.constructor.name,
-                        url: c.client.clientConfig.baseUrl,
-                        lifetime: {
-                            total: await getRequestsCount(
-                                c.client.clientConfig.baseUrl,
-                            ),
-                            successful: await getRequestsCount(
-                                c.client.clientConfig.baseUrl,
-                                undefined,
-                                successfulStatusCodes,
-                            ),
-                            failed: await getRequestsCount(
-                                c.client.clientConfig.baseUrl,
-                                undefined,
-                                failedStatusCodes,
-                            ),
-                        },
-                        session: {
-                            total: await getRequestsCount(
-                                c.client.clientConfig.baseUrl,
-                                applicationStartTime,
-                            ),
-                            succsesful: await getRequestsCount(
-                                c.client.clientConfig.baseUrl,
-                                applicationStartTime,
-                                successfulStatusCodes,
-                            ),
-                            failed: await getRequestsCount(
-                                c.client.clientConfig.baseUrl,
-                                applicationStartTime,
-                                failedStatusCodes,
-                            ),
-                        },
+                        url: baseUrl,
+                        onCooldownUntil: c.client.onCooldownUntil(),
+                        rateLimit: c.client.getCapacities(),
+                        requests: stats,
                     };
                 }),
             ),
-            activeMethods: this.clients
-                .map((client) => client.client.clientConfig.abilities)
-                .flat()
-                .filter((value, index, self) => self.indexOf(value) === index)
-                .map((a) => ClientAbilities[a]),
+            rateLimitsTotal: this.clients.reduce(
+                (acc, c) => {
+                    const clientCapacities = c.client.getCapacities();
+
+                    for (const capacity of clientCapacities) {
+                        if (!acc[capacity.ability]) {
+                            acc[capacity.ability] = { total: 0, remaining: 0 };
+                        }
+                        acc[capacity.ability].total += capacity.limit;
+                        acc[capacity.ability].remaining += capacity.remaining;
+                    }
+
+                    return acc;
+                },
+                {} as Record<string, { total: number; remaining: number }>,
+            ),
         };
     }