diff --git a/package-lock.json b/package-lock.json index 96cd2986..1139d006 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "@notionhq/client": "^2.2.3", "@sentry/node": "^7.50.0", "@temporalio/client": "^1.11.3", - "@togethercrew.dev/db": "^3.2.3", + "@togethercrew.dev/db": "^3.3.0", "@togethercrew.dev/tc-messagebroker": "^0.0.50", "@types/express-session": "^1.17.7", "@types/morgan": "^1.9.5", @@ -3659,9 +3659,9 @@ "dev": true }, "node_modules/@togethercrew.dev/db": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/@togethercrew.dev/db/-/db-3.2.3.tgz", - "integrity": "sha512-V2NqrrXyuJKornzhAfZymWYATp6KkjXrMxNPgb0c7LCsIo+tRLlqqQIkwYt5htI9ujBP5bemgtYHYykhQhWdpA==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@togethercrew.dev/db/-/db-3.3.0.tgz", + "integrity": "sha512-5UYnxR0P5N8KAEsGUfIkbGT8B+wu4sU9hiKvLp5J9MeaebnPHp9U/Lie30ofjP3eOVetpE3SwO/HVuBrR4aYaw==", "license": "ISC", "dependencies": { "discord.js": "^14.7.1", diff --git a/package.json b/package.json index 7457546a..77b93b8b 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "@notionhq/client": "^2.2.3", "@sentry/node": "^7.50.0", "@temporalio/client": "^1.11.3", - "@togethercrew.dev/db": "^3.2.3", + "@togethercrew.dev/db": "^3.3.0", "@togethercrew.dev/tc-messagebroker": "^0.0.50", "@types/express-session": "^1.17.7", "@types/morgan": "^1.9.5", diff --git a/src/docs/module.doc.yml b/src/docs/module.doc.yml index 202870f6..c19c0fcc 100644 --- a/src/docs/module.doc.yml +++ b/src/docs/module.doc.yml @@ -169,7 +169,8 @@ paths: name: type: string description: Name of the platform. - enum: ['discord', 'google', 'github', 'notion', 'mediaWiki', 'discourse'] + enum: + ['discord', 'google', 'github', 'notion', 'mediaWiki', 'discourse', 'telegram', 'website'] metadata: type: object description: Metadata specific to the module and platform. Varies depending on the platform name and module name. @@ -236,6 +237,8 @@ paths: items: type: string description: Metadata for the hivemind module on MediaWiki. + - type: object + description: Metadata for the hivemind module on website. - type: object properties: selectedEmails: diff --git a/src/docs/platform.doc.yml b/src/docs/platform.doc.yml index 82264c58..de8cadbd 100644 --- a/src/docs/platform.doc.yml +++ b/src/docs/platform.doc.yml @@ -69,6 +69,7 @@ paths: - mediaWiki - discourse - telegram + - website description: Name of the platform to create. Must be one of the supported platforms. community: type: string @@ -194,7 +195,17 @@ paths: properties: chat: type: object - description: Metadata for Telegram. + description: Metadata for Website. + - type: object + required: [resources] + properties: + resources: + type: array + items: + type: string + format: uri + description: Metadata for Website. + responses: '201': description: Platform created successfully. @@ -227,6 +238,8 @@ paths: - github - notion - mediaWiki + - telegram + - website - in: query name: community schema: diff --git a/src/services/index.ts b/src/services/index.ts index 5c514d0f..75fd628e 100644 --- a/src/services/index.ts +++ b/src/services/index.ts @@ -13,6 +13,7 @@ import platformService from './platform.service'; import reputationScoreService from './reputationScore.service'; import sagaService from './saga.service'; import telegramService from './telegram'; +import websiteService from './temporal/website.service'; import tokenService from './token.service'; import twitterService from './twitter.service'; import userService from './user.service'; @@ -36,4 +37,5 @@ export { discourseService, telegramService, reputationScoreService, + websiteService, }; diff --git a/src/services/module.service.ts b/src/services/module.service.ts index a8b9e8ea..dd81f4df 100644 --- a/src/services/module.service.ts +++ b/src/services/module.service.ts @@ -1,6 +1,9 @@ -import { FilterQuery, HydratedDocument, Types } from 'mongoose'; +import { FilterQuery, HydratedDocument, ObjectId, Types } from 'mongoose'; -import { IModule, IModuleUpdateBody, Module } from '@togethercrew.dev/db'; +import { IModule, IModuleUpdateBody, Module, ModuleNames, PlatformNames } from '@togethercrew.dev/db'; + +import platformService from './platform.service'; +import websiteService from './website'; /** * Create a module @@ -60,7 +63,6 @@ const updateModule = async ( module: HydratedDocument, updateBody: Partial, ): Promise> => { - // Check if `options.platforms` is in the updateBody if (updateBody.options && updateBody.options.platforms) { if (updateBody.options.platforms[0].name == undefined) { { @@ -69,15 +71,18 @@ const updateModule = async ( else module.options?.platforms.push(updateBody.options.platforms[0]); } } else { - // Iterate through each platform in the incoming update for (const newPlatform of updateBody.options.platforms) { const existingPlatform = module.options?.platforms.find((p) => p.name === newPlatform.name); if (existingPlatform) { - // If the platform already exists, update it existingPlatform.metadata = newPlatform.metadata; } else { - // If the platform does not exist, add new module.options?.platforms.push(newPlatform); + if (module.name === ModuleNames.Hivemind && newPlatform.name === PlatformNames.Website) { + const scheduleId = await websiteService.coreService.createWebsiteSchedule(newPlatform.platform); + const platform = await platformService.getPlatformById(newPlatform.platform); + platform?.set('metadata.scheduleId', scheduleId); + await platform?.save(); + } } } } diff --git a/src/services/platform.service.ts b/src/services/platform.service.ts index f9e40f5b..bf1a60ff 100644 --- a/src/services/platform.service.ts +++ b/src/services/platform.service.ts @@ -178,6 +178,8 @@ function getMetadataKey(platformName: string): string { return 'baseURL'; case PlatformNames.Discourse: return 'id'; + case PlatformNames.Website: + return 'resources'; default: throw new Error('Unsupported platform'); } @@ -210,6 +212,7 @@ const managePlatformConnection = async ( const activePlatformOtherCommunity = await Platform.findOne({ community: { $ne: communityId }, [`metadata.${metadataKey}`]: metadataId, + name: platformData.name, }); if (activePlatformOtherCommunity) { diff --git a/src/services/temporal/discourse.service.ts b/src/services/temporal/discourse.service.ts index 4c621781..b53b5bb9 100644 --- a/src/services/temporal/discourse.service.ts +++ b/src/services/temporal/discourse.service.ts @@ -1,6 +1,11 @@ +import parentLogger from 'src/config/logger'; + import { CalendarSpec, Client, ScheduleHandle, ScheduleOverlapPolicy } from '@temporalio/client'; -import { TemporalCoreService } from './core.service'; + import config from '../../config'; +import { TemporalCoreService } from './core.service'; + +const logger = parentLogger.child({ module: 'DiscourseTemporalService' }); class TemporalDiscourseService extends TemporalCoreService { public async createSchedule(platformId: string, endpoint: string): Promise { diff --git a/src/services/temporal/website.service.ts b/src/services/temporal/website.service.ts new file mode 100644 index 00000000..869e81b1 --- /dev/null +++ b/src/services/temporal/website.service.ts @@ -0,0 +1,64 @@ +import { Types } from 'mongoose'; +import parentLogger from 'src/config/logger'; + +import { CalendarSpec, Client, ScheduleHandle, ScheduleOverlapPolicy } from '@temporalio/client'; + +import config from '../../config'; +import { TemporalCoreService } from './core.service'; + +const logger = parentLogger.child({ module: 'WebsiteTemporalService' }); + +class TemporalWebsiteService extends TemporalCoreService { + public async createSchedule(platformId: Types.ObjectId): Promise { + const initiationTime = new Date(); + const dayNumber = initiationTime.getUTCDay(); + const hour = initiationTime.getUTCHours(); + const minute = initiationTime.getUTCMinutes(); + const DAY_NAMES = ['SUNDAY', 'MONDAY', 'TUESDAY', 'WEDNESDAY', 'THURSDAY', 'FRIDAY', 'SATURDAY'] as const; + const dayOfWeek = DAY_NAMES[dayNumber]; + + const calendarSpec: CalendarSpec = { + dayOfWeek, + hour, + minute, + comment: `Weekly schedule for ${dayOfWeek} at ${hour}:${minute} UTC`, + }; + + try { + const client: Client = await this.getClient(); + + return client.schedule.create({ + scheduleId: `website/${platformId}`, + spec: { + calendars: [calendarSpec], + }, + action: { + type: 'startWorkflow', + workflowType: 'WebsiteIngestionSchedulerWorkflow', + args: [{ platformId }], + taskQueue: config.temporal.heavyQueue, + }, + policies: { + catchupWindow: '1 day', + overlap: ScheduleOverlapPolicy.SKIP, + }, + }); + } catch (error) { + throw new Error(`Failed to create or update website ingestion schedule: ${(error as Error).message}`); + } + } + + public async pauseSchedule(scheduleId: string): Promise { + const client: Client = await this.getClient(); + const handle = client.schedule.getHandle(scheduleId); + await handle.pause(); + } + + public async deleteSchedule(scheduleId: string): Promise { + const client: Client = await this.getClient(); + const handle = client.schedule.getHandle(scheduleId); + await handle.delete(); + } +} + +export default new TemporalWebsiteService(); diff --git a/src/services/website/core.service.ts b/src/services/website/core.service.ts new file mode 100644 index 00000000..3761f523 --- /dev/null +++ b/src/services/website/core.service.ts @@ -0,0 +1,33 @@ +import { Types } from 'mongoose'; + +import parentLogger from '../../config/logger'; +import { ApiError } from '../../utils'; +import temporalWebsite from '../temporal/website.service'; + +const logger = parentLogger.child({ module: 'WebsiteCoreService' }); + +async function createWebsiteSchedule(platformId: Types.ObjectId): Promise { + try { + const schedule = await temporalWebsite.createSchedule(platformId); + logger.info(`Started schedule '${schedule.scheduleId}'`); + await schedule.trigger(); + return schedule.scheduleId; + } catch (error) { + logger.error(error, 'Failed to trigger website schedule.'); + throw new ApiError(590, 'Failed to create website schedule.'); + } +} + +async function deleteWebsiteSchedule(scheduleId: string): Promise { + try { + await temporalWebsite.deleteSchedule(scheduleId); + } catch (error) { + logger.error(error, 'Failed to delete website schedule.'); + throw new ApiError(590, 'Failed to delete website schedule.'); + } +} + +export default { + createWebsiteSchedule, + deleteWebsiteSchedule, +}; diff --git a/src/services/website/index.ts b/src/services/website/index.ts new file mode 100644 index 00000000..2c70ec6e --- /dev/null +++ b/src/services/website/index.ts @@ -0,0 +1,5 @@ +import coreService from './core.service'; + +export default { + coreService, +}; diff --git a/src/validations/module.validation.ts b/src/validations/module.validation.ts index 8fda1cd5..64037deb 100644 --- a/src/validations/module.validation.ts +++ b/src/validations/module.validation.ts @@ -1,12 +1,14 @@ import Joi from 'joi'; -import { objectId } from './custom.validation'; + import { - PlatformNames, - ModuleNames, HivemindPlatformNames, + ModuleNames, + PlatformNames, ViolationDetectionPlatformNames, } from '@togethercrew.dev/db'; +import { objectId } from './custom.validation'; + const createModule = { body: Joi.object().keys({ name: Joi.string() @@ -77,6 +79,9 @@ const hivemindMediaWikiMetadata = () => { }); }; +const websiteMediaWikiMetadata = () => { + return Joi.object().keys({}); +}; const hivemindOptions = () => { return Joi.object().keys({ platforms: Joi.array().items( @@ -107,6 +112,10 @@ const hivemindOptions = () => { is: PlatformNames.MediaWiki, then: hivemindMediaWikiMetadata(), }, + { + is: PlatformNames.Website, + then: websiteMediaWikiMetadata(), + }, ], otherwise: Joi.any().forbidden(), }).required(), diff --git a/src/validations/platform.validation.ts b/src/validations/platform.validation.ts index 1ce249e5..002c1227 100644 --- a/src/validations/platform.validation.ts +++ b/src/validations/platform.validation.ts @@ -28,6 +28,12 @@ const discordUpdateMetadata = () => { analyzerStartedAt: Joi.date(), }); }; + +const websiteUpdateMetadata = () => { + return Joi.object().keys({ + resources: Joi.array().items(Joi.string().uri({ scheme: ['http', 'https'] })), + }); +}; const twitterMetadata = () => { return Joi.object().keys({ id: Joi.string().required(), @@ -92,6 +98,14 @@ const discourseMetadata = () => { }); }; +const websiteMetadata = () => { + return Joi.object().keys({ + resources: Joi.array() + .items(Joi.string().uri({ scheme: ['http', 'https'] })) + .required(), + }); +}; + const createPlatform = { body: Joi.object().keys({ name: Joi.string() @@ -130,7 +144,11 @@ const createPlatform = { }, { is: PlatformNames.Telegram, - then: telegramMetadata, + then: telegramMetadata(), + }, + { + is: PlatformNames.Website, + then: websiteMetadata(), }, ], }).required(), @@ -201,6 +219,16 @@ const dynamicUpdatePlatform = (req: Request) => { }), }; } + case PlatformNames.Website: { + return { + params: Joi.object().keys({ + platformId: Joi.required().custom(objectId), + }), + body: Joi.object().required().keys({ + metadata: websiteUpdateMetadata(), + }), + }; + } default: req.allowInput = false; return {};