diff --git a/lib/mongo.ts b/lib/mongo.ts index f5bbe22..7f3b416 100644 --- a/lib/mongo.ts +++ b/lib/mongo.ts @@ -9,6 +9,7 @@ import Profile from "models/Profile"; import City from "models/City"; import Log from "models/Log"; import RegionInfo from "models/RegionInfo"; +import LegacyImage from "models/LegacyImage"; import Regions from "data/regions.json"; import SyncRegions from "data/sync-regions.json"; import { @@ -613,20 +614,26 @@ export async function deleteHotspot(hotspot: HotspotType) { export const getHotspotImages = async (locationId: string) => { await connect(); - const [ebirdImages, hotspot] = await Promise.all([ + const [ebirdImages, hotspot, legacyImages] = await Promise.all([ getBestImages(locationId as string), Hotspot.findOne({ locationId }, [ "featuredImg", - "images", "featuredImg1", "featuredImg2", "featuredImg3", "featuredImg4", ]).lean(), + LegacyImage.find({ + locationId, + type: "hotspot", + isMap: { $ne: true }, + isMigrated: { $ne: true }, + }) + .sort({ order: 1, _id: 1 }) + .lean(), ]); if (!hotspot) throw new Error("Hotspot not found"); - const legacyImages = hotspot?.images?.filter((it) => !it.isMap && !it.isMigrated) || []; const { featuredImg1, featuredImg2, featuredImg3, featuredImg4 } = hotspot; const shouldShowEbirdImages = !featuredImg1; diff --git a/lib/types.ts b/lib/types.ts index a311831..35b89b4 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -6,7 +6,7 @@ export type Image = { _id?: string; xsUrl?: string; smUrl: string; - lgUrl: string; + lgUrl?: string; by?: string; email?: string; uid?: string; @@ -28,6 +28,32 @@ export type Image = { isMigrated?: boolean; }; +export type LegacyImage = { + _id?: string; + locationId: string; + type: "hotspot" | "group" | "drive"; + xsUrl?: string; + smUrl: string; + lgUrl?: string; + by?: string; + email?: string; + uid?: string; + isMap?: boolean; + isStreetview?: boolean; + isPublicDomain?: boolean; + width?: number; + height?: number; + size?: number; + caption?: string; + legacy?: boolean; + isNew?: boolean; + id?: string; + streetviewData?: any; + hideFromChildren?: boolean; + isMigrated?: boolean; + order?: number; +}; + export type MlImage = { id: number; caption: string; diff --git a/models/LegacyImage.ts b/models/LegacyImage.ts new file mode 100644 index 0000000..6b51f36 --- /dev/null +++ b/models/LegacyImage.ts @@ -0,0 +1,45 @@ +import mongoose from "mongoose"; +const { Schema, model, models } = mongoose; +import { LegacyImage as LegacyImageT } from "lib/types"; + +const LegacyImageSchema = new Schema( + { + locationId: { + type: String, + required: true, + }, + type: { + type: String, + enum: ["hotspot", "group"], + required: true, + }, + xsUrl: String, + smUrl: { + type: String, + required: true, + }, + lgUrl: String, + by: String, + email: String, + uid: String, + caption: String, + width: Number, + height: Number, + size: Number, + isMap: Boolean, + isStreetview: Boolean, + isPublicDomain: Boolean, + legacy: Boolean, + streetviewData: Object, + isMigrated: Boolean, + hideFromChildren: Boolean, + order: Number, + }, + { timestamps: true } +); + +LegacyImageSchema.index({ locationId: 1, type: 1, isMap: 1, isMigrated: 1, order: 1 }); + +const LegacyImage = models.LegacyImage || model("LegacyImage", LegacyImageSchema); + +export default LegacyImage as mongoose.Model; diff --git a/package.json b/package.json index fe235c7..7437db3 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,8 @@ "export-hotspots-aggregated": "tsx ./scripts/export-hotspots-aggregated.ts", "export-hotspots-duplicated": "tsx ./scripts/export-hotspots-duplicated.ts", "export-sample-data": "tsx ./scripts/export-sample-data.ts", - "generate-mapkit-token": "tsx ./scripts/generate-mapkit-token.ts" + "generate-mapkit-token": "tsx ./scripts/generate-mapkit-token.ts", + "migrate-images": "tsx ./scripts/migrate-images.ts" }, "dependencies": { "@aws-sdk/client-s3": "^3.367.0", diff --git a/scripts/migrate-images.ts b/scripts/migrate-images.ts new file mode 100644 index 0000000..e370d1c --- /dev/null +++ b/scripts/migrate-images.ts @@ -0,0 +1,124 @@ +import mongoose from "mongoose"; +import * as dotenv from "dotenv"; +import Hotspot from "../models/Hotspot"; +import Group from "../models/Group"; +import LegacyImage from "../models/LegacyImage"; + +type ModelName = "Hotspot" | "Group"; +type SourceIdField = "locationId"; + +dotenv.config(); + +const MODEL_TO_MIGRATE: ModelName = "Group"; +const BATCH_SIZE = 1000; + +const URI = process.env.MONGO_URI; +const connect = async () => (URI ? mongoose.connect(URI) : null); + +type LegacyType = "hotspot" | "group"; + +const MODEL_CONFIG: Record< + ModelName, + { model: mongoose.Model; sourceIdField: SourceIdField; legacyType: LegacyType } +> = { + Hotspot: { model: Hotspot, sourceIdField: "locationId", legacyType: "hotspot" }, + Group: { model: Group, sourceIdField: "locationId", legacyType: "group" }, +}; + +const migrateImages = async () => { + await connect(); + + const { model: Model, sourceIdField, legacyType } = MODEL_CONFIG[MODEL_TO_MIGRATE]; + + let lastCursor: string | null = null; + let totalDocs = 0; + let totalImages = 0; + let totalWrites = 0; + let totalSkipped = 0; + + while (true) { + const query: any = { + images: { $exists: true, $ne: [] }, + [sourceIdField]: { $exists: true, $ne: "" }, + }; + if (lastCursor) { + query[sourceIdField] = { $gt: lastCursor }; + } + + const docs = await Model.find(query) + .select({ images: 1, [sourceIdField]: 1 }) + .sort({ [sourceIdField]: 1 }) + .limit(BATCH_SIZE) + .lean(); + if (!docs.length) { + break; + } + + const legacyDocs: any[] = []; + + for (const doc of docs) { + totalDocs += 1; + const sourceId = String(doc[sourceIdField]); + lastCursor = sourceId; + + const images = Array.isArray(doc.images) ? doc.images : []; + images.forEach((image: any, index: number) => { + if (!!image?.ebirdId) { + totalSkipped += 1; + return; + } + if (!image?.smUrl) { + totalSkipped += 1; + return; + } + + totalImages += 1; + + const legacyDoc = { + locationId: sourceId, + type: legacyType, + xsUrl: image.xsUrl, + smUrl: image.smUrl, + lgUrl: image.lgUrl, + by: image.by, + email: image.email, + uid: image.uid, + caption: image.caption, + width: image.width, + height: image.height, + size: image.size, + isMap: legacyType === "group" ? true : image.isMap, + isStreetview: image.isStreetview, + isPublicDomain: image.isPublicDomain, + legacy: image.legacy, + streetviewData: image.streetviewData, + isMigrated: image.isMigrated, + hideFromChildren: image.hideFromChildren, + order: index, + }; + + legacyDocs.push(legacyDoc); + }); + } + + if (legacyDocs.length) { + try { + const inserted = await LegacyImage.insertMany(legacyDocs, { ordered: false }); + totalWrites += inserted.length; + } catch (err: any) { + const insertedCount = err?.insertedDocs?.length ?? 0; + totalWrites += insertedCount; + console.warn(`InsertMany had errors; inserted ${insertedCount} docs in this batch.`); + } + } + + console.log( + `Processed ${totalDocs} docs | migrated ${totalImages} images | skipped ${totalSkipped} | writes ${totalWrites}` + ); + } + + console.log("Done!"); + process.exit(); +}; + +migrateImages();