Migrate legacy images

lib/mongo.ts

@@ -9,6 +9,7 @@ import Profile from "models/Profile";
 import City from "models/City";
 import Log from "models/Log";
 import RegionInfo from "models/RegionInfo";
+import LegacyImage from "models/LegacyImage";
 import Regions from "data/regions.json";
 import SyncRegions from "data/sync-regions.json";
 import {
@@ -613,20 +614,26 @@ export async function deleteHotspot(hotspot: HotspotType) {
 export const getHotspotImages = async (locationId: string) => {
   await connect();
 
-  const [ebirdImages, hotspot] = await Promise.all([
+  const [ebirdImages, hotspot, legacyImages] = await Promise.all([
     getBestImages(locationId as string),
     Hotspot.findOne({ locationId }, [
       "featuredImg",
-      "images",
       "featuredImg1",
       "featuredImg2",
       "featuredImg3",
       "featuredImg4",
     ]).lean(),
+    LegacyImage.find({
+      locationId,
+      type: "hotspot",
+      isMap: { $ne: true },
+      isMigrated: { $ne: true },
+    })
+      .sort({ order: 1, _id: 1 })
+      .lean(),
   ]);
 
   if (!hotspot) throw new Error("Hotspot not found");
-  const legacyImages = hotspot?.images?.filter((it) => !it.isMap && !it.isMigrated) || [];
 
   const { featuredImg1, featuredImg2, featuredImg3, featuredImg4 } = hotspot;
   const shouldShowEbirdImages = !featuredImg1;

lib/types.ts

@@ -6,7 +6,7 @@ export type Image = {
   _id?: string;
   xsUrl?: string;
   smUrl: string;
-  lgUrl: string;
+  lgUrl?: string;
   by?: string;
   email?: string;
   uid?: string;
@@ -28,6 +28,32 @@ export type Image = {
   isMigrated?: boolean;
 };
 
+export type LegacyImage = {
+  _id?: string;
+  locationId: string;
+  type: "hotspot" | "group" | "drive";
+  xsUrl?: string;
+  smUrl: string;
+  lgUrl?: string;
+  by?: string;
+  email?: string;
+  uid?: string;
+  isMap?: boolean;
+  isStreetview?: boolean;
+  isPublicDomain?: boolean;
+  width?: number;
+  height?: number;
+  size?: number;
+  caption?: string;
+  legacy?: boolean;
+  isNew?: boolean;
+  id?: string;
+  streetviewData?: any;
+  hideFromChildren?: boolean;
+  isMigrated?: boolean;
+  order?: number;
+};
+
 export type MlImage = {
   id: number;
   caption: string;

models/LegacyImage.ts

@@ -0,0 +1,45 @@
+import mongoose from "mongoose";
+const { Schema, model, models } = mongoose;
+import { LegacyImage as LegacyImageT } from "lib/types";
+
+const LegacyImageSchema = new Schema(
+  {
+    locationId: {
+      type: String,
+      required: true,
+    },
+    type: {
+      type: String,
+      enum: ["hotspot", "group"],
+      required: true,
+    },
+    xsUrl: String,
+    smUrl: {
+      type: String,
+      required: true,
+    },
+    lgUrl: String,
+    by: String,
+    email: String,
+    uid: String,
+    caption: String,
+    width: Number,
+    height: Number,
+    size: Number,
+    isMap: Boolean,
+    isStreetview: Boolean,
+    isPublicDomain: Boolean,
+    legacy: Boolean,
+    streetviewData: Object,
+    isMigrated: Boolean,
+    hideFromChildren: Boolean,
+    order: Number,
+  },
+  { timestamps: true }
+);
+
+LegacyImageSchema.index({ locationId: 1, type: 1, isMap: 1, isMigrated: 1, order: 1 });
+
+const LegacyImage = models.LegacyImage || model("LegacyImage", LegacyImageSchema);
+
+export default LegacyImage as mongoose.Model<LegacyImageT>;

package.json

@@ -17,7 +17,8 @@
     "export-hotspots-aggregated": "tsx ./scripts/export-hotspots-aggregated.ts",
     "export-hotspots-duplicated": "tsx ./scripts/export-hotspots-duplicated.ts",
     "export-sample-data": "tsx ./scripts/export-sample-data.ts",
-    "generate-mapkit-token": "tsx ./scripts/generate-mapkit-token.ts"
+    "generate-mapkit-token": "tsx ./scripts/generate-mapkit-token.ts",
+    "migrate-images": "tsx ./scripts/migrate-images.ts"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.367.0",

scripts/migrate-images.ts

@@ -0,0 +1,124 @@
+import mongoose from "mongoose";
+import * as dotenv from "dotenv";
+import Hotspot from "../models/Hotspot";
+import Group from "../models/Group";
+import LegacyImage from "../models/LegacyImage";
+
+type ModelName = "Hotspot" | "Group";
+type SourceIdField = "locationId";
+
+dotenv.config();
+
+const MODEL_TO_MIGRATE: ModelName = "Group";
+const BATCH_SIZE = 1000;
+
+const URI = process.env.MONGO_URI;
+const connect = async () => (URI ? mongoose.connect(URI) : null);
+
+type LegacyType = "hotspot" | "group";
+
+const MODEL_CONFIG: Record<
+  ModelName,
+  { model: mongoose.Model<any>; sourceIdField: SourceIdField; legacyType: LegacyType }
+> = {
+  Hotspot: { model: Hotspot, sourceIdField: "locationId", legacyType: "hotspot" },
+  Group: { model: Group, sourceIdField: "locationId", legacyType: "group" },
+};
+
+const migrateImages = async () => {
+  await connect();
+
+  const { model: Model, sourceIdField, legacyType } = MODEL_CONFIG[MODEL_TO_MIGRATE];
+
+  let lastCursor: string | null = null;
+  let totalDocs = 0;
+  let totalImages = 0;
+  let totalWrites = 0;
+  let totalSkipped = 0;
+
+  while (true) {
+    const query: any = {
+      images: { $exists: true, $ne: [] },
+      [sourceIdField]: { $exists: true, $ne: "" },
+    };
+    if (lastCursor) {
+      query[sourceIdField] = { $gt: lastCursor };
+    }
+
+    const docs = await Model.find(query)
+      .select({ images: 1, [sourceIdField]: 1 })
+      .sort({ [sourceIdField]: 1 })
+      .limit(BATCH_SIZE)
+      .lean();
+    if (!docs.length) {
+      break;
+    }
+
+    const legacyDocs: any[] = [];
+
+    for (const doc of docs) {
+      totalDocs += 1;
+      const sourceId = String(doc[sourceIdField]);
+      lastCursor = sourceId;
+
+      const images = Array.isArray(doc.images) ? doc.images : [];
+      images.forEach((image: any, index: number) => {
+        if (!!image?.ebirdId) {
+          totalSkipped += 1;
+          return;
+        }
+        if (!image?.smUrl) {
+          totalSkipped += 1;
+          return;
+        }
+
+        totalImages += 1;
+
+        const legacyDoc = {
+          locationId: sourceId,
+          type: legacyType,
+          xsUrl: image.xsUrl,
+          smUrl: image.smUrl,
+          lgUrl: image.lgUrl,
+          by: image.by,
+          email: image.email,
+          uid: image.uid,
+          caption: image.caption,
+          width: image.width,
+          height: image.height,
+          size: image.size,
+          isMap: legacyType === "group" ? true : image.isMap,
+          isStreetview: image.isStreetview,
+          isPublicDomain: image.isPublicDomain,
+          legacy: image.legacy,
+          streetviewData: image.streetviewData,
+          isMigrated: image.isMigrated,
+          hideFromChildren: image.hideFromChildren,
+          order: index,
+        };
+
+        legacyDocs.push(legacyDoc);
+      });
+    }
+
+    if (legacyDocs.length) {
+      try {
+        const inserted = await LegacyImage.insertMany(legacyDocs, { ordered: false });
+        totalWrites += inserted.length;
+      } catch (err: any) {
+        const insertedCount = err?.insertedDocs?.length ?? 0;
+        totalWrites += insertedCount;
+        console.warn(`InsertMany had errors; inserted ${insertedCount} docs in this batch.`);
+      }
+    }
+
+    console.log(
+      `Processed ${totalDocs} docs | migrated ${totalImages} images | skipped ${totalSkipped} | writes ${totalWrites}`
+    );
+  }
+
+  console.log("Done!");
+  process.exit();
+};
+
+migrateImages();