Update workspace-tools and simplify hasher logic

change/change-979f3af9-c39b-4355-be59-58895bb8bd4e.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "type": "minor",
+      "comment": "Simplify internal logic, remove `@rushstack/package-deps-hash` dependency, and add exports for functions duplicated by lage",
+      "packageName": "backfill-hasher",
+      "email": "elcraig@microsoft.com",
+      "dependentChangeType": "patch"
+    }
+  ]
+}

change/change-c8a59fd6-80f6-4f9d-864d-afb7de5f225a.json

@@ -2,7 +2,7 @@
   "changes": [
     {
       "type": "patch",
-      "comment": "Update dependency workspace-tools to ^0.40.0",
+      "comment": "Update dependency workspace-tools to ^0.41.0",
       "packageName": "backfill-hasher",
       "email": "email not defined",
       "dependentChangeType": "patch"

packages/hasher/package.json

@@ -15,13 +15,10 @@
     "watch": "tsc -b -w"
   },
   "dependencies": {
-    "@rushstack/package-deps-hash": "^3.2.4",
     "backfill-logger": "^5.4.0",
-    "fs-extra": "^8.1.0",
-    "workspace-tools": "^0.40.0"
+    "workspace-tools": "^0.41.0"
   },
   "devDependencies": {
-    "@types/fs-extra": "^8.0.0",
     "@types/jest": "^30.0.0",
     "backfill-utils-test": "*",
     "backfill-utils-tsconfig": "*",

packages/hasher/src/Hasher.ts

@@ -1,83 +1,100 @@
-import { Logger } from "backfill-logger";
-import { findWorkspacePath, WorkspaceInfo } from "workspace-tools";
+import path from "path";
+import type { Logger } from "backfill-logger";
+import { type PackageInfos, findPackageRoot } from "workspace-tools";
 import { generateHashOfFiles } from "./hashOfFiles";
 import {
-  PackageHashInfo,
+  type PackageHashInfo,
   getPackageHash,
   generateHashOfInternalPackages,
 } from "./hashOfPackage";
-import { hashStrings, getPackageRoot } from "./helpers";
-import { RepoInfo, getRepoInfo, getRepoInfoNoCache } from "./repoInfo";
+import { hashStrings } from "./hashStrings";
+import { getRepoInfo, getRepoInfoNoCache } from "./repoInfo";
 
 export interface IHasher {
   createPackageHash: (salt: string) => Promise<string>;
   hashOfOutput: () => Promise<string>;
 }
 
-function isDone(done: PackageHashInfo[], packageName: string): boolean {
-  return Boolean(done.find(({ name }) => name === packageName));
-}
-
-function isInQueue(queue: string[], packagePath: string): boolean {
-  return queue.indexOf(packagePath) >= 0;
-}
-
-export function addToQueue(
-  dependencyNames: string[],
-  queue: string[],
-  done: PackageHashInfo[],
-  workspaces: WorkspaceInfo
-): void {
-  dependencyNames.forEach((name) => {
-    const dependencyPath = findWorkspacePath(workspaces, name);
-
-    if (dependencyPath) {
-      if (!isDone(done, name) && !isInQueue(queue, dependencyPath)) {
-        queue.push(dependencyPath);
-      }
+/**
+ * Add repo-internal dependencies to the queue, if not already done.
+ */
+export function _addToQueue(params: {
+  /** Dependency names (will be filtered to internal ones) */
+  dependencyNames: string[];
+  /** Package path queue */
+  queue: string[];
+  /** Packages that are already done */
+  done: PackageHashInfo[];
+  /** Package infos internal to the repo */
+  packageInfos: PackageInfos;
+}): void {
+  const { dependencyNames, queue, done, packageInfos } = params;
+
+  for (const dependencyName of dependencyNames) {
+    const dependencyInfo = packageInfos[dependencyName];
+    const dependencyPath =
+      dependencyInfo && path.dirname(dependencyInfo.packageJsonPath);
+
+    if (
+      dependencyPath &&
+      !done.some((p) => p.name === dependencyName) &&
+      !queue.includes(dependencyPath)
+    ) {
+      queue.push(dependencyPath);
     }
-  });
+  }
 }
 
 export class Hasher implements IHasher {
   private packageRoot: string;
-  private repoInfo?: RepoInfo;
 
   constructor(
-    private options: {
+    options: {
       packageRoot: string;
     },
     private logger: Logger
   ) {
-    this.packageRoot = this.options.packageRoot;
+    this.packageRoot = options.packageRoot;
   }
 
+  // TODO: This implementation is a bit odd... If the hasher is being reused for many packages,
+  // the repoInfo should be created once and passed in. Otherwise it's a massive perf penalty to
+  // recalculate for every package in a large repo. There's potentially also the opportunity to
+  // reduce the perf penalty by only getting hashes for relevant packages per recursively walking
+  // internal dependencies. (Fixing is lower priority since this is no longer used by lage.)
   public async createPackageHash(salt: string): Promise<string> {
     const tracer = this.logger.setTime("hashTime");
 
-    const packageRoot = await getPackageRoot(this.packageRoot);
+    // TODO: not sure why it's getting the root if this is already the root...?
+    const entryPackageRoot = findPackageRoot(this.packageRoot);
+    if (!entryPackageRoot) {
+      throw new Error(
+        `Could not find package.json inside ${this.packageRoot}.`
+      );
+    }
 
-    this.repoInfo = await getRepoInfo(packageRoot);
+    const repoInfo = await getRepoInfo(entryPackageRoot);
 
-    const { workspaceInfo } = this.repoInfo;
+    const { packageInfos } = repoInfo;
 
-    const queue = [packageRoot];
+    const queue: string[] = [entryPackageRoot];
     const done: PackageHashInfo[] = [];
 
     while (queue.length > 0) {
-      const nextPackageRoot = queue.shift();
-
-      if (!nextPackageRoot) {
-        continue;
-      }
+      const nextPackageRoot = queue.shift()!; // eslint-disable-line @typescript-eslint/no-non-null-assertion
 
       const packageHash = await getPackageHash(
         nextPackageRoot,
-        this.repoInfo,
+        repoInfo,
         this.logger
       );
 
-      addToQueue(packageHash.internalDependencies, queue, done, workspaceInfo);
+      _addToQueue({
+        dependencyNames: packageHash.internalDependencies,
+        queue,
+        done,
+        packageInfos,
+      });
 
       done.push(packageHash);
     }

packages/hasher/src/__tests__/Hasher.test.ts

@@ -1,62 +1,59 @@
 import path from "path";
 
-import { setupFixture } from "backfill-utils-test";
+import { removeTempDir, setupFixture } from "backfill-utils-test";
 import { makeLogger } from "backfill-logger";
 
-import { WorkspaceInfo } from "workspace-tools";
-import { PackageHashInfo } from "../hashOfPackage";
-import { Hasher, addToQueue } from "../Hasher";
+import { getPackageInfos } from "workspace-tools";
+import { Hasher, _addToQueue } from "../Hasher";
 
 const logger = makeLogger("mute");
 
-describe("addToQueue", () => {
-  const setupAddToQueue = async () => {
-    const packageRoot = await setupFixture("monorepo");
+type QueueParams = Parameters<typeof _addToQueue>[0];
 
-    const packageToAdd = "package-a";
-    const packagePath = path.join(packageRoot, "packages", packageToAdd);
-    const workspaces: WorkspaceInfo = [
-      {
-        name: packageToAdd,
-        path: packagePath,
-        packageJson: {
-          name: "",
-          packageJsonPath: "",
-          version: "",
-        },
-      },
-    ];
-    const internalDependencies = [packageToAdd];
+describe("_addToQueue", () => {
+  let root = "";
+
+  afterEach(() => {
+    root && removeTempDir(root);
+    root = "";
+  });
 
-    const queue: string[] = [];
-    const done: PackageHashInfo[] = [];
+  const initFixture = () => {
+    root = setupFixture("monorepo");
+
+    const packageToAdd = "package-a";
+    const packageInfos = getPackageInfos(root);
+    const packagePath = path.dirname(
+      packageInfos[packageToAdd].packageJsonPath
+    );
+
+    const queueParams: QueueParams = {
+      dependencyNames: [packageToAdd],
+      queue: [],
+      done: [],
+      packageInfos,
+    };
 
     return {
-      internalDependencies,
-      queue,
-      done,
-      workspaces,
+      queueParams,
       packageToAdd,
       packagePath,
     };
   };
 
   it("adds internal dependencies to the queue", async () => {
-    const { internalDependencies, queue, done, workspaces, packagePath } =
-      await setupAddToQueue();
+    const { queueParams, packagePath } = initFixture();
 
-    addToQueue(internalDependencies, queue, done, workspaces);
+    _addToQueue(queueParams);
 
-    const expectedQueue = [packagePath];
-    expect(queue).toEqual(expectedQueue);
+    expect(queueParams.queue).toEqual([packagePath]);
   });
 
   it("doesn't add to the queue if the package has been evaluated", async () => {
-    let { internalDependencies, queue, done, workspaces, packageToAdd } =
-      await setupAddToQueue();
+    const { queueParams, packageToAdd } = initFixture();
 
     // Override
-    done = [
+    queueParams.done = [
       {
         name: packageToAdd,
         filesHash: "",
@@ -65,28 +62,36 @@ describe("addToQueue", () => {
       },
     ];
 
-    addToQueue(internalDependencies, queue, done, workspaces);
+    _addToQueue(queueParams);
 
-    expect(queue).toEqual([]);
+    expect(queueParams.queue).toEqual([]);
   });
 
   it("doesn't add to the queue if the package is already in the queue", async () => {
-    let { internalDependencies, queue, done, workspaces, packagePath } =
-      await setupAddToQueue();
+    const { queueParams, packagePath } = initFixture();
 
     // Override
-    queue = [packagePath];
+    queueParams.queue = [packagePath];
 
-    addToQueue(internalDependencies, queue, done, workspaces);
+    _addToQueue(queueParams);
 
-    const expectedQueue = [packagePath];
-    expect(queue).toEqual(expectedQueue);
+    expect(queueParams.queue).toEqual([packagePath]);
   });
 });
 
-describe("The main Hasher class", () => {
+describe("Hasher", () => {
+  let roots: string[] = [];
+
+  afterEach(() => {
+    for (const root of roots) {
+      removeTempDir(root);
+    }
+    roots = [];
+  });
+
   const setupFixtureAndReturnHash = async (fixture = "monorepo") => {
-    const packageRoot = await setupFixture(fixture);
+    const packageRoot = setupFixture(fixture);
+    roots.push(packageRoot);
 
     const options = { packageRoot, outputGlob: ["lib/**"] };
     const buildSignature = "yarn build";