From 4c6172840bcf32734d8485c5caceeff4d4343431 Mon Sep 17 00:00:00 2001 From: Elizabeth Craig Date: Thu, 26 Feb 2026 15:24:37 -0800 Subject: [PATCH] Update workspace-tools and simplify hasher logic --- ...-979f3af9-c39b-4355-be59-58895bb8bd4e.json | 11 + ...-c8a59fd6-80f6-4f9d-864d-afb7de5f225a.json | 2 +- packages/hasher/package.json | 5 +- packages/hasher/src/Hasher.ts | 99 +++++--- packages/hasher/src/__tests__/Hasher.test.ts | 95 +++---- .../src/__tests__/createPackageHashes.test.ts | 68 ++--- .../hasher/src/__tests__/getFileHashes.ts | 233 ++++++++++++++++++ .../hasher/src/__tests__/hashOfFiles.test.ts | 111 ++++----- .../{helpers.test.ts => hashStrings.test.ts} | 2 +- .../hasher/src/__tests__/repoInfo.test.ts | 91 +++++++ .../__tests__/resolveDependenciesHelper.ts | 59 ----- .../resolveExternalDependencies.test.ts | 166 ++++--------- .../resolveInternalDependencies.test.ts | 160 ------------ packages/hasher/src/createPackageHashes.ts | 59 +++-- packages/hasher/src/getFileHashes.ts | 224 +++++++++++++++++ packages/hasher/src/hashOfFiles.ts | 19 +- packages/hasher/src/hashOfPackage.ts | 29 ++- packages/hasher/src/hashStrings.ts | 14 ++ packages/hasher/src/helpers.ts | 27 -- packages/hasher/src/index.ts | 3 + packages/hasher/src/repoInfo.ts | 90 +++---- .../hasher/src/resolveExternalDependencies.ts | 105 ++++---- .../hasher/src/resolveInternalDependencies.ts | 25 -- packages/hasher/src/types.ts | 23 ++ .../hasher-nested-test-project/package.json | 3 + .../hasher-nested-test-project/src/file 1.txt | 1 + .../hasher-test-project/file 2.txt | 1 + .../hasher-test-project/file1.txt | 1 + .../file\350\235\264\350\235\266.txt" | 1 + .../hasher-test-project/package.json | 3 + yarn.lock | 101 ++------ 31 files changed, 1017 insertions(+), 814 deletions(-) create mode 100644 change/change-979f3af9-c39b-4355-be59-58895bb8bd4e.json create mode 100644 packages/hasher/src/__tests__/getFileHashes.ts rename packages/hasher/src/__tests__/{helpers.test.ts => hashStrings.test.ts} (93%) create mode 100644 packages/hasher/src/__tests__/repoInfo.test.ts delete mode 100644 packages/hasher/src/__tests__/resolveDependenciesHelper.ts delete mode 100644 packages/hasher/src/__tests__/resolveInternalDependencies.test.ts create mode 100644 packages/hasher/src/getFileHashes.ts create mode 100644 packages/hasher/src/hashStrings.ts delete mode 100644 packages/hasher/src/helpers.ts delete mode 100644 packages/hasher/src/resolveInternalDependencies.ts create mode 100644 packages/hasher/src/types.ts create mode 100644 packages/utils-test/__fixtures__/hasher-nested-test-project/package.json create mode 100644 packages/utils-test/__fixtures__/hasher-nested-test-project/src/file 1.txt create mode 100644 packages/utils-test/__fixtures__/hasher-test-project/file 2.txt create mode 100644 packages/utils-test/__fixtures__/hasher-test-project/file1.txt create mode 100644 "packages/utils-test/__fixtures__/hasher-test-project/file\350\235\264\350\235\266.txt" create mode 100644 packages/utils-test/__fixtures__/hasher-test-project/package.json diff --git a/change/change-979f3af9-c39b-4355-be59-58895bb8bd4e.json b/change/change-979f3af9-c39b-4355-be59-58895bb8bd4e.json new file mode 100644 index 00000000..a407ea62 --- /dev/null +++ b/change/change-979f3af9-c39b-4355-be59-58895bb8bd4e.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "type": "minor", + "comment": "Simplify internal logic, remove `@rushstack/package-deps-hash` dependency, and add exports for functions duplicated by lage", + "packageName": "backfill-hasher", + "email": "elcraig@microsoft.com", + "dependentChangeType": "patch" + } + ] +} \ No newline at end of file diff --git a/change/change-c8a59fd6-80f6-4f9d-864d-afb7de5f225a.json b/change/change-c8a59fd6-80f6-4f9d-864d-afb7de5f225a.json index f40a8d0c..8401da28 100644 --- a/change/change-c8a59fd6-80f6-4f9d-864d-afb7de5f225a.json +++ b/change/change-c8a59fd6-80f6-4f9d-864d-afb7de5f225a.json @@ -2,7 +2,7 @@ "changes": [ { "type": "patch", - "comment": "Update dependency workspace-tools to ^0.40.0", + "comment": "Update dependency workspace-tools to ^0.41.0", "packageName": "backfill-hasher", "email": "email not defined", "dependentChangeType": "patch" diff --git a/packages/hasher/package.json b/packages/hasher/package.json index 8365292b..32e80329 100644 --- a/packages/hasher/package.json +++ b/packages/hasher/package.json @@ -15,13 +15,10 @@ "watch": "tsc -b -w" }, "dependencies": { - "@rushstack/package-deps-hash": "^3.2.4", "backfill-logger": "^5.4.0", - "fs-extra": "^8.1.0", - "workspace-tools": "^0.40.0" + "workspace-tools": "^0.41.0" }, "devDependencies": { - "@types/fs-extra": "^8.0.0", "@types/jest": "^30.0.0", "backfill-utils-test": "*", "backfill-utils-tsconfig": "*", diff --git a/packages/hasher/src/Hasher.ts b/packages/hasher/src/Hasher.ts index 196ffe28..a9c28d61 100644 --- a/packages/hasher/src/Hasher.ts +++ b/packages/hasher/src/Hasher.ts @@ -1,83 +1,100 @@ -import { Logger } from "backfill-logger"; -import { findWorkspacePath, WorkspaceInfo } from "workspace-tools"; +import path from "path"; +import type { Logger } from "backfill-logger"; +import { type PackageInfos, findPackageRoot } from "workspace-tools"; import { generateHashOfFiles } from "./hashOfFiles"; import { - PackageHashInfo, + type PackageHashInfo, getPackageHash, generateHashOfInternalPackages, } from "./hashOfPackage"; -import { hashStrings, getPackageRoot } from "./helpers"; -import { RepoInfo, getRepoInfo, getRepoInfoNoCache } from "./repoInfo"; +import { hashStrings } from "./hashStrings"; +import { getRepoInfo, getRepoInfoNoCache } from "./repoInfo"; export interface IHasher { createPackageHash: (salt: string) => Promise; hashOfOutput: () => Promise; } -function isDone(done: PackageHashInfo[], packageName: string): boolean { - return Boolean(done.find(({ name }) => name === packageName)); -} - -function isInQueue(queue: string[], packagePath: string): boolean { - return queue.indexOf(packagePath) >= 0; -} - -export function addToQueue( - dependencyNames: string[], - queue: string[], - done: PackageHashInfo[], - workspaces: WorkspaceInfo -): void { - dependencyNames.forEach((name) => { - const dependencyPath = findWorkspacePath(workspaces, name); - - if (dependencyPath) { - if (!isDone(done, name) && !isInQueue(queue, dependencyPath)) { - queue.push(dependencyPath); - } +/** + * Add repo-internal dependencies to the queue, if not already done. + */ +export function _addToQueue(params: { + /** Dependency names (will be filtered to internal ones) */ + dependencyNames: string[]; + /** Package path queue */ + queue: string[]; + /** Packages that are already done */ + done: PackageHashInfo[]; + /** Package infos internal to the repo */ + packageInfos: PackageInfos; +}): void { + const { dependencyNames, queue, done, packageInfos } = params; + + for (const dependencyName of dependencyNames) { + const dependencyInfo = packageInfos[dependencyName]; + const dependencyPath = + dependencyInfo && path.dirname(dependencyInfo.packageJsonPath); + + if ( + dependencyPath && + !done.some((p) => p.name === dependencyName) && + !queue.includes(dependencyPath) + ) { + queue.push(dependencyPath); } - }); + } } export class Hasher implements IHasher { private packageRoot: string; - private repoInfo?: RepoInfo; constructor( - private options: { + options: { packageRoot: string; }, private logger: Logger ) { - this.packageRoot = this.options.packageRoot; + this.packageRoot = options.packageRoot; } + // TODO: This implementation is a bit odd... If the hasher is being reused for many packages, + // the repoInfo should be created once and passed in. Otherwise it's a massive perf penalty to + // recalculate for every package in a large repo. There's potentially also the opportunity to + // reduce the perf penalty by only getting hashes for relevant packages per recursively walking + // internal dependencies. (Fixing is lower priority since this is no longer used by lage.) public async createPackageHash(salt: string): Promise { const tracer = this.logger.setTime("hashTime"); - const packageRoot = await getPackageRoot(this.packageRoot); + // TODO: not sure why it's getting the root if this is already the root...? + const entryPackageRoot = findPackageRoot(this.packageRoot); + if (!entryPackageRoot) { + throw new Error( + `Could not find package.json inside ${this.packageRoot}.` + ); + } - this.repoInfo = await getRepoInfo(packageRoot); + const repoInfo = await getRepoInfo(entryPackageRoot); - const { workspaceInfo } = this.repoInfo; + const { packageInfos } = repoInfo; - const queue = [packageRoot]; + const queue: string[] = [entryPackageRoot]; const done: PackageHashInfo[] = []; while (queue.length > 0) { - const nextPackageRoot = queue.shift(); - - if (!nextPackageRoot) { - continue; - } + const nextPackageRoot = queue.shift()!; // eslint-disable-line @typescript-eslint/no-non-null-assertion const packageHash = await getPackageHash( nextPackageRoot, - this.repoInfo, + repoInfo, this.logger ); - addToQueue(packageHash.internalDependencies, queue, done, workspaceInfo); + _addToQueue({ + dependencyNames: packageHash.internalDependencies, + queue, + done, + packageInfos, + }); done.push(packageHash); } diff --git a/packages/hasher/src/__tests__/Hasher.test.ts b/packages/hasher/src/__tests__/Hasher.test.ts index 7335cdbc..f96c6419 100644 --- a/packages/hasher/src/__tests__/Hasher.test.ts +++ b/packages/hasher/src/__tests__/Hasher.test.ts @@ -1,62 +1,59 @@ import path from "path"; -import { setupFixture } from "backfill-utils-test"; +import { removeTempDir, setupFixture } from "backfill-utils-test"; import { makeLogger } from "backfill-logger"; -import { WorkspaceInfo } from "workspace-tools"; -import { PackageHashInfo } from "../hashOfPackage"; -import { Hasher, addToQueue } from "../Hasher"; +import { getPackageInfos } from "workspace-tools"; +import { Hasher, _addToQueue } from "../Hasher"; const logger = makeLogger("mute"); -describe("addToQueue", () => { - const setupAddToQueue = async () => { - const packageRoot = await setupFixture("monorepo"); +type QueueParams = Parameters[0]; - const packageToAdd = "package-a"; - const packagePath = path.join(packageRoot, "packages", packageToAdd); - const workspaces: WorkspaceInfo = [ - { - name: packageToAdd, - path: packagePath, - packageJson: { - name: "", - packageJsonPath: "", - version: "", - }, - }, - ]; - const internalDependencies = [packageToAdd]; +describe("_addToQueue", () => { + let root = ""; + + afterEach(() => { + root && removeTempDir(root); + root = ""; + }); - const queue: string[] = []; - const done: PackageHashInfo[] = []; + const initFixture = () => { + root = setupFixture("monorepo"); + + const packageToAdd = "package-a"; + const packageInfos = getPackageInfos(root); + const packagePath = path.dirname( + packageInfos[packageToAdd].packageJsonPath + ); + + const queueParams: QueueParams = { + dependencyNames: [packageToAdd], + queue: [], + done: [], + packageInfos, + }; return { - internalDependencies, - queue, - done, - workspaces, + queueParams, packageToAdd, packagePath, }; }; it("adds internal dependencies to the queue", async () => { - const { internalDependencies, queue, done, workspaces, packagePath } = - await setupAddToQueue(); + const { queueParams, packagePath } = initFixture(); - addToQueue(internalDependencies, queue, done, workspaces); + _addToQueue(queueParams); - const expectedQueue = [packagePath]; - expect(queue).toEqual(expectedQueue); + expect(queueParams.queue).toEqual([packagePath]); }); it("doesn't add to the queue if the package has been evaluated", async () => { - let { internalDependencies, queue, done, workspaces, packageToAdd } = - await setupAddToQueue(); + const { queueParams, packageToAdd } = initFixture(); // Override - done = [ + queueParams.done = [ { name: packageToAdd, filesHash: "", @@ -65,28 +62,36 @@ describe("addToQueue", () => { }, ]; - addToQueue(internalDependencies, queue, done, workspaces); + _addToQueue(queueParams); - expect(queue).toEqual([]); + expect(queueParams.queue).toEqual([]); }); it("doesn't add to the queue if the package is already in the queue", async () => { - let { internalDependencies, queue, done, workspaces, packagePath } = - await setupAddToQueue(); + const { queueParams, packagePath } = initFixture(); // Override - queue = [packagePath]; + queueParams.queue = [packagePath]; - addToQueue(internalDependencies, queue, done, workspaces); + _addToQueue(queueParams); - const expectedQueue = [packagePath]; - expect(queue).toEqual(expectedQueue); + expect(queueParams.queue).toEqual([packagePath]); }); }); -describe("The main Hasher class", () => { +describe("Hasher", () => { + let roots: string[] = []; + + afterEach(() => { + for (const root of roots) { + removeTempDir(root); + } + roots = []; + }); + const setupFixtureAndReturnHash = async (fixture = "monorepo") => { - const packageRoot = await setupFixture(fixture); + const packageRoot = setupFixture(fixture); + roots.push(packageRoot); const options = { packageRoot, outputGlob: ["lib/**"] }; const buildSignature = "yarn build"; diff --git a/packages/hasher/src/__tests__/createPackageHashes.test.ts b/packages/hasher/src/__tests__/createPackageHashes.test.ts index 93f2fb5a..aeaaa240 100644 --- a/packages/hasher/src/__tests__/createPackageHashes.test.ts +++ b/packages/hasher/src/__tests__/createPackageHashes.test.ts @@ -1,50 +1,50 @@ +import path from "path"; import { createPackageHashes } from "../createPackageHashes"; describe("createPackageHashes", () => { it("creates packages hashes for repo hashes", () => { const packageHashes = createPackageHashes( - "/repo", - [ - { - path: "/repo/packages/package-a", + // Normalize paths for the OS + path.resolve("/repo"), + { + "package-a": { name: "package-a", - packageJson: { - name: "package-a", - packageJsonPath: "/packages/package-a/package.json", - version: "1.0.0", - }, + packageJsonPath: path.resolve( + "/repo/packages/package-a/package.json" + ), + version: "1.0.0", }, - { - path: "/repo/packages/package-b", + "package-b": { name: "package-b", - packageJson: { - name: "package-b", - packageJsonPath: "/packages/package-b/package.json", - version: "1.0.0", - }, + packageJsonPath: path.resolve( + "/repo/packages/package-b/package.json" + ), + version: "1.0.0", }, - ], + }, { - "packages/package-a/foo.ts": "hash-a-foo.ts", - "packages/package-a/package.json": "hash-a-package.json", - "packages/package-b/1.ts": "hash-b-1.ts", - "packages/package-b/2.ts": "hash-b-2.ts", - "packages/package-b/3.ts": "hash-b-3.ts", + // RepoHashes keys have forward slashes, and the values would be real hashes + "packages/package-a/foo.ts": "hash-a-foo-ts", + "packages/package-a/package.json": "hash-a-package-json", + "packages/package-b/1.ts": "hash-b-1-ts", + "packages/package-b/2.ts": "hash-b-2-ts", + "packages/package-b/3.ts": "hash-b-3-ts", + "other-file.js": "hash-other-file-js", } ); - expect(packageHashes["packages/package-a"].length).toEqual(2); - - // packageHashes["packageName"] is an array of tuples of the form [filePath, hash] - expect(packageHashes["packages/package-a"][0][1]).toEqual("hash-a-foo.ts"); - expect(packageHashes["packages/package-a"][1][1]).toEqual( - "hash-a-package.json" - ); - - expect(packageHashes["packages/package-b"].length).toEqual(3); // packageHashes["packageName"] is an array of tuples of the form [filePath, hash] - expect(packageHashes["packages/package-b"][0][1]).toEqual("hash-b-1.ts"); - expect(packageHashes["packages/package-b"][1][1]).toEqual("hash-b-2.ts"); - expect(packageHashes["packages/package-b"][2][1]).toEqual("hash-b-3.ts"); + expect(packageHashes).toEqual({ + "packages/package-a": [ + ["packages/package-a/foo.ts", "hash-a-foo-ts"], + ["packages/package-a/package.json", "hash-a-package-json"], + ], + "packages/package-b": [ + ["packages/package-b/1.ts", "hash-b-1-ts"], + ["packages/package-b/2.ts", "hash-b-2-ts"], + ["packages/package-b/3.ts", "hash-b-3-ts"], + ], + "": [["other-file.js", "hash-other-file-js"]], + }); }); }); diff --git a/packages/hasher/src/__tests__/getFileHashes.ts b/packages/hasher/src/__tests__/getFileHashes.ts new file mode 100644 index 00000000..bcc15328 --- /dev/null +++ b/packages/hasher/src/__tests__/getFileHashes.ts @@ -0,0 +1,233 @@ +import fs from "fs"; +import path from "path"; +import { removeTempDir, setupFixture } from "backfill-utils-test"; +import { + getFileHashes, + _parseGitFilename, + _parseGitLsTree, +} from "../getFileHashes"; + +describe(_parseGitFilename.name, () => { + it("can parse backslash-escaped filenames", () => { + expect(_parseGitFilename("some/path/to/a/file name")).toEqual( + "some/path/to/a/file name" + ); + expect(_parseGitFilename('"some/path/to/a/file?name"')).toEqual( + "some/path/to/a/file?name" + ); + expect(_parseGitFilename('"some/path/to/a/file\\\\name"')).toEqual( + "some/path/to/a/file\\name" + ); + expect(_parseGitFilename('"some/path/to/a/file\\"name"')).toEqual( + 'some/path/to/a/file"name' + ); + expect(_parseGitFilename('"some/path/to/a/file\\"name"')).toEqual( + 'some/path/to/a/file"name' + ); + expect( + _parseGitFilename( + '"some/path/to/a/file\\347\\275\\221\\347\\275\\221name"' + ) + ).toEqual("some/path/to/a/file网网name"); + expect( + _parseGitFilename('"some/path/to/a/file\\\\347\\\\\\347\\275\\221name"') + ).toEqual("some/path/to/a/file\\347\\网name"); + expect( + _parseGitFilename( + '"some/path/to/a/file\\\\\\347\\275\\221\\347\\275\\221name"' + ) + ).toEqual("some/path/to/a/file\\网网name"); + }); +}); + +describe(_parseGitLsTree.name, () => { + it("can handle a blob", () => { + const filename = "src/typings/tsd.d.ts"; + const hash = "3451bccdc831cb43d7a70ed8e628dcf9c7f888c8"; + + const output = `100644 blob ${hash}\t${filename}`; + const changes = _parseGitLsTree(output); + + expect(changes).toEqual({ [filename]: hash }); + }); + + it("can handle a submodule", () => { + const filename = "rushstack"; + const hash = "c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac"; + + const output = `160000 commit ${hash}\t${filename}`; + const changes = _parseGitLsTree(output); + + expect(changes).toEqual({ [filename]: hash }); + }); + + it("can handle multiple lines", () => { + const filename1 = "src/typings/tsd.d.ts"; + const hash1 = "3451bccdc831cb43d7a70ed8e628dcf9c7f888c8"; + + const filename2 = "src/foo bar/tsd.d.ts"; + const hash2 = "0123456789abcdef1234567890abcdef01234567"; + + const output = `100644 blob ${hash1}\t${filename1}\n100666 blob ${hash2}\t${filename2}`; + const changes = _parseGitLsTree(output); + + expect(changes).toEqual({ [filename1]: hash1, [filename2]: hash2 }); + }); + + it("throws with malformed input", () => { + expect( + _parseGitLsTree.bind(undefined, "some super malformed input") + ).toThrow(); + }); +}); + +describe(getFileHashes.name, () => { + let root = ""; + + afterEach(() => { + root && removeTempDir(root); + root = ""; + }); + + it("can parse committed file", async () => { + root = setupFixture("hasher-test-project"); + + const results = getFileHashes(root); + expect(results).toEqual({ + "file1.txt": "c7b2f707ac99ca522f965210a7b6b0b109863f34", + "file 2.txt": "a385f754ec4fede884a4864d090064d9aeef8ccb", + "file蝴蝶.txt": "ae814af81e16cb2ae8c57503c77e2cab6b5462ba", + "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", + }); + }); + + it("can handle files in subfolders", async () => { + root = setupFixture("hasher-nested-test-project"); + + const results = getFileHashes(root); + expect(results).toEqual({ + // This should use a forward slash + "src/file 1.txt": "c7b2f707ac99ca522f965210a7b6b0b109863f34", + "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", + }); + }); + + it("can handle adding one file", async () => { + root = setupFixture("hasher-test-project"); + + const tempFilePath = path.join(root, "a.txt"); + + fs.writeFileSync(tempFilePath, "a"); + + const results = getFileHashes(root); + expect(results).toEqual({ + "a.txt": "2e65efe2a145dda7ee51d1741299f848e5bf752e", + "file1.txt": "c7b2f707ac99ca522f965210a7b6b0b109863f34", + "file 2.txt": "a385f754ec4fede884a4864d090064d9aeef8ccb", + "file蝴蝶.txt": "ae814af81e16cb2ae8c57503c77e2cab6b5462ba", + "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", + }); + }); + + it("can handle adding two files", async () => { + root = setupFixture("hasher-test-project"); + + const tempFilePath1 = path.join(root, "a.txt"); + const tempFilePath2 = path.join(root, "b.txt"); + + fs.writeFileSync(tempFilePath1, "a"); + fs.writeFileSync(tempFilePath2, "a"); + + const results = getFileHashes(root); + expect(results).toEqual({ + "a.txt": "2e65efe2a145dda7ee51d1741299f848e5bf752e", + "b.txt": "2e65efe2a145dda7ee51d1741299f848e5bf752e", + "file1.txt": "c7b2f707ac99ca522f965210a7b6b0b109863f34", + "file 2.txt": "a385f754ec4fede884a4864d090064d9aeef8ccb", + "file蝴蝶.txt": "ae814af81e16cb2ae8c57503c77e2cab6b5462ba", + "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", + }); + }); + + it("can handle removing one file", async () => { + root = setupFixture("hasher-test-project"); + + const testFilePath = path.join(root, "file1.txt"); + + fs.rmSync(testFilePath); + + const results = getFileHashes(root); + expect(results).toEqual({ + "file 2.txt": "a385f754ec4fede884a4864d090064d9aeef8ccb", + "file蝴蝶.txt": "ae814af81e16cb2ae8c57503c77e2cab6b5462ba", + "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", + }); + }); + + it("can handle changing one file", async () => { + root = setupFixture("hasher-test-project"); + + const testFilePath = path.join(root, "file1.txt"); + + fs.writeFileSync(testFilePath, "abc"); + + const results = getFileHashes(root); + expect(results).toEqual({ + "file1.txt": "f2ba8f84ab5c1bce84a7b441cb1959cfc7093b7f", + "file 2.txt": "a385f754ec4fede884a4864d090064d9aeef8ccb", + "file蝴蝶.txt": "ae814af81e16cb2ae8c57503c77e2cab6b5462ba", + "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", + }); + }); + + it("can handle a filename with spaces", async () => { + root = setupFixture("hasher-test-project"); + + const tempFilePath = path.join(root, "a file.txt"); + + fs.writeFileSync(tempFilePath, "a"); + + const results = getFileHashes(root); + expect(results).toEqual({ + "file1.txt": "c7b2f707ac99ca522f965210a7b6b0b109863f34", + "file 2.txt": "a385f754ec4fede884a4864d090064d9aeef8ccb", + "file蝴蝶.txt": "ae814af81e16cb2ae8c57503c77e2cab6b5462ba", + "a file.txt": "2e65efe2a145dda7ee51d1741299f848e5bf752e", + "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", + }); + }); + + it("can handle a filename with multiple spaces", async () => { + root = setupFixture("hasher-test-project"); + + const tempFilePath = path.join(root, "a file name.txt"); + + fs.writeFileSync(tempFilePath, "a"); + + const results = getFileHashes(root); + expect(results).toEqual({ + "file1.txt": "c7b2f707ac99ca522f965210a7b6b0b109863f34", + "file 2.txt": "a385f754ec4fede884a4864d090064d9aeef8ccb", + "file蝴蝶.txt": "ae814af81e16cb2ae8c57503c77e2cab6b5462ba", + "a file name.txt": "2e65efe2a145dda7ee51d1741299f848e5bf752e", + "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", + }); + }); + + it("can handle a filename with non-standard characters", async () => { + root = setupFixture("hasher-test-project"); + + const tempFilePath = path.join(root, "newFile批把.txt"); + + fs.writeFileSync(tempFilePath, "a"); + + const results = getFileHashes(root); + expect(results).toEqual({ + "file1.txt": "c7b2f707ac99ca522f965210a7b6b0b109863f34", + "file 2.txt": "a385f754ec4fede884a4864d090064d9aeef8ccb", + "file蝴蝶.txt": "ae814af81e16cb2ae8c57503c77e2cab6b5462ba", + "newFile批把.txt": "2e65efe2a145dda7ee51d1741299f848e5bf752e", + "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", + }); + }); +}); diff --git a/packages/hasher/src/__tests__/hashOfFiles.test.ts b/packages/hasher/src/__tests__/hashOfFiles.test.ts index 750db6f8..20b0a9bb 100644 --- a/packages/hasher/src/__tests__/hashOfFiles.test.ts +++ b/packages/hasher/src/__tests__/hashOfFiles.test.ts @@ -1,63 +1,58 @@ import path from "path"; -import fs from "fs-extra"; +import fs from "fs"; -import { setupFixture } from "backfill-utils-test"; +import { removeTempDir, setupFixture } from "backfill-utils-test"; import { generateHashOfFiles } from "../hashOfFiles"; import { getRepoInfoNoCache } from "../repoInfo"; describe("generateHashOfFiles()", () => { - it("creates different hashes for different hashes", async () => { - const packageRoot = await setupFixture("monorepo"); - let repoInfo = await getRepoInfoNoCache(packageRoot); + let root = ""; - const hashOfPackage = await generateHashOfFiles(packageRoot, repoInfo); + afterEach(() => { + root && removeTempDir(root); + root = ""; + }); - fs.writeFileSync(path.join(packageRoot, "foo.txt"), "bar"); - repoInfo = await getRepoInfoNoCache(packageRoot); + it("creates different hashes for different contents", async () => { + root = setupFixture("monorepo"); + let repoInfo = await getRepoInfoNoCache(root); - const hashOfPackageWithFoo = await generateHashOfFiles( - packageRoot, - repoInfo - ); + const hashOfPackage = await generateHashOfFiles(root, repoInfo); + + fs.writeFileSync(path.join(root, "foo.txt"), "bar"); + repoInfo = await getRepoInfoNoCache(root); + + const hashOfPackageWithFoo = await generateHashOfFiles(root, repoInfo); expect(hashOfPackage).not.toEqual(hashOfPackageWithFoo); - fs.writeFileSync(path.join(packageRoot, "foo.txt"), "foo"); - repoInfo = await getRepoInfoNoCache(packageRoot); - const hashOfPackageWithFoo2 = await generateHashOfFiles( - packageRoot, - repoInfo - ); + fs.writeFileSync(path.join(root, "foo.txt"), "foo"); + repoInfo = await getRepoInfoNoCache(root); + const hashOfPackageWithFoo2 = await generateHashOfFiles(root, repoInfo); expect(hashOfPackageWithFoo).not.toEqual(hashOfPackageWithFoo2); - fs.unlinkSync(path.join(packageRoot, "foo.txt")); - repoInfo = await getRepoInfoNoCache(packageRoot); - const hashOfPackageWithoutFoo = await generateHashOfFiles( - packageRoot, - repoInfo - ); + fs.unlinkSync(path.join(root, "foo.txt")); + repoInfo = await getRepoInfoNoCache(root); + const hashOfPackageWithoutFoo = await generateHashOfFiles(root, repoInfo); expect(hashOfPackage).toEqual(hashOfPackageWithoutFoo); }); it("is not confused by package names being substring of other packages", async () => { - const packageRoot = await setupFixture("monorepo"); + root = setupFixture("monorepo"); - let repoInfo = await getRepoInfoNoCache(packageRoot); + let repoInfo = await getRepoInfoNoCache(root); const hashOfPackageA = await generateHashOfFiles( - path.join(packageRoot, "packages", "package-a"), + path.join(root, "packages", "package-a"), repoInfo ); - await fs.mkdir(path.join(packageRoot, "packages", "package-abc")); - await fs.writeFile( - path.join(packageRoot, "packages", "package-abc", "foo"), - "bar" - ); + fs.mkdirSync(path.join(root, "packages", "package-abc")); + fs.writeFileSync(path.join(root, "packages", "package-abc", "foo"), "bar"); - repoInfo = await getRepoInfoNoCache(packageRoot); + repoInfo = await getRepoInfoNoCache(root); const newHashOfPackageA = await generateHashOfFiles( - path.join(packageRoot, "packages", "package-a"), + path.join(root, "packages", "package-a"), repoInfo ); @@ -65,56 +60,50 @@ describe("generateHashOfFiles()", () => { }); it("file paths are included in hash", async () => { - const packageRoot = await setupFixture("empty"); + root = setupFixture("empty"); - fs.writeFileSync(path.join(packageRoot, "foo.txt"), "bar"); - let repoInfo = await getRepoInfoNoCache(packageRoot); + fs.writeFileSync(path.join(root, "foo.txt"), "bar"); + let repoInfo = await getRepoInfoNoCache(root); - const hashOfPackageWithFoo = await generateHashOfFiles( - packageRoot, - repoInfo - ); + const hashOfPackageWithFoo = await generateHashOfFiles(root, repoInfo); - fs.unlinkSync(path.join(packageRoot, "foo.txt")); - fs.writeFileSync(path.join(packageRoot, "bar.txt"), "bar"); - repoInfo = await getRepoInfoNoCache(packageRoot); + fs.unlinkSync(path.join(root, "foo.txt")); + fs.writeFileSync(path.join(root, "bar.txt"), "bar"); + repoInfo = await getRepoInfoNoCache(root); - const hashOfPackageWithBar = await generateHashOfFiles( - packageRoot, - repoInfo - ); + const hashOfPackageWithBar = await generateHashOfFiles(root, repoInfo); expect(hashOfPackageWithFoo).not.toEqual(hashOfPackageWithBar); }); // This test will be run on Windows and on Linux on the CI it("file paths are consistent across platforms", async () => { - const packageRoot = await setupFixture("empty"); + root = setupFixture("empty"); // Create a folder to make sure we get folder separators as part of the file name - const folder = path.join(packageRoot, "foo"); + const folder = path.join(root, "foo"); - fs.mkdirpSync(folder); + fs.mkdirSync(folder, { recursive: true }); fs.writeFileSync(path.join(folder, "foo.txt"), "bar"); - const repoInfo = await getRepoInfoNoCache(packageRoot); + const repoInfo = await getRepoInfoNoCache(root); - const hashOfPackage = await generateHashOfFiles(packageRoot, repoInfo); + const hashOfPackage = await generateHashOfFiles(root, repoInfo); expect(hashOfPackage).toEqual("4d4ca2ecc436e1198554f5d03236ea8f956ac0c4"); }); // This test will be run on Windows and on Linux on the CI - it("file paths in a package not defined in a workspace (malformed monorepo) are consistent across platforms (uses slow path)", async () => { - const workspaceRoot = await setupFixture("empty"); + it("file paths not defined in a package (malformed monorepo) are consistent across platforms (uses slow path)", async () => { + root = setupFixture("empty"); // Create a folder to make sure we get folder separators as part of the file name - const folder = path.join(workspaceRoot, "packages", "foo"); + const folder = path.join(root, "packages", "foo"); - fs.mkdirpSync(folder); + fs.mkdirSync(folder, { recursive: true }); fs.writeFileSync(path.join(folder, "foo.txt"), "bar"); - const repoInfo = await getRepoInfoNoCache(workspaceRoot); + const repoInfo = await getRepoInfoNoCache(root); const hashOfPackage = await generateHashOfFiles(folder, repoInfo); @@ -122,12 +111,12 @@ describe("generateHashOfFiles()", () => { }); it("file paths in a monorepo are consistent across platforms (uses fast path)", async () => { - const workspaceRoot = await setupFixture("monorepo"); + root = setupFixture("monorepo"); - const folder = path.join(workspaceRoot, "packages", "package-a"); + const folder = path.join(root, "packages", "package-a"); fs.writeFileSync(path.join(folder, "foo.txt"), "bar"); - const repoInfo = await getRepoInfoNoCache(workspaceRoot); + const repoInfo = await getRepoInfoNoCache(root); const hashOfPackage = await generateHashOfFiles(folder, repoInfo); diff --git a/packages/hasher/src/__tests__/helpers.test.ts b/packages/hasher/src/__tests__/hashStrings.test.ts similarity index 93% rename from packages/hasher/src/__tests__/helpers.test.ts rename to packages/hasher/src/__tests__/hashStrings.test.ts index 534ec9ac..e2df2878 100644 --- a/packages/hasher/src/__tests__/helpers.test.ts +++ b/packages/hasher/src/__tests__/hashStrings.test.ts @@ -1,4 +1,4 @@ -import { hashStrings } from "../helpers"; +import { hashStrings } from "../hashStrings"; describe("hashStrings()", () => { it("creates different hashes given different lists", () => { diff --git a/packages/hasher/src/__tests__/repoInfo.test.ts b/packages/hasher/src/__tests__/repoInfo.test.ts new file mode 100644 index 00000000..d5e56a0b --- /dev/null +++ b/packages/hasher/src/__tests__/repoInfo.test.ts @@ -0,0 +1,91 @@ +import { removeTempDir, setupFixture } from "backfill-utils-test"; +import { getRepoInfo } from "../repoInfo"; + +describe("getRepoInfo()", () => { + let root = ""; + + afterEach(() => { + root && removeTempDir(root); + root = ""; + }); + + it("works on a monorepo", async () => { + root = setupFixture("monorepo"); + + const repoInfo = await getRepoInfo(root); + // This is essentially a snapshot of the results for the whole fixture + expect(repoInfo).toEqual({ + packageHashes: { + "": [ + ["package.json", "93fb6d8d4fdf54a913d0ebc60425ed12bd5784e2"], + ["yarn.lock", "525969d57151bd6826e7cc743fa1451e7646c807"], + ], + "packages/package-a": [ + [ + "packages/package-a/node_modules/.bin/copy", + "5c079d52564adf9932e18d7486d5b00d2e34f59a", + ], + [ + "packages/package-a/package.json", + "fab1e7352e3df61824d457a83a6a3ef1fbf1a7a5", + ], + [ + "packages/package-a/src/index.ts", + "85ce559e8f22b7dee6a5ed4be983fcafbeef9c72", + ], + ], + "packages/package-b": [ + [ + "packages/package-b/node_modules/.bin/copy", + "5c079d52564adf9932e18d7486d5b00d2e34f59a", + ], + [ + "packages/package-b/package.json", + "83913856d9a5efa4a33cfb41825b8d8fee0c71a6", + ], + [ + "packages/package-b/src/index.ts", + "85ce559e8f22b7dee6a5ed4be983fcafbeef9c72", + ], + ], + }, + packageInfos: { + // This is basic workspace-tools logic that doesn't need to be tested + "package-a": expect.anything(), + "package-b": expect.anything(), + }, + parsedLock: { + // This is also from workspace-tools. The parsing logic is reliable in this basic example + // with yarn, but less so with pnpm or scenarios with peer deps. + object: { + "foo@1.0.0": { + dependencies: { bar: "^1.0.0" }, + version: "1.0.0", + }, + }, + type: "success", + }, + repoHashes: { + "package.json": "93fb6d8d4fdf54a913d0ebc60425ed12bd5784e2", + "packages/package-a/node_modules/.bin/copy": + "5c079d52564adf9932e18d7486d5b00d2e34f59a", + "packages/package-a/package.json": + "fab1e7352e3df61824d457a83a6a3ef1fbf1a7a5", + "packages/package-a/src/index.ts": + "85ce559e8f22b7dee6a5ed4be983fcafbeef9c72", + "packages/package-b/node_modules/.bin/copy": + "5c079d52564adf9932e18d7486d5b00d2e34f59a", + "packages/package-b/package.json": + "83913856d9a5efa4a33cfb41825b8d8fee0c71a6", + "packages/package-b/src/index.ts": + "85ce559e8f22b7dee6a5ed4be983fcafbeef9c72", + "yarn.lock": "525969d57151bd6826e7cc743fa1451e7646c807", + }, + root, + }); + + // Keys are sorted using basic ordering + const repoHashesKeys = Object.keys(repoInfo.repoHashes); + expect(repoHashesKeys).toEqual([...repoHashesKeys].sort()); + }); +}); diff --git a/packages/hasher/src/__tests__/resolveDependenciesHelper.ts b/packages/hasher/src/__tests__/resolveDependenciesHelper.ts deleted file mode 100644 index a947e38a..00000000 --- a/packages/hasher/src/__tests__/resolveDependenciesHelper.ts +++ /dev/null @@ -1,59 +0,0 @@ -/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ -import { setupFixture } from "backfill-utils-test"; - -import { - getYarnWorkspaces, - getPnpmWorkspaces, - getRushWorkspaces, -} from "workspace-tools"; - -export async function filterDependenciesInYarnFixture( - fixture: string, - filterFunction: any -) { - const packageRoot = await setupFixture(fixture); - const workspacesPackageInfo = getYarnWorkspaces(packageRoot); - - const dependencies = { "package-a": "1.0.0", foo: "1.0.0" }; - - const filteredDependencies = filterFunction( - dependencies, - workspacesPackageInfo - ); - - return filteredDependencies; -} - -export async function filterDependenciesInPnpmFixture( - fixture: string, - filterFunction: any -) { - const packageRoot = await setupFixture(fixture); - const workspacesPackageInfo = getPnpmWorkspaces(packageRoot); - - const dependencies = { "package-a": "1.0.0", foo: "1.0.0" }; - - const filteredDependencies = filterFunction( - dependencies, - workspacesPackageInfo - ); - - return filteredDependencies; -} - -export async function filterDependenciesInRushFixture( - fixture: string, - filterFunction: any -) { - const packageRoot = await setupFixture(fixture); - const workspacesPackageInfo = getRushWorkspaces(packageRoot); - - const dependencies = { "package-a": "1.0.0", foo: "1.0.0" }; - - const filteredDependencies = filterFunction( - dependencies, - workspacesPackageInfo - ); - - return filteredDependencies; -} diff --git a/packages/hasher/src/__tests__/resolveExternalDependencies.test.ts b/packages/hasher/src/__tests__/resolveExternalDependencies.test.ts index 20b60f84..6be2bd5f 100644 --- a/packages/hasher/src/__tests__/resolveExternalDependencies.test.ts +++ b/packages/hasher/src/__tests__/resolveExternalDependencies.test.ts @@ -1,147 +1,89 @@ -import { setupFixture } from "backfill-utils-test"; +import { setupFixture, removeTempDir } from "backfill-utils-test"; +import { parseLockFile, getPackageInfos } from "workspace-tools"; import { - getPnpmWorkspaces, - getRushWorkspaces, - getYarnWorkspaces, - parseLockFile, -} from "workspace-tools"; - -import { - filterExternalDependencies, resolveExternalDependencies, - addToQueue, + _addToQueue, + type DependencySpec, + type DependencyQueue, } from "../resolveExternalDependencies"; -import { filterDependenciesInYarnFixture } from "./resolveDependenciesHelper"; - -describe("filterExternalDependencies()", () => { - it("only lists external dependencies", async () => { - const results = await filterDependenciesInYarnFixture( - "monorepo", - filterExternalDependencies - ); - expect(results).toEqual({ foo: "1.0.0" }); - }); - it("identifies all dependencies as external packages if there are no workspaces", async () => { - const results = await filterDependenciesInYarnFixture( - "basic", - filterExternalDependencies - ); - expect(results).toEqual({ foo: "1.0.0", "package-a": "1.0.0" }); - }); -}); - -describe("addToQueue()", () => { +describe("_addToQueue", () => { it("adds external dependencies to queue", () => { const externalDependencies = { foo: "1.0.0" }; - const done: string[] = []; - const queue: [string, string][] = []; + const done = new Set(); + const queue: DependencyQueue = []; - addToQueue(externalDependencies, done, queue); + _addToQueue(externalDependencies, done, queue); - const expectedQueue = [["foo", "1.0.0"]]; - expect(queue).toEqual(expectedQueue); + expect(queue).toEqual([["foo", "1.0.0"]]); }); it("doesn't add to the queue if the dependency has been visited", () => { const externalDependencies = { foo: "1.0.0" }; - const done: string[] = ["foo@1.0.0"]; - const queue: [string, string][] = []; + const done = new Set(["foo@1.0.0"]); + const queue: DependencyQueue = []; - addToQueue(externalDependencies, done, queue); + _addToQueue(externalDependencies, done, queue); expect(queue).toEqual([]); }); it("doesn't add to queue if the dependency is already in the queue", () => { const externalDependencies = { foo: "1.0.0" }; - const done: string[] = []; - const queue: [string, string][] = [["foo", "1.0.0"]]; + const done: Set = new Set(); + const queue: DependencyQueue = [["foo", "1.0.0"]]; - addToQueue(externalDependencies, done, queue); + _addToQueue(externalDependencies, done, queue); - const expectedQueue = [["foo", "1.0.0"]]; - expect(queue).toEqual(expectedQueue); + expect(queue).toEqual([["foo", "1.0.0"]]); }); }); -describe("resolveExternalDependencies() - yarn", () => { - it("given a list of external dependencies and a parsed Lock file, add all dependencies, transitively", async () => { - const packageRoot = await setupFixture("monorepo"); - const workspaces = getYarnWorkspaces(packageRoot); - - const allDependencies = { "package-a": "1.0.0", foo: "1.0.0" }; - const parsedLockFile = await parseLockFile(packageRoot); +describe("resolveExternalDependencies", () => { + let root = ""; - const resolvedDependencies = resolveExternalDependencies( - allDependencies, - workspaces, - parsedLockFile - ); - - expect(resolvedDependencies).toEqual(["foo@1.0.0", "bar@^1.0.0"]); + afterEach(() => { + root && removeTempDir(root); + root = ""; }); -}); - -describe("resolveExternalDependencies() - pnpm", () => { - it("given a list of external dependencies and a parsed Lock file, add all dependencies, transitively", async () => { - const packageRoot = await setupFixture("monorepo-pnpm"); - const workspaces = getPnpmWorkspaces(packageRoot); - - const allDependencies = { - "package-a": "1.0.0", - once: "1.4.0", - }; - const parsedLockFile = await parseLockFile(packageRoot); - - const resolvedDependencies = resolveExternalDependencies( - allDependencies, - workspaces, - parsedLockFile - ); - - expect(resolvedDependencies).toEqual(["once@1.4.0", "wrappy@1.0.2"]); - }); -}); - -describe("resolveExternalDependencies() - rush+pnpm", () => { - it("given a list of external dependencies and a parsed Lock file, add all dependencies, transitively", async () => { - const packageRoot = await setupFixture("monorepo-rush-pnpm"); - const workspaces = getRushWorkspaces(packageRoot); - - const allDependencies = { - "package-a": "1.0.0", - once: "1.4.0", - }; - const parsedLockFile = await parseLockFile(packageRoot); - - const resolvedDependencies = resolveExternalDependencies( - allDependencies, - workspaces, - parsedLockFile - ); - - expect(resolvedDependencies).toEqual(["once@1.4.0", "wrappy@1.0.2"]); - }); -}); - -describe("resolveExternalDependencies() - rush+yarn", () => { - it("given a list of external dependencies and a parsed Lock file, add all dependencies, transitively", async () => { - const packageRoot = await setupFixture("monorepo-rush-yarn"); - const workspaces = getRushWorkspaces(packageRoot); - const allDependencies = { - "package-a": "1.0.0", - once: "1.4.0", - }; - const parsedLockFile = await parseLockFile(packageRoot); + it.each<{ + manager: "yarn" | "pnpm" | "rush"; + name: string; + fixture: string; + // only specified if different than the most common case + allDependencies?: Record; + expected?: DependencySpec[]; + }>([ + { + manager: "yarn", + name: "yarn", + fixture: "monorepo", + allDependencies: { "package-a": "1.0.0", foo: "1.0.0" }, + expected: ["foo@1.0.0", "bar@^1.0.0"], + }, + { manager: "pnpm", name: "pnpm", fixture: "monorepo-pnpm" }, + { manager: "rush", name: "rush+pnpm", fixture: "monorepo-rush-pnpm" }, + { manager: "rush", name: "rush+yarn", fixture: "monorepo-rush-yarn" }, + ])("transitively adds all external dependencies ($name)", async (params) => { + const { + fixture, + // These are used in most of the test cases + allDependencies = { "package-a": "1.0.0", once: "1.4.0" }, + expected = ["once@1.4.0", "wrappy@1.0.2"], + } = params; + + root = setupFixture(fixture); + + const parsedLockFile = await parseLockFile(root); + const packageInfos = getPackageInfos(root, params.manager); const resolvedDependencies = resolveExternalDependencies( allDependencies, - workspaces, + packageInfos, parsedLockFile ); - expect(resolvedDependencies).toEqual(["once@1.4.0", "wrappy@1.0.2"]); + expect(resolvedDependencies).toEqual(expected); }); }); diff --git a/packages/hasher/src/__tests__/resolveInternalDependencies.test.ts b/packages/hasher/src/__tests__/resolveInternalDependencies.test.ts deleted file mode 100644 index 6d553faf..00000000 --- a/packages/hasher/src/__tests__/resolveInternalDependencies.test.ts +++ /dev/null @@ -1,160 +0,0 @@ -import { setupFixture } from "backfill-utils-test"; -import { - getPnpmWorkspaces, - getRushWorkspaces, - getYarnWorkspaces, -} from "workspace-tools"; - -import { - filterInternalDependencies, - resolveInternalDependencies, -} from "../resolveInternalDependencies"; -import { - filterDependenciesInYarnFixture, - filterDependenciesInPnpmFixture, - filterDependenciesInRushFixture, -} from "./resolveDependenciesHelper"; - -describe("filterInternalDependencies() for yarn", () => { - it("only lists internal dependencies", async () => { - const results = await filterDependenciesInYarnFixture( - "monorepo", - filterInternalDependencies - ); - - expect(results).toEqual(["package-a"]); - }); - - it("lists no internal packages if there are no workspaces", async () => { - const results = await filterDependenciesInYarnFixture( - "basic", - filterInternalDependencies - ); - - expect(results).toEqual([]); - }); -}); - -describe("resolveInternalDependencies() for yarn", () => { - it("adds internal dependency names to the processedPackages list", async () => { - const packageRoot = await setupFixture("monorepo"); - const workspaces = getYarnWorkspaces(packageRoot); - - const dependencies = { "package-a": "1.0.0", foo: "1.0.0" }; - - const resolvedDependencies = resolveInternalDependencies( - dependencies, - workspaces - ); - - expect(resolvedDependencies).toEqual(["package-a"]); - }); -}); - -describe("filterInternalDependencies() for pnpm", () => { - it("only lists internal dependencies", async () => { - const results = await filterDependenciesInPnpmFixture( - "monorepo-pnpm", - filterInternalDependencies - ); - - expect(results).toEqual(["package-a"]); - }); - - it("lists no internal packages if there are no workspaces", async () => { - const results = await filterDependenciesInPnpmFixture( - "basic", - filterInternalDependencies - ); - - expect(results).toEqual([]); - }); -}); - -describe("resolveInternalDependencies() for pnpm", () => { - it("adds internal dependency names to the processedPackages list", async () => { - const packageRoot = await setupFixture("monorepo-pnpm"); - const workspaces = getPnpmWorkspaces(packageRoot); - - const dependencies = { "package-a": "1.0.0", foo: "1.0.0" }; - - const resolvedDependencies = resolveInternalDependencies( - dependencies, - workspaces - ); - - expect(resolvedDependencies).toEqual(["package-a"]); - }); -}); - -describe("filterInternalDependencies() for rush+pnpm", () => { - it("only lists internal dependencies", async () => { - const results = await filterDependenciesInRushFixture( - "monorepo-rush-pnpm", - filterInternalDependencies - ); - - expect(results).toEqual(["package-a"]); - }); - - it("lists no internal packages if there are no workspaces", async () => { - const results = await filterDependenciesInRushFixture( - "basic", - filterInternalDependencies - ); - - expect(results).toEqual([]); - }); -}); - -describe("resolveInternalDependencies() for rush+pnpm", () => { - it("adds internal dependency names to the processedPackages list", async () => { - const packageRoot = await setupFixture("monorepo-rush-pnpm"); - const workspaces = getRushWorkspaces(packageRoot); - - const dependencies = { "package-a": "1.0.0", foo: "1.0.0" }; - - const resolvedDependencies = resolveInternalDependencies( - dependencies, - workspaces - ); - - expect(resolvedDependencies).toEqual(["package-a"]); - }); -}); - -describe("filterInternalDependencies() for rush+yarn", () => { - it("only lists internal dependencies", async () => { - const results = await filterDependenciesInRushFixture( - "monorepo-rush-yarn", - filterInternalDependencies - ); - - expect(results).toEqual(["package-a"]); - }); - - it("lists no internal packages if there are no workspaces", async () => { - const results = await filterDependenciesInRushFixture( - "basic", - filterInternalDependencies - ); - - expect(results).toEqual([]); - }); -}); - -describe("resolveInternalDependencies() for rush+yarn", () => { - it("adds internal dependency names to the processedPackages list", async () => { - const packageRoot = await setupFixture("monorepo-rush-yarn"); - const workspaces = getRushWorkspaces(packageRoot); - - const dependencies = { "package-a": "1.0.0", foo: "1.0.0" }; - - const resolvedDependencies = resolveInternalDependencies( - dependencies, - workspaces - ); - - expect(resolvedDependencies).toEqual(["package-a"]); - }); -}); diff --git a/packages/hasher/src/createPackageHashes.ts b/packages/hasher/src/createPackageHashes.ts index 3fcec33a..6dbb3990 100644 --- a/packages/hasher/src/createPackageHashes.ts +++ b/packages/hasher/src/createPackageHashes.ts @@ -1,51 +1,62 @@ import path from "path"; -import { WorkspaceInfo } from "workspace-tools"; +import type { PackageInfos } from "workspace-tools"; +import type { PackageHashes, RepoHashes } from "./types"; +/** + * This is a trie that looks like this: + * { + * "packages": { + * "experiences": { + * "react-web-client": {} + * } + * } + * } + */ +interface PathNode { + [key: string]: PathNode; +} + +/** + * Reformat `repoHashes` into a mapping of hashes for files in each package. + * @param root Repo root + * @param packageInfos Repo packages + * @param repoHashes Mapping from relative file path in the repo to its hash + * @returns Mapping from repo-relative package path to list of hashes for files in the package, + * in the form `[repo-relative path, hash]` (all with forward slashes) + */ export function createPackageHashes( root: string, - workspaceInfo: WorkspaceInfo, - repoHashes: { [key: string]: string } -): Record { - /** - * This is a trie that looks like this: - * { - * "packages": { - * "experiences": { - * "react-web-client": {} - * } - * } - * } - */ - interface PathNode { - [key: string]: PathNode; - } - + packageInfos: PackageInfos, + repoHashes: RepoHashes +): PackageHashes { const pathTree: PathNode = {}; // Generate path tree of all packages in workspace (scale: ~2000 * ~3) - for (const workspace of workspaceInfo) { - const pathParts = path.relative(root, workspace.path).split(/[\\/]/); + for (const packageInfo of Object.values(packageInfos)) { + const pathParts = path + .relative(root, path.dirname(packageInfo.packageJsonPath)) + .split(/[\\/]/); let currentNode = pathTree; for (const part of pathParts) { - currentNode[part] = currentNode[part] || {}; + currentNode[part] ??= {}; currentNode = currentNode[part]; } } // key: path/to/package (packageRoot), value: array of a tuple of [file, hash] - const packageHashes: Record = {}; + const packageHashes: PackageHashes = {}; for (const [entry, value] of Object.entries(repoHashes)) { const pathParts = entry.split(/[\\/]/); let node = pathTree; - const packagePathParts = []; + const packagePathParts: string[] = []; for (const part of pathParts) { if (node[part]) { - node = node[part] as PathNode; + node = node[part]; packagePathParts.push(part); } else { break; diff --git a/packages/hasher/src/getFileHashes.ts b/packages/hasher/src/getFileHashes.ts new file mode 100644 index 00000000..e66014e9 --- /dev/null +++ b/packages/hasher/src/getFileHashes.ts @@ -0,0 +1,224 @@ +// This file is based on @rushstack/package-deps-hash getPackageDeps but avoids its extra dependencies +// and clarifies naming and docs. The original usage was intended for a single package, but the way +// it's historically been used by backfill is to get hashes for the entire repo's files. +// https://github.com/microsoft/rushstack/blob/main/libraries/package-deps-hash/src/getPackageDeps.ts + +import * as path from "path"; +import { git } from "workspace-tools"; + +/** + * Parses a quoted filename sourced from the output of the "git status" command. + * + * Paths with non-standard characters will be enclosed with double-quotes, and non-standard + * characters will be backslash escaped (ex. double-quotes, non-ASCII characters). The + * escaped chars can be included in one of two ways: + * - backslash-escaped chars (ex. `\"`) + * - octal encoded chars (ex. `\347`) + * + * See documentation: https://git-scm.com/docs/git-status + */ +export function _parseGitFilename(filename: string): string { + // If there are no double-quotes around the string, then there are no escaped characters + // to decode, so just return + if (!(filename[0] === '"' && filename.endsWith('"'))) { + return filename; + } + + // Need to hex encode '%' since we will be decoding the converted octal values from hex + filename = filename.replace(/%/g, "%25"); + // Replace all instances of octal literals with percent-encoded hex (ex. '\347\275\221' -> '%E7%BD%91'). + // This is done because the octal literals represent UTF-8 bytes, and by converting them to percent-encoded + // hex, we can use decodeURIComponent to get the Unicode chars. + filename = filename.replace( + /(?:\\(\d{1,3}))/g, + (match, ...[octalValue, index, source]) => { + // We need to make sure that the backslash is intended to escape the octal value. To do this, walk + // backwards from the match to ensure that it's already escaped. + const trailingBackslashes: RegExpMatchArray | null = (source as string) + .slice(0, index as number) + .match(/\\*$/); + return trailingBackslashes?.length && + trailingBackslashes[0].length % 2 === 0 + ? `%${parseInt(octalValue, 8).toString(16)}` + : match; + } + ); + + // Finally, decode the filename and unescape the escaped UTF-8 chars + return JSON.parse(decodeURIComponent(filename)); +} + +/** + * Parses the output of the "git ls-tree" command + */ +export function _parseGitLsTree(output: string): Record { + const changes: Record = {}; + + if (!output) { + return changes; + } + + // A line is expected to look like: + // 100644 blob 3451bccdc831cb43d7a70ed8e628dcf9c7f888c8 src/typings/tsd.d.ts + // 160000 commit c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac rushstack + const gitRegex = /([0-9]{6})\s(blob|commit)\s([a-f0-9]{40})\s*(.*)/; + + // Note: The output of git ls-tree uses \n newlines regardless of OS. + const outputLines = output.trim().split("\n"); + for (const line of outputLines) { + if (!line) continue; + + // Take everything after the "100644 blob", which is just the hash and filename + const matches = line.match(gitRegex); + if (matches && matches[3] && matches[4]) { + const hash: string = matches[3]; + const filename: string = _parseGitFilename(matches[4]); + + changes[filename] = hash; + } else { + throw new Error(`Cannot parse git ls-tree input: "${line}"`); + } + } + + return changes; +} + +/** + * Parses the output of the "git status" command + */ +function _parseGitStatus(output: string): Map { + const changes = new Map(); + + /* + * Typically, output will look something like: + * M temp_modules/rush-package-deps-hash/package.json + * D package-deps-hash/src/index.ts + */ + + // If there was an issue with `git ls-tree`, or there are no current changes, processOutputBlocks[1] + // will be empty or undefined + if (!output) { + return changes; + } + + // Note: The output of git hash-object uses \n newlines regardless of OS. + const outputLines = output.trim().split("\n"); + for (const line of outputLines) { + /* + * changeType is in the format of "XY" where "X" is the status of the file in the index and "Y" is the status of + * the file in the working tree. Some example statuses: + * - 'D' == deletion + * - 'M' == modification + * - 'A' == addition + * - '??' == untracked + * - 'R' == rename + * - 'RM' == rename with modifications + * - '[MARC]D' == deleted in work tree + * Full list of examples: https://git-scm.com/docs/git-status#_short_format + */ + const match = line.match(/("(\\"|[^"])+")|(\S+\s*)/g); + + if (match && match.length > 1) { + const [changeType, ...filenameMatches] = match; + + // We always care about the last filename in the filenames array. In the case of non-rename changes, + // the filenames array only contains one file, so we can join all segments that were split on spaces. + // In the case of rename changes, the last item in the array is the path to the file in the working tree, + // which is the only one that we care about. It is also surrounded by double-quotes if spaces are + // included, so no need to worry about joining different segments + let lastFilename: string = changeType.startsWith("R") + ? filenameMatches[filenameMatches.length - 1] + : filenameMatches.join(""); + lastFilename = _parseGitFilename(lastFilename); + + changes.set(lastFilename, changeType.trimRight()); + } + } + + return changes; +} + +/** + * Takes a list of files and returns the current git hashes for them + */ +function _getGitHashForFiles( + filesToHash: string[], + cwd: string +): Map { + const changes = new Map(); + + if (!filesToHash.length) { + return changes; + } + + // Use --stdin-paths arg to pass the list of files to git in order to avoid issues with + // command length + const result = git(["hash-object", "--stdin-paths"], { + input: filesToHash.map((x) => path.resolve(cwd, x)).join("\n"), + throwOnError: true, + cwd, + }); + + // The result of "git hash-object" will be a list of file hashes delimited by newlines + const hashes = result.stdout.trim().split("\n"); + + if (hashes.length !== filesToHash.length) { + throw new Error( + `Passed ${filesToHash.length} file paths to Git to hash, but received ${hashes.length} hashes.` + ); + } + + for (let i = 0; i < hashes.length; i++) { + changes.set(filesToHash[i], hashes[i]); + } + + return changes; +} + +/** + * Builds an object containing hashes (`git hash-object`) for the files under `cwd`. + * This includes tracked and untracked files, but not ignored files. + * @param cwd - Include hashes of files under this folder + * @returns Mapping from file path relative to `cwd` (forward slashes) to hash + */ +export function getFileHashes(cwd: string): Record { + const gitLsOutput = git(["ls-tree", "HEAD", "-r"], { + cwd, + throwOnError: true, + }).stdout; + + // Add all the checked in hashes + const result = _parseGitLsTree(gitLsOutput); + + // Update the checked in hashes with the current repo status. + // -s - Short format. Will be printed as 'XY PATH' or 'XY ORIG_PATH -> PATH'. Paths with non-standard + // characters will be escaped using double-quotes, and non-standard characters will be backslash + // escaped (ex. spaces, tabs, double-quotes) + // -u - Untracked files are included + const gitStatusOutput = git(["status", "-s", "-u", "."], { + cwd, + throwOnError: true, + }).stdout; + + const currentlyChangedFiles = _parseGitStatus(gitStatusOutput); + const filesToHash: string[] = []; + + for (const [filename, changeType] of currentlyChangedFiles) { + // See comments inside parseGitStatus() for more information + if ( + changeType === "D" || + (changeType.length === 2 && changeType.charAt(1) === "D") + ) { + delete result[filename]; + } else { + filesToHash.push(filename); + } + } + + const currentlyChangedFileHashes = _getGitHashForFiles(filesToHash, cwd); + for (const [filename, hash] of currentlyChangedFileHashes) { + result[filename] = hash; + } + + return result; +} diff --git a/packages/hasher/src/hashOfFiles.ts b/packages/hasher/src/hashOfFiles.ts index 62adebfd..cd8d8d8a 100644 --- a/packages/hasher/src/hashOfFiles.ts +++ b/packages/hasher/src/hashOfFiles.ts @@ -1,6 +1,6 @@ -import path, { sep } from "path"; -import { hashStrings } from "./helpers"; -import { RepoInfo } from "./repoInfo"; +import path from "path"; +import { hashStrings } from "./hashStrings"; +import type { RepoInfo } from "./types"; /** * Generates a hash string based on files in a package @@ -9,8 +9,6 @@ import { RepoInfo } from "./repoInfo"; * in the repo, caching this result so repeated calls to this function will be * a simple lookup. * - * Note: We have to force the types because globby types are wrong - * * @param packageRoot The root of the package * @param repoInfo The repoInfo that carries information about repo-wide hashes */ @@ -32,13 +30,12 @@ export async function generateHashOfFiles( } } else { // Slow old path: if files are not clearly inside a package (mostly the case for malformed monorepos, like tests) - const normalized = path.normalize(packageRoot) + sep; - - const files: string[] = Object.keys(repoHashes).filter((f) => - path.join(root, f).includes(normalized) - ); + const normalized = path.normalize(packageRoot) + path.sep; - files.sort((a, b) => a.localeCompare(b)); + const files = Object.keys(repoHashes) + .filter((f) => path.join(root, f).includes(normalized)) + // Sort to ensure consistent ordering/hashing (use basic sorting since locale correctness doesn't matter) + .sort(); for (const file of files) { hashes.push(file, repoHashes[file]); diff --git a/packages/hasher/src/hashOfPackage.ts b/packages/hasher/src/hashOfPackage.ts index 09650971..4ff837d8 100644 --- a/packages/hasher/src/hashOfPackage.ts +++ b/packages/hasher/src/hashOfPackage.ts @@ -1,14 +1,13 @@ import crypto from "crypto"; import path from "path"; -import { Logger } from "backfill-logger"; -import { resolveInternalDependencies } from "./resolveInternalDependencies"; +import type { Logger } from "backfill-logger"; import { resolveExternalDependencies, - Dependencies, + type Dependencies, } from "./resolveExternalDependencies"; import { generateHashOfFiles } from "./hashOfFiles"; -import { hashStrings } from "./helpers"; -import { RepoInfo } from "./repoInfo"; +import { hashStrings } from "./hashStrings"; +import type { RepoInfo } from "./types"; export type PackageHashInfo = { name: string; @@ -20,14 +19,15 @@ export type PackageHashInfo = { export function generateHashOfInternalPackages( internalPackages: PackageHashInfo[] ): string { - internalPackages.sort((a, b) => a.name.localeCompare(b.name)); + // Sort to ensure consistent ordering/hashing (use basic sorting since locale correctness doesn't matter) + internalPackages = [...internalPackages].sort(); const hasher = crypto.createHash("sha1"); - internalPackages.forEach((pkg) => { + for (const pkg of internalPackages) { hasher.update(pkg.name); hasher.update(pkg.filesHash); hasher.update(pkg.dependenciesHash); - }); + } return hasher.digest("hex"); } @@ -39,7 +39,7 @@ export async function getPackageHash( repoInfo: RepoInfo, logger: Logger ): Promise { - const { workspaceInfo, parsedLock } = repoInfo; + const { packageInfos, parsedLock } = repoInfo; const memoizationKey = path.resolve(packageRoot); @@ -56,20 +56,19 @@ export async function getPackageHash( ...devDependencies, }; - const internalDependencies = resolveInternalDependencies( - allDependencies, - workspaceInfo + const internalDependencies = Object.keys(allDependencies).filter( + (dependency) => packageInfos[dependency] ); - const externalDeoendencies = resolveExternalDependencies( + const externalDependencies = resolveExternalDependencies( allDependencies, - workspaceInfo, + packageInfos, parsedLock ); const resolvedDependencies = [ ...internalDependencies, - ...externalDeoendencies, + ...externalDependencies, ]; const filesHash = await generateHashOfFiles(packageRoot, repoInfo); diff --git a/packages/hasher/src/hashStrings.ts b/packages/hasher/src/hashStrings.ts new file mode 100644 index 00000000..6d916c22 --- /dev/null +++ b/packages/hasher/src/hashStrings.ts @@ -0,0 +1,14 @@ +import crypto from "crypto"; + +export function hashStrings(strings: string | string[]): string { + const hasher = crypto.createHash("sha1"); + + // Sort to ensure consistent ordering/hashing (use basic sorting since locale correctness doesn't matter) + const elements = + typeof strings === "string" ? [strings] : [...strings].sort(); + for (const element of elements) { + hasher.update(element); + } + + return hasher.digest("hex"); +} diff --git a/packages/hasher/src/helpers.ts b/packages/hasher/src/helpers.ts deleted file mode 100644 index 63940c6e..00000000 --- a/packages/hasher/src/helpers.ts +++ /dev/null @@ -1,27 +0,0 @@ -import crypto from "crypto"; -import { findPackageRoot } from "workspace-tools"; - -export function hashStrings(strings: string | string[]): string { - const hasher = crypto.createHash("sha1"); - - const anArray = typeof strings === "string" ? [strings] : strings; - const elements = [...anArray]; - elements.sort((a, b) => a.localeCompare(b)); - elements.forEach((element) => hasher.update(element)); - - return hasher.digest("hex"); -} - -export async function getPackageRoot(cwd: string): Promise { - const packageRoot = findPackageRoot(cwd); - - if (!packageRoot) { - throw new Error(`Could not find package.json inside ${cwd}.`); - } - - return packageRoot; -} - -export function nameAtVersion(name: string, version: string): string { - return `${name}@${version}`; -} diff --git a/packages/hasher/src/index.ts b/packages/hasher/src/index.ts index 82f4b644..c3499f65 100644 --- a/packages/hasher/src/index.ts +++ b/packages/hasher/src/index.ts @@ -1 +1,4 @@ export { Hasher, type IHasher } from "./Hasher"; +// Helpers reused by lage (v2 does not use Hasher) +export { getFileHashes } from "./getFileHashes"; +export { resolveExternalDependencies } from "./resolveExternalDependencies"; diff --git a/packages/hasher/src/repoInfo.ts b/packages/hasher/src/repoInfo.ts index c163c53a..1283a24d 100644 --- a/packages/hasher/src/repoInfo.ts +++ b/packages/hasher/src/repoInfo.ts @@ -1,64 +1,52 @@ import { - WorkspaceInfo, - ParsedLock, - getWorkspaceRoot, - getWorkspaces, + getWorkspaceManagerRoot, parseLockFile, + type PackageInfos, + getWorkspaceInfos, } from "workspace-tools"; -import { getPackageDeps } from "@rushstack/package-deps-hash"; +import { getFileHashes } from "./getFileHashes"; import { createPackageHashes } from "./createPackageHashes"; - -export interface RepoInfo { - root: string; - workspaceInfo: WorkspaceInfo; - parsedLock: ParsedLock; - repoHashes: { [key: string]: string }; - packageHashes: Record; -} +import type { RepoHashes, RepoInfo } from "./types"; const repoInfoCache: RepoInfo[] = []; /** - * repoInfo cache lookup - it is specialized to be using a substring match to make it run as fast as possible - * @param packageRoot + * Calculate the repo info for `cwd`. + * Note that this DOES update the cache at the end; it just doesn't check for a cached result first. */ -function searchRepoInfoCache(packageRoot: string) { - for (const repoInfo of repoInfoCache) { - if (repoInfo.workspaceInfo && packageRoot.startsWith(repoInfo.root)) { - return repoInfo; - } - } -} - export async function getRepoInfoNoCache(cwd: string): Promise { - const root = getWorkspaceRoot(cwd); + const root = getWorkspaceManagerRoot(cwd); if (!root) { throw new Error("Cannot initialize Repo class without a workspace root"); } - // Assuming the package-deps-hash package returns a map of files to hashes that are unordered - const unorderedRepoHashes = Object.fromEntries(getPackageDeps(root)); + const unorderedRepoHashes = getFileHashes(root); + + // Sorting repoHash by key because we want to consistent hashing based on the order of the files. + // (Just use basic sorting instead of localeCompare since all that matters is stability. + // There might be a lot of files, so use plain loops instead of function helpers.) + const sortedFiles = Object.keys(unorderedRepoHashes).sort(); + const repoHashes: RepoHashes = {}; + for (const file of sortedFiles) { + repoHashes[file] = unorderedRepoHashes[file]; + } - // Sorting repoHash by key because we want to consistent hashing based on the order of the files - const repoHashes = Object.keys(unorderedRepoHashes) - .sort((a, b) => a.localeCompare(b)) - .reduce( - (obj, key) => { - obj[key] = unorderedRepoHashes[key]; - return obj; - }, - {} as Record - ); + // Use getWorkspaceInfos to exclude the root package (it will return undefined if not a monorepo), + // but convert into PackageInfos format because this works better for subsequent usage. + const workspaceInfos = getWorkspaceInfos(root) || []; + const packageInfos: PackageInfos = {}; + for (const info of workspaceInfos) { + packageInfos[info.name] = info.packageJson; + } - const workspaceInfo = getWorkspaces(root); const parsedLock = await parseLockFile(root); - const packageHashes = createPackageHashes(root, workspaceInfo, repoHashes); + const packageHashes = createPackageHashes(root, packageInfos, repoHashes); - const repoInfo = { + const repoInfo: RepoInfo = { root, - workspaceInfo, + packageInfos, parsedLock, repoHashes, packageHashes, @@ -69,25 +57,19 @@ export async function getRepoInfoNoCache(cwd: string): Promise { return repoInfo; } -// A promise to guarantee the getRepoInfo is done one at a time +/** A promise to guarantee the getRepoInfo is done one at a time */ let oneAtATime: Promise = Promise.resolve(); /** - * Retrieves the repoInfo, one at a time - * - * No parallel of this function is allowed; this maximizes the cache hit even - * though the getWorkspaces and parseLockFile are async functions from workspace-tools - * - * @param cwd + * Get the repo info for `cwd`. + * This function internally prevents parallel calls to maximize cache hits. */ export async function getRepoInfo(cwd: string): Promise { - oneAtATime = oneAtATime.then(async () => { - const searchResult = searchRepoInfoCache(cwd); - if (searchResult) { - return searchResult; - } - return getRepoInfoNoCache(cwd); - }); + oneAtATime = oneAtATime.then( + () => + repoInfoCache.find((repoInfo) => cwd.startsWith(repoInfo.root)) || + getRepoInfoNoCache(cwd) + ); return oneAtATime; } diff --git a/packages/hasher/src/resolveExternalDependencies.ts b/packages/hasher/src/resolveExternalDependencies.ts index 72fc3861..c41ab467 100644 --- a/packages/hasher/src/resolveExternalDependencies.ts +++ b/packages/hasher/src/resolveExternalDependencies.ts @@ -1,96 +1,87 @@ -import type { ParsedLock, WorkspaceInfo } from "workspace-tools"; -import { queryLockFile, listOfWorkspacePackageNames } from "workspace-tools"; -import { nameAtVersion } from "./helpers"; +import { + queryLockFile, + type PackageInfos, + type ParsedLock, +} from "workspace-tools"; -export type Dependencies = { [key in string]: string }; +export type Dependencies = Record; -export type ExternalDependenciesQueue = { - name: string; - versionRange: string; -}[]; +export type DependencyQueue = [name: string, versionRange: string][]; -export function filterExternalDependencies( +export type DependencySpec = `${string}@${string}`; + +/** Filter the `dependencies` object to only contain deps from outside the repo. */ +function _filterExternalDependencies( dependencies: Dependencies, - workspaces: WorkspaceInfo + packageInfos: PackageInfos ): Dependencies { - const workspacePackageNames = listOfWorkspacePackageNames(workspaces); const externalDependencies: Dependencies = {}; - Object.entries(dependencies).forEach(([name, versionRange]) => { - if (workspacePackageNames.indexOf(name) < 0) { + for (const [name, versionRange] of Object.entries(dependencies)) { + if (!packageInfos[name]) { externalDependencies[name] = versionRange; } - }); + } return externalDependencies; } -function isDone(done: string[], key: string): boolean { - return done.indexOf(key) >= 0; -} - -function isInQueue(queue: [string, string][], key: string): boolean { - return Boolean( - queue.find( - ([name, versionRange]) => nameAtVersion(name, versionRange) === key - ) - ); -} - -export function addToQueue( +export function _addToQueue( dependencies: Dependencies | undefined, - done: string[], - queue: [string, string][] + done: Set, + queue: DependencyQueue ): void { - if (dependencies) { - Object.entries(dependencies).forEach(([name, versionRange]) => { - const versionRangeSignature = nameAtVersion(name, versionRange); - - if ( - !isDone(done, versionRangeSignature) && - !isInQueue(queue, versionRangeSignature) - ) { - queue.push([name, versionRange]); - } - }); + if (!dependencies) return; + + for (const [name, versionRange] of Object.entries(dependencies)) { + const versionRangeSignature = `${name}@${versionRange}` as const; + + if ( + !done.has(versionRangeSignature) && + !queue.some(([n, v]) => `${n}@${v}` === versionRangeSignature) + ) { + queue.push([name, versionRange]); + } } } +/** + * Resolve versions for external (outside repo) dependencies and their transitive dependencies + * using the lock file. + * @returns Array of strings in the format `name@version` + */ export function resolveExternalDependencies( allDependencies: Dependencies, - workspaces: WorkspaceInfo, + packageInfos: PackageInfos, lockInfo: ParsedLock -): string[] { - const externalDependencies = filterExternalDependencies( +): DependencySpec[] { + const externalDependencies = _filterExternalDependencies( allDependencies, - workspaces + packageInfos ); - const done = []; - const doneRange = []; - const queue = Object.entries(externalDependencies); + const done = new Set(); + const doneRange = new Set(); + const queue: DependencyQueue = Object.entries(externalDependencies); while (queue.length > 0) { - const next = queue.shift(); - - if (!next) { - continue; - } + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- verified above + const next = queue.shift()!; const [name, versionRange] = next; - doneRange.push(nameAtVersion(name, versionRange)); + doneRange.add(`${name}@${versionRange}`); const lockFileResult = queryLockFile(name, versionRange, lockInfo); if (lockFileResult) { const { version, dependencies } = lockFileResult; - addToQueue(dependencies, doneRange, queue); - done.push(nameAtVersion(name, version)); + _addToQueue(dependencies, doneRange, queue); + done.add(`${name}@${version}`); } else { - done.push(nameAtVersion(name, versionRange)); + done.add(`${name}@${versionRange}`); } } - return done; + return [...done]; } diff --git a/packages/hasher/src/resolveInternalDependencies.ts b/packages/hasher/src/resolveInternalDependencies.ts deleted file mode 100644 index d26afd5c..00000000 --- a/packages/hasher/src/resolveInternalDependencies.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { WorkspaceInfo, listOfWorkspacePackageNames } from "workspace-tools"; - -export type Dependencies = { [key in string]: string }; - -export function filterInternalDependencies( - dependencies: Dependencies, - workspaces: WorkspaceInfo -): string[] { - const workspacePackageNames = listOfWorkspacePackageNames(workspaces); - return Object.keys(dependencies).filter( - (dependency) => workspacePackageNames.indexOf(dependency) >= 0 - ); -} - -export function resolveInternalDependencies( - allDependencies: Dependencies, - workspaces: WorkspaceInfo -): string[] { - const dependencyNames = filterInternalDependencies( - allDependencies, - workspaces - ); - - return dependencyNames; -} diff --git a/packages/hasher/src/types.ts b/packages/hasher/src/types.ts new file mode 100644 index 00000000..90a08eb9 --- /dev/null +++ b/packages/hasher/src/types.ts @@ -0,0 +1,23 @@ +import type { PackageInfos, ParsedLock } from "workspace-tools"; + +/** Mapping from repo-relative path (forward slashes) to hash for every file in the repo */ +export type RepoHashes = Record; + +/** + * Mapping from repo-relative package path to list of hashes for files in the package, + * in the form `[repo-relative path, hash]` (all with forward slashes) + */ +export type PackageHashes = Record; + +export interface RepoInfo { + root: string; + packageInfos: PackageInfos; + parsedLock: ParsedLock; + /** Mapping from repo-relative path (forward slashes) to hash for every file in the repo */ + repoHashes: RepoHashes; + /** + * Mapping from repo-relative package path to list of hashes for files in the package, + * in the form `[repo-relative path, hash]` (all with forward slashes) + */ + packageHashes: PackageHashes; +} diff --git a/packages/utils-test/__fixtures__/hasher-nested-test-project/package.json b/packages/utils-test/__fixtures__/hasher-nested-test-project/package.json new file mode 100644 index 00000000..18a1e415 --- /dev/null +++ b/packages/utils-test/__fixtures__/hasher-nested-test-project/package.json @@ -0,0 +1,3 @@ +{ + "dependencies": {} +} diff --git a/packages/utils-test/__fixtures__/hasher-nested-test-project/src/file 1.txt b/packages/utils-test/__fixtures__/hasher-nested-test-project/src/file 1.txt new file mode 100644 index 00000000..c7b2f707 --- /dev/null +++ b/packages/utils-test/__fixtures__/hasher-nested-test-project/src/file 1.txt @@ -0,0 +1 @@ +file1. \ No newline at end of file diff --git a/packages/utils-test/__fixtures__/hasher-test-project/file 2.txt b/packages/utils-test/__fixtures__/hasher-test-project/file 2.txt new file mode 100644 index 00000000..a385f754 --- /dev/null +++ b/packages/utils-test/__fixtures__/hasher-test-project/file 2.txt @@ -0,0 +1 @@ +file 2. \ No newline at end of file diff --git a/packages/utils-test/__fixtures__/hasher-test-project/file1.txt b/packages/utils-test/__fixtures__/hasher-test-project/file1.txt new file mode 100644 index 00000000..c7b2f707 --- /dev/null +++ b/packages/utils-test/__fixtures__/hasher-test-project/file1.txt @@ -0,0 +1 @@ +file1. \ No newline at end of file diff --git "a/packages/utils-test/__fixtures__/hasher-test-project/file\350\235\264\350\235\266.txt" "b/packages/utils-test/__fixtures__/hasher-test-project/file\350\235\264\350\235\266.txt" new file mode 100644 index 00000000..ae814af8 --- /dev/null +++ "b/packages/utils-test/__fixtures__/hasher-test-project/file\350\235\264\350\235\266.txt" @@ -0,0 +1 @@ +file蝴蝶. \ No newline at end of file diff --git a/packages/utils-test/__fixtures__/hasher-test-project/package.json b/packages/utils-test/__fixtures__/hasher-test-project/package.json new file mode 100644 index 00000000..18a1e415 --- /dev/null +++ b/packages/utils-test/__fixtures__/hasher-test-project/package.json @@ -0,0 +1,3 @@ +{ + "dependencies": {} +} diff --git a/yarn.lock b/yarn.lock index 980081ff..e5a4f50f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -789,27 +789,6 @@ resolved "https://registry.yarnpkg.com/@rtsao/scc/-/scc-1.1.0.tgz#927dd2fae9bc3361403ac2c7a00c32ddce9ad7e8" integrity sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g== -"@rushstack/node-core-library@3.53.3": - version "3.53.3" - resolved "https://registry.yarnpkg.com/@rushstack/node-core-library/-/node-core-library-3.53.3.tgz#e78e0dc1545f6cd7d80b0408cf534aefc62fbbe2" - integrity sha512-H0+T5koi5MFhJUd5ND3dI3bwLhvlABetARl78L3lWftJVQEPyzcgTStvTTRiIM5mCltyTM8VYm6BuCtNUuxD0Q== - dependencies: - "@types/node" "12.20.24" - colors "~1.2.1" - fs-extra "~7.0.1" - import-lazy "~4.0.0" - jju "~1.4.0" - resolve "~1.17.0" - semver "~7.3.0" - z-schema "~5.0.2" - -"@rushstack/package-deps-hash@^3.2.4": - version "3.2.67" - resolved "https://registry.yarnpkg.com/@rushstack/package-deps-hash/-/package-deps-hash-3.2.67.tgz#00e361cd1020ef36ed6e9a0863adff25bea6575d" - integrity sha512-PLp/cCmTl+HN8wLAhuElE8W+7YA0HJ7Bhx1FtKS1GBMXU01a9AVfJTKnHFylsV2rJs18QORyDaftl0XNS2adgw== - dependencies: - "@rushstack/node-core-library" "3.53.3" - "@sinclair/typebox@^0.34.0": version "0.34.41" resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.34.41.tgz#aa51a6c1946df2c5a11494a2cdb9318e026db16c" @@ -918,11 +897,6 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.63.tgz#1788fa8da838dbb5f9ea994b834278205db6ca2b" integrity sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ== -"@types/node@12.20.24": - version "12.20.24" - resolved "https://registry.yarnpkg.com/@types/node/-/node-12.20.24.tgz#c37ac69cb2948afb4cef95f424fa0037971a9a5c" - integrity sha512-yxDeaQIAJlMav7fH5AQqPH1u8YIuhYJXYBzxaQ4PifsU0GDO38MSdmEDeRlIxrKbC6NbEaaEHDanWb+y30U8SQ== - "@types/parse-author@^2.0.0": version "2.0.3" resolved "https://registry.yarnpkg.com/@types/parse-author/-/parse-author-2.0.3.tgz#fbf92ab0770f057483ba535c3087664f4b2800d5" @@ -1711,12 +1685,7 @@ colorette@^2.0.16: resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.20.tgz#9eb793e6833067f7235902fcd3b09917a000a95a" integrity sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w== -colors@~1.2.1: - version "1.2.5" - resolved "https://registry.yarnpkg.com/colors/-/colors-1.2.5.tgz#89c7ad9a374bc030df8013241f68136ed8835afc" - integrity sha512-erNRLao/Y3Fv54qUa0LBB+//Uf3YwMUmdJinN20yMXm9zdKKqH9wt7R9IIVZ+K7ShzfpLV/Zg8+VyrBJYB4lpg== - -commander@10.0.0, commander@^10.0.0: +commander@10.0.0: version "10.0.0" resolved "https://registry.yarnpkg.com/commander/-/commander-10.0.0.tgz#71797971162cd3cf65f0b9d24eb28f8d303acdf1" integrity sha512-zS5PnTI22FIRM6ylNW8G4Ap0IEOyk62fhLSD0+uHRT9McRCLGpkVNvao4bjimpK/GShynyQkFFxHhwMcETmduA== @@ -2464,15 +2433,6 @@ fs-extra@^8.1.0: jsonfile "^4.0.0" universalify "^0.1.0" -fs-extra@~7.0.1: - version "7.0.1" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-7.0.1.tgz#4f189c44aa123b895f722804f55ea23eadc348e9" - integrity sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw== - dependencies: - graceful-fs "^4.1.2" - jsonfile "^4.0.0" - universalify "^0.1.0" - fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -2695,7 +2655,7 @@ gopd@^1.0.1, gopd@^1.2.0: resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.2.0.tgz#89f56b8217bdbc8802bd299df6d7f1081d7e51a1" integrity sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg== -graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.11, graceful-fs@^4.2.4: +graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.11, graceful-fs@^4.2.4: version "4.2.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== @@ -2809,11 +2769,6 @@ import-fresh@^3.2.1, import-fresh@^3.3.0: parent-module "^1.0.0" resolve-from "^4.0.0" -import-lazy@~4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/import-lazy/-/import-lazy-4.0.0.tgz#e8eb627483a0a43da3c03f3e35548be5cb0cc153" - integrity sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw== - import-local@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/import-local/-/import-local-3.2.0.tgz#c3d5c745798c02a6f8b897726aba5100186ee260" @@ -3527,7 +3482,7 @@ jest@^30.0.0: import-local "^3.2.0" jest-cli "30.2.0" -jju@^1.4.0, jju@~1.4.0: +jju@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/jju/-/jju-1.4.0.tgz#a3abe2718af241a2b2904f84a625970f389ae32a" integrity sha512-8wb9Yw966OSxApiCt0K3yNJL8pnNeIv+OEq2YMidz4FKP6nonSRoOXc80iXY4JaN2FC11B9qsNmDsm+ZOfMROA== @@ -3697,16 +3652,6 @@ locate-path@^6.0.0: dependencies: p-locate "^5.0.0" -lodash.get@^4.4.2: - version "4.4.2" - resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99" - integrity sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ== - -lodash.isequal@^4.5.0: - version "4.5.0" - resolved "https://registry.yarnpkg.com/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0" - integrity sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ== - lodash.memoize@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/lodash.memoize/-/lodash.memoize-4.1.2.tgz#bcc6c49a42a2840ed997f323eada5ecd182e0bfe" @@ -4078,7 +4023,7 @@ path-key@^3.0.0, path-key@^3.1.0: resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== -path-parse@^1.0.6, path-parse@^1.0.7: +path-parse@^1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== @@ -4289,13 +4234,6 @@ resolve@^1.22.4: path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" -resolve@~1.17.0: - version "1.17.0" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.17.0.tgz#b25941b54968231cc2d1bb76a79cb7f2c0bf8444" - integrity sha512-ic+7JYiV8Vi2yzQGFWOkiZD5Z9z7O2Zhm9XMaTxdJExKasieFCr+yXZ/WmXsckHiKl12ar0y6XiXDx3m4RHn1w== - dependencies: - path-parse "^1.0.6" - restore-cursor@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-3.1.0.tgz#39f67c54b3a7a58cea5236d95cf0034239631f7e" @@ -4368,7 +4306,7 @@ safe-regex-test@^1.1.0: es-errors "^1.3.0" is-regex "^1.2.1" -semver@7.3.8, semver@~7.3.0: +semver@7.3.8: version "7.3.8" resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.8.tgz#07a78feafb3f7b32347d725e33de7e2a2df67798" integrity sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A== @@ -5052,11 +4990,6 @@ v8-to-istanbul@^9.0.1: "@types/istanbul-lib-coverage" "^2.0.1" convert-source-map "^2.0.0" -validator@^13.7.0: - version "13.15.26" - resolved "https://registry.yarnpkg.com/validator/-/validator-13.15.26.tgz#36c3deeab30e97806a658728a155c66fcaa5b944" - integrity sha512-spH26xU080ydGggxRyR1Yhcbgx+j3y5jbNXk/8L+iRvdIEQ4uTRH2Sgf2dokud6Q4oAtsbNvJ1Ft+9xmm6IZcA== - walker@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/walker/-/walker-1.0.8.tgz#bd498db477afe573dc04185f011d3ab8a8d7653f" @@ -5147,6 +5080,19 @@ workspace-tools@^0.40.0: js-yaml "^4.1.0" micromatch "^4.0.0" +workspace-tools@^0.41.0: + version "0.41.0" + resolved "https://registry.yarnpkg.com/workspace-tools/-/workspace-tools-0.41.0.tgz#b9389f7af1ca79bf102ff613ce21bf2d02c4010a" + integrity sha512-iBB6LNqtJpfjTWnyjlgOwdJmf1wBTUsIr1V5phOBCJMywubpYAijDUqhgz7RfrTYdscA4vEFyhjiy4OSAGULcA== + dependencies: + "@yarnpkg/lockfile" "^1.1.0" + fast-glob "^3.3.1" + git-url-parse "^16.0.0" + globby "^11.0.0" + jju "^1.4.0" + js-yaml "^4.1.0" + micromatch "^4.0.0" + "wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" @@ -5256,14 +5202,3 @@ yocto-queue@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== - -z-schema@~5.0.2: - version "5.0.6" - resolved "https://registry.yarnpkg.com/z-schema/-/z-schema-5.0.6.tgz#46d6a687b15e4a4369e18d6cb1c7b8618fc256c5" - integrity sha512-+XR1GhnWklYdfr8YaZv/iu+vY+ux7V5DS5zH1DQf6bO5ufrt/5cgNhVO5qyhsjFXvsqQb/f08DWE9b6uPscyAg== - dependencies: - lodash.get "^4.4.2" - lodash.isequal "^4.5.0" - validator "^13.7.0" - optionalDependencies: - commander "^10.0.0"