From fecd7e8a2c1a6b4ca4d0a942763752a65895181c Mon Sep 17 00:00:00 2001 From: Elizabeth Craig Date: Thu, 26 Feb 2026 17:04:24 -0800 Subject: [PATCH] Reuse utilities from backfill-hasher --- ...-af1026b1-217e-47f5-9bf6-56d614cb46c9.json | 32 ++ docs/docs/guides/installation.md | 6 +- docs/docs/quick-start.md | 6 +- docs/docs/reference/config.md | 6 +- packages/cache/package.json | 2 +- packages/cli/src/commands/init/action.ts | 6 +- packages/hasher/README.md | 2 +- packages/hasher/package.json | 1 + packages/hasher/src/FileHasher.ts | 12 +- packages/hasher/src/TargetHasher.ts | 20 +- .../src/__tests__/getPackageDeps.test.ts | 329 ------------------ .../resolveExternalDependencies.test.ts | 121 ------- packages/hasher/src/getPackageDeps.ts | 247 ------------- .../hasher/src/resolveExternalDependencies.ts | 77 ---- packages/lage/README.md | 6 +- yarn.lock | 23 +- 16 files changed, 87 insertions(+), 809 deletions(-) create mode 100644 change/change-af1026b1-217e-47f5-9bf6-56d614cb46c9.json delete mode 100644 packages/hasher/src/__tests__/getPackageDeps.test.ts delete mode 100644 packages/hasher/src/__tests__/resolveExternalDependencies.test.ts delete mode 100644 packages/hasher/src/getPackageDeps.ts delete mode 100644 packages/hasher/src/resolveExternalDependencies.ts diff --git a/change/change-af1026b1-217e-47f5-9bf6-56d614cb46c9.json b/change/change-af1026b1-217e-47f5-9bf6-56d614cb46c9.json new file mode 100644 index 000000000..53de9432f --- /dev/null +++ b/change/change-af1026b1-217e-47f5-9bf6-56d614cb46c9.json @@ -0,0 +1,32 @@ +{ + "changes": [ + { + "type": "patch", + "comment": "Update backfill-cache to 5.11.3", + "packageName": "@lage-run/cache", + "email": "elcraig@microsoft.com", + "dependentChangeType": "patch" + }, + { + "type": "patch", + "comment": "Update config generated by init", + "packageName": "@lage-run/cli", + "email": "elcraig@microsoft.com", + "dependentChangeType": "patch" + }, + { + "type": "patch", + "comment": "Reuse utilities from backfill-hasher", + "packageName": "@lage-run/hasher", + "email": "elcraig@microsoft.com", + "dependentChangeType": "patch" + }, + { + "type": "none", + "comment": "Update readme", + "packageName": "lage", + "email": "elcraig@microsoft.com", + "dependentChangeType": "none" + } + ] +} \ No newline at end of file diff --git a/docs/docs/guides/installation.md b/docs/docs/guides/installation.md index 57837cc3a..42be35ba2 100644 --- a/docs/docs/guides/installation.md +++ b/docs/docs/guides/installation.md @@ -73,8 +73,10 @@ const config = { // (relative to package root; folders must end with **/*) outputGlob: ["lib/**/*"], // Changes to any of these files/globs will invalidate the cache (relative to repo root; - // folders must end with **/*). This should include your lock file and any other repo-wide - // configs or scripts that are outside a package but could invalidate previous output. + // folders must end with **/*). This should include any repo-wide configs or scripts that + // are outside a package but could invalidate previous output. Including the lock file is + // optional--lage attempts to more granularly check resolved dependency changes, but this + // isn't entirely reliable, especially for peerDependencies. environmentGlob: ["package.json", "yarn.lock", "lage.config.js"] } }; diff --git a/docs/docs/quick-start.md b/docs/docs/quick-start.md index 2954a23f1..f5a1ed4ce 100644 --- a/docs/docs/quick-start.md +++ b/docs/docs/quick-start.md @@ -32,8 +32,10 @@ const config = { // (relative to package root; folders must end with **/*) outputGlob: ["lib/**/*"], // Changes to any of these files/globs will invalidate the cache (relative to repo root; - // folders must end with **/*). This should include your lock file and any other repo-wide - // configs or scripts that are outside a package but could invalidate previous output. + // folders must end with **/*). This should include any repo-wide configs or scripts that + // are outside a package but could invalidate previous output. Including the lock file is + // optional--lage attempts to more granularly check resolved dependency changes, but this + // isn't entirely reliable, especially for peerDependencies. environmentGlob: ["package.json", "yarn.lock", "lage.config.js"] } }; diff --git a/docs/docs/reference/config.md b/docs/docs/reference/config.md index 7a0e80d95..73efcb301 100644 --- a/docs/docs/reference/config.md +++ b/docs/docs/reference/config.md @@ -21,8 +21,10 @@ const config = { // (relative to package root; folders must end with **/*) outputGlob: ["lib/**/*"], // Changes to any of these files/globs will invalidate the cache (relative to repo root; - // folders must end with **/*). This should include your lock file and any other repo-wide - // configs or scripts that are outside a package but could invalidate previous output. + // folders must end with **/*). This should include any repo-wide configs or scripts that + // are outside a package but could invalidate previous output. Including the lock file is + // optional--lage attempts to more granularly check resolved dependency changes, but this + // isn't entirely reliable, especially for peerDependencies. environmentGlob: ["package.json", "yarn.lock", "lage.config.js"] } }; diff --git a/packages/cache/package.json b/packages/cache/package.json index bff9b3f9f..3caf46562 100644 --- a/packages/cache/package.json +++ b/packages/cache/package.json @@ -23,7 +23,7 @@ "@lage-run/config": "workspace:^", "@lage-run/logger": "workspace:^", "@lage-run/target-graph": "workspace:^", - "backfill-cache": "5.11.2", + "backfill-cache": "5.11.3", "backfill-config": "6.7.1", "backfill-logger": "5.4.0", "glob-hasher": "^1.4.2" diff --git a/packages/cli/src/commands/init/action.ts b/packages/cli/src/commands/init/action.ts index 93aa07288..169e9659e 100644 --- a/packages/cli/src/commands/init/action.ts +++ b/packages/cli/src/commands/init/action.ts @@ -51,8 +51,10 @@ const config = { // (relative to package root; folders must end with **/*) outputGlob: ["lib/**/*"], // Changes to any of these files/globs will invalidate the cache (relative to repo root; - // folders must end with **/*). This should include your lock file and any other repo-wide - // configs or scripts that are outside a package but could invalidate previous output. + // folders must end with **/*). This should include any repo-wide configs or scripts that + // are outside a package but could invalidate previous output. Including the lock file is + // optional--lage attempts to more granularly check resolved dependency changes, but this + // isn't entirely reliable, especially for peerDependencies. environmentGlob: ${JSON.stringify(["package.json", lockFile, "lage.config.js"].filter(Boolean))}, }, }; diff --git a/packages/hasher/README.md b/packages/hasher/README.md index da494839f..94e471c0a 100644 --- a/packages/hasher/README.md +++ b/packages/hasher/README.md @@ -1,3 +1,3 @@ # @lage-run/hasher -This package takes code from both backfill-hasher & package-dep-hash and strips out the extraneous dependencies: backfill-logger and @rushstack/node-core-lib. This is done so that `lage` can become a pure ES Module package. It also allows us to control how the hashing works should lage gains the ability to customize the `inputs` in the future. +This package builds on top of some helpers from `backfill-hasher` but adds customization for hashing targets with custom inputs and other logic. diff --git a/packages/hasher/package.json b/packages/hasher/package.json index a01e20f74..b9462dc88 100644 --- a/packages/hasher/package.json +++ b/packages/hasher/package.json @@ -19,6 +19,7 @@ "@lage-run/globby": "workspace:^", "@lage-run/logger": "workspace:^", "@lage-run/target-graph": "workspace:^", + "backfill-hasher": "6.7.0", "execa": "5.1.1", "glob-hasher": "^1.4.2", "graceful-fs": "4.2.11", diff --git a/packages/hasher/src/FileHasher.ts b/packages/hasher/src/FileHasher.ts index 15c7e4e0e..0b79f66a7 100644 --- a/packages/hasher/src/FileHasher.ts +++ b/packages/hasher/src/FileHasher.ts @@ -1,8 +1,8 @@ -import fs from "graceful-fs"; -import path from "path"; +import { getFileHashes } from "backfill-hasher"; import { hash as fastHash, stat } from "glob-hasher"; +import fs from "graceful-fs"; import { createInterface } from "node:readline"; -import { getPackageDeps } from "./getPackageDeps.js"; +import path from "path"; interface FileHashStoreOptions { root: string; @@ -29,12 +29,12 @@ export class FileHasher { private getHashesFromGit(): void { const { root } = this.options; - const fileHashes = getPackageDeps(root); - const files = [...fileHashes.keys()]; + const fileHashes = getFileHashes(root); + const files = Object.keys(fileHashes); const fileStats = stat(files, { cwd: root }) ?? {}; for (const [relativePath, fileStat] of Object.entries(fileStats)) { - const hash = fileHashes.get(relativePath); + const hash = fileHashes[relativePath]; if (hash) { const { size, mtime } = fileStat; diff --git a/packages/hasher/src/TargetHasher.ts b/packages/hasher/src/TargetHasher.ts index acdad8f54..c49a2c722 100644 --- a/packages/hasher/src/TargetHasher.ts +++ b/packages/hasher/src/TargetHasher.ts @@ -1,25 +1,23 @@ -import type { Target } from "@lage-run/target-graph"; -import { hash } from "glob-hasher"; import { globAsync } from "@lage-run/globby"; - +import type { Logger } from "@lage-run/logger"; +import type { Target } from "@lage-run/target-graph"; +import { resolveExternalDependencies } from "backfill-hasher"; import fs from "fs"; +import { hash } from "glob-hasher"; import path from "path"; import { - type DependencyMap, - type ParsedLock, - type PackageInfos, - parseLockFile, createDependencyMap, + type DependencyMap, getPackageInfo, getPackageInfosAsync, + type PackageInfos, + type ParsedLock, + parseLockFile, } from "workspace-tools"; - -import { hashStrings } from "./hashStrings.js"; -import { resolveExternalDependencies } from "./resolveExternalDependencies.js"; import { FileHasher } from "./FileHasher.js"; -import type { Logger } from "@lage-run/logger"; import { PackageTree } from "./PackageTree.js"; import { getInputFiles } from "./getInputFiles.js"; +import { hashStrings } from "./hashStrings.js"; export interface TargetHasherOptions { root: string; diff --git a/packages/hasher/src/__tests__/getPackageDeps.test.ts b/packages/hasher/src/__tests__/getPackageDeps.test.ts deleted file mode 100644 index b4bbd8f89..000000000 --- a/packages/hasher/src/__tests__/getPackageDeps.test.ts +++ /dev/null @@ -1,329 +0,0 @@ -import { Monorepo } from "@lage-run/monorepo-fixture"; -import fs from "fs"; -import path from "path"; -import { getPackageDeps, parseGitFilename, parseGitLsTree } from "../getPackageDeps.js"; - -const SOURCE_PATH: string = path.join(__dirname, "..", "__fixtures__"); - -const TEST_PROJECT_PATH: string = path.join(SOURCE_PATH, "test-project"); -const NESTED_TEST_PROJECT_PATH: string = path.join(SOURCE_PATH, "nested-test-project"); - -const FileSystem = { - writeFile: fs.writeFileSync, - deleteFile: fs.rmSync, -}; - -describe(parseGitFilename.name, () => { - it("can parse backslash-escaped filenames", () => { - expect(parseGitFilename("some/path/to/a/file name")).toEqual("some/path/to/a/file name"); - expect(parseGitFilename('"some/path/to/a/file?name"')).toEqual("some/path/to/a/file?name"); - expect(parseGitFilename('"some/path/to/a/file\\\\name"')).toEqual("some/path/to/a/file\\name"); - expect(parseGitFilename('"some/path/to/a/file\\"name"')).toEqual('some/path/to/a/file"name'); - expect(parseGitFilename('"some/path/to/a/file\\"name"')).toEqual('some/path/to/a/file"name'); - expect(parseGitFilename('"some/path/to/a/file\\347\\275\\221\\347\\275\\221name"')).toEqual("some/path/to/a/file网网name"); - expect(parseGitFilename('"some/path/to/a/file\\\\347\\\\\\347\\275\\221name"')).toEqual("some/path/to/a/file\\347\\网name"); - expect(parseGitFilename('"some/path/to/a/file\\\\\\347\\275\\221\\347\\275\\221name"')).toEqual("some/path/to/a/file\\网网name"); - }); -}); - -describe(parseGitLsTree.name, () => { - it("can handle a blob", () => { - const filename = "src/typings/tsd.d.ts"; - const hash = "3451bccdc831cb43d7a70ed8e628dcf9c7f888c8"; - - const output = `100644 blob ${hash}\t${filename}`; - const changes: Map = parseGitLsTree(output); - - expect(changes.size).toEqual(1); // Expect there to be exactly 1 change - expect(changes.get(filename)).toEqual(hash); // Expect the hash to be ${hash} - }); - - it("can handle a submodule", () => { - const filename = "rushstack"; - const hash = "c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac"; - - const output = `160000 commit ${hash}\t${filename}`; - const changes: Map = parseGitLsTree(output); - - expect(changes.size).toEqual(1); // Expect there to be exactly 1 change - expect(changes.get(filename)).toEqual(hash); // Expect the hash to be ${hash} - }); - - it("can handle multiple lines", () => { - const filename1 = "src/typings/tsd.d.ts"; - const hash1 = "3451bccdc831cb43d7a70ed8e628dcf9c7f888c8"; - - const filename2 = "src/foo bar/tsd.d.ts"; - const hash2 = "0123456789abcdef1234567890abcdef01234567"; - - const output = `100644 blob ${hash1}\t${filename1}\n100666 blob ${hash2}\t${filename2}`; - const changes: Map = parseGitLsTree(output); - - expect(changes.size).toEqual(2); // Expect there to be exactly 2 changes - expect(changes.get(filename1)).toEqual(hash1); // Expect the hash to be ${hash1} - expect(changes.get(filename2)).toEqual(hash2); // Expect the hash to be ${hash2} - }); - - it("throws with malformed input", () => { - expect(parseGitLsTree.bind(undefined, "some super malformed input")).toThrow(); - }); -}); - -describe(getPackageDeps.name, () => { - it("can parse committed file", async () => { - const monorepo = new Monorepo("parse-commited-file"); - await monorepo.init(TEST_PROJECT_PATH); - - const results: Map = getPackageDeps(monorepo.root); - const expectedFiles: { [key: string]: string } = { - "file1.txt": "c7b2f707ac99ca522f965210a7b6b0b109863f34", - "file 2.txt": "a385f754ec4fede884a4864d090064d9aeef8ccb", - "file蝴蝶.txt": "ae814af81e16cb2ae8c57503c77e2cab6b5462ba", - "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", - }; - const filePaths: string[] = Array.from(results.keys()).sort(); - - filePaths.forEach((filePath) => expect(results.get(filePath)).toEqual(expectedFiles[filePath])); - }); - - it("can handle files in subfolders", async () => { - const monorepo = new Monorepo("files-in-subfolders"); - await monorepo.init(NESTED_TEST_PROJECT_PATH); - - const results: Map = getPackageDeps(monorepo.root); - const expectedFiles: { [key: string]: string } = { - "src/file 1.txt": "c7b2f707ac99ca522f965210a7b6b0b109863f34", - "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", - }; - const filePaths: string[] = Array.from(results.keys()).sort(); - - filePaths.forEach((filePath) => expect(results.get(filePath)).toEqual(expectedFiles[filePath])); - - await monorepo.cleanup(); - }, 60_000); - - it("can handle adding one file", async () => { - const monorepo = new Monorepo("add-one-file"); - await monorepo.init(TEST_PROJECT_PATH); - - const tempFilePath: string = path.join(monorepo.root, "a.txt"); - - FileSystem.writeFile(tempFilePath, "a"); - - const results: Map = getPackageDeps(monorepo.root); - try { - const expectedFiles: { [key: string]: string } = { - "a.txt": "2e65efe2a145dda7ee51d1741299f848e5bf752e", - "file1.txt": "c7b2f707ac99ca522f965210a7b6b0b109863f34", - "file 2.txt": "a385f754ec4fede884a4864d090064d9aeef8ccb", - "file蝴蝶.txt": "ae814af81e16cb2ae8c57503c77e2cab6b5462ba", - "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", - }; - const filePaths: string[] = Array.from(results.keys()).sort(); - - filePaths.forEach((filePath) => expect(results.get(filePath)).toEqual(expectedFiles[filePath])); - } finally { - FileSystem.deleteFile(tempFilePath); - await monorepo.cleanup(); - } - }); - - it("can handle adding two files", async () => { - const monorepo = new Monorepo("add-two-files"); - await monorepo.init(TEST_PROJECT_PATH); - - const tempFilePath1: string = path.join(monorepo.root, "a.txt"); - const tempFilePath2: string = path.join(monorepo.root, "b.txt"); - - FileSystem.writeFile(tempFilePath1, "a"); - FileSystem.writeFile(tempFilePath2, "a"); - - const results: Map = getPackageDeps(monorepo.root); - try { - const expectedFiles: { [key: string]: string } = { - "a.txt": "2e65efe2a145dda7ee51d1741299f848e5bf752e", - "b.txt": "2e65efe2a145dda7ee51d1741299f848e5bf752e", - "file1.txt": "c7b2f707ac99ca522f965210a7b6b0b109863f34", - "file 2.txt": "a385f754ec4fede884a4864d090064d9aeef8ccb", - "file蝴蝶.txt": "ae814af81e16cb2ae8c57503c77e2cab6b5462ba", - "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", - }; - const filePaths: string[] = Array.from(results.keys()).sort(); - - filePaths.forEach((filePath) => expect(results.get(filePath)).toEqual(expectedFiles[filePath])); - } finally { - FileSystem.deleteFile(tempFilePath1); - FileSystem.deleteFile(tempFilePath2); - await monorepo.cleanup(); - } - }); - - it("can handle removing one file", async () => { - const monorepo = new Monorepo("remove-one-file"); - await monorepo.init(TEST_PROJECT_PATH); - - const testFilePath: string = path.join(monorepo.root, "file1.txt"); - - FileSystem.deleteFile(testFilePath); - - const results: Map = getPackageDeps(monorepo.root); - try { - const expectedFiles: { [key: string]: string } = { - "file 2.txt": "a385f754ec4fede884a4864d090064d9aeef8ccb", - "file蝴蝶.txt": "ae814af81e16cb2ae8c57503c77e2cab6b5462ba", - "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", - }; - const filePaths: string[] = Array.from(results.keys()).sort(); - - filePaths.forEach((filePath) => expect(results.get(filePath)).toEqual(expectedFiles[filePath])); - } finally { - await monorepo.cleanup(); - } - }); - - it("can handle changing one file", async () => { - const monorepo = new Monorepo("change-one-file"); - await monorepo.init(TEST_PROJECT_PATH); - - const testFilePath: string = path.join(monorepo.root, "file1.txt"); - - FileSystem.writeFile(testFilePath, "abc"); - - const results: Map = getPackageDeps(monorepo.root); - try { - const expectedFiles: { [key: string]: string } = { - "file1.txt": "f2ba8f84ab5c1bce84a7b441cb1959cfc7093b7f", - "file 2.txt": "a385f754ec4fede884a4864d090064d9aeef8ccb", - "file蝴蝶.txt": "ae814af81e16cb2ae8c57503c77e2cab6b5462ba", - "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", - }; - const filePaths: string[] = Array.from(results.keys()).sort(); - - filePaths.forEach((filePath) => expect(results.get(filePath)).toEqual(expectedFiles[filePath])); - } finally { - await monorepo.cleanup(); - } - }); - - it("can exclude a committed file", async () => { - const monorepo = new Monorepo("exclude-comitted-file"); - await monorepo.init(TEST_PROJECT_PATH); - - const results: Map = getPackageDeps(monorepo.root, ["file1.txt", "file 2.txt", "file蝴蝶.txt"]); - - const expectedFiles: { [key: string]: string } = { - "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", - }; - const filePaths: string[] = Array.from(results.keys()).sort(); - - filePaths.forEach((filePath) => expect(results.get(filePath)).toEqual(expectedFiles[filePath])); - await monorepo.cleanup(); - }); - - it("can exclude an added file", async () => { - const monorepo = new Monorepo("exclude-added-file"); - await monorepo.init(TEST_PROJECT_PATH); - - const tempFilePath: string = path.join(monorepo.root, "a.txt"); - - FileSystem.writeFile(tempFilePath, "a"); - - const results: Map = getPackageDeps(monorepo.root, ["a.txt"]); - try { - const expectedFiles: { [key: string]: string } = { - "file1.txt": "c7b2f707ac99ca522f965210a7b6b0b109863f34", - "file 2.txt": "a385f754ec4fede884a4864d090064d9aeef8ccb", - "file蝴蝶.txt": "ae814af81e16cb2ae8c57503c77e2cab6b5462ba", - "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", - }; - const filePaths: string[] = Array.from(results.keys()).sort(); - - expect(filePaths).toHaveLength(Object.keys(expectedFiles).length); - - filePaths.forEach((filePath) => expect(results.get(filePath)).toEqual(expectedFiles[filePath])); - } finally { - await monorepo.cleanup(); - } - }); - - it("can handle a filename with spaces", async () => { - const monorepo = new Monorepo("filename-with-spaces"); - await monorepo.init(TEST_PROJECT_PATH); - - const tempFilePath: string = path.join(monorepo.root, "a file.txt"); - - FileSystem.writeFile(tempFilePath, "a"); - - const results: Map = getPackageDeps(monorepo.root); - try { - const expectedFiles: { [key: string]: string } = { - "file1.txt": "c7b2f707ac99ca522f965210a7b6b0b109863f34", - "file 2.txt": "a385f754ec4fede884a4864d090064d9aeef8ccb", - "file蝴蝶.txt": "ae814af81e16cb2ae8c57503c77e2cab6b5462ba", - "a file.txt": "2e65efe2a145dda7ee51d1741299f848e5bf752e", - "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", - }; - const filePaths: string[] = Array.from(results.keys()).sort(); - - expect(filePaths).toHaveLength(Object.keys(expectedFiles).length); - - filePaths.forEach((filePath) => expect(results.get(filePath)).toEqual(expectedFiles[filePath])); - } finally { - await monorepo.cleanup(); - } - }); - - it("can handle a filename with multiple spaces", async () => { - const monorepo = new Monorepo("filename-multiple-spaces"); - await monorepo.init(TEST_PROJECT_PATH); - - const tempFilePath: string = path.join(monorepo.root, "a file name.txt"); - - FileSystem.writeFile(tempFilePath, "a"); - - const results: Map = getPackageDeps(monorepo.root); - try { - const expectedFiles: { [key: string]: string } = { - "file1.txt": "c7b2f707ac99ca522f965210a7b6b0b109863f34", - "file 2.txt": "a385f754ec4fede884a4864d090064d9aeef8ccb", - "file蝴蝶.txt": "ae814af81e16cb2ae8c57503c77e2cab6b5462ba", - "a file name.txt": "2e65efe2a145dda7ee51d1741299f848e5bf752e", - "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", - }; - const filePaths: string[] = Array.from(results.keys()).sort(); - - expect(filePaths).toHaveLength(Object.keys(expectedFiles).length); - - filePaths.forEach((filePath) => expect(results.get(filePath)).toEqual(expectedFiles[filePath])); - } finally { - await monorepo.cleanup(); - } - }); - - it("can handle a filename with non-standard characters", async () => { - const monorepo = new Monorepo("non-standard-characters"); - await monorepo.init(TEST_PROJECT_PATH); - - const tempFilePath: string = path.join(monorepo.root, "newFile批把.txt"); - - FileSystem.writeFile(tempFilePath, "a"); - - const results: Map = getPackageDeps(monorepo.root); - try { - const expectedFiles: { [key: string]: string } = { - "file1.txt": "c7b2f707ac99ca522f965210a7b6b0b109863f34", - "file 2.txt": "a385f754ec4fede884a4864d090064d9aeef8ccb", - "file蝴蝶.txt": "ae814af81e16cb2ae8c57503c77e2cab6b5462ba", - "newFile批把.txt": "2e65efe2a145dda7ee51d1741299f848e5bf752e", - "package.json": "18a1e415e56220fa5122428a4ef8eb8874756576", - }; - const filePaths: string[] = Array.from(results.keys()).sort(); - - expect(filePaths).toHaveLength(Object.keys(expectedFiles).length); - - filePaths.forEach((filePath) => expect(results.get(filePath)).toEqual(expectedFiles[filePath])); - } finally { - await monorepo.cleanup(); - } - }); -}); diff --git a/packages/hasher/src/__tests__/resolveExternalDependencies.test.ts b/packages/hasher/src/__tests__/resolveExternalDependencies.test.ts deleted file mode 100644 index 76d15dadb..000000000 --- a/packages/hasher/src/__tests__/resolveExternalDependencies.test.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { parseLockFile, getPackageInfos } from "workspace-tools"; -import { - _filterExternalDependencies, - resolveExternalDependencies, - _addToQueue, - type DependencySpec, - type DependencyQueue, -} from "../resolveExternalDependencies.js"; -import path from "path"; -import { Monorepo } from "@lage-run/monorepo-fixture"; - -const fixturesPath = path.join(__dirname, "../__fixtures__"); - -describe("_filterExternalDependencies", () => { - let monorepo: Monorepo | undefined; - - afterEach(async () => { - await monorepo?.cleanup(); - monorepo = undefined; - }); - - it("only lists external dependencies", async () => { - monorepo = new Monorepo("monorepo"); - await monorepo.init(path.join(fixturesPath, "monorepo")); - - const dependencies = { "package-a": "1.0.0", foo: "1.0.0" }; - const results = _filterExternalDependencies(dependencies, getPackageInfos(monorepo.root)); - - expect(results).toEqual({ foo: "1.0.0" }); - }); - - it("identifies all dependencies as external packages if there are no workspaces", async () => { - monorepo = new Monorepo("monorepo"); - await monorepo.init(path.join(fixturesPath, "basic")); - - const dependencies = { "package-a": "1.0.0", foo: "1.0.0" }; - const results = _filterExternalDependencies(dependencies, getPackageInfos(monorepo.root)); - - expect(results).toEqual(dependencies); - }); -}); - -describe("_addToQueue", () => { - it("adds external dependencies to queue", () => { - const externalDependencies = { foo: "1.0.0" }; - const done = new Set(); - const queue: DependencyQueue = []; - - _addToQueue(externalDependencies, done, queue); - - expect(queue).toEqual([["foo", "1.0.0"]]); - }); - - it("doesn't add to the queue if the dependency has been visited", () => { - const externalDependencies = { foo: "1.0.0" }; - const done = new Set(["foo@1.0.0"]); - const queue: DependencyQueue = []; - - _addToQueue(externalDependencies, done, queue); - - expect(queue).toEqual([]); - }); - - it("doesn't add to queue if the dependency is already in the queue", () => { - const externalDependencies = { foo: "1.0.0" }; - const done: Set = new Set(); - const queue: DependencyQueue = [["foo", "1.0.0"]]; - - _addToQueue(externalDependencies, done, queue); - - expect(queue).toEqual([["foo", "1.0.0"]]); - }); -}); - -describe("resolveExternalDependencies", () => { - let monorepo: Monorepo | undefined; - - afterEach(async () => { - await monorepo?.cleanup(); - monorepo = undefined; - }); - - it.each<{ - manager: "yarn" | "pnpm" | "rush"; - name: string; - fixture: string; - // only specified if different than the most common case - allDependencies?: Record; - expected?: DependencySpec[]; - }>([ - { - manager: "yarn", - name: "yarn", - fixture: "monorepo", - allDependencies: { "package-a": "1.0.0", foo: "1.0.0" }, - expected: ["foo@1.0.0", "bar@^1.0.0"], - }, - { manager: "pnpm", name: "pnpm", fixture: "monorepo-pnpm" }, - { manager: "rush", name: "rush+pnpm", fixture: "monorepo-rush-pnpm" }, - { manager: "rush", name: "rush+yarn", fixture: "monorepo-rush-yarn" }, - ])("transitively adds all external dependencies ($name)", async (params) => { - const { - fixture, - // These are used in most of the test cases - allDependencies = { "package-a": "1.0.0", once: "1.4.0" }, - expected = ["once@1.4.0", "wrappy@1.0.2"], - } = params; - - monorepo = new Monorepo("monorepo"); - await monorepo.init(path.join(fixturesPath, fixture)); - - const parsedLockFile = await parseLockFile(monorepo.root); - // TODO: use getPackageInfos(monorepo.root, manager) once available - // to ensure it's testing with the expected manager - const packageInfos = getPackageInfos(monorepo.root); - - const resolvedDependencies = resolveExternalDependencies(allDependencies, packageInfos, parsedLockFile); - - expect(resolvedDependencies).toEqual(expected); - }); -}); diff --git a/packages/hasher/src/getPackageDeps.ts b/packages/hasher/src/getPackageDeps.ts deleted file mode 100644 index 71670821e..000000000 --- a/packages/hasher/src/getPackageDeps.ts +++ /dev/null @@ -1,247 +0,0 @@ -import * as path from "path"; -import execa from "execa"; - -/** - * Parses a quoted filename sourced from the output of the "git status" command. - * - * Paths with non-standard characters will be enclosed with double-quotes, and non-standard - * characters will be backslash escaped (ex. double-quotes, non-ASCII characters). The - * escaped chars can be included in one of two ways: - * - backslash-escaped chars (ex. \") - * - octal encoded chars (ex. \347) - * - * See documentation: https://git-scm.com/docs/git-status - */ -export function parseGitFilename(filename: string): string { - // If there are no double-quotes around the string, then there are no escaped characters - // to decode, so just return - if (!filename.match(/^".+"$/)) { - return filename; - } - - // Need to hex encode '%' since we will be decoding the converted octal values from hex - filename = filename.replace(/%/g, "%25"); - // Replace all instances of octal literals with percent-encoded hex (ex. '\347\275\221' -> '%E7%BD%91'). - // This is done because the octal literals represent UTF-8 bytes, and by converting them to percent-encoded - // hex, we can use decodeURIComponent to get the Unicode chars. - filename = filename.replace(/(?:\\(\d{1,3}))/g, (match, ...[octalValue, index, source]) => { - // We need to make sure that the backslash is intended to escape the octal value. To do this, walk - // backwards from the match to ensure that it's already escaped. - const trailingBackslashes: RegExpMatchArray | null = (source as string).slice(0, index as number).match(/\\*$/); - return trailingBackslashes && trailingBackslashes.length > 0 && trailingBackslashes[0].length % 2 === 0 - ? `%${parseInt(octalValue, 8).toString(16)}` - : match; - }); - - // Finally, decode the filename and unescape the escaped UTF-8 chars - return JSON.parse(decodeURIComponent(filename)); -} - -/** - * Parses the output of the "git ls-tree" command - */ -export function parseGitLsTree(output: string): Map { - const changes: Map = new Map(); - - if (output) { - // A line is expected to look like: - // 100644 blob 3451bccdc831cb43d7a70ed8e628dcf9c7f888c8 src/typings/tsd.d.ts - // 160000 commit c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac rushstack - const gitRegex = /([0-9]{6})\s(blob|commit)\s([a-f0-9]{40})\s*(.*)/; - - // Note: The output of git ls-tree uses \n newlines regardless of OS. - const outputLines: string[] = output.trim().split("\n"); - for (const line of outputLines) { - if (line) { - // Take everything after the "100644 blob", which is just the hash and filename - const matches: RegExpMatchArray | null = line.match(gitRegex); - if (matches && matches[3] && matches[4]) { - const hash: string = matches[3]; - const filename: string = parseGitFilename(matches[4]); - - changes.set(filename, hash); - } else { - throw new Error(`Cannot parse git ls-tree input: "${line}"`); - } - } - } - } - - return changes; -} - -/** - * Parses the output of the "git status" command - */ -export function parseGitStatus(output: string): Map { - const changes: Map = new Map(); - - /* - * Typically, output will look something like: - * M temp_modules/rush-package-deps-hash/package.json - * D package-deps-hash/src/index.ts - */ - - // If there was an issue with `git ls-tree`, or there are no current changes, processOutputBlocks[1] - // will be empty or undefined - if (!output) { - return changes; - } - - // Note: The output of git hash-object uses \n newlines regardless of OS. - const outputLines: string[] = output.trim().split("\n"); - for (const line of outputLines) { - /* - * changeType is in the format of "XY" where "X" is the status of the file in the index and "Y" is the status of - * the file in the working tree. Some example statuses: - * - 'D' == deletion - * - 'M' == modification - * - 'A' == addition - * - '??' == untracked - * - 'R' == rename - * - 'RM' == rename with modifications - * - '[MARC]D' == deleted in work tree - * Full list of examples: https://git-scm.com/docs/git-status#_short_format - */ - const match: RegExpMatchArray | null = line.match(/("(\\"|[^"])+")|(\S+\s*)/g); - - if (match && match.length > 1) { - const [changeType, ...filenameMatches] = match; - - // We always care about the last filename in the filenames array. In the case of non-rename changes, - // the filenames array only contains one file, so we can join all segments that were split on spaces. - // In the case of rename changes, the last item in the array is the path to the file in the working tree, - // which is the only one that we care about. It is also surrounded by double-quotes if spaces are - // included, so no need to worry about joining different segments - let lastFilename: string = changeType.startsWith("R") ? filenameMatches[filenameMatches.length - 1] : filenameMatches.join(""); - lastFilename = parseGitFilename(lastFilename); - - changes.set(lastFilename, changeType.trimRight()); - } - } - - return changes; -} - -/** - * Takes a list of files and returns the current git hashes for them - * - * @public - */ -export function getGitHashForFiles(filesToHash: string[], packagePath: string, gitPath?: string): Map { - const changes: Map = new Map(); - - if (filesToHash.length) { - // Use --stdin-paths arg to pass the list of files to git in order to avoid issues with - // command length - const result = execa.sync(gitPath || "git", ["hash-object", "--stdin-paths"], { - input: filesToHash.map((x) => path.resolve(packagePath, x)).join("\n"), - }); - - if (result.exitCode !== 0) { - throw new Error(`git hash-object exited with status ${result.exitCode}: ${result.stderr}`); - } - - const hashStdout: string = result.stdout.trim(); - - // The result of "git hash-object" will be a list of file hashes delimited by newlines - const hashes: string[] = hashStdout.split("\n"); - - if (hashes.length !== filesToHash.length) { - throw new Error(`Passed ${filesToHash.length} file paths to Git to hash, but received ${hashes.length} hashes.`); - } - - for (let i = 0; i < hashes.length; i++) { - const hash: string = hashes[i]; - const filePath: string = filesToHash[i]; - changes.set(filePath, hash); - } - } - - return changes; -} - -/** - * Executes "git ls-tree" in a folder - */ -export function gitLsTree(path: string, gitPath?: string): string { - const result = execa.sync(gitPath || "git", ["ls-tree", "HEAD", "-r"], { - cwd: path, - }); - - if (result.exitCode !== 0) { - throw new Error(`git ls-tree exited with status ${result.exitCode}: ${result.stderr}`); - } - - return result.stdout; -} - -/** - * Executes "git status" in a folder - */ -export function gitStatus(path: string, gitPath?: string): string { - /** - * -s - Short format. Will be printed as 'XY PATH' or 'XY ORIG_PATH -> PATH'. Paths with non-standard - * characters will be escaped using double-quotes, and non-standard characters will be backslash - * escaped (ex. spaces, tabs, double-quotes) - * -u - Untracked files are included - * - * See documentation here: https://git-scm.com/docs/git-status - */ - const result = execa.sync(gitPath || "git", ["status", "-s", "-u", "."], { - cwd: path, - }); - - if (result.exitCode !== 0) { - throw new Error(`git status exited with status ${result.exitCode}: ${result.stderr}`); - } - - return result.stdout; -} - -/** - * Builds an object containing hashes for the files under the specified `packagePath` folder. - * @param packagePath - The folder path to derive the package dependencies from. This is typically the folder - * containing package.json. If omitted, the default value is the current working directory. - * @param excludedPaths - An optional array of file path exclusions. If a file should be omitted from the list - * of dependencies, use this to exclude it. - * @returns the package-deps.json file content - * - * @public - */ -export function getPackageDeps(packagePath: string = process.cwd(), excludedPaths?: string[], gitPath?: string): Map { - const gitLsOutput: string = gitLsTree(packagePath, gitPath); - - // Add all the checked in hashes - const result: Map = parseGitLsTree(gitLsOutput); - - // Remove excluded paths - if (excludedPaths) { - for (const excludedPath of excludedPaths) { - result.delete(excludedPath); - } - } - - // Update the checked in hashes with the current repo status - const gitStatusOutput: string = gitStatus(packagePath, gitPath); - const currentlyChangedFiles: Map = parseGitStatus(gitStatusOutput); - const filesToHash: string[] = []; - const excludedPathSet: Set = new Set(excludedPaths); - for (const [filename, changeType] of currentlyChangedFiles) { - // See comments inside parseGitStatus() for more information - if (changeType === "D" || (changeType.length === 2 && changeType.charAt(1) === "D")) { - result.delete(filename); - } else { - if (!excludedPathSet.has(filename)) { - filesToHash.push(filename); - } - } - } - - const currentlyChangedFileHashes: Map = getGitHashForFiles(filesToHash, packagePath, gitPath); - for (const [filename, hash] of currentlyChangedFileHashes) { - result.set(filename, hash); - } - - return result; -} diff --git a/packages/hasher/src/resolveExternalDependencies.ts b/packages/hasher/src/resolveExternalDependencies.ts deleted file mode 100644 index b5ccd3b13..000000000 --- a/packages/hasher/src/resolveExternalDependencies.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { queryLockFile, type PackageInfos, type ParsedLock } from "workspace-tools"; - -type Dependencies = Record; - -export type DependencyQueue = [name: string, versionRange: string][]; - -export type DependencySpec = `${string}@${string}`; - -/** Filter the `dependencies` object to only contain deps from outside the repo. */ -export function _filterExternalDependencies(dependencies: Dependencies, packageInfos: PackageInfos): Dependencies { - const externalDependencies: Dependencies = {}; - - for (const [name, versionRange] of Object.entries(dependencies)) { - if (!packageInfos[name]) { - externalDependencies[name] = versionRange; - } - } - - return externalDependencies; -} - -function isInQueue(queue: DependencyQueue, key: string): boolean { - return queue.some(([name, versionRange]) => `${name}@${versionRange}` === key); -} - -export function _addToQueue(dependencies: Dependencies | undefined, done: Set, queue: DependencyQueue): void { - if (dependencies) { - for (const [name, versionRange] of Object.entries(dependencies)) { - const versionRangeSignature = `${name}@${versionRange}` as const; - - if (!done.has(versionRangeSignature) && !isInQueue(queue, versionRangeSignature)) { - queue.push([name, versionRange]); - } - } - } -} - -/** - * Resolve versions for external (outside repo) dependencies and their transitive dependencies - * using the lock file. - * @returns Array of strings in the format `name@version` - */ -export function resolveExternalDependencies( - allDependencies: Dependencies, - packageInfos: PackageInfos, - lockInfo: ParsedLock -): DependencySpec[] { - const externalDependencies = _filterExternalDependencies(allDependencies, packageInfos); - - const done = new Set(); - const doneRange = new Set(); - const queue: DependencyQueue = Object.entries(externalDependencies); - - while (queue.length > 0) { - const next = queue.shift(); - - if (!next) { - continue; - } - - const [name, versionRange] = next; - doneRange.add(`${name}@${versionRange}`); - - const lockFileResult = queryLockFile(name, versionRange, lockInfo); - - if (lockFileResult) { - const { version, dependencies } = lockFileResult; - - _addToQueue(dependencies, doneRange, queue); - done.add(`${name}@${version}`); - } else { - done.add(`${name}@${versionRange}`); - } - } - - return [...done]; -} diff --git a/packages/lage/README.md b/packages/lage/README.md index fee8170a8..a7fe48b50 100644 --- a/packages/lage/README.md +++ b/packages/lage/README.md @@ -62,8 +62,10 @@ const config = { // (relative to package root; folders must end with **/*) outputGlob: ["lib/**/*"], // Changes to any of these files/globs will invalidate the cache (relative to repo root; - // folders must end with **/*). This should include your lock file and any other repo-wide - // configs or scripts that are outside a package but could invalidate previous output. + // folders must end with **/*). This should include any repo-wide configs or scripts that + // are outside a package but could invalidate previous output. Including the lock file is + // optional--lage attempts to more granularly check resolved dependency changes, but this + // isn't entirely reliable, especially for peerDependencies. environmentGlob: ["package.json", "yarn.lock", "lage.config.js"], }, }; diff --git a/yarn.lock b/yarn.lock index 205918bdf..c41c5ec11 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1572,7 +1572,7 @@ __metadata: "@lage-run/monorepo-fixture": "workspace:^" "@lage-run/monorepo-scripts": "workspace:^" "@lage-run/target-graph": "workspace:^" - backfill-cache: "npm:5.11.2" + backfill-cache: "npm:5.11.3" backfill-config: "npm:6.7.1" backfill-logger: "npm:5.4.0" glob-hasher: "npm:^1.4.2" @@ -1672,6 +1672,7 @@ __metadata: "@lage-run/target-graph": "workspace:^" "@types/graceful-fs": "npm:^4.1.6" "@types/micromatch": "npm:^4.0.0" + backfill-hasher: "npm:6.7.0" execa: "npm:5.1.1" glob-hasher: "npm:^1.4.2" graceful-fs: "npm:4.2.11" @@ -3042,9 +3043,9 @@ __metadata: languageName: node linkType: hard -"backfill-cache@npm:5.11.2": - version: 5.11.2 - resolution: "backfill-cache@npm:5.11.2" +"backfill-cache@npm:5.11.3": + version: 5.11.3 + resolution: "backfill-cache@npm:5.11.3" dependencies: "@azure/storage-blob": "npm:^12.15.0" backfill-config: "npm:^6.7.1" @@ -3054,7 +3055,7 @@ __metadata: globby: "npm:^11.0.0" p-limit: "npm:^3.0.0" tar-fs: "npm:^2.1.0" - checksum: 10c0/2e57b265d3e63aa426dbd4ac41f91725f9fc002dcc11126dd3573e59f2fd5e0001ad5a16fb16ad1107904ef8a584f45645b580a5f399822f691828073c5d433d + checksum: 10c0/8d685384676549b10e14cdca2eeb820ded1cceb7dde28a3eaaf2a81fb4631d758c567f1867e0f23fa78f8c290366ca37aab34ee44feb0dc0b981e7ed5d25958e languageName: node linkType: hard @@ -3069,6 +3070,16 @@ __metadata: languageName: node linkType: hard +"backfill-hasher@npm:6.7.0": + version: 6.7.0 + resolution: "backfill-hasher@npm:6.7.0" + dependencies: + backfill-logger: "npm:^5.4.0" + workspace-tools: "npm:^0.41.0" + checksum: 10c0/5ba758976980d7467763c13ca9c4cf049785d97e004ef437fa8e961f0d653a76876218ac7364a8d54853328e91454aa30ef6c95d0cd8b26f208c0903d15cb099 + languageName: node + linkType: hard + "backfill-logger@npm:5.4.0, backfill-logger@npm:^5.4.0": version: 5.4.0 resolution: "backfill-logger@npm:5.4.0" @@ -8490,7 +8501,7 @@ __metadata: languageName: node linkType: hard -"workspace-tools@npm:0.41.0": +"workspace-tools@npm:0.41.0, workspace-tools@npm:^0.41.0": version: 0.41.0 resolution: "workspace-tools@npm:0.41.0" dependencies: