diff --git a/.chronus/changes/http-client-python_refactor-regenerate-common-2026-02-13-0-0-0.md b/.chronus/changes/http-client-python_refactor-regenerate-common-2026-02-13-0-0-0.md new file mode 100644 index 00000000000..0c0b5d7a4a7 --- /dev/null +++ b/.chronus/changes/http-client-python_refactor-regenerate-common-2026-02-13-0-0-0.md @@ -0,0 +1,7 @@ +--- +changeKind: internal +packages: + - "@typespec/http-client-python" +--- + +Refactor regenerate script to extract shared constants, interfaces, and utilities into a reusable `regenerate-common` module. diff --git a/packages/http-client-python/eng/scripts/ci/regenerate-common.ts b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts new file mode 100644 index 00000000000..9f9a91266d6 --- /dev/null +++ b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts @@ -0,0 +1,472 @@ +import { promises } from "fs"; +import { dirname, join, relative, resolve } from "path"; + +// ---- Shared constants ---- + +export const SKIP_SPECS: string[] = ["type/file"]; + +export const SpecialFlags: Record> = { + azure: { + "generate-test": true, + "generate-sample": true, + }, +}; + +// ---- Base emitter options (shared across repos) ---- + +export const BASE_AZURE_EMITTER_OPTIONS: Record< + string, + Record | Record[] +> = { + "azure/client-generator-core/access": { + namespace: "specs.azure.clientgenerator.core.access", + }, + "azure/client-generator-core/alternate-type": { + namespace: "specs.azure.clientgenerator.core.alternatetype", + }, + "azure/client-generator-core/api-version": { + namespace: "specs.azure.clientgenerator.core.apiversion", + }, + "azure/client-generator-core/client-initialization/default": { + namespace: "specs.azure.clientgenerator.core.clientinitialization.default", + }, + "azure/client-generator-core/client-initialization/individually": { + namespace: "specs.azure.clientgenerator.core.clientinitialization.individually", + }, + "azure/client-generator-core/client-initialization/individuallyParent": { + namespace: "specs.azure.clientgenerator.core.clientinitialization.individuallyparent", + }, + "azure/client-generator-core/client-location": { + namespace: "specs.azure.clientgenerator.core.clientlocation", + }, + "azure/client-generator-core/deserialize-empty-string-as-null": { + namespace: "specs.azure.clientgenerator.core.emptystring", + }, + "azure/client-generator-core/flatten-property": { + namespace: "specs.azure.clientgenerator.core.flattenproperty", + }, + "azure/client-generator-core/usage": { + namespace: "specs.azure.clientgenerator.core.usage", + }, + "azure/client-generator-core/override": { + namespace: "specs.azure.clientgenerator.core.override", + }, + "azure/client-generator-core/hierarchy-building": { + namespace: "specs.azure.clientgenerator.core.hierarchybuilding", + }, + "azure/core/basic": { + namespace: "specs.azure.core.basic", + }, + "azure/core/lro/rpc": { + namespace: "specs.azure.core.lro.rpc", + }, + "azure/core/lro/standard": { + namespace: "specs.azure.core.lro.standard", + }, + "azure/core/model": { + namespace: "specs.azure.core.model", + }, + "azure/core/page": { + namespace: "specs.azure.core.page", + }, + "azure/core/scalar": { + namespace: "specs.azure.core.scalar", + }, + "azure/core/traits": { + namespace: "specs.azure.core.traits", + }, + "azure/encode/duration": { + namespace: "specs.azure.encode.duration", + }, + "azure/example/basic": { + namespace: "specs.azure.example.basic", + }, + "azure/payload/pageable": { + namespace: "specs.azure.payload.pageable", + }, + "azure/versioning/previewVersion": { + namespace: "specs.azure.versioning.previewversion", + }, + "client/structure/default": { + namespace: "client.structure.service", + }, + "client/structure/multi-client": { + "package-name": "client-structure-multiclient", + namespace: "client.structure.multiclient", + }, + "client/structure/renamed-operation": { + "package-name": "client-structure-renamedoperation", + namespace: "client.structure.renamedoperation", + }, + "client/structure/two-operation-group": { + "package-name": "client-structure-twooperationgroup", + namespace: "client.structure.twooperationgroup", + }, + "client/naming": { + namespace: "client.naming.main", + }, + "client/overload": { + namespace: "client.overload", + }, + "encode/duration": { + namespace: "encode.duration", + }, + "encode/numeric": { + namespace: "encode.numeric", + }, + "parameters/basic": { + namespace: "parameters.basic", + }, + "parameters/spread": { + namespace: "parameters.spread", + }, + "payload/content-negotiation": { + namespace: "payload.contentnegotiation", + }, + "payload/multipart": { + namespace: "payload.multipart", + }, + "serialization/encoded-name/json": { + namespace: "serialization.encodedname.json", + }, + "special-words": { + namespace: "specialwords", + }, + "service/multi-service": { + namespace: "service.multiservice", + }, +}; + +export const BASE_EMITTER_OPTIONS: Record< + string, + Record | Record[] +> = { + "resiliency/srv-driven/old.tsp": { + "package-name": "resiliency-srv-driven1", + namespace: "resiliency.srv.driven1", + "package-mode": "azure-dataplane", + "package-pprint-name": "ResiliencySrvDriven1", + }, + "resiliency/srv-driven": { + "package-name": "resiliency-srv-driven2", + namespace: "resiliency.srv.driven2", + "package-mode": "azure-dataplane", + "package-pprint-name": "ResiliencySrvDriven2", + }, + "authentication/api-key": { + "clear-output-folder": "true", + }, + "authentication/http/custom": { + "package-name": "authentication-http-custom", + namespace: "authentication.http.custom", + "package-pprint-name": "Authentication Http Custom", + }, + "authentication/union": [ + { + "package-name": "authentication-union", + namespace: "authentication.union", + }, + { + "package-name": "setuppy-authentication-union", + namespace: "setuppy.authentication.union", + "keep-setup-py": "true", + }, + ], + "type/array": { + "package-name": "typetest-array", + namespace: "typetest.array", + }, + "type/dictionary": { + "package-name": "typetest-dictionary", + namespace: "typetest.dictionary", + }, + "type/enum/extensible": { + "package-name": "typetest-enum-extensible", + namespace: "typetest.enum.extensible", + }, + "type/enum/fixed": { + "package-name": "typetest-enum-fixed", + namespace: "typetest.enum.fixed", + }, + "type/model/empty": { + "package-name": "typetest-model-empty", + namespace: "typetest.model.empty", + }, + "type/model/inheritance/enum-discriminator": { + "package-name": "typetest-model-enumdiscriminator", + namespace: "typetest.model.enumdiscriminator", + }, + "type/model/inheritance/nested-discriminator": { + "package-name": "typetest-model-nesteddiscriminator", + namespace: "typetest.model.nesteddiscriminator", + }, + "type/model/inheritance/not-discriminated": { + "package-name": "typetest-model-notdiscriminated", + namespace: "typetest.model.notdiscriminated", + }, + "type/model/inheritance/single-discriminator": { + "package-name": "typetest-model-singlediscriminator", + namespace: "typetest.model.singlediscriminator", + }, + "type/model/inheritance/recursive": { + "package-name": "typetest-model-recursive", + namespace: "typetest.model.recursive", + }, + "type/model/usage": { + "package-name": "typetest-model-usage", + namespace: "typetest.model.usage", + }, + "type/model/visibility": [ + { + "package-name": "typetest-model-visibility", + namespace: "typetest.model.visibility", + }, + { + "package-name": "headasbooleantrue", + namespace: "headasbooleantrue", + "head-as-boolean": "true", + }, + { + "package-name": "headasbooleanfalse", + namespace: "headasbooleanfalse", + "head-as-boolean": "false", + }, + ], + "type/property/nullable": { + "package-name": "typetest-property-nullable", + namespace: "typetest.property.nullable", + }, + "type/property/optionality": { + "package-name": "typetest-property-optional", + namespace: "typetest.property.optional", + }, + "type/property/additional-properties": { + "package-name": "typetest-property-additionalproperties", + namespace: "typetest.property.additionalproperties", + }, + "type/scalar": { + "package-name": "typetest-scalar", + namespace: "typetest.scalar", + }, + "type/property/value-types": { + "package-name": "typetest-property-valuetypes", + namespace: "typetest.property.valuetypes", + }, + "type/union": { + "package-name": "typetest-union", + namespace: "typetest.union", + }, + "type/union/discriminated": { + "package-name": "typetest-discriminatedunion", + namespace: "typetest.discriminatedunion", + }, + "type/file": { + "package-name": "typetest-file", + namespace: "typetest.file", + }, + documentation: { + "package-name": "specs-documentation", + namespace: "specs.documentation", + }, +}; + +// ---- Shared interfaces ---- + +export interface TspCommand { + outputDir: string; + command: string | string[]; +} + +export interface RegenerateFlagsInput { + flavor?: string; + debug?: boolean; + name?: string; + pyodide?: boolean; +} + +export interface RegenerateFlags { + flavor: string; + debug: boolean; + name?: string; + pyodide?: boolean; +} + +export interface ProcessedEmitterOption { + options: Record; + outputDir: string; +} + +export interface RegenerateConfig { + azureHttpSpecs: string; + httpSpecs: string; + emitterOptions: Record | Record[]>; + azureEmitterOptions: Record | Record[]>; + preprocess: (flags: RegenerateFlagsInput) => Promise; + getCmdList: (spec: string, flags: RegenerateFlags) => TspCommand[]; + executeCommand: (cmd: TspCommand) => Promise; +} + +// ---- Shared utility functions ---- + +export function toPosix(dir: string): string { + return dir.replace(/\\/g, "/"); +} + +export function getEmitterOption( + spec: string, + flavor: string, + config: RegenerateConfig, +): Record[] { + const specDir = spec.includes("azure") ? config.azureHttpSpecs : config.httpSpecs; + const relativeSpec = toPosix(relative(specDir, spec)); + const key = relativeSpec.includes("resiliency/srv-driven/old.tsp") + ? relativeSpec + : dirname(relativeSpec); + const emitter_options = config.emitterOptions[key] || + (flavor === "azure" ? config.azureEmitterOptions[key] : [{}]) || [{}]; + return Array.isArray(emitter_options) ? emitter_options : [emitter_options]; +} + +export async function getSubdirectories( + baseDir: string, + flags: RegenerateFlags, +): Promise { + const subdirectories: string[] = []; + + async function searchDir(currentDir: string) { + const items = await promises.readdir(currentDir, { withFileTypes: true }); + + const promisesArray = items.map(async (item) => { + const subDirPath = join(currentDir, item.name); + if (item.isDirectory()) { + const mainTspPath = join(subDirPath, "main.tsp"); + const clientTspPath = join(subDirPath, "client.tsp"); + + const mainTspRelativePath = toPosix(relative(baseDir, mainTspPath)); + + if (SKIP_SPECS.some((skipSpec) => mainTspRelativePath.includes(skipSpec))) return; + + const hasMainTsp = await promises + .access(mainTspPath) + .then(() => true) + .catch(() => false); + const hasClientTsp = await promises + .access(clientTspPath) + .then(() => true) + .catch(() => false); + + if (mainTspRelativePath.toLowerCase().includes(flags.name || "")) { + if (mainTspRelativePath.includes("resiliency/srv-driven")) { + subdirectories.push(resolve(subDirPath, "old.tsp")); + } + if (hasClientTsp) { + subdirectories.push(resolve(subDirPath, "client.tsp")); + } else if (hasMainTsp) { + subdirectories.push(resolve(subDirPath, "main.tsp")); + } + } + + // Recursively search in the subdirectory + await searchDir(subDirPath); + } + }); + + await Promise.all(promisesArray); + } + + await searchDir(baseDir); + return subdirectories; +} + +export function defaultPackageName(spec: string, config: RegenerateConfig): string { + const specDir = spec.includes("azure") ? config.azureHttpSpecs : config.httpSpecs; + return toPosix(relative(specDir, dirname(spec))) + .replace(/\//g, "-") + .toLowerCase(); +} + +export function buildOptions( + spec: string, + generatedFolder: string, + flags: RegenerateFlags, + config: RegenerateConfig, +): ProcessedEmitterOption[] { + const results: ProcessedEmitterOption[] = []; + for (const emitterConfig of getEmitterOption(spec, flags.flavor, config)) { + const options: Record = { ...emitterConfig }; + if (flags.pyodide) { + options["use-pyodide"] = "true"; + } + options["flavor"] = flags.flavor; + for (const [k, v] of Object.entries(SpecialFlags[flags.flavor] ?? {})) { + options[k] = v; + } + if (options["emitter-output-dir"] === undefined) { + const packageName = options["package-name"] || defaultPackageName(spec, config); + options["emitter-output-dir"] = toPosix( + `${generatedFolder}/test/${flags.flavor}/generated/${packageName}`, + ); + } + if (flags.debug) { + options["debug"] = "true"; + } + options["examples-dir"] = toPosix(join(dirname(spec), "examples")); + results.push({ + options, + outputDir: options["emitter-output-dir"], + }); + } + return results; +} + +export async function runTaskPool( + tasks: Array<() => Promise>, + poolLimit: number, +): Promise { + async function worker(start: number, end: number) { + while (start < end) { + await tasks[start](); + start++; + } + } + + const workers = []; + let start = 0; + while (start < tasks.length) { + const end = Math.min(start + poolLimit, tasks.length); + workers.push((async () => await worker(start, end))()); + start = end; + } + await Promise.all(workers); +} + +export async function regenerate( + flags: RegenerateFlagsInput, + config: RegenerateConfig, +): Promise { + if (flags.flavor === undefined) { + await regenerate({ flavor: "azure", ...flags }, config); + await regenerate({ flavor: "unbranded", ...flags }, config); + } else { + await config.preprocess(flags); + + const flagsResolved: RegenerateFlags = { debug: false, flavor: flags.flavor, ...flags }; + const subdirectoriesForAzure = await getSubdirectories(config.azureHttpSpecs, flagsResolved); + const subdirectoriesForNonAzure = await getSubdirectories(config.httpSpecs, flagsResolved); + const subdirectories = + flags.flavor === "azure" + ? [...subdirectoriesForAzure, ...subdirectoriesForNonAzure] + : subdirectoriesForNonAzure; + const cmdList: TspCommand[] = subdirectories.flatMap((subdirectory) => + config.getCmdList(subdirectory, flagsResolved), + ); + + // Create tasks as functions for the pool + const tasks: Array<() => Promise> = cmdList.map((tspCommand) => { + return () => config.executeCommand(tspCommand); + }); + + // Run tasks with a concurrency limit + await runTaskPool(tasks, 30); + } +} diff --git a/packages/http-client-python/eng/scripts/ci/regenerate.ts b/packages/http-client-python/eng/scripts/ci/regenerate.ts index 631a6ad384b..84bc53727bd 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate.ts @@ -2,9 +2,20 @@ import chalk from "chalk"; import { execFile } from "child_process"; import { promises, rmSync } from "fs"; -import { dirname, join, relative, resolve } from "path"; +import { join, resolve } from "path"; import { fileURLToPath } from "url"; import { parseArgs, promisify } from "util"; +import { + BASE_AZURE_EMITTER_OPTIONS, + BASE_EMITTER_OPTIONS, + buildOptions, + regenerate, + toPosix, + type RegenerateConfig, + type RegenerateFlags, + type RegenerateFlagsInput, + type TspCommand, +} from "./regenerate-common.js"; // PARSE INPUT ARGUMENTS @@ -21,9 +32,6 @@ const argv = parseArgs({ }, }); -// Add this near the top with other constants -const SKIP_SPECS: string[] = ["type/file"]; - // Get the directory of the current file const PLUGIN_DIR = argv.values.pluginDir ? resolve(argv.values.pluginDir) @@ -33,290 +41,38 @@ const HTTP_SPECS = resolve(PLUGIN_DIR, "node_modules/@typespec/http-specs/specs" const GENERATED_FOLDER = argv.values.generatedFolder ? resolve(argv.values.generatedFolder) : resolve(PLUGIN_DIR, "generator"); - -interface TspCommand { - outputDir: string; - command: string[]; -} +const EMITTER_NAME = argv.values.emitterName || "@typespec/http-client-python"; const AZURE_EMITTER_OPTIONS: Record | Record[]> = { - "azure/client-generator-core/access": { - namespace: "specs.azure.clientgenerator.core.access", - }, - "azure/client-generator-core/alternate-type": { - namespace: "specs.azure.clientgenerator.core.alternatetype", - }, - "azure/client-generator-core/api-version": { - namespace: "specs.azure.clientgenerator.core.apiversion", - }, - "azure/client-generator-core/client-initialization/default": { - namespace: "specs.azure.clientgenerator.core.clientinitialization.default", - }, - "azure/client-generator-core/client-initialization/individually": { - namespace: "specs.azure.clientgenerator.core.clientinitialization.individually", - }, - "azure/client-generator-core/client-initialization/individuallyParent": { - namespace: "specs.azure.clientgenerator.core.clientinitialization.individuallyparent", - }, - "azure/client-generator-core/client-location": { - namespace: "specs.azure.clientgenerator.core.clientlocation", - }, - "azure/client-generator-core/deserialize-empty-string-as-null": { - namespace: "specs.azure.clientgenerator.core.emptystring", - }, - "azure/client-generator-core/flatten-property": { - namespace: "specs.azure.clientgenerator.core.flattenproperty", - }, - "azure/client-generator-core/usage": { - namespace: "specs.azure.clientgenerator.core.usage", - }, - "azure/client-generator-core/override": { - namespace: "specs.azure.clientgenerator.core.override", - }, - "azure/client-generator-core/hierarchy-building": { - namespace: "specs.azure.clientgenerator.core.hierarchybuilding", - }, - "azure/core/basic": { - namespace: "specs.azure.core.basic", - }, - "azure/core/lro/rpc": { - namespace: "specs.azure.core.lro.rpc", - }, - "azure/core/lro/standard": { - namespace: "specs.azure.core.lro.standard", - }, - "azure/core/model": { - namespace: "specs.azure.core.model", - }, - "azure/core/page": { - namespace: "specs.azure.core.page", - }, - "azure/core/scalar": { - namespace: "specs.azure.core.scalar", - }, - "azure/core/traits": { - namespace: "specs.azure.core.traits", - }, - "azure/encode/duration": { - namespace: "specs.azure.encode.duration", - }, - "azure/example/basic": { - namespace: "specs.azure.example.basic", - }, - "azure/payload/pageable": { - namespace: "specs.azure.payload.pageable", - }, - "azure/versioning/previewVersion": { - namespace: "specs.azure.versioning.previewversion", - }, - "client/structure/default": { - namespace: "client.structure.service", - }, + ...BASE_AZURE_EMITTER_OPTIONS, "client/structure/client-operation-group": { "package-name": "client-structure-clientoperationgroup", namespace: "client.structure.clientoperationgroup", }, - "client/structure/multi-client": { - "package-name": "client-structure-multiclient", - namespace: "client.structure.multiclient", - }, - "client/structure/renamed-operation": { - "package-name": "client-structure-renamedoperation", - namespace: "client.structure.renamedoperation", - }, - "client/structure/two-operation-group": { - "package-name": "client-structure-twooperationgroup", - namespace: "client.structure.twooperationgroup", - }, - "client/naming": { - namespace: "client.naming.main", - }, - "client/overload": { - namespace: "client.overload", - }, - "encode/duration": { - namespace: "encode.duration", - }, - "encode/numeric": { - namespace: "encode.numeric", - }, - "parameters/basic": { - namespace: "parameters.basic", - }, - "parameters/spread": { - namespace: "parameters.spread", - }, - "payload/content-negotiation": { - namespace: "payload.contentnegotiation", - }, - "payload/multipart": { - namespace: "payload.multipart", - }, - "serialization/encoded-name/json": { - namespace: "serialization.encodedname.json", - }, - "special-words": { - namespace: "specialwords", - }, - "service/multi-service": { - namespace: "service.multiservice", - }, }; const EMITTER_OPTIONS: Record | Record[]> = { - "resiliency/srv-driven/old.tsp": { - "package-name": "resiliency-srv-driven1", - namespace: "resiliency.srv.driven1", - "package-mode": "azure-dataplane", - "package-pprint-name": "ResiliencySrvDriven1", - }, - "resiliency/srv-driven": { - "package-name": "resiliency-srv-driven2", - namespace: "resiliency.srv.driven2", - "package-mode": "azure-dataplane", - "package-pprint-name": "ResiliencySrvDriven2", - }, - "authentication/api-key": { - "clear-output-folder": "true", - }, - "authentication/http/custom": { - "package-name": "authentication-http-custom", - namespace: "authentication.http.custom", - "package-pprint-name": "Authentication Http Custom", - }, - "authentication/union": [ - { - "package-name": "authentication-union", - namespace: "authentication.union", - }, - { - "package-name": "setuppy-authentication-union", - namespace: "setuppy.authentication.union", - "keep-setup-py": "true", - }, - ], + ...BASE_EMITTER_OPTIONS, "type/array": { "package-name": "typetest-array", namespace: "typetest.array", "use-pyodide": "true", }, - "type/dictionary": { - "package-name": "typetest-dictionary", - namespace: "typetest.dictionary", - }, - "type/enum/extensible": { - "package-name": "typetest-enum-extensible", - namespace: "typetest.enum.extensible", - }, - "type/enum/fixed": { - "package-name": "typetest-enum-fixed", - namespace: "typetest.enum.fixed", - }, - "type/model/empty": { - "package-name": "typetest-model-empty", - namespace: "typetest.model.empty", - }, - "type/model/inheritance/enum-discriminator": { - "package-name": "typetest-model-enumdiscriminator", - namespace: "typetest.model.enumdiscriminator", - }, - "type/model/inheritance/nested-discriminator": { - "package-name": "typetest-model-nesteddiscriminator", - namespace: "typetest.model.nesteddiscriminator", - }, - "type/model/inheritance/not-discriminated": { - "package-name": "typetest-model-notdiscriminated", - namespace: "typetest.model.notdiscriminated", - }, - "type/model/inheritance/single-discriminator": { - "package-name": "typetest-model-singlediscriminator", - namespace: "typetest.model.singlediscriminator", - }, "type/model/inheritance/recursive": { "package-name": "typetest-model-recursive", namespace: "typetest.model.recursive", "use-pyodide": "true", }, - "type/model/usage": { - "package-name": "typetest-model-usage", - namespace: "typetest.model.usage", - }, - "type/model/visibility": [ - { - "package-name": "typetest-model-visibility", - namespace: "typetest.model.visibility", - }, - { - "package-name": "headasbooleantrue", - namespace: "headasbooleantrue", - "head-as-boolean": "true", - }, - { - "package-name": "headasbooleanfalse", - namespace: "headasbooleanfalse", - "head-as-boolean": "false", - }, - ], - "type/property/nullable": { - "package-name": "typetest-property-nullable", - namespace: "typetest.property.nullable", - }, - "type/property/optionality": { - "package-name": "typetest-property-optional", - namespace: "typetest.property.optional", - }, - "type/property/additional-properties": { - "package-name": "typetest-property-additionalproperties", - namespace: "typetest.property.additionalproperties", - }, - "type/scalar": { - "package-name": "typetest-scalar", - namespace: "typetest.scalar", - }, - "type/property/value-types": { - "package-name": "typetest-property-valuetypes", - namespace: "typetest.property.valuetypes", - }, - "type/union": { - "package-name": "typetest-union", - namespace: "typetest.union", - }, - "type/union/discriminated": { - "package-name": "typetest-discriminatedunion", - namespace: "typetest.discriminatedunion", - }, - "type/file": { - "package-name": "typetest-file", - namespace: "typetest.file", - }, - documentation: { - "package-name": "specs-documentation", - namespace: "specs.documentation", - }, }; -function toPosix(dir: string): string { - return dir.replace(/\\/g, "/"); -} - -function getEmitterOption(spec: string, flavor: string): Record[] { - const specDir = spec.includes("azure") ? AZURE_HTTP_SPECS : HTTP_SPECS; - const relativeSpec = toPosix(relative(specDir, spec)); - const key = relativeSpec.includes("resiliency/srv-driven/old.tsp") - ? relativeSpec - : dirname(relativeSpec); - const emitter_options = EMITTER_OPTIONS[key] || - (flavor === "azure" ? AZURE_EMITTER_OPTIONS[key] : [{}]) || [{}]; - return Array.isArray(emitter_options) ? emitter_options : [emitter_options]; -} - // Function to execute CLI commands asynchronously async function executeCommand(tspCommand: TspCommand): Promise { + const cmd = tspCommand.command as string[]; const execFileAsync = promisify(execFile); try { - console.log(chalk.green(`start tsp ${tspCommand.command.join(" ")}`)); - await execFileAsync("tsp", tspCommand.command, { shell: true }); - console.log(chalk.green(`tsp ${tspCommand.command.join(" ")} succeeded`)); + console.log(chalk.green(`start tsp ${cmd.join(" ")}`)); + await execFileAsync("tsp", cmd, { shell: true }); + console.log(chalk.green(`tsp ${cmd.join(" ")} succeeded`)); } catch (err) { rmSync(tspCommand.outputDir, { recursive: true, force: true }); console.error(chalk.red(`exec error: ${err}`)); @@ -324,153 +80,6 @@ async function executeCommand(tspCommand: TspCommand): Promise { } } -interface RegenerateFlagsInput { - flavor?: string; - debug?: boolean; - name?: string; - pyodide?: boolean; -} - -interface RegenerateFlags { - flavor: string; - debug: boolean; - name?: string; - pyodide?: boolean; -} - -const SpecialFlags: Record> = { - azure: { - "generate-test": true, - "generate-sample": true, - }, -}; - -async function getSubdirectories(baseDir: string, flags: RegenerateFlags): Promise { - const subdirectories: string[] = []; - - async function searchDir(currentDir: string) { - const items = await promises.readdir(currentDir, { withFileTypes: true }); - - const promisesArray = items.map(async (item) => { - const subDirPath = join(currentDir, item.name); - if (item.isDirectory()) { - const mainTspPath = join(subDirPath, "main.tsp"); - const clientTspPath = join(subDirPath, "client.tsp"); - - const mainTspRelativePath = toPosix(relative(baseDir, mainTspPath)); - - // Replace the individual skip checks with: - if (SKIP_SPECS.some((skipSpec) => mainTspRelativePath.includes(skipSpec))) return; - - const hasMainTsp = await promises - .access(mainTspPath) - .then(() => true) - .catch(() => false); - const hasClientTsp = await promises - .access(clientTspPath) - .then(() => true) - .catch(() => false); - - if (mainTspRelativePath.toLowerCase().includes(flags.name || "")) { - if (mainTspRelativePath.includes("resiliency/srv-driven")) { - subdirectories.push(resolve(subDirPath, "old.tsp")); - } - if (hasClientTsp) { - subdirectories.push(resolve(subDirPath, "client.tsp")); - } else if (hasMainTsp) { - subdirectories.push(resolve(subDirPath, "main.tsp")); - } - } - - // Recursively search in the subdirectory - await searchDir(subDirPath); - } - }); - - await Promise.all(promisesArray); - } - - await searchDir(baseDir); - return subdirectories; -} - -function defaultPackageName(spec: string): string { - const specDir = spec.includes("azure") ? AZURE_HTTP_SPECS : HTTP_SPECS; - return toPosix(relative(specDir, dirname(spec))) - .replace(/\//g, "-") - .toLowerCase(); -} - -interface EmitterConfig { - options: string[]; - outputDir: string; -} - -function addOptions( - spec: string, - generatedFolder: string, - flags: RegenerateFlags, -): EmitterConfig[] { - const emitterConfigs: EmitterConfig[] = []; - for (const config of getEmitterOption(spec, flags.flavor)) { - const options: Record = { ...config }; - if (flags.pyodide) { - options["use-pyodide"] = "true"; - } - options["flavor"] = flags.flavor; - for (const [k, v] of Object.entries(SpecialFlags[flags.flavor] ?? {})) { - options[k] = v; - } - if (options["emitter-output-dir"] === undefined) { - const packageName = options["package-name"] || defaultPackageName(spec); - options["emitter-output-dir"] = toPosix( - `${generatedFolder}/test/${flags.flavor}/generated/${packageName}`, - ); - } - if (flags.debug) { - options["debug"] = "true"; - } - options["examples-dir"] = toPosix(join(dirname(spec), "examples")); - const configs = Object.entries(options).flatMap(([k, v]) => { - return [ - "--option", - `${argv.values.emitterName || "@typespec/http-client-python"}.${k}="${v}"`, - ]; - }); - emitterConfigs.push({ - options: configs, - outputDir: options["emitter-output-dir"], - }); - } - return emitterConfigs; -} -function _getCmdList(spec: string, flags: RegenerateFlags): TspCommand[] { - return addOptions(spec, GENERATED_FOLDER, flags).map((option) => { - return { - outputDir: option.outputDir, - command: ["compile", spec, "--emit", toPosix(PLUGIN_DIR), ...option.options], - }; - }); -} - -async function runTaskPool(tasks: Array<() => Promise>, poolLimit: number): Promise { - async function worker(start: number, end: number) { - while (start < end) { - await tasks[start](); - start++; - } - } - - const workers = []; - let start = 0; - while (start < tasks.length) { - const end = Math.min(start + poolLimit, tasks.length); - workers.push((async () => await worker(start, end))()); - start = end; - } - await Promise.all(workers); -} - // create some files before regeneration. After regeneration, these files should be deleted and we will test it // in test case async function preprocess(flags: RegenerateFlagsInput): Promise { @@ -493,36 +102,31 @@ async function preprocess(flags: RegenerateFlagsInput): Promise { } } -async function regenerate(flags: RegenerateFlagsInput): Promise { - if (flags.flavor === undefined) { - await regenerate({ flavor: "azure", ...flags }); - await regenerate({ flavor: "unbranded", ...flags }); - } else { - await preprocess(flags); - - const flagsResolved = { debug: false, flavor: flags.flavor, ...flags }; - const subdirectoriesForAzure = await getSubdirectories(AZURE_HTTP_SPECS, flagsResolved); - const subdirectoriesForNonAzure = await getSubdirectories(HTTP_SPECS, flagsResolved); - const subdirectories = - flags.flavor === "azure" - ? [...subdirectoriesForAzure, ...subdirectoriesForNonAzure] - : subdirectoriesForNonAzure; - const cmdList: TspCommand[] = subdirectories.flatMap((subdirectory) => - _getCmdList(subdirectory, flagsResolved), - ); - - // Create tasks as functions for the pool - const tasks: Array<() => Promise> = cmdList.map((tspCommand) => { - return () => executeCommand(tspCommand); - }); - - // Run tasks with a concurrency limit - await runTaskPool(tasks, 30); - } +function _getCmdList(spec: string, flags: RegenerateFlags): TspCommand[] { + return buildOptions(spec, GENERATED_FOLDER, flags, config).map((po) => { + const optionArgs = Object.entries(po.options).flatMap(([k, v]) => [ + "--option", + `${EMITTER_NAME}.${k}="${v}"`, + ]); + return { + outputDir: po.outputDir, + command: ["compile", spec, "--emit", toPosix(PLUGIN_DIR), ...optionArgs], + }; + }); } +const config: RegenerateConfig = { + azureHttpSpecs: AZURE_HTTP_SPECS, + httpSpecs: HTTP_SPECS, + emitterOptions: EMITTER_OPTIONS, + azureEmitterOptions: AZURE_EMITTER_OPTIONS, + preprocess, + getCmdList: _getCmdList, + executeCommand, +}; + const start = performance.now(); -regenerate(argv.values) +regenerate(argv.values, config) .then(() => console.log( chalk.green( diff --git a/packages/http-client-python/generator/test/azure/mock_api_tests/test_clear_output_folder.py b/packages/http-client-python/generator/test/azure/mock_api_tests/test_clear_output_folder.py index 5c85955dbe0..c49c8bf1d94 100644 --- a/packages/http-client-python/generator/test/azure/mock_api_tests/test_clear_output_folder.py +++ b/packages/http-client-python/generator/test/azure/mock_api_tests/test_clear_output_folder.py @@ -12,3 +12,12 @@ def test_clear_output_folder(): folder = GENERATED_PATH / "authentication-api-key/authentication/apikey/_operations" assert folder.exists(), "Operations folder should exist" assert not (folder / "to_be_deleted.py").exists(), "File to_be_deleted.py should be deleted after regeneration" + + if (GENERATED_PATH / "generation-subdir").exists(): + assert (GENERATED_PATH / "generation-subdir/generated_tests").exists() + assert not (GENERATED_PATH / "generation-subdir/generated_tests/to_be_deleted.py").exists() + + assert (GENERATED_PATH / "generation-subdir/generation/subdir/_generated").exists() + assert not (GENERATED_PATH / "generation-subdir/generation/subdir/_generated/to_be_deleted.py").exists() + + assert (GENERATED_PATH / "generation-subdir/generation/subdir/to_be_kept.py").exists() diff --git a/packages/http-client-python/generator/test/azure/requirements.txt b/packages/http-client-python/generator/test/azure/requirements.txt index 0a0439bdccb..996b8b767cb 100644 --- a/packages/http-client-python/generator/test/azure/requirements.txt +++ b/packages/http-client-python/generator/test/azure/requirements.txt @@ -2,6 +2,7 @@ -e ../../ azure-mgmt-core==1.6.0 +# === common azure dependencies across repos === # only for azure -e ./generated/azure-client-generator-core-access -e ./generated/azure-client-generator-core-api-version-header @@ -55,8 +56,9 @@ azure-mgmt-core==1.6.0 -e ./generated/resiliency-srv-driven1 -e ./generated/resiliency-srv-driven2 -e ./generated/service-multi-service +# === end common azure dependencies across repos === -# common test case +# === common test dependencies across repos === -e ./generated/authentication-api-key -e ./generated/authentication-http-custom -e ./generated/authentication-noauth-union @@ -121,3 +123,4 @@ azure-mgmt-core==1.6.0 -e ./generated/versioning-renamedfrom -e ./generated/versioning-returntypechangedfrom -e ./generated/versioning-typechangedfrom +# === end common test dependencies across repos === diff --git a/packages/http-client-python/generator/test/generic_mock_api_tests/asynctests/test_encode_bytes_async.py b/packages/http-client-python/generator/test/generic_mock_api_tests/asynctests/test_encode_bytes_async.py index 93f8f62c68d..584b277edd9 100644 --- a/packages/http-client-python/generator/test/generic_mock_api_tests/asynctests/test_encode_bytes_async.py +++ b/packages/http-client-python/generator/test/generic_mock_api_tests/asynctests/test_encode_bytes_async.py @@ -13,7 +13,6 @@ Base64urlArrayBytesProperty, ) - FILE_FOLDER = Path(__file__).parent.parent @@ -98,6 +97,12 @@ async def test_header(client: BytesClient): ) +@pytest.fixture +def png_data() -> bytes: + with open(str(FILE_FOLDER / "data/image.png"), "rb") as file_in: + return file_in.read() + + @pytest.mark.asyncio async def test_request_body(client: BytesClient, png_data: bytes): await client.request_body.default( diff --git a/packages/http-client-python/generator/test/generic_mock_api_tests/data/image.jpg b/packages/http-client-python/generator/test/generic_mock_api_tests/data/image.jpg new file mode 100644 index 00000000000..b95b3e7b582 Binary files /dev/null and b/packages/http-client-python/generator/test/generic_mock_api_tests/data/image.jpg differ diff --git a/packages/http-client-python/generator/test/generic_mock_api_tests/data/image.png b/packages/http-client-python/generator/test/generic_mock_api_tests/data/image.png new file mode 100644 index 00000000000..42fe8dc1456 Binary files /dev/null and b/packages/http-client-python/generator/test/generic_mock_api_tests/data/image.png differ diff --git a/packages/http-client-python/generator/test/generic_mock_api_tests/test_encode_bytes.py b/packages/http-client-python/generator/test/generic_mock_api_tests/test_encode_bytes.py index 615d4f050e0..2ca75a22c31 100644 --- a/packages/http-client-python/generator/test/generic_mock_api_tests/test_encode_bytes.py +++ b/packages/http-client-python/generator/test/generic_mock_api_tests/test_encode_bytes.py @@ -95,6 +95,12 @@ def test_header(client: BytesClient): ) +@pytest.fixture +def png_data() -> bytes: + with open(str(FILE_FOLDER / "data/image.png"), "rb") as file_in: + return file_in.read() + + def test_request_body(client: BytesClient, png_data: bytes): client.request_body.default( value=png_data, diff --git a/packages/http-client-python/generator/test/unbranded/mock_api_tests/test_unbranded.py b/packages/http-client-python/generator/test/unbranded/mock_api_tests/test_unbranded.py index a37c81ccef3..19d767860f8 100644 --- a/packages/http-client-python/generator/test/unbranded/mock_api_tests/test_unbranded.py +++ b/packages/http-client-python/generator/test/unbranded/mock_api_tests/test_unbranded.py @@ -54,11 +54,12 @@ def test_sensitive_word(): check_folder = (Path(os.path.dirname(__file__)) / "../generated").resolve() assert [] == check_sensitive_word(check_folder, "azure") # after update spector, it shall also equal to [] - assert sorted( - [ - "authentication-oauth2", - "authentication-noauth-union", - "authentication-union", - "setuppy-authentication-union", - ] - ) == sorted(check_sensitive_word(check_folder, "microsoft")) + expected = [ + "authentication-oauth2", + "authentication-noauth-union", + "authentication-union", + "setuppy-authentication-union", + ] + if (check_folder / "generation-subdir").exists(): + expected.append("generation-subdir") + assert sorted(expected) == sorted(check_sensitive_word(check_folder, "microsoft")) diff --git a/packages/http-client-python/generator/test/unbranded/requirements.txt b/packages/http-client-python/generator/test/unbranded/requirements.txt index 7dfecf1f78b..9fd6dd58cbc 100644 --- a/packages/http-client-python/generator/test/unbranded/requirements.txt +++ b/packages/http-client-python/generator/test/unbranded/requirements.txt @@ -1,7 +1,7 @@ -r ../dev_requirements.txt -e ../../ -# common test case +# === common test dependencies across repos === -e ./generated/authentication-api-key -e ./generated/authentication-http-custom -e ./generated/authentication-noauth-union @@ -66,3 +66,4 @@ -e ./generated/versioning-renamedfrom -e ./generated/versioning-returntypechangedfrom -e ./generated/versioning-typechangedfrom +# === end common test dependencies across repos ===