From 6124c496f1a9442468c5024042d88e9360de7f2c Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Sat, 11 Oct 2025 13:42:10 +0800 Subject: [PATCH 1/7] add clear output folder logic for tsp --- .../eng/scripts/ci/regenerate.ts | 24 +++++++++++++++++++ .../generator/pygen/__init__.py | 15 ++++++++++++ .../pygen/codegen/models/operation.py | 4 ++-- .../pygen/codegen/serializers/__init__.py | 20 ++++++++++++++-- .../codegen/serializers/builder_serializer.py | 14 ++++++----- .../test_clear_output_folder.py | 14 +++++++++++ 6 files changed, 81 insertions(+), 10 deletions(-) create mode 100644 packages/http-client-python/generator/test/azure/mock_api_tests/test_clear_output_folder.py diff --git a/packages/http-client-python/eng/scripts/ci/regenerate.ts b/packages/http-client-python/eng/scripts/ci/regenerate.ts index 6e5fd8627a1..c003eea4249 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate.ts @@ -447,11 +447,35 @@ async function runTaskPool(tasks: Array<() => Promise>, poolLimit: number) await Promise.all(workers); } +// create some files before regeneration. After regeneration, these files should be deleted and we will test it +// in test case +async function preprocess(flags: RegenerateFlagsInput): Promise { + if (flags.flavor === "azure") { + // create folder if not exists + const folderParts = [ + "test", + "azure", + "generated", + "authentication-api-key", + "authentication", + "apiKey", + "_operations", + ]; + await promises.mkdir(join(GENERATED_FOLDER, ...folderParts), { recursive: true }); + await promises.writeFile( + join(GENERATED_FOLDER, ...folderParts, "to_be_deleted.py"), + "# This file is to be deleted after regeneration", + ); + } +} + async function regenerate(flags: RegenerateFlagsInput): Promise { if (flags.flavor === undefined) { await regenerate({ flavor: "azure", ...flags }); await regenerate({ flavor: "unbranded", ...flags }); } else { + await preprocess(flags); + const flagsResolved = { debug: false, flavor: flags.flavor, ...flags }; const subdirectoriesForAzure = await getSubdirectories(AZURE_HTTP_SPECS, flagsResolved); const subdirectoriesForNonAzure = await getSubdirectories(HTTP_SPECS, flagsResolved); diff --git a/packages/http-client-python/generator/pygen/__init__.py b/packages/http-client-python/generator/pygen/__init__.py index ebafa1aab7e..09fef604388 100644 --- a/packages/http-client-python/generator/pygen/__init__.py +++ b/packages/http-client-python/generator/pygen/__init__.py @@ -3,6 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- +import shutil from collections.abc import ItemsView, KeysView, MutableMapping, ValuesView import logging from pathlib import Path @@ -248,9 +249,23 @@ def remove_file(self, filename: Union[str, Path]) -> None: except FileNotFoundError: pass + def remove_folder(self, foldername: Union[str, Path]) -> None: + try: + folder_path = self.output_folder / Path(foldername) + if folder_path.exists() and folder_path.is_dir(): + shutil.rmtree(folder_path) + except FileNotFoundError: + pass + def list_file(self) -> list[str]: return [str(f.relative_to(self.output_folder)) for f in self.output_folder.glob("**/*") if f.is_file()] + def list_file_of_folder(self, foldername: Union[str, Path]) -> list[str]: + folder_path = self.output_folder / Path(foldername) + if folder_path.exists() and folder_path.is_dir(): + return [str(f.relative_to(self.output_folder)) for f in folder_path.glob("**/*") if f.is_file()] + return [] + class Plugin(ReaderAndWriter, ABC): """A base class for autorest plugin. diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation.py b/packages/http-client-python/generator/pygen/codegen/models/operation.py index 94036287ed4..f9e5e623825 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation.py @@ -200,8 +200,8 @@ def default_error_deserialization(self, serialize_namespace: str) -> Optional[st if isinstance(exception_schema, ModelType): pylint_disable = " # pylint: disable=protected-access" if exception_schema.internal else "" return ( - f"{exception_schema.type_annotation(skip_quote=True, serialize_namespace=serialize_namespace)}," - f"{pylint_disable}" + f"{exception_schema.type_annotation(skip_quote=True, serialize_namespace=serialize_namespace)}," + f"{pylint_disable}" ) return None if self.code_model.options["models-mode"] == "dpg" else "'object'," diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index fbcbb83be31..909712165d5 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -123,6 +123,19 @@ def keep_version_file(self) -> bool: return False def serialize(self) -> None: + # remove existing folders when generate from tsp + if self.code_model.is_tsp and self.code_model.is_azure_flavor: + # remove generated_samples and generated_tests folder + self.remove_folder(self._generated_tests_samples_folder("generated_samples")) + self.remove_folder(self._generated_tests_samples_folder("generated_tests")) + + # remove generated sdk files + generation_path = self.code_model.get_generation_dir("") + for file in self.list_file_of_folder(generation_path): + if file.endswith(".py") and "_patch.py" not in file: + self.remove_file(file) + + # serialize logic env = Environment( loader=PackageLoader("pygen.codegen", "templates"), keep_trailing_newline=True, @@ -519,8 +532,11 @@ def sample_additional_folder(self) -> Path: return Path("/".join(namespace_config.split(".")[num_of_package_namespace:])) return Path("") + def _generated_tests_samples_folder(self, folder_name: str) -> Path: + return self._root_of_sdk / folder_name + def _serialize_and_write_sample(self, env: Environment): - out_path = self._root_of_sdk / "generated_samples" + out_path = self._generated_tests_samples_folder("generated_samples") for client in self.code_model.clients: for op_group in client.operation_groups: for operation in op_group.operations: @@ -549,7 +565,7 @@ def _serialize_and_write_sample(self, env: Environment): def _serialize_and_write_test(self, env: Environment): self.code_model.for_test = True - out_path = self._root_of_sdk / "generated_tests" + out_path = self._generated_tests_samples_folder("generated_tests") general_serializer = TestGeneralSerializer(code_model=self.code_model, env=env) self.write_file(out_path / "conftest.py", general_serializer.serialize_conftest()) if not self.code_model.options["azure-arm"]: diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py index d630c015df9..ae12128a3b1 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py @@ -1117,12 +1117,14 @@ def handle_error_response( # pylint: disable=too-many-statements, too-many-bran if builder.non_default_errors: retval.append(" else:") if self.code_model.options["models-mode"] == "dpg": - retval.extend([ - f"{indent}error = _failsafe_deserialize(", - f"{indent} {default_error_deserialization}", - f"{indent} response,", - f"{indent})", - ]) + retval.extend( + [ + f"{indent}error = _failsafe_deserialize(", + f"{indent} {default_error_deserialization}", + f"{indent} response,", + f"{indent})", + ] + ) else: retval.extend( [ diff --git a/packages/http-client-python/generator/test/azure/mock_api_tests/test_clear_output_folder.py b/packages/http-client-python/generator/test/azure/mock_api_tests/test_clear_output_folder.py new file mode 100644 index 00000000000..86f75c09ba6 --- /dev/null +++ b/packages/http-client-python/generator/test/azure/mock_api_tests/test_clear_output_folder.py @@ -0,0 +1,14 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from pathlib import Path + +GENERATED_PATH = Path(__file__).parent.resolve() / "generated" + + +def test_clear_output_folder(): + assert not ( + GENERATED_PATH / "authentication-api-key/authentication/apiKey/_operations/to_be_deleted.py" + ).exists(), "File to_be_deleted.py should be deleted after regeneration" From 6efef019e4316fa3aee0104aebc5d3d7fe5e04d1 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Sat, 11 Oct 2025 13:43:00 +0800 Subject: [PATCH 2/7] add changelog --- .../changes/clear-output-dir-python-2025-9-11-13-42-52.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .chronus/changes/clear-output-dir-python-2025-9-11-13-42-52.md diff --git a/.chronus/changes/clear-output-dir-python-2025-9-11-13-42-52.md b/.chronus/changes/clear-output-dir-python-2025-9-11-13-42-52.md new file mode 100644 index 00000000000..2834afa138f --- /dev/null +++ b/.chronus/changes/clear-output-dir-python-2025-9-11-13-42-52.md @@ -0,0 +1,7 @@ +--- +changeKind: feature +packages: + - "@typespec/http-client-python" +--- + +Add logic to clear output folder \ No newline at end of file From 2a0f7c30cb0a2a9fcac59791b0a4fc5c65d47d02 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Sat, 11 Oct 2025 13:54:54 +0800 Subject: [PATCH 3/7] fix lint issue --- .../generator/pygen/codegen/serializers/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 909712165d5..f84f8dd54e0 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -122,6 +122,7 @@ def keep_version_file(self) -> bool: # If parsing the version fails, we assume the version file is not valid and overwrite. return False + # pylint: disable=too-many-branches def serialize(self) -> None: # remove existing folders when generate from tsp if self.code_model.is_tsp and self.code_model.is_azure_flavor: From 5a5ff5ec7d342f2c3a8ae0323336a4d211195ed9 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Sat, 11 Oct 2025 15:41:16 +0800 Subject: [PATCH 4/7] fix regeneration --- packages/http-client-python/eng/scripts/ci/regenerate.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/http-client-python/eng/scripts/ci/regenerate.ts b/packages/http-client-python/eng/scripts/ci/regenerate.ts index c003eea4249..6c8bd1ab79a 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate.ts @@ -458,7 +458,7 @@ async function preprocess(flags: RegenerateFlagsInput): Promise { "generated", "authentication-api-key", "authentication", - "apiKey", + "apikey", "_operations", ]; await promises.mkdir(join(GENERATED_FOLDER, ...folderParts), { recursive: true }); From 15ce0b5b05088af99be549fbcb19963bbba66291 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Sat, 11 Oct 2025 17:46:36 +0800 Subject: [PATCH 5/7] fix test case --- .../test/azure/mock_api_tests/test_clear_output_folder.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/http-client-python/generator/test/azure/mock_api_tests/test_clear_output_folder.py b/packages/http-client-python/generator/test/azure/mock_api_tests/test_clear_output_folder.py index 86f75c09ba6..775a920519d 100644 --- a/packages/http-client-python/generator/test/azure/mock_api_tests/test_clear_output_folder.py +++ b/packages/http-client-python/generator/test/azure/mock_api_tests/test_clear_output_folder.py @@ -5,10 +5,10 @@ # -------------------------------------------------------------------------- from pathlib import Path -GENERATED_PATH = Path(__file__).parent.resolve() / "generated" +GENERATED_PATH = Path(__file__).parent.parent.resolve() / "generated" def test_clear_output_folder(): - assert not ( - GENERATED_PATH / "authentication-api-key/authentication/apiKey/_operations/to_be_deleted.py" - ).exists(), "File to_be_deleted.py should be deleted after regeneration" + folder = GENERATED_PATH / "authentication-api-key/authentication/apiKey/_operations" + assert folder.exists(), "Operations folder should exist" + assert not (folder / "to_be_deleted.py").exists(), "File to_be_deleted.py should be deleted after regeneration" From cb3f4db6c41b7eff3d026ca9572b2fdf24649e95 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Sat, 11 Oct 2025 17:49:59 +0800 Subject: [PATCH 6/7] fix test case --- .../test/azure/mock_api_tests/test_clear_output_folder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/http-client-python/generator/test/azure/mock_api_tests/test_clear_output_folder.py b/packages/http-client-python/generator/test/azure/mock_api_tests/test_clear_output_folder.py index 775a920519d..5c85955dbe0 100644 --- a/packages/http-client-python/generator/test/azure/mock_api_tests/test_clear_output_folder.py +++ b/packages/http-client-python/generator/test/azure/mock_api_tests/test_clear_output_folder.py @@ -9,6 +9,6 @@ def test_clear_output_folder(): - folder = GENERATED_PATH / "authentication-api-key/authentication/apiKey/_operations" + folder = GENERATED_PATH / "authentication-api-key/authentication/apikey/_operations" assert folder.exists(), "Operations folder should exist" assert not (folder / "to_be_deleted.py").exists(), "File to_be_deleted.py should be deleted after regeneration" From 1d565adde3dbe502dacc27555fd699adf9a7281e Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 29 Oct 2025 13:11:36 +0800 Subject: [PATCH 7/7] only remove files of inner path (e.g. azure/mgmt/test/...) --- .../generator/pygen/codegen/serializers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index f84f8dd54e0..4ec430ae5a5 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -131,7 +131,7 @@ def serialize(self) -> None: self.remove_folder(self._generated_tests_samples_folder("generated_tests")) # remove generated sdk files - generation_path = self.code_model.get_generation_dir("") + generation_path = self.code_model.get_generation_dir(self.code_model.namespace) for file in self.list_file_of_folder(generation_path): if file.endswith(".py") and "_patch.py" not in file: self.remove_file(file)