[http-client-python] add pyproject toml

.chronus/changes/http-client-python-add-pyproject-toml-2025-6-16-18-9-43.md

@@ -0,0 +1,7 @@
+---
+changeKind: feature
+packages:
+  - "@typespec/http-client-python"
+---
+
+Adding pyproject.toml generation and optional keep-setup-py flag

cspell.yaml

@@ -199,6 +199,7 @@ words:
   - pyright
   - pyrightconfig
   - pytest
+  - pytyped
   - pyyaml
   - rcfile
   - reactivex
@@ -218,6 +219,7 @@ words:
   - safeint
   - sdkcore
   - segmentof
+  - setuppy
   - serde
   - sfixed
   - shiki

packages/http-client-python/emitter/src/emitter.ts

@@ -152,6 +152,7 @@ async function onEmitMain(context: EmitContext<PythonEmitterOptions>) {
   }
   if (resolvedOptions["generate-packaging-files"]) {
     commandArgs["package-mode"] = sdkContext.arm ? "azure-mgmt" : "azure-dataplane";
+    commandArgs["keep-setup-py"] = resolvedOptions["keep-setup-py"] === true ? "true" : "false";
   }
   if (sdkContext.arm === true) {
     commandArgs["azure-arm"] = "true";

packages/http-client-python/emitter/src/lib.ts

@@ -22,6 +22,7 @@ export interface PythonEmitterOptions {
   "package-pprint-name"?: string;
   "head-as-boolean"?: boolean;
   "use-pyodide"?: boolean;
+  "keep-setup-py"?: boolean;
 }
 
 export interface PythonSdkContext extends SdkContext<PythonEmitterOptions> {
@@ -86,6 +87,12 @@ export const PythonEmitterOptionsSchema: JSONSchemaType<PythonEmitterOptions> =
       description:
         "Whether to validate the versioning of the package. Defaults to `true`. If set to `false`, we will not validate the versioning of the package.",
     },
+    "keep-setup-py": {
+      type: "boolean",
+      nullable: true,
+      description:
+        "Whether to keep the existing `setup.py` when `generate-packaging-files` is `true`. If set to `false` and by default, `pyproject.toml` will be generated instead. To generate `setup.py`, use `basic-setup-py`.",
+    },
   },
   required: [],
 };

packages/http-client-python/eng/scripts/ci/regenerate.ts

@@ -153,10 +153,17 @@ const EMITTER_OPTIONS: Record<string, Record<string, string> | Record<string, st
     namespace: "authentication.http.custom",
     "package-pprint-name": "Authentication Http Custom",
   },
-  "authentication/union": {
-    "package-name": "authentication-union",
-    namespace: "authentication.union",
-  },
+  "authentication/union": [
+    {
+      "package-name": "authentication-union",
+      namespace: "authentication.union",
+    },
+    {
+      "package-name": "setuppy-authentication-union",
+      namespace: "setuppy.authentication.union",
+      "keep-setup-py": "true",
+    },
+  ],
   "type/array": {
     "package-name": "typetest-array",
     namespace: "typetest.array",

packages/http-client-python/generator/pygen/__init__.py

@@ -29,6 +29,7 @@ class OptionsDict(MutableMapping):
         "flavor": "azure",  # need to default to azure in shared code so we don't break swagger generation
         "from-typespec": False,
         "generate-sample": False,
+        "keep-setup-py": False,
         "generate-test": False,
         "head-as-boolean": True,
         "keep-version-file": False,
@@ -134,6 +135,10 @@ def _validate_combinations(self) -> None:
                 "We are working on creating a new multiapi SDK for version tolerant and it is not available yet."
             )
 
+        # If multiapi, do not generate default pyproject.toml
+        if self.get("multiapi"):
+            self["keep-setup-py"] = True
+
         if self.get("client-side-validation") and self.get("version-tolerant"):
             raise ValueError("Can not generate version tolerant with --client-side-validation. ")
 
@@ -210,6 +215,9 @@ def __init__(self, *, output_folder: Union[str, Path], **kwargs: Any) -> None:
             _LOGGER.warning("Loading python.json file. This behavior will be depreacted")
         self.options.update(python_json)
 
+    def get_output_folder(self) -> Path:
+        return self.output_folder
+
     def read_file(self, path: Union[str, Path]) -> str:
         """Directly reading from disk"""
         # make path relative to output folder
@@ -227,6 +235,14 @@ def write_file(self, filename: Union[str, Path], file_content: str) -> None:
         with open(self.output_folder / Path(filename), "w", encoding="utf-8") as fd:
             fd.write(file_content)
 
+    def remove_file(self, filename: Union[str, Path]) -> None:
+        try:
+            file_path = self.output_folder / Path(filename)
+            if file_path.is_file():
+                file_path.unlink()
+        except FileNotFoundError:
+            pass
+
     def list_file(self) -> List[str]:
         return [str(f.relative_to(self.output_folder)) for f in self.output_folder.glob("**/*") if f.is_file()]
 

packages/http-client-python/generator/pygen/codegen/serializers/__init__.py

@@ -9,6 +9,7 @@
 import re
 from typing import List, Any, Union
 from pathlib import Path
+from packaging.version import parse as parse_version
 from jinja2 import PackageLoader, Environment, FileSystemLoader, StrictUndefined
 
 from ... import ReaderAndWriter
@@ -52,10 +53,9 @@
     "LICENSE.jinja2",
     "MANIFEST.in.jinja2",
     "README.md.jinja2",
-    "setup.py.jinja2",
 ]
 
-_REGENERATE_FILES = {"setup.py", "MANIFEST.in"}
+_REGENERATE_FILES = {"MANIFEST.in"}
 AsyncInfo = namedtuple("AsyncInfo", ["async_mode", "async_path"])
 
 
@@ -80,6 +80,15 @@ def __init__(
     ) -> None:
         super().__init__(output_folder=output_folder, **kwargs)
         self.code_model = code_model
+        self._regenerate_setup_py()
+
+    def _regenerate_setup_py(self):
+        if self.code_model.options["keep-setup-py"] or self.code_model.options["basic-setup-py"]:
+            _PACKAGE_FILES.append("setup.py.jinja2")
+            _REGENERATE_FILES.add("setup.py")
+        else:
+            _PACKAGE_FILES.append("pyproject.toml.jinja2")
+            _REGENERATE_FILES.add("pyproject.toml")
 
     @property
     def has_aio_folder(self) -> bool:
@@ -106,7 +115,11 @@ def keep_version_file(self) -> bool:
             serialized_version = match.group(1) if match else ""
         except (FileNotFoundError, IndexError):
             serialized_version = ""
-        return serialized_version > self.code_model.options.get("package-version", "")
+        try:
+            return parse_version(serialized_version) > parse_version(self.code_model.options.get("package-version", ""))
+        except Exception:  # pylint: disable=broad-except
+            # If parsing the version fails, we assume the version file is not valid and overwrite.
+            return False
 
     def serialize(self) -> None:
         env = Environment(
@@ -122,9 +135,12 @@ def serialize(self) -> None:
         for client_namespace, client_namespace_type in self.code_model.client_namespace_types.items():
             exec_path = self.exec_path(client_namespace)
             if client_namespace == "":
-                # Write the setup file
                 if self.code_model.options["basic-setup-py"]:
+                    # Write the setup file
                     self.write_file(exec_path / Path("setup.py"), general_serializer.serialize_setup_file())
+                elif not self.code_model.options["keep-setup-py"]:
+                    # remove setup.py file
+                    self.remove_file(exec_path / Path("setup.py"))
 
                 # add packaging files in root namespace (e.g. setup.py, README.md, etc.)
                 if self.code_model.options.get("package-mode"):
@@ -231,9 +247,10 @@ def _serialize_and_write_package_files(self, client_namespace: str) -> None:
                 if self.keep_version_file and file == "setup.py" and not self.code_model.options["azure-arm"]:
                     # don't regenerate setup.py file if the version file is more up to date for data-plane
                     continue
+                file_path = self.get_output_folder() / Path(output_name)
                 self.write_file(
                     output_name,
-                    serializer.serialize_package_file(template_name, **params),
+                    serializer.serialize_package_file(template_name, file_path, **params),
                 )
 
     def _keep_patch_file(self, path_file: Path, env: Environment):

packages/http-client-python/generator/pygen/codegen/serializers/general_serializer.py

@@ -4,7 +4,10 @@
 # license information.
 # --------------------------------------------------------------------------
 import json
-from typing import Any, List
+from typing import Any, List, TYPE_CHECKING
+import re
+import tomli as tomllib
+from packaging.version import parse as parse_version
 from .import_serializer import FileImportSerializer, TypingSection
 from ..models.imports import MsrestImportType, FileImport
 from ..models import (
@@ -16,6 +19,9 @@
 from .client_serializer import ClientSerializer, ConfigSerializer
 from .base_serializer import BaseSerializer
 
+if TYPE_CHECKING:
+    from pathlib import Path
+
 VERSION_MAP = {
     "msrest": "0.7.1",
     "isodate": "0.6.1",
@@ -42,8 +48,73 @@ def serialize_setup_file(self) -> str:
         params.update({"options": self.code_model.options})
         return template.render(code_model=self.code_model, **params)
 
-    def serialize_package_file(self, template_name: str, **kwargs: Any) -> str:
+    def _extract_min_dependency(self, s):
+        # Extract the minimum version from a dependency string.
+        #
+        # Handles formats like:
+        # - >=1.2.3
+        # - >=0.1.0b1 (beta versions)
+        # - >=1.2.3rc2 (release candidates)
+        #
+        # Returns the parsed version if found, otherwise version "0".
+        m = re.search(r"[>=]=?([\d.]+(?:[a-z]+\d+)?)", s)
+        return parse_version(m.group(1)) if m else parse_version("0")
+
+    def _keep_pyproject_fields(self, file_path: "Path") -> dict:
+        # Load the pyproject.toml file if it exists and extract fields to keep.
+        result: dict = {"KEEP_FIELDS": {}}
+        try:
+            with open(file_path, "rb") as f:
+                loaded_pyproject_toml = tomllib.load(f)
+        except Exception:  # pylint: disable=broad-except
+            # If parsing the pyproject.toml fails, we assume the it does not exist or is incorrectly formatted.
+            return result
+
+        # Keep azure-sdk-build configuration
+        if "tool" in loaded_pyproject_toml and "azure-sdk-build" in loaded_pyproject_toml["tool"]:
+            result["KEEP_FIELDS"]["tool.azure-sdk-build"] = loaded_pyproject_toml["tool"]["azure-sdk-build"]
+
+        # Process dependencies
+        if "project" in loaded_pyproject_toml:
+            # Handle main dependencies
+            if "dependencies" in loaded_pyproject_toml["project"]:
+                kept_deps = []
+                for dep in loaded_pyproject_toml["project"]["dependencies"]:
+                    dep_name = re.split(r"[<>=\[]", dep)[0].strip()
+
+                    # Check if dependency is one we track in VERSION_MAP
+                    if dep_name in VERSION_MAP:
+                        # For tracked dependencies, check if the version is higher than our default
+                        default_version = parse_version(VERSION_MAP[dep_name])
+                        dep_version = self._extract_min_dependency(dep)
+                        # If the version is higher than the default, update VERSION_MAP
+                        # with higher min dependency version
+                        if dep_version > default_version:
+                            VERSION_MAP[dep_name] = str(dep_version)
+                    else:
+                        # Keep non-default dependencies
+                        kept_deps.append(dep)
+
+                if kept_deps:
+                    result["KEEP_FIELDS"]["project.dependencies"] = kept_deps
+
+            # Keep optional dependencies
+            if "optional-dependencies" in loaded_pyproject_toml["project"]:
+                result["KEEP_FIELDS"]["project.optional-dependencies"] = loaded_pyproject_toml["project"][
+                    "optional-dependencies"
+                ]
+
+        return result
+
+    def serialize_package_file(self, template_name: str, file_path: "Path", **kwargs: Any) -> str:
         template = self.env.get_template(template_name)
+
+        # Add fields to keep from an existing pyproject.toml
+        if template_name == "pyproject.toml.jinja2":
+            params = self._keep_pyproject_fields(file_path)
+        else:
+            params = {}
+
         package_parts = (
             self.code_model.namespace.split(".")[:-1]
             if self.code_model.is_tsp
@@ -57,17 +128,19 @@ def serialize_package_file(self, template_name: str, **kwargs: Any) -> str:
             dev_status = "4 - Beta"
         else:
             dev_status = "5 - Production/Stable"
-        params = {
-            "code_model": self.code_model,
-            "dev_status": dev_status,
-            "token_credential": token_credential,
-            "pkgutil_names": [".".join(package_parts[: i + 1]) for i in range(len(package_parts))],
-            "init_names": ["/".join(package_parts[: i + 1]) + "/__init__.py" for i in range(len(package_parts))],
-            "client_name": self.code_model.clients[0].name if self.code_model.clients else "",
-            "VERSION_MAP": VERSION_MAP,
-            "MIN_PYTHON_VERSION": MIN_PYTHON_VERSION,
-            "MAX_PYTHON_VERSION": MAX_PYTHON_VERSION,
-        }
+        params.update(
+            {
+                "code_model": self.code_model,
+                "dev_status": dev_status,
+                "token_credential": token_credential,
+                "pkgutil_names": [".".join(package_parts[: i + 1]) for i in range(len(package_parts))],
+                "init_names": ["/".join(package_parts[: i + 1]) + "/__init__.py" for i in range(len(package_parts))],
+                "client_name": self.code_model.clients[0].name if self.code_model.clients else "",
+                "VERSION_MAP": VERSION_MAP,
+                "MIN_PYTHON_VERSION": MIN_PYTHON_VERSION,
+                "MAX_PYTHON_VERSION": MAX_PYTHON_VERSION,
+            }
+        )
         params.update({"options": self.code_model.options})
         params.update(kwargs)
         return template.render(file_import=FileImport(self.code_model), **params)