From e895e870b0f4dbc7608b323b3f11f68756e40c3f Mon Sep 17 00:00:00 2001 From: xnuinside Date: Sun, 18 Jan 2026 16:29:57 +0300 Subject: [PATCH] Add Pydal model conversion support (issue #30) - Add convert_models() support for Pydal table definitions - Support conversion to all model types: SQLAlchemy, SQLAlchemy v2, Gino, Pydantic, Pydantic v2, Dataclass, SQLModel - Handle Pydal-specific types: id (primary key), string, text, integer, boolean, datetime, date, float, decimal - Convert 'reference table_name' to ForeignKey - Strip quotes from py_models_parser output - Preserve table names without pluralization (Pydal names are table names) - Add functional tests for all model types - Add integration tests for SQLAlchemy, Pydantic, and Dataclass --- CHANGELOG.md | 7 + omymodels/converter.py | 72 +++++- .../converter/test_pydal_conversion.py | 242 ++++++++++++++++++ tests/integration/converter/__init__.py | 0 tests/integration/converter/conftest.py | 26 ++ .../converter/test_pydal_conversion.py | 182 +++++++++++++ 6 files changed, 527 insertions(+), 2 deletions(-) create mode 100644 tests/functional/converter/test_pydal_conversion.py create mode 100644 tests/integration/converter/__init__.py create mode 100644 tests/integration/converter/conftest.py create mode 100644 tests/integration/converter/test_pydal_conversion.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dcc27c5..5bd03fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Works with both `sqlalchemy` and `sqlalchemy_v2` model types - File naming: `{schema_name}_{base_filename}.py` (e.g., `schema1_models.py`) +**Pydal Model Conversion (issue #30)** +- Convert Pydal table definitions to any supported model type using `convert_models()` +- Supports all output formats: SQLAlchemy, SQLAlchemy v2, Gino, Pydantic, Pydantic v2, Dataclass, SQLModel +- Handles Pydal types: `id`, `string`, `text`, `integer`, `boolean`, `datetime`, `date`, `float`, `decimal` +- Pydal's `id` type maps to primary key +- Pydal's `reference table_name` type maps to foreign key + **SQLModel Improvements** - Fixed array type generation (issue #66) - Arrays now properly generate `List[T]` with correct SQLAlchemy ARRAY type diff --git a/omymodels/converter.py b/omymodels/converter.py index bbb895e..bcad3ff 100644 --- a/omymodels/converter.py +++ b/omymodels/converter.py @@ -8,6 +8,69 @@ from omymodels.models.enum import core as enum +def _strip_quotes(value: str) -> str: + """Strip surrounding quotes from a string value.""" + if value and isinstance(value, str): + return value.strip("'\"") + return value + + +def _is_pydal_result(data: List[Dict]) -> bool: + """Check if parsed data is from Pydal (names contain quotes).""" + if data and data[0].get("name", ""): + name = data[0]["name"] + return name.startswith(("'", '"')) and name.endswith(("'", '"')) + return False + + +def _process_pydal_type(col_type: str, attr: Dict) -> str: + """Process Pydal column type and handle special types. + + - 'id' type: Pydal's auto-generated primary key (maps to integer + primary_key) + - 'reference table_name': Foreign key to another table (maps to integer + FK) + """ + if col_type == "id": + # Pydal 'id' type is auto-generated primary key + attr["properties"]["primary_key"] = True + return "integer" + + if col_type and col_type.startswith("reference "): + ref_table = col_type.split(" ", 1)[1].strip() + # Store reference info for foreign key generation + # All keys expected by add_reference_to_the_column must be present + attr["references"] = { + "table": ref_table, + "column": "id", # Pydal references default to 'id' column + "schema": None, + "on_delete": None, + "on_update": None, + } + return "integer" # Reference fields are integers + + return col_type + + +def _clean_pydal_data(data: List[Dict]) -> List[Dict]: + """Clean Pydal parsed data by stripping quotes from names and types. + + For Pydal, the model name is already the table name, so we set table_name + directly to avoid incorrect pluralization. + """ + for model in data: + table_name = _strip_quotes(model.get("name", "")) + model["name"] = table_name + # For Pydal, the name is already the table name - mark it to skip pluralization + model["table_name"] = table_name + for attr in model.get("attrs", []): + attr["name"] = _strip_quotes(attr.get("name", "")) + if attr.get("type"): + col_type = _strip_quotes(attr["type"]) + # Handle special Pydal types (id, reference) + col_type = _process_pydal_type(col_type, attr) + attr["type"] = col_type + return data + + def get_primary_keys(columns: List[Dict]) -> List[str]: primary_keys = [] for column in columns: @@ -29,7 +92,9 @@ def models_to_meta(data: List[Dict]) -> List[TableMeta]: types = [] for model in data: if "Enum" not in model["parents"]: - model["table_name"] = from_class_to_table_name(model["name"]) + # Use existing table_name if set (e.g., from Pydal), otherwise derive it + if not model.get("table_name"): + model["table_name"] = from_class_to_table_name(model["name"]) model["columns"] = prepare_columns_data(model["attrs"]) model["properties"]["indexes"] = model["properties"].get("indexes") or [] model["primary_key"] = get_primary_keys(model["columns"]) @@ -43,6 +108,9 @@ def models_to_meta(data: List[Dict]) -> List[TableMeta]: def convert_models(model_from: str, models_type: str = "gino") -> str: result = parse(model_from) + # Clean up Pydal parsed data (strip quotes from names/types) + if _is_pydal_result(result): + result = _clean_pydal_data(result) tables, types = models_to_meta(result) generator = get_generator_by_type(models_type) models_str = "" @@ -56,7 +124,7 @@ def convert_models(model_from: str, models_type: str = "gino") -> str: for table in tables: models_str += generator.generate_model(table) - header += generator.create_header(tables) + header += generator.create_header(tables, models_str=models_str) else: header += enum.create_header(generator.enum_imports) models_type = "enum" diff --git a/tests/functional/converter/test_pydal_conversion.py b/tests/functional/converter/test_pydal_conversion.py new file mode 100644 index 0000000..69f1386 --- /dev/null +++ b/tests/functional/converter/test_pydal_conversion.py @@ -0,0 +1,242 @@ +"""Tests for Pydal model conversion to various output formats.""" + +from omymodels import convert_models + + +# === SQLAlchemy Tests === + + +def test_basic_pydal_to_sqlalchemy(): + """Test basic Pydal table conversion to SQLAlchemy.""" + # Using "id" type which is Pydal's primary key type + pydal_code = ( + '''db.define_table("users", Field("id", "id"), ''' + '''Field("name", "string"), Field("email", "string"))''' + ) + + result = convert_models(pydal_code, models_type="sqlalchemy") + + assert "class User(Base):" in result + assert "__tablename__ = 'users'" in result + assert "id = sa.Column(sa.Integer(), primary_key=True)" in result + assert "name = sa.Column(sa.String())" in result + assert "email = sa.Column(sa.String())" in result + + +def test_pydal_to_sqlalchemy_v2(): + """Test Pydal table conversion to SQLAlchemy 2.0 style.""" + pydal_code = '''db.define_table("users", Field("id", "id"), Field("name", "string"))''' + + result = convert_models(pydal_code, models_type="sqlalchemy_v2") + + assert "class User(Base):" in result + assert "__tablename__ = 'users'" in result + assert "primary_key=True" in result + assert "name: Mapped[str | None] = mapped_column(String)" in result + + +def test_pydal_multiple_tables(): + """Test conversion of multiple Pydal tables.""" + pydal_code = '''db.define_table("users", Field("id", "id"), Field("name", "string")) + +db.define_table("posts", Field("id", "id"), Field("title", "string"))''' + + result = convert_models(pydal_code, models_type="sqlalchemy") + + assert "class User(Base):" in result + assert "class Post(Base):" in result + assert "__tablename__ = 'users'" in result + assert "__tablename__ = 'posts'" in result + + +def test_pydal_foreign_key(): + """Test Pydal reference type conversion to SQLAlchemy ForeignKey.""" + pydal_code = '''db.define_table("users", Field("id", "id"), Field("name", "string")) + +db.define_table("posts", Field("id", "id"), Field("title", "string"), \ +Field("user_id", "reference users"))''' + + result = convert_models(pydal_code, models_type="sqlalchemy") + + assert "class User(Base):" in result + assert "class Post(Base):" in result + assert "user_id = sa.Column(sa.Integer(), sa.ForeignKey('users.id'))" in result + + +def test_pydal_foreign_key_v2(): + """Test Pydal reference type conversion to SQLAlchemy v2 ForeignKey.""" + pydal_code = '''db.define_table("users", Field("id", "id"), Field("name", "string")) + +db.define_table("posts", Field("id", "id"), Field("user_id", "reference users"))''' + + result = convert_models(pydal_code, models_type="sqlalchemy_v2") + + assert "from sqlalchemy import ForeignKey" in result + assert "user_id: Mapped[int | None] = mapped_column(Integer, ForeignKey('users.id'))" in result + + +def test_pydal_various_types(): + """Test conversion of various Pydal types.""" + pydal_code = '''db.define_table("test_types", Field("id", "id"), + Field("col_string", "string"), + Field("col_text", "text"), + Field("col_integer", "integer"), + Field("col_boolean", "boolean"), + Field("col_datetime", "datetime"), + Field("col_date", "date"), + Field("col_float", "float"), + Field("col_decimal", "decimal"))''' + + result = convert_models(pydal_code, models_type="sqlalchemy") + + assert "col_string = sa.Column(sa.String())" in result + assert "col_text = sa.Column(sa.Text())" in result + assert "col_integer = sa.Column(sa.Integer())" in result + assert "col_boolean = sa.Column(sa.Boolean())" in result + assert "col_datetime = sa.Column(sa.DateTime())" in result + assert "col_date = sa.Column(sa.Date())" in result + assert "col_float = sa.Column(sa.Float())" in result + assert "col_decimal = sa.Column(sa.Numeric())" in result + + +def test_pydal_table_name_preserved(): + """Test that Pydal table names are preserved without pluralization.""" + pydal_code = '''db.define_table("my_table", Field("id", "id"))''' + + result = convert_models(pydal_code, models_type="sqlalchemy") + + # Table name should be preserved as 'my_table', not pluralized + assert "__tablename__ = 'my_table'" in result + # Class name should be derived from table name + assert "class MyTable(Base):" in result + + +def test_pydal_id_type_is_primary_key(): + """Test that Pydal 'id' type creates a primary key.""" + pydal_code = '''db.define_table("users", Field("id", "id"), Field("name", "string"))''' + + result = convert_models(pydal_code, models_type="sqlalchemy") + + assert "primary_key=True" in result + + +# === Gino Tests === + + +def test_pydal_to_gino(): + """Test Pydal table conversion to Gino ORM.""" + pydal_code = ( + '''db.define_table("users", Field("id", "id"), ''' + '''Field("name", "string"), Field("email", "string"))''' + ) + + result = convert_models(pydal_code, models_type="gino") + + assert "from gino import Gino" in result + assert "db = Gino()" in result + assert "class User(db.Model):" in result + assert "__tablename__ = 'users'" in result + assert "id = db.Column(db.Integer(), primary_key=True)" in result + assert "name = db.Column(db.String())" in result + assert "email = db.Column(db.String())" in result + + +def test_pydal_to_gino_with_foreign_key(): + """Test Pydal reference type conversion to Gino ForeignKey.""" + pydal_code = '''db.define_table("users", Field("id", "id"), Field("name", "string")) + +db.define_table("posts", Field("id", "id"), Field("user_id", "reference users"))''' + + result = convert_models(pydal_code, models_type="gino") + + assert "class User(db.Model):" in result + assert "class Post(db.Model):" in result + assert "user_id = db.Column(db.Integer(), db.ForeignKey('users.id'))" in result + + +# === Pydantic Tests === + + +def test_pydal_to_pydantic(): + """Test Pydal table conversion to Pydantic.""" + pydal_code = ( + '''db.define_table("users", Field("id", "id"), ''' + '''Field("name", "string"), Field("is_active", "boolean"))''' + ) + + result = convert_models(pydal_code, models_type="pydantic") + + assert "from pydantic import BaseModel" in result + assert "class User(BaseModel):" in result + assert "id: Optional[int]" in result + assert "name: Optional[str]" in result + assert "is_active: Optional[bool]" in result + + +def test_pydal_to_pydantic_v2(): + """Test Pydal table conversion to Pydantic v2.""" + pydal_code = ( + '''db.define_table("users", Field("id", "id"), ''' + '''Field("name", "string"), Field("is_active", "boolean"))''' + ) + + result = convert_models(pydal_code, models_type="pydantic_v2") + + assert "from pydantic import BaseModel" in result + assert "class User(BaseModel):" in result + assert "id: int | None" in result + assert "name: str | None" in result + assert "is_active: bool | None" in result + + +# === Dataclass Tests === + + +def test_pydal_to_dataclass(): + """Test Pydal table conversion to Python dataclass.""" + pydal_code = ( + '''db.define_table("users", Field("id", "id"), ''' + '''Field("name", "string"), Field("score", "float"))''' + ) + + result = convert_models(pydal_code, models_type="dataclass") + + assert "from dataclasses import dataclass" in result + assert "@dataclass" in result + assert "class User:" in result + assert "id: int" in result + assert "name: str" in result + assert "score: float" in result + + +# === SQLModel Tests === + + +def test_pydal_to_sqlmodel(): + """Test Pydal table conversion to SQLModel.""" + pydal_code = ( + '''db.define_table("users", Field("id", "id"), ''' + '''Field("name", "string"), Field("email", "string"))''' + ) + + result = convert_models(pydal_code, models_type="sqlmodel") + + assert "from sqlmodel import" in result + assert "SQLModel" in result + assert "class User(SQLModel, table=True):" in result + assert "__tablename__ = 'users'" in result + assert "id: int | None" in result or "id: Optional[int]" in result + assert "name: str | None" in result or "name: Optional[str]" in result + + +def test_pydal_to_sqlmodel_with_foreign_key(): + """Test Pydal reference type conversion to SQLModel ForeignKey.""" + pydal_code = '''db.define_table("users", Field("id", "id"), Field("name", "string")) + +db.define_table("posts", Field("id", "id"), Field("user_id", "reference users"))''' + + result = convert_models(pydal_code, models_type="sqlmodel") + + assert "class User(SQLModel, table=True):" in result + assert "class Post(SQLModel, table=True):" in result + assert "foreign_key='users.id'" in result diff --git a/tests/integration/converter/__init__.py b/tests/integration/converter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/converter/conftest.py b/tests/integration/converter/conftest.py new file mode 100644 index 0000000..64cbefd --- /dev/null +++ b/tests/integration/converter/conftest.py @@ -0,0 +1,26 @@ +import importlib +import os +import uuid +from types import ModuleType +from typing import Optional + +import pytest + +current_path = os.path.dirname(os.path.abspath(__file__)) +package = os.path.dirname(os.path.relpath(__file__)).replace("/", ".") + + +@pytest.fixture +def load_generated_code(): + def _inner(code_text: str, module_name: Optional[str] = None) -> ModuleType: + if not module_name: + module_name = f"module_{uuid.uuid1()}" + + with open(os.path.join(current_path, f"{module_name}.py"), "w+") as f: + f.write(code_text) + + module = importlib.import_module(f"{package}.{module_name}") + + return module + + yield _inner diff --git a/tests/integration/converter/test_pydal_conversion.py b/tests/integration/converter/test_pydal_conversion.py new file mode 100644 index 0000000..3f37ed7 --- /dev/null +++ b/tests/integration/converter/test_pydal_conversion.py @@ -0,0 +1,182 @@ +"""Integration tests for Pydal to SQLAlchemy model conversion.""" + +import os + +import pytest + +from omymodels import convert_models + +try: + import sqlalchemy # noqa: F401 + HAS_SQLALCHEMY = True +except ImportError: + HAS_SQLALCHEMY = False + +pytestmark = pytest.mark.skipif( + not HAS_SQLALCHEMY, + reason="SQLAlchemy is not installed" +) + + +def test_pydal_to_sqlalchemy_valid_model(load_generated_code) -> None: + """Integration test: verify converted Pydal models are valid SQLAlchemy.""" + # Using "id" type which is Pydal's primary key type + pydal_code = ( + '''db.define_table("users", Field("id", "id"), ''' + '''Field("name", "string"), Field("email", "string"))''' + ) + + result = convert_models(pydal_code, models_type="sqlalchemy") + + module = load_generated_code(result) + + # Verify Base class exists + assert hasattr(module, "Base") + + # Verify model class exists + assert hasattr(module, "User") + + # Check the model has correct __tablename__ + assert module.User.__tablename__ == "users" + + # Verify columns exist in __table__.columns + column_names = [c.name for c in module.User.__table__.columns] + assert "id" in column_names + assert "name" in column_names + assert "email" in column_names + + # Verify id is a primary key + assert module.User.__table__.columns["id"].primary_key + + os.remove(os.path.abspath(module.__file__)) + + +def test_pydal_to_sqlalchemy_foreign_key(load_generated_code) -> None: + """Integration test: verify Pydal references become valid SQLAlchemy ForeignKeys.""" + pydal_code = '''db.define_table("users", Field("id", "id"), Field("name", "string")) + +db.define_table("posts", Field("id", "id"), Field("title", "string"), \ +Field("user_id", "reference users"))''' + + result = convert_models(pydal_code, models_type="sqlalchemy") + + module = load_generated_code(result) + + # Verify both models exist + assert hasattr(module, "User") + assert hasattr(module, "Post") + + # Check foreign key exists on posts + user_id_col = module.Post.__table__.columns["user_id"] + assert len(user_id_col.foreign_keys) == 1 + + # Verify foreign key references the correct table + fk = list(user_id_col.foreign_keys)[0] + assert str(fk.column) == "users.id" + + os.remove(os.path.abspath(module.__file__)) + + +def test_pydal_to_sqlalchemy_multiple_types(load_generated_code) -> None: + """Integration test: verify various Pydal types convert correctly.""" + # Include id field for primary key + pydal_code = '''db.define_table("test_types", Field("id", "id"), + Field("col_string", "string"), + Field("col_text", "text"), + Field("col_integer", "integer"), + Field("col_boolean", "boolean"), + Field("col_float", "float"))''' + + result = convert_models(pydal_code, models_type="sqlalchemy") + + module = load_generated_code(result) + + # Verify model exists + assert hasattr(module, "TestType") + + # Verify all columns exist and have correct types + table = module.TestType.__table__ + + from sqlalchemy import String, Text, Integer, Boolean, Float + + assert isinstance(table.columns["col_string"].type, String) + assert isinstance(table.columns["col_text"].type, Text) + assert isinstance(table.columns["col_integer"].type, Integer) + assert isinstance(table.columns["col_boolean"].type, Boolean) + assert isinstance(table.columns["col_float"].type, Float) + + os.remove(os.path.abspath(module.__file__)) + + +def test_pydal_to_sqlalchemy_v2_valid_model(load_generated_code) -> None: + """Integration test: verify converted Pydal models are valid SQLAlchemy v2.""" + pydal_code = '''db.define_table("users", Field("id", "id"), Field("name", "string"))''' + + result = convert_models(pydal_code, models_type="sqlalchemy_v2") + + module = load_generated_code(result) + + # Verify Base class exists with DeclarativeBase pattern + assert hasattr(module, "Base") + + # Verify model class exists + assert hasattr(module, "User") + + # Check the model has correct __tablename__ + assert module.User.__tablename__ == "users" + + os.remove(os.path.abspath(module.__file__)) + + +def test_pydal_to_pydantic_valid_model(load_generated_code) -> None: + """Integration test: verify converted Pydal models are valid Pydantic.""" + pydal_code = ( + '''db.define_table("users", Field("id", "id"), ''' + '''Field("name", "string"), Field("is_active", "boolean"))''' + ) + + result = convert_models(pydal_code, models_type="pydantic") + + module = load_generated_code(result) + + # Verify model class exists + assert hasattr(module, "User") + + # Verify it's a Pydantic model by checking it has model_fields (Pydantic v2) + # or __fields__ (Pydantic v1) + assert hasattr(module.User, "model_fields") or hasattr(module.User, "__fields__") + + # Create an instance to verify the model works + user = module.User(id=1, name="Test", is_active=True) + assert user.id == 1 + assert user.name == "Test" + assert user.is_active is True + + os.remove(os.path.abspath(module.__file__)) + + +def test_pydal_to_dataclass_valid_model(load_generated_code) -> None: + """Integration test: verify converted Pydal models are valid dataclasses.""" + pydal_code = ( + '''db.define_table("users", Field("id", "id"), ''' + '''Field("name", "string"), Field("score", "float"))''' + ) + + result = convert_models(pydal_code, models_type="dataclass") + + module = load_generated_code(result) + + # Verify model class exists + assert hasattr(module, "User") + + # Verify it's a dataclass + from dataclasses import is_dataclass + assert is_dataclass(module.User) + + # Create an instance to verify the dataclass works + user = module.User(id=1, name="Test", score=9.5) + assert user.id == 1 + assert user.name == "Test" + assert user.score == 9.5 + + os.remove(os.path.abspath(module.__file__))