From c0c6942bf299099fbd3cbfb57625df58b1a2fa17 Mon Sep 17 00:00:00 2001 From: wtbates99 Date: Sat, 22 Mar 2025 19:17:45 -0400 Subject: [PATCH 01/10] feat: testing out cicd workflow --- .github/workflows/test.yml | 41 +++++ requirements-dev.txt | 2 + tabletalk/providers/postgres_provider.py | 11 +- tabletalk/tests/providers/test_postgres.py | 145 ++++++++++++++++++ ...test_sqlite_provider.py => test_sqlite.py} | 0 5 files changed, 195 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 tabletalk/tests/providers/test_postgres.py rename tabletalk/tests/providers/{test_sqlite_provider.py => test_sqlite.py} (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..555b46a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,41 @@ +name: Python Tests + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:latest + env: + POSTGRES_USER: test + POSTGRES_PASSWORD: test + POSTGRES_DB: test + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + - name: Run tests + env: + CI: true + run: pytest -v \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index 864a7dc..876f8c1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,3 +8,5 @@ anthropic click flask pytest +pytest-postgresql +psycopg2-binary diff --git a/tabletalk/providers/postgres_provider.py b/tabletalk/providers/postgres_provider.py index c334481..e53b54d 100644 --- a/tabletalk/providers/postgres_provider.py +++ b/tabletalk/providers/postgres_provider.py @@ -7,23 +7,26 @@ class PostgresProvider(DatabaseProvider): - def __init__(self, host: str, database: str, user: str, password: str): + def __init__(self, host: str, port: int, dbname: str, user: str, password: str): """ Initialize PostgreSQL provider with connection string. Args: host (str): PostgreSQL host - database (str): PostgreSQL database name + port (int): PostgreSQL port + dbname (str): PostgreSQL database name user (str): PostgreSQL user password (str): PostgreSQL password """ self.host = host - self.database = database + self.port = port + self.dbname = dbname self.user = user self.password = password self.connection = psycopg2.connect( host=self.host, - database=self.database, + port=self.port, + dbname=self.dbname, user=self.user, password=self.password, ) diff --git a/tabletalk/tests/providers/test_postgres.py b/tabletalk/tests/providers/test_postgres.py new file mode 100644 index 0000000..b337408 --- /dev/null +++ b/tabletalk/tests/providers/test_postgres.py @@ -0,0 +1,145 @@ +import pytest +from typing import Generator, Dict +from contextlib import closing +import uuid +from pytest_postgresql import factories + +from tabletalk.providers.postgres_provider import PostgresProvider + +# Create a postgresql factory that will create a fresh database for each test session +postgresql_proc = factories.postgresql_proc( + port=None, # Let pytest-postgresql choose a random available port + password="test_password", # Set a known password for the test database +) +postgresql = factories.postgresql("postgresql_proc") + + +@pytest.fixture(scope="function") +def postgres_db(postgresql) -> Generator[Dict[str, str], None, None]: + """Create a temporary PostgreSQL database with test data""" + with closing(postgresql.cursor()) as cur: + cur.execute( + """ + CREATE TABLE users ( + id SERIAL PRIMARY KEY, + name TEXT, + age INTEGER, + created_at TIMESTAMP + ) + """ + ) + cur.execute( + """ + INSERT INTO users (name, age, created_at) + VALUES + ('Alice', 30, '2024-01-01 10:00:00'), + ('Bob', 25, '2024-01-02 11:00:00') + """ + ) + cur.execute( + """ + CREATE VIEW adult_users AS + SELECT name, age + FROM users + WHERE age >= 18 + """ + ) + postgresql.commit() + yield postgresql.dsn + + +@pytest.fixture(scope="function") +def postgres_provider( + postgres_db: Dict[str, str], +) -> Generator[PostgresProvider, None, None]: + """Create PostgresProvider instance with connection pooling""" + provider = PostgresProvider( + host=postgres_db["host"], + port=postgres_db["port"], + dbname=postgres_db["dbname"], + user=postgres_db["user"], + password=postgres_db["password"], + min_connections=1, + max_connections=5, + ) + yield provider + provider.close() + + +# Tests +def test_execute_query_table(postgres_provider: PostgresProvider) -> None: + """Test executing query on a table""" + results = postgres_provider.execute_query("SELECT * FROM users ORDER BY id") + assert len(results) == 2 + assert results[0]["name"] == "Alice" + assert results[0]["age"] == 30 + assert results[1]["name"] == "Bob" + assert results[1]["age"] == 25 + + +def test_execute_query_view(postgres_provider: PostgresProvider) -> None: + """Test executing query on a view""" + results = postgres_provider.execute_query("SELECT * FROM adult_users ORDER BY age") + assert len(results) == 2 + assert results[0]["name"] == "Bob" + assert results[0]["age"] == 25 + assert results[1]["name"] == "Alice" + assert results[1]["age"] == 30 + + +def test_get_compact_tables_all(postgres_provider: PostgresProvider) -> None: + """Test getting schema for all tables and views""" + results = postgres_provider.get_compact_tables(schema_name="public") + assert len(results) == 2 + users_table = next(t for t in results if t["t"] == "users") + assert users_table["t"] == "users" + assert len(users_table["f"]) == 4 + assert {"n": "id", "t": "I"} in users_table["f"] + assert {"n": "name", "t": "S"} in users_table["f"] + assert {"n": "age", "t": "I"} in users_table["f"] + assert {"n": "created_at", "t": "TS"} in users_table["f"] + view_table = next(t for t in results if t["t"] == "adult_users") + assert view_table["t"] == "adult_users" + assert len(view_table["f"]) == 2 + assert {"n": "name", "t": "S"} in view_table["f"] + assert {"n": "age", "t": "I"} in view_table["f"] + + +def test_get_compact_tables_specific(postgres_provider: PostgresProvider) -> None: + """Test getting schema for specific tables""" + results = postgres_provider.get_compact_tables( + schema_name="public", table_names=["users"] + ) + assert len(results) == 1 + table = results[0] + assert table["t"] == "users" + assert len(table["f"]) == 4 + + +def test_get_database_type_map(postgres_provider: PostgresProvider) -> None: + """Test database type mapping""" + type_map = postgres_provider.get_database_type_map() + assert type_map["text"] == "S" + assert type_map["integer"] == "I" + assert type_map["timestamp"] == "TS" + + +def test_database_cleanup(postgres_provider: PostgresProvider) -> None: + """Verify that database is properly isolated between tests""" + results = postgres_provider.execute_query("SELECT COUNT(*) as count FROM users") + assert results[0]["count"] == 2 + + +# Register custom marker for CI/CD +def pytest_configure(config): + config.addinivalue_line( + "markers", "postgresql: mark test as requiring postgresql fixture" + ) + + +@pytest.mark.postgresql +@pytest.mark.parametrize("postgres_db", ["postgresql"], indirect=True) +def test_ci_environment(postgres_provider: PostgresProvider) -> None: + """Test that runs only in CI with real postgresql fixture""" + results = postgres_provider.execute_query("SELECT * FROM users") + assert len(results) == 2 diff --git a/tabletalk/tests/providers/test_sqlite_provider.py b/tabletalk/tests/providers/test_sqlite.py similarity index 100% rename from tabletalk/tests/providers/test_sqlite_provider.py rename to tabletalk/tests/providers/test_sqlite.py From 06ba178c7ec30f170648bfac604d6a2be6208eb9 Mon Sep 17 00:00:00 2001 From: wtbates99 Date: Sat, 22 Mar 2025 19:18:05 -0400 Subject: [PATCH 02/10] feat: testing out cicd workflow --- tabletalk/tests/providers/test_postgres.py | 123 +++++++++++++++++---- 1 file changed, 100 insertions(+), 23 deletions(-) diff --git a/tabletalk/tests/providers/test_postgres.py b/tabletalk/tests/providers/test_postgres.py index b337408..b1930a4 100644 --- a/tabletalk/tests/providers/test_postgres.py +++ b/tabletalk/tests/providers/test_postgres.py @@ -1,23 +1,46 @@ import pytest -from typing import Generator, Dict +from typing import Generator, Dict, Optional from contextlib import closing -import uuid -from pytest_postgresql import factories +import os from tabletalk.providers.postgres_provider import PostgresProvider -# Create a postgresql factory that will create a fresh database for each test session -postgresql_proc = factories.postgresql_proc( - port=None, # Let pytest-postgresql choose a random available port - password="test_password", # Set a known password for the test database -) -postgresql = factories.postgresql("postgresql_proc") +# Configuration for local development fallback +LOCAL_PG_CONFIG = { + "host": os.getenv("POSTGRES_HOST", "localhost"), + "port": os.getenv("POSTGRES_PORT", "5432"), + "dbname": os.getenv("POSTGRES_DB", "test_db"), + "user": os.getenv("POSTGRES_USER", "test"), + "password": os.getenv("POSTGRES_PASSWORD", "test"), +} + + +def setup_local_database(config: Dict[str, str]) -> None: + """Set up a local test database if not using pytest-postgresql""" + from psycopg2 import connect + from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT + + # Use a separate connection to create the database if it doesn't exist + admin_conn = connect( + host=config["host"], + port=config["port"], + user=config["user"], + password=config["password"], + database="postgres", # Connect to default db to create test db + ) + admin_conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) + with closing(admin_conn.cursor()) as cur: + cur.execute(f"DROP DATABASE IF EXISTS {config['dbname']}") + cur.execute(f"CREATE DATABASE {config['dbname']}") -@pytest.fixture(scope="function") -def postgres_db(postgresql) -> Generator[Dict[str, str], None, None]: - """Create a temporary PostgreSQL database with test data""" - with closing(postgresql.cursor()) as cur: + admin_conn.close() + + # Now connect to the test database and set up schema + conn = connect( + **{k: v for k, v in config.items() if k != "dbname"}, database=config["dbname"] + ) + with closing(conn.cursor()) as cur: cur.execute( """ CREATE TABLE users ( @@ -44,8 +67,50 @@ def postgres_db(postgresql) -> Generator[Dict[str, str], None, None]: WHERE age >= 18 """ ) - postgresql.commit() - yield postgresql.dsn + conn.commit() + conn.close() + + +@pytest.fixture(scope="function") +def postgres_db(request) -> Generator[Dict[str, str], None, None]: + """Create a temporary PostgreSQL database with test data""" + if hasattr(request, "param") and request.param == "postgresql": + # CI/CD environment with pytest-postgresql + postgresql = request.getfixturevalue("postgresql") + with closing(postgresql.cursor()) as cur: + cur.execute( + """ + CREATE TABLE users ( + id SERIAL PRIMARY KEY, + name TEXT, + age INTEGER, + created_at TIMESTAMP + ) + """ + ) + cur.execute( + """ + INSERT INTO users (name, age, created_at) + VALUES + ('Alice', 30, '2024-01-01 10:00:00'), + ('Bob', 25, '2024-01-02 11:00:00') + """ + ) + cur.execute( + """ + CREATE VIEW adult_users AS + SELECT name, age + FROM users + WHERE age >= 18 + """ + ) + postgresql.commit() + yield postgresql.dsn + else: + # Local development environment + config = LOCAL_PG_CONFIG.copy() + setup_local_database(config) + yield config @pytest.fixture(scope="function") @@ -58,7 +123,7 @@ def postgres_provider( port=postgres_db["port"], dbname=postgres_db["dbname"], user=postgres_db["user"], - password=postgres_db["password"], + password=postgres_db.get("password", ""), min_connections=1, max_connections=5, ) @@ -90,7 +155,10 @@ def test_execute_query_view(postgres_provider: PostgresProvider) -> None: def test_get_compact_tables_all(postgres_provider: PostgresProvider) -> None: """Test getting schema for all tables and views""" results = postgres_provider.get_compact_tables(schema_name="public") - assert len(results) == 2 + + assert len(results) == 2 # Should find both the table and view + + # Find the users table schema users_table = next(t for t in results if t["t"] == "users") assert users_table["t"] == "users" assert len(users_table["f"]) == 4 @@ -98,6 +166,8 @@ def test_get_compact_tables_all(postgres_provider: PostgresProvider) -> None: assert {"n": "name", "t": "S"} in users_table["f"] assert {"n": "age", "t": "I"} in users_table["f"] assert {"n": "created_at", "t": "TS"} in users_table["f"] + + # Find the view schema view_table = next(t for t in results if t["t"] == "adult_users") assert view_table["t"] == "adult_users" assert len(view_table["f"]) == 2 @@ -110,6 +180,7 @@ def test_get_compact_tables_specific(postgres_provider: PostgresProvider) -> Non results = postgres_provider.get_compact_tables( schema_name="public", table_names=["users"] ) + assert len(results) == 1 table = results[0] assert table["t"] == "users" @@ -130,15 +201,21 @@ def test_database_cleanup(postgres_provider: PostgresProvider) -> None: assert results[0]["count"] == 2 -# Register custom marker for CI/CD +# Pytest configuration for CI/CD def pytest_configure(config): - config.addinivalue_line( - "markers", "postgresql: mark test as requiring postgresql fixture" - ) + """Configure pytest to use postgresql fixture in CI/CD""" + if os.getenv("CI"): + config.addinivalue_line( + "markers", "postgresql: mark test as requiring postgresql" + ) -@pytest.mark.postgresql -@pytest.mark.parametrize("postgres_db", ["postgresql"], indirect=True) +# Parametrize for CI/CD environment +@pytest.mark.parametrize( + "postgres_db", + [pytest.param("postgresql", marks=pytest.mark.postgresql)], + indirect=True, +) def test_ci_environment(postgres_provider: PostgresProvider) -> None: """Test that runs only in CI with real postgresql fixture""" results = postgres_provider.execute_query("SELECT * FROM users") From 79342bdf3642b21450f3b0df86037abbf121d181 Mon Sep 17 00:00:00 2001 From: wtbates99 Date: Sat, 22 Mar 2025 19:20:40 -0400 Subject: [PATCH 03/10] test 2 --- .github/workflows/test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 555b46a..9806806 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ on: jobs: test: runs-on: ubuntu-latest - + services: postgres: image: postgres:latest @@ -35,7 +35,8 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements-dev.txt + pip install -e . - name: Run tests env: CI: true - run: pytest -v \ No newline at end of file + run: pytest -v From fd6d5425fb1e2f33f2aaac9b276c54fac1d7f3ef Mon Sep 17 00:00:00 2001 From: wtbates99 Date: Sat, 22 Mar 2025 19:22:50 -0400 Subject: [PATCH 04/10] yeet --- tabletalk/tests/providers/test_postgres.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tabletalk/tests/providers/test_postgres.py b/tabletalk/tests/providers/test_postgres.py index b1930a4..e87b7c1 100644 --- a/tabletalk/tests/providers/test_postgres.py +++ b/tabletalk/tests/providers/test_postgres.py @@ -1,7 +1,8 @@ -import pytest -from typing import Generator, Dict, Optional -from contextlib import closing import os +from contextlib import closing +from typing import Dict, Generator + +import pytest from tabletalk.providers.postgres_provider import PostgresProvider @@ -105,7 +106,15 @@ def postgres_db(request) -> Generator[Dict[str, str], None, None]: """ ) postgresql.commit() - yield postgresql.dsn + # Instead of postgresql.dsn, create a config dict + config = { + "host": postgresql.get_dsn_parameters()["host"], + "port": postgresql.get_dsn_parameters()["port"], + "dbname": postgresql.get_dsn_parameters()["dbname"], + "user": postgresql.get_dsn_parameters()["user"], + "password": postgresql.get_dsn_parameters().get("password", ""), + } + yield config else: # Local development environment config = LOCAL_PG_CONFIG.copy() @@ -124,8 +133,6 @@ def postgres_provider( dbname=postgres_db["dbname"], user=postgres_db["user"], password=postgres_db.get("password", ""), - min_connections=1, - max_connections=5, ) yield provider provider.close() From 2cf545f54c2528f8acb784ec762a7f520b406cb5 Mon Sep 17 00:00:00 2001 From: wtbates99 Date: Sat, 22 Mar 2025 19:26:28 -0400 Subject: [PATCH 05/10] yeet --- .github/workflows/test.yml | 13 +- tabletalk/tests/providers/test_postgres.py | 230 ++++----------------- 2 files changed, 42 insertions(+), 201 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9806806..2488308 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,24 +19,15 @@ jobs: POSTGRES_DB: test ports: - 5432:5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 steps: - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 + - uses: actions/setup-python@v4 with: python-version: '3.11' - name: Install dependencies run: | - python -m pip install --upgrade pip pip install -r requirements-dev.txt pip install -e . - name: Run tests - env: - CI: true - run: pytest -v + run: pytest \ No newline at end of file diff --git a/tabletalk/tests/providers/test_postgres.py b/tabletalk/tests/providers/test_postgres.py index e87b7c1..98db493 100644 --- a/tabletalk/tests/providers/test_postgres.py +++ b/tabletalk/tests/providers/test_postgres.py @@ -1,229 +1,79 @@ import os -from contextlib import closing -from typing import Dict, Generator - import pytest +from psycopg2 import connect +from psycopg2.extras import RealDictCursor from tabletalk.providers.postgres_provider import PostgresProvider -# Configuration for local development fallback -LOCAL_PG_CONFIG = { - "host": os.getenv("POSTGRES_HOST", "localhost"), - "port": os.getenv("POSTGRES_PORT", "5432"), - "dbname": os.getenv("POSTGRES_DB", "test_db"), - "user": os.getenv("POSTGRES_USER", "test"), - "password": os.getenv("POSTGRES_PASSWORD", "test"), +# Basic test configuration +TEST_CONFIG = { + "host": "localhost", + "port": "5432", + "dbname": "test_db", + "user": "test", + "password": "test", } -def setup_local_database(config: Dict[str, str]) -> None: - """Set up a local test database if not using pytest-postgresql""" - from psycopg2 import connect - from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT - - # Use a separate connection to create the database if it doesn't exist +@pytest.fixture(scope="function") +def postgres_db(): + """Set up a simple test database""" + # Create fresh database admin_conn = connect( - host=config["host"], - port=config["port"], - user=config["user"], - password=config["password"], - database="postgres", # Connect to default db to create test db + host=TEST_CONFIG["host"], + port=TEST_CONFIG["port"], + user=TEST_CONFIG["user"], + password=TEST_CONFIG["password"], + database="postgres", ) - admin_conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) - - with closing(admin_conn.cursor()) as cur: - cur.execute(f"DROP DATABASE IF EXISTS {config['dbname']}") - cur.execute(f"CREATE DATABASE {config['dbname']}") - + admin_conn.autocommit = True + with admin_conn.cursor() as cur: + cur.execute(f"DROP DATABASE IF EXISTS {TEST_CONFIG['dbname']}") + cur.execute(f"CREATE DATABASE {TEST_CONFIG['dbname']}") admin_conn.close() - # Now connect to the test database and set up schema - conn = connect( - **{k: v for k, v in config.items() if k != "dbname"}, database=config["dbname"] - ) - with closing(conn.cursor()) as cur: + # Set up schema and data + conn = connect(**TEST_CONFIG) + with conn.cursor() as cur: cur.execute( """ CREATE TABLE users ( id SERIAL PRIMARY KEY, name TEXT, - age INTEGER, - created_at TIMESTAMP + age INTEGER ) """ ) cur.execute( """ - INSERT INTO users (name, age, created_at) - VALUES - ('Alice', 30, '2024-01-01 10:00:00'), - ('Bob', 25, '2024-01-02 11:00:00') - """ - ) - cur.execute( - """ - CREATE VIEW adult_users AS - SELECT name, age - FROM users - WHERE age >= 18 + INSERT INTO users (name, age) + VALUES ('Alice', 30), ('Bob', 25) """ ) conn.commit() conn.close() - -@pytest.fixture(scope="function") -def postgres_db(request) -> Generator[Dict[str, str], None, None]: - """Create a temporary PostgreSQL database with test data""" - if hasattr(request, "param") and request.param == "postgresql": - # CI/CD environment with pytest-postgresql - postgresql = request.getfixturevalue("postgresql") - with closing(postgresql.cursor()) as cur: - cur.execute( - """ - CREATE TABLE users ( - id SERIAL PRIMARY KEY, - name TEXT, - age INTEGER, - created_at TIMESTAMP - ) - """ - ) - cur.execute( - """ - INSERT INTO users (name, age, created_at) - VALUES - ('Alice', 30, '2024-01-01 10:00:00'), - ('Bob', 25, '2024-01-02 11:00:00') - """ - ) - cur.execute( - """ - CREATE VIEW adult_users AS - SELECT name, age - FROM users - WHERE age >= 18 - """ - ) - postgresql.commit() - # Instead of postgresql.dsn, create a config dict - config = { - "host": postgresql.get_dsn_parameters()["host"], - "port": postgresql.get_dsn_parameters()["port"], - "dbname": postgresql.get_dsn_parameters()["dbname"], - "user": postgresql.get_dsn_parameters()["user"], - "password": postgresql.get_dsn_parameters().get("password", ""), - } - yield config - else: - # Local development environment - config = LOCAL_PG_CONFIG.copy() - setup_local_database(config) - yield config + yield TEST_CONFIG -@pytest.fixture(scope="function") -def postgres_provider( - postgres_db: Dict[str, str], -) -> Generator[PostgresProvider, None, None]: - """Create PostgresProvider instance with connection pooling""" - provider = PostgresProvider( - host=postgres_db["host"], - port=postgres_db["port"], - dbname=postgres_db["dbname"], - user=postgres_db["user"], - password=postgres_db.get("password", ""), - ) +@pytest.fixture +def postgres_provider(postgres_db): + """Create PostgresProvider instance""" + provider = PostgresProvider(**postgres_db) yield provider provider.close() -# Tests -def test_execute_query_table(postgres_provider: PostgresProvider) -> None: - """Test executing query on a table""" +# Simplified tests +def test_basic_query(postgres_provider): results = postgres_provider.execute_query("SELECT * FROM users ORDER BY id") assert len(results) == 2 assert results[0]["name"] == "Alice" - assert results[0]["age"] == 30 assert results[1]["name"] == "Bob" - assert results[1]["age"] == 25 -def test_execute_query_view(postgres_provider: PostgresProvider) -> None: - """Test executing query on a view""" - results = postgres_provider.execute_query("SELECT * FROM adult_users ORDER BY age") - assert len(results) == 2 - assert results[0]["name"] == "Bob" - assert results[0]["age"] == 25 - assert results[1]["name"] == "Alice" - assert results[1]["age"] == 30 - - -def test_get_compact_tables_all(postgres_provider: PostgresProvider) -> None: - """Test getting schema for all tables and views""" - results = postgres_provider.get_compact_tables(schema_name="public") - - assert len(results) == 2 # Should find both the table and view - - # Find the users table schema - users_table = next(t for t in results if t["t"] == "users") - assert users_table["t"] == "users" - assert len(users_table["f"]) == 4 - assert {"n": "id", "t": "I"} in users_table["f"] - assert {"n": "name", "t": "S"} in users_table["f"] - assert {"n": "age", "t": "I"} in users_table["f"] - assert {"n": "created_at", "t": "TS"} in users_table["f"] - - # Find the view schema - view_table = next(t for t in results if t["t"] == "adult_users") - assert view_table["t"] == "adult_users" - assert len(view_table["f"]) == 2 - assert {"n": "name", "t": "S"} in view_table["f"] - assert {"n": "age", "t": "I"} in view_table["f"] - - -def test_get_compact_tables_specific(postgres_provider: PostgresProvider) -> None: - """Test getting schema for specific tables""" - results = postgres_provider.get_compact_tables( - schema_name="public", table_names=["users"] - ) - - assert len(results) == 1 - table = results[0] - assert table["t"] == "users" - assert len(table["f"]) == 4 - - -def test_get_database_type_map(postgres_provider: PostgresProvider) -> None: - """Test database type mapping""" - type_map = postgres_provider.get_database_type_map() - assert type_map["text"] == "S" - assert type_map["integer"] == "I" - assert type_map["timestamp"] == "TS" - - -def test_database_cleanup(postgres_provider: PostgresProvider) -> None: - """Verify that database is properly isolated between tests""" - results = postgres_provider.execute_query("SELECT COUNT(*) as count FROM users") - assert results[0]["count"] == 2 - - -# Pytest configuration for CI/CD -def pytest_configure(config): - """Configure pytest to use postgresql fixture in CI/CD""" - if os.getenv("CI"): - config.addinivalue_line( - "markers", "postgresql: mark test as requiring postgresql" - ) - - -# Parametrize for CI/CD environment -@pytest.mark.parametrize( - "postgres_db", - [pytest.param("postgresql", marks=pytest.mark.postgresql)], - indirect=True, -) -def test_ci_environment(postgres_provider: PostgresProvider) -> None: - """Test that runs only in CI with real postgresql fixture""" - results = postgres_provider.execute_query("SELECT * FROM users") - assert len(results) == 2 +def test_table_schema(postgres_provider): + schemas = postgres_provider.get_compact_tables() + assert len(schemas) == 1 + assert schemas[0]["t"] == "users" + assert len(schemas[0]["f"]) == 3 From 93e57342ea972d53a55372ac7b76ac5317c99d3f Mon Sep 17 00:00:00 2001 From: wtbates99 Date: Sat, 22 Mar 2025 19:28:43 -0400 Subject: [PATCH 06/10] yeet --- tabletalk/tests/providers/test_postgres.py | 43 +++++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/tabletalk/tests/providers/test_postgres.py b/tabletalk/tests/providers/test_postgres.py index 98db493..de1613e 100644 --- a/tabletalk/tests/providers/test_postgres.py +++ b/tabletalk/tests/providers/test_postgres.py @@ -35,6 +35,7 @@ def postgres_db(): # Set up schema and data conn = connect(**TEST_CONFIG) with conn.cursor() as cur: + # Create users table cur.execute( """ CREATE TABLE users ( @@ -50,6 +51,14 @@ def postgres_db(): VALUES ('Alice', 30), ('Bob', 25) """ ) + + # Create a view + cur.execute( + """ + CREATE VIEW adult_users AS + SELECT name, age FROM users WHERE age >= 18 + """ + ) conn.commit() conn.close() @@ -61,7 +70,8 @@ def postgres_provider(postgres_db): """Create PostgresProvider instance""" provider = PostgresProvider(**postgres_db) yield provider - provider.close() + # Close the underlying connection instead of calling close() + provider.conn.close() # Simplified tests @@ -72,8 +82,31 @@ def test_basic_query(postgres_provider): assert results[1]["name"] == "Bob" -def test_table_schema(postgres_provider): +def test_table_and_view_schema(postgres_provider): + """Test that both tables and views are returned by get_compact_tables""" schemas = postgres_provider.get_compact_tables() - assert len(schemas) == 1 - assert schemas[0]["t"] == "users" - assert len(schemas[0]["f"]) == 3 + + # Should return both the table and view + assert len(schemas) == 2 + + # Find table and view in results + table_schema = next(s for s in schemas if s["t"] == "users") + view_schema = next(s for s in schemas if s["t"] == "adult_users") + + # Verify table schema + assert len(table_schema["f"]) == 3 + assert [f["n"] for f in table_schema["f"]] == ["id", "name", "age"] + + # Verify view schema + assert len(view_schema["f"]) == 2 + assert [f["n"] for f in view_schema["f"]] == ["name", "age"] + + +def test_view_query(postgres_provider): + """Test querying the view""" + results = postgres_provider.execute_query("SELECT * FROM adult_users ORDER BY name") + assert len(results) == 2 + assert results[0]["name"] == "Alice" + assert results[0]["age"] == 30 + assert results[1]["name"] == "Bob" + assert results[1]["age"] == 25 From d012ce497260b5c2cd4c9649463bc786de9b6a25 Mon Sep 17 00:00:00 2001 From: wtbates99 Date: Sat, 22 Mar 2025 19:41:03 -0400 Subject: [PATCH 07/10] working postgres runs --- tabletalk/factories.py | 3 +- tabletalk/tests/providers/test_postgres.py | 56 +++++++++++----------- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/tabletalk/factories.py b/tabletalk/factories.py index f37155a..2a85abd 100644 --- a/tabletalk/factories.py +++ b/tabletalk/factories.py @@ -55,7 +55,8 @@ def get_db_provider(config: Dict[str, Any]) -> DatabaseProvider: elif provider_type == "postgres": return PostgresProvider( host=config["host"], - database=config["database"], + port=int(config.get("port", 5432)), + dbname=config["database"], user=config["user"], password=config["password"], ) diff --git a/tabletalk/tests/providers/test_postgres.py b/tabletalk/tests/providers/test_postgres.py index de1613e..2cb5995 100644 --- a/tabletalk/tests/providers/test_postgres.py +++ b/tabletalk/tests/providers/test_postgres.py @@ -1,14 +1,13 @@ -import os +from typing import Any, Dict, Generator + import pytest from psycopg2 import connect -from psycopg2.extras import RealDictCursor from tabletalk.providers.postgres_provider import PostgresProvider -# Basic test configuration TEST_CONFIG = { "host": "localhost", - "port": "5432", + "port": 5432, "dbname": "test_db", "user": "test", "password": "test", @@ -16,7 +15,7 @@ @pytest.fixture(scope="function") -def postgres_db(): +def postgres_db() -> Generator[Dict[str, Any], None, None]: """Set up a simple test database""" # Create fresh database admin_conn = connect( @@ -24,7 +23,7 @@ def postgres_db(): port=TEST_CONFIG["port"], user=TEST_CONFIG["user"], password=TEST_CONFIG["password"], - database="postgres", + dbname="postgres", ) admin_conn.autocommit = True with admin_conn.cursor() as cur: @@ -33,9 +32,14 @@ def postgres_db(): admin_conn.close() # Set up schema and data - conn = connect(**TEST_CONFIG) + conn = connect( + host=TEST_CONFIG["host"], + port=TEST_CONFIG["port"], + dbname=TEST_CONFIG["dbname"], + user=TEST_CONFIG["user"], + password=TEST_CONFIG["password"], + ) with conn.cursor() as cur: - # Create users table cur.execute( """ CREATE TABLE users ( @@ -51,8 +55,6 @@ def postgres_db(): VALUES ('Alice', 30), ('Bob', 25) """ ) - - # Create a view cur.execute( """ CREATE VIEW adult_users AS @@ -65,45 +67,41 @@ def postgres_db(): yield TEST_CONFIG +# Rest of the code remains unchanged @pytest.fixture -def postgres_provider(postgres_db): - """Create PostgresProvider instance""" - provider = PostgresProvider(**postgres_db) +def postgres_provider( + postgres_db: Dict[str, Any], +) -> Generator[PostgresProvider, None, None]: + provider = PostgresProvider( + host=postgres_db["host"], + port=postgres_db["port"], + dbname=postgres_db["dbname"], + user=postgres_db["user"], + password=postgres_db["password"], + ) yield provider - # Close the underlying connection instead of calling close() - provider.conn.close() + provider.connection.close() -# Simplified tests -def test_basic_query(postgres_provider): +def test_basic_query(postgres_provider: PostgresProvider) -> None: results = postgres_provider.execute_query("SELECT * FROM users ORDER BY id") assert len(results) == 2 assert results[0]["name"] == "Alice" assert results[1]["name"] == "Bob" -def test_table_and_view_schema(postgres_provider): - """Test that both tables and views are returned by get_compact_tables""" +def test_table_and_view_schema(postgres_provider: PostgresProvider) -> None: schemas = postgres_provider.get_compact_tables() - - # Should return both the table and view assert len(schemas) == 2 - - # Find table and view in results table_schema = next(s for s in schemas if s["t"] == "users") view_schema = next(s for s in schemas if s["t"] == "adult_users") - - # Verify table schema assert len(table_schema["f"]) == 3 assert [f["n"] for f in table_schema["f"]] == ["id", "name", "age"] - - # Verify view schema assert len(view_schema["f"]) == 2 assert [f["n"] for f in view_schema["f"]] == ["name", "age"] -def test_view_query(postgres_provider): - """Test querying the view""" +def test_view_query(postgres_provider: PostgresProvider) -> None: results = postgres_provider.execute_query("SELECT * FROM adult_users ORDER BY name") assert len(results) == 2 assert results[0]["name"] == "Alice" From 8d6271899cfab57a8726787f43f0ecac4a634f73 Mon Sep 17 00:00:00 2001 From: wtbates99 Date: Sat, 22 Mar 2025 20:01:30 -0400 Subject: [PATCH 08/10] working mysql test --- .github/workflows/test.yml | 20 ++++- tabletalk/tests/providers/test_mysql.py | 104 ++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 tabletalk/tests/providers/test_mysql.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2488308..3cdf46d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,6 +19,24 @@ jobs: POSTGRES_DB: test ports: - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + mysql: + image: mysql:latest + env: + MYSQL_ROOT_PASSWORD: test + MYSQL_DATABASE: test + ports: + - 3306:3306 + options: >- + --health-cmd="mysqladmin ping" + --health-interval=10s + --health-timeout=5s + --health-retries=3 steps: - uses: actions/checkout@v3 @@ -30,4 +48,4 @@ jobs: pip install -r requirements-dev.txt pip install -e . - name: Run tests - run: pytest \ No newline at end of file + run: pytest diff --git a/tabletalk/tests/providers/test_mysql.py b/tabletalk/tests/providers/test_mysql.py new file mode 100644 index 0000000..9f5f9ea --- /dev/null +++ b/tabletalk/tests/providers/test_mysql.py @@ -0,0 +1,104 @@ +from typing import Any, Dict, Generator + +import mysql.connector +import pytest +from mysql.connector import MySQLConnection + +from tabletalk.providers.mysql_provider import MySQLProvider + +TEST_CONFIG = { + "host": "localhost", + "port": 3306, + "database": "test", + "user": "root", + "password": "test", +} + + +@pytest.fixture(scope="function") +def mysql_db() -> Generator[Dict[str, Any], None, None]: + """Set up a simple test database""" + # Create fresh database + conn = mysql.connector.connect( + host=TEST_CONFIG["host"], + port=TEST_CONFIG["port"], + user=TEST_CONFIG["user"], + password=TEST_CONFIG["password"], + ) + assert isinstance(conn, MySQLConnection) + conn.autocommit = True + + with conn.cursor() as cur: + cur.execute(f"DROP DATABASE IF EXISTS {TEST_CONFIG['database']}") + cur.execute(f"CREATE DATABASE {TEST_CONFIG['database']}") + + # Set up schema and data + conn.database = str(TEST_CONFIG["database"]) + with conn.cursor() as cur: + cur.execute( + """ + CREATE TABLE users ( + id INT AUTO_INCREMENT PRIMARY KEY, + name TEXT, + age INTEGER + ) + """ + ) + cur.execute( + """ + INSERT INTO users (name, age) + VALUES ('Alice', 30), ('Bob', 25) + """ + ) + cur.execute( + """ + CREATE VIEW adult_users AS + SELECT name, age FROM users WHERE age >= 18 + """ + ) + conn.commit() + conn.close() + + yield TEST_CONFIG + + +@pytest.fixture +def mysql_provider( + mysql_db: Dict[str, Any], +) -> Generator[MySQLProvider, None, None]: + provider = MySQLProvider( + host=mysql_db["host"], + port=mysql_db["port"], + database=mysql_db["database"], + user=mysql_db["user"], + password=mysql_db["password"], + ) + yield provider + provider.connection.close() + + +def test_basic_query(mysql_provider: MySQLProvider) -> None: + results = mysql_provider.execute_query("SELECT * FROM users ORDER BY id") + assert len(results) == 2 + assert results[0]["name"] == "Alice" + assert results[1]["name"] == "Bob" + + +def test_table_and_view_schema(mysql_provider: MySQLProvider) -> None: + schemas = mysql_provider.get_compact_tables() + assert len(schemas) == 2 + table_schema = next(s for s in schemas if s["t"] == "users") + view_schema = next(s for s in schemas if s["t"] == "adult_users") + assert len(table_schema["f"]) == 3 + assert [f["n"] for f in table_schema["f"]] == ["id", "name", "age"] + assert len(view_schema["f"]) == 2 + assert [f["n"] for f in view_schema["f"]] == ["name", "age"] + + +def test_view_query(mysql_provider: MySQLProvider) -> None: + results = mysql_provider.execute_query("SELECT * FROM adult_users ORDER BY name") + assert len(results) == 2 + assert results[0]["name"] == "Alice" + assert results[0]["age"] == 30 + assert results[1]["name"] == "Bob" + assert results[1]["age"] == 25 From 0e6689db20516310b662d65eeae72952e47200e4 Mon Sep 17 00:00:00 2001 From: wtbates99 Date: Sat, 22 Mar 2025 20:08:55 -0400 Subject: [PATCH 09/10] working? --- .../{test.yml => db_provider_tests.yml} | 2 +- tabletalk/tests/providers/test_mysql.py | 37 +++++++++++++++---- 2 files changed, 30 insertions(+), 9 deletions(-) rename .github/workflows/{test.yml => db_provider_tests.yml} (97%) diff --git a/.github/workflows/test.yml b/.github/workflows/db_provider_tests.yml similarity index 97% rename from .github/workflows/test.yml rename to .github/workflows/db_provider_tests.yml index 3cdf46d..bebf2ac 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/db_provider_tests.yml @@ -1,4 +1,4 @@ -name: Python Tests +name: DB Provider Tests on: push: diff --git a/tabletalk/tests/providers/test_mysql.py b/tabletalk/tests/providers/test_mysql.py index 9f5f9ea..6a25037 100644 --- a/tabletalk/tests/providers/test_mysql.py +++ b/tabletalk/tests/providers/test_mysql.py @@ -1,8 +1,9 @@ -from typing import Any, Dict, Generator +from typing import Any, Dict, Generator, Union import mysql.connector import pytest -from mysql.connector import MySQLConnection +from mysql.connector.abstracts import MySQLConnectionAbstract +from mysql.connector.pooling import PooledMySQLConnection from tabletalk.providers.mysql_provider import MySQLProvider @@ -14,26 +15,46 @@ "password": "test", } +ConnectionType = Union[PooledMySQLConnection, MySQLConnectionAbstract] + @pytest.fixture(scope="function") def mysql_db() -> Generator[Dict[str, Any], None, None]: """Set up a simple test database""" - # Create fresh database - conn = mysql.connector.connect( + conn: ConnectionType = mysql.connector.connect( host=TEST_CONFIG["host"], port=TEST_CONFIG["port"], user=TEST_CONFIG["user"], password=TEST_CONFIG["password"], ) - assert isinstance(conn, MySQLConnection) - conn.autocommit = True + + # Handle autocommit based on connection type + if isinstance(conn, PooledMySQLConnection): + real_conn = conn.get_connection() + real_conn.autocommit = True + else: + conn.autocommit = True with conn.cursor() as cur: cur.execute(f"DROP DATABASE IF EXISTS {TEST_CONFIG['database']}") cur.execute(f"CREATE DATABASE {TEST_CONFIG['database']}") - # Set up schema and data - conn.database = str(TEST_CONFIG["database"]) + conn.close() + conn = mysql.connector.connect( + host=TEST_CONFIG["host"], + port=TEST_CONFIG["port"], + user=TEST_CONFIG["user"], + password=TEST_CONFIG["password"], + database=TEST_CONFIG["database"], + ) + + # Handle autocommit again for the new connection + if isinstance(conn, PooledMySQLConnection): + real_conn = conn.get_connection() + real_conn.autocommit = True + else: + conn.autocommit = True + with conn.cursor() as cur: cur.execute( """ From 710c6c38fa9f939b063cb6062651dde098d1bf72 Mon Sep 17 00:00:00 2001 From: wtbates99 Date: Sat, 22 Mar 2025 20:11:37 -0400 Subject: [PATCH 10/10] fix cicd --- .github/workflows/db_provider_tests.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/db_provider_tests.yml b/.github/workflows/db_provider_tests.yml index bebf2ac..50d00a4 100644 --- a/.github/workflows/db_provider_tests.yml +++ b/.github/workflows/db_provider_tests.yml @@ -3,11 +3,13 @@ name: DB Provider Tests on: push: branches: [main] - pull_request: + pull_request_review: + types: [submitted] branches: [main] jobs: test: + if: github.event.review.state == 'approved' || github.event_name == 'push' runs-on: ubuntu-latest services: @@ -39,13 +41,13 @@ jobs: --health-retries=3 steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - name: Install dependencies - run: | - pip install -r requirements-dev.txt - pip install -e . - - name: Run tests - run: pytest + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Install dependencies + run: | + pip install -r requirements-dev.txt + pip install -e . + - name: Run tests + run: pytest