From 66433f4179b482f17cba4c7bd64ff64e0d9968f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Ayuso?= Date: Mon, 12 May 2025 19:50:53 +0200 Subject: [PATCH 01/81] Pre-release version. Errors contained --- .github/workflows/ci.yml | 146 +++ .gitignore | 3 + CONTRIBUTING.md | 52 + README.md | 198 +++- corebrain/__init__.py | 86 ++ corebrain/cli.py | 8 + corebrain/cli/__init__.py | 57 ++ corebrain/cli/__main__.py | 12 + corebrain/cli/auth/__init__.py | 22 + corebrain/cli/auth/api_keys.py | 299 ++++++ corebrain/cli/auth/sso.py | 346 +++++++ corebrain/cli/commands.py | 172 ++++ corebrain/cli/common.py | 10 + corebrain/cli/config.py | 489 ++++++++++ corebrain/cli/utils.py | 595 ++++++++++++ corebrain/config/__init__.py | 10 + corebrain/config/manager.py | 191 ++++ corebrain/core/__init__.py | 20 + corebrain/core/client.py | 1364 +++++++++++++++++++++++++++ corebrain/core/common.py | 225 +++++ corebrain/core/query.py | 1037 ++++++++++++++++++++ corebrain/core/test_utils.py | 157 +++ corebrain/db/__init__.py | 26 + corebrain/db/connector.py | 33 + corebrain/db/connectors/__init__.py | 28 + corebrain/db/connectors/mongodb.py | 474 ++++++++++ corebrain/db/connectors/sql.py | 598 ++++++++++++ corebrain/db/engines.py | 16 + corebrain/db/factory.py | 29 + corebrain/db/interface.py | 36 + corebrain/db/schema/__init__.py | 11 + corebrain/db/schema/extractor.py | 123 +++ corebrain/db/schema/optimizer.py | 157 +++ corebrain/db/schema_file.py | 583 ++++++++++++ corebrain/network/__init__.py | 22 + corebrain/network/client.py | 502 ++++++++++ corebrain/sdk.py | 8 + corebrain/services/schema.py | 31 + corebrain/utils/__init__.py | 66 ++ corebrain/utils/encrypter.py | 264 ++++++ corebrain/utils/logging.py | 243 +++++ corebrain/utils/serializer.py | 33 + examples/add_config.py | 27 + examples/complex.py | 23 + examples/list_schema.py | 162 ++++ examples/simple.py | 15 + health.py | 47 + pyproject.toml | 82 ++ setup.py | 38 + 49 files changed, 9175 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci.yml create mode 100644 CONTRIBUTING.md create mode 100644 corebrain/__init__.py create mode 100644 corebrain/cli.py create mode 100644 corebrain/cli/__init__.py create mode 100644 corebrain/cli/__main__.py create mode 100644 corebrain/cli/auth/__init__.py create mode 100644 corebrain/cli/auth/api_keys.py create mode 100644 corebrain/cli/auth/sso.py create mode 100644 corebrain/cli/commands.py create mode 100644 corebrain/cli/common.py create mode 100644 corebrain/cli/config.py create mode 100644 corebrain/cli/utils.py create mode 100644 corebrain/config/__init__.py create mode 100644 corebrain/config/manager.py create mode 100644 corebrain/core/__init__.py create mode 100644 corebrain/core/client.py create mode 100644 corebrain/core/common.py create mode 100644 corebrain/core/query.py create mode 100644 corebrain/core/test_utils.py create mode 100644 corebrain/db/__init__.py create mode 100644 corebrain/db/connector.py create mode 100644 corebrain/db/connectors/__init__.py create mode 100644 corebrain/db/connectors/mongodb.py create mode 100644 corebrain/db/connectors/sql.py create mode 100644 corebrain/db/engines.py create mode 100644 corebrain/db/factory.py create mode 100644 corebrain/db/interface.py create mode 100644 corebrain/db/schema/__init__.py create mode 100644 corebrain/db/schema/extractor.py create mode 100644 corebrain/db/schema/optimizer.py create mode 100644 corebrain/db/schema_file.py create mode 100644 corebrain/network/__init__.py create mode 100644 corebrain/network/client.py create mode 100644 corebrain/sdk.py create mode 100644 corebrain/services/schema.py create mode 100644 corebrain/utils/__init__.py create mode 100644 corebrain/utils/encrypter.py create mode 100644 corebrain/utils/logging.py create mode 100644 corebrain/utils/serializer.py create mode 100644 examples/add_config.py create mode 100644 examples/complex.py create mode 100644 examples/list_schema.py create mode 100644 examples/simple.py create mode 100644 health.py create mode 100644 pyproject.toml create mode 100644 setup.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a2db687 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,146 @@ +name: Corebrain SDK CI/CD + +on: + push: + branches: [ main, develop ] + tags: + - 'v*' + pull_request: + branches: [ main, develop ] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.8', '3.9', '3.10', '3.11'] + + services: + # PostgreSQL service for integration tests + postgres: + image: postgres:13 + env: + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + POSTGRES_DB: test_db + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + # MongoDB service for NoSQL integration tests + mongodb: + image: mongo:4.4 + ports: + - 27017:27017 + options: >- + --health-cmd "mongo --eval 'db.runCommand({ ping: 1 })'" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install .[dev,all_db] + + - name: Lint with flake8 + run: | + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + + - name: Type check with mypy + run: | + mypy core db cli utils + + - name: Format check with black + run: | + black --check . + + - name: Test with pytest + run: | + pytest --cov=. --cov-report=xml + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + fail_ci_if_error: false + + build-and-publish: + needs: test + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/v') + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build package + run: | + python -m build + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + skip_existing: true + + docker: + needs: test + runs-on: ubuntu-latest + if: | + (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop')) || + startsWith(github.ref, 'refs/tags/v') + + steps: + - uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: corebrain/sdk + tags: | + type=ref,event=branch + type=ref,event=tag + type=semver,pattern={{version}} + type=sha,format=short + + - name: Build and push + uses: docker/build-push-action@v3 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 0a19790..a9285cc 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ __pycache__/ *.py[cod] *$py.class +venv/ +.tofix/ +README-no-valid.md # C extensions *.so diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..19f0904 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,52 @@ +# Guía de contribución a CoreBrain SDK + +¡Gracias por tu interés en contribuir a CoreBrain SDK! Este documento proporciona directrices para contribuir al proyecto. + +## Código de conducta + +Al participar en este proyecto, te comprometes a mantener un entorno respetuoso y colaborativo. + +## Cómo contribuir + +### Reportar bugs + +1. Verifica que el bug no haya sido reportado ya en los [issues](https://github.com/corebrain/sdk/issues) +2. Usa la plantilla de bug para crear un nuevo issue +3. Incluye tanto detalle como sea posible: pasos para reproducir, entorno, versiones, etc. +4. Si es posible, incluye un ejemplo mínimo que reproduzca el problema + +### Sugerir mejoras + +1. Revisa los [issues](https://github.com/corebrain/sdk/issues) para ver si ya se ha sugerido +2. Usa la plantilla de feature para crear un nuevo issue +3. Describe claramente la mejora y justifica su valor + +### Enviar cambios + +1. Haz fork del repositorio +2. Crea una rama para tu cambio (`git checkout -b feature/amazing-feature`) +3. Realiza tus cambios siguiendo las convenciones de código +4. Escribe tests para tus cambios +5. Asegúrate de que todos los tests pasan +6. Haz commit de tus cambios (`git commit -m 'Add amazing feature'`) +7. Sube tu rama (`git push origin feature/amazing-feature`) +8. Abre un Pull Request + +## Entorno de desarrollo + +### Instalación para desarrollo + +```bash +# Clonar el repositorio +git clone https://github.com/corebrain/sdk.git +cd sdk + +# Crear entorno virtual +python -m venv venv +source venv/bin/activate # En Windows: venv\Scripts\activate + +# Instalar para desarrollo +pip install -e ".[dev]" +``` + +### Estructura del proyecto diff --git a/README.md b/README.md index 42f5614..97dba0a 100644 --- a/README.md +++ b/README.md @@ -1 +1,197 @@ -# Corebrain \ No newline at end of file +# Corebrain SDK + +![CI Status](https://github.com/ceoweggo/Corebrain/workflows/Corebrain%20SDK%20CI/CD/badge.svg) +[![PyPI version](https://badge.fury.io/py/corebrain.svg)](https://badge.fury.io/py/corebrain) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) + +SDK for natural language queries to relational and non-relational databases. Enables interaction with databases using natural language questions. + +## ✨ Features + +- **Natural Language Queries**: Transforms human language questions into database queries (SQL/NoSQL) +- **Multi-Database Support**: Compatible with SQLite, MySQL, PostgreSQL, and MongoDB +- **Unified Interface**: Consistent API across different database types +- **Built-in CLI**: Interact with your databases directly from the terminal +- **Strong Security**: Robust authentication and secure credential management +- **Highly Extensible**: Designed for easy integration with new engines and features +- **Comprehensive Documentation**: Usage examples, API reference, and step-by-step guides + +## 📋 Requirements + +- Python 3.8+ +- Specific dependencies based on the database engine: + - **SQLite**: Included in Python + - **PostgreSQL**: `psycopg2-binary` + - **MySQL**: `mysql-connector-python` + - **MongoDB**: `pymongo` + +## 🔧 Installation + +### From PyPI (recommended) + +```bash +# Minimal installation +pip install corebrain + +### From source code + +```bash +git clone https://github.com/ceoweggo/Corebrain.git +pip install -e . +``` + +## 🚀 Quick Start Guide + +### Initialization + +```python +from corebrain import init + +# Initialize with a previously saved configuration +corebrain = init( + api_key="your_api_key", + config_id="your_config_id" +) +``` + +### Making Natural Language Queries + +```python +# Simple query +result = client.ask("How many active users are there?") +print(result["explanation"]) # Natural language explanation +print(result["query"]) # Generated SQL/NoSQL query +print(result["results"]) # Query results + +# Query with additional parameters +result = client.ask( + "Show the last 5 orders", + collection_name="orders", + limit=5, + filters={"status": "completed"} +) + +# Iterate over the results +for item in result["results"]: + print(item) +``` + +### Getting the Database Schema + +```python +# Get the complete schema +schema = client.db_schema + +# List all tables/collections +tables = client.list_collections_name() +print(tables) +``` + +### Closing the Connection + +```python +# It's recommended to close the connection when finished +client.close() + +# Or use the with context +with init(api_key="your_api_key", config_id="your_config_id") as client: + result = client.ask("How many users are there?") + print(result["explanation"]) +``` + +## 🖥️ Command Line Interface Usage + +### Configure Connection + +```bash +# Init configuration +corebrain --configure +``` + +### Display Database Schema + +```bash +# Show complete schema +corebrain --show-schema +``` + +### List Configurations + +```bash +# List all configurations +corebrain --list-configs +``` + +## 📝 Advanced Documentation + +### Configuration Management + +```python +from corebrain import list_configurations, remove_configuration, get_config + +# List all configurations +configs = list_configurations(api_token="your_api_token") +print(configs) + +# Get details of a configuration +config = get_config(api_token="your_api_token", config_id="your_config_id") +print(config) + +# Remove a configuration +removed = remove_configuration(api_token="your_api_token", config_id="your_config_id") +print(f"Configuration removed: {removed}") +``` + +## 🧪 Testing and Development + +### Development Installation + +```bash +# Clone the repository +git clone https://github.com/ceoweggo/Corebrain.git +cd corebrain + +# Install in development mode with extra tools +pip install -e ".[dev,all_db]" +``` + +### Verifying Style and Typing + +```bash +# Check style with flake8 +flake8 . + +# Check typing with mypy +mypy core db cli utils + +# Format code with black +black . +``` + +### Continuous Integration and Deployment (CI/CD) + +The project uses GitHub Actions to automate: + +1. **Testing**: Runs tests on multiple Python versions (3.8-3.11) +2. **Quality Verification**: Checks style, typing, and formatting +3. **Coverage**: Generates code coverage reports +4. **Automatic Publication**: Publishes new versions to PyPI when tags are created +5. **Docker Images**: Builds and publishes Docker images with each version + +You can see the complete configuration in `.github/workflows/ci.yml`. + +## 🛠️ Contributions + +Contributions are welcome! To contribute: + +1. Fork the repository +2. Create a branch for your feature (`git checkout -b feature/amazing-feature`) +3. Commit your changes (`git commit -m 'Add some amazing feature'`) +4. Push to the branch (`git push origin feature/amazing-feature`) +5. Open a Pull Request + +Please make sure your changes pass all tests and comply with the style guidelines. + +## 📄 License + +Distributed under the MIT License. See `LICENSE` for more information. \ No newline at end of file diff --git a/corebrain/__init__.py b/corebrain/__init__.py new file mode 100644 index 0000000..5d21ae4 --- /dev/null +++ b/corebrain/__init__.py @@ -0,0 +1,86 @@ +""" +Corebrain SDK. + +This package provides a Python SDK for interacting with the Corebrain API +and enables natural language queries to relational and non-relational databases. +""" +import logging +from typing import Dict, Any, List, Optional + +# Configuración básica de logging +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + +# Importaciones seguras (sin dependencias circulares) +from corebrain.db.engines import get_available_engines +from corebrain.core.client import Corebrain +from corebrain.config.manager import ConfigManager + +# Exportación explícita de componentes públicos +__all__ = [ + 'init', + 'extract_db_schema', + 'list_configurations', + 'remove_configuration', + 'get_available_engines', + 'get_config', + '__version__' +] + +# Variable de versión +__version__ = "1.0.0" + +def init(api_key: str, config_id: str, skip_verification: bool = False) -> Corebrain: + """ + Initialize the Corebrain SDK with the provided API key and configuration. + + Args: + api_key: API Key de Corebrain + config_id: ID de la configuración a usar + + Returns: + Instancia de Corebrain configurada + """ + return Corebrain(api_key=api_key, config_id=config_id, skip_verification=skip_verification) + +# Funciones de conveniencia a nivel de paquete +def list_configurations(api_key: str) -> List[str]: + """ + Lista las configuraciones disponibles para una API key. + + Args: + api_key: API Key de Corebrain + + Returns: + Lista de IDs de configuración disponibles + """ + config_manager = ConfigManager() + return config_manager.list_configs(api_key) + +def remove_configuration(api_key: str, config_id: str) -> bool: + """ + Elimina una configuración específica. + + Args: + api_key: API Key de Corebrain + config_id: ID de la configuración a eliminar + + Returns: + True si se eliminó correctamente, False en caso contrario + """ + config_manager = ConfigManager() + return config_manager.remove_config(api_key, config_id) + +def get_config(api_key: str, config_id: str) -> Optional[Dict[str, Any]]: + """ + Obtiene una configuración específica. + + Args: + api_key: API Key de Corebrain + config_id: ID de la configuración a obtener + + Returns: + Diccionario con la configuración o None si no existe + """ + config_manager = ConfigManager() + return config_manager.get_config(api_key, config_id) \ No newline at end of file diff --git a/corebrain/cli.py b/corebrain/cli.py new file mode 100644 index 0000000..0ef90a6 --- /dev/null +++ b/corebrain/cli.py @@ -0,0 +1,8 @@ +""" +Punto de entrada para la CLI de Corebrain para compatibilidad. +""" +from corebrain.cli.__main__ import main + +if __name__ == "__main__": + import sys + sys.exit(main()) \ No newline at end of file diff --git a/corebrain/cli/__init__.py b/corebrain/cli/__init__.py new file mode 100644 index 0000000..24a2c65 --- /dev/null +++ b/corebrain/cli/__init__.py @@ -0,0 +1,57 @@ +""" +Interfaz de línea de comandos para Corebrain SDK. + +Este módulo proporciona una interfaz de línea de comandos para configurar +y usar el SDK de Corebrain para consultas en lenguaje natural a bases de datos. +""" +import sys +from typing import Optional, List + +# Importar componentes principales para CLI +from corebrain.cli.commands import main_cli +from corebrain.cli.utils import print_colored, ProgressTracker, get_free_port +from corebrain.cli.config import ( + configure_sdk, + get_db_type, + get_db_engine, + get_connection_params, + test_database_connection, + select_excluded_tables +) +from corebrain.cli.auth import ( + authenticate_with_sso, + fetch_api_keys, + exchange_sso_token_for_api_token, + verify_api_token +) + + +# Exportación explícita de componentes públicos +__all__ = [ + 'main_cli', + 'run_cli', + 'print_colored', + 'ProgressTracker', + 'get_free_port', + 'configure_sdk', + 'authenticate_with_sso', + 'fetch_api_keys', + 'exchange_sso_token_for_api_token', + 'verify_api_token' +] + +# Función de conveniencia para ejecutar CLI +def run_cli(argv: Optional[List[str]] = None) -> int: + """ + Ejecuta la CLI con los argumentos proporcionados. + + Args: + argv: Lista de argumentos (usa sys.argv si es None) + + Returns: + Código de salida + """ + if argv is None: + argv = sys.argv[1:] + + return main_cli(argv) \ No newline at end of file diff --git a/corebrain/cli/__main__.py b/corebrain/cli/__main__.py new file mode 100644 index 0000000..ea8ee3a --- /dev/null +++ b/corebrain/cli/__main__.py @@ -0,0 +1,12 @@ +""" +Punto de entrada para ejecutar la CLI como módulo. +""" +import sys +from corebrain.cli.commands import main_cli + +def main(): + """Función principal para entry point en pyproject.toml""" + return main_cli() + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/corebrain/cli/auth/__init__.py b/corebrain/cli/auth/__init__.py new file mode 100644 index 0000000..ef50100 --- /dev/null +++ b/corebrain/cli/auth/__init__.py @@ -0,0 +1,22 @@ +""" +Módulos de autenticación para CLI de Corebrain. + +Este paquete proporciona funcionalidades para autenticación, +gestión de tokens y API keys en la CLI de Corebrain. +""" +from corebrain.cli.auth.sso import authenticate_with_sso, TokenHandler +from corebrain.cli.auth.api_keys import ( + fetch_api_keys, + exchange_sso_token_for_api_token, + verify_api_token, + get_api_key_id_from_token +) +# Exportación explícita de componentes públicos +__all__ = [ + 'authenticate_with_sso', + 'TokenHandler', + 'fetch_api_keys', + 'exchange_sso_token_for_api_token', + 'verify_api_token', + 'get_api_key_id_from_token' +] \ No newline at end of file diff --git a/corebrain/cli/auth/api_keys.py b/corebrain/cli/auth/api_keys.py new file mode 100644 index 0000000..6163391 --- /dev/null +++ b/corebrain/cli/auth/api_keys.py @@ -0,0 +1,299 @@ +""" +API Keys Management for the CLI. +""" +import uuid +import httpx + +from typing import Optional, Dict, Any, Tuple + +from corebrain.cli.utils import print_colored +from corebrain.network.client import http_session +from corebrain.core.client import Corebrain + +def verify_api_token(token: str, api_url: Optional[str] = None, user_data: Optional[Dict[str, Any]] = None) -> Tuple[bool, Optional[Dict[str, Any]]]: + """ + Verifies if an API token is valid. + + Args: + token: API token to verify + api_url: Optional API URL + user_data: User data + + Returns: + Tuple with (validity, user information) if valid, (False, None) if not + """ + try: + # Create a temporary SDK instance to verify the token + config = {"type": "test", "config_id": str(uuid.uuid4())} + kwargs = {"api_token": token, "db_config": config} + + if user_data: + kwargs["user_data"] = user_data + + if api_url: + kwargs["api_url"] = api_url + + sdk = Corebrain(**kwargs) + return True, sdk.user_info + except Exception as e: + print_colored(f"Error verifying API token: {str(e)}", "red") + return False, None + +def fetch_api_keys(api_url: str, api_token: str, user_data: Dict[str, Any]) -> Optional[str]: + """ + Retrieves the available API keys for the user and allows selecting one. + + Args: + api_url: Base URL of the Corebrain API + api_token: API token (exchanged from SSO token) + user_data: User data + + Returns: + Selected API key or None if none is selected + """ + if not user_data or 'id' not in user_data: + print_colored("Could not identify the user to retrieve their API keys.", "yellow") + return None + + try: + # Ensure protocol in URL + if not api_url.startswith(("http://", "https://")): + api_url = "https://" + api_url + + # Remove trailing slash if it exists + if api_url.endswith('/'): + api_url = api_url[:-1] + + # Build endpoint to get API keys + endpoint = f"{api_url}/api/auth/api-keys" + + print_colored(f"Requesting user's API keys...", "blue") + + # Configure client with timeout and error handling + headers = { + "Authorization": f"Bearer {api_token}", + "Content-Type": "application/json" + } + + response = http_session.get(endpoint, headers=headers) + + # Verify response + if response.status_code == 200: + try: + api_keys_data = response.json() + # Verify response format + if not isinstance(api_keys_data, (list, dict)): + print_colored(f"Unexpected response format: {type(api_keys_data)}", "yellow") + return None + + # Handle both direct list and dictionary with list + api_keys = api_keys_data if isinstance(api_keys_data, list) else api_keys_data.get("data", []) + + if not api_keys: + print_colored("No API keys available for this user.", "yellow") + return None + + print_colored(f"\nFound {len(api_keys)} API keys", "green") + print_colored("\n=== Available API Keys ===", "blue") + + # Show available API keys + for i, key_info in enumerate(api_keys, 1): + key_id = key_info.get('id', 'No ID') + key_value = key_info.get('key', 'No value') + key_name = key_info.get('name', 'No name') + key_active = key_info.get('active') + + # Show status with color + status_color = "green" if key_active == True else "red" + status_text = "Active" if key_active == True else "Inactive" + + print(f"{i}. {key_name} - {print_colored(status_text, status_color, return_str=True)} (Value: {key_value})") + + # Ask user to select an API key + while True: + try: + choice = input(f"\nSelect an API key (1-{len(api_keys)}) or press Enter to cancel: ").strip() + + # Allow canceling and using API token + if not choice: + print_colored("No API key selected.", "yellow") + return None + + choice_num = int(choice) + if 1 <= choice_num <= len(api_keys): + selected_key = api_keys[choice_num - 1] + + # Verify if the key is active + if selected_key.get('active') != True: + print_colored("⚠️ The selected API key is not active. Select another one.", "yellow") + continue + + # Get information of the selected key + key_name = selected_key.get('name', 'Unknown') + key_value = selected_key.get('key', None) + + if not key_value: + print_colored("⚠️ The selected API key does not have a valid value.", "yellow") + continue + + print_colored(f"✅ You selected: {key_name}", "green") + print_colored("Wait while we assign the API key to your SDK...", "yellow") + + return key_value + else: + print_colored("Invalid option. Try again.", "red") + except ValueError: + print_colored("Please enter a valid number.", "red") + except Exception as e: + print_colored(f"Error processing JSON response: {str(e)}", "red") + return None + else: + # Handle error by status code + error_message = f"Error retrieving API keys: {response.status_code}" + + try: + error_data = response.json() + if "message" in error_data: + error_message += f" - {error_data['message']}" + elif "detail" in error_data: + error_message += f" - {error_data['detail']}" + except: + # If we can't parse JSON, use the full text + error_message += f" - {response.text[:100]}..." + + print_colored(error_message, "red") + + # Try to identify common problems + if response.status_code == 401: + print_colored("The authentication token has expired or is invalid.", "yellow") + elif response.status_code == 403: + print_colored("You don't have permissions to access the API keys.", "yellow") + elif response.status_code == 404: + print_colored("The API keys endpoint doesn't exist. Verify the API URL.", "yellow") + elif response.status_code >= 500: + print_colored("Server error. Try again later.", "yellow") + + return None + + except httpx.RequestError as e: + print_colored(f"Connection error: {str(e)}", "red") + print_colored("Verify the API URL and your internet connection.", "yellow") + return None + except Exception as e: + print_colored(f"Unexpected error retrieving API keys: {str(e)}", "red") + return None + +def get_api_key_id_from_token(sso_token: str, api_token: str, api_url: str) -> Optional[str]: + """ + Gets the ID of an API key from its token. + + Args: + sso_token: SSO token + api_token: API token + api_url: API URL + + Returns: + API key ID or None if it cannot be obtained + """ + try: + # Endpoint to get information of the current user + endpoint = f"{api_url}/api/auth/api-keys/{api_token}" + + headers = { + "Authorization": f"Bearer {api_token}", + "Content-Type": "application/json" + } + + response = httpx.get( + endpoint, + headers=headers + ) + + print("API keys response: ", response.json()) + + if response.status_code == 200: + key_data = response.json() + key_id = key_data.get("id") + return key_id + else: + print_colored("⚠️ Could not find the API key ID", "yellow") + return None + + except Exception as e: + print_colored(f"Error getting API key ID: {str(e)}", "red") + return None + +def exchange_sso_token_for_api_token(api_url: str, sso_token: str, user_data: Dict[str, Any]) -> Optional[str]: + """ + Exchanges a Globodain SSO token for a Corebrain API token. + + Args: + api_url: Base URL of the Corebrain API + sso_token: Globodain SSO token + user_data: User data + + Returns: + API token or None if it fails + """ + try: + # Ensure protocol in URL + if not api_url.startswith(("http://", "https://")): + api_url = "https://" + api_url + + # Remove trailing slash if it exists + if api_url.endswith('/'): + api_url = api_url[:-1] + + # Endpoint to exchange token + endpoint = f"{api_url}/api/auth/sso/token" + + print_colored(f"Exchanging SSO token for API token...", "blue") + + # Configure client with timeout and error handling + headers = { + 'Authorization': f'Bearer {sso_token}', + 'Content-Type': 'application/json' + } + body = { + "user_data": user_data + } + + response = http_session.post(endpoint, json=body, headers=headers) + + if response.status_code == 200: + try: + token_data = response.json() + api_token = token_data.get("access_token") + + if not api_token: + print_colored("The response does not contain a valid API token", "red") + return None + + print_colored("✅ API token successfully obtained", "green") + return api_token + except Exception as e: + print_colored(f"Error processing JSON response: {str(e)}", "red") + return None + else: + # Handle error by status code + error_message = f"Error exchanging token: {response.status_code}" + + try: + error_data = response.json() + if "message" in error_data: + error_message += f" - {error_data['message']}" + elif "detail" in error_data: + error_message += f" - {error_data['detail']}" + except: + # If we can't parse JSON, use the full text + error_message += f" - {response.text[:100]}..." + + print_colored(error_message, "red") + return None + + except httpx.RequestError as e: + print_colored(f"Connection error: {str(e)}", "red") + return None + except Exception as e: + print_colored(f"Unexpected error exchanging token: {str(e)}", "red") + return None \ No newline at end of file diff --git a/corebrain/cli/auth/sso.py b/corebrain/cli/auth/sso.py new file mode 100644 index 0000000..83bdc55 --- /dev/null +++ b/corebrain/cli/auth/sso.py @@ -0,0 +1,346 @@ +""" +SSO Authentication for the CLI. +""" +import os +import webbrowser +import http.server +import socketserver +import threading +import urllib.parse +import time + +from typing import Tuple, Dict, Any, Optional + +from corebrain.cli.common import DEFAULT_API_URL, DEFAULT_SSO_URL, DEFAULT_PORT, SSO_CLIENT_ID, SSO_CLIENT_SECRET +from corebrain.cli.utils import print_colored +from corebrain.lib.sso.auth import GlobodainSSOAuth + +class TokenHandler(http.server.SimpleHTTPRequestHandler): + """ + Handler for the local HTTP server that processes the SSO authentication callback. + """ + def __init__(self, *args, **kwargs): + self.sso_auth = kwargs.pop('sso_auth', None) + self.result = kwargs.pop('result', {}) + self.session_data = kwargs.pop('session_data', {}) + self.auth_completed = kwargs.pop('auth_completed', None) + super().__init__(*args, **kwargs) + + def do_GET(self): + # Parse the URL to get the parameters + parsed_path = urllib.parse.urlparse(self.path) + + # Check if it's the callback path + if parsed_path.path == "/auth/sso/callback": + query = urllib.parse.parse_qs(parsed_path.query) + + if "code" in query: + code = query["code"][0] + + try: + # Exchange code for token using the sso_auth object + token_data = self.sso_auth.exchange_code_for_token(code) + + if not token_data: + raise ValueError("Could not obtain the token") + + # Save token in the result and session + access_token = token_data.get('access_token') + if not access_token: + raise ValueError("The received token does not contain an access_token") + + # Updated: save as sso_token for clarity + self.result["sso_token"] = access_token + self.session_data['sso_token'] = token_data + + # Get user information + user_info = self.sso_auth.get_user_info(access_token) + if user_info: + self.session_data['user'] = user_info + # Extract email to identify the user + if 'email' in user_info: + self.session_data['email'] = user_info['email'] + + # Signal that authentication has completed + self.auth_completed.set() + + # Send a success response to the browser + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + success_html = """ + + + Corebrain - Authentication Completed + + + +
+

Authentication Completed

+

You have successfully logged in to Corebrain CLI.

+

You can close this window and return to the terminal.

+
+ + + """ + self.wfile.write(success_html.encode()) + except Exception as e: + # If there's an error, show error message + self.send_response(400) + self.send_header("Content-type", "text/html") + self.end_headers() + error_html = f""" + + + Corebrain - Authentication Error + + + +
+

Authentication Error

+

Error: {str(e)}

+

Please close this window and try again.

+
+ + + """ + self.wfile.write(error_html.encode()) + else: + # If there's no code, it's an error + self.send_response(400) + self.send_header("Content-type", "text/html") + self.end_headers() + error_html = """ + + + Corebrain - Authentication Error + + + +
+

Authentication Error

+

Could not complete the authentication process.

+

Please close this window and try again.

+
+ + + """ + self.wfile.write(error_html.encode()) + else: + # For any other path, show a 404 error + self.send_response(404) + self.end_headers() + self.wfile.write(b"Not Found") + + def log_message(self, format, *args): + # Silence server logs + return + +def authenticate_with_sso(sso_url: str) -> Tuple[Optional[str], Optional[Dict[str, Any]], Optional[str]]: + """ + Initiates an SSO authentication flow through the browser and uses the callback system. + + Args: + sso_url: Base URL of the SSO service + + Returns: + Tuple with (api_key, user_data, api_token) or (None, None, None) if it fails + - api_key: Selected API key to use with the SDK + - user_data: Authenticated user data + - api_token: API token obtained from SSO for general authentication + """ + # Import inside the function to avoid circular dependencies + from corebrain.cli.auth.api_keys import fetch_api_keys, exchange_sso_token_for_api_token + + # Token to store the result + result = {"sso_token": None} # Renamed for clarity + auth_completed = threading.Event() + session_data = {} + + # Find an available port + #port = get_free_port(DEFAULT_PORT) + + # SSO client configuration + auth_config = { + 'GLOBODAIN_SSO_URL': sso_url or DEFAULT_SSO_URL, + 'GLOBODAIN_CLIENT_ID': SSO_CLIENT_ID, + 'GLOBODAIN_CLIENT_SECRET': SSO_CLIENT_SECRET, + 'GLOBODAIN_REDIRECT_URI': f"http://localhost:{DEFAULT_PORT}/auth/sso/callback", + 'GLOBODAIN_SUCCESS_REDIRECT': 'https://sso.globodain.com/cli/success' + } + + sso_auth = GlobodainSSOAuth(config=auth_config) + + # Factory to create TokenHandler instances with the desired parameters + def handler_factory(*args, **kwargs): + return TokenHandler( + *args, + sso_auth=sso_auth, + result=result, + session_data=session_data, + auth_completed=auth_completed, + **kwargs + ) + + # Start server in the background + server = socketserver.TCPServer(("", DEFAULT_PORT), handler_factory) + server_thread = threading.Thread(target=server.serve_forever) + server_thread.daemon = True + server_thread.start() + + try: + # Build complete URL with protocol if missing + if sso_url and not sso_url.startswith(("http://", "https://")): + sso_url = "https://" + sso_url + + # URL to start the SSO flow + login_url = sso_auth.get_login_url() + auth_url = login_url + + print_colored(f"Opening browser for SSO authentication...", "blue") + print_colored(f"If the browser doesn't open automatically, visit:", "blue") + print_colored(f"{auth_url}", "bold") + + # Try to open the browser + if not webbrowser.open(auth_url): + print_colored("Could not open the browser automatically.", "yellow") + print_colored(f"Please copy and paste the following URL into your browser:", "yellow") + print_colored(f"{auth_url}", "bold") + + # Tell the user to wait + print_colored("\nWaiting for you to complete authentication in the browser...", "blue") + + # Wait for authentication to complete (with timeout) + timeout_seconds = 60 + start_time = time.time() + + # We use a loop with better feedback + while not auth_completed.is_set() and (time.time() - start_time < timeout_seconds): + elapsed = int(time.time() - start_time) + if elapsed % 5 == 0: # Every 5 seconds we show a message + remaining = timeout_seconds - elapsed + #print_colored(f"Waiting for authentication... ({remaining}s remaining)", "yellow") + + # Check every 0.5 seconds for better reactivity + auth_completed.wait(0.5) + + # Verify if authentication was completed + if auth_completed.is_set(): + user_data = None + if 'user' in session_data: + user_data = session_data['user'] + + print_colored("✅ SSO authentication completed successfully!", "green") + + # Get and select an API key + api_url = os.environ.get("COREBRAIN_API_URL", DEFAULT_API_URL) + + # Now we use the SSO token to get an API token and then the API keys + # First we verify that we have a token + if result["sso_token"]: + api_token = exchange_sso_token_for_api_token(api_url, result["sso_token"], user_data) + + if not api_token: + print_colored("⚠️ Could not obtain an API Token with the SSO Token", "yellow") + return None, None, None + + # Now that we have the API Token, we get the available API Keys + api_key_selected = fetch_api_keys(api_url, api_token, user_data) + + if api_key_selected: + # We return the selected api_key + return api_key_selected, user_data, api_token + else: + print_colored("⚠️ Could not obtain an API Key. Create a new one using the command", "yellow") + return None, user_data, api_token + else: + print_colored("❌ No valid token was obtained during authentication.", "red") + return None, None, None + + # We don't have a token or user data + print_colored("❌ Authentication did not produce a valid token.", "red") + return None, None, None + else: + print_colored(f"❌ Could not complete SSO authentication in {timeout_seconds} seconds.", "red") + print_colored("You can try again or use a token manually.", "yellow") + return None, None, None + except Exception as e: + print_colored(f"❌ Error during SSO authentication: {str(e)}", "red") + return None, None, None + finally: + # Stop the server + try: + server.shutdown() + server.server_close() + except: + # If there's any error closing the server, we ignore it + pass \ No newline at end of file diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py new file mode 100644 index 0000000..d9adac9 --- /dev/null +++ b/corebrain/cli/commands.py @@ -0,0 +1,172 @@ +""" +Main commands for the Corebrain CLI. +""" +import argparse +import os +import sys +import webbrowser + +from typing import Optional, List + +from corebrain.cli.common import DEFAULT_API_URL, DEFAULT_SSO_URL, DEFAULT_PORT, SSO_CLIENT_ID, SSO_CLIENT_SECRET +from corebrain.cli.auth.sso import authenticate_with_sso +from corebrain.cli.config import configure_sdk, get_api_credential +from corebrain.cli.utils import print_colored +from corebrain.config.manager import ConfigManager +from corebrain.lib.sso.auth import GlobodainSSOAuth + +def main_cli(argv: Optional[List[str]] = None) -> int: + """ + Main entry point for the Corebrain CLI. + + Args: + argv: List of command line arguments (defaults to sys.argv[1:]) + + Returns: + Exit code (0 for success, other value for error) + """ + + # Package version + __version__ = "0.1.0" + + try: + print_colored("Corebrain CLI started. Version ", __version__, "blue") + + if argv is None: + argv = sys.argv[1:] + + # Argument parser configuration + parser = argparse.ArgumentParser(description="Corebrain SDK CLI") + parser.add_argument("--version", action="store_true", help="Show SDK version") + parser.add_argument("--configure", action="store_true", help="Configure the Corebrain SDK") + parser.add_argument("--list-configs", action="store_true", help="List available configurations") + parser.add_argument("--remove-config", action="store_true", help="Remove a configuration") + parser.add_argument("--show-schema", action="store_true", help="Show the schema of the configured database") + parser.add_argument("--extract-schema", action="store_true", help="Extract the database schema and save it to a file") + parser.add_argument("--output-file", help="File to save the extracted schema") + parser.add_argument("--config-id", help="Specific configuration ID to use") + parser.add_argument("--token", help="Corebrain API token (any type)") + parser.add_argument("--api-key", help="Specific API Key for Corebrain") + parser.add_argument("--api-url", help="Corebrain API URL") + parser.add_argument("--sso-url", help="Globodain SSO service URL") + parser.add_argument("--login", action="store_true", help="Login via SSO") + parser.add_argument("--test-auth", action="store_true", help="Test SSO authentication system") + + args = parser.parse_args(argv) + + # Show version + if args.version: + try: + from importlib.metadata import version + sdk_version = version("corebrain") + print(f"Corebrain SDK version {sdk_version}") + except Exception: + print(f"Corebrain SDK version {__version__}") + return 0 + + # Test SSO authentication + if args.test_auth: + sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + + print_colored("Testing SSO authentication...", "blue") + + # Authentication configuration + auth_config = { + 'GLOBODAIN_SSO_URL': sso_url, + 'GLOBODAIN_CLIENT_ID': SSO_CLIENT_ID, + 'GLOBODAIN_CLIENT_SECRET': SSO_CLIENT_SECRET, + 'GLOBODAIN_REDIRECT_URI': f"http://localhost:{DEFAULT_PORT}/auth/sso/callback", + 'GLOBODAIN_SUCCESS_REDIRECT': f"http://localhost:{DEFAULT_PORT}/auth/sso/callback" + } + + try: + # Instantiate authentication client + sso_auth = GlobodainSSOAuth(config=auth_config) + + # Get login URL + login_url = sso_auth.get_login_url() + + print_colored(f"Login URL: {login_url}", "blue") + print_colored("Opening browser for login...", "blue") + + # Open browser + webbrowser.open(login_url) + + print_colored("Please complete the login process in the browser.", "blue") + input("\nPress Enter when you've completed the process or to cancel...") + + print_colored("✅ SSO authentication test completed!", "green") + return 0 + except Exception as e: + print_colored(f"❌ Error during test: {str(e)}", "red") + return 1 + + # Login via SSO + if args.login: + sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + api_key, user_data, api_token = authenticate_with_sso(sso_url) + + if api_token: + # Save the general token for future use + os.environ["COREBRAIN_API_TOKEN"] = api_token + + if api_key: + # Save the specific API key for future use + os.environ["COREBRAIN_API_KEY"] = api_key + print_colored("✅ API Key successfully saved. You can use the SDK now.", "green") + + # If configuration was also requested, continue with the process + if args.configure: + api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + configure_sdk(api_token, api_key, api_url, sso_url, user_data) + + return 0 + else: + print_colored("❌ Could not obtain an API Key via SSO.", "red") + if api_token: + print_colored("A general API token was obtained, but not a specific API Key.", "yellow") + print_colored("You can create an API Key in the Corebrain dashboard.", "yellow") + return 1 + + # Operations that require credentials: configure, list, remove or show schema + if args.configure or args.list_configs or args.remove_config or args.show_schema or args.extract_schema: + # Get URLs + api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + + # Prioritize api_key if explicitly provided + token_arg = args.api_key if args.api_key else args.token + + # Get API credentials + api_key, user_data, api_token = get_api_credential(token_arg, sso_url) + + if not api_key: + print_colored("Error: An API Key is required. You can generate one at dashboard.corebrain.com", "red") + print_colored("Or use the 'corebrain --login' command to login via SSO.", "blue") + return 1 + + from corebrain.db.schema_file import show_db_schema, extract_schema_to_file + + # Execute the selected operation + if args.configure: + configure_sdk(api_token, api_key, api_url, sso_url, user_data) + elif args.list_configs: + ConfigManager.list_configs(api_key, api_url) + elif args.remove_config: + ConfigManager.remove_config(api_key, api_url) + elif args.show_schema: + show_db_schema(api_key, args.config_id, api_url) + elif args.extract_schema: + extract_schema_to_file(api_key, args.config_id, args.output_file, api_url) + + else: + # If no option was specified, show help + parser.print_help() + print_colored("\nTip: Use 'corebrain --login' to login via SSO.", "blue") + + return 0 + except Exception as e: + print_colored(f"Error: {str(e)}", "red") + import traceback + traceback.print_exc() + return 1 \ No newline at end of file diff --git a/corebrain/cli/common.py b/corebrain/cli/common.py new file mode 100644 index 0000000..9dd55e9 --- /dev/null +++ b/corebrain/cli/common.py @@ -0,0 +1,10 @@ +""" +Default values for SSO and API connection +""" + +DEFAULT_API_URL = "http://localhost:5000" +DEFAULT_SSO_URL = "http://localhost:3000" +DEFAULT_PORT = 8765 +DEFAULT_TIMEOUT = 10 +SSO_CLIENT_ID = '401dca6e-3f3b-4458-b3ef-f87eaae0398d' +SSO_CLIENT_SECRET = 'f9d315ea-5a65-4e3f-be35-b27a933dfb5b' \ No newline at end of file diff --git a/corebrain/cli/config.py b/corebrain/cli/config.py new file mode 100644 index 0000000..ca3dc13 --- /dev/null +++ b/corebrain/cli/config.py @@ -0,0 +1,489 @@ +""" +Configuration functions for the CLI. +""" +import json +import uuid +import getpass +import os +from typing import Dict, Any, List, Optional, Tuple +from datetime import datetime + +from corebrain.cli.common import DEFAULT_API_URL, DEFAULT_SSO_URL +from corebrain.cli.auth import authenticate_with_sso +from corebrain.cli.utils import print_colored, ProgressTracker +from corebrain.db.engines import get_available_engines +from corebrain.config.manager import ConfigManager +from corebrain.network.client import http_session +from corebrain.core.test_utils import test_natural_language_query +from corebrain.db.schema_file import extract_db_schema + +def get_api_credential(args_token: Optional[str] = None, sso_url: Optional[str] = None) -> Tuple[Optional[str], Optional[Dict[str, Any]], Optional[str]]: + """ + Obtains the API credential (API key), trying several methods in order: + 1. Token provided as argument + 2. Environment variable + 3. SSO authentication + 4. Manual user input + + Args: + args_token: Token provided as argument + sso_url: SSO service URL + + Returns: + Tuple with (api_key, user_data, api_token) or (None, None, None) if couldn't be obtained + - api_key: API key to use with SDK + - user_data: User data + - api_token: API token for general authentication + """ + # 1. Check if provided as argument + if args_token: + print_colored("Using token provided as argument.", "blue") + # Assume the provided token is directly an API key + return args_token, None, args_token + + # 2. Check environment variable for API key + env_api_key = os.environ.get("COREBRAIN_API_KEY") + if env_api_key: + print_colored("Using API key from COREBRAIN_API_KEY environment variable.", "blue") + return env_api_key, None, env_api_key + + # 3. Check environment variable for API token + env_api_token = os.environ.get("COREBRAIN_API_TOKEN") + if env_api_token: + print_colored("Using API token from COREBRAIN_API_TOKEN environment variable.", "blue") + # Note: Here we return the same value as api_key and api_token + # because we have no way to obtain a specific api_key + return env_api_token, None, env_api_token + + # 4. Try SSO authentication + print_colored("Attempting authentication via SSO...", "blue") + api_key, user_data, api_token = authenticate_with_sso(sso_url or DEFAULT_SSO_URL) + print("Exit from authenticate_with_sso: ", datetime.now()) + if api_key: + # Save for future use + os.environ["COREBRAIN_API_KEY"] = api_key + os.environ["COREBRAIN_API_TOKEN"] = api_token + return api_key, user_data, api_token + + # 5. Request manually + print_colored("\nCouldn't complete SSO authentication.", "yellow") + print_colored("You can directly enter an API key:", "blue") + manual_input = input("Enter your Corebrain API key: ").strip() + if manual_input: + # Assume manual input is an API key + return manual_input, None, manual_input + + # If we got here, we couldn't get a credential + return None, None, None + +def get_db_type() -> str: + """ + Prompts the user to select a database type. + + Returns: + Selected database type + """ + print_colored("\n=== Select the database type ===", "blue") + print("1. SQL (SQLite, MySQL, PostgreSQL)") + print("2. NoSQL (MongoDB)") + + while True: + try: + choice = int(input("\nSelect an option (1-2): ").strip()) + if choice == 1: + return "sql" + elif choice == 2: + return "nosql" + else: + print_colored("Invalid option. Try again.", "red") + except ValueError: + print_colored("Please enter a number.", "red") + +def get_db_engine(db_type: str) -> str: + """ + Prompts the user to select a database engine. + + Args: + db_type: Selected database type + + Returns: + Selected database engine + """ + engines = get_available_engines() + + if db_type == "sql": + available_engines = engines["sql"] + print_colored("\n=== Select the SQL engine ===", "blue") + for i, engine in enumerate(available_engines, 1): + print(f"{i}. {engine.capitalize()}") + + while True: + try: + choice = int(input(f"\nSelect an option (1-{len(available_engines)}): ").strip()) + if 1 <= choice <= len(available_engines): + return available_engines[choice - 1] + else: + print_colored("Invalid option. Try again.", "red") + except ValueError: + print_colored("Please enter a number.", "red") + else: + # For NoSQL, we only have MongoDB for now + return "mongodb" + +def get_connection_params(db_type: str, engine: str) -> Dict[str, Any]: + """ + Prompts for connection parameters according to the database type and engine. + + Args: + db_type: Database type + engine: Database engine + + Returns: + Dictionary with connection parameters + """ + params = {"type": db_type, "engine": engine} + + # Specific parameters by type and engine + if db_type == "sql": + if engine == "sqlite": + path = input("\nPath to SQLite database file: ").strip() + params["database"] = path + else: + # MySQL or PostgreSQL + print_colored("\n=== Connection Parameters ===", "blue") + params["host"] = input("Host (default: localhost): ").strip() or "localhost" + + if engine == "mysql": + params["port"] = int(input("Port (default: 3306): ").strip() or "3306") + else: # PostgreSQL + params["port"] = int(input("Port (default: 5432): ").strip() or "5432") + + params["user"] = input("User: ").strip() + params["password"] = getpass.getpass("Password: ") + params["database"] = input("Database name: ").strip() + else: + # MongoDB + print_colored("\n=== MongoDB Connection Parameters ===", "blue") + use_connection_string = input("Use connection string? (y/n): ").strip().lower() == "y" + + if use_connection_string: + params["connection_string"] = input("MongoDB connection string: ").strip() + else: + params["host"] = input("Host (default: localhost): ").strip() or "localhost" + params["port"] = int(input("Port (default: 27017): ").strip() or "27017") + + use_auth = input("Use authentication? (y/n): ").strip().lower() == "y" + if use_auth: + params["user"] = input("User: ").strip() + params["password"] = getpass.getpass("Password: ") + + params["database"] = input("Database name: ").strip() + + # Add configuration ID + params["config_id"] = str(uuid.uuid4()) + params["excluded_tables"] = [] + + return params + +def test_database_connection(api_token: str, db_config: Dict[str, Any], api_url: Optional[str] = None, user_data: Optional[Dict[str, Any]] = None) -> bool: + """ + Tests the database connection without verifying the API token. + + Args: + api_token: API token + db_config: Database configuration + api_url: Optional API URL + user_data: User data + + Returns: + True if connection is successful, False otherwise + """ + try: + print_colored("\nTesting database connection...", "blue") + + db_type = db_config["type"].lower() + engine = db_config.get("engine", "").lower() + + if db_type == "sql": + if engine == "sqlite": + import sqlite3 + conn = sqlite3.connect(db_config.get("database", "")) + cursor = conn.cursor() + cursor.execute("SELECT 1") + cursor.close() + conn.close() + + elif engine == "mysql": + import mysql.connector + if "connection_string" in db_config: + conn = mysql.connector.connect(connection_string=db_config["connection_string"]) + else: + conn = mysql.connector.connect( + host=db_config.get("host", "localhost"), + user=db_config.get("user", ""), + password=db_config.get("password", ""), + database=db_config.get("database", ""), + port=db_config.get("port", 3306) + ) + cursor = conn.cursor() + cursor.execute("SELECT 1") + cursor.close() + conn.close() + + elif engine == "postgresql": + import psycopg2 + if "connection_string" in db_config: + conn = psycopg2.connect(db_config["connection_string"]) + else: + conn = psycopg2.connect( + host=db_config.get("host", "localhost"), + user=db_config.get("user", ""), + password=db_config.get("password", ""), + dbname=db_config.get("database", ""), + port=db_config.get("port", 5432) + ) + cursor = conn.cursor() + cursor.execute("SELECT 1") + cursor.close() + conn.close() + + elif db_type == "nosql" and engine == "mongodb": + import pymongo + if "connection_string" in db_config: + client = pymongo.MongoClient(db_config["connection_string"]) + else: + client = pymongo.MongoClient( + host=db_config.get("host", "localhost"), + port=db_config.get("port", 27017), + username=db_config.get("user"), + password=db_config.get("password") + ) + + # Verify connection by trying to access the database + db = client[db_config["database"]] + # List collections to verify we can access + _ = db.list_collection_names() + client.close() + + # If we got here, the connection was successful + print_colored("✅ Database connection successful!", "green") + return True + except Exception as e: + print_colored(f"❌ Error connecting to the database: {str(e)}", "red") + return False + +def select_excluded_tables(api_token: str, db_config: Dict[str, Any], api_url: Optional[str] = None, user_data: Optional[Dict[str, Any]] = None) -> List[str]: + + """ + Allows the user to select tables/collections to exclude. + + Args: + api_token: API token + db_config: Database configuration + api_url: Optional API URL + user_data: User data + + Returns: + List of excluded tables/collections + """ + print_colored("\nRetrieving database schema...", "blue") + + # Get the database schema directly + schema = extract_db_schema(db_config) + + if not schema or not schema.get("tables"): + print_colored("No tables/collections found.", "yellow") + return [] + + print_colored("\n=== Tables/Collections found ===", "blue") + print("Mark with 'n' the tables that should NOT be accessible (y for accessible)") + + # Use the tables list instead of the dictionary + tables_list = schema.get("tables_list", []) + excluded_tables = [] + + if not tables_list: + # If there's no table list, convert the tables dictionary to a list + tables = schema.get("tables", {}) + for table_name in tables: + choice = input(f"{table_name} (accessible? y/n): ").strip().lower() + if choice == "n": + excluded_tables.append(table_name) + else: + # If there's a table list, use it directly + for i, table in enumerate(tables_list, 1): + table_name = table["name"] + choice = input(f"{i}. {table_name} (accessible? y/n): ").strip().lower() + if choice == "n": + excluded_tables.append(table_name) + + print_colored(f"\n{len(excluded_tables)} tables/collections have been excluded", "green") + return excluded_tables + +def save_configuration(sso_token: str, api_key: str, db_config: Dict[str, Any], api_url: Optional[str] = None) -> bool: + """ + Saves the configuration locally and syncs it with the API server. + + Args: + sso_token: SSO authentication token + api_key: API Key to identify the configuration + db_config: Database configuration + api_url: Optional API URL + + Returns: + True if saved correctly, False otherwise + """ + config_id = db_config.get("config_id") + if not config_id: + config_id = str(uuid.uuid4()) + db_config["config_id"] = config_id + + print_colored(f"\nSaving configuration with ID: {config_id}...", "blue") + + try: + config_manager = ConfigManager() + config_manager.add_config(api_key, db_config, config_id) + + # 2. Verify that the configuration was saved locally + saved_config = config_manager.get_config(api_key, config_id) + if not saved_config: + print_colored("⚠️ Could not verify local saving of configuration", "yellow") + else: + print_colored("✅ Configuration saved locally successfully", "green") + + # 3. Try to sync with the server + try: + if api_url: + print_colored("Syncing configuration with server...", "blue") + + # Prepare URL + if not api_url.startswith(("http://", "https://")): + api_url = "https://" + api_url + + if api_url.endswith('/'): + api_url = api_url[:-1] + + # Endpoint to update API key + endpoint = f"{api_url}/api/auth/api-keys/{api_key}" + + # Create ApiKeyUpdate object according to your model + update_data = { + "metadata": { + "config_id": config_id, + "db_config": db_config, + "corebrain_sdk": { + "version": "1.0.0", + "updated_at": datetime.now().isoformat() + } + } + } + + print_colored(f"Updating API key with ID: {api_key}", "blue") + + # Send to server + headers = { + "Authorization": f"Bearer {sso_token}", + "Content-Type": "application/json" + } + + response = http_session.put( + endpoint, + headers=headers, + json=update_data, + timeout=5.0 + ) + + if response.status_code in [200, 201, 204]: + print_colored("✅ Configuration successfully synced with server", "green") + else: + print_colored(f"⚠️ Error syncing with server (Code: {response.status_code})", "yellow") + print_colored(f"Response: {response.text[:200]}...", "yellow") + + except Exception as e: + print_colored(f"⚠️ Error syncing with server: {str(e)}", "yellow") + print_colored("The configuration is still saved locally", "green") + + return True + + except Exception as e: + print_colored(f"❌ Error saving configuration: {str(e)}", "red") + return False + +def configure_sdk(api_token: str, api_key: str, api_url: Optional[str] = None, sso_url: Optional[str] = None, user_data: Optional[Dict[str, Any]] = None) -> None: + """ + Configures the Corebrain SDK with a step-by-step wizard. + + Args: + api_token: API token for general authentication (obtained from SSO) + api_key: Specific API key selected to use with the SDK + api_url: Corebrain API URL + sso_url: Globodain SSO service URL + user_data: User data obtained from SSO + """ + # Ensure default values for URLs + api_url = api_url or DEFAULT_API_URL + sso_url = sso_url or DEFAULT_SSO_URL + + print_colored("\n=== COREBRAIN SDK CONFIGURATION WIZARD ===", "bold") + + # PHASE 1-3: Already completed - User authentication + + # PHASE 4: Select database type + print_colored("\n2. Selecting database type...", "blue") + db_type = get_db_type() + + # PHASE 4: Select database engine + print_colored("\n3. Selecting database engine...", "blue") + engine = get_db_engine(db_type) + + # PHASE 5: Configure connection parameters + print_colored("\n4. Configuring connection parameters...", "blue") + db_config = get_connection_params(db_type, engine) + + # PHASE 5: Verify database connection + print_colored("\n5. Verifying database connection...", "blue") + if not test_database_connection(api_key, db_config, api_url, user_data): + print_colored("❌ Configuration not completed due to connection errors.", "red") + return + + # PHASE 6: Define non-accessible tables/collections + print_colored("\n6. Defining non-accessible tables/collections...", "blue") + excluded_tables = select_excluded_tables(api_key, db_config, api_url, user_data) + db_config["excluded_tables"] = excluded_tables + + # PHASE 7: Save configuration + print_colored("\n7. Saving configuration...", "blue") + config_id = db_config["config_id"] + + # Save the configuration + if not save_configuration(api_token, api_key, db_config, api_url): + print_colored("❌ Error saving configuration.", "red") + return + + """ # * --> Deactivated + # PHASE 8: Test natural language query (optional depending on API status) + try: + print_colored("\n8. Testing natural language query...", "blue") + test_natural_language_query(api_key, db_config, api_url, user_data) + except Exception as e: + print_colored(f"⚠️ Could not perform the query test: {str(e)}", "yellow") + print_colored("This does not affect the saved configuration.", "yellow") + """ + + # Final message + print_colored("\n✅ Configuration completed successfully!", "green") + print_colored(f"\nYou can use this SDK in your code with:", "blue") + print(f""" + from corebrain import init + + # Initialize the SDK with API key and configuration ID + corebrain = init( + api_key="{api_key}", + config_id="{config_id}" + ) + + # Perform natural language queries + result = corebrain.ask("Your question in natural language") + print(result["explanation"]) + """ + ) \ No newline at end of file diff --git a/corebrain/cli/utils.py b/corebrain/cli/utils.py new file mode 100644 index 0000000..6c0ccac --- /dev/null +++ b/corebrain/cli/utils.py @@ -0,0 +1,595 @@ +""" +Utilities for the Corebrain CLI. + +This module provides utility functions and classes for the +Corebrain command-line interface. +""" +import sys +import time +import socket +import random +import logging +import threading +import socketserver + +from typing import Optional, Dict, Any, List, Union +from pathlib import Path + +from corebrain.cli.common import DEFAULT_PORT, DEFAULT_TIMEOUT + +logger = logging.getLogger(__name__) + +# Terminal color definitions +COLORS = { + "default": "\033[0m", + "bold": "\033[1m", + "green": "\033[92m", + "red": "\033[91m", + "yellow": "\033[93m", + "blue": "\033[94m", + "magenta": "\033[95m", + "cyan": "\033[96m", + "white": "\033[97m", + "black": "\033[30m", + "bg_green": "\033[42m", + "bg_red": "\033[41m", + "bg_yellow": "\033[43m", + "bg_blue": "\033[44m", +} + +def print_colored(text: str, color: str = "default", return_str: bool = False) -> Optional[str]: + """ + Prints colored text in the terminal or returns the colored text. + + Args: + text: Text to color + color: Color to use (default, green, red, yellow, blue, bold, etc.) + return_str: If True, returns the colored text instead of printing it + + Returns: + If return_str is True, returns the colored text, otherwise None + """ + try: + # Get color code + start_color = COLORS.get(color, COLORS["default"]) + end_color = COLORS["default"] + + # Compose colored text + colored_text = f"{start_color}{text}{end_color}" + + # Return or print + if return_str: + return colored_text + else: + print(colored_text) + return None + except Exception as e: + # If there's an error with colors (e.g., terminal that doesn't support them) + logger.debug(f"Error using colors: {e}") + if return_str: + return text + else: + print(text) + return None + +def format_table(data: List[Dict[str, Any]], columns: Optional[List[str]] = None, + max_width: int = 80) -> str: + """ + Formats data as a text table for display in the terminal. + + Args: + data: List of dictionaries with the data + columns: List of columns to display (if None, uses all columns) + max_width: Maximum width of the table + + Returns: + Table formatted as text + """ + if not data: + return "No data to display" + + # Determine columns to display + if not columns: + # Use all columns from the first element + columns = list(data[0].keys()) + + # Get the maximum width for each column + widths = {col: len(col) for col in columns} + for row in data: + for col in columns: + if col in row: + val = str(row[col]) + widths[col] = max(widths[col], min(len(val), 30)) # Limit to 30 characters + + # Adjust widths if they exceed the maximum + total_width = sum(widths.values()) + (3 * len(columns)) - 1 + if total_width > max_width: + # Reduce proportionally + ratio = max_width / total_width + for col in widths: + widths[col] = max(8, int(widths[col] * ratio)) + + # Header + header = " | ".join(col.ljust(widths[col]) for col in columns) + separator = "-+-".join("-" * widths[col] for col in columns) + + # Rows + rows = [] + for row in data: + row_str = " | ".join( + str(row.get(col, "")).ljust(widths[col])[:widths[col]] + for col in columns + ) + rows.append(row_str) + + # Compose table + return "\n".join([header, separator] + rows) + +def get_free_port(start_port: int = DEFAULT_PORT) -> int: + """ + Finds an available port, starting with the suggested port. + + Args: + start_port: Initial port to try + + Returns: + Available port + """ + try: + # Try with the suggested port first + with socketserver.TCPServer(("", start_port), None) as _: + return start_port # The port is available + except OSError: + # If the suggested port is busy, look for a free one + for _ in range(10): # Try 10 times + # Choose a random port between 8000 and 9000 + port = random.randint(8000, 9000) + try: + with socketserver.TCPServer(("", port), None) as _: + return port # Port available + except OSError: + continue # Try with another port + + # If we can't find a free port, use a default high one + return 10000 + random.randint(0, 1000) + +def is_port_in_use(port: int) -> bool: + """ + Checks if a port is in use. + + Args: + port: Port number to check + + Returns: + True if the port is in use + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + return s.connect_ex(('localhost', port)) == 0 + +def is_interactive() -> bool: + """ + Determines if the current session is interactive. + + Returns: + True if the session is interactive + """ + return sys.stdin.isatty() and sys.stdout.isatty() + +def confirm_action(message: str, default: bool = False) -> bool: + """ + Asks the user for confirmation of an action. + + Args: + message: Confirmation message + default: Default value if the user just presses Enter + + Returns: + True if the user confirms, False otherwise + """ + if not is_interactive(): + return default + + default_text = "Y/n" if default else "y/N" + response = input(f"{message} ({default_text}): ").strip().lower() + + if not response: + return default + + return response.startswith('y') + +def get_input_with_default(prompt: str, default: Optional[str] = None) -> str: + """ + Requests input from the user with a default value. + + Args: + prompt: Request message + default: Default value + + Returns: + Value entered by the user or default value + """ + if default: + full_prompt = f"{prompt} (default: {default}): " + else: + full_prompt = f"{prompt}: " + + response = input(full_prompt).strip() + + return response if response else (default or "") + +def get_password_input(prompt: str = "Password") -> str: + """ + Requests a password from the user without displaying it on screen. + + Args: + prompt: Request message + + Returns: + Password entered + """ + import getpass + return getpass.getpass(f"{prompt}: ") + +def truncate_text(text: str, max_length: int = 100, suffix: str = "...") -> str: + """ + Truncates text if it exceeds the maximum length. + + Args: + text: Text to truncate + max_length: Maximum length + suffix: Suffix to add if the text is truncated + + Returns: + Truncated text if necessary + """ + if not text or len(text) <= max_length: + return text + + return text[:max_length - len(suffix)] + suffix + +def ensure_dir(path: Union[str, Path]) -> Path: + """ + Ensures that a directory exists, creating it if necessary. + + Args: + path: Directory path + + Returns: + Path object of the directory + """ + path_obj = Path(path) + path_obj.mkdir(parents=True, exist_ok=True) + return path_obj + +class ProgressTracker: + """ + Displays progress of CLI operations with colors and timing. + """ + + def __init__(self, verbose: bool = False, spinner: bool = True): + """ + Initializes the progress tracker. + + Args: + verbose: Whether to show detailed information + spinner: Whether to show an animated spinner + """ + self.verbose = verbose + self.use_spinner = spinner and is_interactive() + self.start_time = None + self.current_task = None + self.total = None + self.steps = 0 + self.spinner_thread = None + self.stop_spinner = threading.Event() + self.last_update_time = 0 + self.update_interval = 0.2 # Seconds between updates + + def _run_spinner(self): + """Displays an animated spinner in the console.""" + spinner_chars = "|/-\\" + idx = 0 + + while not self.stop_spinner.is_set(): + if self.current_task: + elapsed = time.time() - self.start_time + status = f"{self.steps}/{self.total}" if self.total else f"step {self.steps}" + sys.stdout.write(f"\r{COLORS['blue']}[{spinner_chars[idx]}] {self.current_task} ({status}, {elapsed:.1f}s){COLORS['default']} ") + sys.stdout.flush() + idx = (idx + 1) % len(spinner_chars) + time.sleep(0.1) + + def start(self, task: str, total: Optional[int] = None) -> None: + """ + Starts tracking a task. + + Args: + task: Task description + total: Total number of steps (optional) + """ + self.reset() # Ensure there's no previous task + + self.current_task = task + self.total = total + self.start_time = time.time() + self.steps = 0 + self.last_update_time = self.start_time + + # Show initial message + print_colored(f"▶ {task}...", "blue") + + # Start spinner if enabled + if self.use_spinner: + self.stop_spinner.clear() + self.spinner_thread = threading.Thread(target=self._run_spinner) + self.spinner_thread.daemon = True + self.spinner_thread.start() + + def update(self, message: Optional[str] = None, increment: int = 1) -> None: + """ + Updates progress with optional message. + + Args: + message: Progress message + increment: Step increment + """ + if not self.start_time: + return # No active task + + self.steps += increment + current_time = time.time() + + # Limit update frequency to avoid saturating the output + if (current_time - self.last_update_time < self.update_interval) and not message: + return + + self.last_update_time = current_time + + # If there's an active spinner, temporarily stop it to show the message + if self.use_spinner and self.spinner_thread and self.spinner_thread.is_alive(): + sys.stdout.write("\r" + " " * 80 + "\r") # Clear current line + sys.stdout.flush() + + if message or self.verbose: + elapsed = current_time - self.start_time + status = f"{self.steps}/{self.total}" if self.total else f"step {self.steps}" + + if message: + print_colored(f" • {message} ({status}, {elapsed:.1f}s)", "blue") + elif self.verbose: + print_colored(f" • Progress: {status}, {elapsed:.1f}s", "blue") + + def finish(self, message: Optional[str] = None) -> None: + """ + Finishes a task with success message. + + Args: + message: Final message + """ + if not self.start_time: + return # No active task + + # Stop spinner if active + self._stop_spinner() + + elapsed = time.time() - self.start_time + msg = message or f"{self.current_task} completed" + print_colored(f"✅ {msg} in {elapsed:.2f}s", "green") + + self.reset() + + def fail(self, message: Optional[str] = None) -> None: + """ + Marks a task as failed. + + Args: + message: Error message + """ + if not self.start_time: + return # No active task + + # Stop spinner if active + self._stop_spinner() + + elapsed = time.time() - self.start_time + msg = message or f"{self.current_task} failed" + print_colored(f"❌ {msg} after {elapsed:.2f}s", "red") + + self.reset() + + def _stop_spinner(self) -> None: + """Stops the spinner if active.""" + if self.use_spinner and self.spinner_thread and self.spinner_thread.is_alive(): + self.stop_spinner.set() + self.spinner_thread.join(timeout=0.5) + + # Clear spinner line + sys.stdout.write("\r" + " " * 80 + "\r") + sys.stdout.flush() + + def reset(self) -> None: + """Resets the tracker.""" + self._stop_spinner() + self.start_time = None + self.current_task = None + self.total = None + self.steps = 0 + self.spinner_thread = None + +class CliConfig: + """ + Manages the CLI configuration. + """ + + def __init__(self, config_dir: Optional[Union[str, Path]] = None): + """ + Initializes the CLI configuration. + + Args: + config_dir: Directory for configuration files + """ + if config_dir: + self.config_dir = Path(config_dir) + else: + self.config_dir = Path.home() / ".corebrain" / "cli" + + self.config_file = self.config_dir / "config.json" + self.config = self._load_config() + + def _load_config(self) -> Dict[str, Any]: + """ + Loads configuration from file. + + Returns: + Loaded configuration + """ + if not self.config_file.exists(): + return self._create_default_config() + + try: + import json + with open(self.config_file, 'r') as f: + return json.load(f) + except Exception as e: + logger.warning(f"Error loading configuration: {e}") + return self._create_default_config() + + def _create_default_config(self) -> Dict[str, Any]: + """ + Creates a default configuration. + + Returns: + Default configuration + """ + from corebrain.cli.common import DEFAULT_API_URL, DEFAULT_SSO_URL + + config = { + "api_url": DEFAULT_API_URL, + "sso_url": DEFAULT_SSO_URL, + "verbose": False, + "timeout": DEFAULT_TIMEOUT, + "last_used": { + "api_key": None, + "config_id": None + }, + "ui": { + "use_colors": True, + "use_spinner": True, + "verbose": False + } + } + + # Ensure the directory exists + ensure_dir(self.config_dir) + + # Save default configuration + try: + import json + with open(self.config_file, 'w') as f: + json.dump(config, f, indent=2) + except Exception as e: + logger.warning(f"Error saving configuration: {e}") + + return config + + def save(self) -> bool: + """ + Saves current configuration. + + Returns: + True if saved correctly + """ + try: + # Ensure the directory exists + ensure_dir(self.config_dir) + + import json + with open(self.config_file, 'w') as f: + json.dump(self.config, f, indent=2) + return True + except Exception as e: + logger.error(f"Error saving configuration: {e}") + return False + + def get(self, key: str, default: Any = None) -> Any: + """ + Gets a configuration value. + + Args: + key: Configuration key + default: Default value + + Returns: + Configuration value + """ + # Support for nested keys with dots + if "." in key: + parts = key.split(".") + current = self.config + for part in parts: + if part not in current: + return default + current = current[part] + return current + + return self.config.get(key, default) + + def set(self, key: str, value: Any) -> bool: + """ + Sets a configuration value. + + Args: + key: Configuration key + value: Value to set + + Returns: + True if set correctly + """ + # Support for nested keys with dots + if "." in key: + parts = key.split(".") + current = self.config + for part in parts[:-1]: + if part not in current: + current[part] = {} + current = current[part] + current[parts[-1]] = value + else: + self.config[key] = value + + return self.save() + + def update(self, config_dict: Dict[str, Any]) -> bool: + """ + Updates configuration with a dictionary. + + Args: + config_dict: Configuration dictionary + + Returns: + True if updated correctly + """ + self.config.update(config_dict) + return self.save() + + def update_last_used(self, api_key: Optional[str] = None, config_id: Optional[str] = None) -> bool: + """ + Updates the last used configuration. + + Args: + api_key: API key used + config_id: Configuration ID used + + Returns: + True if updated correctly + """ + if not self.config.get("last_used"): + self.config["last_used"] = {} + + if api_key: + self.config["last_used"]["api_key"] = api_key + + if config_id: + self.config["last_used"]["config_id"] = config_id + + return self.save() \ No newline at end of file diff --git a/corebrain/config/__init__.py b/corebrain/config/__init__.py new file mode 100644 index 0000000..8caa1ed --- /dev/null +++ b/corebrain/config/__init__.py @@ -0,0 +1,10 @@ +""" +Gestión de configuración para Corebrain SDK. + +Este paquete proporciona funcionalidades para gestionar configuraciones +de conexión a bases de datos y preferencias del SDK. +""" +from .manager import ConfigManager + +# Exportación explícita de componentes públicos +__all__ = ['ConfigManager'] \ No newline at end of file diff --git a/corebrain/config/manager.py b/corebrain/config/manager.py new file mode 100644 index 0000000..1c0a0e5 --- /dev/null +++ b/corebrain/config/manager.py @@ -0,0 +1,191 @@ +""" +Gestor de configuraciones para Corebrain SDK. +""" + +import json +import uuid +from pathlib import Path +from typing import Dict, Any, List, Optional +from cryptography.fernet import Fernet + +from corebrain.utils.serializer import serialize_to_json +from corebrain.core.common import logger + +# Función para imprimir mensajes coloreados +def _print_colored(message: str, color: str) -> None: + """Versión simplificada de _print_colored que no depende de cli.utils""" + colors = { + "red": "\033[91m", + "green": "\033[92m", + "yellow": "\033[93m", + "blue": "\033[94m", + "default": "\033[0m" + } + color_code = colors.get(color, colors["default"]) + print(f"{color_code}{message}{colors['default']}") + +class ConfigManager: + """Gestor de configuraciones del SDK con seguridad y rendimiento mejorados""" + + CONFIG_DIR = Path.home() / ".corebrain" + CONFIG_FILE = CONFIG_DIR / "config.json" + SECRET_KEY_FILE = CONFIG_DIR / "secret.key" + + def __init__(self): + self.configs = {} + self.cipher = None + self._ensure_config_dir() + self._load_secret_key() + self._load_configs() + + def _ensure_config_dir(self) -> None: + """Asegura que existe el directorio de configuración.""" + try: + self.CONFIG_DIR.mkdir(parents=True, exist_ok=True) + logger.debug(f"Directorio de configuración asegurado: {self.CONFIG_DIR}") + _print_colored(f"Directorio de configuración asegurado: {self.CONFIG_DIR}", "blue") + except Exception as e: + logger.error(f"Error al crear directorio de configuración: {str(e)}") + _print_colored(f"Error al crear directorio de configuración: {str(e)}", "red") + + def _load_secret_key(self) -> None: + """Carga o genera la clave secreta para encriptar datos sensibles.""" + try: + if not self.SECRET_KEY_FILE.exists(): + key = Fernet.generate_key() + with open(self.SECRET_KEY_FILE, 'wb') as key_file: + key_file.write(key) + _print_colored(f"Nueva clave secreta generada en: {self.SECRET_KEY_FILE}", "green") + + with open(self.SECRET_KEY_FILE, 'rb') as key_file: + self.secret_key = key_file.read() + + self.cipher = Fernet(self.secret_key) + except Exception as e: + _print_colored(f"Error al cargar/generar clave secreta: {str(e)}", "red") + # Fallback a una clave temporal (menos segura pero funcional) + self.secret_key = Fernet.generate_key() + self.cipher = Fernet(self.secret_key) + + def _load_configs(self) -> Dict[str, Dict[str, Any]]: + """Carga las configuraciones guardadas.""" + if not self.CONFIG_FILE.exists(): + _print_colored(f"Archivo de configuración no encontrado: {self.CONFIG_FILE}", "yellow") + return {} + + try: + with open(self.CONFIG_FILE, 'r') as f: + encrypted_data = f.read() + + if not encrypted_data: + _print_colored("Archivo de configuración vacío", "yellow") + return {} + + try: + # Intentar descifrar los datos + decrypted_data = self.cipher.decrypt(encrypted_data.encode()).decode() + configs = json.loads(decrypted_data) + except Exception as e: + # Si falla el descifrado, intentar cargar como JSON plano + logger.warning(f"Error al descifrar configuración: {e}") + configs = json.loads(encrypted_data) + + if isinstance(configs, str): + configs = json.loads(configs) + + _print_colored(f"Configuración cargada", "green") + self.configs = configs + return configs + except Exception as e: + _print_colored(f"Error al cargar configuraciones: {str(e)}", "red") + return {} + + def _save_configs(self) -> None: + """Guarda las configuraciones actuales.""" + try: + configs_json = serialize_to_json(self.configs) + encrypted_data = self.cipher.encrypt(json.dumps(configs_json).encode()).decode() + + with open(self.CONFIG_FILE, 'w') as f: + f.write(encrypted_data) + + _print_colored(f"Configuraciones guardadas en: {self.CONFIG_FILE}", "green") + except Exception as e: + _print_colored(f"Error al guardar configuraciones: {str(e)}", "red") + + def add_config(self, api_key: str, db_config: Dict[str, Any], config_id: Optional[str] = None) -> str: + """ + Añade una nueva configuración. + + Args: + api_key: API Key seleccionada + db_config: Configuración de la base de datos + config_id: ID opcional para la configuración (se genera uno si no se proporciona) + + Returns: + ID de la configuración + """ + if not config_id: + config_id = str(uuid.uuid4()) + db_config["config_id"] = config_id + + # Crear o actualizar la entrada para este token + if api_key not in self.configs: + self.configs[api_key] = {} + + # Añadir la configuración + self.configs[api_key][config_id] = db_config + self._save_configs() + + _print_colored(f"Configuración agregada: {config_id} para la API Key: {api_key[:8]}...", "green") + return config_id + + def get_config(self, api_key_selected: str, config_id: str) -> Optional[Dict[str, Any]]: + """ + Obtiene una configuración específica. + + Args: + api_key_selected: API Key seleccionada + config_id: ID de la configuración + + Returns: + Configuración o None si no existe + """ + return self.configs.get(api_key_selected, {}).get(config_id) + + def list_configs(self, api_key_selected: str) -> List[str]: + """ + Lista los IDs de configuración disponibles para una API Key. + + Args: + api_key_selected: API Key seleccionada + + Returns: + Lista de IDs de configuración + """ + return list(self.configs.get(api_key_selected, {}).keys()) + + def remove_config(self, api_key_selected: str, config_id: str) -> bool: + """ + Elimina una configuración. + + Args: + api_key_selected: API Key seleccionada + config_id: ID de la configuración + + Returns: + True si se eliminó correctamente, False en caso contrario + """ + if api_key_selected in self.configs and config_id in self.configs[api_key_selected]: + del self.configs[api_key_selected][config_id] + + # Si no quedan configuraciones para este token, eliminar la entrada + if not self.configs[api_key_selected]: + del self.configs[api_key_selected] + + self._save_configs() + _print_colored(f"Configuración {config_id} eliminada para API Key: {api_key_selected[:8]}...", "green") + return True + + _print_colored(f"Configuración {config_id} no encontrada para API Key: {api_key_selected[:8]}...", "yellow") + return False \ No newline at end of file diff --git a/corebrain/core/__init__.py b/corebrain/core/__init__.py new file mode 100644 index 0000000..b345d11 --- /dev/null +++ b/corebrain/core/__init__.py @@ -0,0 +1,20 @@ +""" +Componentes principales del SDK de Corebrain. + +Este paquete contiene los componentes centrales del SDK, +incluyendo el cliente principal y el manejo de schemas. +""" +from corebrain.core.client import Corebrain, init +from corebrain.core.query import QueryCache, QueryAnalyzer, QueryTemplate +from corebrain.core.test_utils import test_natural_language_query, generate_test_question_from_schema + +# Exportación explícita de componentes públicos +__all__ = [ + 'Corebrain', + 'init', + 'QueryCache', + 'QueryAnalyzer', + 'QueryTemplate', + 'test_natural_language_query', + 'generate_test_question_from_schema' +] \ No newline at end of file diff --git a/corebrain/core/client.py b/corebrain/core/client.py new file mode 100644 index 0000000..61e4bdb --- /dev/null +++ b/corebrain/core/client.py @@ -0,0 +1,1364 @@ +""" +Corebrain SDK Main Client. + +This module provides the main interface to interact with the Corebrain API +and enables natural language queries to relational and non-relational databases. +""" +import uuid +import re +import logging +import requests +import httpx +import sqlite3 +import mysql.connector +import psycopg2 +import pymongo +import json +from typing import Dict, Any, List, Optional +from sqlalchemy import create_engine, inspect +from pathlib import Path +from datetime import datetime + +from corebrain.core.common import logger, CorebrainError + +class Corebrain: + """ + Main client for the Corebrain SDK for natural language database queries. + + This class provides a unified interface to interact with different types of databases + (SQL and NoSQL) using natural language. It manages the connection, schema extraction, + and query processing through the Corebrain API. + + Attributes: + api_key (str): Authentication key for the Corebrain API. + db_config (Dict[str, Any]): Database connection configuration. + config_id (str): Unique identifier for the current configuration. + api_url (str): Base URL for the Corebrain API. + user_info (Dict[str, Any]): Information about the authenticated user. + db_connection: Active database connection. + db_schema (Dict[str, Any]): Extracted database schema. + + Examples: + SQLite initialization: + ```python + from corebrain import init + + # Connect to a SQLite database + client = init( + api_key="your_api_key", + db_config={ + "type": "sql", + "engine": "sqlite", + "database": "my_database.db" + } + ) + + # Make a query + result = client.ask("How many registered users are there?") + print(result["explanation"]) + ``` + + PostgreSQL initialization: + ```python + # Connect to PostgreSQL + client = init( + api_key="your_api_key", + db_config={ + "type": "sql", + "engine": "postgresql", + "host": "localhost", + "port": 5432, + "user": "postgres", + "password": "your_password", + "database": "my_database" + } + ) + ``` + + MongoDB initialization: + ```python + # Connect to MongoDB + client = init( + api_key="your_api_key", + db_config={ + "type": "mongodb", + "host": "localhost", + "port": 27017, + "database": "my_database" + } + ) + ``` + """ + + def __init__( + self, + api_key: str, + db_config: Optional[Dict[str, Any]] = None, + config_id: Optional[str] = None, + user_data: Optional[Dict[str, Any]] = None, + api_url: str = "http://localhost:5000", + skip_verification: bool = False + ): + """ + Initialize the Corebrain SDK client. + + Args: + api_key (str): Required API key for authentication with the Corebrain service. + Can be generated from the dashboard at https://dashboard.corebrain.com. + + db_config (Dict[str, Any], optional): Database configuration to query. + This parameter is required if config_id is not provided. Must contain at least: + - "type": Database type ("sql" or "mongodb") + - For SQL: "engine" ("sqlite", "postgresql", "mysql") + - Specific connection parameters depending on type and engine + + Example for SQLite: + ``` + { + "type": "sql", + "engine": "sqlite", + "database": "path/to/database.db" + } + ``` + + Example for PostgreSQL: + ``` + { + "type": "sql", + "engine": "postgresql", + "host": "localhost", + "port": 5432, + "user": "postgres", + "password": "password", + "database": "db_name" + } + ``` + + config_id (str, optional): Identifier for a previously saved configuration. + If provided, this configuration will be used instead of db_config. + Useful for maintaining persistent configurations between sessions. + + user_data (Dict[str, Any], optional): Additional user information for verification. + Can contain data like "email" for more precise token validation. + + api_url (str, optional): Base URL for the Corebrain API. + Defaults to "http://localhost:5000" for local development. + In production, it is typically "https://api.corebrain.com". + + skip_verification (bool, optional): If True, skips token verification with the server. + Useful in offline environments or for local testing. + Defaults to False. + + Raises: + ValueError: If required parameters are missing or if the configuration is invalid. + CorebrainError: If there are issues with the API connection or database. + + Example: + ```python + from corebrain import Corebrain + + # Basic initialization with SQLite + client = Corebrain( + api_key="your_api_key", + db_config={ + "type": "sql", + "engine": "sqlite", + "database": "my_db.db" + } + ) + ``` + """ + self.api_key = api_key + self.user_data = user_data + self.api_url = api_url.rstrip('/') + self.db_connection = None + self.db_schema = None + + # Import ConfigManager dynamically to avoid circular dependency + try: + from corebrain.config.manager import ConfigManager + self.config_manager = ConfigManager() + except ImportError as e: + logger.error(f"Error importing ConfigManager: {e}") + raise CorebrainError(f"Could not load configuration manager: {e}") + + # Determine which configuration to use + if config_id: + saved_config = self.config_manager.get_config(api_key, config_id) + if not saved_config: + # Try to load from old format + old_config = self._load_old_config(api_key, config_id) + if old_config: + self.db_config = old_config + self.config_id = config_id + # Save in new format + self.config_manager.add_config(api_key, old_config, config_id) + else: + raise ValueError(f"Configuration with ID {config_id} not found for the provided key") + else: + self.db_config = saved_config + self.config_id = config_id + elif db_config: + self.db_config = db_config + + # Generate config ID if it doesn't exist + if "config_id" in db_config: + self.config_id = db_config["config_id"] + else: + self.config_id = str(uuid.uuid4()) + db_config["config_id"] = self.config_id + + # Save the configuration + self.config_manager.add_config(api_key, db_config, self.config_id) + else: + raise ValueError("db_config or config_id must be provided") + + # Validate configuration + self._validate_config() + + # Verify the API token (only if necessary) + if not skip_verification: + self._verify_api_token() + else: + # Initialize user_info with basic information if not verifying + self.user_info = {"token": api_key} + + # Connect to the database + self._connect_to_database() + + # Extract database schema + self.db_schema = self._extract_db_schema() + + self.metadata = { + "config_id": self.config_id, + "api_key": api_key, + "db_config": self.db_config + } + + def _load_old_config(self, api_key: str, config_id: str) -> Optional[Dict[str, Any]]: + """ + Try to load configuration from old format. + + Args: + api_key: API key + config_id: Configuration ID + + Returns: + Configuration dictionary if found, None otherwise + """ + try: + # Try to load from old config file + old_config_path = Path.home() / ".corebrain" / "config.json" + if old_config_path.exists(): + with open(old_config_path, 'r') as f: + old_configs = json.load(f) + if api_key in old_configs and config_id in old_configs[api_key]: + return old_configs[api_key][config_id] + except Exception as e: + logger.warning(f"Error loading old config: {e}") + return None + + def _validate_config(self) -> None: + """ + Validate the provided configuration. + + This internal function verifies that the database configuration + contains all necessary fields according to the specified database type. + + Raises: + ValueError: If the database configuration is invalid or incomplete. + """ + if not self.api_key: + raise ValueError("API key is required. Generate one at dashboard.corebrain.com") + + if not self.db_config: + raise ValueError("Database configuration is required") + + if "type" not in self.db_config: + raise ValueError("Database type is required in db_config") + + if "connection_string" not in self.db_config and self.db_config["type"] != "sqlite_memory": + if self.db_config["type"] == "sql": + if "engine" not in self.db_config: + raise ValueError("Database engine is required for 'sql' type") + + # Verify alternative configuration for SQL engines + if self.db_config["engine"] == "mysql" or self.db_config["engine"] == "postgresql": + if not ("host" in self.db_config and "user" in self.db_config and + "password" in self.db_config and "database" in self.db_config): + raise ValueError("host, user, password, and database are required for MySQL/PostgreSQL") + elif self.db_config["engine"] == "sqlite": + if "database" not in self.db_config: + raise ValueError("database field is required for SQLite") + elif self.db_config["type"] == "mongodb": + if "database" not in self.db_config: + raise ValueError("database field is required for MongoDB") + + if "connection_string" not in self.db_config: + if not ("host" in self.db_config and "port" in self.db_config): + raise ValueError("host and port are required for MongoDB without connection_string") + + def _verify_api_token(self) -> None: + """ + Verify the API token with the server. + + This internal function sends a request to the Corebrain server + to validate that the provided API token is valid. + If the user provided additional information (like email), + it will be used for more precise verification. + + The verification results are stored in self.user_info. + + Raises: + ValueError: If the API token is invalid. + """ + try: + # Use the user's email for verification if available + if self.user_data and 'email' in self.user_data: + endpoint = f"{self.api_url}/api/auth/users/{self.user_data['email']}" + + response = httpx.get( + endpoint, + headers={"X-API-Key": self.api_key}, + timeout=10.0 + ) + + if response.status_code != 200: + raise ValueError(f"Invalid API token. Error code: {response.status_code}") + + # Store user information + self.user_info = response.json() + else: + # If no email, do a simple verification with a generic endpoint + endpoint = f"{self.api_url}/api/auth/verify" + + try: + response = httpx.get( + endpoint, + headers={"X-API-Key": self.api_key}, + timeout=5.0 + ) + + if response.status_code == 200: + self.user_info = response.json() + else: + # If it fails, just store basic information + self.user_info = {"token": self.api_key} + except Exception as e: + # If there's a connection error, don't fail, just store basic info + logger.warning(f"Could not verify token: {str(e)}") + self.user_info = {"token": self.api_key} + + except httpx.RequestError as e: + # Connection error shouldn't be fatal if we already have a configuration + logger.warning(f"Error connecting to API: {str(e)}") + self.user_info = {"token": self.api_key} + except Exception as e: + # Other errors are logged but not fatal + logger.warning(f"Error in token verification: {str(e)}") + self.user_info = {"token": self.api_key} + + def _connect_to_database(self) -> None: + """ + Establish a connection to the database according to the configuration. + + This internal function creates a database connection using the parameters + defined in self.db_config. It supports various database types: + - SQLite (file or in-memory) + - PostgreSQL + - MySQL + - MongoDB + + The connection is stored in self.db_connection for later use. + + Raises: + CorebrainError: If the connection to the database cannot be established. + NotImplementedError: If the database type is not supported. + """ + db_type = self.db_config["type"].lower() + + try: + if db_type == "sql": + engine = self.db_config.get("engine", "").lower() + + if engine == "sqlite": + database = self.db_config.get("database", "") + if database: + self.db_connection = sqlite3.connect(database) + else: + self.db_connection = sqlite3.connect(self.db_config.get("connection_string", "")) + + elif engine == "mysql": + if "connection_string" in self.db_config: + self.db_connection = mysql.connector.connect( + connection_string=self.db_config["connection_string"] + ) + else: + self.db_connection = mysql.connector.connect( + host=self.db_config.get("host", "localhost"), + user=self.db_config.get("user", ""), + password=self.db_config.get("password", ""), + database=self.db_config.get("database", ""), + port=self.db_config.get("port", 3306) + ) + + elif engine == "postgresql": + if "connection_string" in self.db_config: + self.db_connection = psycopg2.connect(self.db_config["connection_string"]) + else: + self.db_connection = psycopg2.connect( + host=self.db_config.get("host", "localhost"), + user=self.db_config.get("user", ""), + password=self.db_config.get("password", ""), + dbname=self.db_config.get("database", ""), + port=self.db_config.get("port", 5432) + ) + + else: + # Use SQLAlchemy for other engines + self.db_connection = create_engine(self.db_config["connection_string"]) + + # Improved code for MongoDB + elif db_type == "nosql" or db_type == "mongodb": + # If engine is mongodb or the type is directly mongodb + engine = self.db_config.get("engine", "").lower() + if not engine or engine == "mongodb": + # Create connection parameters + mongo_params = {} + + if "connection_string" in self.db_config: + # Save the MongoDB client to be able to close it correctly later + self.mongo_client = pymongo.MongoClient(self.db_config["connection_string"]) + else: + # Configure host and port + mongo_params["host"] = self.db_config.get("host", "localhost") + if "port" in self.db_config: + mongo_params["port"] = self.db_config.get("port") + + # Add credentials if available + if "user" in self.db_config and self.db_config["user"]: + mongo_params["username"] = self.db_config["user"] + if "password" in self.db_config and self.db_config["password"]: + mongo_params["password"] = self.db_config["password"] + + # Create MongoDB client + self.mongo_client = pymongo.MongoClient(**mongo_params) + + # Get the database + db_name = self.db_config.get("database", "") + if db_name: + # Save reference to the database + self.db_connection = self.mongo_client[db_name] + else: + # If there's no database name, use 'admin' as fallback + logger.warning("Database name not specified for MongoDB, using 'admin'") + self.db_connection = self.mongo_client["admin"] + else: + raise ValueError(f"Unsupported NoSQL database engine: {engine}") + + elif db_type == "sqlite_memory": + self.db_connection = sqlite3.connect(":memory:") + + else: + raise ValueError(f"Unsupported database type: {db_type}. Valid types: 'sql', 'nosql', 'mongodb'") + + except Exception as e: + logger.error(f"Error connecting to database: {str(e)}") + raise ConnectionError(f"Error connecting to database: {str(e)}") + + def _extract_db_schema(self, detail_level: str = "full", specific_collections: List[str] = None) -> Dict[str, Any]: + """ + Extrae la estructura de la base de datos para proporcionar contexto a la IA. + + Returns: + Dict con la estructura de la base de datos organizada por tablas/colecciones + """ + logger.info(f"Extrayendo esquema de base de datos. Tipo: {self.db_config['type']}, Motor: {self.db_config.get('engine')}") + + db_type = self.db_config["type"].lower() + schema = { + "type": db_type, + "database": self.db_config.get("database", ""), + "tables": {}, + "total_collections": 0, # Añadir contador total + "included_collections": 0 # Contador de incluidas + } + excluded_tables = set(self.db_config.get("excluded_tables", [])) + logger.info(f"Tablas excluidas: {excluded_tables}") + + try: + if db_type == "sql": + engine = self.db_config.get("engine", "").lower() + logger.info(f"Procesando base de datos SQL con motor: {engine}") + + if engine in ["sqlite", "mysql", "postgresql"]: + cursor = self.db_connection.cursor() + + if engine == "sqlite": + logger.info("Obteniendo tablas de SQLite") + # Obtener listado de tablas + cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") + tables = cursor.fetchall() + logger.info(f"Tablas encontradas en SQLite: {tables}") + + elif engine == "mysql": + logger.info("Obteniendo tablas de MySQL") + cursor.execute("SHOW TABLES;") + tables = cursor.fetchall() + logger.info(f"Tablas encontradas en MySQL: {tables}") + + elif engine == "postgresql": + logger.info("Obteniendo tablas de PostgreSQL") + cursor.execute(""" + SELECT table_name FROM information_schema.tables + WHERE table_schema = 'public'; + """) + tables = cursor.fetchall() + logger.info(f"Tablas encontradas en PostgreSQL: {tables}") + + # Procesar las tablas encontradas + for table in tables: + table_name = table[0] + logger.info(f"Procesando tabla: {table_name}") + + # Saltar tablas excluidas + if table_name in excluded_tables: + logger.info(f"Saltando tabla excluida: {table_name}") + continue + + try: + # Obtener información de columnas según el motor + if engine == "sqlite": + cursor.execute(f"PRAGMA table_info({table_name});") + elif engine == "mysql": + cursor.execute(f"DESCRIBE {table_name};") + elif engine == "postgresql": + cursor.execute(f""" + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_name = '{table_name}'; + """) + + columns = cursor.fetchall() + logger.info(f"Columnas encontradas para {table_name}: {columns}") + + # Estructura de columnas según el motor + if engine == "sqlite": + column_info = [{"name": col[1], "type": col[2]} for col in columns] + elif engine == "mysql": + column_info = [{"name": col[0], "type": col[1]} for col in columns] + elif engine == "postgresql": + column_info = [{"name": col[0], "type": col[1]} for col in columns] + + # Guardar información de la tabla + schema["tables"][table_name] = { + "columns": column_info, + "sample_data": [] # No obtenemos datos de muestra por defecto + } + + except Exception as e: + logger.error(f"Error procesando tabla {table_name}: {str(e)}") + + else: + # Usando SQLAlchemy + logger.info("Usando SQLAlchemy para obtener el esquema") + inspector = inspect(self.db_connection) + table_names = inspector.get_table_names() + logger.info(f"Tablas encontradas con SQLAlchemy: {table_names}") + + for table_name in table_names: + if table_name in excluded_tables: + logger.info(f"Saltando tabla excluida: {table_name}") + continue + + try: + columns = inspector.get_columns(table_name) + column_info = [{"name": col["name"], "type": str(col["type"])} for col in columns] + + schema["tables"][table_name] = { + "columns": column_info, + "sample_data": [] + } + except Exception as e: + logger.error(f"Error procesando tabla {table_name} con SQLAlchemy: {str(e)}") + + elif db_type in ["nosql", "mongodb"]: + logger.info("Procesando base de datos MongoDB") + if not hasattr(self, 'db_connection') or self.db_connection is None: + logger.error("La conexión a MongoDB no está disponible") + return schema + + try: + collection_names = [] + try: + collection_names = self.db_connection.list_collection_names() + schema["total_collections"] = len(collection_names) + logger.info(f"Colecciones encontradas en MongoDB: {collection_names}") + except Exception as e: + logger.error(f"Error al obtener colecciones MongoDB: {str(e)}") + return schema + + # Si solo queremos los nombres + if detail_level == "names_only": + schema["collection_names"] = collection_names + return schema + + # Procesar cada colección + for collection_name in collection_names: + if collection_name in excluded_tables: + logger.info(f"Saltando colección excluida: {collection_name}") + continue + + try: + collection = self.db_connection[collection_name] + # Obtener un documento para inferir estructura + first_doc = collection.find_one() + + if first_doc: + fields = [] + for field, value in first_doc.items(): + if field != "_id": + field_type = type(value).__name__ + fields.append({"name": field, "type": field_type}) + + schema["tables"][collection_name] = { + "fields": fields, + "doc_count": collection.estimated_document_count() + } + logger.info(f"Procesada colección {collection_name} con {len(fields)} campos") + else: + logger.info(f"Colección {collection_name} está vacía") + schema["tables"][collection_name] = { + "fields": [], + "doc_count": 0 + } + except Exception as e: + logger.error(f"Error procesando colección {collection_name}: {str(e)}") + + except Exception as e: + logger.error(f"Error general procesando MongoDB: {str(e)}") + + # Convertir el diccionario de tablas en una lista + table_list = [] + for table_name, table_info in schema["tables"].items(): + table_data = {"name": table_name} + table_data.update(table_info) + table_list.append(table_data) + + schema["tables_list"] = table_list + logger.info(f"Esquema final - Tablas encontradas: {len(schema['tables'])}") + logger.info(f"Nombres de tablas: {list(schema['tables'].keys())}") + + return schema + + except Exception as e: + logger.error(f"Error al extraer el esquema de la base de datos: {str(e)}") + return {"type": db_type, "tables": {}, "tables_list": []} + + def list_collections_name(self) -> List[str]: + """ + Devuelve una lista de las colecciones o tablas disponibles en la base de datos. + + Returns: + Lista de collections o tablas + """ + print("Excluded tables: ", self.db_schema.get("excluded_tables", [])) + return self.db_schema.get("tables", []) + + def ask(self, question: str, **kwargs) -> Dict: + """ + Realizar una consulta en lenguaje natural a la base de datos. + + Args: + question: La pregunta en lenguaje natural + **kwargs: Parámetros adicionales: + - collection_name: Para MongoDB, la colección a consultar + - limit: Número máximo de resultados + - detail_level: Nivel de detalle del esquema ("names_only", "structure", "full") + - auto_select: Si seleccionar automáticamente colecciones + - max_collections: Número máximo de colecciones a incluir + - execute_query: Si ejecutar la consulta (True por defecto) + - explain_results: Si generar explicación de resultados (True por defecto) + + Returns: + Dict con los resultados de la consulta y la explicación + """ + try: + # Verificar opciones de comportamiento + execute_query = kwargs.get("execute_query", True) + explain_results = kwargs.get("explain_results", True) + + # Obtener esquema con el nivel de detalle apropiado + detail_level = kwargs.get("detail_level", "full") + schema = self._extract_db_schema(detail_level=detail_level) + + # Validar que el esquema tiene tablas/colecciones + if not schema.get("tables"): + print("Error: No se encontraron tablas/colecciones en la base de datos") + return {"error": True, "explanation": "No se encontraron tablas/colecciones en la base de datos"} + + # Obtener nombres de tablas disponibles para validación + available_tables = set() + if isinstance(schema.get("tables"), dict): + available_tables.update(schema["tables"].keys()) + elif isinstance(schema.get("tables_list"), list): + available_tables.update(table["name"] for table in schema["tables_list"]) + + # Preparar datos de la solicitud con información de esquema mejorada + request_data = { + "question": question, + "db_schema": schema, + "config_id": self.config_id, + "metadata": { + "type": self.db_config["type"].lower(), + "engine": self.db_config.get("engine", "").lower(), + "database": self.db_config.get("database", ""), + "available_tables": list(available_tables), + "collections": list(available_tables) + } + } + + # Añadir configuración de la base de datos al request + # Esto permite a la API ejecutar directamente las consultas si es necesario + if execute_query: + request_data["db_config"] = self.db_config + + # Añadir datos de usuario si están disponibles + if self.user_data: + request_data["user_data"] = self.user_data + + # Preparar headers para la solicitud + headers = { + "X-API-Key": self.api_key, + "Content-Type": "application/json" + } + + # Determinar el endpoint adecuado según el modo de ejecución + if execute_query: + # Usar el endpoint de ejecución completa + endpoint = f"{self.api_url}/api/database/sdk/query" + else: + # Usar el endpoint de solo generación de consulta + endpoint = f"{self.api_url}/api/database/generate" + + # Realizar solicitud a la API + response = httpx.post( + endpoint, + headers=headers, + content=json.dumps(request_data, default=str), + timeout=60.0 + ) + + # Verificar respuesta + if response.status_code != 200: + error_msg = f"Error {response.status_code} al realizar la consulta" + try: + error_data = response.json() + if isinstance(error_data, dict): + error_msg += f": {error_data.get('detail', error_data.get('message', response.text))}" + except: + error_msg += f": {response.text}" + return {"error": True, "explanation": error_msg} + + # Procesar respuesta de la API + api_response = response.json() + + # Verificar si la API reportó un error + if api_response.get("error", False): + return api_response + + # Verificar si se generó una consulta válida + if "query" not in api_response: + return { + "error": True, + "explanation": "La API no generó una consulta válida." + } + + # Si se debe ejecutar la consulta pero la API no lo hizo + # (esto ocurriría solo en caso de cambios de configuración o fallbacks) + if execute_query and "result" not in api_response: + try: + # Preparar la consulta para ejecución local + query_type = self.db_config.get("engine", "").lower() if self.db_config["type"].lower() == "sql" else self.db_config["type"].lower() + query_value = api_response["query"] + + # Para SQL, asegurarse de que la consulta es un string + if query_type in ["sqlite", "mysql", "postgresql"]: + if isinstance(query_value, dict): + sql_candidate = query_value.get("sql") or query_value.get("query") + if isinstance(sql_candidate, str): + query_value = sql_candidate + else: + raise CorebrainError(f"La consulta SQL generada no es un string: {query_value}") + + # Preparar la consulta con el formato adecuado + query_to_execute = { + "type": query_type, + "query": query_value + } + + # Para MongoDB, añadir información específica + if query_type in ["nosql", "mongodb"]: + # Obtener nombre de colección + collection_name = None + if isinstance(api_response["query"], dict): + collection_name = api_response["query"].get("collection") + if not collection_name and "collection_name" in kwargs: + collection_name = kwargs["collection_name"] + if not collection_name and "collection" in self.db_config: + collection_name = self.db_config["collection"] + if not collection_name and available_tables: + collection_name = list(available_tables)[0] + + # Validar nombre de colección + if not collection_name: + raise CorebrainError("No se especificó colección y no se encontraron colecciones en el esquema") + if not isinstance(collection_name, str) or not collection_name.strip(): + raise CorebrainError("Nombre de colección inválido: debe ser un string no vacío") + + # Añadir colección a la consulta + query_to_execute["collection"] = collection_name + + # Añadir tipo de operación + if isinstance(api_response["query"], dict): + query_to_execute["operation"] = api_response["query"].get("operation", "find") + + # Añadir límite si se especifica + if "limit" in kwargs: + query_to_execute["limit"] = kwargs["limit"] + + # Ejecutar la consulta + start_time = datetime.now() + query_result = self._execute_query(query_to_execute) + query_time_ms = int((datetime.now() - start_time).total_seconds() * 1000) + + # Actualizar la respuesta con los resultados + api_response["result"] = { + "data": query_result, + "count": len(query_result) if isinstance(query_result, list) else 1, + "query_time_ms": query_time_ms, + "has_more": False + } + + # Si se debe generar explicación pero la API no lo hizo + if explain_results and ( + "explanation" not in api_response or + not isinstance(api_response.get("explanation"), str) or + len(str(api_response.get("explanation", ""))) < 15 # Detectar explicaciones numéricas o muy cortas + ): + # Preparar datos para obtener explicación + explanation_data = { + "question": question, + "query": api_response["query"], + "result": query_result, + "query_time_ms": query_time_ms, + "config_id": self.config_id, + "metadata": { + "collections_used": [query_to_execute.get("collection")] if query_to_execute.get("collection") else [], + "execution_time_ms": query_time_ms, + "available_tables": list(available_tables) + } + } + + try: + # Obtener explicación de la API + explanation_response = httpx.post( + f"{self.api_url}/api/database/sdk/query/explain", + headers=headers, + content=json.dumps(explanation_data, default=str), + timeout=30.0 + ) + + if explanation_response.status_code == 200: + explanation_result = explanation_response.json() + api_response["explanation"] = explanation_result.get("explanation", "No se pudo generar una explicación.") + else: + api_response["explanation"] = self._generate_fallback_explanation(query_to_execute, query_result) + except Exception as explain_error: + logger.error(f"Error al obtener explicación: {str(explain_error)}") + api_response["explanation"] = self._generate_fallback_explanation(query_to_execute, query_result) + + except Exception as e: + error_msg = f"Error al ejecutar la consulta: {str(e)}" + logger.error(error_msg) + return { + "error": True, + "explanation": error_msg, + "query": api_response.get("query", {}), + "metadata": { + "available_tables": list(available_tables) + } + } + + # Verificar si la explicación es un número (probablemente el tiempo de ejecución) y corregirlo + if "explanation" in api_response and not isinstance(api_response["explanation"], str): + # Si la explicación es un número, reemplazarla con una explicación generada + try: + is_sql = False + if "query" in api_response: + if isinstance(api_response["query"], dict) and "sql" in api_response["query"]: + is_sql = True + + if "result" in api_response: + result_data = api_response["result"] + if isinstance(result_data, dict) and "data" in result_data: + result_data = result_data["data"] + + if is_sql: + sql_query = api_response["query"].get("sql", "") + api_response["explanation"] = self._generate_sql_explanation(sql_query, result_data) + else: + # Para MongoDB o genérico + api_response["explanation"] = self._generate_generic_explanation(api_response["query"], result_data) + else: + api_response["explanation"] = "La consulta se ha ejecutado correctamente." + except Exception as exp_fix_error: + logger.error(f"Error al corregir explicación: {str(exp_fix_error)}") + api_response["explanation"] = "La consulta se ha ejecutado correctamente." + + # Preparar la respuesta final + result = { + "question": question, + "query": api_response["query"], + "config_id": self.config_id, + "metadata": { + "available_tables": list(available_tables) + } + } + + # Añadir resultados si están disponibles + if "result" in api_response: + if isinstance(api_response["result"], dict) and "data" in api_response["result"]: + result["result"] = api_response["result"] + else: + result["result"] = { + "data": api_response["result"], + "count": len(api_response["result"]) if isinstance(api_response["result"], list) else 1, + "query_time_ms": api_response.get("query_time_ms", 0), + "has_more": False + } + + # Añadir explicación si está disponible + if "explanation" in api_response: + result["explanation"] = api_response["explanation"] + + return result + + except httpx.TimeoutException: + return {"error": True, "explanation": "Tiempo de espera agotado al conectar con el servidor."} + + except httpx.RequestError as e: + return {"error": True, "explanation": f"Error de conexión con el servidor: {str(e)}"} + + except Exception as e: + import traceback + error_details = traceback.format_exc() + logger.error(f"Error inesperado en ask(): {error_details}") + return {"error": True, "explanation": f"Error inesperado: {str(e)}"} + + def _generate_fallback_explanation(self, query, results): + """ + Genera una explicación de respaldo cuando falla la generación de explicaciones. + + Args: + query: La consulta ejecutada + results: Los resultados obtenidos + + Returns: + Explicación generada + """ + # Determinar si es SQL o MongoDB + if isinstance(query, dict): + query_type = query.get("type", "").lower() + + if query_type in ["sqlite", "mysql", "postgresql"]: + return self._generate_sql_explanation(query.get("query", ""), results) + elif query_type in ["nosql", "mongodb"]: + return self._generate_mongodb_explanation(query, results) + + # Fallback genérico + result_count = len(results) if isinstance(results, list) else (1 if results else 0) + return f"La consulta devolvió {result_count} resultados." + + def _generate_sql_explanation(self, sql_query, results): + """ + Genera una explicación simple para consultas SQL. + + Args: + sql_query: La consulta SQL ejecutada + results: Los resultados obtenidos + + Returns: + Explicación generada + """ + sql_lower = sql_query.lower() if isinstance(sql_query, str) else "" + result_count = len(results) if isinstance(results, list) else (1 if results else 0) + + # Extraer nombres de tablas si es posible + tables = [] + from_match = re.search(r'from\s+([a-zA-Z0-9_]+)', sql_lower) + if from_match: + tables.append(from_match.group(1)) + + join_matches = re.findall(r'join\s+([a-zA-Z0-9_]+)', sql_lower) + if join_matches: + tables.extend(join_matches) + + # Detectar tipo de consulta + if "select" in sql_lower: + if "join" in sql_lower: + if len(tables) > 1: + if "where" in sql_lower: + return f"Se encontraron {result_count} registros que cumplen con los criterios especificados, relacionando información de las tablas {', '.join(tables)}." + else: + return f"Se obtuvieron {result_count} registros relacionando información de las tablas {', '.join(tables)}." + else: + return f"Se obtuvieron {result_count} registros relacionando datos entre tablas." + + elif "where" in sql_lower: + return f"Se encontraron {result_count} registros que cumplen con los criterios de búsqueda." + + else: + return f"La consulta devolvió {result_count} registros de la base de datos." + + # Para otros tipos de consultas (INSERT, UPDATE, DELETE) + if "insert" in sql_lower: + return "Se insertaron correctamente los datos en la base de datos." + elif "update" in sql_lower: + return "Se actualizaron correctamente los datos en la base de datos." + elif "delete" in sql_lower: + return "Se eliminaron correctamente los datos de la base de datos." + + # Fallback genérico + return f"La consulta SQL se ejecutó correctamente y devolvió {result_count} resultados." + + + def _generate_mongodb_explanation(self, query, results): + """ + Genera una explicación simple para consultas MongoDB. + + Args: + query: La consulta MongoDB ejecutada + results: Los resultados obtenidos + + Returns: + Explicación generada + """ + collection = query.get("collection", "la colección") + operation = query.get("operation", "find") + result_count = len(results) if isinstance(results, list) else (1 if results else 0) + + # Generar explicación según la operación + if operation == "find": + return f"Se encontraron {result_count} documentos en la colección {collection} que coinciden con los criterios de búsqueda." + elif operation == "findOne": + if result_count > 0: + return f"Se encontró el documento solicitado en la colección {collection}." + else: + return f"No se encontró ningún documento en la colección {collection} que coincida con los criterios." + elif operation == "aggregate": + return f"La agregación en la colección {collection} devolvió {result_count} resultados." + elif operation == "insertOne": + return f"Se ha insertado correctamente un nuevo documento en la colección {collection}." + elif operation == "updateOne": + return f"Se ha actualizado correctamente un documento en la colección {collection}." + elif operation == "deleteOne": + return f"Se ha eliminado correctamente un documento de la colección {collection}." + + # Fallback genérico + return f"La operación {operation} se ejecutó correctamente en la colección {collection} y devolvió {result_count} resultados." + + + def _generate_generic_explanation(self, query, results): + """ + Genera una explicación genérica cuando no se puede determinar el tipo de consulta. + + Args: + query: La consulta ejecutada + results: Los resultados obtenidos + + Returns: + Explicación generada + """ + result_count = len(results) if isinstance(results, list) else (1 if results else 0) + + if result_count == 0: + return "La consulta no devolvió ningún resultado." + elif result_count == 1: + return "La consulta devolvió 1 resultado." + else: + return f"La consulta devolvió {result_count} resultados." + + + def close(self) -> None: + """ + Close the database connection and release resources. + + This method should be called when the client is no longer needed to + ensure proper cleanup of resources. + """ + if self.db_connection: + db_type = self.db_config["type"].lower() + + try: + if db_type == "sql": + engine = self.db_config.get("engine", "").lower() + if engine in ["sqlite", "mysql", "postgresql"]: + self.db_connection.close() + else: + # SQLAlchemy engine + self.db_connection.dispose() + + elif db_type == "nosql" or db_type == "mongodb": + # For MongoDB, we close the client + if hasattr(self, 'mongo_client') and self.mongo_client: + self.mongo_client.close() + + elif db_type == "sqlite_memory": + self.db_connection.close() + + except Exception as e: + logger.warning(f"Error closing database connection: {str(e)}") + + self.db_connection = None + logger.info("Database connection closed") + + def _execute_query(self, query: Dict[str, Any]) -> List[Dict[str, Any]]: + """ + Execute a query based on its type. + + Args: + query: Dictionary containing query information + + Returns: + List of dictionaries containing query results + """ + query_type = query.get("type", "").lower() + + if query_type in ["sqlite", "mysql", "postgresql"]: + return self._execute_sql_query(query) + elif query_type in ["nosql", "mongodb"]: + return self._execute_mongodb_query(query) + else: + raise CorebrainError(f"Unsupported query type: {query_type}") + + def _execute_sql_query(self, query: Dict[str, Any]) -> List[Dict[str, Any]]: + """ + Execute a SQL query. + + Args: + query: Dictionary containing SQL query information + + Returns: + List of dictionaries containing query results + """ + query_type = query.get("type", "").lower() + + if query_type in ["sqlite", "mysql", "postgresql"]: + sql_query = query.get("query", "") + if not sql_query: + raise CorebrainError("No SQL query provided") + + engine = self.db_config.get("engine", "").lower() + + if engine == "sqlite": + return self._execute_sqlite_query(sql_query) + elif engine == "mysql": + return self._execute_mysql_query(sql_query) + elif engine == "postgresql": + return self._execute_postgresql_query(sql_query) + else: + raise CorebrainError(f"Unsupported SQL engine: {engine}") + + else: + raise CorebrainError(f"Unsupported SQL query type: {query_type}") + + def _execute_sqlite_query(self, sql_query: str) -> List[Dict[str, Any]]: + """ + Execute a SQLite query. + + Args: + sql_query (str): SQL query to execute + + Returns: + List[Dict[str, Any]]: List of results as dictionaries + """ + cursor = self.db_connection.cursor() + cursor.execute(sql_query) + + # Get column names + columns = [description[0] for description in cursor.description] + + # Convert results to list of dictionaries + results = [] + for row in cursor.fetchall(): + result = {} + for i, value in enumerate(row): + # Convert datetime objects to strings + if hasattr(value, 'isoformat'): + result[columns[i]] = value.isoformat() + else: + result[columns[i]] = value + results.append(result) + + return results + + def _execute_mysql_query(self, sql_query: str) -> List[Dict[str, Any]]: + """ + Execute a MySQL query. + + Args: + sql_query (str): SQL query to execute + + Returns: + List[Dict[str, Any]]: List of results as dictionaries + """ + cursor = self.db_connection.cursor(dictionary=True) + cursor.execute(sql_query) + + # Convert results to list of dictionaries + results = [] + for row in cursor.fetchall(): + result = {} + for key, value in row.items(): + # Convert datetime objects to strings + if hasattr(value, 'isoformat'): + result[key] = value.isoformat() + else: + result[key] = value + results.append(result) + + return results + + def _execute_postgresql_query(self, sql_query: str) -> List[Dict[str, Any]]: + """ + Execute a PostgreSQL query. + + Args: + sql_query (str): SQL query to execute + + Returns: + List[Dict[str, Any]]: List of results as dictionaries + """ + cursor = self.db_connection.cursor() + cursor.execute(sql_query) + + # Get column names + columns = [description[0] for description in cursor.description] + + # Convert results to list of dictionaries + results = [] + for row in cursor.fetchall(): + result = {} + for i, value in enumerate(row): + # Convert datetime objects to strings + if hasattr(value, 'isoformat'): + result[columns[i]] = value.isoformat() + else: + result[columns[i]] = value + results.append(result) + + return results + + def _execute_mongodb_query(self, query: Dict[str, Any]) -> List[Dict[str, Any]]: + """ + Execute a MongoDB query. + + Args: + query: Dictionary containing MongoDB query information + + Returns: + List of dictionaries containing query results + """ + try: + # Get collection name from query or use default + collection_name = query.get("collection") + if not collection_name: + raise CorebrainError("No collection specified for MongoDB query") + + # Get MongoDB collection + collection = self.mongo_client[self.db_config.get("database", "")][collection_name] + + # Execute query based on operation type + operation = query.get("operation", "find") + + if operation == "find": + # Handle find operation + cursor = collection.find( + query.get("query", {}), + projection=query.get("projection"), + sort=query.get("sort"), + limit=query.get("limit", 10), + skip=query.get("skip", 0) + ) + results = list(cursor) + + elif operation == "aggregate": + # Handle aggregate operation + pipeline = query.get("pipeline", []) + cursor = collection.aggregate(pipeline) + results = list(cursor) + + else: + raise CorebrainError(f"Unsupported MongoDB operation: {operation}") + + # Convert results to dictionaries and handle datetime serialization + serialized_results = [] + for doc in results: + # Convert ObjectId to string + if "_id" in doc: + doc["_id"] = str(doc["_id"]) + + # Handle datetime objects + for key, value in doc.items(): + if hasattr(value, 'isoformat'): + doc[key] = value.isoformat() + + serialized_results.append(doc) + + return serialized_results + + except Exception as e: + raise CorebrainError(f"Error executing MongoDB query: {str(e)}") + + +def init( + api_key: str = None, + db_config: Dict = None, + config_id: str = None, + user_data: Dict = None, + api_url: str = None, + skip_verification: bool = False +) -> Corebrain: + """ + Initialize and return a Corebrain client instance. + + This function creates a new Corebrain SDK client with the provided configuration. + It's a convenient factory function that wraps the Corebrain class initialization. + + Args: + api_key (str, optional): Corebrain API key. If not provided, it will attempt + to read from the COREBRAIN_API_KEY environment variable. + db_config (Dict, optional): Database configuration dictionary. If not provided, + it will attempt to read from the COREBRAIN_DB_CONFIG environment variable + (expected in JSON format). + config_id (str, optional): Configuration ID for saving/loading configurations. + user_data (Dict, optional): Optional user data for personalization. + api_url (str, optional): Corebrain API URL. Defaults to the production API. + skip_verification (bool, optional): Skip API token verification. Default False. + + Returns: + Corebrain: An initialized Corebrain client instance. + + Example: + >>> client = init(api_key="your_api_key", db_config={"type": "sql", "engine": "sqlite", "database": "example.db"}) + """ + return Corebrain( + api_key=api_key, + db_config=db_config, + config_id=config_id, + user_data=user_data, + api_url=api_url, + skip_verification=skip_verification + ) + diff --git a/corebrain/core/common.py b/corebrain/core/common.py new file mode 100644 index 0000000..3d75c8e --- /dev/null +++ b/corebrain/core/common.py @@ -0,0 +1,225 @@ +""" +Core functionalities shared across the Corebrain SDK. + +This module contains common elements used throughout the SDK, including: +- Logging system configuration +- Common type definitions and aliases +- Custom exceptions for better error handling +- Component registry system for dependency management + +These elements provide a common foundation for implementing +the rest of the SDK modules, ensuring consistency and facilitating +maintenance. +""" +import logging +from typing import Dict, Any, Optional, List, Callable, TypeVar, Union + +# Global logging configuration +logger = logging.getLogger("corebrain") +logger.addHandler(logging.NullHandler()) + +# Type aliases to improve readability and maintenance +ConfigDict = Dict[str, Any] +""" +Type representing a configuration as a key-value dictionary. + +Example: +```python +config: ConfigDict = { + "type": "sql", + "engine": "postgresql", + "host": "localhost", + "port": 5432, + "user": "postgres", + "password": "password", + "database": "mydatabase" +} +``` +""" + +SchemaDict = Dict[str, Any] +""" +Type representing a database schema as a dictionary. + +Example: +```python +schema: SchemaDict = { + "tables": [ + { + "name": "users", + "columns": [ + {"name": "id", "type": "INTEGER", "primary_key": True}, + {"name": "name", "type": "TEXT"}, + {"name": "email", "type": "TEXT"} + ] + } + ] +} +``` +""" + +# Generic component for typing +T = TypeVar('T') + +# SDK exceptions +class CorebrainError(Exception): + """ + Base exception for all Corebrain SDK errors. + + All other specific exceptions inherit from this class, + allowing you to catch any SDK error with a single + except block. + + Example: + ```python + try: + result = client.ask("How many users are there?") + except CorebrainError as e: + print(f"Corebrain error: {e}") + ``` + """ + pass + +class ConfigError(CorebrainError): + """ + Error related to SDK configuration. + + Raised when there are issues with the provided configuration, + such as invalid credentials, missing parameters, or incorrect formats. + + Example: + ```python + try: + client = init(api_key="invalid_key", db_config={}) + except ConfigError as e: + print(f"Configuration error: {e}") + ``` + """ + pass + +class DatabaseError(CorebrainError): + """ + Error related to database connection or query. + + Raised when there are problems connecting to the database, + executing queries, or extracting schema information. + + Example: + ```python + try: + result = client.ask("select * from a_table_that_does_not_exist") + except DatabaseError as e: + print(f"Database error: {e}") + ``` + """ + pass + +class APIError(CorebrainError): + """ + Error related to communication with the Corebrain API. + + Raised when there are issues in communicating with the service, + such as network errors, authentication failures, or unexpected responses. + + Example: + ```python + try: + result = client.ask("How many users are there?") + except APIError as e: + print(f"API error: {e}") + if e.status_code == 401: + print("Please verify your API key") + ``` + """ + def __init__(self, message: str, status_code: Optional[int] = None, response: Optional[Dict[str, Any]] = None): + """ + Initialize an APIError exception. + + Args: + message: Descriptive error message + status_code: Optional HTTP status code (e.g., 401, 404, 500) + response: Server response content if available + """ + self.status_code = status_code + self.response = response + super().__init__(message) + +# Component registry (to avoid circular imports) +_registry: Dict[str, Any] = {} + +def register_component(name: str, component: Any) -> None: + """ + Register a component in the global registry. + + This mechanism resolves circular dependencies between modules + by providing a way to access components without importing them directly. + + Args: + name: Unique name to identify the component + component: The component to register (can be any object) + + Example: + ```python + # In the module that defines the component + from core.common import register_component + + class DatabaseConnector: + def connect(self): + pass + + # Register the component + connector = DatabaseConnector() + register_component("db_connector", connector) + ``` + """ + _registry[name] = component + +def get_component(name: str) -> Any: + """ + Get a component from the global registry. + + Args: + name: Name of the component to retrieve + + Returns: + The registered component or None if it doesn't exist + + Example: + ```python + # In another module that needs to use the component + from core.common import get_component + + # Get the component + connector = get_component("db_connector") + if connector: + connector.connect() + ``` + """ + return _registry.get(name) + +def safely_get_component(name: str, default: Optional[T] = None) -> Union[Any, T]: + """ + Safely get a component from the global registry. + + If the component doesn't exist, it returns the provided default + value instead of None. + + Args: + name: Name of the component to retrieve + default: Default value to return if the component doesn't exist + + Returns: + The registered component or the default value + + Example: + ```python + # In another module + from core.common import safely_get_component + + # Get the component with a default value + connector = safely_get_component("db_connector", MyDefaultConnector()) + connector.connect() # Guaranteed not to be None + ``` + """ + component = _registry.get(name) + return component if component is not None else default \ No newline at end of file diff --git a/corebrain/core/query.py b/corebrain/core/query.py new file mode 100644 index 0000000..8747c6f --- /dev/null +++ b/corebrain/core/query.py @@ -0,0 +1,1037 @@ +""" +Componentes para manejo y análisis de consultas. +""" +import os +import json +import time +import re +import sqlite3 +import pickle +import hashlib + +from typing import Dict, Any, List, Optional, Tuple, Callable +from datetime import datetime +from pathlib import Path + +from corebrain.cli.utils import print_colored + +class QueryCache: + """Sistema de caché multinivel para consultas.""" + + def __init__(self, cache_dir: str = None, ttl: int = 86400, memory_limit: int = 100): + """ + Inicializa el sistema de caché. + + Args: + cache_dir: Directorio para el caché persistente + ttl: Tiempo de vida del caché en segundos (default: 24 horas) + memory_limit: Límite de entradas en caché de memoria + """ + # Caché en memoria (más rápido, pero volátil) + self.memory_cache = {} + self.memory_timestamps = {} + self.memory_limit = memory_limit + self.memory_lru = [] # Lista para seguimiento de menos usados recientemente + + # Caché persistente (más lento, pero permanente) + self.ttl = ttl + if cache_dir: + self.cache_dir = Path(cache_dir) + else: + self.cache_dir = Path.home() / ".corebrain_cache" + + # Crear directorio de caché si no existe + self.cache_dir.mkdir(parents=True, exist_ok=True) + + # Inicializar base de datos SQLite para metadatos + self.db_path = self.cache_dir / "cache_metadata.db" + self._init_db() + + print_colored(f"Caché inicializado en {self.cache_dir}", "blue") + + def _init_db(self): + """Inicializa la base de datos SQLite para metadatos de caché.""" + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + + # Crear tabla de metadatos si no existe + cursor.execute(''' + CREATE TABLE IF NOT EXISTS cache_metadata ( + query_hash TEXT PRIMARY KEY, + query TEXT, + config_id TEXT, + created_at TIMESTAMP, + last_accessed TIMESTAMP, + hit_count INTEGER DEFAULT 1 + ) + ''') + + conn.commit() + conn.close() + + def _get_hash(self, query: str, config_id: str, collection_name: Optional[str] = None) -> str: + """Genera un hash único para la consulta.""" + # Normalizar la consulta (eliminar espacios extra, convertir a minúsculas) + normalized_query = re.sub(r'\s+', ' ', query.lower().strip()) + + # Crear string compuesto para el hash + hash_input = f"{normalized_query}|{config_id}" + if collection_name: + hash_input += f"|{collection_name}" + + # Generar el hash + return hashlib.md5(hash_input.encode()).hexdigest() + + def _get_cache_path(self, query_hash: str) -> Path: + """Obtiene la ruta del archivo de caché para un hash dado.""" + # Usar los primeros caracteres del hash para crear subdirectorios + # Esto evita tener demasiados archivos en un solo directorio + subdir = query_hash[:2] + cache_subdir = self.cache_dir / subdir + cache_subdir.mkdir(exist_ok=True) + + return cache_subdir / f"{query_hash}.cache" + + def _update_metadata(self, query_hash: str, query: str, config_id: str): + """Actualiza los metadatos en la base de datos.""" + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + + now = datetime.now().isoformat() + + # Verificar si el hash ya existe + cursor.execute("SELECT hit_count FROM cache_metadata WHERE query_hash = ?", (query_hash,)) + result = cursor.fetchone() + + if result: + # Actualizar entrada existente + hit_count = result[0] + 1 + cursor.execute(''' + UPDATE cache_metadata + SET last_accessed = ?, hit_count = ? + WHERE query_hash = ? + ''', (now, hit_count, query_hash)) + else: + # Insertar nueva entrada + cursor.execute(''' + INSERT INTO cache_metadata (query_hash, query, config_id, created_at, last_accessed, hit_count) + VALUES (?, ?, ?, ?, ?, 1) + ''', (query_hash, query, config_id, now, now)) + + conn.commit() + conn.close() + + def _update_memory_lru(self, query_hash: str): + """Actualiza la lista LRU (Least Recently Used) para el caché en memoria.""" + if query_hash in self.memory_lru: + # Mover al final (más recientemente usado) + self.memory_lru.remove(query_hash) + + self.memory_lru.append(query_hash) + + # Si excedemos el límite, eliminar el elemento menos usado recientemente + if len(self.memory_lru) > self.memory_limit: + oldest_hash = self.memory_lru.pop(0) + if oldest_hash in self.memory_cache: + del self.memory_cache[oldest_hash] + del self.memory_timestamps[oldest_hash] + + def get(self, query: str, config_id: str, collection_name: Optional[str] = None) -> Optional[Dict[str, Any]]: + """ + Obtiene un resultado cacheado si existe y no ha expirado. + + Args: + query: Consulta en lenguaje natural + config_id: ID de configuración de la base de datos + collection_name: Nombre de la colección/tabla (opcional) + + Returns: + Resultado cacheado o None si no existe o ha expirado + """ + query_hash = self._get_hash(query, config_id, collection_name) + + # 1. Verificar caché en memoria (más rápido) + if query_hash in self.memory_cache: + timestamp = self.memory_timestamps[query_hash] + if (time.time() - timestamp) < self.ttl: + self._update_memory_lru(query_hash) + self._update_metadata(query_hash, query, config_id) + print_colored(f"Cache hit (memory): {query[:30]}...", "green") + return self.memory_cache[query_hash] + else: + # Expirado en memoria + del self.memory_cache[query_hash] + del self.memory_timestamps[query_hash] + if query_hash in self.memory_lru: + self.memory_lru.remove(query_hash) + + # 2. Verificar caché en disco + cache_path = self._get_cache_path(query_hash) + if cache_path.exists(): + # Verificar edad del archivo + file_age = time.time() - cache_path.stat().st_mtime + if file_age < self.ttl: + try: + with open(cache_path, 'rb') as f: + result = pickle.load(f) + + # Guardar también en caché de memoria + self.memory_cache[query_hash] = result + self.memory_timestamps[query_hash] = time.time() + self._update_memory_lru(query_hash) + self._update_metadata(query_hash, query, config_id) + + print_colored(f"Cache hit (disk): {query[:30]}...", "green") + return result + except Exception as e: + print_colored(f"Error al cargar caché: {str(e)}", "red") + # Si hay error al cargar, eliminar el archivo corrupto + cache_path.unlink(missing_ok=True) + else: + # Archivo expirado, eliminarlo + cache_path.unlink(missing_ok=True) + + return None + + def set(self, query: str, config_id: str, result: Dict[str, Any], collection_name: Optional[str] = None): + """ + Guarda un resultado en el caché. + + Args: + query: Consulta en lenguaje natural + config_id: ID de configuración + result: Resultado a cachear + collection_name: Nombre de la colección/tabla (opcional) + """ + query_hash = self._get_hash(query, config_id, collection_name) + + # 1. Guardar en caché de memoria + self.memory_cache[query_hash] = result + self.memory_timestamps[query_hash] = time.time() + self._update_memory_lru(query_hash) + + # 2. Guardar en caché persistente + try: + cache_path = self._get_cache_path(query_hash) + with open(cache_path, 'wb') as f: + pickle.dump(result, f) + + # 3. Actualizar metadatos + self._update_metadata(query_hash, query, config_id) + + print_colored(f"Cached: {query[:30]}...", "green") + except Exception as e: + print_colored(f"Error al guardar en caché: {str(e)}", "red") + + def clear(self, older_than: int = None): + """ + Limpia el caché. + + Args: + older_than: Limpiar solo entradas más antiguas que esta cantidad de segundos + """ + # Limpiar caché en memoria + if older_than: + current_time = time.time() + keys_to_remove = [ + k for k, timestamp in self.memory_timestamps.items() + if (current_time - timestamp) > older_than + ] + + for k in keys_to_remove: + if k in self.memory_cache: + del self.memory_cache[k] + if k in self.memory_timestamps: + del self.memory_timestamps[k] + if k in self.memory_lru: + self.memory_lru.remove(k) + else: + self.memory_cache.clear() + self.memory_timestamps.clear() + self.memory_lru.clear() + + # Limpiar caché en disco + if older_than: + cutoff_time = time.time() - older_than + + # Usar la base de datos para encontrar archivos antiguos + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + + # Convertir cutoff_time a formato ISO + cutoff_datetime = datetime.fromtimestamp(cutoff_time).isoformat() + + cursor.execute( + "SELECT query_hash FROM cache_metadata WHERE last_accessed < ?", + (cutoff_datetime,) + ) + + old_hashes = [row[0] for row in cursor.fetchall()] + + # Eliminar archivos antiguos + for query_hash in old_hashes: + cache_path = self._get_cache_path(query_hash) + if cache_path.exists(): + cache_path.unlink() + + # Eliminar de la base de datos + cursor.execute( + "DELETE FROM cache_metadata WHERE query_hash = ?", + (query_hash,) + ) + + conn.commit() + conn.close() + else: + # Eliminar todos los archivos de caché + for subdir in self.cache_dir.iterdir(): + if subdir.is_dir(): + for cache_file in subdir.glob("*.cache"): + cache_file.unlink() + + # Reiniciar la base de datos + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + cursor.execute("DELETE FROM cache_metadata") + conn.commit() + conn.close() + + def get_stats(self) -> Dict[str, Any]: + """Obtiene estadísticas del caché.""" + # Contar archivos en disco + disk_count = 0 + for subdir in self.cache_dir.iterdir(): + if subdir.is_dir(): + disk_count += len(list(subdir.glob("*.cache"))) + + # Obtener estadísticas de la base de datos + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + + # Total de entradas + cursor.execute("SELECT COUNT(*) FROM cache_metadata") + total_entries = cursor.fetchone()[0] + + # Consultas más frecuentes + cursor.execute( + "SELECT query, hit_count FROM cache_metadata ORDER BY hit_count DESC LIMIT 5" + ) + top_queries = cursor.fetchall() + + # Edad promedio + cursor.execute( + "SELECT AVG(strftime('%s', 'now') - strftime('%s', created_at)) FROM cache_metadata" + ) + avg_age = cursor.fetchone()[0] + + conn.close() + + return { + "memory_cache_size": len(self.memory_cache), + "disk_cache_size": disk_count, + "total_entries": total_entries, + "top_queries": top_queries, + "average_age_seconds": avg_age, + "cache_directory": str(self.cache_dir) + } + +class QueryTemplate: + """Plantilla de consulta predefinida para patrones comunes.""" + + def __init__(self, pattern: str, description: str, + sql_template: Optional[str] = None, + generator_func: Optional[Callable] = None, + db_type: str = "sql", + applicable_tables: Optional[List[str]] = None): + """ + Inicializa una plantilla de consulta. + + Args: + pattern: Patrón de lenguaje natural que coincide con esta plantilla + description: Descripción de la plantilla + sql_template: Plantilla SQL con marcadores para parámetros + generator_func: Función alternativa para generar la consulta + db_type: Tipo de base de datos (sql, mongodb) + applicable_tables: Lista de tablas a las que aplica esta plantilla + """ + self.pattern = pattern + self.description = description + self.sql_template = sql_template + self.generator_func = generator_func + self.db_type = db_type + self.applicable_tables = applicable_tables or [] + + # Compilar expresión regular para el patrón + self.regex = self._compile_pattern(pattern) + + def _compile_pattern(self, pattern: str) -> re.Pattern: + """Compila el patrón en una expresión regular.""" + # Reemplazar marcadores especiales con grupos de captura + regex_pattern = pattern + + # {table} se convierte en grupo de captura para el nombre de tabla + regex_pattern = regex_pattern.replace("{table}", r"(\w+)") + + # {field} se convierte en grupo de captura para el nombre de campo + regex_pattern = regex_pattern.replace("{field}", r"(\w+)") + + # {value} se convierte en grupo de captura para un valor + regex_pattern = regex_pattern.replace("{value}", r"([^,.\s]+)") + + # {number} se convierte en grupo de captura para un número + regex_pattern = regex_pattern.replace("{number}", r"(\d+)") + + # Hacer coincidir el patrón completo + regex_pattern = f"^{regex_pattern}$" + + return re.compile(regex_pattern, re.IGNORECASE) + + def matches(self, query: str) -> Tuple[bool, List[str]]: + """ + Verifica si una consulta coincide con esta plantilla. + + Args: + query: Consulta a verificar + + Returns: + Tupla de (coincide, [parámetros capturados]) + """ + match = self.regex.match(query) + if match: + return True, list(match.groups()) + return False, [] + + def generate_query(self, params: List[str], db_schema: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """ + Genera una consulta a partir de los parámetros capturados. + + Args: + params: Parámetros capturados del patrón + db_schema: Esquema de la base de datos + + Returns: + Consulta generada o None si no se puede generar + """ + if self.generator_func: + # Usar función personalizada + return self.generator_func(params, db_schema) + + if not self.sql_template: + return None + + # Intentar aplicar la plantilla SQL con los parámetros + try: + sql_query = self.sql_template + + # Reemplazar parámetros en la plantilla + for i, param in enumerate(params): + placeholder = f"${i+1}" + sql_query = sql_query.replace(placeholder, param) + + # Verificar si hay algún parámetro sin reemplazar + if "$" in sql_query: + return None + + return {"sql": sql_query} + except Exception: + return None + +class QueryAnalyzer: + """Analiza patrones de consultas para sugerir optimizaciones.""" + + def __init__(self, query_log_path: str = None, template_path: str = None): + """ + Inicializa el analizador de consultas. + + Args: + query_log_path: Ruta al archivo de registro de consultas + template_path: Ruta al archivo de plantillas + """ + self.query_log_path = query_log_path or os.path.join( + Path.home(), ".corebrain_cache", "query_log.db" + ) + + self.template_path = template_path or os.path.join( + Path.home(), ".corebrain_cache", "templates.json" + ) + + # Inicializar base de datos + self._init_db() + + # Plantillas predefinidas para consultas comunes + self.templates = self._load_default_templates() + + # Cargar plantillas personalizadas + self._load_custom_templates() + + # Plantillas comunes para identificar patrones + self.common_patterns = [ + r"muestra\s+(?:todos\s+)?los\s+(\w+)", + r"lista\s+(?:de\s+)?(?:todos\s+)?los\s+(\w+)", + r"busca\s+(\w+)\s+donde", + r"cu[aá]ntos\s+(\w+)\s+hay", + r"total\s+de\s+(\w+)" + ] + + def _init_db(self): + """Inicializa la base de datos para el registro de consultas.""" + # Asegurar que el directorio existe + os.makedirs(os.path.dirname(self.query_log_path), exist_ok=True) + + conn = sqlite3.connect(self.query_log_path) + cursor = conn.cursor() + + # Crear tabla de registro si no existe + cursor.execute(''' + CREATE TABLE IF NOT EXISTS query_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + query TEXT, + config_id TEXT, + collection_name TEXT, + timestamp TIMESTAMP, + execution_time REAL, + cost REAL, + result_count INTEGER, + pattern TEXT + ) + ''') + + # Crear tabla de patrones detectados + cursor.execute(''' + CREATE TABLE IF NOT EXISTS query_patterns ( + pattern TEXT PRIMARY KEY, + count INTEGER, + avg_execution_time REAL, + avg_cost REAL, + last_updated TIMESTAMP + ) + ''') + + conn.commit() + conn.close() + + def _load_default_templates(self) -> List[QueryTemplate]: + """Carga las plantillas predefinidas para consultas comunes.""" + templates = [] + + # Listar todos los registros de una tabla + templates.append( + QueryTemplate( + pattern="muestra todos los {table}", + description="Listar todos los registros de una tabla", + sql_template="SELECT * FROM $1 LIMIT 100", + db_type="sql" + ) + ) + + # Contar registros + templates.append( + QueryTemplate( + pattern="cuántos {table} hay", + description="Contar registros en una tabla", + sql_template="SELECT COUNT(*) FROM $1", + db_type="sql" + ) + ) + + # Buscar por ID + templates.append( + QueryTemplate( + pattern="busca el {table} con id {value}", + description="Buscar registro por ID", + sql_template="SELECT * FROM $1 WHERE id = $2", + db_type="sql" + ) + ) + + # Listar ordenados + templates.append( + QueryTemplate( + pattern="lista los {table} ordenados por {field}", + description="Listar registros ordenados por campo", + sql_template="SELECT * FROM $1 ORDER BY $2 LIMIT 100", + db_type="sql" + ) + ) + + # Buscar por email + templates.append( + QueryTemplate( + pattern="busca el usuario con email {value}", + description="Buscar usuario por email", + sql_template="SELECT * FROM users WHERE email = '$2'", + db_type="sql" + ) + ) + + # Contar por campo + templates.append( + QueryTemplate( + pattern="cuántos {table} hay por {field}", + description="Contar registros agrupados por campo", + sql_template="SELECT $2, COUNT(*) FROM $1 GROUP BY $2", + db_type="sql" + ) + ) + + # Contar usuarios activos + templates.append( + QueryTemplate( + pattern="cuántos usuarios activos hay", + description="Contar usuarios activos", + sql_template="SELECT COUNT(*) FROM users WHERE is_active = TRUE", + db_type="sql", + applicable_tables=["users"] + ) + ) + + # Listar usuarios por fecha de registro + templates.append( + QueryTemplate( + pattern="usuarios registrados en los últimos {number} días", + description="Listar usuarios recientes", + sql_template=""" + SELECT * FROM users + WHERE created_at >= datetime('now', '-$2 days') + ORDER BY created_at DESC + LIMIT 100 + """, + db_type="sql", + applicable_tables=["users"] + ) + ) + + # Buscar empresas + templates.append( + QueryTemplate( + pattern="usuarios que tienen empresa", + description="Buscar usuarios con empresa asignada", + sql_template=""" + SELECT u.* FROM users u + INNER JOIN businesses b ON u.id = b.owner_id + WHERE u.is_business = TRUE + LIMIT 100 + """, + db_type="sql", + applicable_tables=["users", "businesses"] + ) + ) + + # Buscar negocios + templates.append( + QueryTemplate( + pattern="busca negocios en {value}", + description="Buscar negocios por ubicación", + sql_template=""" + SELECT * FROM businesses + WHERE address_city LIKE '%$2%' OR address_province LIKE '%$2%' + LIMIT 100 + """, + db_type="sql", + applicable_tables=["businesses"] + ) + ) + + # MongoDB: Listar documentos + templates.append( + QueryTemplate( + pattern="muestra todos los documentos de {table}", + description="Listar documentos en una colección", + db_type="mongodb", + generator_func=lambda params, schema: { + "collection": params[0], + "operation": "find", + "query": {}, + "limit": 100 + } + ) + ) + + return templates + + def _load_custom_templates(self): + """Carga plantillas personalizadas desde el archivo.""" + if not os.path.exists(self.template_path): + return + + try: + with open(self.template_path, 'r') as f: + custom_templates = json.load(f) + + for template_data in custom_templates: + # Crear plantilla desde datos JSON + template = QueryTemplate( + pattern=template_data.get("pattern", ""), + description=template_data.get("description", ""), + sql_template=template_data.get("sql_template"), + db_type=template_data.get("db_type", "sql"), + applicable_tables=template_data.get("applicable_tables", []) + ) + + self.templates.append(template) + + except Exception as e: + print_colored(f"Error al cargar plantillas personalizadas: {str(e)}", "red") + + def save_custom_template(self, template: QueryTemplate) -> bool: + """ + Guarda una plantilla personalizada. + + Args: + template: Plantilla a guardar + + Returns: + True si se guardó correctamente + """ + # Cargar plantillas existentes + custom_templates = [] + if os.path.exists(self.template_path): + try: + with open(self.template_path, 'r') as f: + custom_templates = json.load(f) + except: + custom_templates = [] + + # Convertir plantilla a diccionario + template_data = { + "pattern": template.pattern, + "description": template.description, + "sql_template": template.sql_template, + "db_type": template.db_type, + "applicable_tables": template.applicable_tables + } + + # Verificar si ya existe una plantilla con el mismo patrón + for i, existing in enumerate(custom_templates): + if existing.get("pattern") == template.pattern: + # Actualizar existente + custom_templates[i] = template_data + break + else: + # Agregar nueva + custom_templates.append(template_data) + + # Guardar plantillas + try: + with open(self.template_path, 'w') as f: + json.dump(custom_templates, f, indent=2) + + # Actualizar lista de plantillas + self.templates.append(template) + + return True + except Exception as e: + print_colored(f"Error al guardar plantilla personalizada: {str(e)}", "red") + return False + + def find_matching_template(self, query: str, db_schema: Dict[str, Any]) -> Optional[Tuple[QueryTemplate, List[str]]]: + """ + Busca una plantilla que coincida con la consulta. + + Args: + query: Consulta en lenguaje natural + db_schema: Esquema de la base de datos + + Returns: + Tupla de (plantilla, parámetros) o None si no hay coincidencia + """ + for template in self.templates: + matches, params = template.matches(query) + if matches: + # Verificar si la plantilla es aplicable a las tablas existentes + if template.applicable_tables: + available_tables = set(db_schema.get("tables", {}).keys()) + if not any(table in available_tables for table in template.applicable_tables): + continue + + return template, params + + return None + + def log_query(self, query: str, config_id: str, collection_name: str = None, + execution_time: float = 0, cost: float = 0.09, result_count: int = 0): + """ + Registra una consulta para análisis. + + Args: + query: Consulta en lenguaje natural + config_id: ID de configuración + collection_name: Nombre de la colección/tabla + execution_time: Tiempo de ejecución en segundos + cost: Costo estimado de la consulta + result_count: Número de resultados obtenidos + """ + # Detectar patrón + pattern = self._detect_pattern(query) + + # Registrar en la base de datos + conn = sqlite3.connect(self.query_log_path) + cursor = conn.cursor() + + cursor.execute(''' + INSERT INTO query_log (query, config_id, collection_name, timestamp, execution_time, cost, result_count, pattern) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ''', ( + query, config_id, collection_name, datetime.now().isoformat(), + execution_time, cost, result_count, pattern + )) + + # Actualizar estadísticas de patrones + if pattern: + cursor.execute( + "SELECT count, avg_execution_time, avg_cost FROM query_patterns WHERE pattern = ?", + (pattern,) + ) + result = cursor.fetchone() + + if result: + # Actualizar patrón existente + count, avg_exec_time, avg_cost = result + new_count = count + 1 + new_avg_exec_time = (avg_exec_time * count + execution_time) / new_count + new_avg_cost = (avg_cost * count + cost) / new_count + + cursor.execute(''' + UPDATE query_patterns + SET count = ?, avg_execution_time = ?, avg_cost = ?, last_updated = ? + WHERE pattern = ? + ''', (new_count, new_avg_exec_time, new_avg_cost, datetime.now().isoformat(), pattern)) + else: + # Insertar nuevo patrón + cursor.execute(''' + INSERT INTO query_patterns (pattern, count, avg_execution_time, avg_cost, last_updated) + VALUES (?, 1, ?, ?, ?) + ''', (pattern, execution_time, cost, datetime.now().isoformat())) + + conn.commit() + conn.close() + + def _detect_pattern(self, query: str) -> Optional[str]: + """ + Detecta un patrón en la consulta. + + Args: + query: Consulta a analizar + + Returns: + Patrón detectado o None + """ + normalized_query = query.lower() + + # Comprobar patrones predefinidos + for pattern in self.common_patterns: + match = re.search(pattern, normalized_query) + if match: + # Devolver el patrón con comodines + entity = match.group(1) + return pattern.replace(r'(\w+)', f"{entity}") + + # Si no se detecta ningún patrón predefinido, intentar generalizar + words = normalized_query.split() + if len(words) < 3: + return None + + # Intentar generalizar consultas simples + if "mostrar" in words or "muestra" in words or "listar" in words or "lista" in words: + for i, word in enumerate(words): + if word in ["de", "los", "las", "todos", "todas"]: + if i+1 < len(words): + return f"lista_de_{words[i+1]}" + + return None + + def get_common_patterns(self, limit: int = 5) -> List[Dict[str, Any]]: + """ + Obtiene los patrones de consulta más comunes. + + Args: + limit: Número máximo de patrones a devolver + + Returns: + Lista de patrones más comunes + """ + conn = sqlite3.connect(self.query_log_path) + cursor = conn.cursor() + + cursor.execute(''' + SELECT pattern, count, avg_execution_time, avg_cost + FROM query_patterns + ORDER BY count DESC + LIMIT ? + ''', (limit,)) + + patterns = [] + for row in cursor.fetchall(): + pattern, count, avg_time, avg_cost = row + patterns.append({ + "pattern": pattern, + "count": count, + "avg_execution_time": avg_time, + "avg_cost": avg_cost, + "estimated_monthly_cost": round(avg_cost * count * 30 / 7, 2) # Estimación mensual + }) + + conn.close() + return patterns + + def suggest_new_template(self, query: str, sql_query: str) -> Optional[QueryTemplate]: + """ + Sugiere una nueva plantilla basada en una consulta exitosa. + + Args: + query: Consulta en lenguaje natural + sql_query: Consulta SQL generada + + Returns: + Plantilla sugerida o None + """ + # Detectar patrón + pattern = self._detect_pattern(query) + if not pattern: + return None + + # Generalizar la consulta SQL + generalized_sql = sql_query + + # Reemplazar valores específicos con marcadores + # Esto es una simplificación, idealmente se usaría un parser SQL + tokens = query.lower().split() + + # Identificar posibles valores a parametrizar + for i, token in enumerate(tokens): + if token.isdigit(): + # Reemplazar números + generalized_sql = re.sub(r'\b' + re.escape(token) + r'\b', '$1', generalized_sql) + pattern = pattern.replace(token, "{number}") + elif '@' in token and '.' in token: + # Reemplazar emails + generalized_sql = re.sub(r'\b' + re.escape(token) + r'\b', '$1', generalized_sql) + pattern = pattern.replace(token, "{value}") + elif token.startswith('"') or token.startswith("'"): + # Reemplazar strings + value = token.strip('"\'') + if len(value) > 2: # Evitar reemplazar strings muy cortos + generalized_sql = re.sub(r'[\'"]' + re.escape(value) + r'[\'"]', "'$1'", generalized_sql) + pattern = pattern.replace(token, "{value}") + + # Crear plantilla + return QueryTemplate( + pattern=pattern, + description=f"Plantilla generada automáticamente para: {pattern}", + sql_template=generalized_sql, + db_type="sql" + ) + + def get_optimization_suggestions(self) -> List[Dict[str, Any]]: + """ + Genera sugerencias para optimizar consultas. + + Returns: + Lista de sugerencias de optimización + """ + suggestions = [] + + # Calcular estadísticas generales + conn = sqlite3.connect(self.query_log_path) + cursor = conn.cursor() + + # Total de consultas y costo en los últimos 30 días + cursor.execute(''' + SELECT COUNT(*) as query_count, SUM(cost) as total_cost + FROM query_log + WHERE timestamp > datetime('now', '-30 day') + ''') + + row = cursor.fetchone() + if row: + query_count, total_cost = row + + if query_count and query_count > 100: + # Si hay muchas consultas en total, sugerir plan de volumen + suggestions.append({ + "type": "volume_plan", + "query_count": query_count, + "total_cost": round(total_cost, 2) if total_cost else 0, + "suggestion": f"Considerar negociar un plan por volumen. Actualmente ~{query_count} consultas/mes." + }) + + # Sugerir ajustar el TTL del caché según frecuencia + avg_queries_per_day = query_count / 30 + suggested_ttl = max(3600, min(86400 * 3, 86400 * (100 / avg_queries_per_day))) + + suggestions.append({ + "type": "cache_adjustment", + "current_rate": f"{avg_queries_per_day:.1f} consultas/día", + "suggestion": f"Ajustar TTL del caché a {suggested_ttl/3600:.1f} horas basado en su patrón de uso" + }) + + # Obtener patrones comunes + common_patterns = self.get_common_patterns(10) + + for pattern in common_patterns: + if pattern["count"] >= 5: + # Si un patrón se repite mucho, sugerir precompilación + suggestions.append({ + "type": "precompile", + "pattern": pattern["pattern"], + "count": pattern["count"], + "estimated_savings": round(pattern["avg_cost"] * pattern["count"] * 0.9, 2), # 90% de ahorro + "suggestion": f"Crear una plantilla SQL para consultas del tipo '{pattern['pattern']}'" + }) + + # Si un patrón es costoso pero poco frecuente + if pattern["avg_cost"] > 0.1 and pattern["count"] < 5: + suggestions.append({ + "type": "analyze", + "pattern": pattern["pattern"], + "avg_cost": pattern["avg_cost"], + "suggestion": f"Revisar manualmente consultas del tipo '{pattern['pattern']}' para optimizar" + }) + + # Buscar períodos con alta carga para ajustar parámetros + cursor.execute(''' + SELECT strftime('%Y-%m-%d %H', timestamp) as hour, COUNT(*) as count, SUM(cost) as total_cost + FROM query_log + WHERE timestamp > datetime('now', '-7 day') + GROUP BY hour + ORDER BY count DESC + LIMIT 5 + ''') + + for row in cursor.fetchall(): + hour, count, total_cost = row + if count > 20: # Si hay más de 20 consultas en una hora + suggestions.append({ + "type": "load_balancing", + "hour": hour, + "query_count": count, + "total_cost": round(total_cost, 2), + "suggestion": f"Alta carga de consultas detectada el {hour} ({count} consultas). Considerar técnicas de agrupación." + }) + + # Buscar consultas redundantes (misma consulta en corto tiempo) + cursor.execute(''' + SELECT query, COUNT(*) as count + FROM query_log + WHERE timestamp > datetime('now', '-1 day') + GROUP BY query + HAVING COUNT(*) > 3 + ORDER BY COUNT(*) DESC + LIMIT 5 + ''') + + for row in cursor.fetchall(): + query, count = row + suggestions.append({ + "type": "redundant", + "query": query, + "count": count, + "estimated_savings": round(0.09 * (count - 1), 2), # Ahorro por no repetir + "suggestion": f"Implementar caché para la consulta '{query[:50]}...' que se repitió {count} veces" + }) + + conn.close() + return suggestions + + + \ No newline at end of file diff --git a/corebrain/core/test_utils.py b/corebrain/core/test_utils.py new file mode 100644 index 0000000..543eee8 --- /dev/null +++ b/corebrain/core/test_utils.py @@ -0,0 +1,157 @@ +""" +Utilidades para pruebas y validación de componentes. +""" +import json +import random +from typing import Dict, Any, Optional + +from corebrain.cli.utils import print_colored +from corebrain.cli.common import DEFAULT_API_URL +from corebrain.network.client import http_session + +def generate_test_question_from_schema(schema: Dict[str, Any]) -> str: + """ + Genera una pregunta de prueba basada en el esquema de la base de datos. + + Args: + schema: Esquema de la base de datos + + Returns: + Pregunta de prueba generada + """ + if not schema or not schema.get("tables"): + return "¿Cuáles son las tablas disponibles?" + + tables = schema["tables"] + + if not tables: + return "¿Cuáles son las tablas disponibles?" + + # Seleccionar una tabla aleatoria + table = random.choice(tables) + table_name = table["name"] + + # Determinar el tipo de pregunta + question_types = [ + f"¿Cuántos registros hay en la tabla {table_name}?", + f"Muestra los primeros 5 registros de {table_name}", + f"¿Cuáles son los campos de la tabla {table_name}?", + ] + + # Obtener columnas según la estructura (SQL vs NoSQL) + columns = [] + if "columns" in table and table["columns"]: + columns = table["columns"] + elif "fields" in table and table["fields"]: + columns = table["fields"] + + if columns: + # Si tenemos información de columnas/campos + column_name = columns[0]["name"] if columns else "id" + + # Añadir preguntas específicas con columnas + question_types.extend([ + f"¿Cuál es el valor máximo de {column_name} en {table_name}?", + f"¿Cuáles son los valores únicos de {column_name} en {table_name}?", + ]) + + return random.choice(question_types) + +def test_natural_language_query(api_token: str, db_config: Dict[str, Any], api_url: Optional[str] = None, user_data: Optional[Dict[str, Any]] = None) -> bool: + """ + Prueba una consulta de lenguaje natural. + + Args: + api_token: Token de API + db_config: Configuración de la base de datos + api_url: URL opcional de la API + user_data: Datos del usuario + + Returns: + True si la prueba es exitosa, False en caso contrario + """ + try: + print_colored("\nRealizando prueba de consulta en lenguaje natural...", "blue") + + # Importación dinámica para evitar circular imports + from db.schema_file import extract_db_schema + + # Generar una pregunta de prueba basada en el esquema extraído directamente + schema = extract_db_schema(db_config) + print("REcoge esquema: ", schema) + question = generate_test_question_from_schema(schema) + print(f"Pregunta de prueba: {question}") + + # Preparar los datos para la petición + api_url = api_url or DEFAULT_API_URL + if not api_url.startswith(("http://", "https://")): + api_url = "https://" + api_url + + if api_url.endswith('/'): + api_url = api_url[:-1] + + # Construir endpoint para la consulta + endpoint = f"{api_url}/api/database/sdk/query" + + # Datos para la consulta + request_data = { + "question": question, + "db_schema": schema, + "config_id": db_config["config_id"] + } + + # Realizar la petición al API + headers = { + "Authorization": f"Bearer {api_token}", + "Content-Type": "application/json" + } + + timeout = 15.0 # Tiempo máximo de espera reducido + + try: + print_colored("Enviando consulta al API...", "blue") + response = http_session.post( + endpoint, + headers=headers, + json=request_data, + timeout=timeout + ) + + # Verificar la respuesta + if response.status_code == 200: + result = response.json() + + # Verificar si hay explicación en el resultado + if "explanation" in result: + print_colored("\nRespuesta:", "green") + print(result["explanation"]) + + print_colored("\n✅ Prueba de consulta exitosa!", "green") + return True + else: + # Si no hay explicación pero la API responde, puede ser un formato diferente + print_colored("\nRespuesta recibida del API (formato diferente al esperado):", "yellow") + print(json.dumps(result, indent=2)) + print_colored("\n⚠️ La API respondió, pero con un formato diferente al esperado.", "yellow") + return True + else: + print_colored(f"❌ Error en la respuesta: Código {response.status_code}", "red") + try: + error_data = response.json() + print(json.dumps(error_data, indent=2)) + except: + print(response.text[:500]) + return False + + except http_session.TimeoutException: + print_colored("⚠️ Timeout al realizar la consulta. El API puede estar ocupado o no disponible.", "yellow") + print_colored("Esto no afecta a la configuración guardada.", "yellow") + return False + except http_session.RequestError as e: + print_colored(f"⚠️ Error de conexión: {str(e)}", "yellow") + print_colored("Verifica la URL de la API y tu conexión a internet.", "yellow") + return False + + except Exception as e: + print_colored(f"❌ Error al realizar la consulta: {str(e)}", "red") + return False \ No newline at end of file diff --git a/corebrain/db/__init__.py b/corebrain/db/__init__.py new file mode 100644 index 0000000..e362e76 --- /dev/null +++ b/corebrain/db/__init__.py @@ -0,0 +1,26 @@ +""" +Conectores de bases de datos para Corebrain SDK. + +Este paquete proporciona conectores para diferentes tipos y +motores de bases de datos soportados por Corebrain. +""" +from corebrain.db.connector import DatabaseConnector +from corebrain.db.factory import get_connector +from corebrain.db.engines import get_available_engines +from corebrain.db.connectors.sql import SQLConnector +from corebrain.db.connectors.mongodb import MongoDBConnector +from corebrain.db.schema_file import get_schema_with_dynamic_import +from corebrain.db.schema.optimizer import SchemaOptimizer +from corebrain.db.schema.extractor import extract_db_schema + +# Exportación explícita de componentes públicos +__all__ = [ + 'DatabaseConnector', + 'get_connector', + 'get_available_engines', + 'SQLConnector', + 'MongoDBConnector', + 'SchemaOptimizer', + 'extract_db_schema', + 'get_schema_with_dynamic_import' +] \ No newline at end of file diff --git a/corebrain/db/connector.py b/corebrain/db/connector.py new file mode 100644 index 0000000..4a54f4e --- /dev/null +++ b/corebrain/db/connector.py @@ -0,0 +1,33 @@ +""" +Conectores base para diferentes tipos de bases de datos. +""" +from typing import Dict, Any, List, Optional, Callable + +class DatabaseConnector: + """Clase base para todos los conectores de base de datos""" + + def __init__(self, config: Dict[str, Any], timeout: int = 10): + self.config = config + self.timeout = timeout + self.connection = None + + def connect(self): + """Establece conexión a la base de datos""" + raise NotImplementedError + + def extract_schema(self, sample_limit: int = 5, table_limit: Optional[int] = None, + progress_callback: Optional[Callable] = None) -> Dict[str, Any]: + """Extrae el esquema de la base de datos""" + raise NotImplementedError + + def execute_query(self, query: str) -> List[Dict[str, Any]]: + """Ejecuta una consulta en la base de datos""" + raise NotImplementedError + + def close(self): + """Cierra la conexión""" + if self.connection: + try: + self.connection.close() + except: + pass \ No newline at end of file diff --git a/corebrain/db/connectors/__init__.py b/corebrain/db/connectors/__init__.py new file mode 100644 index 0000000..8b4af97 --- /dev/null +++ b/corebrain/db/connectors/__init__.py @@ -0,0 +1,28 @@ +""" +Conectores de bases de datos para diferentes motores. +""" + +from typing import Dict, Any + +from corebrain.db.connectors.sql import SQLConnector +from corebrain.db.connectors.mongodb import MongoDBConnector + +def get_connector(db_config: Dict[str, Any]): + """ + Obtiene el conector adecuado según la configuración de la base de datos. + + Args: + db_config: Configuración de la base de datos + + Returns: + Instancia del conector apropiado + """ + db_type = db_config.get("type", "").lower() + + if db_type == "sql": + engine = db_config.get("engine", "").lower() + return SQLConnector(db_config, engine) + elif db_type == "nosql" or db_type == "mongodb": + return MongoDBConnector(db_config) + else: + raise ValueError(f"Tipo de base de datos no soportado: {db_type}") \ No newline at end of file diff --git a/corebrain/db/connectors/mongodb.py b/corebrain/db/connectors/mongodb.py new file mode 100644 index 0000000..ae57cc2 --- /dev/null +++ b/corebrain/db/connectors/mongodb.py @@ -0,0 +1,474 @@ +""" +Conector para bases de datos MongoDB. +""" + +import time +import json +import re + +from typing import Dict, Any, List, Optional, Callable, Tuple + +try: + import pymongo + from pymongo.errors import ConnectionFailure, ServerSelectionTimeoutError + PYMONGO_AVAILABLE = True +except ImportError: + PYMONGO_AVAILABLE = False + +from corebrain.db.connector import DatabaseConnector + +class MongoDBConnector(DatabaseConnector): + """Conector optimizado para MongoDB""" + + def __init__(self, config: Dict[str, Any]): + """ + Inicializa el conector MongoDB con la configuración proporcionada. + + Args: + config: Diccionario con la configuración de conexión + """ + super().__init__(config) + self.client = None + self.db = None + self.config = config + self.connection_timeout = 30 # segundos + + if not PYMONGO_AVAILABLE: + print("Advertencia: pymongo no está instalado. Instálalo con 'pip install pymongo'") + + def connect(self) -> bool: + """ + Establece conexión con timeout optimizado + + Returns: + True si la conexión fue exitosa, False en caso contrario + """ + if not PYMONGO_AVAILABLE: + raise ImportError("pymongo no está instalado. Instálalo con 'pip install pymongo'") + + try: + start_time = time.time() + + # Construir los parámetros de conexión + if "connection_string" in self.config: + connection_string = self.config["connection_string"] + # Añadir timeout a la cadena de conexión si no está presente + if "connectTimeoutMS=" not in connection_string: + if "?" in connection_string: + connection_string += "&connectTimeoutMS=10000" # 10 segundos + else: + connection_string += "?connectTimeoutMS=10000" + + # Crear cliente MongoDB con la cadena de conexión + self.client = pymongo.MongoClient(connection_string) + else: + # Diccionario de parámetros para MongoClient + mongo_params = { + "host": self.config.get("host", "localhost"), + "port": int(self.config.get("port", 27017)), + "connectTimeoutMS": 10000, # 10 segundos + "serverSelectionTimeoutMS": 10000 + } + + # Añadir credenciales solo si están presentes + if self.config.get("user"): + mongo_params["username"] = self.config.get("user") + if self.config.get("password"): + mongo_params["password"] = self.config.get("password") + + # Opcionalmente añadir opciones de autenticación + if self.config.get("auth_source"): + mongo_params["authSource"] = self.config.get("auth_source") + if self.config.get("auth_mechanism"): + mongo_params["authMechanism"] = self.config.get("auth_mechanism") + + # Crear cliente MongoDB con parámetros + self.client = pymongo.MongoClient(**mongo_params) + + # Verificar que la conexión funciona + self.client.admin.command('ping') + + # Seleccionar la base de datos + db_name = self.config.get("database", "") + if not db_name: + # Si no hay base de datos especificada, listar las disponibles + db_names = self.client.list_database_names() + if not db_names: + raise ValueError("No se encontraron bases de datos disponibles") + + # Seleccionar la primera que no sea de sistema + system_dbs = ["admin", "local", "config"] + for name in db_names: + if name not in system_dbs: + db_name = name + break + + # Si no encontramos ninguna que no sea de sistema, usar la primera + if not db_name: + db_name = db_names[0] + + print(f"No se especificó base de datos. Usando '{db_name}'") + + # Guardar la referencia a la base de datos + self.db = self.client[db_name] + return True + + except (ConnectionFailure, ServerSelectionTimeoutError) as e: + # Si es un error de timeout, reintentar + if time.time() - start_time < self.connection_timeout: + print(f"Timeout al conectar a MongoDB: {str(e)}. Reintentando...") + time.sleep(2) # Esperar antes de reintentar + return self.connect() + else: + print(f"Error de conexión a MongoDB después de {self.connection_timeout}s: {str(e)}") + self.close() + return False + except Exception as e: + print(f"Error al conectar a MongoDB: {str(e)}") + self.close() + return False + + def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] = None, + progress_callback: Optional[Callable] = None) -> Dict[str, Any]: + """ + Extrae el esquema con límites y progreso para mejorar rendimiento + + Args: + sample_limit: Número máximo de documentos de muestra por colección + collection_limit: Límite de colecciones a procesar (None para todas) + progress_callback: Función opcional para reportar progreso + + Returns: + Diccionario con el esquema de la base de datos + """ + # Asegurar que estamos conectados + if not self.client and not self.connect(): + return {"type": "mongodb", "tables": {}, "tables_list": []} + + # Inicializar el esquema + schema = { + "type": "mongodb", + "database": self.db.name, + "tables": {} # En MongoDB, las "tablas" son colecciones + } + + try: + # Obtener la lista de colecciones + collections = self.db.list_collection_names() + + # Limitar colecciones si es necesario + if collection_limit is not None and collection_limit > 0: + collections = collections[:collection_limit] + + # Procesar cada colección + total_collections = len(collections) + for i, collection_name in enumerate(collections): + # Reportar progreso si hay callback + if progress_callback: + progress_callback(i, total_collections, f"Procesando colección {collection_name}") + + collection = self.db[collection_name] + + try: + # Contar documentos + doc_count = collection.count_documents({}) + + if doc_count > 0: + # Obtener muestra de documentos + sample_docs = list(collection.find().limit(sample_limit)) + + # Extraer campos y sus tipos + fields = {} + for doc in sample_docs: + self._extract_document_fields(doc, fields) + + # Convertir a formato esperado + formatted_fields = [{"name": field, "type": type_name} for field, type_name in fields.items()] + + # Procesar documentos para sample_data + sample_data = [] + for doc in sample_docs: + processed_doc = self._process_document_for_serialization(doc) + sample_data.append(processed_doc) + + # Guardar en el esquema + schema["tables"][collection_name] = { + "fields": formatted_fields, + "sample_data": sample_data, + "count": doc_count + } + else: + # Colección vacía + schema["tables"][collection_name] = { + "fields": [], + "sample_data": [], + "count": 0, + "empty": True + } + + except Exception as e: + print(f"Error al procesar colección {collection_name}: {str(e)}") + schema["tables"][collection_name] = { + "fields": [], + "error": str(e) + } + + # Crear la lista de tablas/colecciones para compatibilidad + table_list = [] + for collection_name, collection_info in schema["tables"].items(): + table_data = {"name": collection_name} + table_data.update(collection_info) + table_list.append(table_data) + + # Guardar también la lista de tablas para compatibilidad + schema["tables_list"] = table_list + + return schema + + except Exception as e: + print(f"Error al extraer el esquema MongoDB: {str(e)}") + return {"type": "mongodb", "tables": {}, "tables_list": []} + + def _extract_document_fields(self, doc: Dict[str, Any], fields: Dict[str, str], + prefix: str = "", max_depth: int = 3, current_depth: int = 0) -> None: + """ + Extrae recursivamente los campos y tipos de un documento MongoDB. + + Args: + doc: Documento a analizar + fields: Diccionario donde guardar los campos y tipos + prefix: Prefijo para campos anidados + max_depth: Profundidad máxima para campos anidados + current_depth: Profundidad actual + """ + if current_depth >= max_depth: + return + + for field, value in doc.items(): + # Para _id y otros campos especiales + if field == "_id": + field_type = "ObjectId" + elif isinstance(value, dict): + if current_depth < max_depth - 1: + # Recursión para campos anidados + self._extract_document_fields(value, fields, + f"{prefix}{field}.", max_depth, current_depth + 1) + field_type = "object" + elif isinstance(value, list): + if value and current_depth < max_depth - 1: + # Si tenemos elementos en la lista, analizar el primero + if isinstance(value[0], dict): + self._extract_document_fields(value[0], fields, + f"{prefix}{field}[].", max_depth, current_depth + 1) + else: + # Para listas de tipos primitivos + field_type = f"array<{type(value[0]).__name__}>" + else: + field_type = "array" + else: + field_type = type(value).__name__ + + # Guardar el tipo del campo actual + field_key = f"{prefix}{field}" + if field_key not in fields: + fields[field_key] = field_type + + def _process_document_for_serialization(self, doc: Dict[str, Any]) -> Dict[str, Any]: + """ + Procesa un documento para ser serializable a JSON. + + Args: + doc: Documento a procesar + + Returns: + Documento procesado + """ + processed_doc = {} + for field, value in doc.items(): + # Convertir ObjectId a string + if field == "_id": + processed_doc[field] = str(value) + # Manejar objetos anidados + elif isinstance(value, dict): + processed_doc[field] = self._process_document_for_serialization(value) + # Manejar arrays + elif isinstance(value, list): + processed_items = [] + for item in value: + if isinstance(item, dict): + processed_items.append(self._process_document_for_serialization(item)) + elif hasattr(item, "__str__"): + processed_items.append(str(item)) + else: + processed_items.append(item) + processed_doc[field] = processed_items + # Convertir fechas a ISO + elif hasattr(value, 'isoformat'): + processed_doc[field] = value.isoformat() + # Otros tipos de datos + else: + processed_doc[field] = value + + return processed_doc + + def execute_query(self, query: str) -> List[Dict[str, Any]]: + """ + Ejecuta una consulta MongoDB con manejo de errores mejorado + + Args: + query: Consulta MongoDB en formato JSON o lenguaje de consulta + + Returns: + Lista de documentos resultantes + """ + if not self.client and not self.connect(): + raise ConnectionError("No se pudo establecer conexión con MongoDB") + + try: + # Determinar si la consulta es un string JSON o una consulta en otro formato + filter_dict, projection, collection_name, limit = self._parse_query(query) + + # Obtener la colección + if not collection_name: + raise ValueError("No se especificó el nombre de la colección en la consulta") + + collection = self.db[collection_name] + + # Ejecutar la consulta + if projection: + cursor = collection.find(filter_dict, projection).limit(limit or 100) + else: + cursor = collection.find(filter_dict).limit(limit or 100) + + # Convertir los resultados a formato serializable + results = [] + for doc in cursor: + processed_doc = self._process_document_for_serialization(doc) + results.append(processed_doc) + + return results + + except Exception as e: + # Intentar reconectar y reintentar una vez + try: + self.close() + if self.connect(): + print("Reconectando y reintentando consulta...") + + # Reintentar la consulta + filter_dict, projection, collection_name, limit = self._parse_query(query) + collection = self.db[collection_name] + + if projection: + cursor = collection.find(filter_dict, projection).limit(limit or 100) + else: + cursor = collection.find(filter_dict).limit(limit or 100) + + results = [] + for doc in cursor: + processed_doc = self._process_document_for_serialization(doc) + results.append(processed_doc) + + return results + except Exception as retry_error: + # Si falla el reintento, propagar el error original + raise Exception(f"Error al ejecutar consulta MongoDB: {str(e)}") + + # Si llegamos aquí, ha habido un error en el reintento + raise Exception(f"Error al ejecutar consulta MongoDB (después de reconexión): {str(e)}") + + def _parse_query(self, query: str) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]], str, Optional[int]]: + """ + Analiza una consulta y extrae los componentes necesarios. + + Args: + query: Consulta en formato string + + Returns: + Tupla con (filtro, proyección, nombre de colección, límite) + """ + # Intentar parsear como JSON + try: + query_dict = json.loads(query) + + # Extraer componentes de la consulta + filter_dict = query_dict.get("filter", {}) + projection = query_dict.get("projection") + collection_name = query_dict.get("collection") + limit = query_dict.get("limit") + + return filter_dict, projection, collection_name, limit + + except json.JSONDecodeError: + # Si no es JSON válido, intentar parsear el formato de consulta alternativo + collection_match = re.search(r'from\s+([a-zA-Z0-9_]+)', query, re.IGNORECASE) + collection_name = collection_match.group(1) if collection_match else None + + # Intentar extraer filtros + filter_match = re.search(r'where\s+(.+?)(?:limit|$)', query, re.IGNORECASE | re.DOTALL) + filter_str = filter_match.group(1).strip() if filter_match else "{}" + + # Intentar parsear los filtros como JSON + try: + filter_dict = json.loads(filter_str) + except json.JSONDecodeError: + # Si no se puede parsear, usar filtro vacío + filter_dict = {} + + # Extraer límite si existe + limit_match = re.search(r'limit\s+(\d+)', query, re.IGNORECASE) + limit = int(limit_match.group(1)) if limit_match else None + + return filter_dict, None, collection_name, limit + + def count_documents(self, collection_name: str, filter_dict: Optional[Dict[str, Any]] = None) -> int: + """ + Cuenta documentos en una colección + + Args: + collection_name: Nombre de la colección + filter_dict: Filtro opcional + + Returns: + Número de documentos + """ + if not self.client and not self.connect(): + raise ConnectionError("No se pudo establecer conexión con MongoDB") + + try: + collection = self.db[collection_name] + return collection.count_documents(filter_dict or {}) + except Exception as e: + print(f"Error al contar documentos: {str(e)}") + return 0 + + def list_collections(self) -> List[str]: + """ + Devuelve una lista de colecciones en la base de datos + + Returns: + Lista de nombres de colecciones + """ + if not self.client and not self.connect(): + raise ConnectionError("No se pudo establecer conexión con MongoDB") + + try: + return self.db.list_collection_names() + except Exception as e: + print(f"Error al listar colecciones: {str(e)}") + return [] + + def close(self) -> None: + """Cierra la conexión a MongoDB""" + if self.client: + try: + self.client.close() + except: + pass + finally: + self.client = None + self.db = None + + def __del__(self): + """Destructor para asegurar que la conexión se cierre""" + self.close() \ No newline at end of file diff --git a/corebrain/db/connectors/sql.py b/corebrain/db/connectors/sql.py new file mode 100644 index 0000000..36f3f89 --- /dev/null +++ b/corebrain/db/connectors/sql.py @@ -0,0 +1,598 @@ +""" +Conector para bases de datos SQL. +""" +import sqlite3 +import time +from typing import Dict, Any, List, Optional, Callable + +try: + import mysql.connector +except ImportError: + pass + +try: + import psycopg2 + import psycopg2.extras +except ImportError: + pass + +from corebrain.db.connector import DatabaseConnector + +class SQLConnector(DatabaseConnector): + """Conector optimizado para bases de datos SQL""" + + def __init__(self, config: Dict[str, Any]): + """ + Inicializa el conector SQL con la configuración proporcionada. + + Args: + config: Diccionario con la configuración de conexión + """ + super().__init__(config) + self.conn = None + self.cursor = None + self.engine = config.get("engine", "").lower() + self.config = config + self.connection_timeout = 30 # segundos + + def connect(self) -> bool: + """ + Establece conexión con timeout optimizado + + Returns: + True si la conexión fue exitosa, False en caso contrario + """ + try: + start_time = time.time() + + # Intentar la conexión con un límite de tiempo + while time.time() - start_time < self.connection_timeout: + try: + if self.engine == "sqlite": + if "connection_string" in self.config: + self.conn = sqlite3.connect(self.config["connection_string"], timeout=10.0) + else: + self.conn = sqlite3.connect(self.config.get("database", ""), timeout=10.0) + + # Configurar para que devuelva filas como diccionarios + self.conn.row_factory = sqlite3.Row + + elif self.engine == "mysql": + if "connection_string" in self.config: + self.conn = mysql.connector.connect( + connection_string=self.config["connection_string"], + connection_timeout=10 + ) + else: + self.conn = mysql.connector.connect( + host=self.config.get("host", "localhost"), + user=self.config.get("user", ""), + password=self.config.get("password", ""), + database=self.config.get("database", ""), + port=self.config.get("port", 3306), + connection_timeout=10 + ) + + elif self.engine == "postgresql": + # Determinar si usar cadena de conexión o parámetros + if "connection_string" in self.config: + # Agregar timeout a la cadena de conexión si no está presente + conn_str = self.config["connection_string"] + if "connect_timeout" not in conn_str: + if "?" in conn_str: + conn_str += "&connect_timeout=10" + else: + conn_str += "?connect_timeout=10" + + self.conn = psycopg2.connect(conn_str) + else: + self.conn = psycopg2.connect( + host=self.config.get("host", "localhost"), + user=self.config.get("user", ""), + password=self.config.get("password", ""), + dbname=self.config.get("database", ""), + port=self.config.get("port", 5432), + connect_timeout=10 + ) + + # Si llegamos aquí, la conexión fue exitosa + if self.conn: + # Verificar conexión con una consulta simple + cursor = self.conn.cursor() + cursor.execute("SELECT 1") + cursor.close() + return True + + except (sqlite3.Error, mysql.connector.Error, psycopg2.Error) as e: + # Si el error no es de timeout, propagar la excepción + if "timeout" not in str(e).lower() and "tiempo de espera" not in str(e).lower(): + raise + + # Si es un error de timeout, esperamos un poco y reintentamos + time.sleep(1.0) + + # Si llegamos aquí, se agotó el tiempo de espera + raise TimeoutError(f"No se pudo conectar a la base de datos en {self.connection_timeout} segundos") + + except Exception as e: + if self.conn: + try: + self.conn.close() + except: + pass + self.conn = None + + print(f"Error al conectar a la base de datos: {str(e)}") + return False + + def extract_schema(self, sample_limit: int = 5, table_limit: Optional[int] = None, + progress_callback: Optional[Callable] = None) -> Dict[str, Any]: + """ + Extrae el esquema con límites y progreso + + Args: + sample_limit: Límite de muestras de datos por tabla + table_limit: Límite de tablas a procesar (None para todas) + progress_callback: Función opcional para reportar progreso + + Returns: + Diccionario con el esquema de la base de datos + """ + # Asegurar que estamos conectados + if not self.conn and not self.connect(): + return {"type": "sql", "tables": {}, "tables_list": []} + + # Inicializar esquema + schema = { + "type": "sql", + "engine": self.engine, + "database": self.config.get("database", ""), + "tables": {} + } + + # Seleccionar la función extractora según el motor + if self.engine == "sqlite": + return self._extract_sqlite_schema(sample_limit, table_limit, progress_callback) + elif self.engine == "mysql": + return self._extract_mysql_schema(sample_limit, table_limit, progress_callback) + elif self.engine == "postgresql": + return self._extract_postgresql_schema(sample_limit, table_limit, progress_callback) + else: + return schema # Esquema vacío si no se reconoce el motor + + def execute_query(self, query: str) -> List[Dict[str, Any]]: + """ + Ejecuta una consulta SQL con manejo de errores mejorado + + Args: + query: Consulta SQL a ejecutar + + Returns: + Lista de filas resultantes como diccionarios + """ + if not self.conn and not self.connect(): + raise ConnectionError("No se pudo establecer conexión con la base de datos") + + try: + # Ejecutar query según el motor + if self.engine == "sqlite": + return self._execute_sqlite_query(query) + elif self.engine == "mysql": + return self._execute_mysql_query(query) + elif self.engine == "postgresql": + return self._execute_postgresql_query(query) + else: + raise ValueError(f"Motor de base de datos no soportado: {self.engine}") + + except Exception as e: + # Intentar reconectar y reintentar una vez + try: + self.close() + if self.connect(): + print("Reconectando y reintentando consulta...") + + if self.engine == "sqlite": + return self._execute_sqlite_query(query) + elif self.engine == "mysql": + return self._execute_mysql_query(query) + elif self.engine == "postgresql": + return self._execute_postgresql_query(query) + + except Exception as retry_error: + # Si falla el reintento, propagar el error original + raise Exception(f"Error al ejecutar consulta: {str(e)}") + + # Si llegamos aquí sin retornar, ha habido un error en el reintento + raise Exception(f"Error al ejecutar consulta (después de reconexión): {str(e)}") + + def _execute_sqlite_query(self, query: str) -> List[Dict[str, Any]]: + """Ejecuta una consulta en SQLite""" + cursor = self.conn.cursor() + cursor.execute(query) + + # Convertir filas a diccionarios + columns = [desc[0] for desc in cursor.description] if cursor.description else [] + rows = cursor.fetchall() + result = [] + + for row in rows: + row_dict = {} + for i, column in enumerate(columns): + row_dict[column] = row[i] + result.append(row_dict) + + cursor.close() + return result + + def _execute_mysql_query(self, query: str) -> List[Dict[str, Any]]: + """Ejecuta una consulta en MySQL""" + cursor = self.conn.cursor(dictionary=True) + cursor.execute(query) + result = cursor.fetchall() + cursor.close() + return result + + def _execute_postgresql_query(self, query: str) -> List[Dict[str, Any]]: + """Ejecuta una consulta en PostgreSQL""" + cursor = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + cursor.execute(query) + results = [dict(row) for row in cursor.fetchall()] + cursor.close() + return results + + def _extract_sqlite_schema(self, sample_limit: int, table_limit: Optional[int], progress_callback: Optional[Callable]) -> Dict[str, Any]: + """ + Extrae schema específico para SQLite + + Args: + sample_limit: Número máximo de filas de muestra por tabla + table_limit: Número máximo de tablas a extraer + progress_callback: Función para reportar progreso + + Returns: + Diccionario con el esquema de la base de datos + """ + schema = { + "type": "sql", + "engine": "sqlite", + "database": self.config.get("database", ""), + "tables": {} + } + + try: + cursor = self.conn.cursor() + + # Obtener la lista de tablas + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;") + tables = [row[0] for row in cursor.fetchall()] + + # Limitar tablas si es necesario + if table_limit is not None and table_limit > 0: + tables = tables[:table_limit] + + # Procesar cada tabla + total_tables = len(tables) + for i, table_name in enumerate(tables): + # Reportar progreso si hay callback + if progress_callback: + progress_callback(i, total_tables, f"Procesando tabla {table_name}") + + # Extraer información de columnas + cursor.execute(f"PRAGMA table_info({table_name});") + columns = [{"name": col[1], "type": col[2]} for col in cursor.fetchall()] + + # Guardar información básica de la tabla + schema["tables"][table_name] = { + "columns": columns, + "sample_data": [] + } + + # Obtener muestra de datos + try: + cursor.execute(f"SELECT * FROM {table_name} LIMIT {sample_limit};") + + # Obtener nombres de columnas + col_names = [desc[0] for desc in cursor.description] + + # Procesar las filas + sample_data = [] + for row in cursor.fetchall(): + row_dict = {} + for j, value in enumerate(row): + # Convertir a string los valores que no son serializable directamente + if isinstance(value, (bytes, bytearray)): + row_dict[col_names[j]] = f"" + else: + row_dict[col_names[j]] = value + sample_data.append(row_dict) + + schema["tables"][table_name]["sample_data"] = sample_data + + except Exception as e: + print(f"Error al obtener muestra de datos para tabla {table_name}: {str(e)}") + + cursor.close() + + except Exception as e: + print(f"Error al extraer esquema SQLite: {str(e)}") + + # Crear la lista de tablas para compatibilidad + table_list = [] + for table_name, table_info in schema["tables"].items(): + table_data = {"name": table_name} + table_data.update(table_info) + table_list.append(table_data) + + schema["tables_list"] = table_list + return schema + + def _extract_mysql_schema(self, sample_limit: int, table_limit: Optional[int], progress_callback: Optional[Callable]) -> Dict[str, Any]: + """ + Extrae schema específico para MySQL + + Args: + sample_limit: Número máximo de filas de muestra por tabla + table_limit: Número máximo de tablas a extraer + progress_callback: Función para reportar progreso + + Returns: + Diccionario con el esquema de la base de datos + """ + schema = { + "type": "sql", + "engine": "mysql", + "database": self.config.get("database", ""), + "tables": {} + } + + try: + cursor = self.conn.cursor(dictionary=True) + + # Obtener la lista de tablas + cursor.execute("SHOW TABLES;") + tables_result = cursor.fetchall() + tables = [] + + # Extraer nombres de tablas (el formato puede variar según versión) + for row in tables_result: + if len(row) == 1: # Si es una lista simple + tables.extend(row.values()) + else: # Si tiene estructura compleja + for value in row.values(): + if isinstance(value, str): + tables.append(value) + break + + # Limitar tablas si es necesario + if table_limit is not None and table_limit > 0: + tables = tables[:table_limit] + + # Procesar cada tabla + total_tables = len(tables) + for i, table_name in enumerate(tables): + # Reportar progreso si hay callback + if progress_callback: + progress_callback(i, total_tables, f"Procesando tabla {table_name}") + + # Extraer información de columnas + cursor.execute(f"DESCRIBE `{table_name}`;") + columns = [{"name": col.get("Field"), "type": col.get("Type")} for col in cursor.fetchall()] + + # Guardar información básica de la tabla + schema["tables"][table_name] = { + "columns": columns, + "sample_data": [] + } + + # Obtener muestra de datos + try: + cursor.execute(f"SELECT * FROM `{table_name}` LIMIT {sample_limit};") + sample_data = cursor.fetchall() + + # Procesar valores que no son JSON serializable + processed_samples = [] + for row in sample_data: + processed_row = {} + for key, value in row.items(): + if isinstance(value, (bytes, bytearray)): + processed_row[key] = f"" + elif hasattr(value, 'isoformat'): # Para fechas y horas + processed_row[key] = value.isoformat() + else: + processed_row[key] = value + processed_samples.append(processed_row) + + schema["tables"][table_name]["sample_data"] = processed_samples + + except Exception as e: + print(f"Error al obtener muestra de datos para tabla {table_name}: {str(e)}") + + cursor.close() + + except Exception as e: + print(f"Error al extraer esquema MySQL: {str(e)}") + + # Crear la lista de tablas para compatibilidad + table_list = [] + for table_name, table_info in schema["tables"].items(): + table_data = {"name": table_name} + table_data.update(table_info) + table_list.append(table_data) + + schema["tables_list"] = table_list + return schema + + def _extract_postgresql_schema(self, sample_limit: int, table_limit: Optional[int], progress_callback: Optional[Callable]) -> Dict[str, Any]: + """ + Extrae schema específico para PostgreSQL con optimizaciones + + Args: + sample_limit: Número máximo de filas de muestra por tabla + table_limit: Número máximo de tablas a extraer + progress_callback: Función para reportar progreso + + Returns: + Diccionario con el esquema de la base de datos + """ + schema = { + "type": "sql", + "engine": "postgresql", + "database": self.config.get("database", ""), + "tables": {} + } + + try: + cursor = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + + # Estrategia 1: Buscar en todos los esquemas accesibles + cursor.execute(""" + SELECT table_schema, table_name + FROM information_schema.tables + WHERE table_schema NOT IN ('pg_catalog', 'information_schema') + AND table_type = 'BASE TABLE' + ORDER BY table_schema, table_name; + """) + tables = cursor.fetchall() + + # Si no se encontraron tablas, intentar estrategia alternativa + if not tables: + cursor.execute(""" + SELECT schemaname AS table_schema, tablename AS table_name + FROM pg_tables + WHERE schemaname NOT IN ('pg_catalog', 'information_schema') + ORDER BY schemaname, tablename; + """) + tables = cursor.fetchall() + + # Si aún no hay tablas, intentar buscar en esquemas específicos + if not tables: + cursor.execute(""" + SELECT DISTINCT table_schema + FROM information_schema.tables + ORDER BY table_schema; + """) + schemas = cursor.fetchall() + + # Intentar con esquemas que no sean del sistema + user_schemas = [s[0] for s in schemas if s[0] not in ('pg_catalog', 'information_schema')] + for schema_name in user_schemas: + cursor.execute(f""" + SELECT '{schema_name}' AS table_schema, table_name + FROM information_schema.tables + WHERE table_schema = '{schema_name}' + AND table_type = 'BASE TABLE'; + """) + schema_tables = cursor.fetchall() + if schema_tables: + tables.extend(schema_tables) + + # Limitar tablas si es necesario + if table_limit is not None and table_limit > 0: + tables = tables[:table_limit] + + # Procesar cada tabla + total_tables = len(tables) + for i, (schema_name, table_name) in enumerate(tables): + # Reportar progreso si hay callback + if progress_callback: + progress_callback(i, total_tables, f"Procesando tabla {schema_name}.{table_name}") + + # Determinar el nombre completo de la tabla + full_name = f"{schema_name}.{table_name}" if schema_name != 'public' else table_name + + # Extraer información de columnas + cursor.execute(f""" + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_schema = '{schema_name}' AND table_name = '{table_name}' + ORDER BY ordinal_position; + """) + + columns_data = cursor.fetchall() + if columns_data: + columns = [{"name": col[0], "type": col[1]} for col in columns_data] + schema["tables"][full_name] = {"columns": columns, "sample_data": []} + + # Obtener muestra de datos + try: + cursor.execute(f""" + SELECT * FROM "{schema_name}"."{table_name}" LIMIT {sample_limit}; + """) + rows = cursor.fetchall() + + # Obtener nombres de columnas + col_names = [desc[0] for desc in cursor.description] + + # Convertir filas a diccionarios + sample_data = [] + for row in rows: + row_dict = {} + for j, value in enumerate(row): + # Convertir a formato serializable + if hasattr(value, 'isoformat'): # Para fechas y horas + row_dict[col_names[j]] = value.isoformat() + elif isinstance(value, (bytes, bytearray)): + row_dict[col_names[j]] = f"" + else: + row_dict[col_names[j]] = str(value) if value is not None else None + sample_data.append(row_dict) + + schema["tables"][full_name]["sample_data"] = sample_data + + except Exception as e: + print(f"Error al obtener muestra de datos para tabla {full_name}: {str(e)}") + else: + # Registrar la tabla aunque no tenga columnas + schema["tables"][full_name] = {"columns": [], "sample_data": []} + + cursor.close() + + except Exception as e: + print(f"Error al extraer esquema PostgreSQL: {str(e)}") + + # Intento de recuperación para diagnosticar problemas + try: + if self.conn and self.conn.closed == 0: # 0 = conexión abierta + recovery_cursor = self.conn.cursor() + + # Verificar versión + recovery_cursor.execute("SELECT version();") + version = recovery_cursor.fetchone() + print(f"Versión PostgreSQL: {version[0] if version else 'Desconocida'}") + + # Verificar permisos + recovery_cursor.execute(""" + SELECT has_schema_privilege(current_user, 'public', 'USAGE') AS has_usage, + has_schema_privilege(current_user, 'public', 'CREATE') AS has_create; + """) + perms = recovery_cursor.fetchone() + if perms: + print(f"Permisos en esquema public: USAGE={perms[0]}, CREATE={perms[1]}") + + recovery_cursor.close() + except Exception as diag_err: + print(f"Error durante el diagnóstico: {str(diag_err)}") + + # Crear la lista de tablas para compatibilidad + table_list = [] + for table_name, table_info in schema["tables"].items(): + table_data = {"name": table_name} + table_data.update(table_info) + table_list.append(table_data) + + schema["tables_list"] = table_list + return schema + + def close(self) -> None: + """Cierra la conexión a la base de datos""" + if self.conn: + try: + self.conn.close() + except: + pass + finally: + self.conn = None + + def __del__(self): + """Destructor para asegurar que la conexión se cierre""" + self.close() \ No newline at end of file diff --git a/corebrain/db/engines.py b/corebrain/db/engines.py new file mode 100644 index 0000000..9b9c866 --- /dev/null +++ b/corebrain/db/engines.py @@ -0,0 +1,16 @@ +""" +Información sobre motores de bases de datos soportados. +""" +from typing import Dict, List + +def get_available_engines() -> Dict[str, List[str]]: + """ + Devuelve los motores de base de datos disponibles por tipo + + Returns: + Dict con tipos de DB y lista de motores por tipo + """ + return { + "sql": ["sqlite", "mysql", "postgresql"], + "nosql": ["mongodb"] + } \ No newline at end of file diff --git a/corebrain/db/factory.py b/corebrain/db/factory.py new file mode 100644 index 0000000..afbc0dd --- /dev/null +++ b/corebrain/db/factory.py @@ -0,0 +1,29 @@ +""" +Fábrica de conectores de base de datos. +""" +from typing import Dict, Any + +from corebrain.db.connector import DatabaseConnector +from corebrain.db.connectors.sql import SQLConnector +from corebrain.db.connectors.mongodb import MongoDBConnector + +def get_connector(db_config: Dict[str, Any], timeout: int = 10) -> DatabaseConnector: + """ + Fábrica de conectores de base de datos según la configuración + + Args: + db_config: Configuración de la base de datos + timeout: Timeout para operaciones de DB + + Returns: + Instancia de conector apropiado + """ + db_type = db_config.get("type", "").lower() + engine = db_config.get("engine", "").lower() + + if db_type == "sql": + return SQLConnector(db_config, timeout) + elif db_type in ["nosql", "mongodb"] or engine == "mongodb": + return MongoDBConnector(db_config, timeout) + else: + raise ValueError(f"Tipo de base de datos no soportado: {db_type}") \ No newline at end of file diff --git a/corebrain/db/interface.py b/corebrain/db/interface.py new file mode 100644 index 0000000..d2b3ae5 --- /dev/null +++ b/corebrain/db/interface.py @@ -0,0 +1,36 @@ +""" +Interfaces abstractas para conexiones de bases de datos. +""" +from typing import Dict, Any, List, Optional, Protocol +from abc import ABC, abstractmethod + +from corebrain.core.common import ConfigDict, SchemaDict + +class DatabaseConnector(ABC): + """Interfaz abstracta para conectores de bases de datos""" + + @abstractmethod + def connect(self, config: ConfigDict) -> Any: + """Establece conexión con la base de datos""" + pass + + @abstractmethod + def extract_schema(self, connection: Any) -> SchemaDict: + """Extrae el esquema de la base de datos""" + pass + + @abstractmethod + def execute_query(self, connection: Any, query: str) -> List[Dict[str, Any]]: + """Ejecuta una consulta y devuelve resultados""" + pass + + @abstractmethod + def close(self, connection: Any) -> None: + """Cierra la conexión""" + pass + +# Posteriormente se podrían implementar conectores específicos: +# - SQLiteConnector +# - MySQLConnector +# - PostgresConnector +# - MongoDBConnector \ No newline at end of file diff --git a/corebrain/db/schema/__init__.py b/corebrain/db/schema/__init__.py new file mode 100644 index 0000000..25d843c --- /dev/null +++ b/corebrain/db/schema/__init__.py @@ -0,0 +1,11 @@ +""" +Componentes para extracción y optimización de esquemas de base de datos. +""" +from .extractor import extract_schema +from .optimizer import SchemaOptimizer + +# Alias para compatibilidad con código existente +extract_db_schema = extract_schema +schemaOptimizer = SchemaOptimizer + +__all__ = ['extract_schema', 'extract_db_schema', 'schemaOptimizer'] \ No newline at end of file diff --git a/corebrain/db/schema/extractor.py b/corebrain/db/schema/extractor.py new file mode 100644 index 0000000..3051951 --- /dev/null +++ b/corebrain/db/schema/extractor.py @@ -0,0 +1,123 @@ +# db/schema/extractor.py (reemplaza la importación circular en db/schema.py) + +""" +Extractor de esquemas de bases de datos independiente. +""" + +from typing import Dict, Any, Optional, Callable + +from corebrain.utils.logging import get_logger + +logger = get_logger(__name__) + +def extract_db_schema(db_config: Dict[str, Any], client_factory: Optional[Callable] = None) -> Dict[str, Any]: + """ + Extrae el esquema de la base de datos con inyección de dependencias. + + Args: + db_config: Configuración de la base de datos + client_factory: Función opcional para crear un cliente (evita importación circular) + + Returns: + Diccionario con la estructura de la base de datos + """ + db_type = db_config.get("type", "").lower() + schema = { + "type": db_type, + "database": db_config.get("database", ""), + "tables": {}, + "tables_list": [] + } + + try: + # Si tenemos un cliente especializado, usarlo + if client_factory: + # La factoría crea un cliente y extrae el esquema + client = client_factory(db_config) + return client.extract_schema() + + # Extracción directa sin usar cliente de Corebrain + if db_type == "sql": + # Código para bases de datos SQL (sin dependencias circulares) + engine = db_config.get("engine", "").lower() + if engine == "sqlite": + # Extraer esquema SQLite + import sqlite3 + # (implementación...) + elif engine == "mysql": + # Extraer esquema MySQL + import mysql.connector + # (implementación...) + elif engine == "postgresql": + # Extraer esquema PostgreSQL + import psycopg2 + # (implementación...) + + elif db_type in ["nosql", "mongodb"]: + # Extraer esquema MongoDB + import pymongo + # (implementación...) + + # Convertir diccionario a lista para compatibilidad + table_list = [] + for table_name, table_info in schema["tables"].items(): + table_data = {"name": table_name} + table_data.update(table_info) + table_list.append(table_data) + + schema["tables_list"] = table_list + return schema + + except Exception as e: + logger.error(f"Error al extraer esquema: {str(e)}") + return {"type": db_type, "tables": {}, "tables_list": []} + + +def create_schema_from_corebrain() -> Callable: + """ + Crea una función de extracción que usa Corebrain internamente. + Carga dinámicamente para evitar importaciones circulares. + + Returns: + Función que extrae schema usando Corebrain + """ + def extract_with_corebrain(db_config: Dict[str, Any]) -> Dict[str, Any]: + # Importar dinámicamente para evitar circular + from corebrain.core.client import Corebrain + + # Crear cliente temporal solo para extraer el schema + try: + client = Corebrain( + api_token="temp_token", + db_config=db_config, + skip_verification=True + ) + schema = client.db_schema + client.close() + return schema + except Exception as e: + logger.error(f"Error al extraer schema con Corebrain: {str(e)}") + return {"type": db_config.get("type", ""), "tables": {}, "tables_list": []} + + return extract_with_corebrain + + +# Función pública expuesta +def extract_schema(db_config: Dict[str, Any], use_corebrain: bool = False) -> Dict[str, Any]: + """ + Función pública que decide cómo extraer el schema. + + Args: + db_config: Configuración de la base de datos + use_corebrain: Si es True, usa la clase Corebrain para extracción + + Returns: + Esquema de la base de datos + """ + if use_corebrain: + # Intentar usar Corebrain si se solicita + factory = create_schema_from_corebrain() + return extract_db_schema(db_config, client_factory=factory) + else: + # Usar extracción directa sin dependencias circulares + return extract_db_schema(db_config) \ No newline at end of file diff --git a/corebrain/db/schema/optimizer.py b/corebrain/db/schema/optimizer.py new file mode 100644 index 0000000..c85f286 --- /dev/null +++ b/corebrain/db/schema/optimizer.py @@ -0,0 +1,157 @@ +""" +Componentes para optimización de esquemas de base de datos. +""" +import re +from typing import Dict, Any, Optional + +from corebrain.utils.logging import get_logger + +logger = get_logger(__name__) + +class SchemaOptimizer: + """Optimiza el esquema de la base de datos para reducir tamaño de contexto.""" + + def __init__(self, max_tables: int = 10, max_columns_per_table: int = 15, max_samples: int = 2): + """ + Inicializa el optimizador de esquema. + + Args: + max_tables: Máximo número de tablas a incluir + max_columns_per_table: Máximo número de columnas por tabla + max_samples: Máximo número de filas de muestra por tabla + """ + self.max_tables = max_tables + self.max_columns_per_table = max_columns_per_table + self.max_samples = max_samples + + # Tablas importantes que siempre deben incluirse si existen + self.priority_tables = set([ + "users", "customers", "products", "orders", "transactions", + "invoices", "accounts", "clients", "employees", "services" + ]) + + # Tablas típicamente menos importantes + self.low_priority_tables = set([ + "logs", "sessions", "tokens", "temp", "cache", "metrics", + "statistics", "audit", "history", "archives", "settings" + ]) + + def optimize_schema(self, db_schema: Dict[str, Any], query: str = None) -> Dict[str, Any]: + """ + Optimiza el esquema para reducir su tamaño. + + Args: + db_schema: Esquema original de la base de datos + query: Consulta del usuario (para priorizar tablas relevantes) + + Returns: + Esquema optimizado + """ + # Crear copia para no modificar el original + optimized_schema = { + "type": db_schema.get("type", ""), + "database": db_schema.get("database", ""), + "engine": db_schema.get("engine", ""), + "tables": {}, + "tables_list": [] + } + + # Determinar tablas relevantes para la consulta + query_relevant_tables = set() + if query: + # Extraer potenciales nombres de tablas de la consulta + normalized_query = query.lower() + + # Obtener nombres de todas las tablas + all_table_names = [ + name.lower() for name in db_schema.get("tables", {}).keys() + ] + + # Buscar menciones a tablas en la consulta + for table_name in all_table_names: + # Buscar el nombre exacto (como palabra completa) + if re.search(r'\b' + re.escape(table_name) + r'\b', normalized_query): + query_relevant_tables.add(table_name) + + # También buscar formas singulares/plurales simples + if table_name.endswith('s') and re.search(r'\b' + re.escape(table_name[:-1]) + r'\b', normalized_query): + query_relevant_tables.add(table_name) + elif not table_name.endswith('s') and re.search(r'\b' + re.escape(table_name + 's') + r'\b', normalized_query): + query_relevant_tables.add(table_name) + + # Priorizar tablas a incluir + table_scores = {} + for table_name in db_schema.get("tables", {}): + score = 0 + + # Tablas mencionadas en la consulta tienen máxima prioridad + if table_name.lower() in query_relevant_tables: + score += 100 + + # Tablas importantes + if table_name.lower() in self.priority_tables: + score += 50 + + # Tablas poco importantes + if table_name.lower() in self.low_priority_tables: + score -= 30 + + # Tablas con más columnas pueden ser más relevantes + table_info = db_schema["tables"].get(table_name, {}) + column_count = len(table_info.get("columns", [])) + score += min(column_count, 20) # Limitar a 20 puntos máximo + + # Guardar puntuación + table_scores[table_name] = score + + # Ordenar tablas por puntuación + sorted_tables = sorted(table_scores.items(), key=lambda x: x[1], reverse=True) + + # Limitar número de tablas + selected_tables = [name for name, _ in sorted_tables[:self.max_tables]] + + # Copiar tablas seleccionadas con optimizaciones + for table_name in selected_tables: + table_info = db_schema["tables"].get(table_name, {}) + + # Optimizar columnas + columns = table_info.get("columns", []) + if len(columns) > self.max_columns_per_table: + # Mantener las columnas más importantes (id, nombre, clave primaria, etc) + important_columns = [] + other_columns = [] + + for col in columns: + col_name = col.get("name", "").lower() + if col_name in ["id", "uuid", "name", "key", "code"] or "id" in col_name: + important_columns.append(col) + else: + other_columns.append(col) + + # Tomar las columnas importantes y completar con otras hasta el límite + optimized_columns = important_columns + remaining_slots = self.max_columns_per_table - len(optimized_columns) + if remaining_slots > 0: + optimized_columns.extend(other_columns[:remaining_slots]) + else: + optimized_columns = columns + + # Optimizar datos de muestra + sample_data = table_info.get("sample_data", []) + optimized_samples = sample_data[:self.max_samples] if sample_data else [] + + # Guardar tabla optimizada + optimized_schema["tables"][table_name] = { + "columns": optimized_columns, + "sample_data": optimized_samples + } + + # Añadir a la lista de tablas + optimized_schema["tables_list"].append({ + "name": table_name, + "columns": optimized_columns, + "sample_data": optimized_samples + }) + + return optimized_schema + diff --git a/corebrain/db/schema_file.py b/corebrain/db/schema_file.py new file mode 100644 index 0000000..e13806c --- /dev/null +++ b/corebrain/db/schema_file.py @@ -0,0 +1,583 @@ +""" +Componentes para extracción y optimización de esquemas de base de datos. +""" +import json + +from typing import Dict, Any, Optional + +def _print_colored(message: str, color: str) -> None: + """Versión simplificada de _print_colored que no depende de cli.utils""" + colors = { + "red": "\033[91m", + "green": "\033[92m", + "yellow": "\033[93m", + "blue": "\033[94m", + "default": "\033[0m" + } + color_code = colors.get(color, colors["default"]) + print(f"{color_code}{message}{colors['default']}") + +def extract_db_schema(db_config: Dict[str, Any]) -> Dict[str, Any]: + """ + Extrae el esquema de la base de datos directamente sin usar el SDK. + + Args: + db_config: Configuración de la base de datos + + Returns: + Diccionario con la estructura de la base de datos organizada por tablas/colecciones + """ + db_type = db_config["type"].lower() + schema = { + "type": db_type, + "database": db_config.get("database", ""), + "tables": {} # Cambiado a diccionario para facilitar el acceso directo a tablas por nombre + } + + try: + if db_type == "sql": + # Código para bases de datos SQL... + # [Se mantiene igual] + pass + + # Manejar tanto "nosql" como "mongodb" como tipos válidos + elif db_type == "nosql" or db_type == "mongodb": + import pymongo + + # Determinar el motor (si existe) + engine = db_config.get("engine", "").lower() + + # Si no se especifica el engine o es mongodb, proceder + if not engine or engine == "mongodb": + if "connection_string" in db_config: + client = pymongo.MongoClient(db_config["connection_string"]) + else: + # Diccionario de parámetros para MongoClient + mongo_params = { + "host": db_config.get("host", "localhost"), + "port": db_config.get("port", 27017) + } + + # Añadir credenciales solo si están presentes + if db_config.get("user"): + mongo_params["username"] = db_config["user"] + if db_config.get("password"): + mongo_params["password"] = db_config["password"] + + client = pymongo.MongoClient(**mongo_params) + + # Obtener la base de datos + db_name = db_config.get("database", "") + if not db_name: + _print_colored("⚠️ Nombre de base de datos no especificado", "yellow") + return schema + + try: + db = client[db_name] + collection_names = db.list_collection_names() + + # Procesar colecciones + for collection_name in collection_names: + collection = db[collection_name] + + # Obtener varios documentos de muestra + try: + sample_docs = list(collection.find().limit(5)) + + # Extraer estructura de campos a partir de los documentos + field_types = {} + + for doc in sample_docs: + for field, value in doc.items(): + if field != "_id": # Ignoramos el _id de MongoDB + # Actualizar el tipo si no existe o combinar si hay diferentes tipos + field_type = type(value).__name__ + if field not in field_types: + field_types[field] = field_type + elif field_types[field] != field_type: + field_types[field] = f"{field_types[field]}|{field_type}" + + # Convertir a formato esperado + fields = [{"name": field, "type": type_name} for field, type_name in field_types.items()] + + # Convertir documentos a formato serializable + sample_data = [] + for doc in sample_docs: + serialized_doc = {} + for key, value in doc.items(): + if key == "_id": + serialized_doc[key] = str(value) + elif isinstance(value, (dict, list)): + serialized_doc[key] = str(value) # Simplificar objetos anidados + else: + serialized_doc[key] = value + sample_data.append(serialized_doc) + + # Guardar información de la colección + schema["tables"][collection_name] = { + "fields": fields, + "sample_data": sample_data + } + except Exception as e: + _print_colored(f"Error al procesar colección {collection_name}: {str(e)}", "red") + schema["tables"][collection_name] = { + "fields": [], + "sample_data": [], + "error": str(e) + } + + except Exception as e: + _print_colored(f"Error al acceder a la base de datos MongoDB '{db_name}': {str(e)}", "red") + + finally: + # Cerrar la conexión + client.close() + else: + _print_colored(f"Motor de base de datos NoSQL no soportado: {engine}", "red") + + # Convertir el diccionario de tablas en una lista para mantener compatibilidad con el formato anterior + table_list = [] + for table_name, table_info in schema["tables"].items(): + table_data = {"name": table_name} + table_data.update(table_info) + table_list.append(table_data) + + # Guardar también la lista de tablas para mantener compatibilidad + schema["tables_list"] = table_list + + return schema + + except Exception as e: + _print_colored(f"Error al extraer el esquema de la base de datos: {str(e)}", "red") + # En caso de error, devolver un esquema vacío + return {"type": db_type, "tables": {}, "tables_list": []} + +def extract_db_schema_direct(db_config: Dict[str, Any]) -> Dict[str, Any]: + """ + Extrae el esquema directamente sin usar el cliente de Corebrain. + Esta es una versión reducida que no requiere importar core. + """ + db_type = db_config["type"].lower() + schema = { + "type": db_type, + "database": db_config.get("database", ""), + "tables": {}, + "tables_list": [] # Lista inicialmente vacía + } + + try: + # [Implementación existente para extraer esquema sin usar Corebrain] + # ... + + return schema + except Exception as e: + _print_colored(f"Error al extraer esquema directamente: {str(e)}", "red") + return {"type": db_type, "tables": {}, "tables_list": []} + +def extract_schema_with_lazy_init(api_key: str, db_config: Dict[str, Any], api_url: Optional[str] = None) -> Dict[str, Any]: + """ + Extrae esquema usando importación tardía del cliente. + + Esta función evita el problema de importación circular cargando + dinámicamente el cliente de Corebrain solo cuando es necesario. + """ + try: + # La importación se mueve aquí para evitar el problema de circular import + # Solo se ejecuta cuando realmente necesitamos crear el cliente + import importlib + core_module = importlib.import_module('core') + init_func = getattr(core_module, 'init') + + # Crear cliente con la configuración + api_url_to_use = api_url or "https://api.corebrain.com" + cb = init_func( + api_token=api_key, + db_config=db_config, + api_url=api_url_to_use, + skip_verification=True # No necesitamos verificar token para extraer schema + ) + + # Obtener el esquema y cerrar cliente + schema = cb.db_schema + cb.close() + + return schema + + except Exception as e: + _print_colored(f"Error al extraer esquema con cliente: {str(e)}", "red") + # Como alternativa, usar extracción directa sin cliente + return extract_db_schema_direct(db_config) + +def extract_schema_to_file(api_key: str, config_id: Optional[str] = None, output_file: Optional[str] = None, api_url: Optional[str] = None) -> bool: + """ + Extrae el esquema de la base de datos y lo guarda en un archivo. + + Args: + api_key: API Key para identificar la configuración + config_id: ID de configuración específico (opcional) + output_file: Ruta al archivo donde guardar el esquema + api_url: URL opcional de la API + + Returns: + True si se extrae correctamente, False en caso contrario + """ + try: + # Importación explícita con try-except para manejar errores + try: + from corebrain.config.manager import ConfigManager + except ImportError as e: + _print_colored(f"Error al importar ConfigManager: {e}", "red") + return False + + # Obtener las configuraciones disponibles + config_manager = ConfigManager() + configs = config_manager.list_configs(api_key) + + if not configs: + _print_colored("No hay configuraciones guardadas para esta API Key.", "yellow") + return False + + selected_config_id = config_id + + # Si no se especifica un config_id, mostrar lista para seleccionar + if not selected_config_id: + _print_colored("\n=== Configuraciones disponibles ===", "blue") + for i, conf_id in enumerate(configs, 1): + print(f"{i}. {conf_id}") + + try: + choice = int(input(f"\nSelecciona una configuración (1-{len(configs)}): ").strip()) + if 1 <= choice <= len(configs): + selected_config_id = configs[choice - 1] + else: + _print_colored("Opción inválida.", "red") + return False + except ValueError: + _print_colored("Por favor, introduce un número válido.", "red") + return False + + # Verificar que el config_id exista + if selected_config_id not in configs: + _print_colored(f"No se encontró la configuración con ID: {selected_config_id}", "red") + return False + + # Obtener la configuración seleccionada + db_config = config_manager.get_config(api_key, selected_config_id) + + if not db_config: + _print_colored(f"Error al obtener la configuración con ID: {selected_config_id}", "red") + return False + + _print_colored(f"\nExtrayendo esquema para configuración: {selected_config_id}", "blue") + print(f"Tipo: {db_config['type'].upper()}, Motor: {db_config.get('engine', 'No especificado').upper()}") + print(f"Base de datos: {db_config.get('database', 'No especificada')}") + + # Extraer el esquema de la base de datos + _print_colored("\nExtrayendo esquema de la base de datos...", "blue") + schema = extract_schema_with_lazy_init(api_key, db_config, api_url) + + # Verificar si se obtuvo un esquema válido + if not schema or not schema.get("tables"): + _print_colored("No se encontraron tablas/colecciones en la base de datos.", "yellow") + return False + + # Guardar el esquema en un archivo + output_path = output_file or "db_schema.json" + try: + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(schema, f, indent=2, default=str) + _print_colored(f"✅ Esquema extraído y guardado en: {output_path}", "green") + except Exception as e: + _print_colored(f"❌ Error al guardar el archivo: {str(e)}", "red") + return False + + # Mostrar un resumen de las tablas/colecciones encontradas + tables = schema.get("tables", {}) + _print_colored(f"\nResumen del esquema extraído: {len(tables)} tablas/colecciones", "green") + + for table_name in tables: + print(f"- {table_name}") + + return True + + except Exception as e: + _print_colored(f"❌ Error al extraer esquema: {str(e)}", "red") + return False + +def show_db_schema(api_token: str, config_id: Optional[str] = None, api_url: Optional[str] = None) -> None: + """ + Muestra el esquema de la base de datos configurada. + + Args: + api_token: Token de API + config_id: ID de configuración específico (opcional) + api_url: URL opcional de la API + """ + try: + # Importación explícita con try-except para manejar errores + try: + from corebrain.config.manager import ConfigManager + except ImportError as e: + _print_colored(f"Error al importar ConfigManager: {e}", "red") + return False + + # Obtener las configuraciones disponibles + config_manager = ConfigManager() + configs = config_manager.list_configs(api_token) + + if not configs: + _print_colored("No hay configuraciones guardadas para este token.", "yellow") + return + + selected_config_id = config_id + + # Si no se especifica un config_id, mostrar lista para seleccionar + if not selected_config_id: + _print_colored("\n=== Configuraciones disponibles ===", "blue") + for i, conf_id in enumerate(configs, 1): + print(f"{i}. {conf_id}") + + try: + choice = int(input(f"\nSelecciona una configuración (1-{len(configs)}): ").strip()) + if 1 <= choice <= len(configs): + selected_config_id = configs[choice - 1] + else: + _print_colored("Opción inválida.", "red") + return + except ValueError: + _print_colored("Por favor, introduce un número válido.", "red") + return + + # Verificar que el config_id exista + if selected_config_id not in configs: + _print_colored(f"No se encontró la configuración con ID: {selected_config_id}", "red") + return + + if config_id and config_id in configs: + db_config = config_manager.get_config(api_token, config_id) + else: + # Obtener la configuración seleccionada + db_config = config_manager.get_config(api_token, selected_config_id) + + if not db_config: + _print_colored(f"Error al obtener la configuración con ID: {selected_config_id}", "red") + return + + _print_colored(f"\nObteniendo esquema para configuración: {selected_config_id}", "blue") + _print_colored("Tipo de base de datos:", "blue") + print(f" {db_config['type'].upper()}") + + if db_config.get('engine'): + _print_colored("Motor:", "blue") + print(f" {db_config['engine'].upper()}") + + _print_colored("Base de datos:", "blue") + print(f" {db_config.get('database', 'No especificada')}") + + # Extraer y mostrar el esquema + _print_colored("\nExtrayendo esquema de la base de datos...", "blue") + + # Intenta conectarse a la base de datos y extraer el esquema + try: + + # Creamos una instancia de Corebrain con la configuración seleccionada + """ + cb = init( + api_token=api_token, + config_id=selected_config_id, + api_url=api_url, + skip_verification=True # Omitimos verificación para simplificar + ) + """ + + import importlib + core_module = importlib.import_module('core.client') + init_func = getattr(core_module, 'init') + + # Creamos una instancia de Corebrain con la configuración seleccionada + cb = init_func( + api_token=api_token, + config_id=config_id, + api_url=api_url, + skip_verification=True # Omitimos verificación para simplificar + ) + + # El esquema se extrae automáticamente al inicializar + schema = get_schema_with_dynamic_import( + api_token=api_token, + config_id=selected_config_id, + db_config=db_config, + api_url=api_url + ) + + # Si no hay esquema, intentamos extraerlo explícitamente + if not schema or not schema.get("tables"): + _print_colored("Intentando extraer esquema explícitamente...", "yellow") + schema = cb._extract_db_schema() + + # Cerramos la conexión + cb.close() + + except Exception as conn_error: + _print_colored(f"Error de conexión: {str(conn_error)}", "red") + print("Intentando método alternativo...") + + # Método alternativo: usar función extract_db_schema directamente + schema = extract_db_schema(db_config) + + # Verificar si se obtuvo un esquema válido + if not schema or not schema.get("tables"): + _print_colored("No se encontraron tablas/colecciones en la base de datos.", "yellow") + + # Información adicional para ayudar a diagnosticar el problema + print("\nInformación de depuración:") + print(f" Tipo de base de datos: {db_config.get('type', 'No especificado')}") + print(f" Motor: {db_config.get('engine', 'No especificado')}") + print(f" Host: {db_config.get('host', 'No especificado')}") + print(f" Puerto: {db_config.get('port', 'No especificado')}") + print(f" Base de datos: {db_config.get('database', 'No especificado')}") + + # Para PostgreSQL, sugerir verificar el esquema + if db_config.get('engine') == 'postgresql': + print("\nPara PostgreSQL, verifica que las tablas existan en el esquema 'public' o") + print("que tengas acceso a los esquemas donde están las tablas.") + print("Puedes verificar los esquemas disponibles con: SELECT DISTINCT table_schema FROM information_schema.tables;") + + return + + # Mostrar información del esquema + tables = schema.get("tables", {}) + + # Separar tablas SQL y colecciones NoSQL para mostrarlas apropiadamente + sql_tables = {} + nosql_collections = {} + + for name, info in tables.items(): + if "columns" in info: + sql_tables[name] = info + elif "fields" in info: + nosql_collections[name] = info + + # Mostrar tablas SQL + if sql_tables: + _print_colored(f"\nSe encontraron {len(sql_tables)} tablas SQL:", "green") + for table_name, table_info in sql_tables.items(): + _print_colored(f"\n=== Tabla: {table_name} ===", "bold") + + # Mostrar columnas + columns = table_info.get("columns", []) + if columns: + _print_colored("Columnas:", "blue") + for column in columns: + print(f" - {column['name']} ({column['type']})") + else: + _print_colored("No se encontraron columnas.", "yellow") + + # Mostrar muestra de datos si está disponible + sample_data = table_info.get("sample_data", []) + if sample_data: + _print_colored("\nMuestra de datos:", "blue") + for i, row in enumerate(sample_data[:2], 1): # Limitar a 2 filas para simplificar + print(f" Registro {i}: {row}") + + if len(sample_data) > 2: + print(f" ... ({len(sample_data) - 2} registros más)") + + # Mostrar colecciones NoSQL + if nosql_collections: + _print_colored(f"\nSe encontraron {len(nosql_collections)} colecciones NoSQL:", "green") + for coll_name, coll_info in nosql_collections.items(): + _print_colored(f"\n=== Colección: {coll_name} ===", "bold") + + # Mostrar campos + fields = coll_info.get("fields", []) + if fields: + _print_colored("Campos:", "blue") + for field in fields: + print(f" - {field['name']} ({field['type']})") + else: + _print_colored("No se encontraron campos.", "yellow") + + # Mostrar muestra de datos si está disponible + sample_data = coll_info.get("sample_data", []) + if sample_data: + _print_colored("\nMuestra de datos:", "blue") + for i, doc in enumerate(sample_data[:2], 1): # Limitar a 2 documentos + # Simplificar la visualización para documentos grandes + if isinstance(doc, dict) and len(doc) > 5: + simplified = {k: doc[k] for k in list(doc.keys())[:5]} + print(f" Documento {i}: {simplified} ... (y {len(doc) - 5} campos más)") + else: + print(f" Documento {i}: {doc}") + + if len(sample_data) > 2: + print(f" ... ({len(sample_data) - 2} documentos más)") + + _print_colored("\n✅ Esquema extraído correctamente!", "green") + + # Preguntar si quiere guardar el esquema en un archivo + save_option = input("\n¿Deseas guardar el esquema en un archivo? (s/n): ").strip().lower() + if save_option == "s": + filename = input("Nombre del archivo (por defecto: db_schema.json): ").strip() or "db_schema.json" + try: + with open(filename, 'w') as f: + json.dump(schema, f, indent=2, default=str) + _print_colored(f"\n✅ Esquema guardado en: {filename}", "green") + except Exception as e: + _print_colored(f"❌ Error al guardar el archivo: {str(e)}", "red") + + except Exception as e: + _print_colored(f"❌ Error al mostrar el esquema: {str(e)}", "red") + import traceback + traceback.print_exc() + + +def get_schema_with_dynamic_import(api_token: str, config_id: str, db_config: Dict[str, Any], api_url: Optional[str] = None) -> Dict[str, Any]: + """ + Obtiene el esquema de la base de datos usando importación dinámica. + + Args: + api_token: Token de API + config_id: ID de configuración + db_config: Configuración de la base de datos + api_url: URL opcional de la API + + Returns: + Esquema de la base de datos + """ + try: + # Importación dinámica del módulo core + import importlib + core_module = importlib.import_module('core.client') + init_func = getattr(core_module, 'init') + + # Creamos una instancia de Corebrain con la configuración seleccionada + cb = init_func( + api_token=api_token, + config_id=config_id, + api_url=api_url, + skip_verification=True # Omitimos verificación para simplificar + ) + + # El esquema se extrae automáticamente al inicializar + schema = cb.db_schema + + # Si no hay esquema, intentamos extraerlo explícitamente + if not schema or not schema.get("tables"): + _print_colored("Intentando extraer esquema explícitamente...", "yellow") + schema = cb._extract_db_schema() + + # Cerramos la conexión + cb.close() + + return schema + + except ImportError: + # Si falla la importación dinámica, intentamos un enfoque alternativo + _print_colored("No se pudo importar el cliente. Usando método alternativo.", "yellow") + return extract_db_schema(db_config) + + except Exception as e: + _print_colored(f"Error al extraer esquema con cliente: {str(e)}", "red") + # Fallback a extracción directa + return extract_db_schema(db_config) \ No newline at end of file diff --git a/corebrain/network/__init__.py b/corebrain/network/__init__.py new file mode 100644 index 0000000..98413c5 --- /dev/null +++ b/corebrain/network/__init__.py @@ -0,0 +1,22 @@ +""" +Componentes de red para Corebrain SDK. + +Este paquete proporciona utilidades y clientes para comunicación +con la API de Corebrain y otros servicios web. +""" +from corebrain.network.client import ( + APIClient, + APIError, + APITimeoutError, + APIConnectionError, + APIAuthError +) + +# Exportación explícita de componentes públicos +__all__ = [ + 'APIClient', + 'APIError', + 'APITimeoutError', + 'APIConnectionError', + 'APIAuthError' +] \ No newline at end of file diff --git a/corebrain/network/client.py b/corebrain/network/client.py new file mode 100644 index 0000000..54e6b02 --- /dev/null +++ b/corebrain/network/client.py @@ -0,0 +1,502 @@ +""" +Cliente HTTP para comunicación con la API de Corebrain. +""" +import time +import logging +import httpx + +from typing import Dict, Any, Optional, List +from urllib.parse import urljoin +from httpx import Response, ConnectError, ReadTimeout, WriteTimeout, PoolTimeout + +logger = logging.getLogger(__name__) +http_session = httpx.Client(timeout=10.0, verify=True) + +def __init__(self, verbose=False): + self.verbose = verbose + +class APIError(Exception): + """Error genérico en la API.""" + def __init__(self, message: str, status_code: Optional[int] = None, + detail: Optional[str] = None, response: Optional[Response] = None): + self.message = message + self.status_code = status_code + self.detail = detail + self.response = response + super().__init__(message) + +class APITimeoutError(APIError): + """Error de timeout en la API.""" + pass + +class APIConnectionError(APIError): + """Error de conexión a la API.""" + pass + +class APIAuthError(APIError): + """Error de autenticación en la API.""" + pass + +class APIClient: + """Cliente HTTP optimizado para comunicación con la API de Corebrain.""" + + # Constantes para manejo de reintentos y errores + MAX_RETRIES = 3 + RETRY_DELAY = 0.5 # segundos + RETRY_STATUS_CODES = [408, 429, 500, 502, 503, 504] + + def __init__(self, base_url: str, default_timeout: int = 10, + verify_ssl: bool = True, user_agent: Optional[str] = None): + """ + Inicializa el cliente API con configuración optimizada. + + Args: + base_url: URL base para todas las peticiones + default_timeout: Tiempo de espera predeterminado en segundos + verify_ssl: Si se debe verificar el certificado SSL + user_agent: Agente de usuario personalizado + """ + # Normalizar URL base para asegurar que termina con '/' + self.base_url = base_url if base_url.endswith('/') else base_url + '/' + self.default_timeout = default_timeout + self.verify_ssl = verify_ssl + + # Headers predeterminados + self.default_headers = { + 'User-Agent': user_agent or 'CorebrainSDK/1.0', + 'Accept': 'application/json', + 'Content-Type': 'application/json' + } + + # Crear sesión HTTP con límites y timeouts optimizados + self.session = httpx.Client( + timeout=httpx.Timeout(timeout=default_timeout), + verify=verify_ssl, + http2=True, # Usar HTTP/2 si está disponible + limits=httpx.Limits(max_connections=100, max_keepalive_connections=20) + ) + + # Estadísticas y métricas + self.request_count = 0 + self.error_count = 0 + self.total_request_time = 0 + + logger.debug(f"Cliente API inicializado con base_url={base_url}, timeout={default_timeout}s") + + def __del__(self): + """Asegurar que la sesión se cierre al eliminar el cliente.""" + self.close() + + def close(self): + """Cierra la sesión HTTP.""" + if hasattr(self, 'session') and self.session: + try: + self.session.close() + logger.debug("Sesión HTTP cerrada correctamente") + except Exception as e: + logger.warning(f"Error al cerrar sesión HTTP: {e}") + + def get_full_url(self, endpoint: str) -> str: + """ + Construye la URL completa para un endpoint. + + Args: + endpoint: Ruta relativa del endpoint + + Returns: + URL completa + """ + # Eliminar '/' inicial si existe para evitar rutas duplicadas + endpoint = endpoint.lstrip('/') + return urljoin(self.base_url, endpoint) + + def prepare_headers(self, headers: Optional[Dict[str, str]] = None, + auth_token: Optional[str] = None) -> Dict[str, str]: + """ + Prepara los headers para una petición. + + Args: + headers: Headers adicionales + auth_token: Token de autenticación + + Returns: + Headers combinados + """ + # Comenzar con headers predeterminados + final_headers = self.default_headers.copy() + + # Añadir headers personalizados + if headers: + final_headers.update(headers) + + # Añadir token de autenticación si se proporciona + if auth_token: + final_headers['Authorization'] = f'Bearer {auth_token}' + + return final_headers + + def handle_response(self, response: Response) -> Response: + """ + Procesa la respuesta para manejar errores comunes. + + Args: + response: Respuesta HTTP + + Returns: + La misma respuesta si no hay errores + + Raises: + APIError: Si hay errores en la respuesta + """ + status_code = response.status_code + + # Procesar errores según código de estado + if 400 <= status_code < 500: + error_detail = None + + # Intentar extraer detalles del error del cuerpo JSON + try: + json_data = response.json() + if isinstance(json_data, dict): + error_detail = ( + json_data.get('detail') or + json_data.get('message') or + json_data.get('error') + ) + except Exception: + # Si no podemos parsear JSON, usar el texto completo + error_detail = response.text[:200] + ('...' if len(response.text) > 200 else '') + + # Errores específicos según código + if status_code == 401: + msg = "Error de autenticación: token inválido o expirado" + logger.error(f"{msg} - {error_detail or ''}") + raise APIAuthError(msg, status_code, error_detail, response) + + elif status_code == 403: + msg = "Acceso prohibido: no tienes permisos suficientes" + logger.error(f"{msg} - {error_detail or ''}") + raise APIAuthError(msg, status_code, error_detail, response) + + elif status_code == 404: + msg = f"Recurso no encontrado: {response.url}" + logger.error(msg) + raise APIError(msg, status_code, error_detail, response) + + elif status_code == 429: + msg = "Demasiadas peticiones: límite de tasa excedido" + logger.warning(msg) + raise APIError(msg, status_code, error_detail, response) + + else: + msg = f"Error del cliente ({status_code}): {error_detail or 'sin detalles'}" + logger.error(msg) + raise APIError(msg, status_code, error_detail, response) + + elif 500 <= status_code < 600: + msg = f"Error del servidor ({status_code}): el servidor API encontró un error" + logger.error(msg) + raise APIError(msg, status_code, response.text[:200], response) + + return response + + def request(self, method: str, endpoint: str, *, + headers: Optional[Dict[str, str]] = None, + json: Optional[Any] = None, + data: Optional[Any] = None, + params: Optional[Dict[str, Any]] = None, + timeout: Optional[int] = None, + auth_token: Optional[str] = None, + retry: bool = True) -> Response: + """ + Realiza una petición HTTP con manejo de errores y reintentos. + + Args: + method: Método HTTP (GET, POST, etc.) + endpoint: Ruta relativa del endpoint + headers: Headers adicionales + json: Datos para enviar como JSON + data: Datos para enviar como form o bytes + params: Parámetros de query string + timeout: Tiempo de espera en segundos (sobreescribe el predeterminado) + auth_token: Token de autenticación + retry: Si se deben reintentar peticiones fallidas + + Returns: + Respuesta HTTP procesada + + Raises: + APIError: Si hay errores en la petición o respuesta + APITimeoutError: Si la petición excede el tiempo de espera + APIConnectionError: Si hay errores de conexión + """ + url = self.get_full_url(endpoint) + final_headers = self.prepare_headers(headers, auth_token) + + # Configurar timeout + request_timeout = timeout or self.default_timeout + + # Contador para reintentos + retries = 0 + last_error = None + + # Registrar inicio de la petición + start_time = time.time() + self.request_count += 1 + + while retries <= (self.MAX_RETRIES if retry else 0): + try: + if retries > 0: + # Esperar antes de reintentar con backoff exponencial + wait_time = self.RETRY_DELAY * (2 ** (retries - 1)) + logger.info(f"Reintentando petición ({retries}/{self.MAX_RETRIES}) a {url} después de {wait_time:.2f}s") + time.sleep(wait_time) + + # Realizar la petición + logger.debug(f"Enviando petición {method} a {url}") + response = self.session.request( + method=method, + url=url, + headers=final_headers, + json=json, + data=data, + params=params, + timeout=request_timeout + ) + + # Verificar si debemos reintentar por código de estado + if response.status_code in self.RETRY_STATUS_CODES and retry and retries < self.MAX_RETRIES: + logger.warning(f"Código de estado {response.status_code} recibido, reintentando") + retries += 1 + continue + + # Procesar la respuesta + processed_response = self.handle_response(response) + + # Registrar tiempo total + elapsed = time.time() - start_time + self.total_request_time += elapsed + logger.debug(f"Petición completada en {elapsed:.3f}s con estado {response.status_code}") + + return processed_response + + except (ConnectError, httpx.HTTPError) as e: + last_error = e + + # Decidir si reintentamos dependiendo del tipo de error + if isinstance(e, (ReadTimeout, WriteTimeout, PoolTimeout, ConnectError)) and retry and retries < self.MAX_RETRIES: + logger.warning(f"Error de conexión: {str(e)}, reintentando {retries+1}/{self.MAX_RETRIES}") + retries += 1 + continue + + # No más reintentos o error no recuperable + self.error_count += 1 + elapsed = time.time() - start_time + + if isinstance(e, (ReadTimeout, WriteTimeout, PoolTimeout)): + logger.error(f"Timeout en petición a {url} después de {elapsed:.3f}s: {str(e)}") + raise APITimeoutError(f"La petición a {endpoint} excedió el tiempo máximo de {request_timeout}s", + response=getattr(e, 'response', None)) + else: + logger.error(f"Error de conexión a {url} después de {elapsed:.3f}s: {str(e)}") + raise APIConnectionError(f"Error de conexión a {endpoint}: {str(e)}", + response=getattr(e, 'response', None)) + + except Exception as e: + # Error inesperado + self.error_count += 1 + elapsed = time.time() - start_time + logger.error(f"Error inesperado en petición a {url} después de {elapsed:.3f}s: {str(e)}") + raise APIError(f"Error inesperado en petición a {endpoint}: {str(e)}") + + # Si llegamos aquí es porque agotamos los reintentos + if last_error: + self.error_count += 1 + raise APIError(f"Petición a {endpoint} falló después de {retries} reintentos: {str(last_error)}") + + # Este punto nunca debería alcanzarse + raise APIError(f"Error inesperado en petición a {endpoint}") + + def get(self, endpoint: str, **kwargs) -> Response: + """Realiza una petición GET.""" + return self.request("GET", endpoint, **kwargs) + + def post(self, endpoint: str, **kwargs) -> Response: + """Realiza una petición POST.""" + return self.request("POST", endpoint, **kwargs) + + def put(self, endpoint: str, **kwargs) -> Response: + """Realiza una petición PUT.""" + return self.request("PUT", endpoint, **kwargs) + + def delete(self, endpoint: str, **kwargs) -> Response: + """Realiza una petición DELETE.""" + return self.request("DELETE", endpoint, **kwargs) + + def patch(self, endpoint: str, **kwargs) -> Response: + """Realiza una petición PATCH.""" + return self.request("PATCH", endpoint, **kwargs) + + def get_json(self, endpoint: str, **kwargs) -> Any: + """ + Realiza una petición GET y devuelve los datos JSON. + + Args: + endpoint: Endpoint a consultar + **kwargs: Argumentos adicionales para request() + + Returns: + Datos JSON parseados + """ + response = self.get(endpoint, **kwargs) + try: + return response.json() + except Exception as e: + raise APIError(f"Error al parsear respuesta JSON: {str(e)}", response=response) + + def post_json(self, endpoint: str, **kwargs) -> Any: + """ + Realiza una petición POST y devuelve los datos JSON. + + Args: + endpoint: Endpoint a consultar + **kwargs: Argumentos adicionales para request() + + Returns: + Datos JSON parseados + """ + response = self.post(endpoint, **kwargs) + try: + return response.json() + except Exception as e: + raise APIError(f"Error al parsear respuesta JSON: {str(e)}", response=response) + + # Métodos de alto nivel para operaciones comunes en la API de Corebrain + + def check_health(self, timeout: int = 5) -> bool: + """ + Comprueba si la API está disponible. + + Args: + timeout: Tiempo máximo de espera + + Returns: + True si la API está disponible + """ + try: + response = self.get("health", timeout=timeout, retry=False) + return response.status_code == 200 + except Exception: + return False + + def verify_token(self, token: str, timeout: int = 5) -> Dict[str, Any]: + """ + Verifica si un token es válido. + + Args: + token: Token a verificar + timeout: Tiempo máximo de espera + + Returns: + Información del usuario si el token es válido + + Raises: + APIAuthError: Si el token no es válido + """ + try: + response = self.get("api/auth/me", auth_token=token, timeout=timeout) + return response.json() + except APIAuthError: + raise + except Exception as e: + raise APIAuthError(f"Error al verificar token: {str(e)}") + + def get_api_keys(self, token: str) -> List[Dict[str, Any]]: + """ + Obtiene las API keys disponibles para un usuario. + + Args: + token: Token de autenticación + + Returns: + Lista de API keys + """ + return self.get_json("api/auth/api-keys", auth_token=token) + + def update_api_key_metadata(self, token: str, api_key: str, metadata: Dict[str, Any]) -> Dict[str, Any]: + """ + Actualiza los metadatos de una API key. + + Args: + token: Token de autenticación + api_key: ID de la API key + metadata: Metadatos a actualizar + + Returns: + Datos actualizados de la API key + """ + data = {"metadata": metadata} + return self.put_json(f"api/auth/api-keys/{api_key}", auth_token=token, json=data) + + def query_database(self, token: str, question: str, db_schema: Dict[str, Any], + config_id: str, timeout: int = 30) -> Dict[str, Any]: + """ + Realiza una consulta en lenguaje natural. + + Args: + token: Token de autenticación + question: Pregunta en lenguaje natural + db_schema: Esquema de la base de datos + config_id: ID de la configuración + timeout: Tiempo máximo de espera + + Returns: + Resultado de la consulta + """ + data = { + "question": question, + "db_schema": db_schema, + "config_id": config_id + } + return self.post_json("api/database/sdk/query", auth_token=token, json=data, timeout=timeout) + + def exchange_sso_token(self, sso_token: str, user_data: Dict[str, Any]) -> Dict[str, Any]: + """ + Intercambia un token SSO por un token API. + + Args: + sso_token: Token SSO + user_data: Datos del usuario + + Returns: + Datos del token API + """ + headers = {"Authorization": f"Bearer {sso_token}"} + data = {"user_data": user_data} + return self.post_json("api/auth/sso/token", headers=headers, json=data) + + # Métodos para estadísticas y diagnóstico + + def get_stats(self) -> Dict[str, Any]: + """ + Obtiene estadísticas de uso del cliente. + + Returns: + Estadísticas de peticiones + """ + avg_time = self.total_request_time / max(1, self.request_count) + error_rate = (self.error_count / max(1, self.request_count)) * 100 + + return { + "request_count": self.request_count, + "error_count": self.error_count, + "error_rate": f"{error_rate:.2f}%", + "total_request_time": f"{self.total_request_time:.3f}s", + "average_request_time": f"{avg_time:.3f}s", + } + + def reset_stats(self) -> None: + """Resetea las estadísticas de uso.""" + self.request_count = 0 + self.error_count = 0 + self.total_request_time = 0 \ No newline at end of file diff --git a/corebrain/sdk.py b/corebrain/sdk.py new file mode 100644 index 0000000..7b4ced1 --- /dev/null +++ b/corebrain/sdk.py @@ -0,0 +1,8 @@ +""" +SDK de Corebrain para compatibilidad. +""" +from corebrain.config.manager import ConfigManager + +# Re-exportar elementos principales +list_configurations = ConfigManager().list_configs +remove_configuration = ConfigManager().remove_config \ No newline at end of file diff --git a/corebrain/services/schema.py b/corebrain/services/schema.py new file mode 100644 index 0000000..5eb92ae --- /dev/null +++ b/corebrain/services/schema.py @@ -0,0 +1,31 @@ + +# Nuevo directorio: services/ +# Nuevo archivo: services/schema_service.py +""" +Servicios para manejo de esquemas de base de datos. +""" +from typing import Dict, Any, Optional + +from corebrain.config.manager import ConfigManager +from corebrain.db.schema import extract_db_schema, SchemaOptimizer + +class SchemaService: + """Servicio para operaciones de esquema de base de datos.""" + + def __init__(self): + self.config_manager = ConfigManager() + self.schema_optimizer = SchemaOptimizer() + + def get_schema(self, api_token: str, config_id: str) -> Optional[Dict[str, Any]]: + """Obtiene el esquema para una configuración específica.""" + config = self.config_manager.get_config(api_token, config_id) + if not config: + return None + + return extract_db_schema(config) + + def optimize_schema(self, schema: Dict[str, Any], query: str = None) -> Dict[str, Any]: + """Optimiza un esquema existente.""" + return self.schema_optimizer.optimize_schema(schema, query) + + # Otros métodos de servicio... \ No newline at end of file diff --git a/corebrain/utils/__init__.py b/corebrain/utils/__init__.py new file mode 100644 index 0000000..0ed2c33 --- /dev/null +++ b/corebrain/utils/__init__.py @@ -0,0 +1,66 @@ +""" +Utilidades generales para Corebrain SDK. + +Este paquete proporciona utilidades compartidas por diferentes +componentes del SDK, como serialización, cifrado y logging. +""" +import logging + +from corebrain.utils.serializer import serialize_to_json, JSONEncoder +from corebrain.utils.encrypter import ( + create_cipher, + generate_key, + derive_key_from_password, + ConfigEncrypter +) + +# Configuración de logging +logger = logging.getLogger('corebrain') + +def setup_logger(level=logging.INFO, + file_path=None, + format_string=None): + """ + Configura el logger principal de Corebrain. + + Args: + level: Nivel de logging + file_path: Ruta a archivo de log (opcional) + format_string: Formato de log personalizado + """ + # Formato predeterminado + fmt = format_string or '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + formatter = logging.Formatter(fmt) + + # Handler de consola + console_handler = logging.StreamHandler() + console_handler.setFormatter(formatter) + + # Configurar logger principal + logger.setLevel(level) + logger.addHandler(console_handler) + + # Handler de archivo si se proporciona ruta + if file_path: + file_handler = logging.FileHandler(file_path) + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + + # Mensajes de diagnóstico + logger.debug(f"Logger configurado con nivel {logging.getLevelName(level)}") + if file_path: + logger.debug(f"Logs escritos a {file_path}") + + return logger + +# Exportación explícita de componentes públicos +__all__ = [ + 'serialize_to_json', + 'JSONEncoder', + 'create_cipher', + 'generate_key', + 'derive_key_from_password', + 'ConfigEncrypter', + 'setup_logger', + 'logger' +] \ No newline at end of file diff --git a/corebrain/utils/encrypter.py b/corebrain/utils/encrypter.py new file mode 100644 index 0000000..1b4cfab --- /dev/null +++ b/corebrain/utils/encrypter.py @@ -0,0 +1,264 @@ +""" +Utilidades de cifrado para Corebrain SDK. +""" +import os +import base64 +import logging + +from pathlib import Path +from typing import Optional, Union +from cryptography.fernet import Fernet, InvalidToken +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC + +logger = logging.getLogger(__name__) + +def derive_key_from_password(password: Union[str, bytes], salt: Optional[bytes] = None) -> bytes: + """ + Deriva una clave de cifrado segura a partir de una contraseña y sal. + + Args: + password: Contraseña o frase de paso + salt: Sal criptográfica (se genera si no se proporciona) + + Returns: + Clave derivada en bytes + """ + if isinstance(password, str): + password = password.encode() + + # Generar sal si no se proporciona + if salt is None: + salt = os.urandom(16) + + # Derivar clave usando PBKDF2 + kdf = PBKDF2HMAC( + algorithm=hashes.SHA256(), + length=32, + salt=salt, + iterations=100000 # Mayor número de iteraciones = mayor seguridad + ) + + key = kdf.derive(password) + return base64.urlsafe_b64encode(key) + +def generate_key() -> str: + """ + Genera una nueva clave de cifrado aleatoria. + + Returns: + Clave de cifrado en formato base64 + """ + key = Fernet.generate_key() + return key.decode() + +def create_cipher(key: Optional[Union[str, bytes]] = None) -> Fernet: + """ + Crea un objeto de cifrado Fernet con la clave dada o genera una nueva. + + Args: + key: Clave de cifrado en formato base64 o None para generar + + Returns: + Objeto Fernet para cifrado/descifrado + """ + if key is None: + key = Fernet.generate_key() + elif isinstance(key, str): + key = key.encode() + + return Fernet(key) + +class ConfigEncrypter: + """ + Gestor de cifrado para configuraciones con manejo de claves. + """ + + def __init__(self, key_path: Optional[Union[str, Path]] = None): + """ + Inicializa el encriptador con ruta de clave opcional. + + Args: + key_path: Ruta al archivo de clave (si no existe, se creará) + """ + self.key_path = Path(key_path) if key_path else None + self.cipher = None + self._init_cipher() + + def _init_cipher(self) -> None: + """Inicializa el objeto de cifrado, creando o cargando la clave según sea necesario.""" + key = None + + # Si hay ruta de clave, intentar cargar o crear + if self.key_path: + try: + if self.key_path.exists(): + with open(self.key_path, 'rb') as f: + key = f.read().strip() + logger.debug(f"Clave cargada desde {self.key_path}") + else: + # Crear directorio padre si no existe + self.key_path.parent.mkdir(parents=True, exist_ok=True) + + # Generar nueva clave + key = Fernet.generate_key() + + # Guardar clave + with open(self.key_path, 'wb') as f: + f.write(key) + + # Asegurar permisos restrictivos (solo el propietario puede leer) + try: + os.chmod(self.key_path, 0o600) + except Exception as e: + logger.warning(f"No se pudieron establecer permisos en archivo de clave: {e}") + + logger.debug(f"Nueva clave generada y guardada en {self.key_path}") + except Exception as e: + logger.error(f"Error al gestionar clave en {self.key_path}: {e}") + # En caso de error, generar clave efímera + key = None + + # Si no tenemos clave, generar una efímera + if not key: + key = Fernet.generate_key() + logger.debug("Usando clave efímera generada") + + self.cipher = Fernet(key) + + def encrypt(self, data: Union[str, bytes]) -> bytes: + """ + Cifra datos. + + Args: + data: Datos a cifrar + + Returns: + Datos cifrados en bytes + """ + if isinstance(data, str): + data = data.encode() + + try: + return self.cipher.encrypt(data) + except Exception as e: + logger.error(f"Error al cifrar datos: {e}") + raise + + def decrypt(self, encrypted_data: Union[str, bytes]) -> bytes: + """ + Descifra datos. + + Args: + encrypted_data: Datos cifrados + + Returns: + Datos descifrados en bytes + """ + if isinstance(encrypted_data, str): + encrypted_data = encrypted_data.encode() + + try: + return self.cipher.decrypt(encrypted_data) + except InvalidToken: + logger.error("Token inválido o datos corruptos") + raise ValueError("Los datos no pueden ser descifrados: token inválido o datos corruptos") + except Exception as e: + logger.error(f"Error al descifrar datos: {e}") + raise + + def encrypt_file(self, input_path: Union[str, Path], output_path: Optional[Union[str, Path]] = None) -> Path: + """ + Cifra un archivo completo. + + Args: + input_path: Ruta al archivo a cifrar + output_path: Ruta para guardar el archivo cifrado (si es None, se añade .enc) + + Returns: + Ruta del archivo cifrado + """ + input_path = Path(input_path) + + if not output_path: + output_path = input_path.with_suffix(input_path.suffix + '.enc') + else: + output_path = Path(output_path) + + try: + with open(input_path, 'rb') as f: + data = f.read() + + encrypted_data = self.encrypt(data) + + with open(output_path, 'wb') as f: + f.write(encrypted_data) + + return output_path + except Exception as e: + logger.error(f"Error al cifrar archivo {input_path}: {e}") + raise + + def decrypt_file(self, input_path: Union[str, Path], output_path: Optional[Union[str, Path]] = None) -> Path: + """ + Descifra un archivo completo. + + Args: + input_path: Ruta al archivo cifrado + output_path: Ruta para guardar el archivo descifrado + + Returns: + Ruta del archivo descifrado + """ + input_path = Path(input_path) + + if not output_path: + # Si termina en .enc, quitar esa extensión + if input_path.suffix == '.enc': + output_path = input_path.with_suffix('') + else: + output_path = input_path.with_suffix(input_path.suffix + '.dec') + else: + output_path = Path(output_path) + + try: + with open(input_path, 'rb') as f: + encrypted_data = f.read() + + decrypted_data = self.decrypt(encrypted_data) + + with open(output_path, 'wb') as f: + f.write(decrypted_data) + + return output_path + except Exception as e: + logger.error(f"Error al descifrar archivo {input_path}: {e}") + raise + + @staticmethod + def generate_key_file(key_path: Union[str, Path]) -> None: + """ + Genera y guarda una nueva clave en un archivo. + + Args: + key_path: Ruta donde guardar la clave + """ + key_path = Path(key_path) + + # Crear directorio padre si no existe + key_path.parent.mkdir(parents=True, exist_ok=True) + + # Generar clave + key = Fernet.generate_key() + + # Guardar clave + with open(key_path, 'wb') as f: + f.write(key) + + # Establecer permisos restrictivos + try: + os.chmod(key_path, 0o600) + except Exception as e: + logger.warning(f"No se pudieron establecer permisos en archivo de clave: {e}") + + logger.info(f"Nueva clave generada y guardada en {key_path}") \ No newline at end of file diff --git a/corebrain/utils/logging.py b/corebrain/utils/logging.py new file mode 100644 index 0000000..82495a9 --- /dev/null +++ b/corebrain/utils/logging.py @@ -0,0 +1,243 @@ +""" +Utilidades de logging para Corebrain SDK. + +Este módulo proporciona funciones y clases para gestionar el logging +dentro del SDK de forma consistente. +""" +import logging +import sys +from datetime import datetime +from pathlib import Path +from typing import Optional, Any, Union + +# Niveles de logging personalizados +VERBOSE = 15 # Entre DEBUG e INFO + +# Configuración predeterminada +DEFAULT_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' +DEFAULT_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' +DEFAULT_LEVEL = logging.INFO +DEFAULT_LOG_DIR = Path.home() / ".corebrain" / "logs" + +# Colores para logging en terminal +LOG_COLORS = { + "DEBUG": "\033[94m", # Azul + "VERBOSE": "\033[96m", # Cian + "INFO": "\033[92m", # Verde + "WARNING": "\033[93m", # Amarillo + "ERROR": "\033[91m", # Rojo + "CRITICAL": "\033[95m", # Magenta + "RESET": "\033[0m" # Reset +} + +class VerboseLogger(logging.Logger): + """Logger personalizado con nivel VERBOSE.""" + + def verbose(self, msg: str, *args: Any, **kwargs: Any) -> None: + """ + Registra un mensaje con nivel VERBOSE. + + Args: + msg: Mensaje a registrar + *args: Argumentos para formatear el mensaje + **kwargs: Argumentos adicionales para el logger + """ + return self.log(VERBOSE, msg, *args, **kwargs) + +class ColoredFormatter(logging.Formatter): + """Formateador que agrega colores a los mensajes de log en terminal.""" + + def __init__(self, fmt: str = DEFAULT_FORMAT, datefmt: str = DEFAULT_DATE_FORMAT, + use_colors: bool = True): + """ + Inicializa el formateador. + + Args: + fmt: Formato del mensaje + datefmt: Formato de fecha + use_colors: Si es True, usa colores en terminal + """ + super().__init__(fmt, datefmt) + self.use_colors = use_colors and sys.stdout.isatty() + + def format(self, record: logging.LogRecord) -> str: + """ + Formatea un registro con colores. + + Args: + record: Registro a formatear + + Returns: + Mensaje formateado + """ + levelname = record.levelname + message = super().format(record) + + if self.use_colors and levelname in LOG_COLORS: + return f"{LOG_COLORS[levelname]}{message}{LOG_COLORS['RESET']}" + return message + +def setup_logger(name: str = "corebrain", + level: int = DEFAULT_LEVEL, + file_path: Optional[Union[str, Path]] = None, + format_string: Optional[str] = None, + use_colors: bool = True, + propagate: bool = False) -> logging.Logger: + """ + Configura un logger con opciones personalizadas. + + Args: + name: Nombre del logger + level: Nivel de logging + file_path: Ruta al archivo de log (opcional) + format_string: Formato de mensajes personalizado + use_colors: Si es True, usa colores en terminal + propagate: Si es True, propaga mensajes a loggers padre + + Returns: + Logger configurado + """ + # Registrar nivel personalizado VERBOSE + if not hasattr(logging, 'VERBOSE'): + logging.addLevelName(VERBOSE, 'VERBOSE') + + # Registrar clase de logger personalizada + logging.setLoggerClass(VerboseLogger) + + # Obtener o crear logger + logger = logging.getLogger(name) + + # Limpiar handlers existentes + for handler in logger.handlers[:]: + logger.removeHandler(handler) + + # Configurar nivel de logging + logger.setLevel(level) + logger.propagate = propagate + + # Formato predeterminado + fmt = format_string or DEFAULT_FORMAT + formatter = ColoredFormatter(fmt, use_colors=use_colors) + + # Handler de consola + console_handler = logging.StreamHandler() + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) + + # Handler de archivo si se proporciona ruta + if file_path: + # Asegurar que el directorio exista + file_path = Path(file_path) + file_path.parent.mkdir(parents=True, exist_ok=True) + + file_handler = logging.FileHandler(file_path) + # Para archivos, usar formateador sin colores + file_formatter = logging.Formatter(fmt) + file_handler.setFormatter(file_formatter) + logger.addHandler(file_handler) + + # Mensajes de diagnóstico + logger.debug(f"Logger '{name}' configurado con nivel {logging.getLevelName(level)}") + if file_path: + logger.debug(f"Logs escritos a {file_path}") + + return logger + +def get_logger(name: str, level: Optional[int] = None) -> logging.Logger: + """ + Obtiene un logger existente o crea uno nuevo. + + Args: + name: Nombre del logger + level: Nivel de logging opcional + + Returns: + Logger configurado + """ + logger = logging.getLogger(name) + + # Si el logger no tiene handlers, configurarlo + if not logger.handlers: + # Determinar si es un logger secundario + if '.' in name: + # Es un sublogger, configurar para propagar a logger padre + logger.propagate = True + if level is not None: + logger.setLevel(level) + else: + # Es un logger principal, configurar completamente + logger = setup_logger(name, level or DEFAULT_LEVEL) + elif level is not None: + # Solo actualizar el nivel si se especifica + logger.setLevel(level) + + return logger + +def enable_file_logging(logger_name: str = "corebrain", + log_dir: Optional[Union[str, Path]] = None, + filename: Optional[str] = None) -> str: + """ + Activa el logging a archivo para un logger existente. + + Args: + logger_name: Nombre del logger + log_dir: Directorio para los logs (opcional) + filename: Nombre de archivo personalizado (opcional) + + Returns: + Ruta al archivo de log + """ + logger = logging.getLogger(logger_name) + + # Determinar la ruta del archivo de log + log_dir = Path(log_dir) if log_dir else DEFAULT_LOG_DIR + log_dir.mkdir(parents=True, exist_ok=True) + + # Generar nombre de archivo si no se proporciona + if not filename: + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + filename = f"{logger_name}_{timestamp}.log" + + file_path = log_dir / filename + + # Verificar si ya hay un FileHandler + for handler in logger.handlers: + if isinstance(handler, logging.FileHandler): + logger.removeHandler(handler) + + # Agregar nuevo FileHandler + file_handler = logging.FileHandler(file_path) + formatter = logging.Formatter(DEFAULT_FORMAT) + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + + logger.info(f"Logging a archivo activado: {file_path}") + return str(file_path) + +def set_log_level(level: Union[int, str], + logger_name: Optional[str] = None) -> None: + """ + Establece el nivel de logging para uno o todos los loggers. + + Args: + level: Nivel de logging (nombre o valor entero) + logger_name: Nombre del logger específico (si es None, afecta a todos) + """ + # Convertir nombre de nivel a valor si es necesario + if isinstance(level, str): + level = getattr(logging, level.upper(), logging.INFO) + + if logger_name: + # Afectar solo al logger especificado + logger = logging.getLogger(logger_name) + logger.setLevel(level) + logger.info(f"Nivel de log cambiado a {logging.getLevelName(level)}") + else: + # Afectar al logger raíz y a todos los loggers existentes + root = logging.getLogger() + root.setLevel(level) + + # También afectar a loggers específicos del SDK + for name in logging.root.manager.loggerDict: + if name.startswith("corebrain"): + logging.getLogger(name).setLevel(level) \ No newline at end of file diff --git a/corebrain/utils/serializer.py b/corebrain/utils/serializer.py new file mode 100644 index 0000000..cf1fb49 --- /dev/null +++ b/corebrain/utils/serializer.py @@ -0,0 +1,33 @@ +""" +Utilidades de serialización para Corebrain SDK. +""" +import json + +from datetime import datetime, date, time +from bson import ObjectId +from decimal import Decimal + +class JSONEncoder(json.JSONEncoder): + """Serializador JSON personalizado para tipos especiales.""" + def default(self, obj): + # Objetos datetime + if isinstance(obj, (datetime, date, time)): + return obj.isoformat() + # Objetos timedelta + elif hasattr(obj, 'total_seconds'): # Para objetos timedelta + return obj.total_seconds() + # ObjectId de MongoDB + elif isinstance(obj, ObjectId): + return str(obj) + # Bytes o bytearray + elif isinstance(obj, (bytes, bytearray)): + return obj.hex() + # Decimal + elif isinstance(obj, Decimal): + return float(obj) + # Otros tipos + return super().default(obj) + +def serialize_to_json(obj): + """Serializa cualquier objeto a JSON usando el encoder personalizado""" + return json.dumps(obj, cls=JSONEncoder) \ No newline at end of file diff --git a/examples/add_config.py b/examples/add_config.py new file mode 100644 index 0000000..963996a --- /dev/null +++ b/examples/add_config.py @@ -0,0 +1,27 @@ +from corebrain import ConfigManager + +# Initialize config manager +config_manager = ConfigManager() + +# API key +api_key = "sk_bH8rnkIHCDF1BlRmgS9s6QAK" + +# Database configuration +db_config = { + "type": "sql", # or "mongodb" for MongoDB + "engine": "postgresql", # or "mysql", "sqlite", etc. + "host": "localhost", + "port": 5432, + "database": "your_database", + "user": "your_username", + "password": "your_password" +} + +# Add configuration +config_id = config_manager.add_config(api_key, db_config) +print(f"Configuration added with ID: {config_id}") + +# List all configurations +print("\nAvailable configurations:") +configs = config_manager.list_configs(api_key) +print(configs) \ No newline at end of file diff --git a/examples/complex.py b/examples/complex.py new file mode 100644 index 0000000..e66c21b --- /dev/null +++ b/examples/complex.py @@ -0,0 +1,23 @@ +from corebrain import init + +api_key = "sk_bH8rnkIHCDF1BlRmgS9s6QAK" +#config_id = "c9913a04-a530-4ae3-a877-8e295be87f78" # MONGODB +config_id = "8bdba894-34a7-4453-b665-e640d11fd463" # POSTGRES + +# Initialize the SDK with API key and configuration ID +corebrain = init( + api_key=api_key, + config_id=config_id +) + +""" +Corebrain possible arguments (all optionals): + +- execute_query (bool) +- explain_results (bool) +- detail_level (string = "full") +""" + +result = corebrain.ask("Devuélveme 5 datos interesantes sobre mis usuarios", detail_level="full") + +print(result['explanation']) diff --git a/examples/list_schema.py b/examples/list_schema.py new file mode 100644 index 0000000..daeba01 --- /dev/null +++ b/examples/list_schema.py @@ -0,0 +1,162 @@ +""" +Example script to list database schema and configuration details. +This helps diagnose issues with database connections and schema extraction. +""" +import os +import json +import logging +import psycopg2 +from corebrain import init + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +def verify_postgres_connection(db_config): + """Verify PostgreSQL connection and list tables directly""" + logger.info("\n=== Direct PostgreSQL Connection Test ===") + try: + # Create connection + conn = psycopg2.connect( + host=db_config.get("host", "localhost"), + user=db_config.get("user", ""), + password=db_config.get("password", ""), + dbname=db_config.get("database", ""), + port=db_config.get("port", 5432) + ) + + # Create cursor + cur = conn.cursor() + + # Test connection + cur.execute("SELECT version();") + version = cur.fetchone() + logger.info(f"PostgreSQL Version: {version[0]}") + + # List all schemas + cur.execute(""" + SELECT schema_name + FROM information_schema.schemata + WHERE schema_name NOT IN ('information_schema', 'pg_catalog'); + """) + schemas = cur.fetchall() + logger.info("\nAvailable Schemas:") + for schema in schemas: + logger.info(f" - {schema[0]}") + + # List all tables in public schema + cur.execute(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public'; + """) + tables = cur.fetchall() + logger.info("\nTables in public schema:") + for table in tables: + logger.info(f" - {table[0]}") + + # Get column info for each table + cur.execute(f""" + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = '{table[0]}'; + """) + columns = cur.fetchall() + logger.info(" Columns:") + for col in columns: + logger.info(f" - {col[0]}: {col[1]}") + + cur.close() + conn.close() + + except Exception as e: + logger.error(f"Error in direct PostgreSQL connection: {str(e)}", exc_info=True) + +def main(): + # Get API key from environment variable + api_key = "sk_bH8rnkIHCDF1BlRmgS9s6QAK" + if not api_key: + raise ValueError("Please set COREBRAIN_API_KEY environment variable") + + # Get config ID from environment variable + config_id = "8bdba894-34a7-4453-b665-e640d11fd463" + if not config_id: + raise ValueError("Please set COREBRAIN_CONFIG_ID environment variable") + + logger.info("Initializing Corebrain SDK...") + try: + corebrain = init( + api_key=api_key, + config_id=config_id, + skip_verification=True # Skip API key verification due to the error + ) + except Exception as e: + logger.error(f"Error initializing SDK: {str(e)}") + return + + # Print configuration details + logger.info("\n=== Configuration Details ===") + logger.info(f"Database Type: {corebrain.db_config.get('type')}") + logger.info(f"Database Engine: {corebrain.db_config.get('engine')}") + logger.info(f"Database Name: {corebrain.db_config.get('database')}") + logger.info(f"Config ID: {corebrain.config_id}") + + # Print full database configuration + logger.info("\n=== Full Database Configuration ===") + logger.info(json.dumps(corebrain.db_config, indent=2)) + + # If PostgreSQL, verify connection directly + if corebrain.db_config.get("type", "").lower() == "sql" and \ + corebrain.db_config.get("engine", "").lower() == "postgresql": + verify_postgres_connection(corebrain.db_config) + + # Extract and print schema + logger.info("\n=== Database Schema ===") + try: + schema = corebrain._extract_db_schema(detail_level="full") + + # Print schema summary + logger.info(f"Schema Type: {schema.get('type')}") + logger.info(f"Total Collections: {schema.get('total_collections', 0)}") + logger.info(f"Included Collections: {schema.get('included_collections', 0)}") + + # Print tables/collections + if schema.get("tables"): + logger.info("\n=== Tables/Collections ===") + for table_name, table_info in schema["tables"].items(): + logger.info(f"\nTable/Collection: {table_name}") + + # Print columns/fields + if "columns" in table_info: + logger.info("Columns:") + for col in table_info["columns"]: + logger.info(f" - {col['name']}: {col['type']}") + elif "fields" in table_info: + logger.info("Fields:") + for field in table_info["fields"]: + logger.info(f" - {field['name']}: {field['type']}") + + # Print document count if available + if "doc_count" in table_info: + logger.info(f"Document Count: {table_info['doc_count']}") + + # Print sample data if available + if "sample_data" in table_info and table_info["sample_data"]: + logger.info("Sample Data:") + for doc in table_info["sample_data"][:2]: # Show only first 2 documents + logger.info(f" {json.dumps(doc, indent=2)}") + else: + logger.warning("No tables/collections found in schema!") + + # Print raw schema for debugging + logger.info("\n=== Raw Schema ===") + logger.info(json.dumps(schema, indent=2)) + except Exception as e: + logger.error(f"Error extracting schema: {str(e)}", exc_info=True) + +if __name__ == "__main__": + try: + main() + except Exception as e: + logger.error(f"Error: {str(e)}", exc_info=True) \ No newline at end of file diff --git a/examples/simple.py b/examples/simple.py new file mode 100644 index 0000000..483c546 --- /dev/null +++ b/examples/simple.py @@ -0,0 +1,15 @@ +from corebrain import init + +api_key = "sk_bH8rnkIHCDF1BlRmgS9s6QAK" +#config_id = "c9913a04-a530-4ae3-a877-8e295be87f78" MONGODB +config_id = "8bdba894-34a7-4453-b665-e640d11fd463" # POSTGRES + +# Initialize the SDK with API key and configuration ID +corebrain = init( + api_key=api_key, + config_id=config_id +) + +result = corebrain.ask("Analiza los usuarios y los servicios asociados a estos usuarios.") +print(result["explanation"]) + diff --git a/health.py b/health.py new file mode 100644 index 0000000..a393671 --- /dev/null +++ b/health.py @@ -0,0 +1,47 @@ +# check_imports.py +import os +import importlib +import sys + +def check_imports(package_name, directory): + """Verifica recursivamente las importaciones en un directorio.""" + for item in os.listdir(directory): + path = os.path.join(directory, item) + + # Ignorar directorios ocultos o __pycache__ + if item.startswith('.') or item == '__pycache__': + continue + + if os.path.isdir(path): + # Es un directorio, intentar importar como subpaquete + if os.path.exists(os.path.join(path, '__init__.py')): + subpackage = f"{package_name}.{item}" + try: + print(f"Verificando subpaquete: {subpackage}") + importlib.import_module(subpackage) + # Verificar recursivamente + check_imports(subpackage, path) + except Exception as e: + print(f"ERROR en {subpackage}: {e}") + + elif item.endswith('.py') and item != '__init__.py': + # Es un archivo Python, intentar importar + module_name = f"{package_name}.{item[:-3]}" # quitar .py + try: + print(f"Verificando módulo: {module_name}") + importlib.import_module(module_name) + except Exception as e: + print(f"ERROR en {module_name}: {e}") + +# Asegurar que el directorio actual esté en el path +sys.path.insert(0, '.') + +# Verificar todos los módulos principales +for pkg in ['corebrain']: + if os.path.exists(pkg): + try: + print(f"\nVerificando paquete: {pkg}") + importlib.import_module(pkg) + check_imports(pkg, pkg) + except Exception as e: + print(f"ERROR en paquete {pkg}: {e}") \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..825f3d5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,82 @@ +[project] +name = "corebrain" +version = "0.1.0" +description = "SDK de Corebrain para consultas en lenguaje natural a bases de datos" +readme = "README.md" +authors = [ + {name = "Rubén Ayuso", email = "ruben@globodain.com"} +] +license = {text = "MIT"} +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] +requires-python = ">=3.8" +dependencies = [ + "httpx>=0.24.0", + "sqlalchemy>=2.0.0", + "pydantic>=2.0.0", + "cryptography>=40.0.0", + "python-dotenv>=1.0.0", + "typing-extensions>=4.4.0", + "requests>=2.28.0", + "asyncio>=3.4.3", + "psycopg2-binary>=2.9.0", # En lugar de psycopg2 para evitar problemas de compilación + "mysql-connector-python>=8.0.23", + "pymongo>=4.4.0", +] + +[project.optional-dependencies] +postgres = ["psycopg2-binary>=2.9.0"] +mongodb = ["pymongo>=4.4.0"] +mysql = ["mysql-connector-python>=8.0.23"] +all_db = [ + "psycopg2-binary>=2.9.0", + "pymongo>=4.4.0", + "mysql-connector-python>=8.0.23", +] +dev = [ + "pytest>=7.0.0", + "pytest-cov>=4.0.0", + "black>=23.0.0", + "isort>=5.12.0", + "mypy>=1.3.0", + "flake8>=6.0.0", +] + +[tool.setuptools] +packages = ["corebrain"] + +[project.urls] +"Homepage" = "https://github.com/ceoweggo/Corebrain" +"Bug Tracker" = "https://github.com/ceoweggo/Corebrain/issues" + +[project.scripts] +corebrain = "corebrain.cli.__main__:main" + +[tool.black] +line-length = 100 +target-version = ["py38"] +include = '\.pyi?$' + +[tool.isort] +profile = "black" +line_length = 100 + +[tool.mypy] +python_version = "3.8" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false +disallow_incomplete_defs = false + +[tool.pytest.ini_options] +minversion = "7.0" +testpaths = ["tests"] +pythonpath = ["."] \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..f133b42 --- /dev/null +++ b/setup.py @@ -0,0 +1,38 @@ +""" +Configuración de instalación para el paquete Corebrain. +""" + +from setuptools import setup, find_packages + +setup( + name="corebrain", + version="1.0.0", + description="SDK para consultas en lenguaje natural a bases de datos", + author="Rubén Ayuso", + author_email="ruben@globodain.com", + packages=find_packages(), + install_requires=[ + "httpx>=0.23.0", + "pymongo>=4.3.0", + "psycopg2-binary>=2.9.5", + "mysql-connector-python>=8.0.31", + "sqlalchemy>=2.0.0", + "cryptography>=39.0.0", + "pydantic>=1.10.0", + ], + python_requires=">=3.8", + entry_points={ + "console_scripts": [ + "corebrain=corebrain.__main__:main", + ], + }, + classifiers=[ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + ], +) \ No newline at end of file From 409e8dfe77cad475e64414bc8d5126c24ae45e5d Mon Sep 17 00:00:00 2001 From: RUBEN AYUSO Date: Wed, 14 May 2025 10:30:08 +0200 Subject: [PATCH 02/81] Git ignore added --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a9285cc..c67bee9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ __pycache__/ venv/ .tofix/ README-no-valid.md +requirements.txt # C extensions *.so From 6d1bbeb87d4922817faeb44318497aa8423e3c03 Mon Sep 17 00:00:00 2001 From: RUBEN AYUSO Date: Wed, 14 May 2025 11:31:27 +0200 Subject: [PATCH 03/81] SSO lib added --- .gitignore | 2 +- corebrain/lib/sso/__init__.py | 4 + corebrain/lib/sso/auth.py | 171 ++++++++++++++++++++++++++++++ corebrain/lib/sso/client.py | 194 ++++++++++++++++++++++++++++++++++ 4 files changed, 370 insertions(+), 1 deletion(-) create mode 100644 corebrain/lib/sso/__init__.py create mode 100644 corebrain/lib/sso/auth.py create mode 100644 corebrain/lib/sso/client.py diff --git a/.gitignore b/.gitignore index c67bee9..cf98110 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,7 @@ dist/ downloads/ eggs/ .eggs/ -lib/ +#lib/ lib64/ parts/ sdist/ diff --git a/corebrain/lib/sso/__init__.py b/corebrain/lib/sso/__init__.py new file mode 100644 index 0000000..033a277 --- /dev/null +++ b/corebrain/lib/sso/__init__.py @@ -0,0 +1,4 @@ +from corebrain.lib.sso.auth import GlobodainSSOAuth +from corebrain.lib.sso.client import GlobodainSSOClient + +__all__ = ['GlobodainSSOAuth', 'GlobodainSSOClient'] \ No newline at end of file diff --git a/corebrain/lib/sso/auth.py b/corebrain/lib/sso/auth.py new file mode 100644 index 0000000..5da318f --- /dev/null +++ b/corebrain/lib/sso/auth.py @@ -0,0 +1,171 @@ +import requests +import logging +from urllib.parse import urlencode + +class GlobodainSSOAuth: + def __init__(self, config=None): + self.config = config or {} + self.logger = logging.getLogger(__name__) + + # Configuración por defecto + self.sso_url = self.config.get('GLOBODAIN_SSO_URL', 'http://localhost:3000/login') + self.client_id = self.config.get('GLOBODAIN_CLIENT_ID', '') + self.client_secret = self.config.get('GLOBODAIN_CLIENT_SECRET', '') + self.redirect_uri = self.config.get('GLOBODAIN_REDIRECT_URI', '') + self.success_redirect = self.config.get('GLOBODAIN_SUCCESS_REDIRECT', 'https://sso.globodain.com/cli/success') + + def requires_auth(self, session_handler): + """ + Decorador genérico que verifica si el usuario está autenticado + + Args: + session_handler: Función que obtiene el objeto de sesión actual + + Returns: + Una función decoradora que puede aplicarse a rutas/vistas + """ + def decorator(func): + def wrapper(*args, **kwargs): + # Obtener la sesión actual usando el manejador proporcionado + session = session_handler() + + if 'user' not in session: + # Aquí retornamos información para que el framework redirija + return { + 'authenticated': False, + 'redirect_url': self.get_login_url() + } + return func(*args, **kwargs) + return wrapper + return decorator + + def get_login_url(self, state=None): + """ + Genera la URL para iniciar la autenticación SSO + + Args: + state: Parámetro opcional para mantener estado entre solicitudes + + Returns: + URL completa para el inicio de sesión SSO + """ + params = { + 'client_id': self.client_id, + 'redirect_uri': self.redirect_uri, + 'response_type': 'code', + } + + if state: + params['state'] = state + + return f"{self.sso_url}/api/auth/authorize?{urlencode(params)}" + + def verify_token(self, token): + """ + Verifica el token con el servidor SSO + + Args: + token: Token de acceso a verificar + + Returns: + Datos del token si es válido, None en caso contrario + """ + try: + response = requests.post( + f"{self.sso_url}/api/auth/service-auth", + headers={'Authorization': f'Bearer {token}'}, + json={'service_id': self.client_id} + ) + if response.status_code == 200: + return response.json() + return None + except Exception as e: + self.logger.error(f"Error verificando token: {str(e)}") + return None + + def get_user_info(self, token): + """ + Obtiene información del usuario con el token + + Args: + token: Token de acceso del usuario + + Returns: + Información del perfil del usuario si el token es válido, None en caso contrario + """ + try: + response = requests.get( + f"{self.sso_url}/api/users/me/profile", + headers={'Authorization': f'Bearer {token}'} + ) + if response.status_code == 200: + return response.json() + return None + except Exception as e: + self.logger.error(f"Error obteniendo info de usuario: {str(e)}") + return None + + def exchange_code_for_token(self, code): + """ + Intercambia el código de autorización por un token de acceso + + Args: + code: Código de autorización recibido del servidor SSO + + Returns: + Datos del token de acceso si el intercambio es exitoso, None en caso contrario + """ + try: + response = requests.post( + f"{self.sso_url}/api/auth/token", + json={ + 'client_id': self.client_id, + 'client_secret': self.client_secret, + 'code': code, + 'grant_type': 'authorization_code', + 'redirect_uri': self.redirect_uri + } + ) + if response.status_code == 200: + return response.json() + return None + except Exception as e: + self.logger.error(f"Error intercambiando código: {str(e)}") + return None + + def handle_callback(self, code, session_handler, store_user_func=None): + """ + Maneja el callback del SSO procesando el código recibido + + Args: + code: Código de autorización recibido + session_handler: Función que obtiene el objeto de sesión actual + store_user_func: Función opcional para almacenar datos de usuario en otro lugar + + Returns: + URL de redirección después de procesar el código + """ + # Intercambiar código por token + token_data = self.exchange_code_for_token(code) + if not token_data: + # Error al obtener el token + return self.get_login_url() + + # Obtener información del usuario + user_info = self.get_user_info(token_data.get('access_token')) + if not user_info: + # Error al obtener información del usuario + return self.get_login_url() + + # Guardar información en la sesión + session = session_handler() + session['user'] = user_info + session['token'] = token_data + + # Si hay una función para almacenar el usuario, ejecutarla + if store_user_func and callable(store_user_func): + store_user_func(user_info, token_data) + + # Redirigir a la URL de éxito o a la URL guardada anteriormente + next_url = session.pop('next_url', self.success_redirect) + return next_url \ No newline at end of file diff --git a/corebrain/lib/sso/client.py b/corebrain/lib/sso/client.py new file mode 100644 index 0000000..ce83058 --- /dev/null +++ b/corebrain/lib/sso/client.py @@ -0,0 +1,194 @@ +# /auth/sso_client.py +import requests + +from typing import Dict, Any +from datetime import datetime, timedelta + +class GlobodainSSOClient: + """ + Cliente SDK para servicios de Globodain que se conectan al SSO central + """ + + def __init__( + self, + sso_url: str, + client_id: str, + client_secret: str, + service_id: int, + redirect_uri: str + ): + """ + Inicializar el cliente SSO + + Args: + sso_url: URL base del servicio SSO (ej: https://sso.globodain.com) + client_id: ID de cliente del servicio + client_secret: Secreto de cliente del servicio + service_id: ID numérico del servicio en la plataforma SSO + redirect_uri: URI de redirección para OAuth + """ + self.sso_url = sso_url.rstrip('/') + self.client_id = client_id + self.client_secret = client_secret + self.service_id = service_id + self.redirect_uri = redirect_uri + self._token_cache = {} # Cache de tokens verificados + + + def get_login_url(self, provider: str = None) -> str: + """ + Obtener URL para iniciar sesión en SSO + + Args: + provider: Proveedor de OAuth (google, microsoft, github) o None para login normal + + Returns: + URL para redireccionar al usuario + """ + if provider: + return f"{self.sso_url}/api/auth/oauth/{provider}?service_id={self.service_id}" + else: + return f"{self.sso_url}/login?service_id={self.service_id}&redirect_uri={self.redirect_uri}" + + def verify_token(self, token: str) -> Dict[str, Any]: + """ + Verificar un token de acceso y obtener información del usuario + + Args: + token: Token JWT a verificar + + Returns: + Información del usuario si el token es válido + + Raises: + Exception: Si el token no es válido + """ + # Verificar si ya tenemos información cacheada y válida del token + now = datetime.now() + if token in self._token_cache: + cache_data = self._token_cache[token] + if cache_data['expires_at'] > now: + return cache_data['user_info'] + else: + # Eliminar token expirado del caché + del self._token_cache[token] + + # Verificar token con el servicio SSO + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json" + } + + response = requests.post( + f"{self.sso_url}/api/auth/service-auth", + headers=headers, + json={"service_id": self.service_id} + ) + + if response.status_code != 200: + raise Exception(f"Token inválido: {response.text}") + + # Obtener información del usuario + user_response = requests.get( + f"{self.sso_url}/api/users/me", + headers=headers + ) + + if user_response.status_code != 200: + raise Exception(f"Error al obtener información del usuario: {user_response.text}") + + user_info = user_response.json() + + # Guardar en caché (15 minutos) + self._token_cache[token] = { + 'user_info': user_info, + 'expires_at': now + timedelta(minutes=15) + } + + return user_info + + def authenticate_service(self, token: str) -> Dict[str, Any]: + """ + Autenticar un token para usarlo con este servicio específico + + Args: + token: Token JWT obtenido del SSO + + Returns: + Nuevo token específico para el servicio + + Raises: + Exception: Si hay un error en la autenticación + """ + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json" + } + + response = requests.post( + f"{self.sso_url}/api/auth/service-auth", + headers=headers, + json={"service_id": self.service_id} + ) + + if response.status_code != 200: + raise Exception(f"Error de autenticación: {response.text}") + + return response.json() + + def refresh_token(self, refresh_token: str) -> Dict[str, Any]: + """ + Renovar un token de acceso usando refresh token + + Args: + refresh_token: Token de refresco + + Returns: + Nuevo token de acceso + + Raises: + Exception: Si hay un error al renovar el token + """ + response = requests.post( + f"{self.sso_url}/api/auth/refresh", + json={"refresh_token": refresh_token} + ) + + if response.status_code != 200: + raise Exception(f"Error al renovar token: {response.text}") + + return response.json() + + def logout(self, refresh_token: str, access_token: str) -> bool: + """ + Cerrar sesión (revoca refresh token) + + Args: + refresh_token: Token de refresco a revocar + access_token: Token de acceso válido + + Returns: + True si se cerró sesión correctamente + + Raises: + Exception: Si hay un error al cerrar sesión + """ + headers = { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json" + } + + response = requests.post( + f"{self.sso_url}/api/auth/logout", + headers=headers, + json={"refresh_token": refresh_token} + ) + + if response.status_code != 200: + raise Exception(f"Error al cerrar sesión: {response.text}") + + # Limpiar cualquier token cacheado + if access_token in self._token_cache: + del self._token_cache[access_token] + + return True \ No newline at end of file From eec1e9c1e131e7d2735da67fa15fe933415f399c Mon Sep 17 00:00:00 2001 From: RUBEN AYUSO Date: Wed, 14 May 2025 16:45:20 +0200 Subject: [PATCH 04/81] Translation, README and CONTRIBUTING modified. Trying Discord implementation. --- CONTRIBUTING.md | 158 ++++++++++++++++++++++++++++++-------- README.md | 6 ++ corebrain/lib/sso/auth.py | 2 +- health.py | 26 +++---- setup.py | 4 +- 5 files changed, 149 insertions(+), 47 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 19f0904..e91b37e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,52 +1,148 @@ -# Guía de contribución a CoreBrain SDK +# How to Contribute to Corebrain SDK -¡Gracias por tu interés en contribuir a CoreBrain SDK! Este documento proporciona directrices para contribuir al proyecto. +Thank you for your interest in contributing to CoreBrain SDK! This document provides guidelines for contributing to the project. -## Código de conducta +## Code of Conduct -Al participar en este proyecto, te comprometes a mantener un entorno respetuoso y colaborativo. +By participating in this project, you commit to maintaining a respectful and collaborative environment. -## Cómo contribuir +## How to Contribute -### Reportar bugs +### Reporting Bugs -1. Verifica que el bug no haya sido reportado ya en los [issues](https://github.com/corebrain/sdk/issues) -2. Usa la plantilla de bug para crear un nuevo issue -3. Incluye tanto detalle como sea posible: pasos para reproducir, entorno, versiones, etc. -4. Si es posible, incluye un ejemplo mínimo que reproduzca el problema +1. Verify that the bug hasn't already been reported in the [issues](https://github.com/ceoweggo/Corebrain/issues) +2. Use the bug template to create a new issue +3. Include as much detail as possible: steps to reproduce, environment, versions, etc. +4. If possible, include a minimal example that reproduces the problem -### Sugerir mejoras +### Suggesting Improvements -1. Revisa los [issues](https://github.com/corebrain/sdk/issues) para ver si ya se ha sugerido -2. Usa la plantilla de feature para crear un nuevo issue -3. Describe claramente la mejora y justifica su valor +1. Check the [issues](https://github.com/ceoweggo/Corebrain/issues) to see if it has already been suggested +2. Use the feature template to create a new issue +3. Clearly describe the improvement and justify its value -### Enviar cambios +### Submitting Changes -1. Haz fork del repositorio -2. Crea una rama para tu cambio (`git checkout -b feature/amazing-feature`) -3. Realiza tus cambios siguiendo las convenciones de código -4. Escribe tests para tus cambios -5. Asegúrate de que todos los tests pasan -6. Haz commit de tus cambios (`git commit -m 'Add amazing feature'`) -7. Sube tu rama (`git push origin feature/amazing-feature`) -8. Abre un Pull Request +1. Fork the repository +2. Create a branch for your change (`git checkout -b feature/amazing-feature`) +3. Make your changes following the code conventions +4. Write tests for your changes +5. Ensure all tests pass +6. Commit your changes (`git commit -m 'Add amazing feature'`) +7. Push your branch (`git push origin feature/amazing-feature`) +8. Open a Pull Request -## Entorno de desarrollo +## Development Environment -### Instalación para desarrollo +### Installation for Development ```bash -# Clonar el repositorio -git clone https://github.com/corebrain/sdk.git +# Clone the repository +git clone https://github.com/ceoweggo/Corebrain.git cd sdk -# Crear entorno virtual +# Create virtual environment python -m venv venv -source venv/bin/activate # En Windows: venv\Scripts\activate +source venv/bin/activate # On Windows: venv\Scripts\activate -# Instalar para desarrollo +# Install for development pip install -e ".[dev]" ``` -### Estructura del proyecto +### Project Structure + +``` +v1/ +├── corebrain/ # Main package +│ ├── __init__.py +│ ├── _pycache_/ +│ ├── cli/ # Command-line interface +│ ├── config/ # Configuration management +│ ├── core/ # Core functionality +│ ├── db/ # Database interactions +│ ├── lib/ # Library components +│ └── SSO/ # Globodain SSO Authentication +│ ├── network/ # Network functionality +│ ├── services/ # Service implementations +│ ├── utils/ # Utility functions +│ ├── cli.py # CLI entry point +│ └── sdk.py # SDK entry point +├── corebrain.egg-info/ # Package metadata +├── docs/ # Documentation +├── examples/ # Usage examples +├── screenshots/ # Project screenshots +├── venv/ # Virtual environment (not to be committed) +├── .github/ # GitHub files directory +├── _pycache_/ # Python cache files +├── .tofix/ # Files to be fixed +├── .gitignore # Git ignore rules +├── CONTRIBUTING.md # Contribution guidelines +├── health.py # Health check script +├── LICENSE # License information +├── pyproject.toml # Project configuration +├── README-no-valid.md # Outdated README +├── README.md # Project overview +├── requirements.txt # Production dependencies +└── setup.py # Package setup +``` + +### Running Tests + +```bash +# Run all tests +pytest + +# Run specific test file +pytest tests/test_specific.py + +# Run tests with coverage +pytest --cov=corebrain +``` + +## Coding Standards + +### Style Guide + +- We follow [PEP 8](https://www.python.org/dev/peps/pep-0008/) for Python code +- Use 4 spaces for indentation +- Maximum line length is 88 characters +- Use descriptive variable and function names + +### Documentation + +- All modules, classes, and functions should have docstrings +- Follow the [Google docstring format](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings) +- Keep documentation up-to-date with code changes + +### Commit Messages + +- Use clear, concise commit messages +- Start with a verb in the present tense (e.g., "Add feature" not "Added feature") +- Reference issue numbers when applicable (e.g., "Fix #123: Resolve memory leak") + +## Pull Request Process + +1. Update documentation if necessary +2. Add or update tests as needed +3. Ensure CI checks pass +4. Request a review from maintainers +5. Address review feedback +6. Maintainers will merge your PR once approved + +## Release Process + +Our maintainers follow semantic versioning (MAJOR.MINOR.PATCH): +- MAJOR version for incompatible API changes +- MINOR version for backward-compatible functionality +- PATCH version for backward-compatible bug fixes + +## Getting Help + + + +If you need help with anything: +- Join our [Discord community](https://discord.gg/corebrain) +- Ask questions in the GitHub Discussions +- Contact the maintainers at ruben@globodain.com + +Thank you for contributing to Corebrain SDK! \ No newline at end of file diff --git a/README.md b/README.md index 97dba0a..8d22293 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,12 @@ pip install -e . ### Initialization +# > **⚠️ IMPORTANT:** +# > * If you don't have an existing configuration, first run `corebrain --configure` +# > * If you need to generate a new API key, use `corebrain --create` +# > * Never share your API key in public repositories. Use environment variables instead. + + ```python from corebrain import init diff --git a/corebrain/lib/sso/auth.py b/corebrain/lib/sso/auth.py index 5da318f..a782d8c 100644 --- a/corebrain/lib/sso/auth.py +++ b/corebrain/lib/sso/auth.py @@ -8,7 +8,7 @@ def __init__(self, config=None): self.logger = logging.getLogger(__name__) # Configuración por defecto - self.sso_url = self.config.get('GLOBODAIN_SSO_URL', 'http://localhost:3000/login') + self.sso_url = self.config.get('GLOBODAIN_SSO_URL', 'http://localhost:3000/login') # URL del SSO self.client_id = self.config.get('GLOBODAIN_CLIENT_ID', '') self.client_secret = self.config.get('GLOBODAIN_CLIENT_SECRET', '') self.redirect_uri = self.config.get('GLOBODAIN_REDIRECT_URI', '') diff --git a/health.py b/health.py index a393671..55ffd9d 100644 --- a/health.py +++ b/health.py @@ -4,44 +4,44 @@ import sys def check_imports(package_name, directory): - """Verifica recursivamente las importaciones en un directorio.""" + """ + Recursively checks imports into a directory. + """ + for item in os.listdir(directory): path = os.path.join(directory, item) - # Ignorar directorios ocultos o __pycache__ + # Ignore hidden folders or __pycache__ if item.startswith('.') or item == '__pycache__': continue if os.path.isdir(path): - # Es un directorio, intentar importar como subpaquete + if os.path.exists(os.path.join(path, '__init__.py')): subpackage = f"{package_name}.{item}" try: - print(f"Verificando subpaquete: {subpackage}") + print(f"Verificating subpackage: {subpackage}") importlib.import_module(subpackage) - # Verificar recursivamente check_imports(subpackage, path) except Exception as e: - print(f"ERROR en {subpackage}: {e}") + print(f"ERROR in {subpackage}: {e}") elif item.endswith('.py') and item != '__init__.py': - # Es un archivo Python, intentar importar module_name = f"{package_name}.{item[:-3]}" # quitar .py try: - print(f"Verificando módulo: {module_name}") + print(f"Verificating module: {module_name}") importlib.import_module(module_name) except Exception as e: - print(f"ERROR en {module_name}: {e}") + print(f"ERROR in {module_name}: {e}") -# Asegurar que el directorio actual esté en el path sys.path.insert(0, '.') -# Verificar todos los módulos principales +# Verify all main modules for pkg in ['corebrain']: if os.path.exists(pkg): try: - print(f"\nVerificando paquete: {pkg}") + print(f"\Verificating pkg: {pkg}") importlib.import_module(pkg) check_imports(pkg, pkg) except Exception as e: - print(f"ERROR en paquete {pkg}: {e}") \ No newline at end of file + print(f"ERROR in pkg {pkg}: {e}") \ No newline at end of file diff --git a/setup.py b/setup.py index f133b42..b14bc71 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ """ -Configuración de instalación para el paquete Corebrain. +Installer configuration for Corebrain package. """ from setuptools import setup, find_packages @@ -7,7 +7,7 @@ setup( name="corebrain", version="1.0.0", - description="SDK para consultas en lenguaje natural a bases de datos", + description="SDK for natural language ask to DB", author="Rubén Ayuso", author_email="ruben@globodain.com", packages=find_packages(), From e980c851d65cf89ce5ab2d51fadd1490b5112622 Mon Sep 17 00:00:00 2001 From: RUBEN AYUSO Date: Wed, 14 May 2025 17:29:34 +0200 Subject: [PATCH 05/81] Discord community updated in CONTRIBUTING --- CONTRIBUTING.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e91b37e..628522b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -138,10 +138,8 @@ Our maintainers follow semantic versioning (MAJOR.MINOR.PATCH): ## Getting Help - - If you need help with anything: -- Join our [Discord community](https://discord.gg/corebrain) +- Join our [Discord community](https://discord.gg/m2AXjPn2yV) - Ask questions in the GitHub Discussions - Contact the maintainers at ruben@globodain.com From 2bd87ed98bdc9522ed66d7951380947b647df8a3 Mon Sep 17 00:00:00 2001 From: RUBEN AYUSO Date: Wed, 14 May 2025 18:20:22 +0200 Subject: [PATCH 06/81] CONTRIBUTING file modified --- CONTRIBUTING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 628522b..47e6927 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -140,6 +140,7 @@ Our maintainers follow semantic versioning (MAJOR.MINOR.PATCH): If you need help with anything: - Join our [Discord community](https://discord.gg/m2AXjPn2yV) +- Join our [Whatsapp Channel](https://whatsapp.com/channel/0029Vap43Vy5EjxvR4rncQ1I) - Ask questions in the GitHub Discussions - Contact the maintainers at ruben@globodain.com From 0ad1714e0646b3946a22039d6f2066326582a73f Mon Sep 17 00:00:00 2001 From: Emilia-Kaczor Date: Fri, 16 May 2025 17:51:40 +0200 Subject: [PATCH 07/81] Add docs --- docs/Makefile | 20 ++++++++ docs/README.md | 13 ++++++ docs/make.bat | 35 ++++++++++++++ docs/source/conf.py | 28 +++++++++++ docs/source/corebrain.cli.auth.rst | 29 ++++++++++++ docs/source/corebrain.cli.rst | 53 +++++++++++++++++++++ docs/source/corebrain.config.rst | 21 +++++++++ docs/source/corebrain.core.rst | 45 ++++++++++++++++++ docs/source/corebrain.db.connectors.rst | 29 ++++++++++++ docs/source/corebrain.db.rst | 62 +++++++++++++++++++++++++ docs/source/corebrain.db.schema.rst | 29 ++++++++++++ docs/source/corebrain.network.rst | 21 +++++++++ docs/source/corebrain.rst | 42 +++++++++++++++++ docs/source/corebrain.utils.rst | 37 +++++++++++++++ docs/source/index.rst | 17 +++++++ docs/source/modules.rst | 7 +++ pyproject.toml | 1 + 17 files changed, 489 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/README.md create mode 100644 docs/make.bat create mode 100644 docs/source/conf.py create mode 100644 docs/source/corebrain.cli.auth.rst create mode 100644 docs/source/corebrain.cli.rst create mode 100644 docs/source/corebrain.config.rst create mode 100644 docs/source/corebrain.core.rst create mode 100644 docs/source/corebrain.db.connectors.rst create mode 100644 docs/source/corebrain.db.rst create mode 100644 docs/source/corebrain.db.schema.rst create mode 100644 docs/source/corebrain.network.rst create mode 100644 docs/source/corebrain.rst create mode 100644 docs/source/corebrain.utils.rst create mode 100644 docs/source/index.rst create mode 100644 docs/source/modules.rst diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..cd094fd --- /dev/null +++ b/docs/README.md @@ -0,0 +1,13 @@ +### Generating docs + +Run in terminal: + +```bash + +.\docs\make.bat html + +``` + + + + diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..dc1312a --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..02b6799 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,28 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'Documentation' +copyright = '2025, Emilia' +author = 'Emilia' +release = '0.1' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [] + +templates_path = ['_templates'] +exclude_patterns = [] + + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'alabaster' +html_static_path = ['_static'] diff --git a/docs/source/corebrain.cli.auth.rst b/docs/source/corebrain.cli.auth.rst new file mode 100644 index 0000000..85bb14a --- /dev/null +++ b/docs/source/corebrain.cli.auth.rst @@ -0,0 +1,29 @@ +corebrain.cli.auth package +========================== + +Submodules +---------- + +corebrain.cli.auth.api\_keys module +----------------------------------- + +.. automodule:: corebrain.cli.auth.api_keys + :members: + :show-inheritance: + :undoc-members: + +corebrain.cli.auth.sso module +----------------------------- + +.. automodule:: corebrain.cli.auth.sso + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain.cli.auth + :members: + :show-inheritance: + :undoc-members: diff --git a/docs/source/corebrain.cli.rst b/docs/source/corebrain.cli.rst new file mode 100644 index 0000000..3fdb48b --- /dev/null +++ b/docs/source/corebrain.cli.rst @@ -0,0 +1,53 @@ +corebrain.cli package +===================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + corebrain.cli.auth + +Submodules +---------- + +corebrain.cli.commands module +----------------------------- + +.. automodule:: corebrain.cli.commands + :members: + :show-inheritance: + :undoc-members: + +corebrain.cli.common module +--------------------------- + +.. automodule:: corebrain.cli.common + :members: + :show-inheritance: + :undoc-members: + +corebrain.cli.config module +--------------------------- + +.. automodule:: corebrain.cli.config + :members: + :show-inheritance: + :undoc-members: + +corebrain.cli.utils module +-------------------------- + +.. automodule:: corebrain.cli.utils + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain.cli + :members: + :show-inheritance: + :undoc-members: diff --git a/docs/source/corebrain.config.rst b/docs/source/corebrain.config.rst new file mode 100644 index 0000000..4168d30 --- /dev/null +++ b/docs/source/corebrain.config.rst @@ -0,0 +1,21 @@ +corebrain.config package +======================== + +Submodules +---------- + +corebrain.config.manager module +------------------------------- + +.. automodule:: corebrain.config.manager + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain.config + :members: + :show-inheritance: + :undoc-members: diff --git a/docs/source/corebrain.core.rst b/docs/source/corebrain.core.rst new file mode 100644 index 0000000..0313ae3 --- /dev/null +++ b/docs/source/corebrain.core.rst @@ -0,0 +1,45 @@ +corebrain.core package +====================== + +Submodules +---------- + +corebrain.core.client module +---------------------------- + +.. automodule:: corebrain.core.client + :members: + :show-inheritance: + :undoc-members: + +corebrain.core.common module +---------------------------- + +.. automodule:: corebrain.core.common + :members: + :show-inheritance: + :undoc-members: + +corebrain.core.query module +--------------------------- + +.. automodule:: corebrain.core.query + :members: + :show-inheritance: + :undoc-members: + +corebrain.core.test\_utils module +--------------------------------- + +.. automodule:: corebrain.core.test_utils + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain.core + :members: + :show-inheritance: + :undoc-members: diff --git a/docs/source/corebrain.db.connectors.rst b/docs/source/corebrain.db.connectors.rst new file mode 100644 index 0000000..d2710b3 --- /dev/null +++ b/docs/source/corebrain.db.connectors.rst @@ -0,0 +1,29 @@ +corebrain.db.connectors package +=============================== + +Submodules +---------- + +corebrain.db.connectors.mongodb module +-------------------------------------- + +.. automodule:: corebrain.db.connectors.mongodb + :members: + :show-inheritance: + :undoc-members: + +corebrain.db.connectors.sql module +---------------------------------- + +.. automodule:: corebrain.db.connectors.sql + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain.db.connectors + :members: + :show-inheritance: + :undoc-members: diff --git a/docs/source/corebrain.db.rst b/docs/source/corebrain.db.rst new file mode 100644 index 0000000..751b1d4 --- /dev/null +++ b/docs/source/corebrain.db.rst @@ -0,0 +1,62 @@ +corebrain.db package +==================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + corebrain.db.connectors + corebrain.db.schema + +Submodules +---------- + +corebrain.db.connector module +----------------------------- + +.. automodule:: corebrain.db.connector + :members: + :show-inheritance: + :undoc-members: + +corebrain.db.engines module +--------------------------- + +.. automodule:: corebrain.db.engines + :members: + :show-inheritance: + :undoc-members: + +corebrain.db.factory module +--------------------------- + +.. automodule:: corebrain.db.factory + :members: + :show-inheritance: + :undoc-members: + +corebrain.db.interface module +----------------------------- + +.. automodule:: corebrain.db.interface + :members: + :show-inheritance: + :undoc-members: + +corebrain.db.schema\_file module +-------------------------------- + +.. automodule:: corebrain.db.schema_file + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain.db + :members: + :show-inheritance: + :undoc-members: diff --git a/docs/source/corebrain.db.schema.rst b/docs/source/corebrain.db.schema.rst new file mode 100644 index 0000000..ccc435b --- /dev/null +++ b/docs/source/corebrain.db.schema.rst @@ -0,0 +1,29 @@ +corebrain.db.schema package +=========================== + +Submodules +---------- + +corebrain.db.schema.extractor module +------------------------------------ + +.. automodule:: corebrain.db.schema.extractor + :members: + :show-inheritance: + :undoc-members: + +corebrain.db.schema.optimizer module +------------------------------------ + +.. automodule:: corebrain.db.schema.optimizer + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain.db.schema + :members: + :show-inheritance: + :undoc-members: diff --git a/docs/source/corebrain.network.rst b/docs/source/corebrain.network.rst new file mode 100644 index 0000000..3d94c4f --- /dev/null +++ b/docs/source/corebrain.network.rst @@ -0,0 +1,21 @@ +corebrain.network package +========================= + +Submodules +---------- + +corebrain.network.client module +------------------------------- + +.. automodule:: corebrain.network.client + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain.network + :members: + :show-inheritance: + :undoc-members: diff --git a/docs/source/corebrain.rst b/docs/source/corebrain.rst new file mode 100644 index 0000000..fb44332 --- /dev/null +++ b/docs/source/corebrain.rst @@ -0,0 +1,42 @@ +corebrain package +================= + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + corebrain.cli + corebrain.config + corebrain.core + corebrain.db + corebrain.network + corebrain.utils + +Submodules +---------- + +corebrain.cli module +-------------------- + +.. automodule:: corebrain.cli + :members: + :show-inheritance: + :undoc-members: + +corebrain.sdk module +-------------------- + +.. automodule:: corebrain.sdk + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain + :members: + :show-inheritance: + :undoc-members: diff --git a/docs/source/corebrain.utils.rst b/docs/source/corebrain.utils.rst new file mode 100644 index 0000000..4294529 --- /dev/null +++ b/docs/source/corebrain.utils.rst @@ -0,0 +1,37 @@ +corebrain.utils package +======================= + +Submodules +---------- + +corebrain.utils.encrypter module +-------------------------------- + +.. automodule:: corebrain.utils.encrypter + :members: + :show-inheritance: + :undoc-members: + +corebrain.utils.logging module +------------------------------ + +.. automodule:: corebrain.utils.logging + :members: + :show-inheritance: + :undoc-members: + +corebrain.utils.serializer module +--------------------------------- + +.. automodule:: corebrain.utils.serializer + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain.utils + :members: + :show-inheritance: + :undoc-members: diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..4f9233d --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,17 @@ +.. Documentation documentation master file, created by + sphinx-quickstart on Fri May 16 16:20:00 2025. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Documentation documentation +=========================== + +Add your content using ``reStructuredText`` syntax. See the +`reStructuredText `_ +documentation for details. + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + diff --git a/docs/source/modules.rst b/docs/source/modules.rst new file mode 100644 index 0000000..7f3849e --- /dev/null +++ b/docs/source/modules.rst @@ -0,0 +1,7 @@ +corebrain +========= + +.. toctree:: + :maxdepth: 4 + + corebrain diff --git a/pyproject.toml b/pyproject.toml index 825f3d5..c07d952 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dev = [ "isort>=5.12.0", "mypy>=1.3.0", "flake8>=6.0.0", + "sphinx>=8.2.3", ] [tool.setuptools] From 4bde6331bb13c501239ba25e14ec681244eacbe1 Mon Sep 17 00:00:00 2001 From: Emilia-Kaczor Date: Fri, 16 May 2025 18:33:41 +0200 Subject: [PATCH 08/81] Add path to sphinx --- corebrain/cli/auth/api_keys.py | 12 ++++++------ docs/source/conf.py | 18 +++++++++--------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/corebrain/cli/auth/api_keys.py b/corebrain/cli/auth/api_keys.py index 6163391..5be72f0 100644 --- a/corebrain/cli/auth/api_keys.py +++ b/corebrain/cli/auth/api_keys.py @@ -13,14 +13,14 @@ def verify_api_token(token: str, api_url: Optional[str] = None, user_data: Optional[Dict[str, Any]] = None) -> Tuple[bool, Optional[Dict[str, Any]]]: """ Verifies if an API token is valid. - + Args: - token: API token to verify - api_url: Optional API URL - user_data: User data - + token (str): API token to verify. + api_url (str, optional): API URL. Defaults to None. + user_data (dict, optional): User data. Defaults to None. + Returns: - Tuple with (validity, user information) if valid, (False, None) if not + tuple: (validity (bool), user information (dict)) if valid, else (False, None). """ try: # Create a temporary SDK instance to verify the token diff --git a/docs/source/conf.py b/docs/source/conf.py index 02b6799..374e9c0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -3,26 +3,26 @@ # For the full list of built-in configuration values, see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html -# -- Project information ----------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information +import os +import sys + +sys.path.insert(0, os.path.abspath('../..')) + +# -- Project information ----------------------------------------------------- project = 'Documentation' copyright = '2025, Emilia' author = 'Emilia' release = '0.1' # -- General configuration --------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration - -extensions = [] +extensions = [ + 'sphinx.ext.autodoc', +] templates_path = ['_templates'] exclude_patterns = [] - - # -- Options for HTML output ------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output - html_theme = 'alabaster' html_static_path = ['_static'] From c9eb503bf35fadd1e500a437d4eaab714aa535d5 Mon Sep 17 00:00:00 2001 From: Emilia-Kaczor Date: Fri, 16 May 2025 18:51:37 +0200 Subject: [PATCH 09/81] Documentation working --- docs/source/conf.py | 16 ++++++++++++++++ docs/source/index.rst | 7 ++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 374e9c0..ba0877a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -25,4 +25,20 @@ # -- Options for HTML output ------------------------------------------------- html_theme = 'alabaster' + +html_theme_options = { + 'show_related': True, + 'sidebar_width': '220px', +} + +# Optional: add extra sidebar content +html_sidebars = { + '**': [ + 'about.html', + 'navigation.html', + 'relations.html', # next/prev links + 'searchbox.html', + ] +} + html_static_path = ['_static'] diff --git a/docs/source/index.rst b/docs/source/index.rst index 4f9233d..434f597 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -6,12 +6,9 @@ Documentation documentation =========================== -Add your content using ``reStructuredText`` syntax. See the -`reStructuredText `_ -documentation for details. - - .. toctree:: :maxdepth: 2 :caption: Contents: + modules + From 599d1b25116e8005b9a289c0d726c5cbbab49cb7 Mon Sep 17 00:00:00 2001 From: KarixD2137 Date: Mon, 19 May 2025 12:36:05 +0200 Subject: [PATCH 10/81] Added CLI argument for API key creation --- corebrain/cli/commands.py | 31 +++++++++++++++++++++++++++++-- corebrain/cli/config.py | 3 ++- corebrain/core/client.py | 19 +++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index d9adac9..d919fab 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -14,6 +14,7 @@ from corebrain.cli.utils import print_colored from corebrain.config.manager import ConfigManager from corebrain.lib.sso.auth import GlobodainSSOAuth +from corebrain.core.client import create_api_key # function that sends request for creating API key def main_cli(argv: Optional[List[str]] = None) -> int: """ @@ -51,6 +52,9 @@ def main_cli(argv: Optional[List[str]] = None) -> int: parser.add_argument("--sso-url", help="Globodain SSO service URL") parser.add_argument("--login", action="store_true", help="Login via SSO") parser.add_argument("--test-auth", action="store_true", help="Test SSO authentication system") + parser.add_argument("--create", action="store_true", help="Create a new API Key") + parser.add_argument("--key-name", help="Name of the new API Key") + parser.add_argument("--key-level", choices=["read", "write", "admin"], default="read", help="Access level for the new API Key") args = parser.parse_args(argv) @@ -158,12 +162,35 @@ def main_cli(argv: Optional[List[str]] = None) -> int: show_db_schema(api_key, args.config_id, api_url) elif args.extract_schema: extract_schema_to_file(api_key, args.config_id, args.output_file, api_url) + + # Creating the API key + if args.create: + + if not args.token: + print_colored("You must provide an API token using --token", "yellow") + return 1 + + if not args.key_name: + print_colored("You must provide a name for the API Key using --key-name", "yellow") + return 1 + + try: + api_key = create_api_key( + api_token=args.token, + name=args.key_name, + level=args.key_level + ) + print_colored("API Key created successfully:", "green") + print(api_key) + return 0 + except Exception as e: + print_colored(f"Failed to create API Key: {str(e)}", "red") + return 1 else: # If no option was specified, show help parser.print_help() - print_colored("\nTip: Use 'corebrain --login' to login via SSO.", "blue") - + print_colored("\nTip: Use 'corebrain --login' to login via SSO.", "blue") return 0 except Exception as e: print_colored(f"Error: {str(e)}", "red") diff --git a/corebrain/cli/config.py b/corebrain/cli/config.py index ca3dc13..8c64cad 100644 --- a/corebrain/cli/config.py +++ b/corebrain/cli/config.py @@ -486,4 +486,5 @@ def configure_sdk(api_token: str, api_key: str, api_url: Optional[str] = None, s result = corebrain.ask("Your question in natural language") print(result["explanation"]) """ - ) \ No newline at end of file + ) + diff --git a/corebrain/core/client.py b/corebrain/core/client.py index 61e4bdb..c425d09 100644 --- a/corebrain/core/client.py +++ b/corebrain/core/client.py @@ -20,6 +20,7 @@ from datetime import datetime from corebrain.core.common import logger, CorebrainError +from corebrain.cli.common import DEFAULT_API_URL # The API URL for running requests class Corebrain: """ @@ -1320,6 +1321,24 @@ def _execute_mongodb_query(self, query: Dict[str, Any]) -> List[Dict[str, Any]]: except Exception as e: raise CorebrainError(f"Error executing MongoDB query: {str(e)}") + + # Sends request to Corebrain-API to create API key + def create_api_key(api_token: str, name: str, level: str = "read") -> dict: + """ + Create an API key using the backend API. + """ + url = f"{DEFAULT_API_URL}/api-keys" + headers = { + "Authorization": f"Bearer {api_token}", + "Content-Type": "application/json" + } + payload = { + "name": name, + "level": level + } + response = requests.post(url, headers=headers, json=payload) + response.raise_for_status() + return response.json() def init( From b1ad3b6725c9821caf376124e33b7da9839742c8 Mon Sep 17 00:00:00 2001 From: barto3214 Date: Mon, 19 May 2025 12:51:56 +0200 Subject: [PATCH 11/81] New commands added --- corebrain/cli/commands.py | 118 +++++++++++++++++++++++++++++++++++- corebrain/config/manager.py | 46 +++++++++++++- 2 files changed, 162 insertions(+), 2 deletions(-) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index d9adac9..1008126 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -5,6 +5,7 @@ import os import sys import webbrowser +import requests from typing import Optional, List @@ -13,6 +14,8 @@ from corebrain.cli.config import configure_sdk, get_api_credential from corebrain.cli.utils import print_colored from corebrain.config.manager import ConfigManager +from config.manager import export_config +from config.manager import validate_config from corebrain.lib.sso.auth import GlobodainSSOAuth def main_cli(argv: Optional[List[str]] = None) -> int: @@ -51,9 +54,29 @@ def main_cli(argv: Optional[List[str]] = None) -> int: parser.add_argument("--sso-url", help="Globodain SSO service URL") parser.add_argument("--login", action="store_true", help="Login via SSO") parser.add_argument("--test-auth", action="store_true", help="Test SSO authentication system") + parser.add_argument("--woami",action="store_true",help="Display information about the current user") + parser.add_argument("--check-status",action="store_true",help="Checks status of task") + parser.add_argument("--task-id",help="ID of the task to check status for") + parser.add_argument("--validate-config",action="store_true",help="Validates the selected configuration without executing any operations") + parser.add_argument("--test-connection",action="store_true",help="Tests the connection to the Corebrain API using the provide credentials") + parser.add_argument("--export-config",action="store_true",help="Exports the current configuration to a file") + args = parser.parse_args(argv) + + # Made by Lukasz + if args.export_config: + export_config(args.export_config) + # --> config/manager.py --> export_config + + if args.validate_config: + if not args.config_id: + print_colored("Error: --config-id is required for validation", "red") + return 1 + return validate_config(args.config_id) + + # Show version if args.version: try: @@ -127,6 +150,70 @@ def main_cli(argv: Optional[List[str]] = None) -> int: print_colored("A general API token was obtained, but not a specific API Key.", "yellow") print_colored("You can create an API Key in the Corebrain dashboard.", "yellow") return 1 + + if args.check_status: + if not args.task_id: + print_colored("❌ Please provide a task ID using --task-id", "red") + return 1 + + # Get URLs + api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + + # Prioritize api_key if explicitly provided + token_arg = args.api_key if args.api_key else args.token + + # Get API credentials + api_key, user_data, api_token = get_api_credential(token_arg, sso_url) + + if not api_key: + print_colored("❌ API Key is required to check task status. Use --api-key or login via --login", "red") + return 1 + + try: + task_id = args.task_id + headers = { + "Authorization": f"Bearer {api_key}", + "Accept": "application/json" + } + url = f"{api_url}/tasks/{task_id}/status" + response = requests.get(url, headers=headers) + + if response.status_code == 404: + print_colored(f"❌ Task with ID '{task_id}' not found.", "red") + return 1 + + response.raise_for_status() + data = response.json() + status = data.get("status", "unknown") + + print_colored(f"✅ Task '{task_id}' status: {status}", "green") + return 0 + except Exception as e: + print_colored(f"❌ Failed to check status: {str(e)}", "red") + return 1 + + if args.woami: + try: + #downloading user data + sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + token_arg = args.api_key if args.api_key else args.token + + #using saved data about user + api_key, user_data, api_token = get_api_credential(token_arg, sso_url) + #printing user data + if user_data: + print_colored("User Data:", "blue") + for k, v in user_data.items(): + print(f"{k}: {v}") + else: + print_colored("❌ Can't find data about user, be sure that you are logged into --login.", "red") + return 1 + + return 0 + except Exception as e: + print_colored(f"❌ Error when downloading data about user {str(e)}", "red") + return 1 # Operations that require credentials: configure, list, remove or show schema if args.configure or args.list_configs or args.remove_config or args.show_schema or args.extract_schema: @@ -158,7 +245,36 @@ def main_cli(argv: Optional[List[str]] = None) -> int: show_db_schema(api_key, args.config_id, api_url) elif args.extract_schema: extract_schema_to_file(api_key, args.config_id, args.output_file, api_url) - + + if args.test_connection: + # Test connection to the Corebrain API + api_url = args.api_url or os.environ.get("COREBRAIN_API_URL", DEFAULT_API_URL) + sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL", DEFAULT_SSO_URL) + + try: + # Retrieve API credentials + api_key, user_data, api_token = get_api_credential(args.token, sso_url) + except Exception as e: + print_colored(f"Error while retrieving API credentials: {e}", "red") + return 1 + + if not api_key: + print_colored( + "Error: An API key is required. You can generate one at dashboard.corebrain.com.", + "red" + ) + return 1 + + try: + # Test the connection + from corebrain.db.schema_file import test_connection + test_connection(api_key, api_url) + print_colored("Successfully connected to Corebrain API.", "green") + except Exception as e: + print_colored(f"Failed to connect to Corebrain API: {e}", "red") + return 1 + + else: # If no option was specified, show help parser.print_help() diff --git a/corebrain/config/manager.py b/corebrain/config/manager.py index 1c0a0e5..245b047 100644 --- a/corebrain/config/manager.py +++ b/corebrain/config/manager.py @@ -7,10 +7,54 @@ from pathlib import Path from typing import Dict, Any, List, Optional from cryptography.fernet import Fernet - from corebrain.utils.serializer import serialize_to_json from corebrain.core.common import logger +# Made by Lukasz +# get data from pyproject.toml +def load_project_metadata(): + pyproject_path = Path(__file__).resolve().parent.parent / "pyproject.toml" + try: + with open(pyproject_path, "rb") as f: + data = tomli.load(f) + return data.get("project", {}) + except (FileNotFoundError, tomli.TOMLDecodeError) as e: + print(f"Warning: Could not load project metadata: {e}") + return {} + +# Made by Lukasz +# get the name, version, etc. +def get_config(): + metadata = load_project_metadata() # ^ + return { + "model": metadata.get("name", "unknown"), + "version": metadata.get("version", "0.0.0"), + "debug": False, + "logging": {"level": "info"} + } + +# Made by Lukasz +# export config to file +def export_config(filepath="config.json"): + config = get_config() # ^ + with open(filepath, "w") as f: + json.dump(config, f, indent=4) + print(f"Configuration exported to {filepath}") + +# Validates that a configuration with the given ID exists. +def validate_config(config_id: str): + # The API key under which configs are stored + api_key = os.environ.get("COREBRAIN_API_KEY", "") + manager = ConfigManager() + cfg = manager.get_config(api_key, config_id) + + if cfg: + print(f"✅ Configuration '{config_id}' is present and valid.") + return 0 + else: + print(f"❌ Configuration '{config_id}' not found.") + return 1 + # Función para imprimir mensajes coloreados def _print_colored(message: str, color: str) -> None: """Versión simplificada de _print_colored que no depende de cli.utils""" From c25c0e520892bbdb6e5b2470d3b99726dd7e2810 Mon Sep 17 00:00:00 2001 From: KarixD2137 Date: Tue, 20 May 2025 09:21:53 +0200 Subject: [PATCH 12/81] Added CLI argument for API key creation --- corebrain/cli/commands.py | 3 ++- corebrain/core/client.py | 36 +++++++++++++++++------------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index d919fab..25c5fde 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -14,7 +14,7 @@ from corebrain.cli.utils import print_colored from corebrain.config.manager import ConfigManager from corebrain.lib.sso.auth import GlobodainSSOAuth -from corebrain.core.client import create_api_key # function that sends request for creating API key +from corebrain.core.client import create_api_key def main_cli(argv: Optional[List[str]] = None) -> int: """ @@ -176,6 +176,7 @@ def main_cli(argv: Optional[List[str]] = None) -> int: try: api_key = create_api_key( + DEFAULT_API_URL, api_token=args.token, name=args.key_name, level=args.key_level diff --git a/corebrain/core/client.py b/corebrain/core/client.py index c425d09..980568a 100644 --- a/corebrain/core/client.py +++ b/corebrain/core/client.py @@ -20,7 +20,6 @@ from datetime import datetime from corebrain.core.common import logger, CorebrainError -from corebrain.cli.common import DEFAULT_API_URL # The API URL for running requests class Corebrain: """ @@ -1321,24 +1320,6 @@ def _execute_mongodb_query(self, query: Dict[str, Any]) -> List[Dict[str, Any]]: except Exception as e: raise CorebrainError(f"Error executing MongoDB query: {str(e)}") - - # Sends request to Corebrain-API to create API key - def create_api_key(api_token: str, name: str, level: str = "read") -> dict: - """ - Create an API key using the backend API. - """ - url = f"{DEFAULT_API_URL}/api-keys" - headers = { - "Authorization": f"Bearer {api_token}", - "Content-Type": "application/json" - } - payload = { - "name": name, - "level": level - } - response = requests.post(url, headers=headers, json=payload) - response.raise_for_status() - return response.json() def init( @@ -1381,3 +1362,20 @@ def init( skip_verification=skip_verification ) +# Sends request to Corebrain-API to create API key +def create_api_key(DEFAULT_API_URL: str, api_token: str, name: str, level: str = "read") -> dict: + """ + Create an API key using the backend API. + """ + url = f"{DEFAULT_API_URL}/api-keys" + headers = { + "Authorization": f"Bearer {api_token}", + "Content-Type": "application/json" + } + payload = { + "name": name, + "level": level + } + response = requests.post(url, headers=headers, json=payload) + response.raise_for_status() + return response.json() \ No newline at end of file From 1025f5be86fb4f8344e80f7ccb481a4990a3bb8f Mon Sep 17 00:00:00 2001 From: KarixD2137 Date: Tue, 20 May 2025 09:31:40 +0200 Subject: [PATCH 13/81] Added CLI argument for API key creation - fixed imports --- corebrain/cli/commands.py | 4 ++-- corebrain/core/client.py | 37 ++++++++++++++++++------------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index 25c5fde..a6acc17 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -14,7 +14,7 @@ from corebrain.cli.utils import print_colored from corebrain.config.manager import ConfigManager from corebrain.lib.sso.auth import GlobodainSSOAuth -from corebrain.core.client import create_api_key +from corebrain.core.client import Corebrain def main_cli(argv: Optional[List[str]] = None) -> int: """ @@ -175,7 +175,7 @@ def main_cli(argv: Optional[List[str]] = None) -> int: return 1 try: - api_key = create_api_key( + api_key = Corebrain.create_api_key( DEFAULT_API_URL, api_token=args.token, name=args.key_name, diff --git a/corebrain/core/client.py b/corebrain/core/client.py index 980568a..9c2a5ee 100644 --- a/corebrain/core/client.py +++ b/corebrain/core/client.py @@ -1321,6 +1321,23 @@ def _execute_mongodb_query(self, query: Dict[str, Any]) -> List[Dict[str, Any]]: except Exception as e: raise CorebrainError(f"Error executing MongoDB query: {str(e)}") + # Sends request to Corebrain-API to create API key + def create_api_key(DEFAULT_API_URL: str, api_token: str, name: str, level: str = "read") -> dict: + """ + Create an API key using the backend API. + """ + url = f"{DEFAULT_API_URL}/api-keys" + headers = { + "Authorization": f"Bearer {api_token}", + "Content-Type": "application/json" + } + payload = { + "name": name, + "level": level + } + response = requests.post(url, headers=headers, json=payload) + response.raise_for_status() + return response.json() def init( api_key: str = None, @@ -1360,22 +1377,4 @@ def init( user_data=user_data, api_url=api_url, skip_verification=skip_verification - ) - -# Sends request to Corebrain-API to create API key -def create_api_key(DEFAULT_API_URL: str, api_token: str, name: str, level: str = "read") -> dict: - """ - Create an API key using the backend API. - """ - url = f"{DEFAULT_API_URL}/api-keys" - headers = { - "Authorization": f"Bearer {api_token}", - "Content-Type": "application/json" - } - payload = { - "name": name, - "level": level - } - response = requests.post(url, headers=headers, json=payload) - response.raise_for_status() - return response.json() \ No newline at end of file + ) \ No newline at end of file From cdeba4128a1f8cdf1c44092a1e807fb132494a16 Mon Sep 17 00:00:00 2001 From: B_Locksmith <144423976+barto3214@users.noreply.github.com> Date: Tue, 20 May 2025 10:10:36 +0200 Subject: [PATCH 14/81] test connection func added --- corebrain/db/schema_file.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/corebrain/db/schema_file.py b/corebrain/db/schema_file.py index e13806c..445681b 100644 --- a/corebrain/db/schema_file.py +++ b/corebrain/db/schema_file.py @@ -207,6 +207,27 @@ def extract_schema_with_lazy_init(api_key: str, db_config: Dict[str, Any], api_u _print_colored(f"Error al extraer esquema con cliente: {str(e)}", "red") # Como alternativa, usar extracción directa sin cliente return extract_db_schema_direct(db_config) +from typing import Dict, Any + +def test_connection(db_config: Dict[str, Any]) -> bool: + try: + if db_config["type"].lower() == "sql": + # Code to test SQL connection... + pass + elif db_config["type"].lower() in ["nosql", "mongodb"]: + import pymongo + + # Create MongoDB client + client = pymongo.MongoClient(db_config["connection_string"]) + client.admin.command('ping') # Test connection + + return True + else: + _print_colored("Unsupported database type.", "red") + return False + except Exception as e: + _print_colored(f"Failed to connect to the database: {str(e)}", "red") + return False def extract_schema_to_file(api_key: str, config_id: Optional[str] = None, output_file: Optional[str] = None, api_url: Optional[str] = None) -> bool: """ @@ -580,4 +601,4 @@ def get_schema_with_dynamic_import(api_token: str, config_id: str, db_config: Di except Exception as e: _print_colored(f"Error al extraer esquema con cliente: {str(e)}", "red") # Fallback a extracción directa - return extract_db_schema(db_config) \ No newline at end of file + return extract_db_schema(db_config) From 3e17b176932cf7c5cf188afc79f1a8006a905ea8 Mon Sep 17 00:00:00 2001 From: Emilia-Kaczor Date: Tue, 20 May 2025 15:52:48 +0200 Subject: [PATCH 15/81] Changed theme of docs --- docs/source/conf.py | 34 +++++----------------------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index ba0877a..2d85d0c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,44 +1,20 @@ -# Configuration file for the Sphinx documentation builder. -# -# For the full list of built-in configuration values, see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - import os import sys - sys.path.insert(0, os.path.abspath('../..')) -# -- Project information ----------------------------------------------------- -project = 'Documentation' -copyright = '2025, Emilia' -author = 'Emilia' +project = 'Corebrain Documentation' +copyright = '2025, Corebrain' +author = 'Corebrain' release = '0.1' -# -- General configuration --------------------------------------------------- extensions = [ - 'sphinx.ext.autodoc', + 'sphinx.ext.autodoc', ] templates_path = ['_templates'] exclude_patterns = [] -# -- Options for HTML output ------------------------------------------------- -html_theme = 'alabaster' - -html_theme_options = { - 'show_related': True, - 'sidebar_width': '220px', -} - -# Optional: add extra sidebar content -html_sidebars = { - '**': [ - 'about.html', - 'navigation.html', - 'relations.html', # next/prev links - 'searchbox.html', - ] -} +html_theme = 'furo' html_static_path = ['_static'] From d4c20f9e236101e701fc1602473a19776c828bab Mon Sep 17 00:00:00 2001 From: Albert Stankowski Date: Tue, 20 May 2025 15:00:02 +0100 Subject: [PATCH 16/81] Add translations to inline docs in cli and config --- corebrain/cli/__init__.py | 12 ++++----- corebrain/cli/__main__.py | 4 +-- corebrain/cli/auth/__init__.py | 6 ++--- corebrain/config/__init__.py | 6 ++--- corebrain/config/manager.py | 46 +++++++++++++++++----------------- corebrain/core/__init__.py | 6 ++--- corebrain/core/client.py | 30 +++++++++++----------- 7 files changed, 55 insertions(+), 55 deletions(-) diff --git a/corebrain/cli/__init__.py b/corebrain/cli/__init__.py index 24a2c65..53672d1 100644 --- a/corebrain/cli/__init__.py +++ b/corebrain/cli/__init__.py @@ -1,8 +1,8 @@ """ -Interfaz de línea de comandos para Corebrain SDK. +Command-line interface for the Corebrain SDK. -Este módulo proporciona una interfaz de línea de comandos para configurar -y usar el SDK de Corebrain para consultas en lenguaje natural a bases de datos. +This module provides a command-line interface to configure +and use the Corebrain SDK for natural language queries to databases. """ import sys from typing import Optional, List @@ -43,13 +43,13 @@ # Función de conveniencia para ejecutar CLI def run_cli(argv: Optional[List[str]] = None) -> int: """ - Ejecuta la CLI con los argumentos proporcionados. + Run the CLI with the provided arguments. Args: - argv: Lista de argumentos (usa sys.argv si es None) + argv: List of arguments (use sys.argv if None) Returns: - Código de salida + Exit code """ if argv is None: argv = sys.argv[1:] diff --git a/corebrain/cli/__main__.py b/corebrain/cli/__main__.py index ea8ee3a..db91155 100644 --- a/corebrain/cli/__main__.py +++ b/corebrain/cli/__main__.py @@ -1,11 +1,11 @@ """ -Punto de entrada para ejecutar la CLI como módulo. +Entry point to run the CLI as a module. """ import sys from corebrain.cli.commands import main_cli def main(): - """Función principal para entry point en pyproject.toml""" + """Main function for the entry point in pyproject.toml.""" return main_cli() if __name__ == "__main__": diff --git a/corebrain/cli/auth/__init__.py b/corebrain/cli/auth/__init__.py index ef50100..873572d 100644 --- a/corebrain/cli/auth/__init__.py +++ b/corebrain/cli/auth/__init__.py @@ -1,8 +1,8 @@ """ -Módulos de autenticación para CLI de Corebrain. +Authentication modules for the Corebrain CLI. -Este paquete proporciona funcionalidades para autenticación, -gestión de tokens y API keys en la CLI de Corebrain. +This package provides functionality for authentication, +token management, and API keys in the Corebrain CLI. """ from corebrain.cli.auth.sso import authenticate_with_sso, TokenHandler from corebrain.cli.auth.api_keys import ( diff --git a/corebrain/config/__init__.py b/corebrain/config/__init__.py index 8caa1ed..7149948 100644 --- a/corebrain/config/__init__.py +++ b/corebrain/config/__init__.py @@ -1,8 +1,8 @@ """ -Gestión de configuración para Corebrain SDK. +Configuration management for the Corebrain SDK. -Este paquete proporciona funcionalidades para gestionar configuraciones -de conexión a bases de datos y preferencias del SDK. +This package provides functionality to manage database connection configurations +and SDK preferences. """ from .manager import ConfigManager diff --git a/corebrain/config/manager.py b/corebrain/config/manager.py index 1c0a0e5..5f4fc90 100644 --- a/corebrain/config/manager.py +++ b/corebrain/config/manager.py @@ -1,5 +1,5 @@ """ -Gestor de configuraciones para Corebrain SDK. +Configuration manager for the Corebrain SDK. """ import json @@ -13,7 +13,7 @@ # Función para imprimir mensajes coloreados def _print_colored(message: str, color: str) -> None: - """Versión simplificada de _print_colored que no depende de cli.utils""" + """Simplified version of _print_colored that does not depend on cli.utils.""" colors = { "red": "\033[91m", "green": "\033[92m", @@ -25,7 +25,7 @@ def _print_colored(message: str, color: str) -> None: print(f"{color_code}{message}{colors['default']}") class ConfigManager: - """Gestor de configuraciones del SDK con seguridad y rendimiento mejorados""" + """SDK configuration manager with improved security and performance.""" CONFIG_DIR = Path.home() / ".corebrain" CONFIG_FILE = CONFIG_DIR / "config.json" @@ -39,7 +39,7 @@ def __init__(self): self._load_configs() def _ensure_config_dir(self) -> None: - """Asegura que existe el directorio de configuración.""" + """Ensures that the configuration directory exists.""" try: self.CONFIG_DIR.mkdir(parents=True, exist_ok=True) logger.debug(f"Directorio de configuración asegurado: {self.CONFIG_DIR}") @@ -49,7 +49,7 @@ def _ensure_config_dir(self) -> None: _print_colored(f"Error al crear directorio de configuración: {str(e)}", "red") def _load_secret_key(self) -> None: - """Carga o genera la clave secreta para encriptar datos sensibles.""" + """Loads or generates the secret key to encrypt sensitive data.""" try: if not self.SECRET_KEY_FILE.exists(): key = Fernet.generate_key() @@ -68,7 +68,7 @@ def _load_secret_key(self) -> None: self.cipher = Fernet(self.secret_key) def _load_configs(self) -> Dict[str, Dict[str, Any]]: - """Carga las configuraciones guardadas.""" + """Loads the saved configurations.""" if not self.CONFIG_FILE.exists(): _print_colored(f"Archivo de configuración no encontrado: {self.CONFIG_FILE}", "yellow") return {} @@ -101,7 +101,7 @@ def _load_configs(self) -> Dict[str, Dict[str, Any]]: return {} def _save_configs(self) -> None: - """Guarda las configuraciones actuales.""" + """Saves the current configurations.""" try: configs_json = serialize_to_json(self.configs) encrypted_data = self.cipher.encrypt(json.dumps(configs_json).encode()).decode() @@ -115,15 +115,15 @@ def _save_configs(self) -> None: def add_config(self, api_key: str, db_config: Dict[str, Any], config_id: Optional[str] = None) -> str: """ - Añade una nueva configuración. + Adds a new configuration. Args: - api_key: API Key seleccionada - db_config: Configuración de la base de datos - config_id: ID opcional para la configuración (se genera uno si no se proporciona) + api_key: Selected API Key + db_config: Database configuration + config_id: Optional ID for the configuration (one is generated if not provided) Returns: - ID de la configuración + Configuration ID """ if not config_id: config_id = str(uuid.uuid4()) @@ -142,39 +142,39 @@ def add_config(self, api_key: str, db_config: Dict[str, Any], config_id: Optiona def get_config(self, api_key_selected: str, config_id: str) -> Optional[Dict[str, Any]]: """ - Obtiene una configuración específica. + Retrieves a specific configuration. Args: - api_key_selected: API Key seleccionada - config_id: ID de la configuración + api_key_selected: Selected API Key + config_id: Configuration ID Returns: - Configuración o None si no existe + Configuration or None if it does not exist """ return self.configs.get(api_key_selected, {}).get(config_id) def list_configs(self, api_key_selected: str) -> List[str]: """ - Lista los IDs de configuración disponibles para una API Key. + Lists the available configuration IDs for an API Key. Args: - api_key_selected: API Key seleccionada + api_key_selected: Selected API Key Returns: - Lista de IDs de configuración + List of configuration IDs """ return list(self.configs.get(api_key_selected, {}).keys()) def remove_config(self, api_key_selected: str, config_id: str) -> bool: """ - Elimina una configuración. + Deletes a configuration. Args: - api_key_selected: API Key seleccionada - config_id: ID de la configuración + api_key_selected: Selected API Key + config_id: Configuration ID Returns: - True si se eliminó correctamente, False en caso contrario + True if deleted successfully, False otherwise """ if api_key_selected in self.configs and config_id in self.configs[api_key_selected]: del self.configs[api_key_selected][config_id] diff --git a/corebrain/core/__init__.py b/corebrain/core/__init__.py index b345d11..a5fb552 100644 --- a/corebrain/core/__init__.py +++ b/corebrain/core/__init__.py @@ -1,8 +1,8 @@ """ -Componentes principales del SDK de Corebrain. +Corebrain SDK main components. -Este paquete contiene los componentes centrales del SDK, -incluyendo el cliente principal y el manejo de schemas. +This package contains the core components of the SDK, +including the main client and schema handling. """ from corebrain.core.client import Corebrain, init from corebrain.core.query import QueryCache, QueryAnalyzer, QueryTemplate diff --git a/corebrain/core/client.py b/corebrain/core/client.py index 61e4bdb..51333a5 100644 --- a/corebrain/core/client.py +++ b/corebrain/core/client.py @@ -468,10 +468,10 @@ def _connect_to_database(self) -> None: def _extract_db_schema(self, detail_level: str = "full", specific_collections: List[str] = None) -> Dict[str, Any]: """ - Extrae la estructura de la base de datos para proporcionar contexto a la IA. + Extracts the database schema to provide context to the AI. Returns: - Dict con la estructura de la base de datos organizada por tablas/colecciones + Dictionary with the database structure organized by tables/collections """ logger.info(f"Extrayendo esquema de base de datos. Tipo: {self.db_config['type']}, Motor: {self.db_config.get('engine')}") @@ -657,31 +657,31 @@ def _extract_db_schema(self, detail_level: str = "full", specific_collections: L def list_collections_name(self) -> List[str]: """ - Devuelve una lista de las colecciones o tablas disponibles en la base de datos. + Returns a list of the available collections or tables in the database. Returns: - Lista de collections o tablas + List of collections or tables """ print("Excluded tables: ", self.db_schema.get("excluded_tables", [])) return self.db_schema.get("tables", []) def ask(self, question: str, **kwargs) -> Dict: """ - Realizar una consulta en lenguaje natural a la base de datos. + Perform a natural language query to the database. Args: - question: La pregunta en lenguaje natural - **kwargs: Parámetros adicionales: - - collection_name: Para MongoDB, la colección a consultar - - limit: Número máximo de resultados - - detail_level: Nivel de detalle del esquema ("names_only", "structure", "full") - - auto_select: Si seleccionar automáticamente colecciones - - max_collections: Número máximo de colecciones a incluir - - execute_query: Si ejecutar la consulta (True por defecto) - - explain_results: Si generar explicación de resultados (True por defecto) + question: The natural language question + **kwargs: Additional parameters: + - collection_name: For MongoDB, the collection to query + - limit: Maximum number of results + - detail_level: Schema detail level ("names_only", "structure", "full") + - auto_select: Whether to automatically select collections + - max_collections: Maximum number of collections to include + - execute_query: Whether to execute the query (True by default) + - explain_results: Whether to generate an explanation of results (True by default) Returns: - Dict con los resultados de la consulta y la explicación + Dictionary with the query results and explanation """ try: # Verificar opciones de comportamiento From 53a71ced43191b05d9a41e749bf6502ff408143d Mon Sep 17 00:00:00 2001 From: Vapniak <113126917+Vapniak@users.noreply.github.com> Date: Tue, 20 May 2025 16:43:15 +0200 Subject: [PATCH 17/81] Fix the issue with wrong import --- corebrain/cli/commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index 1008126..9bd3dbb 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -14,8 +14,8 @@ from corebrain.cli.config import configure_sdk, get_api_credential from corebrain.cli.utils import print_colored from corebrain.config.manager import ConfigManager -from config.manager import export_config -from config.manager import validate_config +from corebrain.config.manager import export_config +from corebrain.config.manager import validate_config from corebrain.lib.sso.auth import GlobodainSSOAuth def main_cli(argv: Optional[List[str]] = None) -> int: From 6dd2473448a2d721e3a5f03fdc8670a0cd50b45e Mon Sep 17 00:00:00 2001 From: Albert Stankowski Date: Tue, 20 May 2025 15:59:41 +0100 Subject: [PATCH 18/81] Add translations to config, core, db --- corebrain/core/client.py | 32 +++---- corebrain/core/query.py | 144 ++++++++++++++-------------- corebrain/core/test_utils.py | 20 ++-- corebrain/db/__init__.py | 6 +- corebrain/db/connector.py | 12 +-- corebrain/db/connectors/__init__.py | 8 +- corebrain/db/connectors/mongodb.py | 68 ++++++------- corebrain/db/connectors/sql.py | 66 ++++++------- corebrain/db/engines.py | 6 +- corebrain/db/factory.py | 10 +- corebrain/db/interface.py | 12 +-- corebrain/db/schema/__init__.py | 2 +- corebrain/db/schema/extractor.py | 24 ++--- corebrain/db/schema/optimizer.py | 20 ++-- corebrain/db/schema_file.py | 54 +++++------ 15 files changed, 242 insertions(+), 242 deletions(-) diff --git a/corebrain/core/client.py b/corebrain/core/client.py index 51333a5..9b65225 100644 --- a/corebrain/core/client.py +++ b/corebrain/core/client.py @@ -958,14 +958,14 @@ def ask(self, question: str, **kwargs) -> Dict: def _generate_fallback_explanation(self, query, results): """ - Genera una explicación de respaldo cuando falla la generación de explicaciones. + Generates a fallback explanation when the explanation generation fails. Args: - query: La consulta ejecutada - results: Los resultados obtenidos + query: The executed query + results: The obtained results Returns: - Explicación generada + Generated explanation """ # Determinar si es SQL o MongoDB if isinstance(query, dict): @@ -982,14 +982,14 @@ def _generate_fallback_explanation(self, query, results): def _generate_sql_explanation(self, sql_query, results): """ - Genera una explicación simple para consultas SQL. + Generates a simple explanation for SQL queries. Args: - sql_query: La consulta SQL ejecutada - results: Los resultados obtenidos + sql_query: The executed SQL query + results: The obtained results Returns: - Explicación generada + Generated explanation """ sql_lower = sql_query.lower() if isinstance(sql_query, str) else "" result_count = len(results) if isinstance(results, list) else (1 if results else 0) @@ -1035,14 +1035,14 @@ def _generate_sql_explanation(self, sql_query, results): def _generate_mongodb_explanation(self, query, results): """ - Genera una explicación simple para consultas MongoDB. + Generates a simple explanation for MongoDB queries. Args: - query: La consulta MongoDB ejecutada - results: Los resultados obtenidos + query: The executed MongoDB query + results: The obtained results Returns: - Explicación generada + Generated explanation """ collection = query.get("collection", "la colección") operation = query.get("operation", "find") @@ -1071,14 +1071,14 @@ def _generate_mongodb_explanation(self, query, results): def _generate_generic_explanation(self, query, results): """ - Genera una explicación genérica cuando no se puede determinar el tipo de consulta. + Generates a generic explanation when the query type cannot be determined. Args: - query: La consulta ejecutada - results: Los resultados obtenidos + query: The executed query + results: The obtained results Returns: - Explicación generada + Generated explanation """ result_count = len(results) if isinstance(results, list) else (1 if results else 0) diff --git a/corebrain/core/query.py b/corebrain/core/query.py index 8747c6f..b23eddb 100644 --- a/corebrain/core/query.py +++ b/corebrain/core/query.py @@ -1,5 +1,5 @@ """ -Componentes para manejo y análisis de consultas. +Components for query handling and analysis. """ import os import json @@ -16,16 +16,16 @@ from corebrain.cli.utils import print_colored class QueryCache: - """Sistema de caché multinivel para consultas.""" + """Multilevel cache system for queries.""" def __init__(self, cache_dir: str = None, ttl: int = 86400, memory_limit: int = 100): """ - Inicializa el sistema de caché. + Initializes the cache system. Args: - cache_dir: Directorio para el caché persistente - ttl: Tiempo de vida del caché en segundos (default: 24 horas) - memory_limit: Límite de entradas en caché de memoria + cache_dir: Directory for persistent cache + ttl: Time-to-live of the cache in seconds (default: 24 hours) + memory_limit: Memory cache entry limit """ # Caché en memoria (más rápido, pero volátil) self.memory_cache = {} @@ -50,7 +50,7 @@ def __init__(self, cache_dir: str = None, ttl: int = 86400, memory_limit: int = print_colored(f"Caché inicializado en {self.cache_dir}", "blue") def _init_db(self): - """Inicializa la base de datos SQLite para metadatos de caché.""" + """Initializes the SQLite database for cache metadata.""" conn = sqlite3.connect(str(self.db_path)) cursor = conn.cursor() @@ -70,7 +70,7 @@ def _init_db(self): conn.close() def _get_hash(self, query: str, config_id: str, collection_name: Optional[str] = None) -> str: - """Genera un hash único para la consulta.""" + """Generates a unique hash for the query.""" # Normalizar la consulta (eliminar espacios extra, convertir a minúsculas) normalized_query = re.sub(r'\s+', ' ', query.lower().strip()) @@ -83,7 +83,7 @@ def _get_hash(self, query: str, config_id: str, collection_name: Optional[str] = return hashlib.md5(hash_input.encode()).hexdigest() def _get_cache_path(self, query_hash: str) -> Path: - """Obtiene la ruta del archivo de caché para un hash dado.""" + """Gets the cache file path for a given hash.""" # Usar los primeros caracteres del hash para crear subdirectorios # Esto evita tener demasiados archivos en un solo directorio subdir = query_hash[:2] @@ -93,7 +93,7 @@ def _get_cache_path(self, query_hash: str) -> Path: return cache_subdir / f"{query_hash}.cache" def _update_metadata(self, query_hash: str, query: str, config_id: str): - """Actualiza los metadatos en la base de datos.""" + """Updates the metadata in the database.""" conn = sqlite3.connect(str(self.db_path)) cursor = conn.cursor() @@ -122,7 +122,7 @@ def _update_metadata(self, query_hash: str, query: str, config_id: str): conn.close() def _update_memory_lru(self, query_hash: str): - """Actualiza la lista LRU (Least Recently Used) para el caché en memoria.""" + """Updates the LRU (Least Recently Used) list for the in-memory cache.""" if query_hash in self.memory_lru: # Mover al final (más recientemente usado) self.memory_lru.remove(query_hash) @@ -138,15 +138,15 @@ def _update_memory_lru(self, query_hash: str): def get(self, query: str, config_id: str, collection_name: Optional[str] = None) -> Optional[Dict[str, Any]]: """ - Obtiene un resultado cacheado si existe y no ha expirado. + Retrieves a cached result if it exists and has not expired. Args: - query: Consulta en lenguaje natural - config_id: ID de configuración de la base de datos - collection_name: Nombre de la colección/tabla (opcional) + query: Natural language query + config_id: Database configuration ID + collection_name: Name of the collection/table (optional) Returns: - Resultado cacheado o None si no existe o ha expirado + Cached result or None if it does not exist or has expired """ query_hash = self._get_hash(query, config_id, collection_name) @@ -195,13 +195,13 @@ def get(self, query: str, config_id: str, collection_name: Optional[str] = None) def set(self, query: str, config_id: str, result: Dict[str, Any], collection_name: Optional[str] = None): """ - Guarda un resultado en el caché. + Saves a result in the cache. Args: - query: Consulta en lenguaje natural - config_id: ID de configuración - result: Resultado a cachear - collection_name: Nombre de la colección/tabla (opcional) + query: Natural language query + config_id: Configuration ID + result: Result to cache + collection_name: Name of the collection/table (optional) """ query_hash = self._get_hash(query, config_id, collection_name) @@ -225,10 +225,10 @@ def set(self, query: str, config_id: str, result: Dict[str, Any], collection_nam def clear(self, older_than: int = None): """ - Limpia el caché. + Clears the cache. Args: - older_than: Limpiar solo entradas más antiguas que esta cantidad de segundos + older_than: Only clear entries older than this number of seconds """ # Limpiar caché en memoria if older_than: @@ -297,7 +297,7 @@ def clear(self, older_than: int = None): conn.close() def get_stats(self) -> Dict[str, Any]: - """Obtiene estadísticas del caché.""" + """Gets cache statistics.""" # Contar archivos en disco disk_count = 0 for subdir in self.cache_dir.iterdir(): @@ -336,7 +336,7 @@ def get_stats(self) -> Dict[str, Any]: } class QueryTemplate: - """Plantilla de consulta predefinida para patrones comunes.""" + """Predefined query template for common patterns.""" def __init__(self, pattern: str, description: str, sql_template: Optional[str] = None, @@ -344,15 +344,15 @@ def __init__(self, pattern: str, description: str, db_type: str = "sql", applicable_tables: Optional[List[str]] = None): """ - Inicializa una plantilla de consulta. + Initializes a query template. Args: - pattern: Patrón de lenguaje natural que coincide con esta plantilla - description: Descripción de la plantilla - sql_template: Plantilla SQL con marcadores para parámetros - generator_func: Función alternativa para generar la consulta - db_type: Tipo de base de datos (sql, mongodb) - applicable_tables: Lista de tablas a las que aplica esta plantilla + pattern: Natural language pattern that matches this template + description: Description of the template + sql_template: SQL template with placeholders for parameters + generator_func: Alternative function to generate the query + db_type: Database type (sql, mongodb) + applicable_tables: List of tables to which this template applies """ self.pattern = pattern self.description = description @@ -365,7 +365,7 @@ def __init__(self, pattern: str, description: str, self.regex = self._compile_pattern(pattern) def _compile_pattern(self, pattern: str) -> re.Pattern: - """Compila el patrón en una expresión regular.""" + """Compiles the pattern into a regular expression.""" # Reemplazar marcadores especiales con grupos de captura regex_pattern = pattern @@ -388,13 +388,13 @@ def _compile_pattern(self, pattern: str) -> re.Pattern: def matches(self, query: str) -> Tuple[bool, List[str]]: """ - Verifica si una consulta coincide con esta plantilla. + Checks if a query matches this template. Args: - query: Consulta a verificar + query: Query to check Returns: - Tupla de (coincide, [parámetros capturados]) + Tuple of (match, [captured parameters]) """ match = self.regex.match(query) if match: @@ -403,14 +403,14 @@ def matches(self, query: str) -> Tuple[bool, List[str]]: def generate_query(self, params: List[str], db_schema: Dict[str, Any]) -> Optional[Dict[str, Any]]: """ - Genera una consulta a partir de los parámetros capturados. + Generates a query from the captured parameters. Args: - params: Parámetros capturados del patrón - db_schema: Esquema de la base de datos + params: Captured parameters from the pattern + db_schema: Database schema Returns: - Consulta generada o None si no se puede generar + Generated query or None if it cannot be generated """ if self.generator_func: # Usar función personalizada @@ -437,15 +437,15 @@ def generate_query(self, params: List[str], db_schema: Dict[str, Any]) -> Option return None class QueryAnalyzer: - """Analiza patrones de consultas para sugerir optimizaciones.""" + """Analyzes query patterns to suggest optimizations.""" def __init__(self, query_log_path: str = None, template_path: str = None): """ - Inicializa el analizador de consultas. + Initializes the query analyzer. Args: - query_log_path: Ruta al archivo de registro de consultas - template_path: Ruta al archivo de plantillas + query_log_path: Path to the query log file + template_path: Path to the template file """ self.query_log_path = query_log_path or os.path.join( Path.home(), ".corebrain_cache", "query_log.db" @@ -474,7 +474,7 @@ def __init__(self, query_log_path: str = None, template_path: str = None): ] def _init_db(self): - """Inicializa la base de datos para el registro de consultas.""" + """Initializes the database for query logging.""" # Asegurar que el directorio existe os.makedirs(os.path.dirname(self.query_log_path), exist_ok=True) @@ -650,7 +650,7 @@ def _load_default_templates(self) -> List[QueryTemplate]: return templates def _load_custom_templates(self): - """Carga plantillas personalizadas desde el archivo.""" + """Loads custom templates from the file.""" if not os.path.exists(self.template_path): return @@ -675,13 +675,13 @@ def _load_custom_templates(self): def save_custom_template(self, template: QueryTemplate) -> bool: """ - Guarda una plantilla personalizada. + Saves a custom template. Args: - template: Plantilla a guardar + template: Template to save Returns: - True si se guardó correctamente + True if saved successfully """ # Cargar plantillas existentes custom_templates = [] @@ -726,14 +726,14 @@ def save_custom_template(self, template: QueryTemplate) -> bool: def find_matching_template(self, query: str, db_schema: Dict[str, Any]) -> Optional[Tuple[QueryTemplate, List[str]]]: """ - Busca una plantilla que coincida con la consulta. + Searches for a template that matches the query. Args: - query: Consulta en lenguaje natural - db_schema: Esquema de la base de datos + query: Natural language query + db_schema: Database schema Returns: - Tupla de (plantilla, parámetros) o None si no hay coincidencia + Tuple of (template, parameters) or None if no match is found """ for template in self.templates: matches, params = template.matches(query) @@ -751,15 +751,15 @@ def find_matching_template(self, query: str, db_schema: Dict[str, Any]) -> Optio def log_query(self, query: str, config_id: str, collection_name: str = None, execution_time: float = 0, cost: float = 0.09, result_count: int = 0): """ - Registra una consulta para análisis. + Registers a query for analysis. Args: - query: Consulta en lenguaje natural - config_id: ID de configuración - collection_name: Nombre de la colección/tabla - execution_time: Tiempo de ejecución en segundos - cost: Costo estimado de la consulta - result_count: Número de resultados obtenidos + query: Natural language query + config_id: Configuration ID + collection_name: Name of the collection/table + execution_time: Execution time in seconds + cost: Estimated cost of the query + result_count: Number of results obtained """ # Detectar patrón pattern = self._detect_pattern(query) @@ -808,13 +808,13 @@ def log_query(self, query: str, config_id: str, collection_name: str = None, def _detect_pattern(self, query: str) -> Optional[str]: """ - Detecta un patrón en la consulta. + Detects a pattern in the query. Args: - query: Consulta a analizar + query: Query to analyze Returns: - Patrón detectado o None + Detected pattern or None """ normalized_query = query.lower() @@ -842,13 +842,13 @@ def _detect_pattern(self, query: str) -> Optional[str]: def get_common_patterns(self, limit: int = 5) -> List[Dict[str, Any]]: """ - Obtiene los patrones de consulta más comunes. + Retrieves the most common query patterns. Args: - limit: Número máximo de patrones a devolver + limit: Maximum number of patterns to return Returns: - Lista de patrones más comunes + List of the most common patterns """ conn = sqlite3.connect(self.query_log_path) cursor = conn.cursor() @@ -876,14 +876,14 @@ def get_common_patterns(self, limit: int = 5) -> List[Dict[str, Any]]: def suggest_new_template(self, query: str, sql_query: str) -> Optional[QueryTemplate]: """ - Sugiere una nueva plantilla basada en una consulta exitosa. + Suggests a new template based on a successful query. Args: - query: Consulta en lenguaje natural - sql_query: Consulta SQL generada + query: Natural language query + sql_query: Generated SQL query Returns: - Plantilla sugerida o None + Suggested template or None """ # Detectar patrón pattern = self._detect_pattern(query) @@ -924,10 +924,10 @@ def suggest_new_template(self, query: str, sql_query: str) -> Optional[QueryTemp def get_optimization_suggestions(self) -> List[Dict[str, Any]]: """ - Genera sugerencias para optimizar consultas. + Generates suggestions to optimize queries. Returns: - Lista de sugerencias de optimización + List of optimization suggestions """ suggestions = [] diff --git a/corebrain/core/test_utils.py b/corebrain/core/test_utils.py index 543eee8..5e334d9 100644 --- a/corebrain/core/test_utils.py +++ b/corebrain/core/test_utils.py @@ -1,5 +1,5 @@ """ -Utilidades para pruebas y validación de componentes. +Utilities for testing and validating components. """ import json import random @@ -11,13 +11,13 @@ def generate_test_question_from_schema(schema: Dict[str, Any]) -> str: """ - Genera una pregunta de prueba basada en el esquema de la base de datos. + Generates a test question based on the database schema. Args: - schema: Esquema de la base de datos + schema: Database schema Returns: - Pregunta de prueba generada + Generated test question """ if not schema or not schema.get("tables"): return "¿Cuáles son las tablas disponibles?" @@ -59,16 +59,16 @@ def generate_test_question_from_schema(schema: Dict[str, Any]) -> str: def test_natural_language_query(api_token: str, db_config: Dict[str, Any], api_url: Optional[str] = None, user_data: Optional[Dict[str, Any]] = None) -> bool: """ - Prueba una consulta de lenguaje natural. + Tests a natural language query. Args: - api_token: Token de API - db_config: Configuración de la base de datos - api_url: URL opcional de la API - user_data: Datos del usuario + api_token: API token + db_config: Database configuration + api_url: Optional API URL + user_data: User data Returns: - True si la prueba es exitosa, False en caso contrario + True if the test is successful, False otherwise """ try: print_colored("\nRealizando prueba de consulta en lenguaje natural...", "blue") diff --git a/corebrain/db/__init__.py b/corebrain/db/__init__.py index e362e76..6ac390d 100644 --- a/corebrain/db/__init__.py +++ b/corebrain/db/__init__.py @@ -1,8 +1,8 @@ """ -Conectores de bases de datos para Corebrain SDK. +Database connectors for Corebrain SDK. -Este paquete proporciona conectores para diferentes tipos y -motores de bases de datos soportados por Corebrain. +This package provides connectors for different types and +database engines supported by Corebrain. """ from corebrain.db.connector import DatabaseConnector from corebrain.db.factory import get_connector diff --git a/corebrain/db/connector.py b/corebrain/db/connector.py index 4a54f4e..886a2a9 100644 --- a/corebrain/db/connector.py +++ b/corebrain/db/connector.py @@ -1,10 +1,10 @@ """ -Conectores base para diferentes tipos de bases de datos. +Base connectors for different types of databases. """ from typing import Dict, Any, List, Optional, Callable class DatabaseConnector: - """Clase base para todos los conectores de base de datos""" + """Base class for all database connectors.""" def __init__(self, config: Dict[str, Any], timeout: int = 10): self.config = config @@ -12,20 +12,20 @@ def __init__(self, config: Dict[str, Any], timeout: int = 10): self.connection = None def connect(self): - """Establece conexión a la base de datos""" + """Establishes a connection to the database.""" raise NotImplementedError def extract_schema(self, sample_limit: int = 5, table_limit: Optional[int] = None, progress_callback: Optional[Callable] = None) -> Dict[str, Any]: - """Extrae el esquema de la base de datos""" + """Extracts the database schema.""" raise NotImplementedError def execute_query(self, query: str) -> List[Dict[str, Any]]: - """Ejecuta una consulta en la base de datos""" + """Executes a query on the database.""" raise NotImplementedError def close(self): - """Cierra la conexión""" + """Closes the connection.""" if self.connection: try: self.connection.close() diff --git a/corebrain/db/connectors/__init__.py b/corebrain/db/connectors/__init__.py index 8b4af97..3db5c71 100644 --- a/corebrain/db/connectors/__init__.py +++ b/corebrain/db/connectors/__init__.py @@ -1,5 +1,5 @@ """ -Conectores de bases de datos para diferentes motores. +Database connectors for different engines. """ from typing import Dict, Any @@ -9,13 +9,13 @@ def get_connector(db_config: Dict[str, Any]): """ - Obtiene el conector adecuado según la configuración de la base de datos. + Gets the appropriate connector based on the database configuration. Args: - db_config: Configuración de la base de datos + db_config: Database configuration Returns: - Instancia del conector apropiado + Instance of the appropriate connector """ db_type = db_config.get("type", "").lower() diff --git a/corebrain/db/connectors/mongodb.py b/corebrain/db/connectors/mongodb.py index ae57cc2..1b98575 100644 --- a/corebrain/db/connectors/mongodb.py +++ b/corebrain/db/connectors/mongodb.py @@ -1,5 +1,5 @@ """ -Conector para bases de datos MongoDB. +Connector for MongoDB databases. """ import time @@ -18,14 +18,14 @@ from corebrain.db.connector import DatabaseConnector class MongoDBConnector(DatabaseConnector): - """Conector optimizado para MongoDB""" + """Optimized connector for MongoDB.""" def __init__(self, config: Dict[str, Any]): """ - Inicializa el conector MongoDB con la configuración proporcionada. + Initializes the MongoDB connector with the provided configuration. Args: - config: Diccionario con la configuración de conexión + config: Dictionary with the connection configuration """ super().__init__(config) self.client = None @@ -38,10 +38,10 @@ def __init__(self, config: Dict[str, Any]): def connect(self) -> bool: """ - Establece conexión con timeout optimizado + Establishes a connection with optimized timeout. Returns: - True si la conexión fue exitosa, False en caso contrario + True if the connection was successful, False otherwise """ if not PYMONGO_AVAILABLE: raise ImportError("pymongo no está instalado. Instálalo con 'pip install pymongo'") @@ -131,15 +131,15 @@ def connect(self) -> bool: def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] = None, progress_callback: Optional[Callable] = None) -> Dict[str, Any]: """ - Extrae el esquema con límites y progreso para mejorar rendimiento + Extracts the schema with limits and progress to improve performance. Args: - sample_limit: Número máximo de documentos de muestra por colección - collection_limit: Límite de colecciones a procesar (None para todas) - progress_callback: Función opcional para reportar progreso + sample_limit: Maximum number of sample documents per collection + collection_limit: Limit of collections to process (None for all) + progress_callback: Optional function to report progress Returns: - Diccionario con el esquema de la base de datos + Dictionary with the database schema """ # Asegurar que estamos conectados if not self.client and not self.connect(): @@ -232,14 +232,14 @@ def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] def _extract_document_fields(self, doc: Dict[str, Any], fields: Dict[str, str], prefix: str = "", max_depth: int = 3, current_depth: int = 0) -> None: """ - Extrae recursivamente los campos y tipos de un documento MongoDB. + Recursively extracts fields and types from a MongoDB document. Args: - doc: Documento a analizar - fields: Diccionario donde guardar los campos y tipos - prefix: Prefijo para campos anidados - max_depth: Profundidad máxima para campos anidados - current_depth: Profundidad actual + doc: Document to analyze + fields: Dictionary to store fields and types + prefix: Prefix for nested fields + max_depth: Maximum depth for nested fields + current_depth: Current depth """ if current_depth >= max_depth: return @@ -275,13 +275,13 @@ def _extract_document_fields(self, doc: Dict[str, Any], fields: Dict[str, str], def _process_document_for_serialization(self, doc: Dict[str, Any]) -> Dict[str, Any]: """ - Procesa un documento para ser serializable a JSON. + Processes a document to be JSON serializable. Args: - doc: Documento a procesar + doc: Document to process Returns: - Documento procesado + Processed document """ processed_doc = {} for field, value in doc.items(): @@ -313,13 +313,13 @@ def _process_document_for_serialization(self, doc: Dict[str, Any]) -> Dict[str, def execute_query(self, query: str) -> List[Dict[str, Any]]: """ - Ejecuta una consulta MongoDB con manejo de errores mejorado + Executes a MongoDB query with improved error handling. Args: - query: Consulta MongoDB en formato JSON o lenguaje de consulta + query: MongoDB query in JSON format or query language Returns: - Lista de documentos resultantes + List of resulting documents """ if not self.client and not self.connect(): raise ConnectionError("No se pudo establecer conexión con MongoDB") @@ -379,13 +379,13 @@ def execute_query(self, query: str) -> List[Dict[str, Any]]: def _parse_query(self, query: str) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]], str, Optional[int]]: """ - Analiza una consulta y extrae los componentes necesarios. + Analyzes a query and extracts the necessary components. Args: - query: Consulta en formato string + query: Query in string format Returns: - Tupla con (filtro, proyección, nombre de colección, límite) + Tuple with (filter, projection, collection name, limit) """ # Intentar parsear como JSON try: @@ -423,14 +423,14 @@ def _parse_query(self, query: str) -> Tuple[Dict[str, Any], Optional[Dict[str, A def count_documents(self, collection_name: str, filter_dict: Optional[Dict[str, Any]] = None) -> int: """ - Cuenta documentos en una colección + Counts documents in a collection. Args: - collection_name: Nombre de la colección - filter_dict: Filtro opcional + collection_name: Name of the collection + filter_dict: Optional filter Returns: - Número de documentos + Number of documents """ if not self.client and not self.connect(): raise ConnectionError("No se pudo establecer conexión con MongoDB") @@ -444,10 +444,10 @@ def count_documents(self, collection_name: str, filter_dict: Optional[Dict[str, def list_collections(self) -> List[str]: """ - Devuelve una lista de colecciones en la base de datos + Returns a list of collections in the database. Returns: - Lista de nombres de colecciones + List of collection names """ if not self.client and not self.connect(): raise ConnectionError("No se pudo establecer conexión con MongoDB") @@ -459,7 +459,7 @@ def list_collections(self) -> List[str]: return [] def close(self) -> None: - """Cierra la conexión a MongoDB""" + """Closes the MongoDB connection.""" if self.client: try: self.client.close() @@ -470,5 +470,5 @@ def close(self) -> None: self.db = None def __del__(self): - """Destructor para asegurar que la conexión se cierre""" + """Destructor to ensure the connection is closed.""" self.close() \ No newline at end of file diff --git a/corebrain/db/connectors/sql.py b/corebrain/db/connectors/sql.py index 36f3f89..82f49bf 100644 --- a/corebrain/db/connectors/sql.py +++ b/corebrain/db/connectors/sql.py @@ -19,14 +19,14 @@ from corebrain.db.connector import DatabaseConnector class SQLConnector(DatabaseConnector): - """Conector optimizado para bases de datos SQL""" + """Optimized connector for SQL databases.""" def __init__(self, config: Dict[str, Any]): """ - Inicializa el conector SQL con la configuración proporcionada. + Initializes the SQL connector with the provided configuration. Args: - config: Diccionario con la configuración de conexión + config: Dictionary with the connection configuration """ super().__init__(config) self.conn = None @@ -37,10 +37,10 @@ def __init__(self, config: Dict[str, Any]): def connect(self) -> bool: """ - Establece conexión con timeout optimizado + Establishes a connection with optimized timeout. Returns: - True si la conexión fue exitosa, False en caso contrario + True if the connection was successful, False otherwise """ try: start_time = time.time() @@ -128,15 +128,15 @@ def connect(self) -> bool: def extract_schema(self, sample_limit: int = 5, table_limit: Optional[int] = None, progress_callback: Optional[Callable] = None) -> Dict[str, Any]: """ - Extrae el esquema con límites y progreso + Extracts the schema with limits and progress. Args: - sample_limit: Límite de muestras de datos por tabla - table_limit: Límite de tablas a procesar (None para todas) - progress_callback: Función opcional para reportar progreso + sample_limit: Data sample limit per table + table_limit: Limit of tables to process (None for all) + progress_callback: Optional function to report progress Returns: - Diccionario con el esquema de la base de datos + Dictionary with the database schema """ # Asegurar que estamos conectados if not self.conn and not self.connect(): @@ -162,13 +162,13 @@ def extract_schema(self, sample_limit: int = 5, table_limit: Optional[int] = Non def execute_query(self, query: str) -> List[Dict[str, Any]]: """ - Ejecuta una consulta SQL con manejo de errores mejorado + Executes an SQL query with improved error handling. Args: - query: Consulta SQL a ejecutar + query: SQL query to execute Returns: - Lista de filas resultantes como diccionarios + List of resulting rows as dictionaries """ if not self.conn and not self.connect(): raise ConnectionError("No se pudo establecer conexión con la base de datos") @@ -206,7 +206,7 @@ def execute_query(self, query: str) -> List[Dict[str, Any]]: raise Exception(f"Error al ejecutar consulta (después de reconexión): {str(e)}") def _execute_sqlite_query(self, query: str) -> List[Dict[str, Any]]: - """Ejecuta una consulta en SQLite""" + """Executes a query in SQLite.""" cursor = self.conn.cursor() cursor.execute(query) @@ -225,7 +225,7 @@ def _execute_sqlite_query(self, query: str) -> List[Dict[str, Any]]: return result def _execute_mysql_query(self, query: str) -> List[Dict[str, Any]]: - """Ejecuta una consulta en MySQL""" + """Executes a query in MySQL.""" cursor = self.conn.cursor(dictionary=True) cursor.execute(query) result = cursor.fetchall() @@ -233,7 +233,7 @@ def _execute_mysql_query(self, query: str) -> List[Dict[str, Any]]: return result def _execute_postgresql_query(self, query: str) -> List[Dict[str, Any]]: - """Ejecuta una consulta en PostgreSQL""" + """Executes a query in PostgreSQL.""" cursor = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) cursor.execute(query) results = [dict(row) for row in cursor.fetchall()] @@ -242,15 +242,15 @@ def _execute_postgresql_query(self, query: str) -> List[Dict[str, Any]]: def _extract_sqlite_schema(self, sample_limit: int, table_limit: Optional[int], progress_callback: Optional[Callable]) -> Dict[str, Any]: """ - Extrae schema específico para SQLite + Extracts specific schema for SQLite. Args: - sample_limit: Número máximo de filas de muestra por tabla - table_limit: Número máximo de tablas a extraer - progress_callback: Función para reportar progreso + sample_limit: Maximum number of sample rows per table + table_limit: Maximum number of tables to extract + progress_callback: Function to report progress Returns: - Diccionario con el esquema de la base de datos + Dictionary with the database schema """ schema = { "type": "sql", @@ -328,15 +328,15 @@ def _extract_sqlite_schema(self, sample_limit: int, table_limit: Optional[int], def _extract_mysql_schema(self, sample_limit: int, table_limit: Optional[int], progress_callback: Optional[Callable]) -> Dict[str, Any]: """ - Extrae schema específico para MySQL + Extracts specific schema for MySQL. Args: - sample_limit: Número máximo de filas de muestra por tabla - table_limit: Número máximo de tablas a extraer - progress_callback: Función para reportar progreso + sample_limit: Maximum number of sample rows per table + table_limit: Maximum number of tables to extract + progress_callback: Function to report progress Returns: - Diccionario con el esquema de la base de datos + Dictionary with the database schema """ schema = { "type": "sql", @@ -424,15 +424,15 @@ def _extract_mysql_schema(self, sample_limit: int, table_limit: Optional[int], p def _extract_postgresql_schema(self, sample_limit: int, table_limit: Optional[int], progress_callback: Optional[Callable]) -> Dict[str, Any]: """ - Extrae schema específico para PostgreSQL con optimizaciones + Extracts specific schema for PostgreSQL with optimizations. Args: - sample_limit: Número máximo de filas de muestra por tabla - table_limit: Número máximo de tablas a extraer - progress_callback: Función para reportar progreso + sample_limit: Maximum number of sample rows per table + table_limit: Maximum number of tables to extract + progress_callback: Function to report progress Returns: - Diccionario con el esquema de la base de datos + Dictionary with the database schema """ schema = { "type": "sql", @@ -584,7 +584,7 @@ def _extract_postgresql_schema(self, sample_limit: int, table_limit: Optional[in return schema def close(self) -> None: - """Cierra la conexión a la base de datos""" + """Closes the database connection.""" if self.conn: try: self.conn.close() @@ -594,5 +594,5 @@ def close(self) -> None: self.conn = None def __del__(self): - """Destructor para asegurar que la conexión se cierre""" + """Destructor to ensure the connection is closed.""" self.close() \ No newline at end of file diff --git a/corebrain/db/engines.py b/corebrain/db/engines.py index 9b9c866..51b51d6 100644 --- a/corebrain/db/engines.py +++ b/corebrain/db/engines.py @@ -1,14 +1,14 @@ """ -Información sobre motores de bases de datos soportados. +Information about supported database engines. """ from typing import Dict, List def get_available_engines() -> Dict[str, List[str]]: """ - Devuelve los motores de base de datos disponibles por tipo + Returns the available database engines by type. Returns: - Dict con tipos de DB y lista de motores por tipo + Dict with DB types and a list of engines per type """ return { "sql": ["sqlite", "mysql", "postgresql"], diff --git a/corebrain/db/factory.py b/corebrain/db/factory.py index afbc0dd..c2c23bc 100644 --- a/corebrain/db/factory.py +++ b/corebrain/db/factory.py @@ -1,5 +1,5 @@ """ -Fábrica de conectores de base de datos. +Database connector factory. """ from typing import Dict, Any @@ -9,14 +9,14 @@ def get_connector(db_config: Dict[str, Any], timeout: int = 10) -> DatabaseConnector: """ - Fábrica de conectores de base de datos según la configuración + Database connector factory based on configuration. Args: - db_config: Configuración de la base de datos - timeout: Timeout para operaciones de DB + db_config: Database configuration + timeout: Timeout for DB operations Returns: - Instancia de conector apropiado + Instance of the appropriate connector """ db_type = db_config.get("type", "").lower() engine = db_config.get("engine", "").lower() diff --git a/corebrain/db/interface.py b/corebrain/db/interface.py index d2b3ae5..d8373ff 100644 --- a/corebrain/db/interface.py +++ b/corebrain/db/interface.py @@ -1,5 +1,5 @@ """ -Interfaces abstractas para conexiones de bases de datos. +Abstract interfaces for database connections. """ from typing import Dict, Any, List, Optional, Protocol from abc import ABC, abstractmethod @@ -7,26 +7,26 @@ from corebrain.core.common import ConfigDict, SchemaDict class DatabaseConnector(ABC): - """Interfaz abstracta para conectores de bases de datos""" + """Abstract interface for database connectors.""" @abstractmethod def connect(self, config: ConfigDict) -> Any: - """Establece conexión con la base de datos""" + """Establishes a connection with the database.""" pass @abstractmethod def extract_schema(self, connection: Any) -> SchemaDict: - """Extrae el esquema de la base de datos""" + """Extracts the database schema.""" pass @abstractmethod def execute_query(self, connection: Any, query: str) -> List[Dict[str, Any]]: - """Ejecuta una consulta y devuelve resultados""" + """Executes a query and returns results.""" pass @abstractmethod def close(self, connection: Any) -> None: - """Cierra la conexión""" + """Closes the connection.""" pass # Posteriormente se podrían implementar conectores específicos: diff --git a/corebrain/db/schema/__init__.py b/corebrain/db/schema/__init__.py index 25d843c..388ee63 100644 --- a/corebrain/db/schema/__init__.py +++ b/corebrain/db/schema/__init__.py @@ -1,5 +1,5 @@ """ -Componentes para extracción y optimización de esquemas de base de datos. +Components for extracting and optimizing database schemas. """ from .extractor import extract_schema from .optimizer import SchemaOptimizer diff --git a/corebrain/db/schema/extractor.py b/corebrain/db/schema/extractor.py index 3051951..c361b83 100644 --- a/corebrain/db/schema/extractor.py +++ b/corebrain/db/schema/extractor.py @@ -1,7 +1,7 @@ # db/schema/extractor.py (reemplaza la importación circular en db/schema.py) """ -Extractor de esquemas de bases de datos independiente. +Independent database schema extractor. """ from typing import Dict, Any, Optional, Callable @@ -12,14 +12,14 @@ def extract_db_schema(db_config: Dict[str, Any], client_factory: Optional[Callable] = None) -> Dict[str, Any]: """ - Extrae el esquema de la base de datos con inyección de dependencias. + Extracts the database schema with dependency injection. Args: - db_config: Configuración de la base de datos - client_factory: Función opcional para crear un cliente (evita importación circular) + db_config: Database configuration + client_factory: Optional function to create a client (avoids circular imports) Returns: - Diccionario con la estructura de la base de datos + Dictionary with the database structure """ db_type = db_config.get("type", "").lower() schema = { @@ -75,11 +75,11 @@ def extract_db_schema(db_config: Dict[str, Any], client_factory: Optional[Callab def create_schema_from_corebrain() -> Callable: """ - Crea una función de extracción que usa Corebrain internamente. - Carga dinámicamente para evitar importaciones circulares. + Creates an extraction function that uses Corebrain internally. + Loads dynamically to avoid circular imports. Returns: - Función que extrae schema usando Corebrain + Function that extracts schema using Corebrain """ def extract_with_corebrain(db_config: Dict[str, Any]) -> Dict[str, Any]: # Importar dinámicamente para evitar circular @@ -105,14 +105,14 @@ def extract_with_corebrain(db_config: Dict[str, Any]) -> Dict[str, Any]: # Función pública expuesta def extract_schema(db_config: Dict[str, Any], use_corebrain: bool = False) -> Dict[str, Any]: """ - Función pública que decide cómo extraer el schema. + Public function that decides how to extract the schema. Args: - db_config: Configuración de la base de datos - use_corebrain: Si es True, usa la clase Corebrain para extracción + db_config: Database configuration + use_corebrain: If True, uses the Corebrain class for extraction Returns: - Esquema de la base de datos + Database schema """ if use_corebrain: # Intentar usar Corebrain si se solicita diff --git a/corebrain/db/schema/optimizer.py b/corebrain/db/schema/optimizer.py index c85f286..c7840b0 100644 --- a/corebrain/db/schema/optimizer.py +++ b/corebrain/db/schema/optimizer.py @@ -1,5 +1,5 @@ """ -Componentes para optimización de esquemas de base de datos. +Components for database schema optimization. """ import re from typing import Dict, Any, Optional @@ -9,16 +9,16 @@ logger = get_logger(__name__) class SchemaOptimizer: - """Optimiza el esquema de la base de datos para reducir tamaño de contexto.""" + """Optimizes the database schema to reduce context size.""" def __init__(self, max_tables: int = 10, max_columns_per_table: int = 15, max_samples: int = 2): """ - Inicializa el optimizador de esquema. + Initializes the schema optimizer. Args: - max_tables: Máximo número de tablas a incluir - max_columns_per_table: Máximo número de columnas por tabla - max_samples: Máximo número de filas de muestra por tabla + max_tables: Maximum number of tables to include + max_columns_per_table: Maximum number of columns per table + max_samples: Maximum number of sample rows per table """ self.max_tables = max_tables self.max_columns_per_table = max_columns_per_table @@ -38,14 +38,14 @@ def __init__(self, max_tables: int = 10, max_columns_per_table: int = 15, max_sa def optimize_schema(self, db_schema: Dict[str, Any], query: str = None) -> Dict[str, Any]: """ - Optimiza el esquema para reducir su tamaño. + Optimizes the schema to reduce its size. Args: - db_schema: Esquema original de la base de datos - query: Consulta del usuario (para priorizar tablas relevantes) + db_schema: Original database schema + query: User query (to prioritize relevant tables) Returns: - Esquema optimizado + Optimized schema """ # Crear copia para no modificar el original optimized_schema = { diff --git a/corebrain/db/schema_file.py b/corebrain/db/schema_file.py index 445681b..c4dc8f7 100644 --- a/corebrain/db/schema_file.py +++ b/corebrain/db/schema_file.py @@ -1,12 +1,12 @@ """ -Componentes para extracción y optimización de esquemas de base de datos. +Components for extracting and optimizing database schemas. """ import json from typing import Dict, Any, Optional def _print_colored(message: str, color: str) -> None: - """Versión simplificada de _print_colored que no depende de cli.utils""" + """Simplified version of _print_colored that doesn't depend on cli.utils.""" colors = { "red": "\033[91m", "green": "\033[92m", @@ -19,13 +19,13 @@ def _print_colored(message: str, color: str) -> None: def extract_db_schema(db_config: Dict[str, Any]) -> Dict[str, Any]: """ - Extrae el esquema de la base de datos directamente sin usar el SDK. + Extracts the database schema directly without using the SDK. Args: - db_config: Configuración de la base de datos + db_config: Database configuration Returns: - Diccionario con la estructura de la base de datos organizada por tablas/colecciones + Dictionary with the database structure organized by tables/collections """ db_type = db_config["type"].lower() schema = { @@ -154,8 +154,8 @@ def extract_db_schema(db_config: Dict[str, Any]) -> Dict[str, Any]: def extract_db_schema_direct(db_config: Dict[str, Any]) -> Dict[str, Any]: """ - Extrae el esquema directamente sin usar el cliente de Corebrain. - Esta es una versión reducida que no requiere importar core. + Extracts the schema directly without using the Corebrain client. + This is a reduced version that doesn't require importing core. """ db_type = db_config["type"].lower() schema = { @@ -176,10 +176,10 @@ def extract_db_schema_direct(db_config: Dict[str, Any]) -> Dict[str, Any]: def extract_schema_with_lazy_init(api_key: str, db_config: Dict[str, Any], api_url: Optional[str] = None) -> Dict[str, Any]: """ - Extrae esquema usando importación tardía del cliente. + Extracts the schema using late import of the client. - Esta función evita el problema de importación circular cargando - dinámicamente el cliente de Corebrain solo cuando es necesario. + This function avoids the circular import issue by dynamically loading + the Corebrain client only when necessary. """ try: # La importación se mueve aquí para evitar el problema de circular import @@ -231,16 +231,16 @@ def test_connection(db_config: Dict[str, Any]) -> bool: def extract_schema_to_file(api_key: str, config_id: Optional[str] = None, output_file: Optional[str] = None, api_url: Optional[str] = None) -> bool: """ - Extrae el esquema de la base de datos y lo guarda en un archivo. + Extracts the database schema and saves it to a file. Args: - api_key: API Key para identificar la configuración - config_id: ID de configuración específico (opcional) - output_file: Ruta al archivo donde guardar el esquema - api_url: URL opcional de la API + api_key: API Key to identify the configuration + config_id: Specific configuration ID (optional) + output_file: Path to the file where the schema will be saved + api_url: Optional API URL Returns: - True si se extrae correctamente, False en caso contrario + True if extraction is successful, False otherwise """ try: # Importación explícita con try-except para manejar errores @@ -327,12 +327,12 @@ def extract_schema_to_file(api_key: str, config_id: Optional[str] = None, output def show_db_schema(api_token: str, config_id: Optional[str] = None, api_url: Optional[str] = None) -> None: """ - Muestra el esquema de la base de datos configurada. + Displays the schema of the configured database. Args: - api_token: Token de API - config_id: ID de configuración específico (opcional) - api_url: URL opcional de la API + api_token: API token + config_id: Specific configuration ID (optional) + api_url: Optional API URL """ try: # Importación explícita con try-except para manejar errores @@ -407,7 +407,7 @@ def show_db_schema(api_token: str, config_id: Optional[str] = None, api_url: Opt api_token=api_token, config_id=selected_config_id, api_url=api_url, - skip_verification=True # Omitimos verificación para simplificar + skip_verification=True # Skip verification for simplicity ) """ @@ -555,16 +555,16 @@ def show_db_schema(api_token: str, config_id: Optional[str] = None, api_url: Opt def get_schema_with_dynamic_import(api_token: str, config_id: str, db_config: Dict[str, Any], api_url: Optional[str] = None) -> Dict[str, Any]: """ - Obtiene el esquema de la base de datos usando importación dinámica. + Retrieves the database schema using dynamic import. Args: - api_token: Token de API - config_id: ID de configuración - db_config: Configuración de la base de datos - api_url: URL opcional de la API + api_token: API token + config_id: Configuration ID + db_config: Database configuration + api_url: Optional API URL Returns: - Esquema de la base de datos + Database schema """ try: # Importación dinámica del módulo core From cfee1214986b7451b03c28237076587778571e63 Mon Sep 17 00:00:00 2001 From: Piotr_Piskorz Date: Tue, 20 May 2025 18:06:37 +0200 Subject: [PATCH 19/81] Add setup scripts --- setup.ps1 | 5 +++++ setup.sh | 6 ++++++ 2 files changed, 11 insertions(+) create mode 100644 setup.ps1 create mode 100644 setup.sh diff --git a/setup.ps1 b/setup.ps1 new file mode 100644 index 0000000..3d031a4 --- /dev/null +++ b/setup.ps1 @@ -0,0 +1,5 @@ +python -m venv venv + +.\venv\Scripts\Activate.ps1 + +pip install -e ".[dev,all_db]" \ No newline at end of file diff --git a/setup.sh b/setup.sh new file mode 100644 index 0000000..d32b7c8 --- /dev/null +++ b/setup.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Utwórz i aktywuj środowisko wirtualne +python3 -m venv venv +source venv/bin/activate + +pip install -e ".[dev,all_db]" \ No newline at end of file From 7ec5d5d543784dbfcd85b6232583cc8d6efacc10 Mon Sep 17 00:00:00 2001 From: Albert Stankowski Date: Tue, 20 May 2025 17:18:55 +0100 Subject: [PATCH 20/81] Add translations to all scripts --- corebrain/__init__.py | 22 ++-- corebrain/cli.py | 2 +- corebrain/lib/sso/auth.py | 56 ++++----- corebrain/lib/sso/client.py | 84 ++++++------- corebrain/network/__init__.py | 6 +- corebrain/network/client.py | 220 +++++++++++++++++----------------- corebrain/sdk.py | 2 +- corebrain/services/schema.py | 8 +- corebrain/utils/encrypter.py | 90 +++++++------- corebrain/utils/logging.py | 94 +++++++-------- corebrain/utils/serializer.py | 6 +- 11 files changed, 295 insertions(+), 295 deletions(-) diff --git a/corebrain/__init__.py b/corebrain/__init__.py index 5d21ae4..6819487 100644 --- a/corebrain/__init__.py +++ b/corebrain/__init__.py @@ -46,41 +46,41 @@ def init(api_key: str, config_id: str, skip_verification: bool = False) -> Coreb # Funciones de conveniencia a nivel de paquete def list_configurations(api_key: str) -> List[str]: """ - Lista las configuraciones disponibles para una API key. + Lists the available configurations for an API key. Args: - api_key: API Key de Corebrain + api_key: Corebrain API Key Returns: - Lista de IDs de configuración disponibles + List of available configuration IDs """ config_manager = ConfigManager() return config_manager.list_configs(api_key) def remove_configuration(api_key: str, config_id: str) -> bool: """ - Elimina una configuración específica. + Deletes a specific configuration. Args: - api_key: API Key de Corebrain - config_id: ID de la configuración a eliminar + api_key: Corebrain API Key + config_id: ID of the configuration to delete Returns: - True si se eliminó correctamente, False en caso contrario + True if deleted successfully, False otherwise """ config_manager = ConfigManager() return config_manager.remove_config(api_key, config_id) def get_config(api_key: str, config_id: str) -> Optional[Dict[str, Any]]: """ - Obtiene una configuración específica. + Retrieves a specific configuration. Args: - api_key: API Key de Corebrain - config_id: ID de la configuración a obtener + api_key: Corebrain API Key + config_id: ID of the configuration to retrieve Returns: - Diccionario con la configuración o None si no existe + Dictionary with the configuration or None if it does not exist """ config_manager = ConfigManager() return config_manager.get_config(api_key, config_id) \ No newline at end of file diff --git a/corebrain/cli.py b/corebrain/cli.py index 0ef90a6..7e17025 100644 --- a/corebrain/cli.py +++ b/corebrain/cli.py @@ -1,5 +1,5 @@ """ -Punto de entrada para la CLI de Corebrain para compatibilidad. +Entry point for the Corebrain CLI for compatibility. """ from corebrain.cli.__main__ import main diff --git a/corebrain/lib/sso/auth.py b/corebrain/lib/sso/auth.py index a782d8c..d065a20 100644 --- a/corebrain/lib/sso/auth.py +++ b/corebrain/lib/sso/auth.py @@ -16,13 +16,13 @@ def __init__(self, config=None): def requires_auth(self, session_handler): """ - Decorador genérico que verifica si el usuario está autenticado + Generic decorator that checks if the user is authenticated Args: - session_handler: Función que obtiene el objeto de sesión actual + session_handler: Function that retrieves the current session object Returns: - Una función decoradora que puede aplicarse a rutas/vistas + A decorator function that can be applied to routes/views """ def decorator(func): def wrapper(*args, **kwargs): @@ -41,13 +41,13 @@ def wrapper(*args, **kwargs): def get_login_url(self, state=None): """ - Genera la URL para iniciar la autenticación SSO + Generates the URL to initiate SSO authentication Args: - state: Parámetro opcional para mantener estado entre solicitudes + state: Optional parameter to maintain state between requests Returns: - URL completa para el inicio de sesión SSO + Full URL for SSO login initiation """ params = { 'client_id': self.client_id, @@ -62,13 +62,13 @@ def get_login_url(self, state=None): def verify_token(self, token): """ - Verifica el token con el servidor SSO - + Verifies the token with the SSO server + Args: - token: Token de acceso a verificar - + token: Access token to verify + Returns: - Datos del token si es válido, None en caso contrario + Token data if valid, None otherwise """ try: response = requests.post( @@ -85,13 +85,13 @@ def verify_token(self, token): def get_user_info(self, token): """ - Obtiene información del usuario con el token - + Retrieves user information using the token + Args: - token: Token de acceso del usuario - + token: User access token + Returns: - Información del perfil del usuario si el token es válido, None en caso contrario + User profile information if the token is valid, None otherwise """ try: response = requests.get( @@ -107,13 +107,13 @@ def get_user_info(self, token): def exchange_code_for_token(self, code): """ - Intercambia el código de autorización por un token de acceso - + Exchanges the authorization code for an access token + Args: - code: Código de autorización recibido del servidor SSO - + code: Authorization code received from the SSO server + Returns: - Datos del token de acceso si el intercambio es exitoso, None en caso contrario + Access token data if the exchange is successful, None otherwise """ try: response = requests.post( @@ -135,15 +135,15 @@ def exchange_code_for_token(self, code): def handle_callback(self, code, session_handler, store_user_func=None): """ - Maneja el callback del SSO procesando el código recibido - + Handles the SSO callback by processing the received code + Args: - code: Código de autorización recibido - session_handler: Función que obtiene el objeto de sesión actual - store_user_func: Función opcional para almacenar datos de usuario en otro lugar - + code: Authorization code received + session_handler: Function that retrieves the current session object + store_user_func: Optional function to store user data elsewhere + Returns: - URL de redirección después de procesar el código + Redirect URL after processing the code """ # Intercambiar código por token token_data = self.exchange_code_for_token(code) diff --git a/corebrain/lib/sso/client.py b/corebrain/lib/sso/client.py index ce83058..0315085 100644 --- a/corebrain/lib/sso/client.py +++ b/corebrain/lib/sso/client.py @@ -6,7 +6,7 @@ class GlobodainSSOClient: """ - Cliente SDK para servicios de Globodain que se conectan al SSO central + SDK client for Globodain services that connect to the central SSO """ def __init__( @@ -18,14 +18,14 @@ def __init__( redirect_uri: str ): """ - Inicializar el cliente SSO - + Initialize the SSO client + Args: - sso_url: URL base del servicio SSO (ej: https://sso.globodain.com) - client_id: ID de cliente del servicio - client_secret: Secreto de cliente del servicio - service_id: ID numérico del servicio en la plataforma SSO - redirect_uri: URI de redirección para OAuth + sso_url: Base URL of the SSO service (e.g., https://sso.globodain.com) + client_id: Client ID of the service + client_secret: Client secret of the service + service_id: Numeric ID of the service on the SSO platform + redirect_uri: Redirect URI for OAuth """ self.sso_url = sso_url.rstrip('/') self.client_id = client_id @@ -37,13 +37,13 @@ def __init__( def get_login_url(self, provider: str = None) -> str: """ - Obtener URL para iniciar sesión en SSO - + Get URL to initiate SSO login + Args: - provider: Proveedor de OAuth (google, microsoft, github) o None para login normal - + provider: OAuth provider (google, microsoft, github) or None for normal login + Returns: - URL para redireccionar al usuario + URL to redirect the user """ if provider: return f"{self.sso_url}/api/auth/oauth/{provider}?service_id={self.service_id}" @@ -52,16 +52,16 @@ def get_login_url(self, provider: str = None) -> str: def verify_token(self, token: str) -> Dict[str, Any]: """ - Verificar un token de acceso y obtener información del usuario - + Verify an access token and retrieve user information + Args: - token: Token JWT a verificar - + token: JWT token to verify + Returns: - Información del usuario si el token es válido - + User information if the token is valid + Raises: - Exception: Si el token no es válido + Exception: If the token is not valid """ # Verificar si ya tenemos información cacheada y válida del token now = datetime.now() @@ -109,16 +109,16 @@ def verify_token(self, token: str) -> Dict[str, Any]: def authenticate_service(self, token: str) -> Dict[str, Any]: """ - Autenticar un token para usarlo con este servicio específico - + Authenticate a token for use with this specific service + Args: - token: Token JWT obtenido del SSO - + token: JWT token obtained from the SSO + Returns: - Nuevo token específico para el servicio - + New service-specific token + Raises: - Exception: Si hay un error en la autenticación + Exception: If there is an authentication error """ headers = { "Authorization": f"Bearer {token}", @@ -138,16 +138,16 @@ def authenticate_service(self, token: str) -> Dict[str, Any]: def refresh_token(self, refresh_token: str) -> Dict[str, Any]: """ - Renovar un token de acceso usando refresh token - + Renew an access token using a refresh token + Args: - refresh_token: Token de refresco - + refresh_token: Refresh token + Returns: - Nuevo token de acceso - + New access token + Raises: - Exception: Si hay un error al renovar el token + Exception: If there is an error renewing the token """ response = requests.post( f"{self.sso_url}/api/auth/refresh", @@ -161,17 +161,17 @@ def refresh_token(self, refresh_token: str) -> Dict[str, Any]: def logout(self, refresh_token: str, access_token: str) -> bool: """ - Cerrar sesión (revoca refresh token) - + Log out (revoke refresh token) + Args: - refresh_token: Token de refresco a revocar - access_token: Token de acceso válido - + refresh_token: Refresh token to revoke + access_token: Valid access token + Returns: - True si se cerró sesión correctamente - + True if the logout was successful + Raises: - Exception: Si hay un error al cerrar sesión + Exception: If there is an error logging out """ headers = { "Authorization": f"Bearer {access_token}", diff --git a/corebrain/network/__init__.py b/corebrain/network/__init__.py index 98413c5..aa079ff 100644 --- a/corebrain/network/__init__.py +++ b/corebrain/network/__init__.py @@ -1,8 +1,8 @@ """ -Componentes de red para Corebrain SDK. +Network components for Corebrain SDK. -Este paquete proporciona utilidades y clientes para comunicación -con la API de Corebrain y otros servicios web. +This package provides utilities and clients for communication +with the Corebrain API and other web services. """ from corebrain.network.client import ( APIClient, diff --git a/corebrain/network/client.py b/corebrain/network/client.py index 54e6b02..1176fb1 100644 --- a/corebrain/network/client.py +++ b/corebrain/network/client.py @@ -1,5 +1,5 @@ """ -Cliente HTTP para comunicación con la API de Corebrain. +HTTP client for communication with the Corebrain API. """ import time import logging @@ -16,7 +16,7 @@ def __init__(self, verbose=False): self.verbose = verbose class APIError(Exception): - """Error genérico en la API.""" + """Generic error in the API.""" def __init__(self, message: str, status_code: Optional[int] = None, detail: Optional[str] = None, response: Optional[Response] = None): self.message = message @@ -26,19 +26,19 @@ def __init__(self, message: str, status_code: Optional[int] = None, super().__init__(message) class APITimeoutError(APIError): - """Error de timeout en la API.""" + """Timeout error in the API.""" pass class APIConnectionError(APIError): - """Error de conexión a la API.""" + """Connection error to the API.""" pass class APIAuthError(APIError): - """Error de autenticación en la API.""" + """Authentication error in the API.""" pass class APIClient: - """Cliente HTTP optimizado para comunicación con la API de Corebrain.""" + """Optimized HTTP client for communication with the Corebrain API.""" # Constantes para manejo de reintentos y errores MAX_RETRIES = 3 @@ -48,13 +48,13 @@ class APIClient: def __init__(self, base_url: str, default_timeout: int = 10, verify_ssl: bool = True, user_agent: Optional[str] = None): """ - Inicializa el cliente API con configuración optimizada. - + Initializes the API client with optimized configuration. + Args: - base_url: URL base para todas las peticiones - default_timeout: Tiempo de espera predeterminado en segundos - verify_ssl: Si se debe verificar el certificado SSL - user_agent: Agente de usuario personalizado + base_url: Base URL for all requests + default_timeout: Default timeout in seconds + verify_ssl: Whether to verify the SSL certificate + user_agent: Custom user agent """ # Normalizar URL base para asegurar que termina con '/' self.base_url = base_url if base_url.endswith('/') else base_url + '/' @@ -84,11 +84,11 @@ def __init__(self, base_url: str, default_timeout: int = 10, logger.debug(f"Cliente API inicializado con base_url={base_url}, timeout={default_timeout}s") def __del__(self): - """Asegurar que la sesión se cierre al eliminar el cliente.""" + """Ensure the session is closed when the client is deleted.""" self.close() def close(self): - """Cierra la sesión HTTP.""" + """Closes the HTTP session.""" if hasattr(self, 'session') and self.session: try: self.session.close() @@ -98,13 +98,13 @@ def close(self): def get_full_url(self, endpoint: str) -> str: """ - Construye la URL completa para un endpoint. - + Builds the full URL for an endpoint. + Args: - endpoint: Ruta relativa del endpoint - + endpoint: Relative path of the endpoint + Returns: - URL completa + Full URL """ # Eliminar '/' inicial si existe para evitar rutas duplicadas endpoint = endpoint.lstrip('/') @@ -113,14 +113,14 @@ def get_full_url(self, endpoint: str) -> str: def prepare_headers(self, headers: Optional[Dict[str, str]] = None, auth_token: Optional[str] = None) -> Dict[str, str]: """ - Prepara los headers para una petición. - + Prepares the headers for a request. + Args: - headers: Headers adicionales - auth_token: Token de autenticación - + headers: Additional headers + auth_token: Authentication token + Returns: - Headers combinados + Combined headers """ # Comenzar con headers predeterminados final_headers = self.default_headers.copy() @@ -137,16 +137,16 @@ def prepare_headers(self, headers: Optional[Dict[str, str]] = None, def handle_response(self, response: Response) -> Response: """ - Procesa la respuesta para manejar errores comunes. - + Processes the response to handle common errors. + Args: - response: Respuesta HTTP - + response: HTTP response + Returns: - La misma respuesta si no hay errores - + The same response if there are no errors + Raises: - APIError: Si hay errores en la respuesta + APIError: If there are errors in the response """ status_code = response.status_code @@ -209,26 +209,26 @@ def request(self, method: str, endpoint: str, *, auth_token: Optional[str] = None, retry: bool = True) -> Response: """ - Realiza una petición HTTP con manejo de errores y reintentos. - + Makes an HTTP request with error handling and retries. + Args: - method: Método HTTP (GET, POST, etc.) - endpoint: Ruta relativa del endpoint - headers: Headers adicionales - json: Datos para enviar como JSON - data: Datos para enviar como form o bytes - params: Parámetros de query string - timeout: Tiempo de espera en segundos (sobreescribe el predeterminado) - auth_token: Token de autenticación - retry: Si se deben reintentar peticiones fallidas - + method: HTTP method (GET, POST, etc.) + endpoint: Relative path of the endpoint + headers: Additional headers + json: Data to send as JSON + data: Data to send as form or bytes + params: Query string parameters + timeout: Timeout in seconds (overrides the default) + auth_token: Authentication token + retry: Whether to retry failed requests + Returns: - Respuesta HTTP procesada - + Processed HTTP response + Raises: - APIError: Si hay errores en la petición o respuesta - APITimeoutError: Si la petición excede el tiempo de espera - APIConnectionError: Si hay errores de conexión + APIError: If there are errors in the request or response + APITimeoutError: If the request exceeds the timeout + APIConnectionError: If there are connection errors """ url = self.get_full_url(endpoint) final_headers = self.prepare_headers(headers, auth_token) @@ -318,35 +318,35 @@ def request(self, method: str, endpoint: str, *, raise APIError(f"Error inesperado en petición a {endpoint}") def get(self, endpoint: str, **kwargs) -> Response: - """Realiza una petición GET.""" + """Makes a GET request.""" return self.request("GET", endpoint, **kwargs) def post(self, endpoint: str, **kwargs) -> Response: - """Realiza una petición POST.""" + """Makes a POST request.""" return self.request("POST", endpoint, **kwargs) def put(self, endpoint: str, **kwargs) -> Response: - """Realiza una petición PUT.""" + """Makes a PUT request.""" return self.request("PUT", endpoint, **kwargs) def delete(self, endpoint: str, **kwargs) -> Response: - """Realiza una petición DELETE.""" + """Makes a DELETE request.""" return self.request("DELETE", endpoint, **kwargs) def patch(self, endpoint: str, **kwargs) -> Response: - """Realiza una petición PATCH.""" + """Makes a PATCH request.""" return self.request("PATCH", endpoint, **kwargs) def get_json(self, endpoint: str, **kwargs) -> Any: """ - Realiza una petición GET y devuelve los datos JSON. - + Makes a GET request and returns the JSON data. + Args: - endpoint: Endpoint a consultar - **kwargs: Argumentos adicionales para request() - + endpoint: Endpoint to query + **kwargs: Additional arguments for request() + Returns: - Datos JSON parseados + Parsed JSON data """ response = self.get(endpoint, **kwargs) try: @@ -356,14 +356,14 @@ def get_json(self, endpoint: str, **kwargs) -> Any: def post_json(self, endpoint: str, **kwargs) -> Any: """ - Realiza una petición POST y devuelve los datos JSON. - + Makes a POST request and returns the JSON data. + Args: - endpoint: Endpoint a consultar - **kwargs: Argumentos adicionales para request() - + endpoint: Endpoint to query + **kwargs: Additional arguments for request() + Returns: - Datos JSON parseados + Parsed JSON data """ response = self.post(endpoint, **kwargs) try: @@ -375,13 +375,13 @@ def post_json(self, endpoint: str, **kwargs) -> Any: def check_health(self, timeout: int = 5) -> bool: """ - Comprueba si la API está disponible. - + Checks if the API is available. + Args: - timeout: Tiempo máximo de espera - + timeout: Maximum wait time + Returns: - True si la API está disponible + True if the API is available """ try: response = self.get("health", timeout=timeout, retry=False) @@ -391,17 +391,17 @@ def check_health(self, timeout: int = 5) -> bool: def verify_token(self, token: str, timeout: int = 5) -> Dict[str, Any]: """ - Verifica si un token es válido. - + Verifies if a token is valid. + Args: - token: Token a verificar - timeout: Tiempo máximo de espera - + token: Token to verify + timeout: Maximum wait time + Returns: - Información del usuario si el token es válido - + User information if the token is valid + Raises: - APIAuthError: Si el token no es válido + APIAuthError: If the token is invalid """ try: response = self.get("api/auth/me", auth_token=token, timeout=timeout) @@ -413,27 +413,27 @@ def verify_token(self, token: str, timeout: int = 5) -> Dict[str, Any]: def get_api_keys(self, token: str) -> List[Dict[str, Any]]: """ - Obtiene las API keys disponibles para un usuario. - + Retrieves the available API keys for a user. + Args: - token: Token de autenticación - + token: Authentication token + Returns: - Lista de API keys + List of API keys """ return self.get_json("api/auth/api-keys", auth_token=token) def update_api_key_metadata(self, token: str, api_key: str, metadata: Dict[str, Any]) -> Dict[str, Any]: """ - Actualiza los metadatos de una API key. - + Updates the metadata of an API key. + Args: - token: Token de autenticación - api_key: ID de la API key - metadata: Metadatos a actualizar - + token: Authentication token + api_key: API key ID + metadata: Metadata to update + Returns: - Datos actualizados de la API key + Updated API key data """ data = {"metadata": metadata} return self.put_json(f"api/auth/api-keys/{api_key}", auth_token=token, json=data) @@ -441,17 +441,17 @@ def update_api_key_metadata(self, token: str, api_key: str, metadata: Dict[str, def query_database(self, token: str, question: str, db_schema: Dict[str, Any], config_id: str, timeout: int = 30) -> Dict[str, Any]: """ - Realiza una consulta en lenguaje natural. - + Makes a natural language query. + Args: - token: Token de autenticación - question: Pregunta en lenguaje natural - db_schema: Esquema de la base de datos - config_id: ID de la configuración - timeout: Tiempo máximo de espera - + token: Authentication token + question: Natural language question + db_schema: Database schema + config_id: Configuration ID + timeout: Maximum wait time + Returns: - Resultado de la consulta + Query result """ data = { "question": question, @@ -462,14 +462,14 @@ def query_database(self, token: str, question: str, db_schema: Dict[str, Any], def exchange_sso_token(self, sso_token: str, user_data: Dict[str, Any]) -> Dict[str, Any]: """ - Intercambia un token SSO por un token API. - + Exchanges an SSO token for an API token. + Args: - sso_token: Token SSO - user_data: Datos del usuario - + sso_token: SSO token + user_data: User data + Returns: - Datos del token API + API token data """ headers = {"Authorization": f"Bearer {sso_token}"} data = {"user_data": user_data} @@ -479,10 +479,10 @@ def exchange_sso_token(self, sso_token: str, user_data: Dict[str, Any]) -> Dict[ def get_stats(self) -> Dict[str, Any]: """ - Obtiene estadísticas de uso del cliente. - + Retrieves client usage statistics. + Returns: - Estadísticas de peticiones + Request statistics """ avg_time = self.total_request_time / max(1, self.request_count) error_rate = (self.error_count / max(1, self.request_count)) * 100 @@ -496,7 +496,7 @@ def get_stats(self) -> Dict[str, Any]: } def reset_stats(self) -> None: - """Resetea las estadísticas de uso.""" + """Resets the usage statistics.""" self.request_count = 0 self.error_count = 0 self.total_request_time = 0 \ No newline at end of file diff --git a/corebrain/sdk.py b/corebrain/sdk.py index 7b4ced1..7de1491 100644 --- a/corebrain/sdk.py +++ b/corebrain/sdk.py @@ -1,5 +1,5 @@ """ -SDK de Corebrain para compatibilidad. +Corebrain SDK for compatibility. """ from corebrain.config.manager import ConfigManager diff --git a/corebrain/services/schema.py b/corebrain/services/schema.py index 5eb92ae..4155fb1 100644 --- a/corebrain/services/schema.py +++ b/corebrain/services/schema.py @@ -2,7 +2,7 @@ # Nuevo directorio: services/ # Nuevo archivo: services/schema_service.py """ -Servicios para manejo de esquemas de base de datos. +Services for managing database schemas. """ from typing import Dict, Any, Optional @@ -10,14 +10,14 @@ from corebrain.db.schema import extract_db_schema, SchemaOptimizer class SchemaService: - """Servicio para operaciones de esquema de base de datos.""" + """Service for database schema operations.""" def __init__(self): self.config_manager = ConfigManager() self.schema_optimizer = SchemaOptimizer() def get_schema(self, api_token: str, config_id: str) -> Optional[Dict[str, Any]]: - """Obtiene el esquema para una configuración específica.""" + """Retrieves the schema for a specific configuration.""" config = self.config_manager.get_config(api_token, config_id) if not config: return None @@ -25,7 +25,7 @@ def get_schema(self, api_token: str, config_id: str) -> Optional[Dict[str, Any]] return extract_db_schema(config) def optimize_schema(self, schema: Dict[str, Any], query: str = None) -> Dict[str, Any]: - """Optimiza un esquema existente.""" + """Optimizes an existing schema.""" return self.schema_optimizer.optimize_schema(schema, query) # Otros métodos de servicio... \ No newline at end of file diff --git a/corebrain/utils/encrypter.py b/corebrain/utils/encrypter.py index 1b4cfab..286a705 100644 --- a/corebrain/utils/encrypter.py +++ b/corebrain/utils/encrypter.py @@ -1,5 +1,5 @@ """ -Utilidades de cifrado para Corebrain SDK. +Encryption utilities for Corebrain SDK. """ import os import base64 @@ -15,14 +15,14 @@ def derive_key_from_password(password: Union[str, bytes], salt: Optional[bytes] = None) -> bytes: """ - Deriva una clave de cifrado segura a partir de una contraseña y sal. - + Derives a secure encryption key from a password and salt. + Args: - password: Contraseña o frase de paso - salt: Sal criptográfica (se genera si no se proporciona) - + password: Password or passphrase + salt: Cryptographic salt (generated if not provided) + Returns: - Clave derivada en bytes + Derived key in bytes """ if isinstance(password, str): password = password.encode() @@ -44,23 +44,23 @@ def derive_key_from_password(password: Union[str, bytes], salt: Optional[bytes] def generate_key() -> str: """ - Genera una nueva clave de cifrado aleatoria. - + Generates a new random encryption key. + Returns: - Clave de cifrado en formato base64 + Encryption key in base64 format """ key = Fernet.generate_key() return key.decode() def create_cipher(key: Optional[Union[str, bytes]] = None) -> Fernet: """ - Crea un objeto de cifrado Fernet con la clave dada o genera una nueva. - + Creates a Fernet encryption object with the given key or generates a new one. + Args: - key: Clave de cifrado en formato base64 o None para generar - + key: Encryption key in base64 format or None to generate a new one + Returns: - Objeto Fernet para cifrado/descifrado + Fernet object for encryption/decryption """ if key is None: key = Fernet.generate_key() @@ -71,22 +71,22 @@ def create_cipher(key: Optional[Union[str, bytes]] = None) -> Fernet: class ConfigEncrypter: """ - Gestor de cifrado para configuraciones con manejo de claves. + Encryption manager for configurations with key management. """ def __init__(self, key_path: Optional[Union[str, Path]] = None): """ - Inicializa el encriptador con ruta de clave opcional. - + Initializes the encryptor with an optional key path. + Args: - key_path: Ruta al archivo de clave (si no existe, se creará) + key_path: Path to the key file (will be created if it doesn't exist) """ self.key_path = Path(key_path) if key_path else None self.cipher = None self._init_cipher() def _init_cipher(self) -> None: - """Inicializa el objeto de cifrado, creando o cargando la clave según sea necesario.""" + """Initializes the encryption object, creating or loading the key as needed.""" key = None # Si hay ruta de clave, intentar cargar o crear @@ -128,13 +128,13 @@ def _init_cipher(self) -> None: def encrypt(self, data: Union[str, bytes]) -> bytes: """ - Cifra datos. - + Encrypts data. + Args: - data: Datos a cifrar - + data: Data to encrypt + Returns: - Datos cifrados en bytes + Encrypted data in bytes """ if isinstance(data, str): data = data.encode() @@ -147,13 +147,13 @@ def encrypt(self, data: Union[str, bytes]) -> bytes: def decrypt(self, encrypted_data: Union[str, bytes]) -> bytes: """ - Descifra datos. - + Decrypts data. + Args: - encrypted_data: Datos cifrados - + encrypted_data: Encrypted data + Returns: - Datos descifrados en bytes + Decrypted data in bytes """ if isinstance(encrypted_data, str): encrypted_data = encrypted_data.encode() @@ -169,14 +169,14 @@ def decrypt(self, encrypted_data: Union[str, bytes]) -> bytes: def encrypt_file(self, input_path: Union[str, Path], output_path: Optional[Union[str, Path]] = None) -> Path: """ - Cifra un archivo completo. - + Encrypts a complete file. + Args: - input_path: Ruta al archivo a cifrar - output_path: Ruta para guardar el archivo cifrado (si es None, se añade .enc) - + input_path: Path to the file to encrypt + output_path: Path to save the encrypted file (if None, .enc is added) + Returns: - Ruta del archivo cifrado + Path of the encrypted file """ input_path = Path(input_path) @@ -201,14 +201,14 @@ def encrypt_file(self, input_path: Union[str, Path], output_path: Optional[Union def decrypt_file(self, input_path: Union[str, Path], output_path: Optional[Union[str, Path]] = None) -> Path: """ - Descifra un archivo completo. - + Decrypts a complete file. + Args: - input_path: Ruta al archivo cifrado - output_path: Ruta para guardar el archivo descifrado - + input_path: Path to the encrypted file + output_path: Path to save the decrypted file + Returns: - Ruta del archivo descifrado + Path of the decrypted file """ input_path = Path(input_path) @@ -238,10 +238,10 @@ def decrypt_file(self, input_path: Union[str, Path], output_path: Optional[Union @staticmethod def generate_key_file(key_path: Union[str, Path]) -> None: """ - Genera y guarda una nueva clave en un archivo. - + Generates and saves a new key to a file. + Args: - key_path: Ruta donde guardar la clave + key_path: Path to save the key """ key_path = Path(key_path) diff --git a/corebrain/utils/logging.py b/corebrain/utils/logging.py index 82495a9..0ba559d 100644 --- a/corebrain/utils/logging.py +++ b/corebrain/utils/logging.py @@ -1,8 +1,8 @@ """ -Utilidades de logging para Corebrain SDK. +Logging utilities for Corebrain SDK. -Este módulo proporciona funciones y clases para gestionar el logging -dentro del SDK de forma consistente. +This module provides functions and classes to manage logging +within the SDK consistently. """ import logging import sys @@ -31,44 +31,44 @@ } class VerboseLogger(logging.Logger): - """Logger personalizado con nivel VERBOSE.""" + """Custom logger with VERBOSE level.""" def verbose(self, msg: str, *args: Any, **kwargs: Any) -> None: """ - Registra un mensaje con nivel VERBOSE. - + Logs a message with VERBOSE level. + Args: - msg: Mensaje a registrar - *args: Argumentos para formatear el mensaje - **kwargs: Argumentos adicionales para el logger + msg: Message to log + *args: Arguments to format the message + **kwargs: Additional arguments for the logger """ return self.log(VERBOSE, msg, *args, **kwargs) class ColoredFormatter(logging.Formatter): - """Formateador que agrega colores a los mensajes de log en terminal.""" + """Formatter that adds colors to log messages in the terminal.""" def __init__(self, fmt: str = DEFAULT_FORMAT, datefmt: str = DEFAULT_DATE_FORMAT, use_colors: bool = True): """ - Inicializa el formateador. - + Initializes the formatter. + Args: - fmt: Formato del mensaje - datefmt: Formato de fecha - use_colors: Si es True, usa colores en terminal + fmt: Message format + datefmt: Date format + use_colors: If True, uses colors in the terminal """ super().__init__(fmt, datefmt) self.use_colors = use_colors and sys.stdout.isatty() def format(self, record: logging.LogRecord) -> str: """ - Formatea un registro con colores. - + Formats a log record with colors. + Args: - record: Registro a formatear - + record: Record to format + Returns: - Mensaje formateado + Formatted message """ levelname = record.levelname message = super().format(record) @@ -84,18 +84,18 @@ def setup_logger(name: str = "corebrain", use_colors: bool = True, propagate: bool = False) -> logging.Logger: """ - Configura un logger con opciones personalizadas. - + Configures a logger with custom options. + Args: - name: Nombre del logger - level: Nivel de logging - file_path: Ruta al archivo de log (opcional) - format_string: Formato de mensajes personalizado - use_colors: Si es True, usa colores en terminal - propagate: Si es True, propaga mensajes a loggers padre - + name: Logger name + level: Logging level + file_path: Path to the log file (optional) + format_string: Custom message format + use_colors: If True, uses colors in the terminal + propagate: If True, propagates messages to parent loggers + Returns: - Logger configurado + Configured logger """ # Registrar nivel personalizado VERBOSE if not hasattr(logging, 'VERBOSE'): @@ -145,14 +145,14 @@ def setup_logger(name: str = "corebrain", def get_logger(name: str, level: Optional[int] = None) -> logging.Logger: """ - Obtiene un logger existente o crea uno nuevo. - + Retrieves an existing logger or creates a new one. + Args: - name: Nombre del logger - level: Nivel de logging opcional - + name: Logger name + level: Optional logging level + Returns: - Logger configurado + Configured logger """ logger = logging.getLogger(name) @@ -177,15 +177,15 @@ def enable_file_logging(logger_name: str = "corebrain", log_dir: Optional[Union[str, Path]] = None, filename: Optional[str] = None) -> str: """ - Activa el logging a archivo para un logger existente. - + Enables file logging for an existing logger. + Args: - logger_name: Nombre del logger - log_dir: Directorio para los logs (opcional) - filename: Nombre de archivo personalizado (opcional) - + logger_name: Logger name + log_dir: Directory for the logs (optional) + filename: Custom file name (optional) + Returns: - Ruta al archivo de log + Path to the log file """ logger = logging.getLogger(logger_name) @@ -217,11 +217,11 @@ def enable_file_logging(logger_name: str = "corebrain", def set_log_level(level: Union[int, str], logger_name: Optional[str] = None) -> None: """ - Establece el nivel de logging para uno o todos los loggers. - + Sets the logging level for one or all loggers. + Args: - level: Nivel de logging (nombre o valor entero) - logger_name: Nombre del logger específico (si es None, afecta a todos) + level: Logging level (name or integer value) + logger_name: Specific logger name (if None, affects all) """ # Convertir nombre de nivel a valor si es necesario if isinstance(level, str): diff --git a/corebrain/utils/serializer.py b/corebrain/utils/serializer.py index cf1fb49..c230c3e 100644 --- a/corebrain/utils/serializer.py +++ b/corebrain/utils/serializer.py @@ -1,5 +1,5 @@ """ -Utilidades de serialización para Corebrain SDK. +Serialization utilities for Corebrain SDK. """ import json @@ -8,7 +8,7 @@ from decimal import Decimal class JSONEncoder(json.JSONEncoder): - """Serializador JSON personalizado para tipos especiales.""" + """Custom JSON serializer for special types.""" def default(self, obj): # Objetos datetime if isinstance(obj, (datetime, date, time)): @@ -29,5 +29,5 @@ def default(self, obj): return super().default(obj) def serialize_to_json(obj): - """Serializa cualquier objeto a JSON usando el encoder personalizado""" + """Serializes any object to JSON using the custom encoder""" return json.dumps(obj, cls=JSONEncoder) \ No newline at end of file From 855bb3ed5f40ab9c88292d468f970332ba028b05 Mon Sep 17 00:00:00 2001 From: Albert Stankowski Date: Tue, 20 May 2025 17:25:31 +0100 Subject: [PATCH 21/81] Add translations to all scripts --- corebrain/utils/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/corebrain/utils/__init__.py b/corebrain/utils/__init__.py index 0ed2c33..3c89186 100644 --- a/corebrain/utils/__init__.py +++ b/corebrain/utils/__init__.py @@ -1,8 +1,8 @@ """ -Utilidades generales para Corebrain SDK. +General utilities for Corebrain SDK. -Este paquete proporciona utilidades compartidas por diferentes -componentes del SDK, como serialización, cifrado y logging. +This package provides utilities shared by different +SDK components, such as serialization, encryption, and logging. """ import logging @@ -21,12 +21,12 @@ def setup_logger(level=logging.INFO, file_path=None, format_string=None): """ - Configura el logger principal de Corebrain. - + Configures the main Corebrain logger. + Args: - level: Nivel de logging - file_path: Ruta a archivo de log (opcional) - format_string: Formato de log personalizado + level: Logging level + file_path: Path to log file (optional) + format_string: Custom log format """ # Formato predeterminado fmt = format_string or '%(asctime)s - %(name)s - %(levelname)s - %(message)s' From 81edc9dcb942171fdc9324907439b562b118f6dd Mon Sep 17 00:00:00 2001 From: Emilia-Kaczor Date: Tue, 20 May 2025 18:35:13 +0200 Subject: [PATCH 22/81] Added extenions --- docs/source/_static/custom.css | 0 docs/source/conf.py | 7 ++++++- docs/source/index.rst | 2 +- pyproject.toml | 2 ++ 4 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 docs/source/_static/custom.css diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css new file mode 100644 index 0000000..e69de29 diff --git a/docs/source/conf.py b/docs/source/conf.py index 2d85d0c..a59ab3a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -10,11 +10,16 @@ extensions = [ 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', + 'sphinx.ext.viewcode', + 'sphinx_copybutton', + 'sphinx_design', ] templates_path = ['_templates'] exclude_patterns = [] html_theme = 'furo' - +html_css_files = ['custom.css'] html_static_path = ['_static'] + diff --git a/docs/source/index.rst b/docs/source/index.rst index 434f597..03ce071 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -3,7 +3,7 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Documentation documentation +Welcome to Corebrain's documentation! =========================== .. toctree:: diff --git a/pyproject.toml b/pyproject.toml index c07d952..76f919d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,8 +49,10 @@ dev = [ "mypy>=1.3.0", "flake8>=6.0.0", "sphinx>=8.2.3", + "furo>=2024.8.6", ] + [tool.setuptools] packages = ["corebrain"] From 9334dc4a3bde84da4295bc16b4b299e099256167 Mon Sep 17 00:00:00 2001 From: Emilia-Kaczor Date: Wed, 21 May 2025 16:11:31 +0200 Subject: [PATCH 23/81] Add extensions to dependencies in pyprojectm.toml --- docs/source/conf.py | 6 +++--- pyproject.toml | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index a59ab3a..be062c8 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -11,9 +11,9 @@ extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.napoleon', - 'sphinx.ext.viewcode', - 'sphinx_copybutton', - 'sphinx_design', + 'sphinx.ext.viewcode', + 'sphinx_copybutton', + 'sphinx_design', ] templates_path = ['_templates'] diff --git a/pyproject.toml b/pyproject.toml index 76f919d..981d509 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,8 @@ dev = [ "flake8>=6.0.0", "sphinx>=8.2.3", "furo>=2024.8.6", + "sphinx-copybutton>=0.5.2", + "sphinx-design>=0.6.1,<0.7.0", ] From 859c7117daa821987617a027d992101e99000926 Mon Sep 17 00:00:00 2001 From: Emilia-Kaczor Date: Wed, 21 May 2025 16:55:35 +0200 Subject: [PATCH 24/81] Added custom.css --- docs/source/_static/custom.css | 31 +++++++++++++++++++++++++++++++ docs/source/conf.py | 2 ++ 2 files changed, 33 insertions(+) diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css index e69de29..ab588d2 100644 --- a/docs/source/_static/custom.css +++ b/docs/source/_static/custom.css @@ -0,0 +1,31 @@ +.sidebar-brand-text { + display: inline-block; + font-weight: 800; + font-size: 1.2rem; /* delikatnie zmniejszona */ + font-family: "Segoe UI", Tahoma, Geneva, Verdana, sans-serif; + padding: 0.4em 0.8em; + border-radius: 10px; + margin: 0.5rem auto; + text-align: center; + letter-spacing: 0.03em; + transition: background-color 0.3s ease, color 0.3s ease, box-shadow 0.3s ease; + user-select: none; + cursor: default; + box-shadow: 0 0 10px rgba(0, 0, 0, 0.04); + overflow: hidden; + text-overflow: ellipsis; + max-width: 100%; +} + +body.light .sidebar-brand-text { + background: linear-gradient(135deg, #3a8dde, #2c6db6); + color: #ffffff; + box-shadow: 0 4px 12px rgba(58, 141, 222, 0.6); +} + + +body.dark .sidebar-brand-text { + background: linear-gradient(135deg, #1e2a47, #122147); + color: #a9c1ff; + box-shadow: 0 4px 12px rgba(35, 56, 93, 0.8); +} diff --git a/docs/source/conf.py b/docs/source/conf.py index be062c8..cb05f07 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -23,3 +23,5 @@ html_css_files = ['custom.css'] html_static_path = ['_static'] +html_title = "Corebrain Documentation 0.1" + From 1c45ea2ece17e74a47b92de90883ed9a0f85bd38 Mon Sep 17 00:00:00 2001 From: Emilia-Kaczor Date: Wed, 21 May 2025 17:21:03 +0200 Subject: [PATCH 25/81] Updated custom.css --- docs/source/_static/custom.css | 38 +++++++++++++++++++++++++++++++++- docs/source/conf.py | 5 ++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css index ab588d2..1b1501e 100644 --- a/docs/source/_static/custom.css +++ b/docs/source/_static/custom.css @@ -1,7 +1,7 @@ .sidebar-brand-text { display: inline-block; font-weight: 800; - font-size: 1.2rem; /* delikatnie zmniejszona */ + font-size: 1.2rem; font-family: "Segoe UI", Tahoma, Geneva, Verdana, sans-serif; padding: 0.4em 0.8em; border-radius: 10px; @@ -29,3 +29,39 @@ body.dark .sidebar-brand-text { color: #a9c1ff; box-shadow: 0 4px 12px rgba(35, 56, 93, 0.8); } + + +h1 { + font-family: 'Inter', 'Segoe UI', Roboto, Helvetica, sans-serif; + font-size: 2.4rem; + font-weight: 900; + letter-spacing: -0.015em; + color: var(--color-foreground-primary, #1a1a1a); + text-align: center; + margin-top: 2rem; + margin-bottom: 1.5rem; + line-height: 1.3; + position: relative; + transition: color 0.3s ease; +} + + +h1 .headerlink { + visibility: hidden; + text-decoration: none; + margin-left: 0.6rem; + font-size: 0.9em; + color: inherit; + transition: opacity 0.2s ease; +} + + +h1:hover .headerlink { + visibility: visible; + opacity: 0.5; +} + + +body.dark h1 { + color: #e6eaf0; /* jasna czcionka dla ciemnego motywu */ +} \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index cb05f07..c3ffdca 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -20,7 +20,10 @@ exclude_patterns = [] html_theme = 'furo' -html_css_files = ['custom.css'] +html_css_files = [ + "custom.css", + "https://fonts.googleapis.com/css2?family=Inter:wght@400;600;800;900&display=swap", +] html_static_path = ['_static'] html_title = "Corebrain Documentation 0.1" From 001c022529dbabbc6e0583a95bf9e4d30f8613db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Ayuso?= Date: Thu, 22 May 2025 15:27:00 +0200 Subject: [PATCH 26/81] Authentication with SSO was splitted. Argument to create user and api key by default was added. --- corebrain/cli/auth/sso.py | 106 ++++++++++++++++++++++++++++++++++++++ corebrain/cli/commands.py | 91 +++++++++++++++++++++++++++++++- corebrain/cli/config.py | 4 +- 3 files changed, 197 insertions(+), 4 deletions(-) diff --git a/corebrain/cli/auth/sso.py b/corebrain/cli/auth/sso.py index 83bdc55..cee535f 100644 --- a/corebrain/cli/auth/sso.py +++ b/corebrain/cli/auth/sso.py @@ -209,6 +209,112 @@ def authenticate_with_sso(sso_url: str) -> Tuple[Optional[str], Optional[Dict[st """ Initiates an SSO authentication flow through the browser and uses the callback system. + Args: + sso_url: Base URL of the SSO service + + Returns: + Tuple with (api_key, user_data, api_token) or (None, None, None) if it fails + - api_key: Selected API key to use with the SDK + - user_data: Authenticated user data + - api_token: API token obtained from SSO for general authentication + """ + + # Token to store the result + result = {"sso_token": None} # Renamed for clarity + auth_completed = threading.Event() + session_data = {} + + # Find an available port + #port = get_free_port(DEFAULT_PORT) + + # SSO client configuration + auth_config = { + 'GLOBODAIN_SSO_URL': sso_url or DEFAULT_SSO_URL, + 'GLOBODAIN_CLIENT_ID': SSO_CLIENT_ID, + 'GLOBODAIN_CLIENT_SECRET': SSO_CLIENT_SECRET, + 'GLOBODAIN_REDIRECT_URI': f"http://localhost:{DEFAULT_PORT}/auth/sso/callback", + 'GLOBODAIN_SUCCESS_REDIRECT': 'https://sso.globodain.com/cli/success' + } + + sso_auth = GlobodainSSOAuth(config=auth_config) + + # Factory to create TokenHandler instances with the desired parameters + def handler_factory(*args, **kwargs): + return TokenHandler( + *args, + sso_auth=sso_auth, + result=result, + session_data=session_data, + auth_completed=auth_completed, + **kwargs + ) + + # Start server in the background + server = socketserver.TCPServer(("", DEFAULT_PORT), handler_factory) + server_thread = threading.Thread(target=server.serve_forever) + server_thread.daemon = True + server_thread.start() + + try: + # Build complete URL with protocol if missing + if sso_url and not sso_url.startswith(("http://", "https://")): + sso_url = "https://" + sso_url + + # URL to start the SSO flow + login_url = sso_auth.get_login_url() + auth_url = login_url + + print_colored(f"Opening browser for SSO authentication...", "blue") + print_colored(f"If the browser doesn't open automatically, visit:", "blue") + print_colored(f"{auth_url}", "bold") + + # Try to open the browser + if not webbrowser.open(auth_url): + print_colored("Could not open the browser automatically.", "yellow") + print_colored(f"Please copy and paste the following URL into your browser:", "yellow") + print_colored(f"{auth_url}", "bold") + + # Tell the user to wait + print_colored("\nWaiting for you to complete authentication in the browser...", "blue") + + # Wait for authentication to complete (with timeout) + timeout_seconds = 60 + start_time = time.time() + + # We use a loop with better feedback + while not auth_completed.is_set() and (time.time() - start_time < timeout_seconds): + elapsed = int(time.time() - start_time) + if elapsed % 5 == 0: # Every 5 seconds we show a message + remaining = timeout_seconds - elapsed + #print_colored(f"Waiting for authentication... ({remaining}s remaining)", "yellow") + + # Check every 0.5 seconds for better reactivity + auth_completed.wait(0.5) + + # Verify if authentication was completed + if auth_completed.is_set(): + print_colored("✅ SSO authentication completed successfully!", "green") + return result["sso_token"], session_data['user'] + else: + print_colored(f"❌ Could not complete SSO authentication in {timeout_seconds} seconds.", "red") + print_colored("You can try again or use a token manually.", "yellow") + return None, None, None + except Exception as e: + print_colored(f"❌ Error during SSO authentication: {str(e)}", "red") + return None, None, None + finally: + # Stop the server + try: + server.shutdown() + server.server_close() + except: + # If there's any error closing the server, we ignore it + pass + +def authenticate_with_sso_and_api_key_request(sso_url: str) -> Tuple[Optional[str], Optional[Dict[str, Any]], Optional[str]]: + """ + Initiates an SSO authentication flow through the browser and uses the callback system. + Args: sso_url: Base URL of the SSO service diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index 9bd3dbb..8ce9071 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -6,11 +6,13 @@ import sys import webbrowser import requests +import random +import string from typing import Optional, List from corebrain.cli.common import DEFAULT_API_URL, DEFAULT_SSO_URL, DEFAULT_PORT, SSO_CLIENT_ID, SSO_CLIENT_SECRET -from corebrain.cli.auth.sso import authenticate_with_sso +from corebrain.cli.auth.sso import authenticate_with_sso, authenticate_with_sso_and_api_key_request from corebrain.cli.config import configure_sdk, get_api_credential from corebrain.cli.utils import print_colored from corebrain.config.manager import ConfigManager @@ -41,6 +43,8 @@ def main_cli(argv: Optional[List[str]] = None) -> int: # Argument parser configuration parser = argparse.ArgumentParser(description="Corebrain SDK CLI") parser.add_argument("--version", action="store_true", help="Show SDK version") + parser.add_argument("--authentication", action="store_true", help="Authenticate with SSO") + parser.add_argument("--create-user", action="store_true", help="Create an user and API Key by default") parser.add_argument("--configure", action="store_true", help="Configure the Corebrain SDK") parser.add_argument("--list-configs", action="store_true", help="List available configurations") parser.add_argument("--remove-config", action="store_true", help="Remove a configuration") @@ -64,6 +68,24 @@ def main_cli(argv: Optional[List[str]] = None) -> int: args = parser.parse_args(argv) + def authentication(): + sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + sso_token, sso_user = authenticate_with_sso(sso_url) + if sso_token: + try: + print_colored("✅ Returning SSO Token.", "green") + print_colored(f"{sso_user}", "blue") + print_colored("✅ Returning User data.", "green") + print_colored(f"{sso_user}", "blue") + return sso_token, sso_user + + except Exception as e: + print_colored("❌ Could not return SSO Token or SSO User data.", "red") + return sso_token, sso_user + + else: + print_colored("❌ Could not authenticate with SSO.", "red") + return None, None # Made by Lukasz if args.export_config: @@ -87,6 +109,71 @@ def main_cli(argv: Optional[List[str]] = None) -> int: print(f"Corebrain SDK version {__version__}") return 0 + # Create an user and API Key by default + if args.authentication: + authentication() + + if args.create_user: + sso_token, sso_user = authentication() # Authentica use with SSO + + if sso_token and sso_user: + print_colored("✅ Enter to create an user and API Key.", "green") + + # Get API URL from environment or use default + api_url = os.environ.get("COREBRAIN_API_URL", DEFAULT_API_URL) + + """ + Create user data with SSO information. + If the user wants to use a different password than their SSO account, + they can specify it here. + """ + # Ask if user wants to use SSO password or create a new one + use_sso_password = input("Do you want to use your SSO password? (y/n): ").lower().strip() == 'y' + + if use_sso_password: + random_password = ''.join(random.choices(string.ascii_letters + string.digits, k=12)) + password = sso_user.get("password", random_password) + else: + while True: + password = input("Enter new password: ").strip() + if len(password) >= 8: + break + print_colored("Password must be at least 8 characters long", "yellow") + + user_data = { + "email": sso_user["email"], + "name": f"{sso_user['first_name']} {sso_user['last_name']}", + "password": password + } + + try: + # Make the API request + response = requests.post( + f"{api_url}/api/auth/users", + json=user_data, + headers={ + "Authorization": f"Bearer {sso_token}", + "Content-Type": "application/json" + } + ) + + # Check if the request was successful + print("response API: ", response) + if response.status_code == 200: + print_colored("✅ User and API Key created successfully!", "green") + return 0 + else: + print_colored(f"❌ Error creating user: {response.text}", "red") + return 1 + + except requests.exceptions.RequestException as e: + print_colored(f"❌ Error connecting to API: {str(e)}", "red") + return 1 + + else: + print_colored("❌ Could not create the user or the API KEY.", "red") + return 1 + # Test SSO authentication if args.test_auth: sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL @@ -127,7 +214,7 @@ def main_cli(argv: Optional[List[str]] = None) -> int: # Login via SSO if args.login: sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL - api_key, user_data, api_token = authenticate_with_sso(sso_url) + api_key, user_data, api_token = authenticate_with_sso_and_api_key_request(sso_url) if api_token: # Save the general token for future use diff --git a/corebrain/cli/config.py b/corebrain/cli/config.py index ca3dc13..cf8a175 100644 --- a/corebrain/cli/config.py +++ b/corebrain/cli/config.py @@ -9,7 +9,7 @@ from datetime import datetime from corebrain.cli.common import DEFAULT_API_URL, DEFAULT_SSO_URL -from corebrain.cli.auth import authenticate_with_sso +from corebrain.cli.auth.sso import authenticate_with_sso, authenticate_with_sso_and_api_key_request from corebrain.cli.utils import print_colored, ProgressTracker from corebrain.db.engines import get_available_engines from corebrain.config.manager import ConfigManager @@ -57,7 +57,7 @@ def get_api_credential(args_token: Optional[str] = None, sso_url: Optional[str] # 4. Try SSO authentication print_colored("Attempting authentication via SSO...", "blue") - api_key, user_data, api_token = authenticate_with_sso(sso_url or DEFAULT_SSO_URL) + api_key, user_data, api_token = authenticate_with_sso_and_api_key_request(sso_url or DEFAULT_SSO_URL) print("Exit from authenticate_with_sso: ", datetime.now()) if api_key: # Save for future use From 84edef343d1d8f7f3e3f31fdf5ba6877fd2ff2da Mon Sep 17 00:00:00 2001 From: Kacper Date: Thu, 22 May 2025 15:46:38 +0200 Subject: [PATCH 27/81] Updated development installation instructions in README.md --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8d22293..750d1be 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,10 @@ pip install corebrain ### From source code ```bash + git clone https://github.com/ceoweggo/Corebrain.git pip install -e . + ``` ## 🚀 Quick Start Guide @@ -158,7 +160,12 @@ git clone https://github.com/ceoweggo/Corebrain.git cd corebrain # Install in development mode with extra tools -pip install -e ".[dev,all_db]" + +# On Windows (use powershell) +.\setup.ps1 + +# On Linux/macOS (use bash) +./setup.sh ``` ### Verifying Style and Typing From 945a3daeed66b80dfc5a93ec937182f32e48e0eb Mon Sep 17 00:00:00 2001 From: Vapniak <113126917+Vapniak@users.noreply.github.com> Date: Thu, 22 May 2025 15:50:05 +0200 Subject: [PATCH 28/81] Add csharp wrappers --- corebrain/wrappers/csharp | 1 + corebrain/wrappers/csharp_cli_api | 1 + 2 files changed, 2 insertions(+) create mode 160000 corebrain/wrappers/csharp create mode 160000 corebrain/wrappers/csharp_cli_api diff --git a/corebrain/wrappers/csharp b/corebrain/wrappers/csharp new file mode 160000 index 0000000..8610d81 --- /dev/null +++ b/corebrain/wrappers/csharp @@ -0,0 +1 @@ +Subproject commit 8610d8138a6d31cf098d7d596a580dd026c00a64 diff --git a/corebrain/wrappers/csharp_cli_api b/corebrain/wrappers/csharp_cli_api new file mode 160000 index 0000000..bc36bcf --- /dev/null +++ b/corebrain/wrappers/csharp_cli_api @@ -0,0 +1 @@ +Subproject commit bc36bcf3fae1337c268b71687dec818974f0da6d From 5ed16d6c3a960f2c6e5195b199d2bbae16bbcddf Mon Sep 17 00:00:00 2001 From: Vapniak <113126917+Vapniak@users.noreply.github.com> Date: Thu, 22 May 2025 16:12:40 +0200 Subject: [PATCH 29/81] Add wrappers --- corebrain/wrappers/csharp | 1 - corebrain/wrappers/csharp/.editorconfig | 432 ++++++++++++++ corebrain/wrappers/csharp/.gitignore | 417 +++++++++++++ .../wrappers/csharp/.vscode/settings.json | 3 + corebrain/wrappers/csharp/.vscode/tasks.json | 32 + .../CorebrainCS.Tests.csproj | 14 + .../csharp/CorebrainCS.Tests/Program.cs | 8 + .../csharp/CorebrainCS.Tests/README.md | 4 + corebrain/wrappers/csharp/CorebrainCS.sln | 48 ++ .../csharp/CorebrainCS/CorebrainCS.cs | 112 ++++ .../csharp/CorebrainCS/CorebrainCS.csproj | 7 + corebrain/wrappers/csharp/LICENSE | 21 + corebrain/wrappers/csharp/README.md | 77 +++ corebrain/wrappers/csharp_cli_api | 1 - corebrain/wrappers/csharp_cli_api/.gitignore | 548 ++++++++++++++++++ .../csharp_cli_api/.vscode/settings.json | 3 + .../csharp_cli_api/CorebrainCLIAPI.sln | 50 ++ corebrain/wrappers/csharp_cli_api/README.md | 18 + .../wrappers/csharp_cli_api/src/.editorconfig | 432 ++++++++++++++ .../src/CorebrainCLIAPI/CommandController.cs | 70 +++ .../CorebrainCLIAPI/CorebrainCLIAPI.csproj | 20 + .../src/CorebrainCLIAPI/CorebrainSettings.cs | 8 + .../src/CorebrainCLIAPI/Program.cs | 49 ++ .../Properties/launchSettings.json | 23 + .../appsettings.Development.json | 8 + .../src/CorebrainCLIAPI/appsettings.json | 14 + 26 files changed, 2418 insertions(+), 2 deletions(-) delete mode 160000 corebrain/wrappers/csharp create mode 100644 corebrain/wrappers/csharp/.editorconfig create mode 100644 corebrain/wrappers/csharp/.gitignore create mode 100644 corebrain/wrappers/csharp/.vscode/settings.json create mode 100644 corebrain/wrappers/csharp/.vscode/tasks.json create mode 100644 corebrain/wrappers/csharp/CorebrainCS.Tests/CorebrainCS.Tests.csproj create mode 100644 corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs create mode 100644 corebrain/wrappers/csharp/CorebrainCS.Tests/README.md create mode 100644 corebrain/wrappers/csharp/CorebrainCS.sln create mode 100644 corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs create mode 100644 corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.csproj create mode 100644 corebrain/wrappers/csharp/LICENSE create mode 100644 corebrain/wrappers/csharp/README.md delete mode 160000 corebrain/wrappers/csharp_cli_api create mode 100644 corebrain/wrappers/csharp_cli_api/.gitignore create mode 100644 corebrain/wrappers/csharp_cli_api/.vscode/settings.json create mode 100644 corebrain/wrappers/csharp_cli_api/CorebrainCLIAPI.sln create mode 100644 corebrain/wrappers/csharp_cli_api/README.md create mode 100644 corebrain/wrappers/csharp_cli_api/src/.editorconfig create mode 100644 corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CommandController.cs create mode 100644 corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainCLIAPI.csproj create mode 100644 corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainSettings.cs create mode 100644 corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Program.cs create mode 100644 corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Properties/launchSettings.json create mode 100644 corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.Development.json create mode 100644 corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.json diff --git a/corebrain/wrappers/csharp b/corebrain/wrappers/csharp deleted file mode 160000 index 8610d81..0000000 --- a/corebrain/wrappers/csharp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8610d8138a6d31cf098d7d596a580dd026c00a64 diff --git a/corebrain/wrappers/csharp/.editorconfig b/corebrain/wrappers/csharp/.editorconfig new file mode 100644 index 0000000..e4eb58c --- /dev/null +++ b/corebrain/wrappers/csharp/.editorconfig @@ -0,0 +1,432 @@ +# This file is the top-most EditorConfig file +root = true + +#All Files +[*] +charset = utf-8 +indent_style = space +indent_size = 2 +insert_final_newline = true +trim_trailing_whitespace = true +end_of_line = lf + +########################################## +# File Extension Settings +########################################## + +# Visual Studio Solution Files +[*.sln] +indent_style = tab + +# Visual Studio XML Project Files +[*.{csproj,vbproj,vcxproj.filters,proj,projitems,shproj}] +indent_size = 2 + +# XML Configuration Files +[*.{xml,config,props,targets,nuspec,resx,ruleset,vsixmanifest,vsct}] +indent_size = 2 + +# JSON Files +[*.{json,json5,webmanifest}] +indent_size = 2 + +# YAML Files +[*.{yml,yaml}] +indent_size = 2 + +# Markdown Files +[*.{md,mdx}] +trim_trailing_whitespace = false + +# Web Files +[*.{htm,html,js,jsm,ts,tsx,cjs,cts,ctsx,mjs,mts,mtsx,css,sass,scss,less,pcss,svg,vue}] +indent_size = 2 + +# Batch Files +[*.{cmd,bat}] +end_of_line = crlf + +# Bash Files +[*.sh] +end_of_line = lf + +# Makefiles +[Makefile] +indent_style = tab + +[{*_Generated.cs, *.g.cs, *.generated.cs}] +# Ignore a lack of documentation for generated code. Doesn't apply to builds, +# just to viewing generation output. +dotnet_diagnostic.CS1591.severity = none + +########################################## +# Default .NET Code Style Severities +########################################## + +[*.{cs,csx,cake,vb,vbx}] +# Default Severity for all .NET Code Style rules below +dotnet_analyzer_diagnostic.severity = warning + +########################################## +# Language Rules +########################################## + +# .NET Style Rules +[*.{cs,csx,cake,vb,vbx}] + +# "this." and "Me." qualifiers +dotnet_style_qualification_for_field = false +dotnet_style_qualification_for_property = false +dotnet_style_qualification_for_method = false +dotnet_style_qualification_for_event = false + +# Language keywords instead of framework type names for type references +dotnet_style_predefined_type_for_locals_parameters_members = true:warning +dotnet_style_predefined_type_for_member_access = true:warning + +# Modifier preferences +dotnet_style_require_accessibility_modifiers = always:warning +csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:warning +visual_basic_preferred_modifier_order = Partial,Default,Private,Protected,Public,Friend,NotOverridable,Overridable,MustOverride,Overloads,Overrides,MustInherit,NotInheritable,Static,Shared,Shadows,ReadOnly,WriteOnly,Dim,Const,WithEvents,Widening,Narrowing,Custom,Async:warning +dotnet_style_readonly_field = true:warning +dotnet_diagnostic.IDE0036.severity = warning + + +# Parentheses preferences +dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:warning +dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:warning +dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:warning +dotnet_style_parentheses_in_other_operators = never_if_unnecessary:warning + +# Expression-level preferences +dotnet_style_object_initializer = true:warning +dotnet_style_collection_initializer = true:warning +dotnet_style_explicit_tuple_names = true:warning +dotnet_style_prefer_inferred_tuple_names = true:warning +dotnet_style_prefer_inferred_anonymous_type_member_names = true:warning +dotnet_style_prefer_auto_properties = true:warning +dotnet_style_prefer_conditional_expression_over_assignment = true:suggestion +dotnet_diagnostic.IDE0045.severity = suggestion +dotnet_style_prefer_conditional_expression_over_return = true:suggestion +dotnet_diagnostic.IDE0046.severity = suggestion +dotnet_style_prefer_compound_assignment = true:warning +dotnet_style_prefer_simplified_interpolation = true:warning +dotnet_style_prefer_simplified_boolean_expressions = true:warning + +# Null-checking preferences +dotnet_style_coalesce_expression = true:warning +dotnet_style_null_propagation = true:warning +dotnet_style_prefer_is_null_check_over_reference_equality_method = true:warning + +# File header preferences +# Keep operators at end of line when wrapping. +dotnet_style_operator_placement_when_wrapping = end_of_line:warning +csharp_style_prefer_null_check_over_type_check = true:warning + +# Code block preferences +csharp_prefer_braces = true:warning +csharp_prefer_simple_using_statement = true:suggestion +dotnet_diagnostic.IDE0063.severity = suggestion + +# C# Style Rules +[*.{cs,csx,cake}] +# 'var' preferences +csharp_style_var_for_built_in_types = true:warning +csharp_style_var_when_type_is_apparent = true:warning +csharp_style_var_elsewhere = true:warning +# Expression-bodied members +csharp_style_expression_bodied_methods = true:warning +csharp_style_expression_bodied_constructors = false:warning +csharp_style_expression_bodied_operators = true:warning +csharp_style_expression_bodied_properties = true:warning +csharp_style_expression_bodied_indexers = true:warning +csharp_style_expression_bodied_accessors = true:warning +csharp_style_expression_bodied_lambdas = true:warning +csharp_style_expression_bodied_local_functions = true:warning +# Pattern matching preferences +csharp_style_pattern_matching_over_is_with_cast_check = true:warning +csharp_style_pattern_matching_over_as_with_null_check = true:warning +csharp_style_prefer_switch_expression = true:warning +csharp_style_prefer_pattern_matching = true:warning +csharp_style_prefer_not_pattern = true:warning +# Expression-level preferences +csharp_style_inlined_variable_declaration = true:warning +csharp_prefer_simple_default_expression = true:warning +csharp_style_pattern_local_over_anonymous_function = true:warning +csharp_style_deconstructed_variable_declaration = true:warning +csharp_style_prefer_index_operator = true:warning +csharp_style_prefer_range_operator = true:warning +csharp_style_implicit_object_creation_when_type_is_apparent = true:warning +# "Null" checking preferences +csharp_style_throw_expression = true:warning +csharp_style_conditional_delegate_call = true:warning +# Code block preferences +csharp_prefer_braces = true:warning +csharp_prefer_simple_using_statement = true:suggestion +dotnet_diagnostic.IDE0063.severity = suggestion +# 'using' directive preferences +csharp_using_directive_placement = inside_namespace:warning +# Modifier preferences +# Don't suggest making public methods static. Very annoying. +csharp_prefer_static_local_function = false +# Only suggest making private methods static (if they don't use instance data). +dotnet_code_quality.CA1822.api_surface = private + +########################################## +# Unnecessary Code Rules +########################################## + +# .NET Unnecessary code rules +[*.{cs,csx,cake,vb,vbx}] + +dotnet_code_quality_unused_parameters = non_public:suggestion +dotnet_remove_unnecessary_suppression_exclusions = none +dotnet_diagnostic.IDE0079.severity = warning + +# C# Unnecessary code rules +[*.{cs,csx,cake}] + + +# Don't remove method parameters that are unused. +dotnet_diagnostic.IDE0060.severity = none +dotnet_diagnostic.RCS1163.severity = none + +# Don't remove methods that are unused. +dotnet_diagnostic.IDE0051.severity = none +dotnet_diagnostic.RCS1213.severity = none + +# Use discard variable for unused expression values. +csharp_style_unused_value_expression_statement_preference = discard_variable + +# .NET formatting rules +[*.{cs,csx,cake,vb,vbx}] + +# Organize using directives +dotnet_sort_system_directives_first = true +dotnet_separate_import_directive_groups = false + +dotnet_sort_accessibility = true + +# Dotnet namespace options +# +# We don't care about namespaces matching folder structure. Games and apps +# are complicated and you are free to organize them however you like. Change +# this if you want to enforce it. +dotnet_style_namespace_match_folder = false +dotnet_diagnostic.IDE0130.severity = none + +# C# formatting rules +[*.{cs,csx,cake}] + +# Newline options +csharp_new_line_before_open_brace = none +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true +csharp_new_line_before_members_in_object_initializers = true +csharp_new_line_before_members_in_anonymous_types = true +csharp_new_line_between_query_expression_clauses = true + +# Indentation options +csharp_indent_switch_labels = true +csharp_indent_case_contents = true +csharp_indent_case_contents_when_block = true +csharp_indent_labels = no_change +csharp_indent_block_contents = true +csharp_indent_braces = false + +# Spacing options +csharp_space_after_cast = false +csharp_space_after_keywords_in_control_flow_statements = true +csharp_space_between_parentheses = false +csharp_space_before_colon_in_inheritance_clause = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_around_binary_operators = before_and_after +csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_between_method_declaration_empty_parameter_list_parentheses = false +csharp_space_between_method_declaration_name_and_open_parenthesis = false +csharp_space_between_method_call_parameter_list_parentheses = false +csharp_space_between_method_call_empty_parameter_list_parentheses = false +csharp_space_between_method_call_name_and_opening_parenthesis = false +csharp_space_after_comma = true +csharp_space_before_comma = false +csharp_space_after_dot = false +csharp_space_before_dot = false +csharp_space_after_semicolon_in_for_statement = true +csharp_space_before_semicolon_in_for_statement = false +csharp_space_around_declaration_statements = false +csharp_space_before_open_square_brackets = false +csharp_space_between_empty_square_brackets = false +csharp_space_between_square_brackets = false + +# Wrap options +csharp_preserve_single_line_statements = false +csharp_preserve_single_line_blocks = true + +# Namespace options +csharp_style_namespace_declarations = file_scoped:warning + +########################################## +# .NET Naming Rules +########################################## +[*.{cs,csx,cake,vb,vbx}] + +# Allow underscores in names. +dotnet_diagnostic.CA1707.severity = none + +# Styles +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +dotnet_naming_style.upper_case_style.capitalization = all_upper +dotnet_naming_style.upper_case_style.word_separator = _ + +dotnet_naming_style.camel_case_style.capitalization = camel_case + +dotnet_naming_style.camel_case_underscore_style.required_prefix = _ +dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case + +# Use uppercase for all constant fields. +dotnet_naming_rule.constants_uppercase.severity = suggestion +dotnet_naming_rule.constants_uppercase.symbols = constant_fields +dotnet_naming_rule.constants_uppercase.style = upper_case_style +dotnet_naming_symbols.constant_fields.applicable_kinds = field +dotnet_naming_symbols.constant_fields.applicable_accessibilities = * +dotnet_naming_symbols.constant_fields.required_modifiers = const + +# Non-public fields should be _camelCase +dotnet_naming_rule.non_public_fields_under_camel.severity = suggestion +dotnet_naming_rule.non_public_fields_under_camel.symbols = non_public_fields +dotnet_naming_rule.non_public_fields_under_camel.style = camel_case_underscore_style +dotnet_naming_symbols.non_public_fields.applicable_kinds = field +dotnet_naming_symbols.non_public_fields.required_modifiers = +dotnet_naming_symbols.non_public_fields.applicable_accessibilities = private,private_protected,internal,protected,protected_internal + +# Public fields should be PascalCase +dotnet_naming_rule.public_fields_pascal.severity = suggestion +dotnet_naming_rule.public_fields_pascal.symbols = public_fields +dotnet_naming_rule.public_fields_pascal.style = pascal_case_style +dotnet_naming_symbols.public_fields.applicable_kinds = field +dotnet_naming_symbols.public_fields.required_modifiers = +dotnet_naming_symbols.public_fields.applicable_accessibilities = public + +# Async methods should have "Async" suffix. +# Disabled because it makes tests too verbose. +# dotnet_naming_style.end_in_async.required_suffix = Async +# dotnet_naming_style.end_in_async.capitalization = pascal_case +# dotnet_naming_rule.methods_end_in_async.symbols = methods_async +# dotnet_naming_rule.methods_end_in_async.style = end_in_async +# dotnet_naming_rule.methods_end_in_async.severity = warning +# dotnet_naming_symbols.methods_async.applicable_kinds = method +# dotnet_naming_symbols.methods_async.required_modifiers = async +# dotnet_naming_symbols.methods_async.applicable_accessibilities = * + +########################################## +# Other Naming Rules +########################################## + +# All of the following must be PascalCase: +dotnet_naming_symbols.element_group.applicable_kinds = namespace, class, enum, struct, delegate, event, method, property +dotnet_naming_rule.element_rule.symbols = element_group +dotnet_naming_rule.element_rule.style = pascal_case_style +dotnet_naming_rule.element_rule.severity = warning + +# Interfaces use PascalCase and are prefixed with uppercase 'I' +# https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-classes-structs-and-interfaces +dotnet_naming_style.prefix_interface_with_i_style.capitalization = pascal_case +dotnet_naming_style.prefix_interface_with_i_style.required_prefix = I +dotnet_naming_symbols.interface_group.applicable_kinds = interface +dotnet_naming_rule.interface_rule.symbols = interface_group +dotnet_naming_rule.interface_rule.style = prefix_interface_with_i_style +dotnet_naming_rule.interface_rule.severity = warning + +# Generics Type Parameters use PascalCase and are prefixed with uppercase 'T' +# https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-classes-structs-and-interfaces +dotnet_naming_style.prefix_type_parameters_with_t_style.capitalization = pascal_case +dotnet_naming_style.prefix_type_parameters_with_t_style.required_prefix = T +dotnet_naming_symbols.type_parameter_group.applicable_kinds = type_parameter +dotnet_naming_rule.type_parameter_rule.symbols = type_parameter_group +dotnet_naming_rule.type_parameter_rule.style = prefix_type_parameters_with_t_style +dotnet_naming_rule.type_parameter_rule.severity = warning + +# Function parameters use camelCase +# https://docs.microsoft.com/dotnet/standard/design-guidelines/naming-parameters +dotnet_naming_symbols.parameters_group.applicable_kinds = parameter +dotnet_naming_rule.parameters_rule.symbols = parameters_group +dotnet_naming_rule.parameters_rule.style = camel_case_style +dotnet_naming_rule.parameters_rule.severity = warning + +# Anything not specified uses camel case. +dotnet_naming_rule.unspecified_naming.severity = warning +dotnet_naming_rule.unspecified_naming.symbols = unspecified +dotnet_naming_rule.unspecified_naming.style = camel_case_style +dotnet_naming_symbols.unspecified.applicable_kinds = * +dotnet_naming_symbols.unspecified.applicable_accessibilities = * + +########################################## +# Rule Overrides +########################################## + +roslyn_correctness.assembly_reference_validation = relaxed + +# Allow using keywords as names +# dotnet_diagnostic.CA1716.severity = none +# Don't require culture info for ToString() +dotnet_diagnostic.CA1304.severity = none +# Don't require a string comparison for comparing strings. +dotnet_diagnostic.CA1310.severity = none +# Don't require a string format specifier. +dotnet_diagnostic.CA1305.severity = none +# Allow protected fields. +dotnet_diagnostic.CA1051.severity = none +# Don't warn about checking values that are supposedly never null. Sometimes +# they are actually null. +dotnet_diagnostic.CS8073.severity = none +# Don't remove seemingly "unnecessary" assignments, as they often have +# intended side-effects. +dotnet_diagnostic.IDE0059.severity = none +# Switch/case should always have a default clause. Tell that to Roslynator. +dotnet_diagnostic.RCS1070.severity = none +# Tell roslynator not to eat unused parameters. +dotnet_diagnostic.RCS1163.severity = none +# Tell dotnet not to remove unused parameters. +dotnet_diagnostic.IDE0060.severity = none +# Tell roslynator not to remove `partial` modifiers. +dotnet_diagnostic.RCS1043.severity = none +# Tell roslynator not to make classes static so aggressively. +dotnet_diagnostic.RCS1102.severity = none +# Roslynator wants to make properties readonly all the time, so stop it. +# The developer knows best when it comes to contract definitions with Godot. +dotnet_diagnostic.RCS1170.severity = none +# Allow expression values to go unused, even without discard variable. +# Otherwise, using Moq would be way too verbose. +dotnet_diagnostic.IDE0058.severity = none +# Don't let roslynator turn every local variable into a const. +# If we did, we'd have to specify the types of local variables far more often, +# and this style prefers type inference. +dotnet_diagnostic.RCS1118.severity = none +# Enums don't need to declare explicit values. Everyone knows they start at 0. +dotnet_diagnostic.RCS1161.severity = none +# Allow unconstrained type parameter to be checked for null. +dotnet_diagnostic.RCS1165.severity = none +# Allow keyword-based names so that parameter names like `@event` can be used. +dotnet_diagnostic.CA1716.severity = none +# Allow me to use the word Collection if I want. +dotnet_diagnostic.CA1711.severity = none +# Not disposing of objects in a test is normal within Godot because of scene tree stuff. +dotnet_diagnostic.CA1001.severity = none +# No primary constructors — not supported well by tooling. +dotnet_diagnostic.IDE0290.severity = none +# Let me comment where I like +dotnet_diagnostic.RCS1181.severity = none +# Let me write dumb if checks, keeps it readable +dotnet_diagnostic.IDE0046.severity = none +# Don't make me use expression bodies for methods +dotnet_diagnostic.IDE0022.severity = none +# Don't use collection shorhand. +dotnet_diagnostic.IDE0300.severity = none +dotnet_diagnostic.IDE0028.severity = none +dotnet_diagnostic.IDE0305.severity = none +# Don't make me populate a switch expression redundantly +dotnet_diagnostic.IDE0072.severity = none +# Leave me alone about primary constructors +dotnet_diagnostic.IDE0290.severity = none diff --git a/corebrain/wrappers/csharp/.gitignore b/corebrain/wrappers/csharp/.gitignore new file mode 100644 index 0000000..3021ab1 --- /dev/null +++ b/corebrain/wrappers/csharp/.gitignore @@ -0,0 +1,417 @@ +# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,csharp +# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,csharp + +### Csharp ### +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Mono auto generated files +mono_crash.* + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUnit +*.VisualState.xml +TestResult.xml +nunit-*.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.log +*.tlog +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage*.json +coverage*.xml +coverage*.info + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# NuGet Symbol Packages +*.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx +*.appxbundle +*.appxupload + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio 6 auto-generated project file (contains which files were open etc.) +*.vbp + +# Visual Studio 6 workspace and project file (working project files containing files to include in project) +*.dsw +*.dsp + +# Visual Studio 6 technical files + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# Visual Studio History (VSHistory) files +.vshistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ + +# Fody - auto-generated XML schema +FodyWeavers.xsd + +# VS Code files for those working on multiple tools +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +.venv + +# Local History for Visual Studio Code +.history/ + +# Windows Installer files from build outputs +*.cab +*.msi +*.msix +*.msm +*.msp + +# JetBrains Rider +*.sln.iml + +### VisualStudioCode ### +!.vscode/*.code-snippets + +# Local History for Visual Studio Code + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,csharp \ No newline at end of file diff --git a/corebrain/wrappers/csharp/.vscode/settings.json b/corebrain/wrappers/csharp/.vscode/settings.json new file mode 100644 index 0000000..d956415 --- /dev/null +++ b/corebrain/wrappers/csharp/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "dotnet.defaultSolution": "CorebrainCS.sln" +} \ No newline at end of file diff --git a/corebrain/wrappers/csharp/.vscode/tasks.json b/corebrain/wrappers/csharp/.vscode/tasks.json new file mode 100644 index 0000000..817885b --- /dev/null +++ b/corebrain/wrappers/csharp/.vscode/tasks.json @@ -0,0 +1,32 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Build Project", + "command": "dotnet", + "type": "process", + "args": [ + "build", + "${workspaceFolder}/CorebrainCS.Tests/CorebrainCS.Tests.csproj", + "--configuration", + "Release" + ], + "problemMatcher": "$msCompile", + "group": { + "kind": "build", + "isDefault": true + } + }, + { + "label": "Run Project", + "command": "dotnet", + "type": "process", + "args": [ + "run", + "--project", + "${workspaceFolder}/CorebrainCS.Tests/CorebrainCS.Tests.csproj" + ], + "dependsOn": ["Build Project"] + }, + ] +} \ No newline at end of file diff --git a/corebrain/wrappers/csharp/CorebrainCS.Tests/CorebrainCS.Tests.csproj b/corebrain/wrappers/csharp/CorebrainCS.Tests/CorebrainCS.Tests.csproj new file mode 100644 index 0000000..0d163c1 --- /dev/null +++ b/corebrain/wrappers/csharp/CorebrainCS.Tests/CorebrainCS.Tests.csproj @@ -0,0 +1,14 @@ + + + + Exe + net9.0 + enable + enable + + + + + + + diff --git a/corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs b/corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs new file mode 100644 index 0000000..d935860 --- /dev/null +++ b/corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs @@ -0,0 +1,8 @@ +using CorebrainCS; + +Console.WriteLine("Hello, World!"); + +// For now it only works on windows +var corebrain = new CorebrainCS.CorebrainCS("../../../../venv/Scripts/python.exe", "../../../cli", false); + +Console.WriteLine(corebrain.Version()); diff --git a/corebrain/wrappers/csharp/CorebrainCS.Tests/README.md b/corebrain/wrappers/csharp/CorebrainCS.Tests/README.md new file mode 100644 index 0000000..e74913d --- /dev/null +++ b/corebrain/wrappers/csharp/CorebrainCS.Tests/README.md @@ -0,0 +1,4 @@ +### Quick start + +* Create venv in the root directory and install all the dependencies. The instalation guide is in corebrain README.md +* Go to the CorebrainCS.Tests directory to see how the program runs and run `dotnet run` \ No newline at end of file diff --git a/corebrain/wrappers/csharp/CorebrainCS.sln b/corebrain/wrappers/csharp/CorebrainCS.sln new file mode 100644 index 0000000..2e5b65b --- /dev/null +++ b/corebrain/wrappers/csharp/CorebrainCS.sln @@ -0,0 +1,48 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CorebrainCS", "CorebrainCS\CorebrainCS.csproj", "{152890AC-4B76-42F7-813B-CB7F3F902B9F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CorebrainCS.Tests", "CorebrainCS.Tests\CorebrainCS.Tests.csproj", "{664BB3EB-0364-4989-879A-D8CCDBCF6B89}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Debug|x64.ActiveCfg = Debug|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Debug|x64.Build.0 = Debug|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Debug|x86.ActiveCfg = Debug|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Debug|x86.Build.0 = Debug|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Release|Any CPU.Build.0 = Release|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Release|x64.ActiveCfg = Release|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Release|x64.Build.0 = Release|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Release|x86.ActiveCfg = Release|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Release|x86.Build.0 = Release|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Debug|Any CPU.Build.0 = Debug|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Debug|x64.ActiveCfg = Debug|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Debug|x64.Build.0 = Debug|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Debug|x86.ActiveCfg = Debug|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Debug|x86.Build.0 = Debug|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Release|Any CPU.ActiveCfg = Release|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Release|Any CPU.Build.0 = Release|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Release|x64.ActiveCfg = Release|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Release|x64.Build.0 = Release|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Release|x86.ActiveCfg = Release|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs b/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs new file mode 100644 index 0000000..23c37d0 --- /dev/null +++ b/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs @@ -0,0 +1,112 @@ +namespace CorebrainCS; + +using System; +using System.Diagnostics; + +/// +/// Creates the main corebrain interface. +/// +/// Path to the python which works with the corebrain cli, for example if you create the ./.venv you pass the path to the ./.venv python executable +/// Path to the corebrain cli script, if you installed it globally you just pass the `corebrain` path +/// +public class CorebrainCS(string pythonPath = "python", string scriptPath = "corebrain", bool verbose = false) { + private readonly string _pythonPath = Path.GetFullPath(pythonPath); + private readonly string _scriptPath = Path.GetFullPath(scriptPath); + private readonly bool _verbose = verbose; + + + public string Help() { + return ExecuteCommand("--help"); + } + + public string Version() { + return ExecuteCommand("--version"); + } + + public string Configure() { + return ExecuteCommand("--configure"); + } + + public string ListConfigs() { + return ExecuteCommand("--list-configs"); + } + + public string RemoveConfig() { + return ExecuteCommand("--remove-config"); + } + + public string ShowSchema() { + return ExecuteCommand("--show-schema"); + } + + public string ExtractSchema() { + return ExecuteCommand("--extract-schema"); + } + + public string ExtractSchemaToDefaultFile() { + return ExecuteCommand("--extract-schema --output-file test"); + } + + public string ConfigID() { + return ExecuteCommand("--extract-schema --config-id config"); + } + + public string SetToken(string token) { + return ExecuteCommand($"--token {token}"); + } + + public string ApiKey(string apikey) { + return ExecuteCommand($"--api-key {apikey}"); + } + + public string ApiUrl(string apiurl) { + if (string.IsNullOrWhiteSpace(apiurl)) { + throw new ArgumentException("API URL cannot be empty or whitespace", nameof(apiurl)); + } + + if (!Uri.TryCreate(apiurl, UriKind.Absolute, out var uriResult) || + (uriResult.Scheme != Uri.UriSchemeHttp && uriResult.Scheme != Uri.UriSchemeHttps)) { + throw new ArgumentException("Invalid API URL format. Must be a valid HTTP/HTTPS URL", nameof(apiurl)); + } + + // Escape the URL for command line safety + var escapedUrl = apiurl.Replace("\"", "\\\""); + return ExecuteCommand($"--api-url \"{escapedUrl}\""); + } + + public string ExecuteCommand(string arguments) { + if (_verbose) { + Console.WriteLine($"Executing: {_pythonPath} {_scriptPath} {arguments}"); + } + + var process = new Process { + StartInfo = new ProcessStartInfo { + FileName = _pythonPath, + Arguments = $"\"{_scriptPath}\" {arguments}", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + process.Start(); + var output = process.StandardOutput.ReadToEnd(); + var error = process.StandardError.ReadToEnd(); + process.WaitForExit(); + + if (_verbose) { + Console.WriteLine("Command output:"); + Console.WriteLine(output); + if (!string.IsNullOrEmpty(error)) { + Console.WriteLine("Error output:\n" + error); + } + } + + if (!string.IsNullOrEmpty(error)) { + throw new InvalidOperationException($"Python CLI error: {error}"); + } + + return output.Trim(); + } +} \ No newline at end of file diff --git a/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.csproj b/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.csproj new file mode 100644 index 0000000..4ef4183 --- /dev/null +++ b/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.csproj @@ -0,0 +1,7 @@ + + + net9.0 + enable + enable + + diff --git a/corebrain/wrappers/csharp/LICENSE b/corebrain/wrappers/csharp/LICENSE new file mode 100644 index 0000000..8e423f6 --- /dev/null +++ b/corebrain/wrappers/csharp/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Oliwier Adamczyk + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/corebrain/wrappers/csharp/README.md b/corebrain/wrappers/csharp/README.md new file mode 100644 index 0000000..e9e2c7a --- /dev/null +++ b/corebrain/wrappers/csharp/README.md @@ -0,0 +1,77 @@ +# CoreBrain-CS + +[![NuGet Version](https://img.shields.io/nuget/v/CorebrainCS.svg)](https://www.nuget.org/packages/CorebrainCS/) +[![Python Requirement](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/) + +A C# wrapper for the CoreBrain Python CLI tool, providing seamless integration between .NET applications and CoreBrain's cognitive computing capabilities. + +## Features + +- 🚀 Native C# interface for CoreBrain functions +- 🛠️ Supports both development and production workflows + +## Installation + +### Prerequisites + +- [.NET 8.0 SDK](https://dotnet.microsoft.com/download) +- [Python 3.8+](https://www.python.org/downloads/) + +## Corebrain installation + +See the main corebrain package installation on https://github.com/ceoweggo/Corebrain/blob/main/README.md#installation + +## Basic Usage + +```csharp +using CorebrainCS; + +// Initialize wrapper (auto-detects Python environment) +var corebrain = new CorebrainCS(); + +// Get version +Console.WriteLine($"CoreBrain version: {corebrain.Version()}"); +``` + +## Advanced Configuration + +```csharp +// Custom configuration +var corebrain = new CorebrainCS( + pythonPath: "path/to/python", // Custom python path + scriptPath: "path/to/cli", // Custom CoreBrain CLI path + verbose: true // Enable debug logging +); +``` + +## Common Commands + +| Command | C# Method | Description | +|---------|-----------|-------------| +| `--version` | `.Version()` | Get CoreBrain version | + + + +### File Structure + +``` +Corebrain-CS/ +├── CorebrainCS/ # C# wrapper library +├── CorebrainCLI/ # Example consumer app +├── corebrain/ # Embedded Python package +``` + +## License + +MIT License - See [LICENSE](LICENSE) for details. diff --git a/corebrain/wrappers/csharp_cli_api b/corebrain/wrappers/csharp_cli_api deleted file mode 160000 index bc36bcf..0000000 --- a/corebrain/wrappers/csharp_cli_api +++ /dev/null @@ -1 +0,0 @@ -Subproject commit bc36bcf3fae1337c268b71687dec818974f0da6d diff --git a/corebrain/wrappers/csharp_cli_api/.gitignore b/corebrain/wrappers/csharp_cli_api/.gitignore new file mode 100644 index 0000000..4ead058 --- /dev/null +++ b/corebrain/wrappers/csharp_cli_api/.gitignore @@ -0,0 +1,548 @@ +# Created by https://www.toptal.com/developers/gitignore/api/csharp,dotnetcore,aspnetcore,visualstudiocode +# Edit at https://www.toptal.com/developers/gitignore?templates=csharp,dotnetcore,aspnetcore,visualstudiocode + +### ASPNETCore ### +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# DNX +project.lock.json +project.fragment.lock.json +artifacts/ + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# TODO: Comment the next line if you want to checkin your web deploy settings +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config +# NuGet v3's project.json files produces more ignoreable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +node_modules/ +orleans.codegen.cs + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush +.cr/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/ + +### Csharp ### +## +## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore + +# User-specific files +*.rsuser + +# User-specific files (MonoDevelop/Xamarin Studio) + +# Mono auto generated files +mono_crash.* + +# Build results +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +# Uncomment if you have tasks that create the project's static files in wwwroot + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results + +# NUnit +nunit-*.xml + +# Build Results of an ATL Project + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_h.h +*.iobj +*.ipdb +*_wpftmp.csproj +*.tlog + +# Chutzpah Test files + +# Visual C++ cache files + +# Visual Studio profiler + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace + +# Guidance Automation Toolkit + +# ReSharper is a .NET coding add-in + +# TeamCity is a build add-in + +# DotCover is a Code Coverage Tool + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage*.json +coverage*.xml +coverage*.info + +# Visual Studio code coverage results + +# NCrunch + +# MightyMoose + +# Web workbench (sass) + +# Installshield output folder + +# DocProject is a documentation generator add-in + +# Click-Once directory + +# Publish Web Output +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted + +# NuGet Packages +# NuGet Symbol Packages +*.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files + +# Microsoft Azure Build Output + +# Microsoft Azure Emulator + +# Windows Store app package directories and files +*.appx +*.appxbundle +*.appxupload + +# Visual Studio cache files +# files ending in .cache can be ignored +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) + +# RIA/Silverlight projects + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.ndf + +# Business Intelligence projects +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes + +# GhostDoc plugin setting file + +# Node.js Tools for Visual Studio + +# Visual Studio 6 build log + +# Visual Studio 6 workspace options file + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio 6 auto-generated project file (contains which files were open etc.) +*.vbp + +# Visual Studio 6 workspace and project file (working project files containing files to include in project) +*.dsw +*.dsp + +# Visual Studio 6 technical files + +# Visual Studio LightSwitch build output + +# Paket dependency manager + +# FAKE - F# Make + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# Visual Studio History (VSHistory) files +.vshistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ + +# Fody - auto-generated XML schema +FodyWeavers.xsd + +# VS Code files for those working on multiple tools +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# Local History for Visual Studio Code +.history/ + +# Windows Installer files from build outputs +*.cab +*.msi +*.msix +*.msm +*.msp + +# JetBrains Rider + +### DotnetCore ### +# .NET Core build folders +bin/ +obj/ + +# Common node modules locations +/node_modules +/wwwroot/node_modules + +### VisualStudioCode ### +!.vscode/*.code-snippets + +# Local History for Visual Studio Code + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +# End of https://www.toptal.com/developers/gitignore/api/csharp,dotnetcore,aspnetcore,visualstudiocode \ No newline at end of file diff --git a/corebrain/wrappers/csharp_cli_api/.vscode/settings.json b/corebrain/wrappers/csharp_cli_api/.vscode/settings.json new file mode 100644 index 0000000..60f6c5c --- /dev/null +++ b/corebrain/wrappers/csharp_cli_api/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "dotnet.defaultSolution": "CorebrainCLIAPI.sln" +} \ No newline at end of file diff --git a/corebrain/wrappers/csharp_cli_api/CorebrainCLIAPI.sln b/corebrain/wrappers/csharp_cli_api/CorebrainCLIAPI.sln new file mode 100644 index 0000000..bd8794a --- /dev/null +++ b/corebrain/wrappers/csharp_cli_api/CorebrainCLIAPI.sln @@ -0,0 +1,50 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CorebrainCLIAPI", "src\CorebrainCLIAPI\CorebrainCLIAPI.csproj", "{3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Debug|x64.ActiveCfg = Debug|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Debug|x64.Build.0 = Debug|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Debug|x86.ActiveCfg = Debug|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Debug|x86.Build.0 = Debug|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Release|Any CPU.Build.0 = Release|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Release|x64.ActiveCfg = Release|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Release|x64.Build.0 = Release|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Release|x86.ActiveCfg = Release|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Release|x86.Build.0 = Release|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Debug|x64.ActiveCfg = Debug|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Debug|x64.Build.0 = Debug|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Debug|x86.ActiveCfg = Debug|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Debug|x86.Build.0 = Debug|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Release|Any CPU.Build.0 = Release|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Release|x64.ActiveCfg = Release|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Release|x64.Build.0 = Release|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Release|x86.ActiveCfg = Release|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {C5CF7B2F-DA16-24C6-929A-8AB8C4831AB0} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {1B7A4995-2D77-4398-BE28-B3B52C1E351B} = {C5CF7B2F-DA16-24C6-929A-8AB8C4831AB0} + EndGlobalSection +EndGlobal diff --git a/corebrain/wrappers/csharp_cli_api/README.md b/corebrain/wrappers/csharp_cli_api/README.md new file mode 100644 index 0000000..4bf45aa --- /dev/null +++ b/corebrain/wrappers/csharp_cli_api/README.md @@ -0,0 +1,18 @@ +# Corebrain CLI API + +## Quick Start + +### Prerequisites + +- Python 3.8+ +- .NET 6.0+ +- Node.js 14+ +- Git + +### Installation + +1. Create **venv** in corebrain directory +2. Continue with installation provided here https://github.com/ceoweggo/Corebrain/blob/pre-release-v0.2.0/README.md#development-installation +3. If you changed the installation directory of venv or corebrain, change the paths in `CorebrainCLIAPI/appsettings.json` +4. Go to `src/CorebrainCLIAPI` +5. run `dotnet run` \ No newline at end of file diff --git a/corebrain/wrappers/csharp_cli_api/src/.editorconfig b/corebrain/wrappers/csharp_cli_api/src/.editorconfig new file mode 100644 index 0000000..b770c5e --- /dev/null +++ b/corebrain/wrappers/csharp_cli_api/src/.editorconfig @@ -0,0 +1,432 @@ +# This file is the top-most EditorConfig file +root = true + +#All Files +[*] +charset = utf-8 +indent_style = space +indent_size = 2 +insert_final_newline = true +trim_trailing_whitespace = true +end_of_line = lf + +########################################## +# File Extension Settings +########################################## + +# Visual Studio Solution Files +[*.sln] +indent_style = tab + +# Visual Studio XML Project Files +[*.{csproj,vbproj,vcxproj.filters,proj,projitems,shproj}] +indent_size = 2 + +# XML Configuration Files +[*.{xml,config,props,targets,nuspec,resx,ruleset,vsixmanifest,vsct}] +indent_size = 2 + +# JSON Files +[*.{json,json5,webmanifest}] +indent_size = 2 + +# YAML Files +[*.{yml,yaml}] +indent_size = 2 + +# Markdown Files +[*.{md,mdx}] +trim_trailing_whitespace = false + +# Web Files +[*.{htm,html,js,jsm,ts,tsx,cjs,cts,ctsx,mjs,mts,mtsx,css,sass,scss,less,pcss,svg,vue}] +indent_size = 2 + +# Batch Files +[*.{cmd,bat}] +end_of_line = crlf + +# Bash Files +[*.sh] +end_of_line = lf + +# Makefiles +[Makefile] +indent_style = tab + +[{*_Generated.cs, *.g.cs, *.generated.cs}] +# Ignore a lack of documentation for generated code. Doesn't apply to builds, +# just to viewing generation output. +dotnet_diagnostic.CS1591.severity = none + +########################################## +# Default .NET Code Style Severities +########################################## + +[*.{cs,csx,cake,vb,vbx}] +# Default Severity for all .NET Code Style rules below +dotnet_analyzer_diagnostic.severity = warning + +########################################## +# Language Rules +########################################## + +# .NET Style Rules +[*.{cs,csx,cake,vb,vbx}] + +# "this." and "Me." qualifiers +dotnet_style_qualification_for_field = false +dotnet_style_qualification_for_property = false +dotnet_style_qualification_for_method = false +dotnet_style_qualification_for_event = false + +# Language keywords instead of framework type names for type references +dotnet_style_predefined_type_for_locals_parameters_members = true:warning +dotnet_style_predefined_type_for_member_access = true:warning + +# Modifier preferences +dotnet_style_require_accessibility_modifiers = always:warning +csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:warning +visual_basic_preferred_modifier_order = Partial,Default,Private,Protected,Public,Friend,NotOverridable,Overridable,MustOverride,Overloads,Overrides,MustInherit,NotInheritable,Static,Shared,Shadows,ReadOnly,WriteOnly,Dim,Const,WithEvents,Widening,Narrowing,Custom,Async:warning +dotnet_style_readonly_field = true:warning +dotnet_diagnostic.IDE0036.severity = warning + + +# Parentheses preferences +dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:warning +dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:warning +dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:warning +dotnet_style_parentheses_in_other_operators = never_if_unnecessary:warning + +# Expression-level preferences +dotnet_style_object_initializer = true:warning +dotnet_style_collection_initializer = true:warning +dotnet_style_explicit_tuple_names = true:warning +dotnet_style_prefer_inferred_tuple_names = true:warning +dotnet_style_prefer_inferred_anonymous_type_member_names = true:warning +dotnet_style_prefer_auto_properties = true:warning +dotnet_style_prefer_conditional_expression_over_assignment = true:suggestion +dotnet_diagnostic.IDE0045.severity = suggestion +dotnet_style_prefer_conditional_expression_over_return = true:suggestion +dotnet_diagnostic.IDE0046.severity = suggestion +dotnet_style_prefer_compound_assignment = true:warning +dotnet_style_prefer_simplified_interpolation = true:warning +dotnet_style_prefer_simplified_boolean_expressions = true:warning + +# Null-checking preferences +dotnet_style_coalesce_expression = true:warning +dotnet_style_null_propagation = true:warning +dotnet_style_prefer_is_null_check_over_reference_equality_method = true:warning + +# File header preferences +# Keep operators at end of line when wrapping. +dotnet_style_operator_placement_when_wrapping = end_of_line:warning +csharp_style_prefer_null_check_over_type_check = true:warning + +# Code block preferences +csharp_prefer_braces = true:warning +csharp_prefer_simple_using_statement = true:suggestion +dotnet_diagnostic.IDE0063.severity = suggestion + +# C# Style Rules +[*.{cs,csx,cake}] +# 'var' preferences +csharp_style_var_for_built_in_types = true:warning +csharp_style_var_when_type_is_apparent = true:warning +csharp_style_var_elsewhere = true:warning +# Expression-bodied members +csharp_style_expression_bodied_methods = true:warning +csharp_style_expression_bodied_constructors = false:warning +csharp_style_expression_bodied_operators = true:warning +csharp_style_expression_bodied_properties = true:warning +csharp_style_expression_bodied_indexers = true:warning +csharp_style_expression_bodied_accessors = true:warning +csharp_style_expression_bodied_lambdas = true:warning +csharp_style_expression_bodied_local_functions = true:warning +# Pattern matching preferences +csharp_style_pattern_matching_over_is_with_cast_check = true:warning +csharp_style_pattern_matching_over_as_with_null_check = true:warning +csharp_style_prefer_switch_expression = true:warning +csharp_style_prefer_pattern_matching = true:warning +csharp_style_prefer_not_pattern = true:warning +# Expression-level preferences +csharp_style_inlined_variable_declaration = true:warning +csharp_prefer_simple_default_expression = true:warning +csharp_style_pattern_local_over_anonymous_function = true:warning +csharp_style_deconstructed_variable_declaration = true:warning +csharp_style_prefer_index_operator = true:warning +csharp_style_prefer_range_operator = true:warning +csharp_style_implicit_object_creation_when_type_is_apparent = true:warning +# "Null" checking preferences +csharp_style_throw_expression = true:warning +csharp_style_conditional_delegate_call = true:warning +# Code block preferences +csharp_prefer_braces = true:warning +csharp_prefer_simple_using_statement = true:suggestion +dotnet_diagnostic.IDE0063.severity = suggestion +# 'using' directive preferences +csharp_using_directive_placement = inside_namespace:warning +# Modifier preferences +# Don't suggest making public methods static. Very annoying. +csharp_prefer_static_local_function = false +# Only suggest making private methods static (if they don't use instance data). +dotnet_code_quality.CA1822.api_surface = private + +########################################## +# Unnecessary Code Rules +########################################## + +# .NET Unnecessary code rules +[*.{cs,csx,cake,vb,vbx}] + +dotnet_code_quality_unused_parameters = non_public:suggestion +dotnet_remove_unnecessary_suppression_exclusions = none +dotnet_diagnostic.IDE0079.severity = warning + +# C# Unnecessary code rules +[*.{cs,csx,cake}] + + +# Don't remove method parameters that are unused. +dotnet_diagnostic.IDE0060.severity = none +dotnet_diagnostic.RCS1163.severity = none + +# Don't remove methods that are unused. +dotnet_diagnostic.IDE0051.severity = none +dotnet_diagnostic.RCS1213.severity = none + +# Use discard variable for unused expression values. +csharp_style_unused_value_expression_statement_preference = discard_variable + +# .NET formatting rules +[*.{cs,csx,cake,vb,vbx}] + +# Organize using directives +dotnet_sort_system_directives_first = true +dotnet_separate_import_directive_groups = false + +dotnet_sort_accessibility = true + +# Dotnet namespace options +# +# We don't care about namespaces matching folder structure. Games and apps +# are complicated and you are free to organize them however you like. Change +# this if you want to enforce it. +dotnet_style_namespace_match_folder = false +dotnet_diagnostic.IDE0130.severity = none + +# C# formatting rules +[*.{cs,csx,cake}] + +# Newline options +csharp_new_line_before_open_brace = none +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true +csharp_new_line_before_members_in_object_initializers = true +csharp_new_line_before_members_in_anonymous_types = true +csharp_new_line_between_query_expression_clauses = true + +# Indentation options +csharp_indent_switch_labels = true +csharp_indent_case_contents = true +csharp_indent_case_contents_when_block = true +csharp_indent_labels = no_change +csharp_indent_block_contents = true +csharp_indent_braces = false + +# Spacing options +csharp_space_after_cast = false +csharp_space_after_keywords_in_control_flow_statements = true +csharp_space_between_parentheses = false +csharp_space_before_colon_in_inheritance_clause = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_around_binary_operators = before_and_after +csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_between_method_declaration_empty_parameter_list_parentheses = false +csharp_space_between_method_declaration_name_and_open_parenthesis = false +csharp_space_between_method_call_parameter_list_parentheses = false +csharp_space_between_method_call_empty_parameter_list_parentheses = false +csharp_space_between_method_call_name_and_opening_parenthesis = false +csharp_space_after_comma = true +csharp_space_before_comma = false +csharp_space_after_dot = false +csharp_space_before_dot = false +csharp_space_after_semicolon_in_for_statement = true +csharp_space_before_semicolon_in_for_statement = false +csharp_space_around_declaration_statements = false +csharp_space_before_open_square_brackets = false +csharp_space_between_empty_square_brackets = false +csharp_space_between_square_brackets = false + +# Wrap options +csharp_preserve_single_line_statements = false +csharp_preserve_single_line_blocks = true + +# Namespace options +csharp_style_namespace_declarations = file_scoped:warning + +########################################## +# .NET Naming Rules +########################################## +[*.{cs,csx,cake,vb,vbx}] + +# Allow underscores in names. +dotnet_diagnostic.CA1707.severity = none + +# Styles +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +dotnet_naming_style.upper_case_style.capitalization = all_upper +dotnet_naming_style.upper_case_style.word_separator = _ + +dotnet_naming_style.camel_case_style.capitalization = camel_case + +dotnet_naming_style.camel_case_underscore_style.required_prefix = _ +dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case + +# Use uppercase for all constant fields. +dotnet_naming_rule.constants_uppercase.severity = suggestion +dotnet_naming_rule.constants_uppercase.symbols = constant_fields +dotnet_naming_rule.constants_uppercase.style = upper_case_style +dotnet_naming_symbols.constant_fields.applicable_kinds = field +dotnet_naming_symbols.constant_fields.applicable_accessibilities = * +dotnet_naming_symbols.constant_fields.required_modifiers = const + +# Non-public fields should be _camelCase +dotnet_naming_rule.non_public_fields_under_camel.severity = suggestion +dotnet_naming_rule.non_public_fields_under_camel.symbols = non_public_fields +dotnet_naming_rule.non_public_fields_under_camel.style = camel_case_underscore_style +dotnet_naming_symbols.non_public_fields.applicable_kinds = field +dotnet_naming_symbols.non_public_fields.required_modifiers = +dotnet_naming_symbols.non_public_fields.applicable_accessibilities = private,private_protected,internal,protected,protected_internal + +# Public fields should be PascalCase +dotnet_naming_rule.public_fields_pascal.severity = suggestion +dotnet_naming_rule.public_fields_pascal.symbols = public_fields +dotnet_naming_rule.public_fields_pascal.style = pascal_case_style +dotnet_naming_symbols.public_fields.applicable_kinds = field +dotnet_naming_symbols.public_fields.required_modifiers = +dotnet_naming_symbols.public_fields.applicable_accessibilities = public + +# Async methods should have "Async" suffix. +# Disabled because it makes tests too verbose. +# dotnet_naming_style.end_in_async.required_suffix = Async +# dotnet_naming_style.end_in_async.capitalization = pascal_case +# dotnet_naming_rule.methods_end_in_async.symbols = methods_async +# dotnet_naming_rule.methods_end_in_async.style = end_in_async +# dotnet_naming_rule.methods_end_in_async.severity = warning +# dotnet_naming_symbols.methods_async.applicable_kinds = method +# dotnet_naming_symbols.methods_async.required_modifiers = async +# dotnet_naming_symbols.methods_async.applicable_accessibilities = * + +########################################## +# Other Naming Rules +########################################## + +# All of the following must be PascalCase: +dotnet_naming_symbols.element_group.applicable_kinds = namespace, class, enum, struct, delegate, event, method, property +dotnet_naming_rule.element_rule.symbols = element_group +dotnet_naming_rule.element_rule.style = pascal_case_style +dotnet_naming_rule.element_rule.severity = warning + +# Interfaces use PascalCase and are prefixed with uppercase 'I' +# https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-classes-structs-and-interfaces +dotnet_naming_style.prefix_interface_with_i_style.capitalization = pascal_case +dotnet_naming_style.prefix_interface_with_i_style.required_prefix = I +dotnet_naming_symbols.interface_group.applicable_kinds = interface +dotnet_naming_rule.interface_rule.symbols = interface_group +dotnet_naming_rule.interface_rule.style = prefix_interface_with_i_style +dotnet_naming_rule.interface_rule.severity = warning + +# Generics Type Parameters use PascalCase and are prefixed with uppercase 'T' +# https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-classes-structs-and-interfaces +dotnet_naming_style.prefix_type_parameters_with_t_style.capitalization = pascal_case +dotnet_naming_style.prefix_type_parameters_with_t_style.required_prefix = T +dotnet_naming_symbols.type_parameter_group.applicable_kinds = type_parameter +dotnet_naming_rule.type_parameter_rule.symbols = type_parameter_group +dotnet_naming_rule.type_parameter_rule.style = prefix_type_parameters_with_t_style +dotnet_naming_rule.type_parameter_rule.severity = warning + +# Function parameters use camelCase +# https://docs.microsoft.com/dotnet/standard/design-guidelines/naming-parameters +dotnet_naming_symbols.parameters_group.applicable_kinds = parameter +dotnet_naming_rule.parameters_rule.symbols = parameters_group +dotnet_naming_rule.parameters_rule.style = camel_case_style +dotnet_naming_rule.parameters_rule.severity = warning + +# Anything not specified uses camel case. +dotnet_naming_rule.unspecified_naming.severity = warning +dotnet_naming_rule.unspecified_naming.symbols = unspecified +dotnet_naming_rule.unspecified_naming.style = camel_case_style +dotnet_naming_symbols.unspecified.applicable_kinds = * +dotnet_naming_symbols.unspecified.applicable_accessibilities = * + +########################################## +# Rule Overrides +########################################## + +roslyn_correctness.assembly_reference_validation = relaxed + +# Allow using keywords as names +# dotnet_diagnostic.CA1716.severity = none +# Don't require culture info for ToString() +dotnet_diagnostic.CA1304.severity = none +# Don't require a string comparison for comparing strings. +dotnet_diagnostic.CA1310.severity = none +# Don't require a string format specifier. +dotnet_diagnostic.CA1305.severity = none +# Allow protected fields. +dotnet_diagnostic.CA1051.severity = none +# Don't warn about checking values that are supposedly never null. Sometimes +# they are actually null. +dotnet_diagnostic.CS8073.severity = none +# Don't remove seemingly "unnecessary" assignments, as they often have +# intended side-effects. +dotnet_diagnostic.IDE0059.severity = none +# Switch/case should always have a default clause. Tell that to Roslynator. +dotnet_diagnostic.RCS1070.severity = none +# Tell roslynator not to eat unused parameters. +dotnet_diagnostic.RCS1163.severity = none +# Tell dotnet not to remove unused parameters. +dotnet_diagnostic.IDE0060.severity = none +# Tell roslynator not to remove `partial` modifiers. +dotnet_diagnostic.RCS1043.severity = none +# Tell roslynator not to make classes static so aggressively. +dotnet_diagnostic.RCS1102.severity = none +# Roslynator wants to make properties readonly all the time, so stop it. +# The developer knows best when it comes to contract definitions with Godot. +dotnet_diagnostic.RCS1170.severity = none +# Allow expression values to go unused, even without discard variable. +# Otherwise, using Moq would be way too verbose. +dotnet_diagnostic.IDE0058.severity = none +# Don't let roslynator turn every local variable into a const. +# If we did, we'd have to specify the types of local variables far more often, +# and this style prefers type inference. +dotnet_diagnostic.RCS1118.severity = none +# Enums don't need to declare explicit values. Everyone knows they start at 0. +dotnet_diagnostic.RCS1161.severity = none +# Allow unconstrained type parameter to be checked for null. +dotnet_diagnostic.RCS1165.severity = none +# Allow keyword-based names so that parameter names like `@event` can be used. +dotnet_diagnostic.CA1716.severity = none +# Allow me to use the word Collection if I want. +dotnet_diagnostic.CA1711.severity = none +# Not disposing of objects in a test is normal within Godot because of scene tree stuff. +dotnet_diagnostic.CA1001.severity = none +# No primary constructors — not supported well by tooling. +dotnet_diagnostic.IDE0290.severity = none +# Let me comment where I like +dotnet_diagnostic.RCS1181.severity = none +# Let me write dumb if checks, keeps it readable +dotnet_diagnostic.IDE0046.severity = none +# Don't make me use expression bodies for methods +dotnet_diagnostic.IDE0022.severity = none +# Don't use collection shorhand. +dotnet_diagnostic.IDE0300.severity = none +dotnet_diagnostic.IDE0028.severity = none +dotnet_diagnostic.IDE0305.severity = none +# Don't make me populate a switch expression redundantly +dotnet_diagnostic.IDE0072.severity = none +# Leave me alone about primary constructors +dotnet_diagnostic.IDE0290.severity = none \ No newline at end of file diff --git a/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CommandController.cs b/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CommandController.cs new file mode 100644 index 0000000..e0236d2 --- /dev/null +++ b/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CommandController.cs @@ -0,0 +1,70 @@ +namespace CorebrainCLIAPI; + +using CorebrainCS; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Options; + +/// +/// Controller for executing Corebrain CLI commands +/// +[ApiController] +[Route("api/[controller]")] +[Produces("application/json")] +public class CommandController : ControllerBase { + private readonly CorebrainCS _corebrain; + + public CommandController(IOptions settings) { + var config = settings.Value; + _corebrain = new CorebrainCS( + config.PythonPath, + config.ScriptPath, + config.Verbose + ); + } + + /// + /// Executes a Corebrain CLI command + /// + /// + /// Sample request: + /// + /// POST /api/command + /// { + /// "arguments": "--help" + /// } + /// + /// + /// Command request containing the arguments + /// The output of the executed command + /// Returns the command output + /// If the arguments are empty + /// If there was an error executing the command + [HttpPost] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + [ProducesResponseType(StatusCodes.Status500InternalServerError)] + public IActionResult ExecuteCommand([FromBody] CommandRequest request) { + if (string.IsNullOrWhiteSpace(request.Arguments)) { + return BadRequest("Command arguments are required"); + } + + try { + var result = _corebrain.ExecuteCommand(request.Arguments); + return Ok(result); + } + catch (Exception ex) { + return StatusCode(500, $"Error executing command: {ex.Message}"); + } + } + + /// + /// Command request model + /// + public class CommandRequest { + /// + /// The arguments to pass to the Corebrain CLI + /// + /// --help + public required string Arguments { get; set; } + } +} diff --git a/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainCLIAPI.csproj b/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainCLIAPI.csproj new file mode 100644 index 0000000..279f7d0 --- /dev/null +++ b/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainCLIAPI.csproj @@ -0,0 +1,20 @@ + + + + net9.0 + enable + enable + true + $(NoWarn);1591 + + + + + + + + + + + + diff --git a/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainSettings.cs b/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainSettings.cs new file mode 100644 index 0000000..82143b9 --- /dev/null +++ b/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainSettings.cs @@ -0,0 +1,8 @@ +namespace CorebrainCLIAPI; + +public class CorebrainSettings +{ + public string PythonPath { get; set; } + public string ScriptPath { get; set; } + public bool Verbose { get; set; } = false; +} diff --git a/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Program.cs b/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Program.cs new file mode 100644 index 0000000..3ddd6a1 --- /dev/null +++ b/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Program.cs @@ -0,0 +1,49 @@ +using System.Reflection; +using CorebrainCLIAPI; +using Microsoft.OpenApi.Models; + +var builder = WebApplication.CreateBuilder(args); + +// CORS policy to allow requests from the frontend +builder.Services.AddCors(options => options.AddPolicy("AllowFrontend", policy => + policy.WithOrigins("http://localhost:5173") + .AllowAnyMethod() + .AllowAnyHeader() +)); + +// Configure controllers and settings +builder.Services.AddControllers(); +builder.Services.Configure( + builder.Configuration.GetSection("CorebrainSettings")); + +// Swagger / OpenAPI +builder.Services.AddEndpointsApiExplorer(); +builder.Services.AddSwaggerGen(c => { + c.SwaggerDoc("v1", new OpenApiInfo { + Title = "Corebrain CLI API", + Version = "v1", + Description = "ASP.NET Core Web API for interfacing with Corebrain CLI commands" + }); + + var xmlFile = $"{Assembly.GetExecutingAssembly().GetName().Name}.xml"; + var xmlPath = Path.Combine(AppContext.BaseDirectory, xmlFile); + if (File.Exists(xmlPath)) { + c.IncludeXmlComments(xmlPath); + } +}); + +var app = builder.Build(); + +// Middleware pipeline +app.UseCors("AllowFrontend"); + +if (app.Environment.IsDevelopment()) { + app.UseSwagger(); + app.UseSwaggerUI(c => + c.SwaggerEndpoint("/swagger/v1/swagger.json", "Corebrain CLI API v1")); +} + +app.UseHttpsRedirection(); +app.UseAuthorization(); +app.MapControllers(); +app.Run(); diff --git a/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Properties/launchSettings.json b/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Properties/launchSettings.json new file mode 100644 index 0000000..cf5accf --- /dev/null +++ b/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Properties/launchSettings.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": false, + "applicationUrl": "http://localhost:5140", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": false, + "applicationUrl": "https://localhost:7261;http://localhost:5140", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.Development.json b/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.Development.json new file mode 100644 index 0000000..ff66ba6 --- /dev/null +++ b/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.json b/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.json new file mode 100644 index 0000000..0ab3335 --- /dev/null +++ b/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.json @@ -0,0 +1,14 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*", + "CorebrainSettings": { + "PythonPath": "../../../../../venv/Scripts/python.exe", + "ScriptPath": "../../../../cli", + "Verbose": false + } +} From 1519cf4bb7f67d5680d36e934ad71236093e1d72 Mon Sep 17 00:00:00 2001 From: Emilia-Kaczor Date: Thu, 22 May 2025 16:13:20 +0200 Subject: [PATCH 30/81] Added css --- docs/source/_static/custom.css | 47 +++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css index 1b1501e..cca03ed 100644 --- a/docs/source/_static/custom.css +++ b/docs/source/_static/custom.css @@ -33,18 +33,43 @@ body.dark .sidebar-brand-text { h1 { font-family: 'Inter', 'Segoe UI', Roboto, Helvetica, sans-serif; - font-size: 2.4rem; + font-size: 2.8rem; font-weight: 900; letter-spacing: -0.015em; - color: var(--color-foreground-primary, #1a1a1a); text-align: center; - margin-top: 2rem; - margin-bottom: 1.5rem; + margin: 3rem auto 2rem; line-height: 1.3; position: relative; - transition: color 0.3s ease; + padding: 1.2rem 2.5rem; + max-width: 90%; + width: fit-content; + + background: rgba(255, 255, 255, 0.1); + backdrop-filter: blur(14px); + -webkit-backdrop-filter: blur(14px); + border-radius: 1.25rem; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.05); + + color: #1a1a1a; + transition: all 0.3s ease; + z-index: 1; +} + +h1::before { + content: ""; + position: absolute; + inset: -2px; + z-index: -1; + background: linear-gradient(135deg, #cfd8e3, #aab8c2); + filter: blur(6px); + opacity: 0.35; + border-radius: inherit; + transition: opacity 0.3s ease; } +h1:hover::before { + opacity: 0.5; +} h1 .headerlink { visibility: hidden; @@ -55,13 +80,17 @@ h1 .headerlink { transition: opacity 0.2s ease; } - h1:hover .headerlink { visibility: visible; opacity: 0.5; } - body.dark h1 { - color: #e6eaf0; /* jasna czcionka dla ciemnego motywu */ -} \ No newline at end of file + background: rgba(0, 0, 0, 0.2); + color: #e6eaf0; +} + +body.dark h1::before { + background: linear-gradient(135deg, #2a2f3a, #3b4a5a); + opacity: 0.25; +} From eb50a369c2d05696d040f0c89a42a0e75175bbd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Kucharczyk?= Date: Thu, 22 May 2025 16:57:44 +0200 Subject: [PATCH 31/81] Add: CLI-UI, --gui command, wrappers for csharp and csharp_cli_api --- .gitmodules | 3 ++ corebrain/CLI-UI | 1 + corebrain/cli/commands.py | 80 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 .gitmodules create mode 160000 corebrain/CLI-UI diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..a034212 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "corebrain/CLI-UI"] + path = corebrain/CLI-UI + url = https://github.com/Luki20091/CLI-UI.git diff --git a/corebrain/CLI-UI b/corebrain/CLI-UI new file mode 160000 index 0000000..1f4167d --- /dev/null +++ b/corebrain/CLI-UI @@ -0,0 +1 @@ +Subproject commit 1f4167dfc3e9d8b8c654983912b8cb4139d63224 diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index 8ce9071..96eac44 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -64,7 +64,8 @@ def main_cli(argv: Optional[List[str]] = None) -> int: parser.add_argument("--validate-config",action="store_true",help="Validates the selected configuration without executing any operations") parser.add_argument("--test-connection",action="store_true",help="Tests the connection to the Corebrain API using the provide credentials") parser.add_argument("--export-config",action="store_true",help="Exports the current configuration to a file") - + parser.add_argument("--gui", action="store_true", help="Check setup and launch the web interface") + args = parser.parse_args(argv) @@ -361,7 +362,84 @@ def authentication(): print_colored(f"Failed to connect to Corebrain API: {e}", "red") return 1 + + + + if args.gui: + import subprocess + from pathlib import Path + + def run_cmd(cmd, cwd=None): + print_colored(f"▶ {cmd}", "yellow") + subprocess.run(cmd, shell=True, cwd=cwd, check=True) + + print("Checking GUI setup...") + + commands_path = Path(__file__).resolve() + corebrain_root = commands_path.parents[1] + + cli_ui_path = corebrain_root / "CLI-UI" + client_path = cli_ui_path / "client" + server_path = cli_ui_path / "server" + api_path = corebrain_root / "wrappers" / "csharp_cli_api" + + # Path validation + if not client_path.exists(): + print_colored(f"Folder {client_path} does not exist!", "red") + sys.exit(1) + if not server_path.exists(): + print_colored(f"Folder {server_path} does not exist!", "red") + sys.exit(1) + if not api_path.exists(): + print_colored(f"Folder {api_path} does not exist!", "red") + sys.exit(1) + + # Setup client + if not (client_path / "node_modules").exists(): + print_colored("Installing frontend (React) dependencies...", "cyan") + run_cmd("npm install", cwd=client_path) + run_cmd("npm install history", cwd=client_path) + run_cmd("npm install --save-dev vite", cwd=client_path) + run_cmd("npm install concurrently --save-dev", cwd=client_path) + + # Setup server + if not (server_path / "node_modules").exists(): + print_colored("Installing backend (Express) dependencies...", "cyan") + run_cmd("npm install", cwd=server_path) + run_cmd("npm install --save-dev ts-node-dev", cwd=server_path) + + # Start GUI: CLI UI + Corebrain API + print("Starting GUI (CLI-UI + Corebrain API)...") + + def run_in_background_silent(cmd, cwd): + return subprocess.Popen( + cmd, + cwd=cwd, + shell=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + + run_in_background_silent("dotnet run", cwd=api_path) + run_in_background_silent( + 'npx concurrently "npm --prefix server run dev" "npm --prefix client run dev"', + cwd=cli_ui_path + ) + + url = "http://localhost:5173/" + print_colored(f"GUI: {url}", "cyan") + webbrowser.open(url) + + + + + + + + + + else: # If no option was specified, show help parser.print_help() From 4133b91b2744aecd200fbc2b0be2f30e43b24740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Ayuso?= Date: Thu, 22 May 2025 17:01:52 +0200 Subject: [PATCH 32/81] SSO remote authentication modified in corebrain/cli/common --- corebrain/cli/common.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/corebrain/cli/common.py b/corebrain/cli/common.py index 9dd55e9..7799dcf 100644 --- a/corebrain/cli/common.py +++ b/corebrain/cli/common.py @@ -3,8 +3,13 @@ """ DEFAULT_API_URL = "http://localhost:5000" -DEFAULT_SSO_URL = "http://localhost:3000" +#DEFAULT_SSO_URL = "http://localhost:3000" # localhost +DEFAULT_SSO_URL = "https://sso.globodain.com" # remote DEFAULT_PORT = 8765 DEFAULT_TIMEOUT = 10 -SSO_CLIENT_ID = '401dca6e-3f3b-4458-b3ef-f87eaae0398d' -SSO_CLIENT_SECRET = 'f9d315ea-5a65-4e3f-be35-b27a933dfb5b' \ No newline at end of file +#SSO_CLIENT_ID = '401dca6e-3f3b-4458-b3ef-f87eaae0398d' # localhost +#SSO_CLIENT_SECRET = 'f9d315ea-5a65-4e3f-be35-b27a933dfb5b' # localhost +SSO_CLIENT_ID = '63d767e9-5a06-4890-a194-8608ae29d426' # remote +SSO_CLIENT_SECRET = '06cf39f6-ca93-466e-955e-cb6ea0a02d4d' # remote +SSO_REDIRECT_URI = 'http://localhost:8765/oauth/callback' +SSO_SERVICE_ID = 2 \ No newline at end of file From 79d148fbfa6d8d9685878bbc7c08f3c288541d88 Mon Sep 17 00:00:00 2001 From: BartekPachniak Date: Thu, 22 May 2025 18:08:06 +0200 Subject: [PATCH 33/81] Add new commands Added --sso-url, --login with username and password or token, --test-auth --- .../csharp/CorebrainCS.Tests/Program.cs | 2 + .../csharp/CorebrainCS/CorebrainCS.cs | 79 +++++++++++++++++-- 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs b/corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs index d935860..d58d0fa 100644 --- a/corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs +++ b/corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs @@ -6,3 +6,5 @@ var corebrain = new CorebrainCS.CorebrainCS("../../../../venv/Scripts/python.exe", "../../../cli", false); Console.WriteLine(corebrain.Version()); + + diff --git a/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs b/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs index 23c37d0..c8b8fb2 100644 --- a/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs +++ b/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs @@ -69,18 +69,78 @@ public string ApiUrl(string apiurl) { throw new ArgumentException("Invalid API URL format. Must be a valid HTTP/HTTPS URL", nameof(apiurl)); } - // Escape the URL for command line safety var escapedUrl = apiurl.Replace("\"", "\\\""); return ExecuteCommand($"--api-url \"{escapedUrl}\""); + } + public string SsoUrl(string ssoUrl) { + if (string.IsNullOrWhiteSpace(ssoUrl)) { + throw new ArgumentException("SSO URL cannot be empty or whitespace", nameof(ssoUrl)); + } + + if (!Uri.TryCreate(ssoUrl, UriKind.Absolute, out var uriResult) || + (uriResult.Scheme != Uri.UriSchemeHttp && uriResult.Scheme != Uri.UriSchemeHttps)) { + throw new ArgumentException("Invalid SSO URL format. Must be a valid HTTP/HTTPS URL", nameof(ssoUrl)); + } + + var escapedUrl = ssoUrl.Replace("\"", "\\\""); + return ExecuteCommand($"--sso-url \"{escapedUrl}\""); } + public string Login(string username, string password){ + if (string.IsNullOrWhiteSpace(username)){ + throw new ArgumentException("Username cannot be empty or whitespace", nameof(username)); + } - public string ExecuteCommand(string arguments) { - if (_verbose) { + if (string.IsNullOrWhiteSpace(password)){ + throw new ArgumentException("Password cannot be empty or whitespace", nameof(password)); + } + + var escapedUsername = username.Replace("\"", "\\\""); + var escapedPassword = password.Replace("\"", "\\\""); + + return ExecuteCommand($"--login --username \"{escapedUsername}\" --password \"{escapedPassword}\""); + } + + public string LoginWithToken(string token) { + if (string.IsNullOrWhiteSpace(token)) { + throw new ArgumentException("Token cannot be empty or whitespace", nameof(token)); + } + + var escapedToken = token.Replace("\"", "\\\""); + return ExecuteCommand($"--login --token \"{escapedToken}\""); + } + + //When youre logged in use this function + public string TestAuth() { + return ExecuteCommand("--test-auth"); + } + + //Without beeing logged + public string TestAuth(string? apiUrl = null, string? token = null) { + var args = new List { "--test-auth" }; + + if (!string.IsNullOrEmpty(apiUrl)) { + if (!Uri.IsWellFormedUriString(apiUrl, UriKind.Absolute)) + throw new ArgumentException("Invalid API URL format", nameof(apiUrl)); + + args.Add($"--api-url \"{apiUrl}\""); + } + + if (!string.IsNullOrEmpty(token)) + args.Add($"--token \"{token}\""); + + return ExecuteCommand(string.Join(" ", args)); + } + public string ExecuteCommand(string arguments) + { + if (_verbose) + { Console.WriteLine($"Executing: {_pythonPath} {_scriptPath} {arguments}"); } - var process = new Process { - StartInfo = new ProcessStartInfo { + var process = new Process + { + StartInfo = new ProcessStartInfo + { FileName = _pythonPath, Arguments = $"\"{_scriptPath}\" {arguments}", RedirectStandardOutput = true, @@ -95,15 +155,18 @@ public string ExecuteCommand(string arguments) { var error = process.StandardError.ReadToEnd(); process.WaitForExit(); - if (_verbose) { + if (_verbose) + { Console.WriteLine("Command output:"); Console.WriteLine(output); - if (!string.IsNullOrEmpty(error)) { + if (!string.IsNullOrEmpty(error)) + { Console.WriteLine("Error output:\n" + error); } } - if (!string.IsNullOrEmpty(error)) { + if (!string.IsNullOrEmpty(error)) + { throw new InvalidOperationException($"Python CLI error: {error}"); } From 26d3b1ac137594bed6563fc16155719be0158849 Mon Sep 17 00:00:00 2001 From: KarixD2137 Date: Fri, 23 May 2025 09:34:59 +0200 Subject: [PATCH 34/81] Added CLI argument for API key creation --- corebrain/cli/commands.py | 20 ++++++++++++-------- corebrain/cli/common.py | 6 +++--- corebrain/lib/sso/auth.py | 7 ++++--- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index a6acc17..71a567c 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -52,7 +52,7 @@ def main_cli(argv: Optional[List[str]] = None) -> int: parser.add_argument("--sso-url", help="Globodain SSO service URL") parser.add_argument("--login", action="store_true", help="Login via SSO") parser.add_argument("--test-auth", action="store_true", help="Test SSO authentication system") - parser.add_argument("--create", action="store_true", help="Create a new API Key") + parser.add_argument("--create-api-key", action="store_true", help="Create a new API Key") parser.add_argument("--key-name", help="Name of the new API Key") parser.add_argument("--key-level", choices=["read", "write", "admin"], default="read", help="Access level for the new API Key") @@ -80,7 +80,8 @@ def main_cli(argv: Optional[List[str]] = None) -> int: 'GLOBODAIN_CLIENT_ID': SSO_CLIENT_ID, 'GLOBODAIN_CLIENT_SECRET': SSO_CLIENT_SECRET, 'GLOBODAIN_REDIRECT_URI': f"http://localhost:{DEFAULT_PORT}/auth/sso/callback", - 'GLOBODAIN_SUCCESS_REDIRECT': f"http://localhost:{DEFAULT_PORT}/auth/sso/callback" + 'GLOBODAIN_SUCCESS_REDIRECT': f"http://localhost:{DEFAULT_PORT}/auth/sso/callback", + "GLOBODAIN_SERVICE_ID": 2, } try: @@ -163,13 +164,16 @@ def main_cli(argv: Optional[List[str]] = None) -> int: elif args.extract_schema: extract_schema_to_file(api_key, args.config_id, args.output_file, api_url) - # Creating the API key - if args.create: - - if not args.token: - print_colored("You must provide an API token using --token", "yellow") + # Handles the CLI command to create a new API key using stored credentials (token from SSO) + if args.create_api_key: + + # Get token from SSO + if not os.environ.get("COREBRAIN_SSO_URL"): + print_colored("You must log in to SSO first using --login to obtain a valid token.", color="yellow") return 1 + token = os.environ.get("COREBRAIN_SSO_URL") + # Checks if the API Key is set in argument if not args.key_name: print_colored("You must provide a name for the API Key using --key-name", "yellow") return 1 @@ -177,7 +181,7 @@ def main_cli(argv: Optional[List[str]] = None) -> int: try: api_key = Corebrain.create_api_key( DEFAULT_API_URL, - api_token=args.token, + api_token=token, name=args.key_name, level=args.key_level ) diff --git a/corebrain/cli/common.py b/corebrain/cli/common.py index 9dd55e9..2f16ad9 100644 --- a/corebrain/cli/common.py +++ b/corebrain/cli/common.py @@ -3,8 +3,8 @@ """ DEFAULT_API_URL = "http://localhost:5000" -DEFAULT_SSO_URL = "http://localhost:3000" +DEFAULT_SSO_URL = "https://sso.globodain.com" DEFAULT_PORT = 8765 DEFAULT_TIMEOUT = 10 -SSO_CLIENT_ID = '401dca6e-3f3b-4458-b3ef-f87eaae0398d' -SSO_CLIENT_SECRET = 'f9d315ea-5a65-4e3f-be35-b27a933dfb5b' \ No newline at end of file +SSO_CLIENT_ID = '63d767e9-5a06-4890-a194-8608ae29d426' +SSO_CLIENT_SECRET = '06cf39f6-ca93-466e-955e-cb6ea0a02d4d' \ No newline at end of file diff --git a/corebrain/lib/sso/auth.py b/corebrain/lib/sso/auth.py index a782d8c..6f80f18 100644 --- a/corebrain/lib/sso/auth.py +++ b/corebrain/lib/sso/auth.py @@ -8,11 +8,12 @@ def __init__(self, config=None): self.logger = logging.getLogger(__name__) # Configuración por defecto - self.sso_url = self.config.get('GLOBODAIN_SSO_URL', 'http://localhost:3000/login') # URL del SSO - self.client_id = self.config.get('GLOBODAIN_CLIENT_ID', '') - self.client_secret = self.config.get('GLOBODAIN_CLIENT_SECRET', '') + self.sso_url = self.config.get('GLOBODAIN_SSO_URL', 'https://sso.globodain.com/login') # URL del SSO + self.client_id = self.config.get('GLOBODAIN_CLIENT_ID', '63d767e9-5a06-4890-a194-8608ae29d426') + self.client_secret = self.config.get('GLOBODAIN_CLIENT_SECRET', '06cf39f6-ca93-466e-955e-cb6ea0a02d4d') self.redirect_uri = self.config.get('GLOBODAIN_REDIRECT_URI', '') self.success_redirect = self.config.get('GLOBODAIN_SUCCESS_REDIRECT', 'https://sso.globodain.com/cli/success') + self.service_id = self.config.get('GLOBODAIN_SERVICE_ID', 2) def requires_auth(self, session_handler): """ From 17ef7d59d7a5a5f91d346d51e8f4ecd3478d70fb Mon Sep 17 00:00:00 2001 From: Pawel Wasilewski Date: Fri, 23 May 2025 10:05:11 +0200 Subject: [PATCH 35/81] adding nosql.py module and edit the __init__.py --- corebrain/.gitignore | 174 +++++++++++++ corebrain/LICENSE | 21 ++ corebrain/README.md | 106 ++++++++ corebrain/db/connectors/__init__.py | 19 +- corebrain/db/connectors/nosql.py | 366 ++++++++++++++++++++++++++++ 5 files changed, 677 insertions(+), 9 deletions(-) create mode 100644 corebrain/.gitignore create mode 100644 corebrain/LICENSE create mode 100644 corebrain/README.md create mode 100644 corebrain/db/connectors/nosql.py diff --git a/corebrain/.gitignore b/corebrain/.gitignore new file mode 100644 index 0000000..0a19790 --- /dev/null +++ b/corebrain/.gitignore @@ -0,0 +1,174 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc diff --git a/corebrain/LICENSE b/corebrain/LICENSE new file mode 100644 index 0000000..30ba189 --- /dev/null +++ b/corebrain/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Rubén Ayuso + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/corebrain/README.md b/corebrain/README.md new file mode 100644 index 0000000..22ebdd7 --- /dev/null +++ b/corebrain/README.md @@ -0,0 +1,106 @@ +# Corebrain + +![Version](https://img.shields.io/badge/version-0.1.0-blue) +![Status](https://img.shields.io/badge/status-alpha-orange) +![License](https://img.shields.io/badge/license-MIT-green) + +## What is Corebrain? + +Corebrain is an open-source enterprise solution designed to centralize and optimize corporate data management. The project offers a scalable architecture for processing, analyzing, and visualizing critical information for decision-making. + +**IMPORTANT NOTE**: In the current version (0.1.0-alpha), only the SQL code is functional. Other modules are under development. + +## Current Status + +- ✅ SQL queries for data extraction +- ✅ Database schemas +- ✅ Authentication service +- ❌ NoSQL (in development) +- ❌ Frontend (in development) +- ❌ REST API (in development) + +## SDK Integration +Corebrain provides SDKs for multiple programming languages, making it easy to integrate with your existing systems. While only SQL in Python functionality is currently available, this SDK will support all features and most common languages as they are developed. + +![Python](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10-blue) + +## Available Versions + +- **`main` Branch**: Stable version with verified functionality (currently only SQL is functional) +- **`pre-release` Branch**: Initial version with all features in development (may contain errors) + +## Getting Started + +### Installation + +```bash +# Clone the repository +git clone https://github.com/your-organization/corebrain.git + +# Enter the directory +cd corebrain + +# Install dependencies +npm install +``` + +### Configuration + +1. Use `corebrain --configure` to start the configuration. +2. Once configuration has been completed, copy the config_id and replace in your example code (see 'examples' folder). +3. Run the example code in Python and enjoy! + +### Basic Usage + +```bash +# Run SQL migrations +npm run migrate + +# Start the SQL service +npm run sql:start +``` + +## Accessing the Pre-release Version + +If you want to test all features under development (including unstable components), you can switch to the pre-release branch: + +```bash +git checkout pre-release +npm install +``` + +**Warning**: The pre-release version contains experimental features with bugs or unexpected behaviors. Not recommended for production environments. + +## Contributing + +Corebrain is an open-source project, and we welcome all contributions. To contribute: + +1. Fork the repository +2. Create a new branch (`git checkout -b feature/new-feature`) +3. Make your changes +4. Run tests (`npm test`) +5. Commit your changes (`git commit -m 'Add new feature'`) +6. Push to your fork (`git push origin feature/new-feature`) +7. Open a Pull Request + +Please read our [contribution guidelines](CONTRIBUTING.md) before you start. + +## Roadmap + +- **0.1.0**: Basic SQL operation. OpenAI connected. Authentication service Globodain SSO integrated. API Keys configuration integrated. +- **0.2.0**: NoSQL (MongoDB) fixed. API Key creation by command "Corebrain --configure". Functional version. +- **0.3.0**: API deployment and integration at source. Functional version for third parties. +... +- **1.0.0**: First stable version with all features. + +You can see the full report at [Project Roadmap](https://github.com/users/ceoweggo/projects/4/views/2) + +## License + +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. + +## Contact + +- **Email**: [ruben@globodain.com](mailto:ruben@globodain.com) +- **Issues**: [Report a problem](https://github.com/ceoweggo/corebrain/issues) + diff --git a/corebrain/db/connectors/__init__.py b/corebrain/db/connectors/__init__.py index 3db5c71..8475616 100644 --- a/corebrain/db/connectors/__init__.py +++ b/corebrain/db/connectors/__init__.py @@ -5,7 +5,7 @@ from typing import Dict, Any from corebrain.db.connectors.sql import SQLConnector -from corebrain.db.connectors.mongodb import MongoDBConnector +from corebrain.db.connectors.nosql import NoSQLConnector def get_connector(db_config: Dict[str, Any]): """ @@ -18,11 +18,12 @@ def get_connector(db_config: Dict[str, Any]): Instance of the appropriate connector """ db_type = db_config.get("type", "").lower() - - if db_type == "sql": - engine = db_config.get("engine", "").lower() - return SQLConnector(db_config, engine) - elif db_type == "nosql" or db_type == "mongodb": - return MongoDBConnector(db_config) - else: - raise ValueError(f"Tipo de base de datos no soportado: {db_type}") \ No newline at end of file + engine = db_config.get("engine", "").lower() + + match db_type: + case "sql": + return SQLConnector(db_config, engine) + case "nosql": + return NoSQLConnector(db_config, engine) + case _: + raise ValueError(f"Unsupported database type: {db_type}") \ No newline at end of file diff --git a/corebrain/db/connectors/nosql.py b/corebrain/db/connectors/nosql.py new file mode 100644 index 0000000..c738946 --- /dev/null +++ b/corebrain/db/connectors/nosql.py @@ -0,0 +1,366 @@ +''' +NoSQL Database Connector +This module provides a basic structure for connecting to a NoSQL database. +It includes methods for connecting, disconnecting, and executing queries. +''' + +import time +import json +import re + +from typing import Dict, Any, List, Optional, Callable, Tuple + +# Try'ies for imports DB's (for now only mongoDB) +try: + import pymongo + from pymongo.errors import ConnectionFailure, ServerSelectionTimeoutError + PYMONGO_IMPORTED = True +except ImportError: + PYMONGO_IMPORTED = False +# Whe nadding new DB type write a try to it from user + +from corebrain.db.connector import DatabaseConnector +class NoSQLConnector(DatabaseConnector): + ''' + NoSQL Database Connector + This class provides a basic structure for connecting to a NoSQL database. + It includes methods for connecting, disconnecting, and executing queries. + ''' + def __init__(self, config: Dict[str, Any]): + ''' + Initialize the NoSQL database connector. + Args: + engine (str): Name of the database. + config (dict): Configuration dictionary containing connection parameters. + ''' + super().__init__(config) + self.engine = config.get("engine", "").lower() + self.client = None + self.db = None + self.config = config + self.connection_timeout = 30 # seconds + + match self.engine: + case "mongodb": + if not PYMONGO_IMPORTED: + raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") + case _: + pass + + + def connect(self) -> bool: + ''' + Connection with NoSQL DB's + Args: + self.engine (str): Name of the database. + ''' + + match self.engine: + case "mongodb": + if not PYMONGO_IMPORTED: + raise ImportError("Pymongo is not installed. Please install it to use MongoDB connector.") + try: + start_time = time.time() + # Check if connection string is provided + if "connection_string" in self.config: + connection_string = self.config["connection_string"] + if "connectTimeoutMS=" not in connection_string: + if "?" in connection_string: + connection_string += "&connectTimeoutMS=10000" + else: + connection_string += "?connectTimeoutMS=10000" + self.client = pymongo.MongoClient(connection_string) + else: + + # Setup for MongoDB connection parameters + + mongo_params = { + "host": self.config.get("host", "localhost"), + "port": int(self.config.get("port", 27017)), + "connection_timeoutMS": 10000, + "serverSelectionTimeoutMS": 10000, + } + + # Required parameters + + if self.config.get("user"): + mongo_params["username"] = self.config["user"] + if self.config.get("password"): + mongo_params["password"] = self.config["password"] + + #Optional parameters + + if self.config.get("authSource"): + mongo_params["authSource"] = self.config["authSource"] + if self.config.get("authMechanism"): + mongo_params["authMechanism"] = self.config["authMechanism"] + + # Insert parameters for MongoDB + self.client = pymongo.MongoClient(**mongo_params) + # Ping test for DB connection + self.client.admin.command('ping') + + db_name = self.config.get("database", "") + + if not db_name: + db_names = self.client.list_database_names() + if not db_names: + raise ValueError("No database names found in the MongoDB server.") + system_dbs = ["admin", "local", "config"] + for name in db_names: + if name not in system_dbs: + db_name = name + break + if not db_name: + db_name = db_names[0] + print(f"Not specified database name. Using the first available database: {db_name}") + self.db = self.client[db_name] + return True + except (ConnectionFailure, ServerSelectionTimeoutError) as e: + if time.time() - start_time > self.connection_timeout: + print(f"Connection to MongoDB timed out after {self.connection_timeout} seconds.") + time.sleep(2) + self.close() + return False + # Add case when is needed new DB type + case _ : + raise ValueError(f"Unsupported NoSQL database: {self.engine}") + pass + + def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] = None, + progress_callback: Optional[Callable] = None) -> Dict[str, Any]: + ''' + Extract schema from the NoSQL database. + Args: + sample_limit (int): Number of samples to extract for schema inference. + collection_limit (int): Maximum number of collections to process. + progress_callback (Callable): Optional callback function for progress updates. + Returns: + Dict[str, Any]: Extracted schema information. + ''' + + match self.engine: + case "mongodb": + if not PYMONGO_IMPORTED: + raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") + if not self.client and not self.connect(): + return { + "type": "mongodb", + "tables": {}, + "tables_list": [] + } + schema = { + "type": "mongodb", + "database": self.db.name, + "tables": {}, # In MongoDB, tables are collections + } + try: + collections = self.db.list_collection_names() + if collection_limit is not None and collection_limit > 0: + collections = collections[:collection_limit] + total_collections = len(collections) + for i, collection_name in enumerate(collections): + if progress_callback: + progress_callback(i, total_collections, f"Processing collection: {collection_name}") + collection = self.db[collection_name] + + try: + doc_count = collection.count_documents({}) + if doc_count <= 0: + schema["tables"][collection_name] = { + "fields": [], + "sample_data": [], + "count": 0, + "empty": True + } + else: + sample_docs = list(collection.find().limit(sample_limit)) + fields = {} + sample_data = [] + + for doc in sample_docs: + self._extract_document_fields(doc, fields) + processed_doc = self._process_document_for_serialization(doc) + sample_data.append(processed_doc) + + formatted_fields = [{"name": field, "type": type_name} for field, type_name in fields.items()] + + schema["tables"][collection_name] = { + "fields": formatted_fields, + "sample_data": sample_data, + "count": doc_count, + } + except Exception as e: + print(f"Error processing collection {collection_name}: {e}") + schema["tables"][collection_name] = { + "fields": [], + "error": str(e) + } + # Convert the schema to a list of tables + table_list = [] + for collection_name, collection_info in schema["tables"].items(): + table_data = {"name": collection_name} + table_data.update(collection_info) + table_list.append(table_data) + schema["tables_list"] = table_list + return schema + except Exception as e: + print(f"Error extracting schema: {e}") + return { + "type": "mongodb", + "tables": {}, + "tabbles_list": [] + } + # Add case when is needed new DB type + case _ : + raise ValueError(f"Unsupported NoSQL database: {self.engine}") + def _extract_document_fields(self, doc: Dict[str, Any], fields: Dict[str, str], + prefix: str = "", max_depth: int = 3, current_depth: int = 0) -> None: + ''' + + Recursively extract fields from a document and determine their types. + Args: + doc (Dict[str, Any]): The document to extract fields from. + fields (Dict[str, str]): Dictionary to store field names and types. + prefix (str): Prefix for nested fields. + max_depth (int): Maximum depth for nested fields. + current_depth (int): Current depth in the recursion. + ''' + match self.engine: + case "mongodb": + if not PYMONGO_IMPORTED: + raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") + if current_depth >= max_depth: + return + for field, value in doc.items(): + if field == "_id": + field_type = "ObjectId" + elif isinstance(value, dict): + if value and current_depth < max_depth - 1: + self._extract_document_fields(value, fields, f"{prefix}{field}.", max_depth, current_depth + 1) + else: + field_type = f"array<{type(value[0]).__name__}>" + else: + field_type = "array" + else: + field_type = type(value).__name__ + + field_key = f"{prefix}{field}" + if field_key not in fields: + fields[field_key] = field_type + # Add case when is needed new DB type + case _ : + raise ValueError(f"Unsupported NoSQL database: {self.engine}") + + def _process_document_for_serialization(self, doc: Dict[str, Any]) -> Dict[str, Any]: + ''' + Proccesig a document for serialization of a JSON. + Args: + doc (Dict[str, Any]): The document to process. + Returns: + Procesed document + ''' + match self.engine: + case "mongodb": + if not PYMONGO_IMPORTED: + raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") + processed_doc = {} + for field, value in doc.items(): + if field == "_id": + processed_doc[field] = self._process_document_for_serialization(value) + elif isinstance(value, list): + processed_items = [] + for item in value: + if isinstance(item, dict): + processed_items.append(self._process_document_for_serialization(item)) + elif hasattr(item, "__str__"): + processed_items.append(str(item)) + else: + processed_items.append(item) + processed_doc[field] = processed_items + # Convert fetch to ISO + elif hasattr(value, 'isoformat'): + processed_doc[field] = value.isoformat() + # Convert data + else: + processed_doc[field] = value + return processed_doc + # Add case when is needed new DB type + case _ : + raise ValueError(f"Unsupported NoSQL database: {self.engine}") + + def execute_query(self, query: str) -> List[Dict[str, Any]]: + """ + Runs a NoSQL (or other) query with improved error handling + + Args: + query: A NoSQL (or other) query in JSON format or query language + + Returns: + List of resulting documents. + """ + + match self.engine: + case "nosql": + if not PYMONGO_IMPORTED: + raise ImportError("Pymongo is not installed. Please install it to use NoSQL connector.") + + if not self.client and not self.connect(): + raise ConnectionError("Couldn't estabilish a connection with NoSQL") + + try: + # Determine whether the query is a JSON string or a query in another format + filter_dict, projection, collection_name, limit = self._parse_query(query) + + # Get the collection + if not collection_name: + raise ValueError("Name of the colletion not specified in the query") + + collection = self.db[collection_name] + + # Execute the query + if projection: + cursor = collection.find(filter_dict, projection).limit(limit or 100) + else: + cursor = collection.find(filter_dict).limit(limit or 100) + + # Convert the results to a serializable format + results = [] + for doc in cursor: + processed_doc = self._process_document_for_serialization(doc) + results.append(processed_doc) + + return results + + except Exception as e: + # Reconnect and retry the query + try: + self.close() + if self.connect(): + print("Reconnecting and retrying the query...") + + # Retry the query + filter_dict, projection, collection_name, limit = self._parse_query(query) + collection = self.db[collection_name] + + if projection: + cursor = collection.find(filter_dict, projection).limit(limit or 100) + else: + cursor = collection.find(filter_dict).limit(limit or 100) + + results = [] + for doc in cursor: + processed_doc = self._process_document_for_serialization(doc) + results.append(processed_doc) + + return results + except Exception as retry_error: + # If retrying fails, show the original error + raise Exception(f"Failed to execute the NoSQL query: {str(e)}") + + # This code is will be executed if the retry fails + raise Exception(f"Failed to execute the NoSQL query (after the reconnection): {str(e)}") + + # Add case when is needed new DB type + case _ : + raise ValueError(f"Unsupported NoSQL database: {self.self.engine}") \ No newline at end of file From d95f8748640cc8dc0227d9dd8b21a46d8d07f8f9 Mon Sep 17 00:00:00 2001 From: Pawel Wasilewski Date: Fri, 23 May 2025 10:08:04 +0200 Subject: [PATCH 36/81] , --- corebrain/cli/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corebrain/cli/common.py b/corebrain/cli/common.py index 7799dcf..d083f32 100644 --- a/corebrain/cli/common.py +++ b/corebrain/cli/common.py @@ -2,7 +2,7 @@ Default values for SSO and API connection """ -DEFAULT_API_URL = "http://localhost:5000" +DEFAULT_API_URL = "http://localhost:8000" #DEFAULT_SSO_URL = "http://localhost:3000" # localhost DEFAULT_SSO_URL = "https://sso.globodain.com" # remote DEFAULT_PORT = 8765 From cb5b1fda57084364c6a72714f830d7acb8b6456c Mon Sep 17 00:00:00 2001 From: Pawel Wasilewski Date: Fri, 23 May 2025 10:58:07 +0200 Subject: [PATCH 37/81] minor changes --- corebrain/.github/workflows/ci.yml | 146 ++ corebrain/.gitignore | 6 +- corebrain/.gitmodules | 3 + corebrain/CONTRIBUTING.md | 147 ++ corebrain/README.md | 227 ++- corebrain/corebrain/__init__.py | 86 ++ corebrain/corebrain/cli.py | 8 + corebrain/corebrain/cli/__init__.py | 57 + corebrain/corebrain/cli/__main__.py | 12 + corebrain/corebrain/cli/auth/__init__.py | 22 + corebrain/corebrain/cli/auth/api_keys.py | 299 ++++ corebrain/corebrain/cli/auth/sso.py | 452 ++++++ corebrain/corebrain/cli/commands.py | 453 ++++++ corebrain/corebrain/cli/common.py | 15 + corebrain/corebrain/cli/config.py | 489 ++++++ corebrain/corebrain/cli/utils.py | 595 +++++++ corebrain/corebrain/config/__init__.py | 10 + corebrain/corebrain/config/manager.py | 235 +++ corebrain/corebrain/core/__init__.py | 20 + corebrain/corebrain/core/client.py | 1364 +++++++++++++++++ corebrain/corebrain/core/common.py | 225 +++ corebrain/corebrain/core/query.py | 1037 +++++++++++++ corebrain/corebrain/core/test_utils.py | 157 ++ corebrain/corebrain/db/__init__.py | 26 + corebrain/corebrain/db/connector.py | 33 + corebrain/corebrain/db/connectors/__init__.py | 28 + corebrain/corebrain/db/connectors/mongodb.py | 474 ++++++ corebrain/corebrain/db/connectors/sql.py | 598 ++++++++ corebrain/corebrain/db/engines.py | 16 + corebrain/corebrain/db/factory.py | 29 + corebrain/corebrain/db/interface.py | 36 + corebrain/corebrain/db/schema/__init__.py | 11 + corebrain/corebrain/db/schema/extractor.py | 123 ++ corebrain/corebrain/db/schema/optimizer.py | 157 ++ corebrain/corebrain/db/schema_file.py | 604 ++++++++ corebrain/corebrain/lib/sso/__init__.py | 4 + corebrain/corebrain/lib/sso/auth.py | 171 +++ corebrain/corebrain/lib/sso/client.py | 194 +++ corebrain/corebrain/network/__init__.py | 22 + corebrain/corebrain/network/client.py | 502 ++++++ corebrain/corebrain/sdk.py | 8 + corebrain/corebrain/services/schema.py | 31 + corebrain/corebrain/utils/__init__.py | 66 + corebrain/corebrain/utils/encrypter.py | 264 ++++ corebrain/corebrain/utils/logging.py | 243 +++ corebrain/corebrain/utils/serializer.py | 33 + .../corebrain/wrappers/csharp/.editorconfig | 432 ++++++ .../corebrain/wrappers/csharp/.gitignore | 417 +++++ .../wrappers/csharp/.vscode/settings.json | 3 + .../wrappers/csharp/.vscode/tasks.json | 32 + .../CorebrainCS.Tests.csproj | 14 + .../csharp/CorebrainCS.Tests/Program.cs | 10 + .../csharp/CorebrainCS.Tests/README.md | 4 + .../corebrain/wrappers/csharp/CorebrainCS.sln | 48 + .../csharp/CorebrainCS/CorebrainCS.cs | 175 +++ .../csharp/CorebrainCS/CorebrainCS.csproj | 7 + corebrain/corebrain/wrappers/csharp/LICENSE | 21 + corebrain/corebrain/wrappers/csharp/README.md | 77 + .../wrappers/csharp_cli_api/.gitignore | 548 +++++++ .../csharp_cli_api/.vscode/settings.json | 3 + .../csharp_cli_api/CorebrainCLIAPI.sln | 50 + .../wrappers/csharp_cli_api/README.md | 18 + .../wrappers/csharp_cli_api/src/.editorconfig | 432 ++++++ .../src/CorebrainCLIAPI/CommandController.cs | 70 + .../CorebrainCLIAPI/CorebrainCLIAPI.csproj | 20 + .../src/CorebrainCLIAPI/CorebrainSettings.cs | 8 + .../src/CorebrainCLIAPI/Program.cs | 49 + .../Properties/launchSettings.json | 23 + .../appsettings.Development.json | 8 + .../src/CorebrainCLIAPI/appsettings.json | 14 + corebrain/docs/Makefile | 20 + corebrain/docs/README.md | 13 + corebrain/docs/make.bat | 35 + corebrain/docs/source/_static/custom.css | 0 corebrain/docs/source/conf.py | 25 + corebrain/docs/source/corebrain.cli.auth.rst | 29 + corebrain/docs/source/corebrain.cli.rst | 53 + corebrain/docs/source/corebrain.config.rst | 21 + corebrain/docs/source/corebrain.core.rst | 45 + .../docs/source/corebrain.db.connectors.rst | 29 + corebrain/docs/source/corebrain.db.rst | 62 + corebrain/docs/source/corebrain.db.schema.rst | 29 + corebrain/docs/source/corebrain.network.rst | 21 + corebrain/docs/source/corebrain.rst | 42 + corebrain/docs/source/corebrain.utils.rst | 37 + corebrain/docs/source/index.rst | 14 + corebrain/docs/source/modules.rst | 7 + corebrain/examples/add_config.py | 27 + corebrain/examples/complex.py | 23 + corebrain/examples/list_schema.py | 162 ++ corebrain/examples/simple.py | 15 + corebrain/health.py | 47 + corebrain/pyproject.toml | 85 + corebrain/setup.ps1 | 5 + corebrain/setup.py | 38 + corebrain/setup.sh | 6 + 96 files changed, 13045 insertions(+), 66 deletions(-) create mode 100644 corebrain/.github/workflows/ci.yml create mode 100644 corebrain/.gitmodules create mode 100644 corebrain/CONTRIBUTING.md create mode 100644 corebrain/corebrain/__init__.py create mode 100644 corebrain/corebrain/cli.py create mode 100644 corebrain/corebrain/cli/__init__.py create mode 100644 corebrain/corebrain/cli/__main__.py create mode 100644 corebrain/corebrain/cli/auth/__init__.py create mode 100644 corebrain/corebrain/cli/auth/api_keys.py create mode 100644 corebrain/corebrain/cli/auth/sso.py create mode 100644 corebrain/corebrain/cli/commands.py create mode 100644 corebrain/corebrain/cli/common.py create mode 100644 corebrain/corebrain/cli/config.py create mode 100644 corebrain/corebrain/cli/utils.py create mode 100644 corebrain/corebrain/config/__init__.py create mode 100644 corebrain/corebrain/config/manager.py create mode 100644 corebrain/corebrain/core/__init__.py create mode 100644 corebrain/corebrain/core/client.py create mode 100644 corebrain/corebrain/core/common.py create mode 100644 corebrain/corebrain/core/query.py create mode 100644 corebrain/corebrain/core/test_utils.py create mode 100644 corebrain/corebrain/db/__init__.py create mode 100644 corebrain/corebrain/db/connector.py create mode 100644 corebrain/corebrain/db/connectors/__init__.py create mode 100644 corebrain/corebrain/db/connectors/mongodb.py create mode 100644 corebrain/corebrain/db/connectors/sql.py create mode 100644 corebrain/corebrain/db/engines.py create mode 100644 corebrain/corebrain/db/factory.py create mode 100644 corebrain/corebrain/db/interface.py create mode 100644 corebrain/corebrain/db/schema/__init__.py create mode 100644 corebrain/corebrain/db/schema/extractor.py create mode 100644 corebrain/corebrain/db/schema/optimizer.py create mode 100644 corebrain/corebrain/db/schema_file.py create mode 100644 corebrain/corebrain/lib/sso/__init__.py create mode 100644 corebrain/corebrain/lib/sso/auth.py create mode 100644 corebrain/corebrain/lib/sso/client.py create mode 100644 corebrain/corebrain/network/__init__.py create mode 100644 corebrain/corebrain/network/client.py create mode 100644 corebrain/corebrain/sdk.py create mode 100644 corebrain/corebrain/services/schema.py create mode 100644 corebrain/corebrain/utils/__init__.py create mode 100644 corebrain/corebrain/utils/encrypter.py create mode 100644 corebrain/corebrain/utils/logging.py create mode 100644 corebrain/corebrain/utils/serializer.py create mode 100644 corebrain/corebrain/wrappers/csharp/.editorconfig create mode 100644 corebrain/corebrain/wrappers/csharp/.gitignore create mode 100644 corebrain/corebrain/wrappers/csharp/.vscode/settings.json create mode 100644 corebrain/corebrain/wrappers/csharp/.vscode/tasks.json create mode 100644 corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/CorebrainCS.Tests.csproj create mode 100644 corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs create mode 100644 corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/README.md create mode 100644 corebrain/corebrain/wrappers/csharp/CorebrainCS.sln create mode 100644 corebrain/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs create mode 100644 corebrain/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.csproj create mode 100644 corebrain/corebrain/wrappers/csharp/LICENSE create mode 100644 corebrain/corebrain/wrappers/csharp/README.md create mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/.gitignore create mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/.vscode/settings.json create mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/CorebrainCLIAPI.sln create mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/README.md create mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/src/.editorconfig create mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CommandController.cs create mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainCLIAPI.csproj create mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainSettings.cs create mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Program.cs create mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Properties/launchSettings.json create mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.Development.json create mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.json create mode 100644 corebrain/docs/Makefile create mode 100644 corebrain/docs/README.md create mode 100644 corebrain/docs/make.bat create mode 100644 corebrain/docs/source/_static/custom.css create mode 100644 corebrain/docs/source/conf.py create mode 100644 corebrain/docs/source/corebrain.cli.auth.rst create mode 100644 corebrain/docs/source/corebrain.cli.rst create mode 100644 corebrain/docs/source/corebrain.config.rst create mode 100644 corebrain/docs/source/corebrain.core.rst create mode 100644 corebrain/docs/source/corebrain.db.connectors.rst create mode 100644 corebrain/docs/source/corebrain.db.rst create mode 100644 corebrain/docs/source/corebrain.db.schema.rst create mode 100644 corebrain/docs/source/corebrain.network.rst create mode 100644 corebrain/docs/source/corebrain.rst create mode 100644 corebrain/docs/source/corebrain.utils.rst create mode 100644 corebrain/docs/source/index.rst create mode 100644 corebrain/docs/source/modules.rst create mode 100644 corebrain/examples/add_config.py create mode 100644 corebrain/examples/complex.py create mode 100644 corebrain/examples/list_schema.py create mode 100644 corebrain/examples/simple.py create mode 100644 corebrain/health.py create mode 100644 corebrain/pyproject.toml create mode 100644 corebrain/setup.ps1 create mode 100644 corebrain/setup.py create mode 100644 corebrain/setup.sh diff --git a/corebrain/.github/workflows/ci.yml b/corebrain/.github/workflows/ci.yml new file mode 100644 index 0000000..a2db687 --- /dev/null +++ b/corebrain/.github/workflows/ci.yml @@ -0,0 +1,146 @@ +name: Corebrain SDK CI/CD + +on: + push: + branches: [ main, develop ] + tags: + - 'v*' + pull_request: + branches: [ main, develop ] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.8', '3.9', '3.10', '3.11'] + + services: + # PostgreSQL service for integration tests + postgres: + image: postgres:13 + env: + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + POSTGRES_DB: test_db + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + # MongoDB service for NoSQL integration tests + mongodb: + image: mongo:4.4 + ports: + - 27017:27017 + options: >- + --health-cmd "mongo --eval 'db.runCommand({ ping: 1 })'" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install .[dev,all_db] + + - name: Lint with flake8 + run: | + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + + - name: Type check with mypy + run: | + mypy core db cli utils + + - name: Format check with black + run: | + black --check . + + - name: Test with pytest + run: | + pytest --cov=. --cov-report=xml + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + fail_ci_if_error: false + + build-and-publish: + needs: test + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/v') + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build package + run: | + python -m build + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + skip_existing: true + + docker: + needs: test + runs-on: ubuntu-latest + if: | + (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop')) || + startsWith(github.ref, 'refs/tags/v') + + steps: + - uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: corebrain/sdk + tags: | + type=ref,event=branch + type=ref,event=tag + type=semver,pattern={{version}} + type=sha,format=short + + - name: Build and push + uses: docker/build-push-action@v3 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/corebrain/.gitignore b/corebrain/.gitignore index 0a19790..cf98110 100644 --- a/corebrain/.gitignore +++ b/corebrain/.gitignore @@ -2,6 +2,10 @@ __pycache__/ *.py[cod] *$py.class +venv/ +.tofix/ +README-no-valid.md +requirements.txt # C extensions *.so @@ -14,7 +18,7 @@ dist/ downloads/ eggs/ .eggs/ -lib/ +#lib/ lib64/ parts/ sdist/ diff --git a/corebrain/.gitmodules b/corebrain/.gitmodules new file mode 100644 index 0000000..a034212 --- /dev/null +++ b/corebrain/.gitmodules @@ -0,0 +1,3 @@ +[submodule "corebrain/CLI-UI"] + path = corebrain/CLI-UI + url = https://github.com/Luki20091/CLI-UI.git diff --git a/corebrain/CONTRIBUTING.md b/corebrain/CONTRIBUTING.md new file mode 100644 index 0000000..47e6927 --- /dev/null +++ b/corebrain/CONTRIBUTING.md @@ -0,0 +1,147 @@ +# How to Contribute to Corebrain SDK + +Thank you for your interest in contributing to CoreBrain SDK! This document provides guidelines for contributing to the project. + +## Code of Conduct + +By participating in this project, you commit to maintaining a respectful and collaborative environment. + +## How to Contribute + +### Reporting Bugs + +1. Verify that the bug hasn't already been reported in the [issues](https://github.com/ceoweggo/Corebrain/issues) +2. Use the bug template to create a new issue +3. Include as much detail as possible: steps to reproduce, environment, versions, etc. +4. If possible, include a minimal example that reproduces the problem + +### Suggesting Improvements + +1. Check the [issues](https://github.com/ceoweggo/Corebrain/issues) to see if it has already been suggested +2. Use the feature template to create a new issue +3. Clearly describe the improvement and justify its value + +### Submitting Changes + +1. Fork the repository +2. Create a branch for your change (`git checkout -b feature/amazing-feature`) +3. Make your changes following the code conventions +4. Write tests for your changes +5. Ensure all tests pass +6. Commit your changes (`git commit -m 'Add amazing feature'`) +7. Push your branch (`git push origin feature/amazing-feature`) +8. Open a Pull Request + +## Development Environment + +### Installation for Development + +```bash +# Clone the repository +git clone https://github.com/ceoweggo/Corebrain.git +cd sdk + +# Create virtual environment +python -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate + +# Install for development +pip install -e ".[dev]" +``` + +### Project Structure + +``` +v1/ +├── corebrain/ # Main package +│ ├── __init__.py +│ ├── _pycache_/ +│ ├── cli/ # Command-line interface +│ ├── config/ # Configuration management +│ ├── core/ # Core functionality +│ ├── db/ # Database interactions +│ ├── lib/ # Library components +│ └── SSO/ # Globodain SSO Authentication +│ ├── network/ # Network functionality +│ ├── services/ # Service implementations +│ ├── utils/ # Utility functions +│ ├── cli.py # CLI entry point +│ └── sdk.py # SDK entry point +├── corebrain.egg-info/ # Package metadata +├── docs/ # Documentation +├── examples/ # Usage examples +├── screenshots/ # Project screenshots +├── venv/ # Virtual environment (not to be committed) +├── .github/ # GitHub files directory +├── _pycache_/ # Python cache files +├── .tofix/ # Files to be fixed +├── .gitignore # Git ignore rules +├── CONTRIBUTING.md # Contribution guidelines +├── health.py # Health check script +├── LICENSE # License information +├── pyproject.toml # Project configuration +├── README-no-valid.md # Outdated README +├── README.md # Project overview +├── requirements.txt # Production dependencies +└── setup.py # Package setup +``` + +### Running Tests + +```bash +# Run all tests +pytest + +# Run specific test file +pytest tests/test_specific.py + +# Run tests with coverage +pytest --cov=corebrain +``` + +## Coding Standards + +### Style Guide + +- We follow [PEP 8](https://www.python.org/dev/peps/pep-0008/) for Python code +- Use 4 spaces for indentation +- Maximum line length is 88 characters +- Use descriptive variable and function names + +### Documentation + +- All modules, classes, and functions should have docstrings +- Follow the [Google docstring format](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings) +- Keep documentation up-to-date with code changes + +### Commit Messages + +- Use clear, concise commit messages +- Start with a verb in the present tense (e.g., "Add feature" not "Added feature") +- Reference issue numbers when applicable (e.g., "Fix #123: Resolve memory leak") + +## Pull Request Process + +1. Update documentation if necessary +2. Add or update tests as needed +3. Ensure CI checks pass +4. Request a review from maintainers +5. Address review feedback +6. Maintainers will merge your PR once approved + +## Release Process + +Our maintainers follow semantic versioning (MAJOR.MINOR.PATCH): +- MAJOR version for incompatible API changes +- MINOR version for backward-compatible functionality +- PATCH version for backward-compatible bug fixes + +## Getting Help + +If you need help with anything: +- Join our [Discord community](https://discord.gg/m2AXjPn2yV) +- Join our [Whatsapp Channel](https://whatsapp.com/channel/0029Vap43Vy5EjxvR4rncQ1I) +- Ask questions in the GitHub Discussions +- Contact the maintainers at ruben@globodain.com + +Thank you for contributing to Corebrain SDK! \ No newline at end of file diff --git a/corebrain/README.md b/corebrain/README.md index 22ebdd7..8d22293 100644 --- a/corebrain/README.md +++ b/corebrain/README.md @@ -1,106 +1,203 @@ -# Corebrain +# Corebrain SDK -![Version](https://img.shields.io/badge/version-0.1.0-blue) -![Status](https://img.shields.io/badge/status-alpha-orange) -![License](https://img.shields.io/badge/license-MIT-green) +![CI Status](https://github.com/ceoweggo/Corebrain/workflows/Corebrain%20SDK%20CI/CD/badge.svg) +[![PyPI version](https://badge.fury.io/py/corebrain.svg)](https://badge.fury.io/py/corebrain) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -## What is Corebrain? +SDK for natural language queries to relational and non-relational databases. Enables interaction with databases using natural language questions. -Corebrain is an open-source enterprise solution designed to centralize and optimize corporate data management. The project offers a scalable architecture for processing, analyzing, and visualizing critical information for decision-making. +## ✨ Features -**IMPORTANT NOTE**: In the current version (0.1.0-alpha), only the SQL code is functional. Other modules are under development. +- **Natural Language Queries**: Transforms human language questions into database queries (SQL/NoSQL) +- **Multi-Database Support**: Compatible with SQLite, MySQL, PostgreSQL, and MongoDB +- **Unified Interface**: Consistent API across different database types +- **Built-in CLI**: Interact with your databases directly from the terminal +- **Strong Security**: Robust authentication and secure credential management +- **Highly Extensible**: Designed for easy integration with new engines and features +- **Comprehensive Documentation**: Usage examples, API reference, and step-by-step guides -## Current Status +## 📋 Requirements -- ✅ SQL queries for data extraction -- ✅ Database schemas -- ✅ Authentication service -- ❌ NoSQL (in development) -- ❌ Frontend (in development) -- ❌ REST API (in development) +- Python 3.8+ +- Specific dependencies based on the database engine: + - **SQLite**: Included in Python + - **PostgreSQL**: `psycopg2-binary` + - **MySQL**: `mysql-connector-python` + - **MongoDB**: `pymongo` -## SDK Integration -Corebrain provides SDKs for multiple programming languages, making it easy to integrate with your existing systems. While only SQL in Python functionality is currently available, this SDK will support all features and most common languages as they are developed. +## 🔧 Installation -![Python](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10-blue) +### From PyPI (recommended) -## Available Versions +```bash +# Minimal installation +pip install corebrain -- **`main` Branch**: Stable version with verified functionality (currently only SQL is functional) -- **`pre-release` Branch**: Initial version with all features in development (may contain errors) +### From source code -## Getting Started +```bash +git clone https://github.com/ceoweggo/Corebrain.git +pip install -e . +``` -### Installation +## 🚀 Quick Start Guide -```bash -# Clone the repository -git clone https://github.com/your-organization/corebrain.git +### Initialization -# Enter the directory -cd corebrain +# > **⚠️ IMPORTANT:** +# > * If you don't have an existing configuration, first run `corebrain --configure` +# > * If you need to generate a new API key, use `corebrain --create` +# > * Never share your API key in public repositories. Use environment variables instead. + + +```python +from corebrain import init + +# Initialize with a previously saved configuration +corebrain = init( + api_key="your_api_key", + config_id="your_config_id" +) +``` -# Install dependencies -npm install +### Making Natural Language Queries + +```python +# Simple query +result = client.ask("How many active users are there?") +print(result["explanation"]) # Natural language explanation +print(result["query"]) # Generated SQL/NoSQL query +print(result["results"]) # Query results + +# Query with additional parameters +result = client.ask( + "Show the last 5 orders", + collection_name="orders", + limit=5, + filters={"status": "completed"} +) + +# Iterate over the results +for item in result["results"]: + print(item) ``` -### Configuration +### Getting the Database Schema -1. Use `corebrain --configure` to start the configuration. -2. Once configuration has been completed, copy the config_id and replace in your example code (see 'examples' folder). -3. Run the example code in Python and enjoy! +```python +# Get the complete schema +schema = client.db_schema -### Basic Usage +# List all tables/collections +tables = client.list_collections_name() +print(tables) +``` + +### Closing the Connection + +```python +# It's recommended to close the connection when finished +client.close() + +# Or use the with context +with init(api_key="your_api_key", config_id="your_config_id") as client: + result = client.ask("How many users are there?") + print(result["explanation"]) +``` + +## 🖥️ Command Line Interface Usage + +### Configure Connection ```bash -# Run SQL migrations -npm run migrate +# Init configuration +corebrain --configure +``` -# Start the SQL service -npm run sql:start +### Display Database Schema + +```bash +# Show complete schema +corebrain --show-schema ``` -## Accessing the Pre-release Version +### List Configurations -If you want to test all features under development (including unstable components), you can switch to the pre-release branch: +```bash +# List all configurations +corebrain --list-configs +``` + +## 📝 Advanced Documentation + +### Configuration Management + +```python +from corebrain import list_configurations, remove_configuration, get_config + +# List all configurations +configs = list_configurations(api_token="your_api_token") +print(configs) + +# Get details of a configuration +config = get_config(api_token="your_api_token", config_id="your_config_id") +print(config) + +# Remove a configuration +removed = remove_configuration(api_token="your_api_token", config_id="your_config_id") +print(f"Configuration removed: {removed}") +``` + +## 🧪 Testing and Development + +### Development Installation ```bash -git checkout pre-release -npm install +# Clone the repository +git clone https://github.com/ceoweggo/Corebrain.git +cd corebrain + +# Install in development mode with extra tools +pip install -e ".[dev,all_db]" ``` -**Warning**: The pre-release version contains experimental features with bugs or unexpected behaviors. Not recommended for production environments. +### Verifying Style and Typing -## Contributing +```bash +# Check style with flake8 +flake8 . -Corebrain is an open-source project, and we welcome all contributions. To contribute: +# Check typing with mypy +mypy core db cli utils -1. Fork the repository -2. Create a new branch (`git checkout -b feature/new-feature`) -3. Make your changes -4. Run tests (`npm test`) -5. Commit your changes (`git commit -m 'Add new feature'`) -6. Push to your fork (`git push origin feature/new-feature`) -7. Open a Pull Request +# Format code with black +black . +``` -Please read our [contribution guidelines](CONTRIBUTING.md) before you start. +### Continuous Integration and Deployment (CI/CD) -## Roadmap +The project uses GitHub Actions to automate: -- **0.1.0**: Basic SQL operation. OpenAI connected. Authentication service Globodain SSO integrated. API Keys configuration integrated. -- **0.2.0**: NoSQL (MongoDB) fixed. API Key creation by command "Corebrain --configure". Functional version. -- **0.3.0**: API deployment and integration at source. Functional version for third parties. -... -- **1.0.0**: First stable version with all features. +1. **Testing**: Runs tests on multiple Python versions (3.8-3.11) +2. **Quality Verification**: Checks style, typing, and formatting +3. **Coverage**: Generates code coverage reports +4. **Automatic Publication**: Publishes new versions to PyPI when tags are created +5. **Docker Images**: Builds and publishes Docker images with each version -You can see the full report at [Project Roadmap](https://github.com/users/ceoweggo/projects/4/views/2) +You can see the complete configuration in `.github/workflows/ci.yml`. -## License +## 🛠️ Contributions -This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. +Contributions are welcome! To contribute: + +1. Fork the repository +2. Create a branch for your feature (`git checkout -b feature/amazing-feature`) +3. Commit your changes (`git commit -m 'Add some amazing feature'`) +4. Push to the branch (`git push origin feature/amazing-feature`) +5. Open a Pull Request -## Contact +Please make sure your changes pass all tests and comply with the style guidelines. -- **Email**: [ruben@globodain.com](mailto:ruben@globodain.com) -- **Issues**: [Report a problem](https://github.com/ceoweggo/corebrain/issues) +## 📄 License +Distributed under the MIT License. See `LICENSE` for more information. \ No newline at end of file diff --git a/corebrain/corebrain/__init__.py b/corebrain/corebrain/__init__.py new file mode 100644 index 0000000..6819487 --- /dev/null +++ b/corebrain/corebrain/__init__.py @@ -0,0 +1,86 @@ +""" +Corebrain SDK. + +This package provides a Python SDK for interacting with the Corebrain API +and enables natural language queries to relational and non-relational databases. +""" +import logging +from typing import Dict, Any, List, Optional + +# Configuración básica de logging +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + +# Importaciones seguras (sin dependencias circulares) +from corebrain.db.engines import get_available_engines +from corebrain.core.client import Corebrain +from corebrain.config.manager import ConfigManager + +# Exportación explícita de componentes públicos +__all__ = [ + 'init', + 'extract_db_schema', + 'list_configurations', + 'remove_configuration', + 'get_available_engines', + 'get_config', + '__version__' +] + +# Variable de versión +__version__ = "1.0.0" + +def init(api_key: str, config_id: str, skip_verification: bool = False) -> Corebrain: + """ + Initialize the Corebrain SDK with the provided API key and configuration. + + Args: + api_key: API Key de Corebrain + config_id: ID de la configuración a usar + + Returns: + Instancia de Corebrain configurada + """ + return Corebrain(api_key=api_key, config_id=config_id, skip_verification=skip_verification) + +# Funciones de conveniencia a nivel de paquete +def list_configurations(api_key: str) -> List[str]: + """ + Lists the available configurations for an API key. + + Args: + api_key: Corebrain API Key + + Returns: + List of available configuration IDs + """ + config_manager = ConfigManager() + return config_manager.list_configs(api_key) + +def remove_configuration(api_key: str, config_id: str) -> bool: + """ + Deletes a specific configuration. + + Args: + api_key: Corebrain API Key + config_id: ID of the configuration to delete + + Returns: + True if deleted successfully, False otherwise + """ + config_manager = ConfigManager() + return config_manager.remove_config(api_key, config_id) + +def get_config(api_key: str, config_id: str) -> Optional[Dict[str, Any]]: + """ + Retrieves a specific configuration. + + Args: + api_key: Corebrain API Key + config_id: ID of the configuration to retrieve + + Returns: + Dictionary with the configuration or None if it does not exist + """ + config_manager = ConfigManager() + return config_manager.get_config(api_key, config_id) \ No newline at end of file diff --git a/corebrain/corebrain/cli.py b/corebrain/corebrain/cli.py new file mode 100644 index 0000000..7e17025 --- /dev/null +++ b/corebrain/corebrain/cli.py @@ -0,0 +1,8 @@ +""" +Entry point for the Corebrain CLI for compatibility. +""" +from corebrain.cli.__main__ import main + +if __name__ == "__main__": + import sys + sys.exit(main()) \ No newline at end of file diff --git a/corebrain/corebrain/cli/__init__.py b/corebrain/corebrain/cli/__init__.py new file mode 100644 index 0000000..53672d1 --- /dev/null +++ b/corebrain/corebrain/cli/__init__.py @@ -0,0 +1,57 @@ +""" +Command-line interface for the Corebrain SDK. + +This module provides a command-line interface to configure +and use the Corebrain SDK for natural language queries to databases. +""" +import sys +from typing import Optional, List + +# Importar componentes principales para CLI +from corebrain.cli.commands import main_cli +from corebrain.cli.utils import print_colored, ProgressTracker, get_free_port +from corebrain.cli.config import ( + configure_sdk, + get_db_type, + get_db_engine, + get_connection_params, + test_database_connection, + select_excluded_tables +) +from corebrain.cli.auth import ( + authenticate_with_sso, + fetch_api_keys, + exchange_sso_token_for_api_token, + verify_api_token +) + + +# Exportación explícita de componentes públicos +__all__ = [ + 'main_cli', + 'run_cli', + 'print_colored', + 'ProgressTracker', + 'get_free_port', + 'configure_sdk', + 'authenticate_with_sso', + 'fetch_api_keys', + 'exchange_sso_token_for_api_token', + 'verify_api_token' +] + +# Función de conveniencia para ejecutar CLI +def run_cli(argv: Optional[List[str]] = None) -> int: + """ + Run the CLI with the provided arguments. + + Args: + argv: List of arguments (use sys.argv if None) + + Returns: + Exit code + """ + if argv is None: + argv = sys.argv[1:] + + return main_cli(argv) \ No newline at end of file diff --git a/corebrain/corebrain/cli/__main__.py b/corebrain/corebrain/cli/__main__.py new file mode 100644 index 0000000..db91155 --- /dev/null +++ b/corebrain/corebrain/cli/__main__.py @@ -0,0 +1,12 @@ +""" +Entry point to run the CLI as a module. +""" +import sys +from corebrain.cli.commands import main_cli + +def main(): + """Main function for the entry point in pyproject.toml.""" + return main_cli() + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/corebrain/corebrain/cli/auth/__init__.py b/corebrain/corebrain/cli/auth/__init__.py new file mode 100644 index 0000000..873572d --- /dev/null +++ b/corebrain/corebrain/cli/auth/__init__.py @@ -0,0 +1,22 @@ +""" +Authentication modules for the Corebrain CLI. + +This package provides functionality for authentication, +token management, and API keys in the Corebrain CLI. +""" +from corebrain.cli.auth.sso import authenticate_with_sso, TokenHandler +from corebrain.cli.auth.api_keys import ( + fetch_api_keys, + exchange_sso_token_for_api_token, + verify_api_token, + get_api_key_id_from_token +) +# Exportación explícita de componentes públicos +__all__ = [ + 'authenticate_with_sso', + 'TokenHandler', + 'fetch_api_keys', + 'exchange_sso_token_for_api_token', + 'verify_api_token', + 'get_api_key_id_from_token' +] \ No newline at end of file diff --git a/corebrain/corebrain/cli/auth/api_keys.py b/corebrain/corebrain/cli/auth/api_keys.py new file mode 100644 index 0000000..5be72f0 --- /dev/null +++ b/corebrain/corebrain/cli/auth/api_keys.py @@ -0,0 +1,299 @@ +""" +API Keys Management for the CLI. +""" +import uuid +import httpx + +from typing import Optional, Dict, Any, Tuple + +from corebrain.cli.utils import print_colored +from corebrain.network.client import http_session +from corebrain.core.client import Corebrain + +def verify_api_token(token: str, api_url: Optional[str] = None, user_data: Optional[Dict[str, Any]] = None) -> Tuple[bool, Optional[Dict[str, Any]]]: + """ + Verifies if an API token is valid. + + Args: + token (str): API token to verify. + api_url (str, optional): API URL. Defaults to None. + user_data (dict, optional): User data. Defaults to None. + + Returns: + tuple: (validity (bool), user information (dict)) if valid, else (False, None). + """ + try: + # Create a temporary SDK instance to verify the token + config = {"type": "test", "config_id": str(uuid.uuid4())} + kwargs = {"api_token": token, "db_config": config} + + if user_data: + kwargs["user_data"] = user_data + + if api_url: + kwargs["api_url"] = api_url + + sdk = Corebrain(**kwargs) + return True, sdk.user_info + except Exception as e: + print_colored(f"Error verifying API token: {str(e)}", "red") + return False, None + +def fetch_api_keys(api_url: str, api_token: str, user_data: Dict[str, Any]) -> Optional[str]: + """ + Retrieves the available API keys for the user and allows selecting one. + + Args: + api_url: Base URL of the Corebrain API + api_token: API token (exchanged from SSO token) + user_data: User data + + Returns: + Selected API key or None if none is selected + """ + if not user_data or 'id' not in user_data: + print_colored("Could not identify the user to retrieve their API keys.", "yellow") + return None + + try: + # Ensure protocol in URL + if not api_url.startswith(("http://", "https://")): + api_url = "https://" + api_url + + # Remove trailing slash if it exists + if api_url.endswith('/'): + api_url = api_url[:-1] + + # Build endpoint to get API keys + endpoint = f"{api_url}/api/auth/api-keys" + + print_colored(f"Requesting user's API keys...", "blue") + + # Configure client with timeout and error handling + headers = { + "Authorization": f"Bearer {api_token}", + "Content-Type": "application/json" + } + + response = http_session.get(endpoint, headers=headers) + + # Verify response + if response.status_code == 200: + try: + api_keys_data = response.json() + # Verify response format + if not isinstance(api_keys_data, (list, dict)): + print_colored(f"Unexpected response format: {type(api_keys_data)}", "yellow") + return None + + # Handle both direct list and dictionary with list + api_keys = api_keys_data if isinstance(api_keys_data, list) else api_keys_data.get("data", []) + + if not api_keys: + print_colored("No API keys available for this user.", "yellow") + return None + + print_colored(f"\nFound {len(api_keys)} API keys", "green") + print_colored("\n=== Available API Keys ===", "blue") + + # Show available API keys + for i, key_info in enumerate(api_keys, 1): + key_id = key_info.get('id', 'No ID') + key_value = key_info.get('key', 'No value') + key_name = key_info.get('name', 'No name') + key_active = key_info.get('active') + + # Show status with color + status_color = "green" if key_active == True else "red" + status_text = "Active" if key_active == True else "Inactive" + + print(f"{i}. {key_name} - {print_colored(status_text, status_color, return_str=True)} (Value: {key_value})") + + # Ask user to select an API key + while True: + try: + choice = input(f"\nSelect an API key (1-{len(api_keys)}) or press Enter to cancel: ").strip() + + # Allow canceling and using API token + if not choice: + print_colored("No API key selected.", "yellow") + return None + + choice_num = int(choice) + if 1 <= choice_num <= len(api_keys): + selected_key = api_keys[choice_num - 1] + + # Verify if the key is active + if selected_key.get('active') != True: + print_colored("⚠️ The selected API key is not active. Select another one.", "yellow") + continue + + # Get information of the selected key + key_name = selected_key.get('name', 'Unknown') + key_value = selected_key.get('key', None) + + if not key_value: + print_colored("⚠️ The selected API key does not have a valid value.", "yellow") + continue + + print_colored(f"✅ You selected: {key_name}", "green") + print_colored("Wait while we assign the API key to your SDK...", "yellow") + + return key_value + else: + print_colored("Invalid option. Try again.", "red") + except ValueError: + print_colored("Please enter a valid number.", "red") + except Exception as e: + print_colored(f"Error processing JSON response: {str(e)}", "red") + return None + else: + # Handle error by status code + error_message = f"Error retrieving API keys: {response.status_code}" + + try: + error_data = response.json() + if "message" in error_data: + error_message += f" - {error_data['message']}" + elif "detail" in error_data: + error_message += f" - {error_data['detail']}" + except: + # If we can't parse JSON, use the full text + error_message += f" - {response.text[:100]}..." + + print_colored(error_message, "red") + + # Try to identify common problems + if response.status_code == 401: + print_colored("The authentication token has expired or is invalid.", "yellow") + elif response.status_code == 403: + print_colored("You don't have permissions to access the API keys.", "yellow") + elif response.status_code == 404: + print_colored("The API keys endpoint doesn't exist. Verify the API URL.", "yellow") + elif response.status_code >= 500: + print_colored("Server error. Try again later.", "yellow") + + return None + + except httpx.RequestError as e: + print_colored(f"Connection error: {str(e)}", "red") + print_colored("Verify the API URL and your internet connection.", "yellow") + return None + except Exception as e: + print_colored(f"Unexpected error retrieving API keys: {str(e)}", "red") + return None + +def get_api_key_id_from_token(sso_token: str, api_token: str, api_url: str) -> Optional[str]: + """ + Gets the ID of an API key from its token. + + Args: + sso_token: SSO token + api_token: API token + api_url: API URL + + Returns: + API key ID or None if it cannot be obtained + """ + try: + # Endpoint to get information of the current user + endpoint = f"{api_url}/api/auth/api-keys/{api_token}" + + headers = { + "Authorization": f"Bearer {api_token}", + "Content-Type": "application/json" + } + + response = httpx.get( + endpoint, + headers=headers + ) + + print("API keys response: ", response.json()) + + if response.status_code == 200: + key_data = response.json() + key_id = key_data.get("id") + return key_id + else: + print_colored("⚠️ Could not find the API key ID", "yellow") + return None + + except Exception as e: + print_colored(f"Error getting API key ID: {str(e)}", "red") + return None + +def exchange_sso_token_for_api_token(api_url: str, sso_token: str, user_data: Dict[str, Any]) -> Optional[str]: + """ + Exchanges a Globodain SSO token for a Corebrain API token. + + Args: + api_url: Base URL of the Corebrain API + sso_token: Globodain SSO token + user_data: User data + + Returns: + API token or None if it fails + """ + try: + # Ensure protocol in URL + if not api_url.startswith(("http://", "https://")): + api_url = "https://" + api_url + + # Remove trailing slash if it exists + if api_url.endswith('/'): + api_url = api_url[:-1] + + # Endpoint to exchange token + endpoint = f"{api_url}/api/auth/sso/token" + + print_colored(f"Exchanging SSO token for API token...", "blue") + + # Configure client with timeout and error handling + headers = { + 'Authorization': f'Bearer {sso_token}', + 'Content-Type': 'application/json' + } + body = { + "user_data": user_data + } + + response = http_session.post(endpoint, json=body, headers=headers) + + if response.status_code == 200: + try: + token_data = response.json() + api_token = token_data.get("access_token") + + if not api_token: + print_colored("The response does not contain a valid API token", "red") + return None + + print_colored("✅ API token successfully obtained", "green") + return api_token + except Exception as e: + print_colored(f"Error processing JSON response: {str(e)}", "red") + return None + else: + # Handle error by status code + error_message = f"Error exchanging token: {response.status_code}" + + try: + error_data = response.json() + if "message" in error_data: + error_message += f" - {error_data['message']}" + elif "detail" in error_data: + error_message += f" - {error_data['detail']}" + except: + # If we can't parse JSON, use the full text + error_message += f" - {response.text[:100]}..." + + print_colored(error_message, "red") + return None + + except httpx.RequestError as e: + print_colored(f"Connection error: {str(e)}", "red") + return None + except Exception as e: + print_colored(f"Unexpected error exchanging token: {str(e)}", "red") + return None \ No newline at end of file diff --git a/corebrain/corebrain/cli/auth/sso.py b/corebrain/corebrain/cli/auth/sso.py new file mode 100644 index 0000000..cee535f --- /dev/null +++ b/corebrain/corebrain/cli/auth/sso.py @@ -0,0 +1,452 @@ +""" +SSO Authentication for the CLI. +""" +import os +import webbrowser +import http.server +import socketserver +import threading +import urllib.parse +import time + +from typing import Tuple, Dict, Any, Optional + +from corebrain.cli.common import DEFAULT_API_URL, DEFAULT_SSO_URL, DEFAULT_PORT, SSO_CLIENT_ID, SSO_CLIENT_SECRET +from corebrain.cli.utils import print_colored +from corebrain.lib.sso.auth import GlobodainSSOAuth + +class TokenHandler(http.server.SimpleHTTPRequestHandler): + """ + Handler for the local HTTP server that processes the SSO authentication callback. + """ + def __init__(self, *args, **kwargs): + self.sso_auth = kwargs.pop('sso_auth', None) + self.result = kwargs.pop('result', {}) + self.session_data = kwargs.pop('session_data', {}) + self.auth_completed = kwargs.pop('auth_completed', None) + super().__init__(*args, **kwargs) + + def do_GET(self): + # Parse the URL to get the parameters + parsed_path = urllib.parse.urlparse(self.path) + + # Check if it's the callback path + if parsed_path.path == "/auth/sso/callback": + query = urllib.parse.parse_qs(parsed_path.query) + + if "code" in query: + code = query["code"][0] + + try: + # Exchange code for token using the sso_auth object + token_data = self.sso_auth.exchange_code_for_token(code) + + if not token_data: + raise ValueError("Could not obtain the token") + + # Save token in the result and session + access_token = token_data.get('access_token') + if not access_token: + raise ValueError("The received token does not contain an access_token") + + # Updated: save as sso_token for clarity + self.result["sso_token"] = access_token + self.session_data['sso_token'] = token_data + + # Get user information + user_info = self.sso_auth.get_user_info(access_token) + if user_info: + self.session_data['user'] = user_info + # Extract email to identify the user + if 'email' in user_info: + self.session_data['email'] = user_info['email'] + + # Signal that authentication has completed + self.auth_completed.set() + + # Send a success response to the browser + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + success_html = """ + + + Corebrain - Authentication Completed + + + +
+

Authentication Completed

+

You have successfully logged in to Corebrain CLI.

+

You can close this window and return to the terminal.

+
+ + + """ + self.wfile.write(success_html.encode()) + except Exception as e: + # If there's an error, show error message + self.send_response(400) + self.send_header("Content-type", "text/html") + self.end_headers() + error_html = f""" + + + Corebrain - Authentication Error + + + +
+

Authentication Error

+

Error: {str(e)}

+

Please close this window and try again.

+
+ + + """ + self.wfile.write(error_html.encode()) + else: + # If there's no code, it's an error + self.send_response(400) + self.send_header("Content-type", "text/html") + self.end_headers() + error_html = """ + + + Corebrain - Authentication Error + + + +
+

Authentication Error

+

Could not complete the authentication process.

+

Please close this window and try again.

+
+ + + """ + self.wfile.write(error_html.encode()) + else: + # For any other path, show a 404 error + self.send_response(404) + self.end_headers() + self.wfile.write(b"Not Found") + + def log_message(self, format, *args): + # Silence server logs + return + +def authenticate_with_sso(sso_url: str) -> Tuple[Optional[str], Optional[Dict[str, Any]], Optional[str]]: + """ + Initiates an SSO authentication flow through the browser and uses the callback system. + + Args: + sso_url: Base URL of the SSO service + + Returns: + Tuple with (api_key, user_data, api_token) or (None, None, None) if it fails + - api_key: Selected API key to use with the SDK + - user_data: Authenticated user data + - api_token: API token obtained from SSO for general authentication + """ + + # Token to store the result + result = {"sso_token": None} # Renamed for clarity + auth_completed = threading.Event() + session_data = {} + + # Find an available port + #port = get_free_port(DEFAULT_PORT) + + # SSO client configuration + auth_config = { + 'GLOBODAIN_SSO_URL': sso_url or DEFAULT_SSO_URL, + 'GLOBODAIN_CLIENT_ID': SSO_CLIENT_ID, + 'GLOBODAIN_CLIENT_SECRET': SSO_CLIENT_SECRET, + 'GLOBODAIN_REDIRECT_URI': f"http://localhost:{DEFAULT_PORT}/auth/sso/callback", + 'GLOBODAIN_SUCCESS_REDIRECT': 'https://sso.globodain.com/cli/success' + } + + sso_auth = GlobodainSSOAuth(config=auth_config) + + # Factory to create TokenHandler instances with the desired parameters + def handler_factory(*args, **kwargs): + return TokenHandler( + *args, + sso_auth=sso_auth, + result=result, + session_data=session_data, + auth_completed=auth_completed, + **kwargs + ) + + # Start server in the background + server = socketserver.TCPServer(("", DEFAULT_PORT), handler_factory) + server_thread = threading.Thread(target=server.serve_forever) + server_thread.daemon = True + server_thread.start() + + try: + # Build complete URL with protocol if missing + if sso_url and not sso_url.startswith(("http://", "https://")): + sso_url = "https://" + sso_url + + # URL to start the SSO flow + login_url = sso_auth.get_login_url() + auth_url = login_url + + print_colored(f"Opening browser for SSO authentication...", "blue") + print_colored(f"If the browser doesn't open automatically, visit:", "blue") + print_colored(f"{auth_url}", "bold") + + # Try to open the browser + if not webbrowser.open(auth_url): + print_colored("Could not open the browser automatically.", "yellow") + print_colored(f"Please copy and paste the following URL into your browser:", "yellow") + print_colored(f"{auth_url}", "bold") + + # Tell the user to wait + print_colored("\nWaiting for you to complete authentication in the browser...", "blue") + + # Wait for authentication to complete (with timeout) + timeout_seconds = 60 + start_time = time.time() + + # We use a loop with better feedback + while not auth_completed.is_set() and (time.time() - start_time < timeout_seconds): + elapsed = int(time.time() - start_time) + if elapsed % 5 == 0: # Every 5 seconds we show a message + remaining = timeout_seconds - elapsed + #print_colored(f"Waiting for authentication... ({remaining}s remaining)", "yellow") + + # Check every 0.5 seconds for better reactivity + auth_completed.wait(0.5) + + # Verify if authentication was completed + if auth_completed.is_set(): + print_colored("✅ SSO authentication completed successfully!", "green") + return result["sso_token"], session_data['user'] + else: + print_colored(f"❌ Could not complete SSO authentication in {timeout_seconds} seconds.", "red") + print_colored("You can try again or use a token manually.", "yellow") + return None, None, None + except Exception as e: + print_colored(f"❌ Error during SSO authentication: {str(e)}", "red") + return None, None, None + finally: + # Stop the server + try: + server.shutdown() + server.server_close() + except: + # If there's any error closing the server, we ignore it + pass + +def authenticate_with_sso_and_api_key_request(sso_url: str) -> Tuple[Optional[str], Optional[Dict[str, Any]], Optional[str]]: + """ + Initiates an SSO authentication flow through the browser and uses the callback system. + + Args: + sso_url: Base URL of the SSO service + + Returns: + Tuple with (api_key, user_data, api_token) or (None, None, None) if it fails + - api_key: Selected API key to use with the SDK + - user_data: Authenticated user data + - api_token: API token obtained from SSO for general authentication + """ + # Import inside the function to avoid circular dependencies + from corebrain.cli.auth.api_keys import fetch_api_keys, exchange_sso_token_for_api_token + + # Token to store the result + result = {"sso_token": None} # Renamed for clarity + auth_completed = threading.Event() + session_data = {} + + # Find an available port + #port = get_free_port(DEFAULT_PORT) + + # SSO client configuration + auth_config = { + 'GLOBODAIN_SSO_URL': sso_url or DEFAULT_SSO_URL, + 'GLOBODAIN_CLIENT_ID': SSO_CLIENT_ID, + 'GLOBODAIN_CLIENT_SECRET': SSO_CLIENT_SECRET, + 'GLOBODAIN_REDIRECT_URI': f"http://localhost:{DEFAULT_PORT}/auth/sso/callback", + 'GLOBODAIN_SUCCESS_REDIRECT': 'https://sso.globodain.com/cli/success' + } + + sso_auth = GlobodainSSOAuth(config=auth_config) + + # Factory to create TokenHandler instances with the desired parameters + def handler_factory(*args, **kwargs): + return TokenHandler( + *args, + sso_auth=sso_auth, + result=result, + session_data=session_data, + auth_completed=auth_completed, + **kwargs + ) + + # Start server in the background + server = socketserver.TCPServer(("", DEFAULT_PORT), handler_factory) + server_thread = threading.Thread(target=server.serve_forever) + server_thread.daemon = True + server_thread.start() + + try: + # Build complete URL with protocol if missing + if sso_url and not sso_url.startswith(("http://", "https://")): + sso_url = "https://" + sso_url + + # URL to start the SSO flow + login_url = sso_auth.get_login_url() + auth_url = login_url + + print_colored(f"Opening browser for SSO authentication...", "blue") + print_colored(f"If the browser doesn't open automatically, visit:", "blue") + print_colored(f"{auth_url}", "bold") + + # Try to open the browser + if not webbrowser.open(auth_url): + print_colored("Could not open the browser automatically.", "yellow") + print_colored(f"Please copy and paste the following URL into your browser:", "yellow") + print_colored(f"{auth_url}", "bold") + + # Tell the user to wait + print_colored("\nWaiting for you to complete authentication in the browser...", "blue") + + # Wait for authentication to complete (with timeout) + timeout_seconds = 60 + start_time = time.time() + + # We use a loop with better feedback + while not auth_completed.is_set() and (time.time() - start_time < timeout_seconds): + elapsed = int(time.time() - start_time) + if elapsed % 5 == 0: # Every 5 seconds we show a message + remaining = timeout_seconds - elapsed + #print_colored(f"Waiting for authentication... ({remaining}s remaining)", "yellow") + + # Check every 0.5 seconds for better reactivity + auth_completed.wait(0.5) + + # Verify if authentication was completed + if auth_completed.is_set(): + user_data = None + if 'user' in session_data: + user_data = session_data['user'] + + print_colored("✅ SSO authentication completed successfully!", "green") + + # Get and select an API key + api_url = os.environ.get("COREBRAIN_API_URL", DEFAULT_API_URL) + + # Now we use the SSO token to get an API token and then the API keys + # First we verify that we have a token + if result["sso_token"]: + api_token = exchange_sso_token_for_api_token(api_url, result["sso_token"], user_data) + + if not api_token: + print_colored("⚠️ Could not obtain an API Token with the SSO Token", "yellow") + return None, None, None + + # Now that we have the API Token, we get the available API Keys + api_key_selected = fetch_api_keys(api_url, api_token, user_data) + + if api_key_selected: + # We return the selected api_key + return api_key_selected, user_data, api_token + else: + print_colored("⚠️ Could not obtain an API Key. Create a new one using the command", "yellow") + return None, user_data, api_token + else: + print_colored("❌ No valid token was obtained during authentication.", "red") + return None, None, None + + # We don't have a token or user data + print_colored("❌ Authentication did not produce a valid token.", "red") + return None, None, None + else: + print_colored(f"❌ Could not complete SSO authentication in {timeout_seconds} seconds.", "red") + print_colored("You can try again or use a token manually.", "yellow") + return None, None, None + except Exception as e: + print_colored(f"❌ Error during SSO authentication: {str(e)}", "red") + return None, None, None + finally: + # Stop the server + try: + server.shutdown() + server.server_close() + except: + # If there's any error closing the server, we ignore it + pass \ No newline at end of file diff --git a/corebrain/corebrain/cli/commands.py b/corebrain/corebrain/cli/commands.py new file mode 100644 index 0000000..96eac44 --- /dev/null +++ b/corebrain/corebrain/cli/commands.py @@ -0,0 +1,453 @@ +""" +Main commands for the Corebrain CLI. +""" +import argparse +import os +import sys +import webbrowser +import requests +import random +import string + +from typing import Optional, List + +from corebrain.cli.common import DEFAULT_API_URL, DEFAULT_SSO_URL, DEFAULT_PORT, SSO_CLIENT_ID, SSO_CLIENT_SECRET +from corebrain.cli.auth.sso import authenticate_with_sso, authenticate_with_sso_and_api_key_request +from corebrain.cli.config import configure_sdk, get_api_credential +from corebrain.cli.utils import print_colored +from corebrain.config.manager import ConfigManager +from corebrain.config.manager import export_config +from corebrain.config.manager import validate_config +from corebrain.lib.sso.auth import GlobodainSSOAuth + +def main_cli(argv: Optional[List[str]] = None) -> int: + """ + Main entry point for the Corebrain CLI. + + Args: + argv: List of command line arguments (defaults to sys.argv[1:]) + + Returns: + Exit code (0 for success, other value for error) + """ + + # Package version + __version__ = "0.1.0" + + try: + print_colored("Corebrain CLI started. Version ", __version__, "blue") + + if argv is None: + argv = sys.argv[1:] + + # Argument parser configuration + parser = argparse.ArgumentParser(description="Corebrain SDK CLI") + parser.add_argument("--version", action="store_true", help="Show SDK version") + parser.add_argument("--authentication", action="store_true", help="Authenticate with SSO") + parser.add_argument("--create-user", action="store_true", help="Create an user and API Key by default") + parser.add_argument("--configure", action="store_true", help="Configure the Corebrain SDK") + parser.add_argument("--list-configs", action="store_true", help="List available configurations") + parser.add_argument("--remove-config", action="store_true", help="Remove a configuration") + parser.add_argument("--show-schema", action="store_true", help="Show the schema of the configured database") + parser.add_argument("--extract-schema", action="store_true", help="Extract the database schema and save it to a file") + parser.add_argument("--output-file", help="File to save the extracted schema") + parser.add_argument("--config-id", help="Specific configuration ID to use") + parser.add_argument("--token", help="Corebrain API token (any type)") + parser.add_argument("--api-key", help="Specific API Key for Corebrain") + parser.add_argument("--api-url", help="Corebrain API URL") + parser.add_argument("--sso-url", help="Globodain SSO service URL") + parser.add_argument("--login", action="store_true", help="Login via SSO") + parser.add_argument("--test-auth", action="store_true", help="Test SSO authentication system") + parser.add_argument("--woami",action="store_true",help="Display information about the current user") + parser.add_argument("--check-status",action="store_true",help="Checks status of task") + parser.add_argument("--task-id",help="ID of the task to check status for") + parser.add_argument("--validate-config",action="store_true",help="Validates the selected configuration without executing any operations") + parser.add_argument("--test-connection",action="store_true",help="Tests the connection to the Corebrain API using the provide credentials") + parser.add_argument("--export-config",action="store_true",help="Exports the current configuration to a file") + parser.add_argument("--gui", action="store_true", help="Check setup and launch the web interface") + + + args = parser.parse_args(argv) + + def authentication(): + sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + sso_token, sso_user = authenticate_with_sso(sso_url) + if sso_token: + try: + print_colored("✅ Returning SSO Token.", "green") + print_colored(f"{sso_user}", "blue") + print_colored("✅ Returning User data.", "green") + print_colored(f"{sso_user}", "blue") + return sso_token, sso_user + + except Exception as e: + print_colored("❌ Could not return SSO Token or SSO User data.", "red") + return sso_token, sso_user + + else: + print_colored("❌ Could not authenticate with SSO.", "red") + return None, None + + # Made by Lukasz + if args.export_config: + export_config(args.export_config) + # --> config/manager.py --> export_config + + if args.validate_config: + if not args.config_id: + print_colored("Error: --config-id is required for validation", "red") + return 1 + return validate_config(args.config_id) + + + # Show version + if args.version: + try: + from importlib.metadata import version + sdk_version = version("corebrain") + print(f"Corebrain SDK version {sdk_version}") + except Exception: + print(f"Corebrain SDK version {__version__}") + return 0 + + # Create an user and API Key by default + if args.authentication: + authentication() + + if args.create_user: + sso_token, sso_user = authentication() # Authentica use with SSO + + if sso_token and sso_user: + print_colored("✅ Enter to create an user and API Key.", "green") + + # Get API URL from environment or use default + api_url = os.environ.get("COREBRAIN_API_URL", DEFAULT_API_URL) + + """ + Create user data with SSO information. + If the user wants to use a different password than their SSO account, + they can specify it here. + """ + # Ask if user wants to use SSO password or create a new one + use_sso_password = input("Do you want to use your SSO password? (y/n): ").lower().strip() == 'y' + + if use_sso_password: + random_password = ''.join(random.choices(string.ascii_letters + string.digits, k=12)) + password = sso_user.get("password", random_password) + else: + while True: + password = input("Enter new password: ").strip() + if len(password) >= 8: + break + print_colored("Password must be at least 8 characters long", "yellow") + + user_data = { + "email": sso_user["email"], + "name": f"{sso_user['first_name']} {sso_user['last_name']}", + "password": password + } + + try: + # Make the API request + response = requests.post( + f"{api_url}/api/auth/users", + json=user_data, + headers={ + "Authorization": f"Bearer {sso_token}", + "Content-Type": "application/json" + } + ) + + # Check if the request was successful + print("response API: ", response) + if response.status_code == 200: + print_colored("✅ User and API Key created successfully!", "green") + return 0 + else: + print_colored(f"❌ Error creating user: {response.text}", "red") + return 1 + + except requests.exceptions.RequestException as e: + print_colored(f"❌ Error connecting to API: {str(e)}", "red") + return 1 + + else: + print_colored("❌ Could not create the user or the API KEY.", "red") + return 1 + + # Test SSO authentication + if args.test_auth: + sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + + print_colored("Testing SSO authentication...", "blue") + + # Authentication configuration + auth_config = { + 'GLOBODAIN_SSO_URL': sso_url, + 'GLOBODAIN_CLIENT_ID': SSO_CLIENT_ID, + 'GLOBODAIN_CLIENT_SECRET': SSO_CLIENT_SECRET, + 'GLOBODAIN_REDIRECT_URI': f"http://localhost:{DEFAULT_PORT}/auth/sso/callback", + 'GLOBODAIN_SUCCESS_REDIRECT': f"http://localhost:{DEFAULT_PORT}/auth/sso/callback" + } + + try: + # Instantiate authentication client + sso_auth = GlobodainSSOAuth(config=auth_config) + + # Get login URL + login_url = sso_auth.get_login_url() + + print_colored(f"Login URL: {login_url}", "blue") + print_colored("Opening browser for login...", "blue") + + # Open browser + webbrowser.open(login_url) + + print_colored("Please complete the login process in the browser.", "blue") + input("\nPress Enter when you've completed the process or to cancel...") + + print_colored("✅ SSO authentication test completed!", "green") + return 0 + except Exception as e: + print_colored(f"❌ Error during test: {str(e)}", "red") + return 1 + + # Login via SSO + if args.login: + sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + api_key, user_data, api_token = authenticate_with_sso_and_api_key_request(sso_url) + + if api_token: + # Save the general token for future use + os.environ["COREBRAIN_API_TOKEN"] = api_token + + if api_key: + # Save the specific API key for future use + os.environ["COREBRAIN_API_KEY"] = api_key + print_colored("✅ API Key successfully saved. You can use the SDK now.", "green") + + # If configuration was also requested, continue with the process + if args.configure: + api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + configure_sdk(api_token, api_key, api_url, sso_url, user_data) + + return 0 + else: + print_colored("❌ Could not obtain an API Key via SSO.", "red") + if api_token: + print_colored("A general API token was obtained, but not a specific API Key.", "yellow") + print_colored("You can create an API Key in the Corebrain dashboard.", "yellow") + return 1 + + if args.check_status: + if not args.task_id: + print_colored("❌ Please provide a task ID using --task-id", "red") + return 1 + + # Get URLs + api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + + # Prioritize api_key if explicitly provided + token_arg = args.api_key if args.api_key else args.token + + # Get API credentials + api_key, user_data, api_token = get_api_credential(token_arg, sso_url) + + if not api_key: + print_colored("❌ API Key is required to check task status. Use --api-key or login via --login", "red") + return 1 + + try: + task_id = args.task_id + headers = { + "Authorization": f"Bearer {api_key}", + "Accept": "application/json" + } + url = f"{api_url}/tasks/{task_id}/status" + response = requests.get(url, headers=headers) + + if response.status_code == 404: + print_colored(f"❌ Task with ID '{task_id}' not found.", "red") + return 1 + + response.raise_for_status() + data = response.json() + status = data.get("status", "unknown") + + print_colored(f"✅ Task '{task_id}' status: {status}", "green") + return 0 + except Exception as e: + print_colored(f"❌ Failed to check status: {str(e)}", "red") + return 1 + + if args.woami: + try: + #downloading user data + sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + token_arg = args.api_key if args.api_key else args.token + + #using saved data about user + api_key, user_data, api_token = get_api_credential(token_arg, sso_url) + #printing user data + if user_data: + print_colored("User Data:", "blue") + for k, v in user_data.items(): + print(f"{k}: {v}") + else: + print_colored("❌ Can't find data about user, be sure that you are logged into --login.", "red") + return 1 + + return 0 + except Exception as e: + print_colored(f"❌ Error when downloading data about user {str(e)}", "red") + return 1 + + # Operations that require credentials: configure, list, remove or show schema + if args.configure or args.list_configs or args.remove_config or args.show_schema or args.extract_schema: + # Get URLs + api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + + # Prioritize api_key if explicitly provided + token_arg = args.api_key if args.api_key else args.token + + # Get API credentials + api_key, user_data, api_token = get_api_credential(token_arg, sso_url) + + if not api_key: + print_colored("Error: An API Key is required. You can generate one at dashboard.corebrain.com", "red") + print_colored("Or use the 'corebrain --login' command to login via SSO.", "blue") + return 1 + + from corebrain.db.schema_file import show_db_schema, extract_schema_to_file + + # Execute the selected operation + if args.configure: + configure_sdk(api_token, api_key, api_url, sso_url, user_data) + elif args.list_configs: + ConfigManager.list_configs(api_key, api_url) + elif args.remove_config: + ConfigManager.remove_config(api_key, api_url) + elif args.show_schema: + show_db_schema(api_key, args.config_id, api_url) + elif args.extract_schema: + extract_schema_to_file(api_key, args.config_id, args.output_file, api_url) + + if args.test_connection: + # Test connection to the Corebrain API + api_url = args.api_url or os.environ.get("COREBRAIN_API_URL", DEFAULT_API_URL) + sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL", DEFAULT_SSO_URL) + + try: + # Retrieve API credentials + api_key, user_data, api_token = get_api_credential(args.token, sso_url) + except Exception as e: + print_colored(f"Error while retrieving API credentials: {e}", "red") + return 1 + + if not api_key: + print_colored( + "Error: An API key is required. You can generate one at dashboard.corebrain.com.", + "red" + ) + return 1 + + try: + # Test the connection + from corebrain.db.schema_file import test_connection + test_connection(api_key, api_url) + print_colored("Successfully connected to Corebrain API.", "green") + except Exception as e: + print_colored(f"Failed to connect to Corebrain API: {e}", "red") + return 1 + + + + + if args.gui: + import subprocess + from pathlib import Path + + def run_cmd(cmd, cwd=None): + print_colored(f"▶ {cmd}", "yellow") + subprocess.run(cmd, shell=True, cwd=cwd, check=True) + + print("Checking GUI setup...") + + commands_path = Path(__file__).resolve() + corebrain_root = commands_path.parents[1] + + cli_ui_path = corebrain_root / "CLI-UI" + client_path = cli_ui_path / "client" + server_path = cli_ui_path / "server" + api_path = corebrain_root / "wrappers" / "csharp_cli_api" + + # Path validation + if not client_path.exists(): + print_colored(f"Folder {client_path} does not exist!", "red") + sys.exit(1) + if not server_path.exists(): + print_colored(f"Folder {server_path} does not exist!", "red") + sys.exit(1) + if not api_path.exists(): + print_colored(f"Folder {api_path} does not exist!", "red") + sys.exit(1) + + # Setup client + if not (client_path / "node_modules").exists(): + print_colored("Installing frontend (React) dependencies...", "cyan") + run_cmd("npm install", cwd=client_path) + run_cmd("npm install history", cwd=client_path) + run_cmd("npm install --save-dev vite", cwd=client_path) + run_cmd("npm install concurrently --save-dev", cwd=client_path) + + # Setup server + if not (server_path / "node_modules").exists(): + print_colored("Installing backend (Express) dependencies...", "cyan") + run_cmd("npm install", cwd=server_path) + run_cmd("npm install --save-dev ts-node-dev", cwd=server_path) + + # Start GUI: CLI UI + Corebrain API + print("Starting GUI (CLI-UI + Corebrain API)...") + + def run_in_background_silent(cmd, cwd): + return subprocess.Popen( + cmd, + cwd=cwd, + shell=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + + run_in_background_silent("dotnet run", cwd=api_path) + run_in_background_silent( + 'npx concurrently "npm --prefix server run dev" "npm --prefix client run dev"', + cwd=cli_ui_path + ) + + url = "http://localhost:5173/" + print_colored(f"GUI: {url}", "cyan") + webbrowser.open(url) + + + + + + + + + + + + else: + # If no option was specified, show help + parser.print_help() + print_colored("\nTip: Use 'corebrain --login' to login via SSO.", "blue") + + return 0 + except Exception as e: + print_colored(f"Error: {str(e)}", "red") + import traceback + traceback.print_exc() + return 1 \ No newline at end of file diff --git a/corebrain/corebrain/cli/common.py b/corebrain/corebrain/cli/common.py new file mode 100644 index 0000000..7799dcf --- /dev/null +++ b/corebrain/corebrain/cli/common.py @@ -0,0 +1,15 @@ +""" +Default values for SSO and API connection +""" + +DEFAULT_API_URL = "http://localhost:5000" +#DEFAULT_SSO_URL = "http://localhost:3000" # localhost +DEFAULT_SSO_URL = "https://sso.globodain.com" # remote +DEFAULT_PORT = 8765 +DEFAULT_TIMEOUT = 10 +#SSO_CLIENT_ID = '401dca6e-3f3b-4458-b3ef-f87eaae0398d' # localhost +#SSO_CLIENT_SECRET = 'f9d315ea-5a65-4e3f-be35-b27a933dfb5b' # localhost +SSO_CLIENT_ID = '63d767e9-5a06-4890-a194-8608ae29d426' # remote +SSO_CLIENT_SECRET = '06cf39f6-ca93-466e-955e-cb6ea0a02d4d' # remote +SSO_REDIRECT_URI = 'http://localhost:8765/oauth/callback' +SSO_SERVICE_ID = 2 \ No newline at end of file diff --git a/corebrain/corebrain/cli/config.py b/corebrain/corebrain/cli/config.py new file mode 100644 index 0000000..cf8a175 --- /dev/null +++ b/corebrain/corebrain/cli/config.py @@ -0,0 +1,489 @@ +""" +Configuration functions for the CLI. +""" +import json +import uuid +import getpass +import os +from typing import Dict, Any, List, Optional, Tuple +from datetime import datetime + +from corebrain.cli.common import DEFAULT_API_URL, DEFAULT_SSO_URL +from corebrain.cli.auth.sso import authenticate_with_sso, authenticate_with_sso_and_api_key_request +from corebrain.cli.utils import print_colored, ProgressTracker +from corebrain.db.engines import get_available_engines +from corebrain.config.manager import ConfigManager +from corebrain.network.client import http_session +from corebrain.core.test_utils import test_natural_language_query +from corebrain.db.schema_file import extract_db_schema + +def get_api_credential(args_token: Optional[str] = None, sso_url: Optional[str] = None) -> Tuple[Optional[str], Optional[Dict[str, Any]], Optional[str]]: + """ + Obtains the API credential (API key), trying several methods in order: + 1. Token provided as argument + 2. Environment variable + 3. SSO authentication + 4. Manual user input + + Args: + args_token: Token provided as argument + sso_url: SSO service URL + + Returns: + Tuple with (api_key, user_data, api_token) or (None, None, None) if couldn't be obtained + - api_key: API key to use with SDK + - user_data: User data + - api_token: API token for general authentication + """ + # 1. Check if provided as argument + if args_token: + print_colored("Using token provided as argument.", "blue") + # Assume the provided token is directly an API key + return args_token, None, args_token + + # 2. Check environment variable for API key + env_api_key = os.environ.get("COREBRAIN_API_KEY") + if env_api_key: + print_colored("Using API key from COREBRAIN_API_KEY environment variable.", "blue") + return env_api_key, None, env_api_key + + # 3. Check environment variable for API token + env_api_token = os.environ.get("COREBRAIN_API_TOKEN") + if env_api_token: + print_colored("Using API token from COREBRAIN_API_TOKEN environment variable.", "blue") + # Note: Here we return the same value as api_key and api_token + # because we have no way to obtain a specific api_key + return env_api_token, None, env_api_token + + # 4. Try SSO authentication + print_colored("Attempting authentication via SSO...", "blue") + api_key, user_data, api_token = authenticate_with_sso_and_api_key_request(sso_url or DEFAULT_SSO_URL) + print("Exit from authenticate_with_sso: ", datetime.now()) + if api_key: + # Save for future use + os.environ["COREBRAIN_API_KEY"] = api_key + os.environ["COREBRAIN_API_TOKEN"] = api_token + return api_key, user_data, api_token + + # 5. Request manually + print_colored("\nCouldn't complete SSO authentication.", "yellow") + print_colored("You can directly enter an API key:", "blue") + manual_input = input("Enter your Corebrain API key: ").strip() + if manual_input: + # Assume manual input is an API key + return manual_input, None, manual_input + + # If we got here, we couldn't get a credential + return None, None, None + +def get_db_type() -> str: + """ + Prompts the user to select a database type. + + Returns: + Selected database type + """ + print_colored("\n=== Select the database type ===", "blue") + print("1. SQL (SQLite, MySQL, PostgreSQL)") + print("2. NoSQL (MongoDB)") + + while True: + try: + choice = int(input("\nSelect an option (1-2): ").strip()) + if choice == 1: + return "sql" + elif choice == 2: + return "nosql" + else: + print_colored("Invalid option. Try again.", "red") + except ValueError: + print_colored("Please enter a number.", "red") + +def get_db_engine(db_type: str) -> str: + """ + Prompts the user to select a database engine. + + Args: + db_type: Selected database type + + Returns: + Selected database engine + """ + engines = get_available_engines() + + if db_type == "sql": + available_engines = engines["sql"] + print_colored("\n=== Select the SQL engine ===", "blue") + for i, engine in enumerate(available_engines, 1): + print(f"{i}. {engine.capitalize()}") + + while True: + try: + choice = int(input(f"\nSelect an option (1-{len(available_engines)}): ").strip()) + if 1 <= choice <= len(available_engines): + return available_engines[choice - 1] + else: + print_colored("Invalid option. Try again.", "red") + except ValueError: + print_colored("Please enter a number.", "red") + else: + # For NoSQL, we only have MongoDB for now + return "mongodb" + +def get_connection_params(db_type: str, engine: str) -> Dict[str, Any]: + """ + Prompts for connection parameters according to the database type and engine. + + Args: + db_type: Database type + engine: Database engine + + Returns: + Dictionary with connection parameters + """ + params = {"type": db_type, "engine": engine} + + # Specific parameters by type and engine + if db_type == "sql": + if engine == "sqlite": + path = input("\nPath to SQLite database file: ").strip() + params["database"] = path + else: + # MySQL or PostgreSQL + print_colored("\n=== Connection Parameters ===", "blue") + params["host"] = input("Host (default: localhost): ").strip() or "localhost" + + if engine == "mysql": + params["port"] = int(input("Port (default: 3306): ").strip() or "3306") + else: # PostgreSQL + params["port"] = int(input("Port (default: 5432): ").strip() or "5432") + + params["user"] = input("User: ").strip() + params["password"] = getpass.getpass("Password: ") + params["database"] = input("Database name: ").strip() + else: + # MongoDB + print_colored("\n=== MongoDB Connection Parameters ===", "blue") + use_connection_string = input("Use connection string? (y/n): ").strip().lower() == "y" + + if use_connection_string: + params["connection_string"] = input("MongoDB connection string: ").strip() + else: + params["host"] = input("Host (default: localhost): ").strip() or "localhost" + params["port"] = int(input("Port (default: 27017): ").strip() or "27017") + + use_auth = input("Use authentication? (y/n): ").strip().lower() == "y" + if use_auth: + params["user"] = input("User: ").strip() + params["password"] = getpass.getpass("Password: ") + + params["database"] = input("Database name: ").strip() + + # Add configuration ID + params["config_id"] = str(uuid.uuid4()) + params["excluded_tables"] = [] + + return params + +def test_database_connection(api_token: str, db_config: Dict[str, Any], api_url: Optional[str] = None, user_data: Optional[Dict[str, Any]] = None) -> bool: + """ + Tests the database connection without verifying the API token. + + Args: + api_token: API token + db_config: Database configuration + api_url: Optional API URL + user_data: User data + + Returns: + True if connection is successful, False otherwise + """ + try: + print_colored("\nTesting database connection...", "blue") + + db_type = db_config["type"].lower() + engine = db_config.get("engine", "").lower() + + if db_type == "sql": + if engine == "sqlite": + import sqlite3 + conn = sqlite3.connect(db_config.get("database", "")) + cursor = conn.cursor() + cursor.execute("SELECT 1") + cursor.close() + conn.close() + + elif engine == "mysql": + import mysql.connector + if "connection_string" in db_config: + conn = mysql.connector.connect(connection_string=db_config["connection_string"]) + else: + conn = mysql.connector.connect( + host=db_config.get("host", "localhost"), + user=db_config.get("user", ""), + password=db_config.get("password", ""), + database=db_config.get("database", ""), + port=db_config.get("port", 3306) + ) + cursor = conn.cursor() + cursor.execute("SELECT 1") + cursor.close() + conn.close() + + elif engine == "postgresql": + import psycopg2 + if "connection_string" in db_config: + conn = psycopg2.connect(db_config["connection_string"]) + else: + conn = psycopg2.connect( + host=db_config.get("host", "localhost"), + user=db_config.get("user", ""), + password=db_config.get("password", ""), + dbname=db_config.get("database", ""), + port=db_config.get("port", 5432) + ) + cursor = conn.cursor() + cursor.execute("SELECT 1") + cursor.close() + conn.close() + + elif db_type == "nosql" and engine == "mongodb": + import pymongo + if "connection_string" in db_config: + client = pymongo.MongoClient(db_config["connection_string"]) + else: + client = pymongo.MongoClient( + host=db_config.get("host", "localhost"), + port=db_config.get("port", 27017), + username=db_config.get("user"), + password=db_config.get("password") + ) + + # Verify connection by trying to access the database + db = client[db_config["database"]] + # List collections to verify we can access + _ = db.list_collection_names() + client.close() + + # If we got here, the connection was successful + print_colored("✅ Database connection successful!", "green") + return True + except Exception as e: + print_colored(f"❌ Error connecting to the database: {str(e)}", "red") + return False + +def select_excluded_tables(api_token: str, db_config: Dict[str, Any], api_url: Optional[str] = None, user_data: Optional[Dict[str, Any]] = None) -> List[str]: + + """ + Allows the user to select tables/collections to exclude. + + Args: + api_token: API token + db_config: Database configuration + api_url: Optional API URL + user_data: User data + + Returns: + List of excluded tables/collections + """ + print_colored("\nRetrieving database schema...", "blue") + + # Get the database schema directly + schema = extract_db_schema(db_config) + + if not schema or not schema.get("tables"): + print_colored("No tables/collections found.", "yellow") + return [] + + print_colored("\n=== Tables/Collections found ===", "blue") + print("Mark with 'n' the tables that should NOT be accessible (y for accessible)") + + # Use the tables list instead of the dictionary + tables_list = schema.get("tables_list", []) + excluded_tables = [] + + if not tables_list: + # If there's no table list, convert the tables dictionary to a list + tables = schema.get("tables", {}) + for table_name in tables: + choice = input(f"{table_name} (accessible? y/n): ").strip().lower() + if choice == "n": + excluded_tables.append(table_name) + else: + # If there's a table list, use it directly + for i, table in enumerate(tables_list, 1): + table_name = table["name"] + choice = input(f"{i}. {table_name} (accessible? y/n): ").strip().lower() + if choice == "n": + excluded_tables.append(table_name) + + print_colored(f"\n{len(excluded_tables)} tables/collections have been excluded", "green") + return excluded_tables + +def save_configuration(sso_token: str, api_key: str, db_config: Dict[str, Any], api_url: Optional[str] = None) -> bool: + """ + Saves the configuration locally and syncs it with the API server. + + Args: + sso_token: SSO authentication token + api_key: API Key to identify the configuration + db_config: Database configuration + api_url: Optional API URL + + Returns: + True if saved correctly, False otherwise + """ + config_id = db_config.get("config_id") + if not config_id: + config_id = str(uuid.uuid4()) + db_config["config_id"] = config_id + + print_colored(f"\nSaving configuration with ID: {config_id}...", "blue") + + try: + config_manager = ConfigManager() + config_manager.add_config(api_key, db_config, config_id) + + # 2. Verify that the configuration was saved locally + saved_config = config_manager.get_config(api_key, config_id) + if not saved_config: + print_colored("⚠️ Could not verify local saving of configuration", "yellow") + else: + print_colored("✅ Configuration saved locally successfully", "green") + + # 3. Try to sync with the server + try: + if api_url: + print_colored("Syncing configuration with server...", "blue") + + # Prepare URL + if not api_url.startswith(("http://", "https://")): + api_url = "https://" + api_url + + if api_url.endswith('/'): + api_url = api_url[:-1] + + # Endpoint to update API key + endpoint = f"{api_url}/api/auth/api-keys/{api_key}" + + # Create ApiKeyUpdate object according to your model + update_data = { + "metadata": { + "config_id": config_id, + "db_config": db_config, + "corebrain_sdk": { + "version": "1.0.0", + "updated_at": datetime.now().isoformat() + } + } + } + + print_colored(f"Updating API key with ID: {api_key}", "blue") + + # Send to server + headers = { + "Authorization": f"Bearer {sso_token}", + "Content-Type": "application/json" + } + + response = http_session.put( + endpoint, + headers=headers, + json=update_data, + timeout=5.0 + ) + + if response.status_code in [200, 201, 204]: + print_colored("✅ Configuration successfully synced with server", "green") + else: + print_colored(f"⚠️ Error syncing with server (Code: {response.status_code})", "yellow") + print_colored(f"Response: {response.text[:200]}...", "yellow") + + except Exception as e: + print_colored(f"⚠️ Error syncing with server: {str(e)}", "yellow") + print_colored("The configuration is still saved locally", "green") + + return True + + except Exception as e: + print_colored(f"❌ Error saving configuration: {str(e)}", "red") + return False + +def configure_sdk(api_token: str, api_key: str, api_url: Optional[str] = None, sso_url: Optional[str] = None, user_data: Optional[Dict[str, Any]] = None) -> None: + """ + Configures the Corebrain SDK with a step-by-step wizard. + + Args: + api_token: API token for general authentication (obtained from SSO) + api_key: Specific API key selected to use with the SDK + api_url: Corebrain API URL + sso_url: Globodain SSO service URL + user_data: User data obtained from SSO + """ + # Ensure default values for URLs + api_url = api_url or DEFAULT_API_URL + sso_url = sso_url or DEFAULT_SSO_URL + + print_colored("\n=== COREBRAIN SDK CONFIGURATION WIZARD ===", "bold") + + # PHASE 1-3: Already completed - User authentication + + # PHASE 4: Select database type + print_colored("\n2. Selecting database type...", "blue") + db_type = get_db_type() + + # PHASE 4: Select database engine + print_colored("\n3. Selecting database engine...", "blue") + engine = get_db_engine(db_type) + + # PHASE 5: Configure connection parameters + print_colored("\n4. Configuring connection parameters...", "blue") + db_config = get_connection_params(db_type, engine) + + # PHASE 5: Verify database connection + print_colored("\n5. Verifying database connection...", "blue") + if not test_database_connection(api_key, db_config, api_url, user_data): + print_colored("❌ Configuration not completed due to connection errors.", "red") + return + + # PHASE 6: Define non-accessible tables/collections + print_colored("\n6. Defining non-accessible tables/collections...", "blue") + excluded_tables = select_excluded_tables(api_key, db_config, api_url, user_data) + db_config["excluded_tables"] = excluded_tables + + # PHASE 7: Save configuration + print_colored("\n7. Saving configuration...", "blue") + config_id = db_config["config_id"] + + # Save the configuration + if not save_configuration(api_token, api_key, db_config, api_url): + print_colored("❌ Error saving configuration.", "red") + return + + """ # * --> Deactivated + # PHASE 8: Test natural language query (optional depending on API status) + try: + print_colored("\n8. Testing natural language query...", "blue") + test_natural_language_query(api_key, db_config, api_url, user_data) + except Exception as e: + print_colored(f"⚠️ Could not perform the query test: {str(e)}", "yellow") + print_colored("This does not affect the saved configuration.", "yellow") + """ + + # Final message + print_colored("\n✅ Configuration completed successfully!", "green") + print_colored(f"\nYou can use this SDK in your code with:", "blue") + print(f""" + from corebrain import init + + # Initialize the SDK with API key and configuration ID + corebrain = init( + api_key="{api_key}", + config_id="{config_id}" + ) + + # Perform natural language queries + result = corebrain.ask("Your question in natural language") + print(result["explanation"]) + """ + ) \ No newline at end of file diff --git a/corebrain/corebrain/cli/utils.py b/corebrain/corebrain/cli/utils.py new file mode 100644 index 0000000..6c0ccac --- /dev/null +++ b/corebrain/corebrain/cli/utils.py @@ -0,0 +1,595 @@ +""" +Utilities for the Corebrain CLI. + +This module provides utility functions and classes for the +Corebrain command-line interface. +""" +import sys +import time +import socket +import random +import logging +import threading +import socketserver + +from typing import Optional, Dict, Any, List, Union +from pathlib import Path + +from corebrain.cli.common import DEFAULT_PORT, DEFAULT_TIMEOUT + +logger = logging.getLogger(__name__) + +# Terminal color definitions +COLORS = { + "default": "\033[0m", + "bold": "\033[1m", + "green": "\033[92m", + "red": "\033[91m", + "yellow": "\033[93m", + "blue": "\033[94m", + "magenta": "\033[95m", + "cyan": "\033[96m", + "white": "\033[97m", + "black": "\033[30m", + "bg_green": "\033[42m", + "bg_red": "\033[41m", + "bg_yellow": "\033[43m", + "bg_blue": "\033[44m", +} + +def print_colored(text: str, color: str = "default", return_str: bool = False) -> Optional[str]: + """ + Prints colored text in the terminal or returns the colored text. + + Args: + text: Text to color + color: Color to use (default, green, red, yellow, blue, bold, etc.) + return_str: If True, returns the colored text instead of printing it + + Returns: + If return_str is True, returns the colored text, otherwise None + """ + try: + # Get color code + start_color = COLORS.get(color, COLORS["default"]) + end_color = COLORS["default"] + + # Compose colored text + colored_text = f"{start_color}{text}{end_color}" + + # Return or print + if return_str: + return colored_text + else: + print(colored_text) + return None + except Exception as e: + # If there's an error with colors (e.g., terminal that doesn't support them) + logger.debug(f"Error using colors: {e}") + if return_str: + return text + else: + print(text) + return None + +def format_table(data: List[Dict[str, Any]], columns: Optional[List[str]] = None, + max_width: int = 80) -> str: + """ + Formats data as a text table for display in the terminal. + + Args: + data: List of dictionaries with the data + columns: List of columns to display (if None, uses all columns) + max_width: Maximum width of the table + + Returns: + Table formatted as text + """ + if not data: + return "No data to display" + + # Determine columns to display + if not columns: + # Use all columns from the first element + columns = list(data[0].keys()) + + # Get the maximum width for each column + widths = {col: len(col) for col in columns} + for row in data: + for col in columns: + if col in row: + val = str(row[col]) + widths[col] = max(widths[col], min(len(val), 30)) # Limit to 30 characters + + # Adjust widths if they exceed the maximum + total_width = sum(widths.values()) + (3 * len(columns)) - 1 + if total_width > max_width: + # Reduce proportionally + ratio = max_width / total_width + for col in widths: + widths[col] = max(8, int(widths[col] * ratio)) + + # Header + header = " | ".join(col.ljust(widths[col]) for col in columns) + separator = "-+-".join("-" * widths[col] for col in columns) + + # Rows + rows = [] + for row in data: + row_str = " | ".join( + str(row.get(col, "")).ljust(widths[col])[:widths[col]] + for col in columns + ) + rows.append(row_str) + + # Compose table + return "\n".join([header, separator] + rows) + +def get_free_port(start_port: int = DEFAULT_PORT) -> int: + """ + Finds an available port, starting with the suggested port. + + Args: + start_port: Initial port to try + + Returns: + Available port + """ + try: + # Try with the suggested port first + with socketserver.TCPServer(("", start_port), None) as _: + return start_port # The port is available + except OSError: + # If the suggested port is busy, look for a free one + for _ in range(10): # Try 10 times + # Choose a random port between 8000 and 9000 + port = random.randint(8000, 9000) + try: + with socketserver.TCPServer(("", port), None) as _: + return port # Port available + except OSError: + continue # Try with another port + + # If we can't find a free port, use a default high one + return 10000 + random.randint(0, 1000) + +def is_port_in_use(port: int) -> bool: + """ + Checks if a port is in use. + + Args: + port: Port number to check + + Returns: + True if the port is in use + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + return s.connect_ex(('localhost', port)) == 0 + +def is_interactive() -> bool: + """ + Determines if the current session is interactive. + + Returns: + True if the session is interactive + """ + return sys.stdin.isatty() and sys.stdout.isatty() + +def confirm_action(message: str, default: bool = False) -> bool: + """ + Asks the user for confirmation of an action. + + Args: + message: Confirmation message + default: Default value if the user just presses Enter + + Returns: + True if the user confirms, False otherwise + """ + if not is_interactive(): + return default + + default_text = "Y/n" if default else "y/N" + response = input(f"{message} ({default_text}): ").strip().lower() + + if not response: + return default + + return response.startswith('y') + +def get_input_with_default(prompt: str, default: Optional[str] = None) -> str: + """ + Requests input from the user with a default value. + + Args: + prompt: Request message + default: Default value + + Returns: + Value entered by the user or default value + """ + if default: + full_prompt = f"{prompt} (default: {default}): " + else: + full_prompt = f"{prompt}: " + + response = input(full_prompt).strip() + + return response if response else (default or "") + +def get_password_input(prompt: str = "Password") -> str: + """ + Requests a password from the user without displaying it on screen. + + Args: + prompt: Request message + + Returns: + Password entered + """ + import getpass + return getpass.getpass(f"{prompt}: ") + +def truncate_text(text: str, max_length: int = 100, suffix: str = "...") -> str: + """ + Truncates text if it exceeds the maximum length. + + Args: + text: Text to truncate + max_length: Maximum length + suffix: Suffix to add if the text is truncated + + Returns: + Truncated text if necessary + """ + if not text or len(text) <= max_length: + return text + + return text[:max_length - len(suffix)] + suffix + +def ensure_dir(path: Union[str, Path]) -> Path: + """ + Ensures that a directory exists, creating it if necessary. + + Args: + path: Directory path + + Returns: + Path object of the directory + """ + path_obj = Path(path) + path_obj.mkdir(parents=True, exist_ok=True) + return path_obj + +class ProgressTracker: + """ + Displays progress of CLI operations with colors and timing. + """ + + def __init__(self, verbose: bool = False, spinner: bool = True): + """ + Initializes the progress tracker. + + Args: + verbose: Whether to show detailed information + spinner: Whether to show an animated spinner + """ + self.verbose = verbose + self.use_spinner = spinner and is_interactive() + self.start_time = None + self.current_task = None + self.total = None + self.steps = 0 + self.spinner_thread = None + self.stop_spinner = threading.Event() + self.last_update_time = 0 + self.update_interval = 0.2 # Seconds between updates + + def _run_spinner(self): + """Displays an animated spinner in the console.""" + spinner_chars = "|/-\\" + idx = 0 + + while not self.stop_spinner.is_set(): + if self.current_task: + elapsed = time.time() - self.start_time + status = f"{self.steps}/{self.total}" if self.total else f"step {self.steps}" + sys.stdout.write(f"\r{COLORS['blue']}[{spinner_chars[idx]}] {self.current_task} ({status}, {elapsed:.1f}s){COLORS['default']} ") + sys.stdout.flush() + idx = (idx + 1) % len(spinner_chars) + time.sleep(0.1) + + def start(self, task: str, total: Optional[int] = None) -> None: + """ + Starts tracking a task. + + Args: + task: Task description + total: Total number of steps (optional) + """ + self.reset() # Ensure there's no previous task + + self.current_task = task + self.total = total + self.start_time = time.time() + self.steps = 0 + self.last_update_time = self.start_time + + # Show initial message + print_colored(f"▶ {task}...", "blue") + + # Start spinner if enabled + if self.use_spinner: + self.stop_spinner.clear() + self.spinner_thread = threading.Thread(target=self._run_spinner) + self.spinner_thread.daemon = True + self.spinner_thread.start() + + def update(self, message: Optional[str] = None, increment: int = 1) -> None: + """ + Updates progress with optional message. + + Args: + message: Progress message + increment: Step increment + """ + if not self.start_time: + return # No active task + + self.steps += increment + current_time = time.time() + + # Limit update frequency to avoid saturating the output + if (current_time - self.last_update_time < self.update_interval) and not message: + return + + self.last_update_time = current_time + + # If there's an active spinner, temporarily stop it to show the message + if self.use_spinner and self.spinner_thread and self.spinner_thread.is_alive(): + sys.stdout.write("\r" + " " * 80 + "\r") # Clear current line + sys.stdout.flush() + + if message or self.verbose: + elapsed = current_time - self.start_time + status = f"{self.steps}/{self.total}" if self.total else f"step {self.steps}" + + if message: + print_colored(f" • {message} ({status}, {elapsed:.1f}s)", "blue") + elif self.verbose: + print_colored(f" • Progress: {status}, {elapsed:.1f}s", "blue") + + def finish(self, message: Optional[str] = None) -> None: + """ + Finishes a task with success message. + + Args: + message: Final message + """ + if not self.start_time: + return # No active task + + # Stop spinner if active + self._stop_spinner() + + elapsed = time.time() - self.start_time + msg = message or f"{self.current_task} completed" + print_colored(f"✅ {msg} in {elapsed:.2f}s", "green") + + self.reset() + + def fail(self, message: Optional[str] = None) -> None: + """ + Marks a task as failed. + + Args: + message: Error message + """ + if not self.start_time: + return # No active task + + # Stop spinner if active + self._stop_spinner() + + elapsed = time.time() - self.start_time + msg = message or f"{self.current_task} failed" + print_colored(f"❌ {msg} after {elapsed:.2f}s", "red") + + self.reset() + + def _stop_spinner(self) -> None: + """Stops the spinner if active.""" + if self.use_spinner and self.spinner_thread and self.spinner_thread.is_alive(): + self.stop_spinner.set() + self.spinner_thread.join(timeout=0.5) + + # Clear spinner line + sys.stdout.write("\r" + " " * 80 + "\r") + sys.stdout.flush() + + def reset(self) -> None: + """Resets the tracker.""" + self._stop_spinner() + self.start_time = None + self.current_task = None + self.total = None + self.steps = 0 + self.spinner_thread = None + +class CliConfig: + """ + Manages the CLI configuration. + """ + + def __init__(self, config_dir: Optional[Union[str, Path]] = None): + """ + Initializes the CLI configuration. + + Args: + config_dir: Directory for configuration files + """ + if config_dir: + self.config_dir = Path(config_dir) + else: + self.config_dir = Path.home() / ".corebrain" / "cli" + + self.config_file = self.config_dir / "config.json" + self.config = self._load_config() + + def _load_config(self) -> Dict[str, Any]: + """ + Loads configuration from file. + + Returns: + Loaded configuration + """ + if not self.config_file.exists(): + return self._create_default_config() + + try: + import json + with open(self.config_file, 'r') as f: + return json.load(f) + except Exception as e: + logger.warning(f"Error loading configuration: {e}") + return self._create_default_config() + + def _create_default_config(self) -> Dict[str, Any]: + """ + Creates a default configuration. + + Returns: + Default configuration + """ + from corebrain.cli.common import DEFAULT_API_URL, DEFAULT_SSO_URL + + config = { + "api_url": DEFAULT_API_URL, + "sso_url": DEFAULT_SSO_URL, + "verbose": False, + "timeout": DEFAULT_TIMEOUT, + "last_used": { + "api_key": None, + "config_id": None + }, + "ui": { + "use_colors": True, + "use_spinner": True, + "verbose": False + } + } + + # Ensure the directory exists + ensure_dir(self.config_dir) + + # Save default configuration + try: + import json + with open(self.config_file, 'w') as f: + json.dump(config, f, indent=2) + except Exception as e: + logger.warning(f"Error saving configuration: {e}") + + return config + + def save(self) -> bool: + """ + Saves current configuration. + + Returns: + True if saved correctly + """ + try: + # Ensure the directory exists + ensure_dir(self.config_dir) + + import json + with open(self.config_file, 'w') as f: + json.dump(self.config, f, indent=2) + return True + except Exception as e: + logger.error(f"Error saving configuration: {e}") + return False + + def get(self, key: str, default: Any = None) -> Any: + """ + Gets a configuration value. + + Args: + key: Configuration key + default: Default value + + Returns: + Configuration value + """ + # Support for nested keys with dots + if "." in key: + parts = key.split(".") + current = self.config + for part in parts: + if part not in current: + return default + current = current[part] + return current + + return self.config.get(key, default) + + def set(self, key: str, value: Any) -> bool: + """ + Sets a configuration value. + + Args: + key: Configuration key + value: Value to set + + Returns: + True if set correctly + """ + # Support for nested keys with dots + if "." in key: + parts = key.split(".") + current = self.config + for part in parts[:-1]: + if part not in current: + current[part] = {} + current = current[part] + current[parts[-1]] = value + else: + self.config[key] = value + + return self.save() + + def update(self, config_dict: Dict[str, Any]) -> bool: + """ + Updates configuration with a dictionary. + + Args: + config_dict: Configuration dictionary + + Returns: + True if updated correctly + """ + self.config.update(config_dict) + return self.save() + + def update_last_used(self, api_key: Optional[str] = None, config_id: Optional[str] = None) -> bool: + """ + Updates the last used configuration. + + Args: + api_key: API key used + config_id: Configuration ID used + + Returns: + True if updated correctly + """ + if not self.config.get("last_used"): + self.config["last_used"] = {} + + if api_key: + self.config["last_used"]["api_key"] = api_key + + if config_id: + self.config["last_used"]["config_id"] = config_id + + return self.save() \ No newline at end of file diff --git a/corebrain/corebrain/config/__init__.py b/corebrain/corebrain/config/__init__.py new file mode 100644 index 0000000..7149948 --- /dev/null +++ b/corebrain/corebrain/config/__init__.py @@ -0,0 +1,10 @@ +""" +Configuration management for the Corebrain SDK. + +This package provides functionality to manage database connection configurations +and SDK preferences. +""" +from .manager import ConfigManager + +# Exportación explícita de componentes públicos +__all__ = ['ConfigManager'] \ No newline at end of file diff --git a/corebrain/corebrain/config/manager.py b/corebrain/corebrain/config/manager.py new file mode 100644 index 0000000..0fa9f65 --- /dev/null +++ b/corebrain/corebrain/config/manager.py @@ -0,0 +1,235 @@ +""" +Configuration manager for the Corebrain SDK. +""" + +import json +import uuid +from pathlib import Path +from typing import Dict, Any, List, Optional +from cryptography.fernet import Fernet +from corebrain.utils.serializer import serialize_to_json +from corebrain.core.common import logger + +# Made by Lukasz +# get data from pyproject.toml +def load_project_metadata(): + pyproject_path = Path(__file__).resolve().parent.parent / "pyproject.toml" + try: + with open(pyproject_path, "rb") as f: + data = tomli.load(f) + return data.get("project", {}) + except (FileNotFoundError, tomli.TOMLDecodeError) as e: + print(f"Warning: Could not load project metadata: {e}") + return {} + +# Made by Lukasz +# get the name, version, etc. +def get_config(): + metadata = load_project_metadata() # ^ + return { + "model": metadata.get("name", "unknown"), + "version": metadata.get("version", "0.0.0"), + "debug": False, + "logging": {"level": "info"} + } + +# Made by Lukasz +# export config to file +def export_config(filepath="config.json"): + config = get_config() # ^ + with open(filepath, "w") as f: + json.dump(config, f, indent=4) + print(f"Configuration exported to {filepath}") + +# Validates that a configuration with the given ID exists. +def validate_config(config_id: str): + # The API key under which configs are stored + api_key = os.environ.get("COREBRAIN_API_KEY", "") + manager = ConfigManager() + cfg = manager.get_config(api_key, config_id) + + if cfg: + print(f"✅ Configuration '{config_id}' is present and valid.") + return 0 + else: + print(f"❌ Configuration '{config_id}' not found.") + return 1 + +# Función para imprimir mensajes coloreados +def _print_colored(message: str, color: str) -> None: + """Simplified version of _print_colored that does not depend on cli.utils.""" + colors = { + "red": "\033[91m", + "green": "\033[92m", + "yellow": "\033[93m", + "blue": "\033[94m", + "default": "\033[0m" + } + color_code = colors.get(color, colors["default"]) + print(f"{color_code}{message}{colors['default']}") + +class ConfigManager: + """SDK configuration manager with improved security and performance.""" + + CONFIG_DIR = Path.home() / ".corebrain" + CONFIG_FILE = CONFIG_DIR / "config.json" + SECRET_KEY_FILE = CONFIG_DIR / "secret.key" + + def __init__(self): + self.configs = {} + self.cipher = None + self._ensure_config_dir() + self._load_secret_key() + self._load_configs() + + def _ensure_config_dir(self) -> None: + """Ensures that the configuration directory exists.""" + try: + self.CONFIG_DIR.mkdir(parents=True, exist_ok=True) + logger.debug(f"Directorio de configuración asegurado: {self.CONFIG_DIR}") + _print_colored(f"Directorio de configuración asegurado: {self.CONFIG_DIR}", "blue") + except Exception as e: + logger.error(f"Error al crear directorio de configuración: {str(e)}") + _print_colored(f"Error al crear directorio de configuración: {str(e)}", "red") + + def _load_secret_key(self) -> None: + """Loads or generates the secret key to encrypt sensitive data.""" + try: + if not self.SECRET_KEY_FILE.exists(): + key = Fernet.generate_key() + with open(self.SECRET_KEY_FILE, 'wb') as key_file: + key_file.write(key) + _print_colored(f"Nueva clave secreta generada en: {self.SECRET_KEY_FILE}", "green") + + with open(self.SECRET_KEY_FILE, 'rb') as key_file: + self.secret_key = key_file.read() + + self.cipher = Fernet(self.secret_key) + except Exception as e: + _print_colored(f"Error al cargar/generar clave secreta: {str(e)}", "red") + # Fallback a una clave temporal (menos segura pero funcional) + self.secret_key = Fernet.generate_key() + self.cipher = Fernet(self.secret_key) + + def _load_configs(self) -> Dict[str, Dict[str, Any]]: + """Loads the saved configurations.""" + if not self.CONFIG_FILE.exists(): + _print_colored(f"Archivo de configuración no encontrado: {self.CONFIG_FILE}", "yellow") + return {} + + try: + with open(self.CONFIG_FILE, 'r') as f: + encrypted_data = f.read() + + if not encrypted_data: + _print_colored("Archivo de configuración vacío", "yellow") + return {} + + try: + # Intentar descifrar los datos + decrypted_data = self.cipher.decrypt(encrypted_data.encode()).decode() + configs = json.loads(decrypted_data) + except Exception as e: + # Si falla el descifrado, intentar cargar como JSON plano + logger.warning(f"Error al descifrar configuración: {e}") + configs = json.loads(encrypted_data) + + if isinstance(configs, str): + configs = json.loads(configs) + + _print_colored(f"Configuración cargada", "green") + self.configs = configs + return configs + except Exception as e: + _print_colored(f"Error al cargar configuraciones: {str(e)}", "red") + return {} + + def _save_configs(self) -> None: + """Saves the current configurations.""" + try: + configs_json = serialize_to_json(self.configs) + encrypted_data = self.cipher.encrypt(json.dumps(configs_json).encode()).decode() + + with open(self.CONFIG_FILE, 'w') as f: + f.write(encrypted_data) + + _print_colored(f"Configuraciones guardadas en: {self.CONFIG_FILE}", "green") + except Exception as e: + _print_colored(f"Error al guardar configuraciones: {str(e)}", "red") + + def add_config(self, api_key: str, db_config: Dict[str, Any], config_id: Optional[str] = None) -> str: + """ + Adds a new configuration. + + Args: + api_key: Selected API Key + db_config: Database configuration + config_id: Optional ID for the configuration (one is generated if not provided) + + Returns: + Configuration ID + """ + if not config_id: + config_id = str(uuid.uuid4()) + db_config["config_id"] = config_id + + # Crear o actualizar la entrada para este token + if api_key not in self.configs: + self.configs[api_key] = {} + + # Añadir la configuración + self.configs[api_key][config_id] = db_config + self._save_configs() + + _print_colored(f"Configuración agregada: {config_id} para la API Key: {api_key[:8]}...", "green") + return config_id + + def get_config(self, api_key_selected: str, config_id: str) -> Optional[Dict[str, Any]]: + """ + Retrieves a specific configuration. + + Args: + api_key_selected: Selected API Key + config_id: Configuration ID + + Returns: + Configuration or None if it does not exist + """ + return self.configs.get(api_key_selected, {}).get(config_id) + + def list_configs(self, api_key_selected: str) -> List[str]: + """ + Lists the available configuration IDs for an API Key. + + Args: + api_key_selected: Selected API Key + + Returns: + List of configuration IDs + """ + return list(self.configs.get(api_key_selected, {}).keys()) + + def remove_config(self, api_key_selected: str, config_id: str) -> bool: + """ + Deletes a configuration. + + Args: + api_key_selected: Selected API Key + config_id: Configuration ID + + Returns: + True if deleted successfully, False otherwise + """ + if api_key_selected in self.configs and config_id in self.configs[api_key_selected]: + del self.configs[api_key_selected][config_id] + + # Si no quedan configuraciones para este token, eliminar la entrada + if not self.configs[api_key_selected]: + del self.configs[api_key_selected] + + self._save_configs() + _print_colored(f"Configuración {config_id} eliminada para API Key: {api_key_selected[:8]}...", "green") + return True + + _print_colored(f"Configuración {config_id} no encontrada para API Key: {api_key_selected[:8]}...", "yellow") + return False \ No newline at end of file diff --git a/corebrain/corebrain/core/__init__.py b/corebrain/corebrain/core/__init__.py new file mode 100644 index 0000000..a5fb552 --- /dev/null +++ b/corebrain/corebrain/core/__init__.py @@ -0,0 +1,20 @@ +""" +Corebrain SDK main components. + +This package contains the core components of the SDK, +including the main client and schema handling. +""" +from corebrain.core.client import Corebrain, init +from corebrain.core.query import QueryCache, QueryAnalyzer, QueryTemplate +from corebrain.core.test_utils import test_natural_language_query, generate_test_question_from_schema + +# Exportación explícita de componentes públicos +__all__ = [ + 'Corebrain', + 'init', + 'QueryCache', + 'QueryAnalyzer', + 'QueryTemplate', + 'test_natural_language_query', + 'generate_test_question_from_schema' +] \ No newline at end of file diff --git a/corebrain/corebrain/core/client.py b/corebrain/corebrain/core/client.py new file mode 100644 index 0000000..9b65225 --- /dev/null +++ b/corebrain/corebrain/core/client.py @@ -0,0 +1,1364 @@ +""" +Corebrain SDK Main Client. + +This module provides the main interface to interact with the Corebrain API +and enables natural language queries to relational and non-relational databases. +""" +import uuid +import re +import logging +import requests +import httpx +import sqlite3 +import mysql.connector +import psycopg2 +import pymongo +import json +from typing import Dict, Any, List, Optional +from sqlalchemy import create_engine, inspect +from pathlib import Path +from datetime import datetime + +from corebrain.core.common import logger, CorebrainError + +class Corebrain: + """ + Main client for the Corebrain SDK for natural language database queries. + + This class provides a unified interface to interact with different types of databases + (SQL and NoSQL) using natural language. It manages the connection, schema extraction, + and query processing through the Corebrain API. + + Attributes: + api_key (str): Authentication key for the Corebrain API. + db_config (Dict[str, Any]): Database connection configuration. + config_id (str): Unique identifier for the current configuration. + api_url (str): Base URL for the Corebrain API. + user_info (Dict[str, Any]): Information about the authenticated user. + db_connection: Active database connection. + db_schema (Dict[str, Any]): Extracted database schema. + + Examples: + SQLite initialization: + ```python + from corebrain import init + + # Connect to a SQLite database + client = init( + api_key="your_api_key", + db_config={ + "type": "sql", + "engine": "sqlite", + "database": "my_database.db" + } + ) + + # Make a query + result = client.ask("How many registered users are there?") + print(result["explanation"]) + ``` + + PostgreSQL initialization: + ```python + # Connect to PostgreSQL + client = init( + api_key="your_api_key", + db_config={ + "type": "sql", + "engine": "postgresql", + "host": "localhost", + "port": 5432, + "user": "postgres", + "password": "your_password", + "database": "my_database" + } + ) + ``` + + MongoDB initialization: + ```python + # Connect to MongoDB + client = init( + api_key="your_api_key", + db_config={ + "type": "mongodb", + "host": "localhost", + "port": 27017, + "database": "my_database" + } + ) + ``` + """ + + def __init__( + self, + api_key: str, + db_config: Optional[Dict[str, Any]] = None, + config_id: Optional[str] = None, + user_data: Optional[Dict[str, Any]] = None, + api_url: str = "http://localhost:5000", + skip_verification: bool = False + ): + """ + Initialize the Corebrain SDK client. + + Args: + api_key (str): Required API key for authentication with the Corebrain service. + Can be generated from the dashboard at https://dashboard.corebrain.com. + + db_config (Dict[str, Any], optional): Database configuration to query. + This parameter is required if config_id is not provided. Must contain at least: + - "type": Database type ("sql" or "mongodb") + - For SQL: "engine" ("sqlite", "postgresql", "mysql") + - Specific connection parameters depending on type and engine + + Example for SQLite: + ``` + { + "type": "sql", + "engine": "sqlite", + "database": "path/to/database.db" + } + ``` + + Example for PostgreSQL: + ``` + { + "type": "sql", + "engine": "postgresql", + "host": "localhost", + "port": 5432, + "user": "postgres", + "password": "password", + "database": "db_name" + } + ``` + + config_id (str, optional): Identifier for a previously saved configuration. + If provided, this configuration will be used instead of db_config. + Useful for maintaining persistent configurations between sessions. + + user_data (Dict[str, Any], optional): Additional user information for verification. + Can contain data like "email" for more precise token validation. + + api_url (str, optional): Base URL for the Corebrain API. + Defaults to "http://localhost:5000" for local development. + In production, it is typically "https://api.corebrain.com". + + skip_verification (bool, optional): If True, skips token verification with the server. + Useful in offline environments or for local testing. + Defaults to False. + + Raises: + ValueError: If required parameters are missing or if the configuration is invalid. + CorebrainError: If there are issues with the API connection or database. + + Example: + ```python + from corebrain import Corebrain + + # Basic initialization with SQLite + client = Corebrain( + api_key="your_api_key", + db_config={ + "type": "sql", + "engine": "sqlite", + "database": "my_db.db" + } + ) + ``` + """ + self.api_key = api_key + self.user_data = user_data + self.api_url = api_url.rstrip('/') + self.db_connection = None + self.db_schema = None + + # Import ConfigManager dynamically to avoid circular dependency + try: + from corebrain.config.manager import ConfigManager + self.config_manager = ConfigManager() + except ImportError as e: + logger.error(f"Error importing ConfigManager: {e}") + raise CorebrainError(f"Could not load configuration manager: {e}") + + # Determine which configuration to use + if config_id: + saved_config = self.config_manager.get_config(api_key, config_id) + if not saved_config: + # Try to load from old format + old_config = self._load_old_config(api_key, config_id) + if old_config: + self.db_config = old_config + self.config_id = config_id + # Save in new format + self.config_manager.add_config(api_key, old_config, config_id) + else: + raise ValueError(f"Configuration with ID {config_id} not found for the provided key") + else: + self.db_config = saved_config + self.config_id = config_id + elif db_config: + self.db_config = db_config + + # Generate config ID if it doesn't exist + if "config_id" in db_config: + self.config_id = db_config["config_id"] + else: + self.config_id = str(uuid.uuid4()) + db_config["config_id"] = self.config_id + + # Save the configuration + self.config_manager.add_config(api_key, db_config, self.config_id) + else: + raise ValueError("db_config or config_id must be provided") + + # Validate configuration + self._validate_config() + + # Verify the API token (only if necessary) + if not skip_verification: + self._verify_api_token() + else: + # Initialize user_info with basic information if not verifying + self.user_info = {"token": api_key} + + # Connect to the database + self._connect_to_database() + + # Extract database schema + self.db_schema = self._extract_db_schema() + + self.metadata = { + "config_id": self.config_id, + "api_key": api_key, + "db_config": self.db_config + } + + def _load_old_config(self, api_key: str, config_id: str) -> Optional[Dict[str, Any]]: + """ + Try to load configuration from old format. + + Args: + api_key: API key + config_id: Configuration ID + + Returns: + Configuration dictionary if found, None otherwise + """ + try: + # Try to load from old config file + old_config_path = Path.home() / ".corebrain" / "config.json" + if old_config_path.exists(): + with open(old_config_path, 'r') as f: + old_configs = json.load(f) + if api_key in old_configs and config_id in old_configs[api_key]: + return old_configs[api_key][config_id] + except Exception as e: + logger.warning(f"Error loading old config: {e}") + return None + + def _validate_config(self) -> None: + """ + Validate the provided configuration. + + This internal function verifies that the database configuration + contains all necessary fields according to the specified database type. + + Raises: + ValueError: If the database configuration is invalid or incomplete. + """ + if not self.api_key: + raise ValueError("API key is required. Generate one at dashboard.corebrain.com") + + if not self.db_config: + raise ValueError("Database configuration is required") + + if "type" not in self.db_config: + raise ValueError("Database type is required in db_config") + + if "connection_string" not in self.db_config and self.db_config["type"] != "sqlite_memory": + if self.db_config["type"] == "sql": + if "engine" not in self.db_config: + raise ValueError("Database engine is required for 'sql' type") + + # Verify alternative configuration for SQL engines + if self.db_config["engine"] == "mysql" or self.db_config["engine"] == "postgresql": + if not ("host" in self.db_config and "user" in self.db_config and + "password" in self.db_config and "database" in self.db_config): + raise ValueError("host, user, password, and database are required for MySQL/PostgreSQL") + elif self.db_config["engine"] == "sqlite": + if "database" not in self.db_config: + raise ValueError("database field is required for SQLite") + elif self.db_config["type"] == "mongodb": + if "database" not in self.db_config: + raise ValueError("database field is required for MongoDB") + + if "connection_string" not in self.db_config: + if not ("host" in self.db_config and "port" in self.db_config): + raise ValueError("host and port are required for MongoDB without connection_string") + + def _verify_api_token(self) -> None: + """ + Verify the API token with the server. + + This internal function sends a request to the Corebrain server + to validate that the provided API token is valid. + If the user provided additional information (like email), + it will be used for more precise verification. + + The verification results are stored in self.user_info. + + Raises: + ValueError: If the API token is invalid. + """ + try: + # Use the user's email for verification if available + if self.user_data and 'email' in self.user_data: + endpoint = f"{self.api_url}/api/auth/users/{self.user_data['email']}" + + response = httpx.get( + endpoint, + headers={"X-API-Key": self.api_key}, + timeout=10.0 + ) + + if response.status_code != 200: + raise ValueError(f"Invalid API token. Error code: {response.status_code}") + + # Store user information + self.user_info = response.json() + else: + # If no email, do a simple verification with a generic endpoint + endpoint = f"{self.api_url}/api/auth/verify" + + try: + response = httpx.get( + endpoint, + headers={"X-API-Key": self.api_key}, + timeout=5.0 + ) + + if response.status_code == 200: + self.user_info = response.json() + else: + # If it fails, just store basic information + self.user_info = {"token": self.api_key} + except Exception as e: + # If there's a connection error, don't fail, just store basic info + logger.warning(f"Could not verify token: {str(e)}") + self.user_info = {"token": self.api_key} + + except httpx.RequestError as e: + # Connection error shouldn't be fatal if we already have a configuration + logger.warning(f"Error connecting to API: {str(e)}") + self.user_info = {"token": self.api_key} + except Exception as e: + # Other errors are logged but not fatal + logger.warning(f"Error in token verification: {str(e)}") + self.user_info = {"token": self.api_key} + + def _connect_to_database(self) -> None: + """ + Establish a connection to the database according to the configuration. + + This internal function creates a database connection using the parameters + defined in self.db_config. It supports various database types: + - SQLite (file or in-memory) + - PostgreSQL + - MySQL + - MongoDB + + The connection is stored in self.db_connection for later use. + + Raises: + CorebrainError: If the connection to the database cannot be established. + NotImplementedError: If the database type is not supported. + """ + db_type = self.db_config["type"].lower() + + try: + if db_type == "sql": + engine = self.db_config.get("engine", "").lower() + + if engine == "sqlite": + database = self.db_config.get("database", "") + if database: + self.db_connection = sqlite3.connect(database) + else: + self.db_connection = sqlite3.connect(self.db_config.get("connection_string", "")) + + elif engine == "mysql": + if "connection_string" in self.db_config: + self.db_connection = mysql.connector.connect( + connection_string=self.db_config["connection_string"] + ) + else: + self.db_connection = mysql.connector.connect( + host=self.db_config.get("host", "localhost"), + user=self.db_config.get("user", ""), + password=self.db_config.get("password", ""), + database=self.db_config.get("database", ""), + port=self.db_config.get("port", 3306) + ) + + elif engine == "postgresql": + if "connection_string" in self.db_config: + self.db_connection = psycopg2.connect(self.db_config["connection_string"]) + else: + self.db_connection = psycopg2.connect( + host=self.db_config.get("host", "localhost"), + user=self.db_config.get("user", ""), + password=self.db_config.get("password", ""), + dbname=self.db_config.get("database", ""), + port=self.db_config.get("port", 5432) + ) + + else: + # Use SQLAlchemy for other engines + self.db_connection = create_engine(self.db_config["connection_string"]) + + # Improved code for MongoDB + elif db_type == "nosql" or db_type == "mongodb": + # If engine is mongodb or the type is directly mongodb + engine = self.db_config.get("engine", "").lower() + if not engine or engine == "mongodb": + # Create connection parameters + mongo_params = {} + + if "connection_string" in self.db_config: + # Save the MongoDB client to be able to close it correctly later + self.mongo_client = pymongo.MongoClient(self.db_config["connection_string"]) + else: + # Configure host and port + mongo_params["host"] = self.db_config.get("host", "localhost") + if "port" in self.db_config: + mongo_params["port"] = self.db_config.get("port") + + # Add credentials if available + if "user" in self.db_config and self.db_config["user"]: + mongo_params["username"] = self.db_config["user"] + if "password" in self.db_config and self.db_config["password"]: + mongo_params["password"] = self.db_config["password"] + + # Create MongoDB client + self.mongo_client = pymongo.MongoClient(**mongo_params) + + # Get the database + db_name = self.db_config.get("database", "") + if db_name: + # Save reference to the database + self.db_connection = self.mongo_client[db_name] + else: + # If there's no database name, use 'admin' as fallback + logger.warning("Database name not specified for MongoDB, using 'admin'") + self.db_connection = self.mongo_client["admin"] + else: + raise ValueError(f"Unsupported NoSQL database engine: {engine}") + + elif db_type == "sqlite_memory": + self.db_connection = sqlite3.connect(":memory:") + + else: + raise ValueError(f"Unsupported database type: {db_type}. Valid types: 'sql', 'nosql', 'mongodb'") + + except Exception as e: + logger.error(f"Error connecting to database: {str(e)}") + raise ConnectionError(f"Error connecting to database: {str(e)}") + + def _extract_db_schema(self, detail_level: str = "full", specific_collections: List[str] = None) -> Dict[str, Any]: + """ + Extracts the database schema to provide context to the AI. + + Returns: + Dictionary with the database structure organized by tables/collections + """ + logger.info(f"Extrayendo esquema de base de datos. Tipo: {self.db_config['type']}, Motor: {self.db_config.get('engine')}") + + db_type = self.db_config["type"].lower() + schema = { + "type": db_type, + "database": self.db_config.get("database", ""), + "tables": {}, + "total_collections": 0, # Añadir contador total + "included_collections": 0 # Contador de incluidas + } + excluded_tables = set(self.db_config.get("excluded_tables", [])) + logger.info(f"Tablas excluidas: {excluded_tables}") + + try: + if db_type == "sql": + engine = self.db_config.get("engine", "").lower() + logger.info(f"Procesando base de datos SQL con motor: {engine}") + + if engine in ["sqlite", "mysql", "postgresql"]: + cursor = self.db_connection.cursor() + + if engine == "sqlite": + logger.info("Obteniendo tablas de SQLite") + # Obtener listado de tablas + cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") + tables = cursor.fetchall() + logger.info(f"Tablas encontradas en SQLite: {tables}") + + elif engine == "mysql": + logger.info("Obteniendo tablas de MySQL") + cursor.execute("SHOW TABLES;") + tables = cursor.fetchall() + logger.info(f"Tablas encontradas en MySQL: {tables}") + + elif engine == "postgresql": + logger.info("Obteniendo tablas de PostgreSQL") + cursor.execute(""" + SELECT table_name FROM information_schema.tables + WHERE table_schema = 'public'; + """) + tables = cursor.fetchall() + logger.info(f"Tablas encontradas en PostgreSQL: {tables}") + + # Procesar las tablas encontradas + for table in tables: + table_name = table[0] + logger.info(f"Procesando tabla: {table_name}") + + # Saltar tablas excluidas + if table_name in excluded_tables: + logger.info(f"Saltando tabla excluida: {table_name}") + continue + + try: + # Obtener información de columnas según el motor + if engine == "sqlite": + cursor.execute(f"PRAGMA table_info({table_name});") + elif engine == "mysql": + cursor.execute(f"DESCRIBE {table_name};") + elif engine == "postgresql": + cursor.execute(f""" + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_name = '{table_name}'; + """) + + columns = cursor.fetchall() + logger.info(f"Columnas encontradas para {table_name}: {columns}") + + # Estructura de columnas según el motor + if engine == "sqlite": + column_info = [{"name": col[1], "type": col[2]} for col in columns] + elif engine == "mysql": + column_info = [{"name": col[0], "type": col[1]} for col in columns] + elif engine == "postgresql": + column_info = [{"name": col[0], "type": col[1]} for col in columns] + + # Guardar información de la tabla + schema["tables"][table_name] = { + "columns": column_info, + "sample_data": [] # No obtenemos datos de muestra por defecto + } + + except Exception as e: + logger.error(f"Error procesando tabla {table_name}: {str(e)}") + + else: + # Usando SQLAlchemy + logger.info("Usando SQLAlchemy para obtener el esquema") + inspector = inspect(self.db_connection) + table_names = inspector.get_table_names() + logger.info(f"Tablas encontradas con SQLAlchemy: {table_names}") + + for table_name in table_names: + if table_name in excluded_tables: + logger.info(f"Saltando tabla excluida: {table_name}") + continue + + try: + columns = inspector.get_columns(table_name) + column_info = [{"name": col["name"], "type": str(col["type"])} for col in columns] + + schema["tables"][table_name] = { + "columns": column_info, + "sample_data": [] + } + except Exception as e: + logger.error(f"Error procesando tabla {table_name} con SQLAlchemy: {str(e)}") + + elif db_type in ["nosql", "mongodb"]: + logger.info("Procesando base de datos MongoDB") + if not hasattr(self, 'db_connection') or self.db_connection is None: + logger.error("La conexión a MongoDB no está disponible") + return schema + + try: + collection_names = [] + try: + collection_names = self.db_connection.list_collection_names() + schema["total_collections"] = len(collection_names) + logger.info(f"Colecciones encontradas en MongoDB: {collection_names}") + except Exception as e: + logger.error(f"Error al obtener colecciones MongoDB: {str(e)}") + return schema + + # Si solo queremos los nombres + if detail_level == "names_only": + schema["collection_names"] = collection_names + return schema + + # Procesar cada colección + for collection_name in collection_names: + if collection_name in excluded_tables: + logger.info(f"Saltando colección excluida: {collection_name}") + continue + + try: + collection = self.db_connection[collection_name] + # Obtener un documento para inferir estructura + first_doc = collection.find_one() + + if first_doc: + fields = [] + for field, value in first_doc.items(): + if field != "_id": + field_type = type(value).__name__ + fields.append({"name": field, "type": field_type}) + + schema["tables"][collection_name] = { + "fields": fields, + "doc_count": collection.estimated_document_count() + } + logger.info(f"Procesada colección {collection_name} con {len(fields)} campos") + else: + logger.info(f"Colección {collection_name} está vacía") + schema["tables"][collection_name] = { + "fields": [], + "doc_count": 0 + } + except Exception as e: + logger.error(f"Error procesando colección {collection_name}: {str(e)}") + + except Exception as e: + logger.error(f"Error general procesando MongoDB: {str(e)}") + + # Convertir el diccionario de tablas en una lista + table_list = [] + for table_name, table_info in schema["tables"].items(): + table_data = {"name": table_name} + table_data.update(table_info) + table_list.append(table_data) + + schema["tables_list"] = table_list + logger.info(f"Esquema final - Tablas encontradas: {len(schema['tables'])}") + logger.info(f"Nombres de tablas: {list(schema['tables'].keys())}") + + return schema + + except Exception as e: + logger.error(f"Error al extraer el esquema de la base de datos: {str(e)}") + return {"type": db_type, "tables": {}, "tables_list": []} + + def list_collections_name(self) -> List[str]: + """ + Returns a list of the available collections or tables in the database. + + Returns: + List of collections or tables + """ + print("Excluded tables: ", self.db_schema.get("excluded_tables", [])) + return self.db_schema.get("tables", []) + + def ask(self, question: str, **kwargs) -> Dict: + """ + Perform a natural language query to the database. + + Args: + question: The natural language question + **kwargs: Additional parameters: + - collection_name: For MongoDB, the collection to query + - limit: Maximum number of results + - detail_level: Schema detail level ("names_only", "structure", "full") + - auto_select: Whether to automatically select collections + - max_collections: Maximum number of collections to include + - execute_query: Whether to execute the query (True by default) + - explain_results: Whether to generate an explanation of results (True by default) + + Returns: + Dictionary with the query results and explanation + """ + try: + # Verificar opciones de comportamiento + execute_query = kwargs.get("execute_query", True) + explain_results = kwargs.get("explain_results", True) + + # Obtener esquema con el nivel de detalle apropiado + detail_level = kwargs.get("detail_level", "full") + schema = self._extract_db_schema(detail_level=detail_level) + + # Validar que el esquema tiene tablas/colecciones + if not schema.get("tables"): + print("Error: No se encontraron tablas/colecciones en la base de datos") + return {"error": True, "explanation": "No se encontraron tablas/colecciones en la base de datos"} + + # Obtener nombres de tablas disponibles para validación + available_tables = set() + if isinstance(schema.get("tables"), dict): + available_tables.update(schema["tables"].keys()) + elif isinstance(schema.get("tables_list"), list): + available_tables.update(table["name"] for table in schema["tables_list"]) + + # Preparar datos de la solicitud con información de esquema mejorada + request_data = { + "question": question, + "db_schema": schema, + "config_id": self.config_id, + "metadata": { + "type": self.db_config["type"].lower(), + "engine": self.db_config.get("engine", "").lower(), + "database": self.db_config.get("database", ""), + "available_tables": list(available_tables), + "collections": list(available_tables) + } + } + + # Añadir configuración de la base de datos al request + # Esto permite a la API ejecutar directamente las consultas si es necesario + if execute_query: + request_data["db_config"] = self.db_config + + # Añadir datos de usuario si están disponibles + if self.user_data: + request_data["user_data"] = self.user_data + + # Preparar headers para la solicitud + headers = { + "X-API-Key": self.api_key, + "Content-Type": "application/json" + } + + # Determinar el endpoint adecuado según el modo de ejecución + if execute_query: + # Usar el endpoint de ejecución completa + endpoint = f"{self.api_url}/api/database/sdk/query" + else: + # Usar el endpoint de solo generación de consulta + endpoint = f"{self.api_url}/api/database/generate" + + # Realizar solicitud a la API + response = httpx.post( + endpoint, + headers=headers, + content=json.dumps(request_data, default=str), + timeout=60.0 + ) + + # Verificar respuesta + if response.status_code != 200: + error_msg = f"Error {response.status_code} al realizar la consulta" + try: + error_data = response.json() + if isinstance(error_data, dict): + error_msg += f": {error_data.get('detail', error_data.get('message', response.text))}" + except: + error_msg += f": {response.text}" + return {"error": True, "explanation": error_msg} + + # Procesar respuesta de la API + api_response = response.json() + + # Verificar si la API reportó un error + if api_response.get("error", False): + return api_response + + # Verificar si se generó una consulta válida + if "query" not in api_response: + return { + "error": True, + "explanation": "La API no generó una consulta válida." + } + + # Si se debe ejecutar la consulta pero la API no lo hizo + # (esto ocurriría solo en caso de cambios de configuración o fallbacks) + if execute_query and "result" not in api_response: + try: + # Preparar la consulta para ejecución local + query_type = self.db_config.get("engine", "").lower() if self.db_config["type"].lower() == "sql" else self.db_config["type"].lower() + query_value = api_response["query"] + + # Para SQL, asegurarse de que la consulta es un string + if query_type in ["sqlite", "mysql", "postgresql"]: + if isinstance(query_value, dict): + sql_candidate = query_value.get("sql") or query_value.get("query") + if isinstance(sql_candidate, str): + query_value = sql_candidate + else: + raise CorebrainError(f"La consulta SQL generada no es un string: {query_value}") + + # Preparar la consulta con el formato adecuado + query_to_execute = { + "type": query_type, + "query": query_value + } + + # Para MongoDB, añadir información específica + if query_type in ["nosql", "mongodb"]: + # Obtener nombre de colección + collection_name = None + if isinstance(api_response["query"], dict): + collection_name = api_response["query"].get("collection") + if not collection_name and "collection_name" in kwargs: + collection_name = kwargs["collection_name"] + if not collection_name and "collection" in self.db_config: + collection_name = self.db_config["collection"] + if not collection_name and available_tables: + collection_name = list(available_tables)[0] + + # Validar nombre de colección + if not collection_name: + raise CorebrainError("No se especificó colección y no se encontraron colecciones en el esquema") + if not isinstance(collection_name, str) or not collection_name.strip(): + raise CorebrainError("Nombre de colección inválido: debe ser un string no vacío") + + # Añadir colección a la consulta + query_to_execute["collection"] = collection_name + + # Añadir tipo de operación + if isinstance(api_response["query"], dict): + query_to_execute["operation"] = api_response["query"].get("operation", "find") + + # Añadir límite si se especifica + if "limit" in kwargs: + query_to_execute["limit"] = kwargs["limit"] + + # Ejecutar la consulta + start_time = datetime.now() + query_result = self._execute_query(query_to_execute) + query_time_ms = int((datetime.now() - start_time).total_seconds() * 1000) + + # Actualizar la respuesta con los resultados + api_response["result"] = { + "data": query_result, + "count": len(query_result) if isinstance(query_result, list) else 1, + "query_time_ms": query_time_ms, + "has_more": False + } + + # Si se debe generar explicación pero la API no lo hizo + if explain_results and ( + "explanation" not in api_response or + not isinstance(api_response.get("explanation"), str) or + len(str(api_response.get("explanation", ""))) < 15 # Detectar explicaciones numéricas o muy cortas + ): + # Preparar datos para obtener explicación + explanation_data = { + "question": question, + "query": api_response["query"], + "result": query_result, + "query_time_ms": query_time_ms, + "config_id": self.config_id, + "metadata": { + "collections_used": [query_to_execute.get("collection")] if query_to_execute.get("collection") else [], + "execution_time_ms": query_time_ms, + "available_tables": list(available_tables) + } + } + + try: + # Obtener explicación de la API + explanation_response = httpx.post( + f"{self.api_url}/api/database/sdk/query/explain", + headers=headers, + content=json.dumps(explanation_data, default=str), + timeout=30.0 + ) + + if explanation_response.status_code == 200: + explanation_result = explanation_response.json() + api_response["explanation"] = explanation_result.get("explanation", "No se pudo generar una explicación.") + else: + api_response["explanation"] = self._generate_fallback_explanation(query_to_execute, query_result) + except Exception as explain_error: + logger.error(f"Error al obtener explicación: {str(explain_error)}") + api_response["explanation"] = self._generate_fallback_explanation(query_to_execute, query_result) + + except Exception as e: + error_msg = f"Error al ejecutar la consulta: {str(e)}" + logger.error(error_msg) + return { + "error": True, + "explanation": error_msg, + "query": api_response.get("query", {}), + "metadata": { + "available_tables": list(available_tables) + } + } + + # Verificar si la explicación es un número (probablemente el tiempo de ejecución) y corregirlo + if "explanation" in api_response and not isinstance(api_response["explanation"], str): + # Si la explicación es un número, reemplazarla con una explicación generada + try: + is_sql = False + if "query" in api_response: + if isinstance(api_response["query"], dict) and "sql" in api_response["query"]: + is_sql = True + + if "result" in api_response: + result_data = api_response["result"] + if isinstance(result_data, dict) and "data" in result_data: + result_data = result_data["data"] + + if is_sql: + sql_query = api_response["query"].get("sql", "") + api_response["explanation"] = self._generate_sql_explanation(sql_query, result_data) + else: + # Para MongoDB o genérico + api_response["explanation"] = self._generate_generic_explanation(api_response["query"], result_data) + else: + api_response["explanation"] = "La consulta se ha ejecutado correctamente." + except Exception as exp_fix_error: + logger.error(f"Error al corregir explicación: {str(exp_fix_error)}") + api_response["explanation"] = "La consulta se ha ejecutado correctamente." + + # Preparar la respuesta final + result = { + "question": question, + "query": api_response["query"], + "config_id": self.config_id, + "metadata": { + "available_tables": list(available_tables) + } + } + + # Añadir resultados si están disponibles + if "result" in api_response: + if isinstance(api_response["result"], dict) and "data" in api_response["result"]: + result["result"] = api_response["result"] + else: + result["result"] = { + "data": api_response["result"], + "count": len(api_response["result"]) if isinstance(api_response["result"], list) else 1, + "query_time_ms": api_response.get("query_time_ms", 0), + "has_more": False + } + + # Añadir explicación si está disponible + if "explanation" in api_response: + result["explanation"] = api_response["explanation"] + + return result + + except httpx.TimeoutException: + return {"error": True, "explanation": "Tiempo de espera agotado al conectar con el servidor."} + + except httpx.RequestError as e: + return {"error": True, "explanation": f"Error de conexión con el servidor: {str(e)}"} + + except Exception as e: + import traceback + error_details = traceback.format_exc() + logger.error(f"Error inesperado en ask(): {error_details}") + return {"error": True, "explanation": f"Error inesperado: {str(e)}"} + + def _generate_fallback_explanation(self, query, results): + """ + Generates a fallback explanation when the explanation generation fails. + + Args: + query: The executed query + results: The obtained results + + Returns: + Generated explanation + """ + # Determinar si es SQL o MongoDB + if isinstance(query, dict): + query_type = query.get("type", "").lower() + + if query_type in ["sqlite", "mysql", "postgresql"]: + return self._generate_sql_explanation(query.get("query", ""), results) + elif query_type in ["nosql", "mongodb"]: + return self._generate_mongodb_explanation(query, results) + + # Fallback genérico + result_count = len(results) if isinstance(results, list) else (1 if results else 0) + return f"La consulta devolvió {result_count} resultados." + + def _generate_sql_explanation(self, sql_query, results): + """ + Generates a simple explanation for SQL queries. + + Args: + sql_query: The executed SQL query + results: The obtained results + + Returns: + Generated explanation + """ + sql_lower = sql_query.lower() if isinstance(sql_query, str) else "" + result_count = len(results) if isinstance(results, list) else (1 if results else 0) + + # Extraer nombres de tablas si es posible + tables = [] + from_match = re.search(r'from\s+([a-zA-Z0-9_]+)', sql_lower) + if from_match: + tables.append(from_match.group(1)) + + join_matches = re.findall(r'join\s+([a-zA-Z0-9_]+)', sql_lower) + if join_matches: + tables.extend(join_matches) + + # Detectar tipo de consulta + if "select" in sql_lower: + if "join" in sql_lower: + if len(tables) > 1: + if "where" in sql_lower: + return f"Se encontraron {result_count} registros que cumplen con los criterios especificados, relacionando información de las tablas {', '.join(tables)}." + else: + return f"Se obtuvieron {result_count} registros relacionando información de las tablas {', '.join(tables)}." + else: + return f"Se obtuvieron {result_count} registros relacionando datos entre tablas." + + elif "where" in sql_lower: + return f"Se encontraron {result_count} registros que cumplen con los criterios de búsqueda." + + else: + return f"La consulta devolvió {result_count} registros de la base de datos." + + # Para otros tipos de consultas (INSERT, UPDATE, DELETE) + if "insert" in sql_lower: + return "Se insertaron correctamente los datos en la base de datos." + elif "update" in sql_lower: + return "Se actualizaron correctamente los datos en la base de datos." + elif "delete" in sql_lower: + return "Se eliminaron correctamente los datos de la base de datos." + + # Fallback genérico + return f"La consulta SQL se ejecutó correctamente y devolvió {result_count} resultados." + + + def _generate_mongodb_explanation(self, query, results): + """ + Generates a simple explanation for MongoDB queries. + + Args: + query: The executed MongoDB query + results: The obtained results + + Returns: + Generated explanation + """ + collection = query.get("collection", "la colección") + operation = query.get("operation", "find") + result_count = len(results) if isinstance(results, list) else (1 if results else 0) + + # Generar explicación según la operación + if operation == "find": + return f"Se encontraron {result_count} documentos en la colección {collection} que coinciden con los criterios de búsqueda." + elif operation == "findOne": + if result_count > 0: + return f"Se encontró el documento solicitado en la colección {collection}." + else: + return f"No se encontró ningún documento en la colección {collection} que coincida con los criterios." + elif operation == "aggregate": + return f"La agregación en la colección {collection} devolvió {result_count} resultados." + elif operation == "insertOne": + return f"Se ha insertado correctamente un nuevo documento en la colección {collection}." + elif operation == "updateOne": + return f"Se ha actualizado correctamente un documento en la colección {collection}." + elif operation == "deleteOne": + return f"Se ha eliminado correctamente un documento de la colección {collection}." + + # Fallback genérico + return f"La operación {operation} se ejecutó correctamente en la colección {collection} y devolvió {result_count} resultados." + + + def _generate_generic_explanation(self, query, results): + """ + Generates a generic explanation when the query type cannot be determined. + + Args: + query: The executed query + results: The obtained results + + Returns: + Generated explanation + """ + result_count = len(results) if isinstance(results, list) else (1 if results else 0) + + if result_count == 0: + return "La consulta no devolvió ningún resultado." + elif result_count == 1: + return "La consulta devolvió 1 resultado." + else: + return f"La consulta devolvió {result_count} resultados." + + + def close(self) -> None: + """ + Close the database connection and release resources. + + This method should be called when the client is no longer needed to + ensure proper cleanup of resources. + """ + if self.db_connection: + db_type = self.db_config["type"].lower() + + try: + if db_type == "sql": + engine = self.db_config.get("engine", "").lower() + if engine in ["sqlite", "mysql", "postgresql"]: + self.db_connection.close() + else: + # SQLAlchemy engine + self.db_connection.dispose() + + elif db_type == "nosql" or db_type == "mongodb": + # For MongoDB, we close the client + if hasattr(self, 'mongo_client') and self.mongo_client: + self.mongo_client.close() + + elif db_type == "sqlite_memory": + self.db_connection.close() + + except Exception as e: + logger.warning(f"Error closing database connection: {str(e)}") + + self.db_connection = None + logger.info("Database connection closed") + + def _execute_query(self, query: Dict[str, Any]) -> List[Dict[str, Any]]: + """ + Execute a query based on its type. + + Args: + query: Dictionary containing query information + + Returns: + List of dictionaries containing query results + """ + query_type = query.get("type", "").lower() + + if query_type in ["sqlite", "mysql", "postgresql"]: + return self._execute_sql_query(query) + elif query_type in ["nosql", "mongodb"]: + return self._execute_mongodb_query(query) + else: + raise CorebrainError(f"Unsupported query type: {query_type}") + + def _execute_sql_query(self, query: Dict[str, Any]) -> List[Dict[str, Any]]: + """ + Execute a SQL query. + + Args: + query: Dictionary containing SQL query information + + Returns: + List of dictionaries containing query results + """ + query_type = query.get("type", "").lower() + + if query_type in ["sqlite", "mysql", "postgresql"]: + sql_query = query.get("query", "") + if not sql_query: + raise CorebrainError("No SQL query provided") + + engine = self.db_config.get("engine", "").lower() + + if engine == "sqlite": + return self._execute_sqlite_query(sql_query) + elif engine == "mysql": + return self._execute_mysql_query(sql_query) + elif engine == "postgresql": + return self._execute_postgresql_query(sql_query) + else: + raise CorebrainError(f"Unsupported SQL engine: {engine}") + + else: + raise CorebrainError(f"Unsupported SQL query type: {query_type}") + + def _execute_sqlite_query(self, sql_query: str) -> List[Dict[str, Any]]: + """ + Execute a SQLite query. + + Args: + sql_query (str): SQL query to execute + + Returns: + List[Dict[str, Any]]: List of results as dictionaries + """ + cursor = self.db_connection.cursor() + cursor.execute(sql_query) + + # Get column names + columns = [description[0] for description in cursor.description] + + # Convert results to list of dictionaries + results = [] + for row in cursor.fetchall(): + result = {} + for i, value in enumerate(row): + # Convert datetime objects to strings + if hasattr(value, 'isoformat'): + result[columns[i]] = value.isoformat() + else: + result[columns[i]] = value + results.append(result) + + return results + + def _execute_mysql_query(self, sql_query: str) -> List[Dict[str, Any]]: + """ + Execute a MySQL query. + + Args: + sql_query (str): SQL query to execute + + Returns: + List[Dict[str, Any]]: List of results as dictionaries + """ + cursor = self.db_connection.cursor(dictionary=True) + cursor.execute(sql_query) + + # Convert results to list of dictionaries + results = [] + for row in cursor.fetchall(): + result = {} + for key, value in row.items(): + # Convert datetime objects to strings + if hasattr(value, 'isoformat'): + result[key] = value.isoformat() + else: + result[key] = value + results.append(result) + + return results + + def _execute_postgresql_query(self, sql_query: str) -> List[Dict[str, Any]]: + """ + Execute a PostgreSQL query. + + Args: + sql_query (str): SQL query to execute + + Returns: + List[Dict[str, Any]]: List of results as dictionaries + """ + cursor = self.db_connection.cursor() + cursor.execute(sql_query) + + # Get column names + columns = [description[0] for description in cursor.description] + + # Convert results to list of dictionaries + results = [] + for row in cursor.fetchall(): + result = {} + for i, value in enumerate(row): + # Convert datetime objects to strings + if hasattr(value, 'isoformat'): + result[columns[i]] = value.isoformat() + else: + result[columns[i]] = value + results.append(result) + + return results + + def _execute_mongodb_query(self, query: Dict[str, Any]) -> List[Dict[str, Any]]: + """ + Execute a MongoDB query. + + Args: + query: Dictionary containing MongoDB query information + + Returns: + List of dictionaries containing query results + """ + try: + # Get collection name from query or use default + collection_name = query.get("collection") + if not collection_name: + raise CorebrainError("No collection specified for MongoDB query") + + # Get MongoDB collection + collection = self.mongo_client[self.db_config.get("database", "")][collection_name] + + # Execute query based on operation type + operation = query.get("operation", "find") + + if operation == "find": + # Handle find operation + cursor = collection.find( + query.get("query", {}), + projection=query.get("projection"), + sort=query.get("sort"), + limit=query.get("limit", 10), + skip=query.get("skip", 0) + ) + results = list(cursor) + + elif operation == "aggregate": + # Handle aggregate operation + pipeline = query.get("pipeline", []) + cursor = collection.aggregate(pipeline) + results = list(cursor) + + else: + raise CorebrainError(f"Unsupported MongoDB operation: {operation}") + + # Convert results to dictionaries and handle datetime serialization + serialized_results = [] + for doc in results: + # Convert ObjectId to string + if "_id" in doc: + doc["_id"] = str(doc["_id"]) + + # Handle datetime objects + for key, value in doc.items(): + if hasattr(value, 'isoformat'): + doc[key] = value.isoformat() + + serialized_results.append(doc) + + return serialized_results + + except Exception as e: + raise CorebrainError(f"Error executing MongoDB query: {str(e)}") + + +def init( + api_key: str = None, + db_config: Dict = None, + config_id: str = None, + user_data: Dict = None, + api_url: str = None, + skip_verification: bool = False +) -> Corebrain: + """ + Initialize and return a Corebrain client instance. + + This function creates a new Corebrain SDK client with the provided configuration. + It's a convenient factory function that wraps the Corebrain class initialization. + + Args: + api_key (str, optional): Corebrain API key. If not provided, it will attempt + to read from the COREBRAIN_API_KEY environment variable. + db_config (Dict, optional): Database configuration dictionary. If not provided, + it will attempt to read from the COREBRAIN_DB_CONFIG environment variable + (expected in JSON format). + config_id (str, optional): Configuration ID for saving/loading configurations. + user_data (Dict, optional): Optional user data for personalization. + api_url (str, optional): Corebrain API URL. Defaults to the production API. + skip_verification (bool, optional): Skip API token verification. Default False. + + Returns: + Corebrain: An initialized Corebrain client instance. + + Example: + >>> client = init(api_key="your_api_key", db_config={"type": "sql", "engine": "sqlite", "database": "example.db"}) + """ + return Corebrain( + api_key=api_key, + db_config=db_config, + config_id=config_id, + user_data=user_data, + api_url=api_url, + skip_verification=skip_verification + ) + diff --git a/corebrain/corebrain/core/common.py b/corebrain/corebrain/core/common.py new file mode 100644 index 0000000..3d75c8e --- /dev/null +++ b/corebrain/corebrain/core/common.py @@ -0,0 +1,225 @@ +""" +Core functionalities shared across the Corebrain SDK. + +This module contains common elements used throughout the SDK, including: +- Logging system configuration +- Common type definitions and aliases +- Custom exceptions for better error handling +- Component registry system for dependency management + +These elements provide a common foundation for implementing +the rest of the SDK modules, ensuring consistency and facilitating +maintenance. +""" +import logging +from typing import Dict, Any, Optional, List, Callable, TypeVar, Union + +# Global logging configuration +logger = logging.getLogger("corebrain") +logger.addHandler(logging.NullHandler()) + +# Type aliases to improve readability and maintenance +ConfigDict = Dict[str, Any] +""" +Type representing a configuration as a key-value dictionary. + +Example: +```python +config: ConfigDict = { + "type": "sql", + "engine": "postgresql", + "host": "localhost", + "port": 5432, + "user": "postgres", + "password": "password", + "database": "mydatabase" +} +``` +""" + +SchemaDict = Dict[str, Any] +""" +Type representing a database schema as a dictionary. + +Example: +```python +schema: SchemaDict = { + "tables": [ + { + "name": "users", + "columns": [ + {"name": "id", "type": "INTEGER", "primary_key": True}, + {"name": "name", "type": "TEXT"}, + {"name": "email", "type": "TEXT"} + ] + } + ] +} +``` +""" + +# Generic component for typing +T = TypeVar('T') + +# SDK exceptions +class CorebrainError(Exception): + """ + Base exception for all Corebrain SDK errors. + + All other specific exceptions inherit from this class, + allowing you to catch any SDK error with a single + except block. + + Example: + ```python + try: + result = client.ask("How many users are there?") + except CorebrainError as e: + print(f"Corebrain error: {e}") + ``` + """ + pass + +class ConfigError(CorebrainError): + """ + Error related to SDK configuration. + + Raised when there are issues with the provided configuration, + such as invalid credentials, missing parameters, or incorrect formats. + + Example: + ```python + try: + client = init(api_key="invalid_key", db_config={}) + except ConfigError as e: + print(f"Configuration error: {e}") + ``` + """ + pass + +class DatabaseError(CorebrainError): + """ + Error related to database connection or query. + + Raised when there are problems connecting to the database, + executing queries, or extracting schema information. + + Example: + ```python + try: + result = client.ask("select * from a_table_that_does_not_exist") + except DatabaseError as e: + print(f"Database error: {e}") + ``` + """ + pass + +class APIError(CorebrainError): + """ + Error related to communication with the Corebrain API. + + Raised when there are issues in communicating with the service, + such as network errors, authentication failures, or unexpected responses. + + Example: + ```python + try: + result = client.ask("How many users are there?") + except APIError as e: + print(f"API error: {e}") + if e.status_code == 401: + print("Please verify your API key") + ``` + """ + def __init__(self, message: str, status_code: Optional[int] = None, response: Optional[Dict[str, Any]] = None): + """ + Initialize an APIError exception. + + Args: + message: Descriptive error message + status_code: Optional HTTP status code (e.g., 401, 404, 500) + response: Server response content if available + """ + self.status_code = status_code + self.response = response + super().__init__(message) + +# Component registry (to avoid circular imports) +_registry: Dict[str, Any] = {} + +def register_component(name: str, component: Any) -> None: + """ + Register a component in the global registry. + + This mechanism resolves circular dependencies between modules + by providing a way to access components without importing them directly. + + Args: + name: Unique name to identify the component + component: The component to register (can be any object) + + Example: + ```python + # In the module that defines the component + from core.common import register_component + + class DatabaseConnector: + def connect(self): + pass + + # Register the component + connector = DatabaseConnector() + register_component("db_connector", connector) + ``` + """ + _registry[name] = component + +def get_component(name: str) -> Any: + """ + Get a component from the global registry. + + Args: + name: Name of the component to retrieve + + Returns: + The registered component or None if it doesn't exist + + Example: + ```python + # In another module that needs to use the component + from core.common import get_component + + # Get the component + connector = get_component("db_connector") + if connector: + connector.connect() + ``` + """ + return _registry.get(name) + +def safely_get_component(name: str, default: Optional[T] = None) -> Union[Any, T]: + """ + Safely get a component from the global registry. + + If the component doesn't exist, it returns the provided default + value instead of None. + + Args: + name: Name of the component to retrieve + default: Default value to return if the component doesn't exist + + Returns: + The registered component or the default value + + Example: + ```python + # In another module + from core.common import safely_get_component + + # Get the component with a default value + connector = safely_get_component("db_connector", MyDefaultConnector()) + connector.connect() # Guaranteed not to be None + ``` + """ + component = _registry.get(name) + return component if component is not None else default \ No newline at end of file diff --git a/corebrain/corebrain/core/query.py b/corebrain/corebrain/core/query.py new file mode 100644 index 0000000..b23eddb --- /dev/null +++ b/corebrain/corebrain/core/query.py @@ -0,0 +1,1037 @@ +""" +Components for query handling and analysis. +""" +import os +import json +import time +import re +import sqlite3 +import pickle +import hashlib + +from typing import Dict, Any, List, Optional, Tuple, Callable +from datetime import datetime +from pathlib import Path + +from corebrain.cli.utils import print_colored + +class QueryCache: + """Multilevel cache system for queries.""" + + def __init__(self, cache_dir: str = None, ttl: int = 86400, memory_limit: int = 100): + """ + Initializes the cache system. + + Args: + cache_dir: Directory for persistent cache + ttl: Time-to-live of the cache in seconds (default: 24 hours) + memory_limit: Memory cache entry limit + """ + # Caché en memoria (más rápido, pero volátil) + self.memory_cache = {} + self.memory_timestamps = {} + self.memory_limit = memory_limit + self.memory_lru = [] # Lista para seguimiento de menos usados recientemente + + # Caché persistente (más lento, pero permanente) + self.ttl = ttl + if cache_dir: + self.cache_dir = Path(cache_dir) + else: + self.cache_dir = Path.home() / ".corebrain_cache" + + # Crear directorio de caché si no existe + self.cache_dir.mkdir(parents=True, exist_ok=True) + + # Inicializar base de datos SQLite para metadatos + self.db_path = self.cache_dir / "cache_metadata.db" + self._init_db() + + print_colored(f"Caché inicializado en {self.cache_dir}", "blue") + + def _init_db(self): + """Initializes the SQLite database for cache metadata.""" + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + + # Crear tabla de metadatos si no existe + cursor.execute(''' + CREATE TABLE IF NOT EXISTS cache_metadata ( + query_hash TEXT PRIMARY KEY, + query TEXT, + config_id TEXT, + created_at TIMESTAMP, + last_accessed TIMESTAMP, + hit_count INTEGER DEFAULT 1 + ) + ''') + + conn.commit() + conn.close() + + def _get_hash(self, query: str, config_id: str, collection_name: Optional[str] = None) -> str: + """Generates a unique hash for the query.""" + # Normalizar la consulta (eliminar espacios extra, convertir a minúsculas) + normalized_query = re.sub(r'\s+', ' ', query.lower().strip()) + + # Crear string compuesto para el hash + hash_input = f"{normalized_query}|{config_id}" + if collection_name: + hash_input += f"|{collection_name}" + + # Generar el hash + return hashlib.md5(hash_input.encode()).hexdigest() + + def _get_cache_path(self, query_hash: str) -> Path: + """Gets the cache file path for a given hash.""" + # Usar los primeros caracteres del hash para crear subdirectorios + # Esto evita tener demasiados archivos en un solo directorio + subdir = query_hash[:2] + cache_subdir = self.cache_dir / subdir + cache_subdir.mkdir(exist_ok=True) + + return cache_subdir / f"{query_hash}.cache" + + def _update_metadata(self, query_hash: str, query: str, config_id: str): + """Updates the metadata in the database.""" + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + + now = datetime.now().isoformat() + + # Verificar si el hash ya existe + cursor.execute("SELECT hit_count FROM cache_metadata WHERE query_hash = ?", (query_hash,)) + result = cursor.fetchone() + + if result: + # Actualizar entrada existente + hit_count = result[0] + 1 + cursor.execute(''' + UPDATE cache_metadata + SET last_accessed = ?, hit_count = ? + WHERE query_hash = ? + ''', (now, hit_count, query_hash)) + else: + # Insertar nueva entrada + cursor.execute(''' + INSERT INTO cache_metadata (query_hash, query, config_id, created_at, last_accessed, hit_count) + VALUES (?, ?, ?, ?, ?, 1) + ''', (query_hash, query, config_id, now, now)) + + conn.commit() + conn.close() + + def _update_memory_lru(self, query_hash: str): + """Updates the LRU (Least Recently Used) list for the in-memory cache.""" + if query_hash in self.memory_lru: + # Mover al final (más recientemente usado) + self.memory_lru.remove(query_hash) + + self.memory_lru.append(query_hash) + + # Si excedemos el límite, eliminar el elemento menos usado recientemente + if len(self.memory_lru) > self.memory_limit: + oldest_hash = self.memory_lru.pop(0) + if oldest_hash in self.memory_cache: + del self.memory_cache[oldest_hash] + del self.memory_timestamps[oldest_hash] + + def get(self, query: str, config_id: str, collection_name: Optional[str] = None) -> Optional[Dict[str, Any]]: + """ + Retrieves a cached result if it exists and has not expired. + + Args: + query: Natural language query + config_id: Database configuration ID + collection_name: Name of the collection/table (optional) + + Returns: + Cached result or None if it does not exist or has expired + """ + query_hash = self._get_hash(query, config_id, collection_name) + + # 1. Verificar caché en memoria (más rápido) + if query_hash in self.memory_cache: + timestamp = self.memory_timestamps[query_hash] + if (time.time() - timestamp) < self.ttl: + self._update_memory_lru(query_hash) + self._update_metadata(query_hash, query, config_id) + print_colored(f"Cache hit (memory): {query[:30]}...", "green") + return self.memory_cache[query_hash] + else: + # Expirado en memoria + del self.memory_cache[query_hash] + del self.memory_timestamps[query_hash] + if query_hash in self.memory_lru: + self.memory_lru.remove(query_hash) + + # 2. Verificar caché en disco + cache_path = self._get_cache_path(query_hash) + if cache_path.exists(): + # Verificar edad del archivo + file_age = time.time() - cache_path.stat().st_mtime + if file_age < self.ttl: + try: + with open(cache_path, 'rb') as f: + result = pickle.load(f) + + # Guardar también en caché de memoria + self.memory_cache[query_hash] = result + self.memory_timestamps[query_hash] = time.time() + self._update_memory_lru(query_hash) + self._update_metadata(query_hash, query, config_id) + + print_colored(f"Cache hit (disk): {query[:30]}...", "green") + return result + except Exception as e: + print_colored(f"Error al cargar caché: {str(e)}", "red") + # Si hay error al cargar, eliminar el archivo corrupto + cache_path.unlink(missing_ok=True) + else: + # Archivo expirado, eliminarlo + cache_path.unlink(missing_ok=True) + + return None + + def set(self, query: str, config_id: str, result: Dict[str, Any], collection_name: Optional[str] = None): + """ + Saves a result in the cache. + + Args: + query: Natural language query + config_id: Configuration ID + result: Result to cache + collection_name: Name of the collection/table (optional) + """ + query_hash = self._get_hash(query, config_id, collection_name) + + # 1. Guardar en caché de memoria + self.memory_cache[query_hash] = result + self.memory_timestamps[query_hash] = time.time() + self._update_memory_lru(query_hash) + + # 2. Guardar en caché persistente + try: + cache_path = self._get_cache_path(query_hash) + with open(cache_path, 'wb') as f: + pickle.dump(result, f) + + # 3. Actualizar metadatos + self._update_metadata(query_hash, query, config_id) + + print_colored(f"Cached: {query[:30]}...", "green") + except Exception as e: + print_colored(f"Error al guardar en caché: {str(e)}", "red") + + def clear(self, older_than: int = None): + """ + Clears the cache. + + Args: + older_than: Only clear entries older than this number of seconds + """ + # Limpiar caché en memoria + if older_than: + current_time = time.time() + keys_to_remove = [ + k for k, timestamp in self.memory_timestamps.items() + if (current_time - timestamp) > older_than + ] + + for k in keys_to_remove: + if k in self.memory_cache: + del self.memory_cache[k] + if k in self.memory_timestamps: + del self.memory_timestamps[k] + if k in self.memory_lru: + self.memory_lru.remove(k) + else: + self.memory_cache.clear() + self.memory_timestamps.clear() + self.memory_lru.clear() + + # Limpiar caché en disco + if older_than: + cutoff_time = time.time() - older_than + + # Usar la base de datos para encontrar archivos antiguos + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + + # Convertir cutoff_time a formato ISO + cutoff_datetime = datetime.fromtimestamp(cutoff_time).isoformat() + + cursor.execute( + "SELECT query_hash FROM cache_metadata WHERE last_accessed < ?", + (cutoff_datetime,) + ) + + old_hashes = [row[0] for row in cursor.fetchall()] + + # Eliminar archivos antiguos + for query_hash in old_hashes: + cache_path = self._get_cache_path(query_hash) + if cache_path.exists(): + cache_path.unlink() + + # Eliminar de la base de datos + cursor.execute( + "DELETE FROM cache_metadata WHERE query_hash = ?", + (query_hash,) + ) + + conn.commit() + conn.close() + else: + # Eliminar todos los archivos de caché + for subdir in self.cache_dir.iterdir(): + if subdir.is_dir(): + for cache_file in subdir.glob("*.cache"): + cache_file.unlink() + + # Reiniciar la base de datos + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + cursor.execute("DELETE FROM cache_metadata") + conn.commit() + conn.close() + + def get_stats(self) -> Dict[str, Any]: + """Gets cache statistics.""" + # Contar archivos en disco + disk_count = 0 + for subdir in self.cache_dir.iterdir(): + if subdir.is_dir(): + disk_count += len(list(subdir.glob("*.cache"))) + + # Obtener estadísticas de la base de datos + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + + # Total de entradas + cursor.execute("SELECT COUNT(*) FROM cache_metadata") + total_entries = cursor.fetchone()[0] + + # Consultas más frecuentes + cursor.execute( + "SELECT query, hit_count FROM cache_metadata ORDER BY hit_count DESC LIMIT 5" + ) + top_queries = cursor.fetchall() + + # Edad promedio + cursor.execute( + "SELECT AVG(strftime('%s', 'now') - strftime('%s', created_at)) FROM cache_metadata" + ) + avg_age = cursor.fetchone()[0] + + conn.close() + + return { + "memory_cache_size": len(self.memory_cache), + "disk_cache_size": disk_count, + "total_entries": total_entries, + "top_queries": top_queries, + "average_age_seconds": avg_age, + "cache_directory": str(self.cache_dir) + } + +class QueryTemplate: + """Predefined query template for common patterns.""" + + def __init__(self, pattern: str, description: str, + sql_template: Optional[str] = None, + generator_func: Optional[Callable] = None, + db_type: str = "sql", + applicable_tables: Optional[List[str]] = None): + """ + Initializes a query template. + + Args: + pattern: Natural language pattern that matches this template + description: Description of the template + sql_template: SQL template with placeholders for parameters + generator_func: Alternative function to generate the query + db_type: Database type (sql, mongodb) + applicable_tables: List of tables to which this template applies + """ + self.pattern = pattern + self.description = description + self.sql_template = sql_template + self.generator_func = generator_func + self.db_type = db_type + self.applicable_tables = applicable_tables or [] + + # Compilar expresión regular para el patrón + self.regex = self._compile_pattern(pattern) + + def _compile_pattern(self, pattern: str) -> re.Pattern: + """Compiles the pattern into a regular expression.""" + # Reemplazar marcadores especiales con grupos de captura + regex_pattern = pattern + + # {table} se convierte en grupo de captura para el nombre de tabla + regex_pattern = regex_pattern.replace("{table}", r"(\w+)") + + # {field} se convierte en grupo de captura para el nombre de campo + regex_pattern = regex_pattern.replace("{field}", r"(\w+)") + + # {value} se convierte en grupo de captura para un valor + regex_pattern = regex_pattern.replace("{value}", r"([^,.\s]+)") + + # {number} se convierte en grupo de captura para un número + regex_pattern = regex_pattern.replace("{number}", r"(\d+)") + + # Hacer coincidir el patrón completo + regex_pattern = f"^{regex_pattern}$" + + return re.compile(regex_pattern, re.IGNORECASE) + + def matches(self, query: str) -> Tuple[bool, List[str]]: + """ + Checks if a query matches this template. + + Args: + query: Query to check + + Returns: + Tuple of (match, [captured parameters]) + """ + match = self.regex.match(query) + if match: + return True, list(match.groups()) + return False, [] + + def generate_query(self, params: List[str], db_schema: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """ + Generates a query from the captured parameters. + + Args: + params: Captured parameters from the pattern + db_schema: Database schema + + Returns: + Generated query or None if it cannot be generated + """ + if self.generator_func: + # Usar función personalizada + return self.generator_func(params, db_schema) + + if not self.sql_template: + return None + + # Intentar aplicar la plantilla SQL con los parámetros + try: + sql_query = self.sql_template + + # Reemplazar parámetros en la plantilla + for i, param in enumerate(params): + placeholder = f"${i+1}" + sql_query = sql_query.replace(placeholder, param) + + # Verificar si hay algún parámetro sin reemplazar + if "$" in sql_query: + return None + + return {"sql": sql_query} + except Exception: + return None + +class QueryAnalyzer: + """Analyzes query patterns to suggest optimizations.""" + + def __init__(self, query_log_path: str = None, template_path: str = None): + """ + Initializes the query analyzer. + + Args: + query_log_path: Path to the query log file + template_path: Path to the template file + """ + self.query_log_path = query_log_path or os.path.join( + Path.home(), ".corebrain_cache", "query_log.db" + ) + + self.template_path = template_path or os.path.join( + Path.home(), ".corebrain_cache", "templates.json" + ) + + # Inicializar base de datos + self._init_db() + + # Plantillas predefinidas para consultas comunes + self.templates = self._load_default_templates() + + # Cargar plantillas personalizadas + self._load_custom_templates() + + # Plantillas comunes para identificar patrones + self.common_patterns = [ + r"muestra\s+(?:todos\s+)?los\s+(\w+)", + r"lista\s+(?:de\s+)?(?:todos\s+)?los\s+(\w+)", + r"busca\s+(\w+)\s+donde", + r"cu[aá]ntos\s+(\w+)\s+hay", + r"total\s+de\s+(\w+)" + ] + + def _init_db(self): + """Initializes the database for query logging.""" + # Asegurar que el directorio existe + os.makedirs(os.path.dirname(self.query_log_path), exist_ok=True) + + conn = sqlite3.connect(self.query_log_path) + cursor = conn.cursor() + + # Crear tabla de registro si no existe + cursor.execute(''' + CREATE TABLE IF NOT EXISTS query_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + query TEXT, + config_id TEXT, + collection_name TEXT, + timestamp TIMESTAMP, + execution_time REAL, + cost REAL, + result_count INTEGER, + pattern TEXT + ) + ''') + + # Crear tabla de patrones detectados + cursor.execute(''' + CREATE TABLE IF NOT EXISTS query_patterns ( + pattern TEXT PRIMARY KEY, + count INTEGER, + avg_execution_time REAL, + avg_cost REAL, + last_updated TIMESTAMP + ) + ''') + + conn.commit() + conn.close() + + def _load_default_templates(self) -> List[QueryTemplate]: + """Carga las plantillas predefinidas para consultas comunes.""" + templates = [] + + # Listar todos los registros de una tabla + templates.append( + QueryTemplate( + pattern="muestra todos los {table}", + description="Listar todos los registros de una tabla", + sql_template="SELECT * FROM $1 LIMIT 100", + db_type="sql" + ) + ) + + # Contar registros + templates.append( + QueryTemplate( + pattern="cuántos {table} hay", + description="Contar registros en una tabla", + sql_template="SELECT COUNT(*) FROM $1", + db_type="sql" + ) + ) + + # Buscar por ID + templates.append( + QueryTemplate( + pattern="busca el {table} con id {value}", + description="Buscar registro por ID", + sql_template="SELECT * FROM $1 WHERE id = $2", + db_type="sql" + ) + ) + + # Listar ordenados + templates.append( + QueryTemplate( + pattern="lista los {table} ordenados por {field}", + description="Listar registros ordenados por campo", + sql_template="SELECT * FROM $1 ORDER BY $2 LIMIT 100", + db_type="sql" + ) + ) + + # Buscar por email + templates.append( + QueryTemplate( + pattern="busca el usuario con email {value}", + description="Buscar usuario por email", + sql_template="SELECT * FROM users WHERE email = '$2'", + db_type="sql" + ) + ) + + # Contar por campo + templates.append( + QueryTemplate( + pattern="cuántos {table} hay por {field}", + description="Contar registros agrupados por campo", + sql_template="SELECT $2, COUNT(*) FROM $1 GROUP BY $2", + db_type="sql" + ) + ) + + # Contar usuarios activos + templates.append( + QueryTemplate( + pattern="cuántos usuarios activos hay", + description="Contar usuarios activos", + sql_template="SELECT COUNT(*) FROM users WHERE is_active = TRUE", + db_type="sql", + applicable_tables=["users"] + ) + ) + + # Listar usuarios por fecha de registro + templates.append( + QueryTemplate( + pattern="usuarios registrados en los últimos {number} días", + description="Listar usuarios recientes", + sql_template=""" + SELECT * FROM users + WHERE created_at >= datetime('now', '-$2 days') + ORDER BY created_at DESC + LIMIT 100 + """, + db_type="sql", + applicable_tables=["users"] + ) + ) + + # Buscar empresas + templates.append( + QueryTemplate( + pattern="usuarios que tienen empresa", + description="Buscar usuarios con empresa asignada", + sql_template=""" + SELECT u.* FROM users u + INNER JOIN businesses b ON u.id = b.owner_id + WHERE u.is_business = TRUE + LIMIT 100 + """, + db_type="sql", + applicable_tables=["users", "businesses"] + ) + ) + + # Buscar negocios + templates.append( + QueryTemplate( + pattern="busca negocios en {value}", + description="Buscar negocios por ubicación", + sql_template=""" + SELECT * FROM businesses + WHERE address_city LIKE '%$2%' OR address_province LIKE '%$2%' + LIMIT 100 + """, + db_type="sql", + applicable_tables=["businesses"] + ) + ) + + # MongoDB: Listar documentos + templates.append( + QueryTemplate( + pattern="muestra todos los documentos de {table}", + description="Listar documentos en una colección", + db_type="mongodb", + generator_func=lambda params, schema: { + "collection": params[0], + "operation": "find", + "query": {}, + "limit": 100 + } + ) + ) + + return templates + + def _load_custom_templates(self): + """Loads custom templates from the file.""" + if not os.path.exists(self.template_path): + return + + try: + with open(self.template_path, 'r') as f: + custom_templates = json.load(f) + + for template_data in custom_templates: + # Crear plantilla desde datos JSON + template = QueryTemplate( + pattern=template_data.get("pattern", ""), + description=template_data.get("description", ""), + sql_template=template_data.get("sql_template"), + db_type=template_data.get("db_type", "sql"), + applicable_tables=template_data.get("applicable_tables", []) + ) + + self.templates.append(template) + + except Exception as e: + print_colored(f"Error al cargar plantillas personalizadas: {str(e)}", "red") + + def save_custom_template(self, template: QueryTemplate) -> bool: + """ + Saves a custom template. + + Args: + template: Template to save + + Returns: + True if saved successfully + """ + # Cargar plantillas existentes + custom_templates = [] + if os.path.exists(self.template_path): + try: + with open(self.template_path, 'r') as f: + custom_templates = json.load(f) + except: + custom_templates = [] + + # Convertir plantilla a diccionario + template_data = { + "pattern": template.pattern, + "description": template.description, + "sql_template": template.sql_template, + "db_type": template.db_type, + "applicable_tables": template.applicable_tables + } + + # Verificar si ya existe una plantilla con el mismo patrón + for i, existing in enumerate(custom_templates): + if existing.get("pattern") == template.pattern: + # Actualizar existente + custom_templates[i] = template_data + break + else: + # Agregar nueva + custom_templates.append(template_data) + + # Guardar plantillas + try: + with open(self.template_path, 'w') as f: + json.dump(custom_templates, f, indent=2) + + # Actualizar lista de plantillas + self.templates.append(template) + + return True + except Exception as e: + print_colored(f"Error al guardar plantilla personalizada: {str(e)}", "red") + return False + + def find_matching_template(self, query: str, db_schema: Dict[str, Any]) -> Optional[Tuple[QueryTemplate, List[str]]]: + """ + Searches for a template that matches the query. + + Args: + query: Natural language query + db_schema: Database schema + + Returns: + Tuple of (template, parameters) or None if no match is found + """ + for template in self.templates: + matches, params = template.matches(query) + if matches: + # Verificar si la plantilla es aplicable a las tablas existentes + if template.applicable_tables: + available_tables = set(db_schema.get("tables", {}).keys()) + if not any(table in available_tables for table in template.applicable_tables): + continue + + return template, params + + return None + + def log_query(self, query: str, config_id: str, collection_name: str = None, + execution_time: float = 0, cost: float = 0.09, result_count: int = 0): + """ + Registers a query for analysis. + + Args: + query: Natural language query + config_id: Configuration ID + collection_name: Name of the collection/table + execution_time: Execution time in seconds + cost: Estimated cost of the query + result_count: Number of results obtained + """ + # Detectar patrón + pattern = self._detect_pattern(query) + + # Registrar en la base de datos + conn = sqlite3.connect(self.query_log_path) + cursor = conn.cursor() + + cursor.execute(''' + INSERT INTO query_log (query, config_id, collection_name, timestamp, execution_time, cost, result_count, pattern) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ''', ( + query, config_id, collection_name, datetime.now().isoformat(), + execution_time, cost, result_count, pattern + )) + + # Actualizar estadísticas de patrones + if pattern: + cursor.execute( + "SELECT count, avg_execution_time, avg_cost FROM query_patterns WHERE pattern = ?", + (pattern,) + ) + result = cursor.fetchone() + + if result: + # Actualizar patrón existente + count, avg_exec_time, avg_cost = result + new_count = count + 1 + new_avg_exec_time = (avg_exec_time * count + execution_time) / new_count + new_avg_cost = (avg_cost * count + cost) / new_count + + cursor.execute(''' + UPDATE query_patterns + SET count = ?, avg_execution_time = ?, avg_cost = ?, last_updated = ? + WHERE pattern = ? + ''', (new_count, new_avg_exec_time, new_avg_cost, datetime.now().isoformat(), pattern)) + else: + # Insertar nuevo patrón + cursor.execute(''' + INSERT INTO query_patterns (pattern, count, avg_execution_time, avg_cost, last_updated) + VALUES (?, 1, ?, ?, ?) + ''', (pattern, execution_time, cost, datetime.now().isoformat())) + + conn.commit() + conn.close() + + def _detect_pattern(self, query: str) -> Optional[str]: + """ + Detects a pattern in the query. + + Args: + query: Query to analyze + + Returns: + Detected pattern or None + """ + normalized_query = query.lower() + + # Comprobar patrones predefinidos + for pattern in self.common_patterns: + match = re.search(pattern, normalized_query) + if match: + # Devolver el patrón con comodines + entity = match.group(1) + return pattern.replace(r'(\w+)', f"{entity}") + + # Si no se detecta ningún patrón predefinido, intentar generalizar + words = normalized_query.split() + if len(words) < 3: + return None + + # Intentar generalizar consultas simples + if "mostrar" in words or "muestra" in words or "listar" in words or "lista" in words: + for i, word in enumerate(words): + if word in ["de", "los", "las", "todos", "todas"]: + if i+1 < len(words): + return f"lista_de_{words[i+1]}" + + return None + + def get_common_patterns(self, limit: int = 5) -> List[Dict[str, Any]]: + """ + Retrieves the most common query patterns. + + Args: + limit: Maximum number of patterns to return + + Returns: + List of the most common patterns + """ + conn = sqlite3.connect(self.query_log_path) + cursor = conn.cursor() + + cursor.execute(''' + SELECT pattern, count, avg_execution_time, avg_cost + FROM query_patterns + ORDER BY count DESC + LIMIT ? + ''', (limit,)) + + patterns = [] + for row in cursor.fetchall(): + pattern, count, avg_time, avg_cost = row + patterns.append({ + "pattern": pattern, + "count": count, + "avg_execution_time": avg_time, + "avg_cost": avg_cost, + "estimated_monthly_cost": round(avg_cost * count * 30 / 7, 2) # Estimación mensual + }) + + conn.close() + return patterns + + def suggest_new_template(self, query: str, sql_query: str) -> Optional[QueryTemplate]: + """ + Suggests a new template based on a successful query. + + Args: + query: Natural language query + sql_query: Generated SQL query + + Returns: + Suggested template or None + """ + # Detectar patrón + pattern = self._detect_pattern(query) + if not pattern: + return None + + # Generalizar la consulta SQL + generalized_sql = sql_query + + # Reemplazar valores específicos con marcadores + # Esto es una simplificación, idealmente se usaría un parser SQL + tokens = query.lower().split() + + # Identificar posibles valores a parametrizar + for i, token in enumerate(tokens): + if token.isdigit(): + # Reemplazar números + generalized_sql = re.sub(r'\b' + re.escape(token) + r'\b', '$1', generalized_sql) + pattern = pattern.replace(token, "{number}") + elif '@' in token and '.' in token: + # Reemplazar emails + generalized_sql = re.sub(r'\b' + re.escape(token) + r'\b', '$1', generalized_sql) + pattern = pattern.replace(token, "{value}") + elif token.startswith('"') or token.startswith("'"): + # Reemplazar strings + value = token.strip('"\'') + if len(value) > 2: # Evitar reemplazar strings muy cortos + generalized_sql = re.sub(r'[\'"]' + re.escape(value) + r'[\'"]', "'$1'", generalized_sql) + pattern = pattern.replace(token, "{value}") + + # Crear plantilla + return QueryTemplate( + pattern=pattern, + description=f"Plantilla generada automáticamente para: {pattern}", + sql_template=generalized_sql, + db_type="sql" + ) + + def get_optimization_suggestions(self) -> List[Dict[str, Any]]: + """ + Generates suggestions to optimize queries. + + Returns: + List of optimization suggestions + """ + suggestions = [] + + # Calcular estadísticas generales + conn = sqlite3.connect(self.query_log_path) + cursor = conn.cursor() + + # Total de consultas y costo en los últimos 30 días + cursor.execute(''' + SELECT COUNT(*) as query_count, SUM(cost) as total_cost + FROM query_log + WHERE timestamp > datetime('now', '-30 day') + ''') + + row = cursor.fetchone() + if row: + query_count, total_cost = row + + if query_count and query_count > 100: + # Si hay muchas consultas en total, sugerir plan de volumen + suggestions.append({ + "type": "volume_plan", + "query_count": query_count, + "total_cost": round(total_cost, 2) if total_cost else 0, + "suggestion": f"Considerar negociar un plan por volumen. Actualmente ~{query_count} consultas/mes." + }) + + # Sugerir ajustar el TTL del caché según frecuencia + avg_queries_per_day = query_count / 30 + suggested_ttl = max(3600, min(86400 * 3, 86400 * (100 / avg_queries_per_day))) + + suggestions.append({ + "type": "cache_adjustment", + "current_rate": f"{avg_queries_per_day:.1f} consultas/día", + "suggestion": f"Ajustar TTL del caché a {suggested_ttl/3600:.1f} horas basado en su patrón de uso" + }) + + # Obtener patrones comunes + common_patterns = self.get_common_patterns(10) + + for pattern in common_patterns: + if pattern["count"] >= 5: + # Si un patrón se repite mucho, sugerir precompilación + suggestions.append({ + "type": "precompile", + "pattern": pattern["pattern"], + "count": pattern["count"], + "estimated_savings": round(pattern["avg_cost"] * pattern["count"] * 0.9, 2), # 90% de ahorro + "suggestion": f"Crear una plantilla SQL para consultas del tipo '{pattern['pattern']}'" + }) + + # Si un patrón es costoso pero poco frecuente + if pattern["avg_cost"] > 0.1 and pattern["count"] < 5: + suggestions.append({ + "type": "analyze", + "pattern": pattern["pattern"], + "avg_cost": pattern["avg_cost"], + "suggestion": f"Revisar manualmente consultas del tipo '{pattern['pattern']}' para optimizar" + }) + + # Buscar períodos con alta carga para ajustar parámetros + cursor.execute(''' + SELECT strftime('%Y-%m-%d %H', timestamp) as hour, COUNT(*) as count, SUM(cost) as total_cost + FROM query_log + WHERE timestamp > datetime('now', '-7 day') + GROUP BY hour + ORDER BY count DESC + LIMIT 5 + ''') + + for row in cursor.fetchall(): + hour, count, total_cost = row + if count > 20: # Si hay más de 20 consultas en una hora + suggestions.append({ + "type": "load_balancing", + "hour": hour, + "query_count": count, + "total_cost": round(total_cost, 2), + "suggestion": f"Alta carga de consultas detectada el {hour} ({count} consultas). Considerar técnicas de agrupación." + }) + + # Buscar consultas redundantes (misma consulta en corto tiempo) + cursor.execute(''' + SELECT query, COUNT(*) as count + FROM query_log + WHERE timestamp > datetime('now', '-1 day') + GROUP BY query + HAVING COUNT(*) > 3 + ORDER BY COUNT(*) DESC + LIMIT 5 + ''') + + for row in cursor.fetchall(): + query, count = row + suggestions.append({ + "type": "redundant", + "query": query, + "count": count, + "estimated_savings": round(0.09 * (count - 1), 2), # Ahorro por no repetir + "suggestion": f"Implementar caché para la consulta '{query[:50]}...' que se repitió {count} veces" + }) + + conn.close() + return suggestions + + + \ No newline at end of file diff --git a/corebrain/corebrain/core/test_utils.py b/corebrain/corebrain/core/test_utils.py new file mode 100644 index 0000000..5e334d9 --- /dev/null +++ b/corebrain/corebrain/core/test_utils.py @@ -0,0 +1,157 @@ +""" +Utilities for testing and validating components. +""" +import json +import random +from typing import Dict, Any, Optional + +from corebrain.cli.utils import print_colored +from corebrain.cli.common import DEFAULT_API_URL +from corebrain.network.client import http_session + +def generate_test_question_from_schema(schema: Dict[str, Any]) -> str: + """ + Generates a test question based on the database schema. + + Args: + schema: Database schema + + Returns: + Generated test question + """ + if not schema or not schema.get("tables"): + return "¿Cuáles son las tablas disponibles?" + + tables = schema["tables"] + + if not tables: + return "¿Cuáles son las tablas disponibles?" + + # Seleccionar una tabla aleatoria + table = random.choice(tables) + table_name = table["name"] + + # Determinar el tipo de pregunta + question_types = [ + f"¿Cuántos registros hay en la tabla {table_name}?", + f"Muestra los primeros 5 registros de {table_name}", + f"¿Cuáles son los campos de la tabla {table_name}?", + ] + + # Obtener columnas según la estructura (SQL vs NoSQL) + columns = [] + if "columns" in table and table["columns"]: + columns = table["columns"] + elif "fields" in table and table["fields"]: + columns = table["fields"] + + if columns: + # Si tenemos información de columnas/campos + column_name = columns[0]["name"] if columns else "id" + + # Añadir preguntas específicas con columnas + question_types.extend([ + f"¿Cuál es el valor máximo de {column_name} en {table_name}?", + f"¿Cuáles son los valores únicos de {column_name} en {table_name}?", + ]) + + return random.choice(question_types) + +def test_natural_language_query(api_token: str, db_config: Dict[str, Any], api_url: Optional[str] = None, user_data: Optional[Dict[str, Any]] = None) -> bool: + """ + Tests a natural language query. + + Args: + api_token: API token + db_config: Database configuration + api_url: Optional API URL + user_data: User data + + Returns: + True if the test is successful, False otherwise + """ + try: + print_colored("\nRealizando prueba de consulta en lenguaje natural...", "blue") + + # Importación dinámica para evitar circular imports + from db.schema_file import extract_db_schema + + # Generar una pregunta de prueba basada en el esquema extraído directamente + schema = extract_db_schema(db_config) + print("REcoge esquema: ", schema) + question = generate_test_question_from_schema(schema) + print(f"Pregunta de prueba: {question}") + + # Preparar los datos para la petición + api_url = api_url or DEFAULT_API_URL + if not api_url.startswith(("http://", "https://")): + api_url = "https://" + api_url + + if api_url.endswith('/'): + api_url = api_url[:-1] + + # Construir endpoint para la consulta + endpoint = f"{api_url}/api/database/sdk/query" + + # Datos para la consulta + request_data = { + "question": question, + "db_schema": schema, + "config_id": db_config["config_id"] + } + + # Realizar la petición al API + headers = { + "Authorization": f"Bearer {api_token}", + "Content-Type": "application/json" + } + + timeout = 15.0 # Tiempo máximo de espera reducido + + try: + print_colored("Enviando consulta al API...", "blue") + response = http_session.post( + endpoint, + headers=headers, + json=request_data, + timeout=timeout + ) + + # Verificar la respuesta + if response.status_code == 200: + result = response.json() + + # Verificar si hay explicación en el resultado + if "explanation" in result: + print_colored("\nRespuesta:", "green") + print(result["explanation"]) + + print_colored("\n✅ Prueba de consulta exitosa!", "green") + return True + else: + # Si no hay explicación pero la API responde, puede ser un formato diferente + print_colored("\nRespuesta recibida del API (formato diferente al esperado):", "yellow") + print(json.dumps(result, indent=2)) + print_colored("\n⚠️ La API respondió, pero con un formato diferente al esperado.", "yellow") + return True + else: + print_colored(f"❌ Error en la respuesta: Código {response.status_code}", "red") + try: + error_data = response.json() + print(json.dumps(error_data, indent=2)) + except: + print(response.text[:500]) + return False + + except http_session.TimeoutException: + print_colored("⚠️ Timeout al realizar la consulta. El API puede estar ocupado o no disponible.", "yellow") + print_colored("Esto no afecta a la configuración guardada.", "yellow") + return False + except http_session.RequestError as e: + print_colored(f"⚠️ Error de conexión: {str(e)}", "yellow") + print_colored("Verifica la URL de la API y tu conexión a internet.", "yellow") + return False + + except Exception as e: + print_colored(f"❌ Error al realizar la consulta: {str(e)}", "red") + return False \ No newline at end of file diff --git a/corebrain/corebrain/db/__init__.py b/corebrain/corebrain/db/__init__.py new file mode 100644 index 0000000..6ac390d --- /dev/null +++ b/corebrain/corebrain/db/__init__.py @@ -0,0 +1,26 @@ +""" +Database connectors for Corebrain SDK. + +This package provides connectors for different types and +database engines supported by Corebrain. +""" +from corebrain.db.connector import DatabaseConnector +from corebrain.db.factory import get_connector +from corebrain.db.engines import get_available_engines +from corebrain.db.connectors.sql import SQLConnector +from corebrain.db.connectors.mongodb import MongoDBConnector +from corebrain.db.schema_file import get_schema_with_dynamic_import +from corebrain.db.schema.optimizer import SchemaOptimizer +from corebrain.db.schema.extractor import extract_db_schema + +# Exportación explícita de componentes públicos +__all__ = [ + 'DatabaseConnector', + 'get_connector', + 'get_available_engines', + 'SQLConnector', + 'MongoDBConnector', + 'SchemaOptimizer', + 'extract_db_schema', + 'get_schema_with_dynamic_import' +] \ No newline at end of file diff --git a/corebrain/corebrain/db/connector.py b/corebrain/corebrain/db/connector.py new file mode 100644 index 0000000..886a2a9 --- /dev/null +++ b/corebrain/corebrain/db/connector.py @@ -0,0 +1,33 @@ +""" +Base connectors for different types of databases. +""" +from typing import Dict, Any, List, Optional, Callable + +class DatabaseConnector: + """Base class for all database connectors.""" + + def __init__(self, config: Dict[str, Any], timeout: int = 10): + self.config = config + self.timeout = timeout + self.connection = None + + def connect(self): + """Establishes a connection to the database.""" + raise NotImplementedError + + def extract_schema(self, sample_limit: int = 5, table_limit: Optional[int] = None, + progress_callback: Optional[Callable] = None) -> Dict[str, Any]: + """Extracts the database schema.""" + raise NotImplementedError + + def execute_query(self, query: str) -> List[Dict[str, Any]]: + """Executes a query on the database.""" + raise NotImplementedError + + def close(self): + """Closes the connection.""" + if self.connection: + try: + self.connection.close() + except: + pass \ No newline at end of file diff --git a/corebrain/corebrain/db/connectors/__init__.py b/corebrain/corebrain/db/connectors/__init__.py new file mode 100644 index 0000000..3db5c71 --- /dev/null +++ b/corebrain/corebrain/db/connectors/__init__.py @@ -0,0 +1,28 @@ +""" +Database connectors for different engines. +""" + +from typing import Dict, Any + +from corebrain.db.connectors.sql import SQLConnector +from corebrain.db.connectors.mongodb import MongoDBConnector + +def get_connector(db_config: Dict[str, Any]): + """ + Gets the appropriate connector based on the database configuration. + + Args: + db_config: Database configuration + + Returns: + Instance of the appropriate connector + """ + db_type = db_config.get("type", "").lower() + + if db_type == "sql": + engine = db_config.get("engine", "").lower() + return SQLConnector(db_config, engine) + elif db_type == "nosql" or db_type == "mongodb": + return MongoDBConnector(db_config) + else: + raise ValueError(f"Tipo de base de datos no soportado: {db_type}") \ No newline at end of file diff --git a/corebrain/corebrain/db/connectors/mongodb.py b/corebrain/corebrain/db/connectors/mongodb.py new file mode 100644 index 0000000..1b98575 --- /dev/null +++ b/corebrain/corebrain/db/connectors/mongodb.py @@ -0,0 +1,474 @@ +""" +Connector for MongoDB databases. +""" + +import time +import json +import re + +from typing import Dict, Any, List, Optional, Callable, Tuple + +try: + import pymongo + from pymongo.errors import ConnectionFailure, ServerSelectionTimeoutError + PYMONGO_AVAILABLE = True +except ImportError: + PYMONGO_AVAILABLE = False + +from corebrain.db.connector import DatabaseConnector + +class MongoDBConnector(DatabaseConnector): + """Optimized connector for MongoDB.""" + + def __init__(self, config: Dict[str, Any]): + """ + Initializes the MongoDB connector with the provided configuration. + + Args: + config: Dictionary with the connection configuration + """ + super().__init__(config) + self.client = None + self.db = None + self.config = config + self.connection_timeout = 30 # segundos + + if not PYMONGO_AVAILABLE: + print("Advertencia: pymongo no está instalado. Instálalo con 'pip install pymongo'") + + def connect(self) -> bool: + """ + Establishes a connection with optimized timeout. + + Returns: + True if the connection was successful, False otherwise + """ + if not PYMONGO_AVAILABLE: + raise ImportError("pymongo no está instalado. Instálalo con 'pip install pymongo'") + + try: + start_time = time.time() + + # Construir los parámetros de conexión + if "connection_string" in self.config: + connection_string = self.config["connection_string"] + # Añadir timeout a la cadena de conexión si no está presente + if "connectTimeoutMS=" not in connection_string: + if "?" in connection_string: + connection_string += "&connectTimeoutMS=10000" # 10 segundos + else: + connection_string += "?connectTimeoutMS=10000" + + # Crear cliente MongoDB con la cadena de conexión + self.client = pymongo.MongoClient(connection_string) + else: + # Diccionario de parámetros para MongoClient + mongo_params = { + "host": self.config.get("host", "localhost"), + "port": int(self.config.get("port", 27017)), + "connectTimeoutMS": 10000, # 10 segundos + "serverSelectionTimeoutMS": 10000 + } + + # Añadir credenciales solo si están presentes + if self.config.get("user"): + mongo_params["username"] = self.config.get("user") + if self.config.get("password"): + mongo_params["password"] = self.config.get("password") + + # Opcionalmente añadir opciones de autenticación + if self.config.get("auth_source"): + mongo_params["authSource"] = self.config.get("auth_source") + if self.config.get("auth_mechanism"): + mongo_params["authMechanism"] = self.config.get("auth_mechanism") + + # Crear cliente MongoDB con parámetros + self.client = pymongo.MongoClient(**mongo_params) + + # Verificar que la conexión funciona + self.client.admin.command('ping') + + # Seleccionar la base de datos + db_name = self.config.get("database", "") + if not db_name: + # Si no hay base de datos especificada, listar las disponibles + db_names = self.client.list_database_names() + if not db_names: + raise ValueError("No se encontraron bases de datos disponibles") + + # Seleccionar la primera que no sea de sistema + system_dbs = ["admin", "local", "config"] + for name in db_names: + if name not in system_dbs: + db_name = name + break + + # Si no encontramos ninguna que no sea de sistema, usar la primera + if not db_name: + db_name = db_names[0] + + print(f"No se especificó base de datos. Usando '{db_name}'") + + # Guardar la referencia a la base de datos + self.db = self.client[db_name] + return True + + except (ConnectionFailure, ServerSelectionTimeoutError) as e: + # Si es un error de timeout, reintentar + if time.time() - start_time < self.connection_timeout: + print(f"Timeout al conectar a MongoDB: {str(e)}. Reintentando...") + time.sleep(2) # Esperar antes de reintentar + return self.connect() + else: + print(f"Error de conexión a MongoDB después de {self.connection_timeout}s: {str(e)}") + self.close() + return False + except Exception as e: + print(f"Error al conectar a MongoDB: {str(e)}") + self.close() + return False + + def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] = None, + progress_callback: Optional[Callable] = None) -> Dict[str, Any]: + """ + Extracts the schema with limits and progress to improve performance. + + Args: + sample_limit: Maximum number of sample documents per collection + collection_limit: Limit of collections to process (None for all) + progress_callback: Optional function to report progress + + Returns: + Dictionary with the database schema + """ + # Asegurar que estamos conectados + if not self.client and not self.connect(): + return {"type": "mongodb", "tables": {}, "tables_list": []} + + # Inicializar el esquema + schema = { + "type": "mongodb", + "database": self.db.name, + "tables": {} # En MongoDB, las "tablas" son colecciones + } + + try: + # Obtener la lista de colecciones + collections = self.db.list_collection_names() + + # Limitar colecciones si es necesario + if collection_limit is not None and collection_limit > 0: + collections = collections[:collection_limit] + + # Procesar cada colección + total_collections = len(collections) + for i, collection_name in enumerate(collections): + # Reportar progreso si hay callback + if progress_callback: + progress_callback(i, total_collections, f"Procesando colección {collection_name}") + + collection = self.db[collection_name] + + try: + # Contar documentos + doc_count = collection.count_documents({}) + + if doc_count > 0: + # Obtener muestra de documentos + sample_docs = list(collection.find().limit(sample_limit)) + + # Extraer campos y sus tipos + fields = {} + for doc in sample_docs: + self._extract_document_fields(doc, fields) + + # Convertir a formato esperado + formatted_fields = [{"name": field, "type": type_name} for field, type_name in fields.items()] + + # Procesar documentos para sample_data + sample_data = [] + for doc in sample_docs: + processed_doc = self._process_document_for_serialization(doc) + sample_data.append(processed_doc) + + # Guardar en el esquema + schema["tables"][collection_name] = { + "fields": formatted_fields, + "sample_data": sample_data, + "count": doc_count + } + else: + # Colección vacía + schema["tables"][collection_name] = { + "fields": [], + "sample_data": [], + "count": 0, + "empty": True + } + + except Exception as e: + print(f"Error al procesar colección {collection_name}: {str(e)}") + schema["tables"][collection_name] = { + "fields": [], + "error": str(e) + } + + # Crear la lista de tablas/colecciones para compatibilidad + table_list = [] + for collection_name, collection_info in schema["tables"].items(): + table_data = {"name": collection_name} + table_data.update(collection_info) + table_list.append(table_data) + + # Guardar también la lista de tablas para compatibilidad + schema["tables_list"] = table_list + + return schema + + except Exception as e: + print(f"Error al extraer el esquema MongoDB: {str(e)}") + return {"type": "mongodb", "tables": {}, "tables_list": []} + + def _extract_document_fields(self, doc: Dict[str, Any], fields: Dict[str, str], + prefix: str = "", max_depth: int = 3, current_depth: int = 0) -> None: + """ + Recursively extracts fields and types from a MongoDB document. + + Args: + doc: Document to analyze + fields: Dictionary to store fields and types + prefix: Prefix for nested fields + max_depth: Maximum depth for nested fields + current_depth: Current depth + """ + if current_depth >= max_depth: + return + + for field, value in doc.items(): + # Para _id y otros campos especiales + if field == "_id": + field_type = "ObjectId" + elif isinstance(value, dict): + if current_depth < max_depth - 1: + # Recursión para campos anidados + self._extract_document_fields(value, fields, + f"{prefix}{field}.", max_depth, current_depth + 1) + field_type = "object" + elif isinstance(value, list): + if value and current_depth < max_depth - 1: + # Si tenemos elementos en la lista, analizar el primero + if isinstance(value[0], dict): + self._extract_document_fields(value[0], fields, + f"{prefix}{field}[].", max_depth, current_depth + 1) + else: + # Para listas de tipos primitivos + field_type = f"array<{type(value[0]).__name__}>" + else: + field_type = "array" + else: + field_type = type(value).__name__ + + # Guardar el tipo del campo actual + field_key = f"{prefix}{field}" + if field_key not in fields: + fields[field_key] = field_type + + def _process_document_for_serialization(self, doc: Dict[str, Any]) -> Dict[str, Any]: + """ + Processes a document to be JSON serializable. + + Args: + doc: Document to process + + Returns: + Processed document + """ + processed_doc = {} + for field, value in doc.items(): + # Convertir ObjectId a string + if field == "_id": + processed_doc[field] = str(value) + # Manejar objetos anidados + elif isinstance(value, dict): + processed_doc[field] = self._process_document_for_serialization(value) + # Manejar arrays + elif isinstance(value, list): + processed_items = [] + for item in value: + if isinstance(item, dict): + processed_items.append(self._process_document_for_serialization(item)) + elif hasattr(item, "__str__"): + processed_items.append(str(item)) + else: + processed_items.append(item) + processed_doc[field] = processed_items + # Convertir fechas a ISO + elif hasattr(value, 'isoformat'): + processed_doc[field] = value.isoformat() + # Otros tipos de datos + else: + processed_doc[field] = value + + return processed_doc + + def execute_query(self, query: str) -> List[Dict[str, Any]]: + """ + Executes a MongoDB query with improved error handling. + + Args: + query: MongoDB query in JSON format or query language + + Returns: + List of resulting documents + """ + if not self.client and not self.connect(): + raise ConnectionError("No se pudo establecer conexión con MongoDB") + + try: + # Determinar si la consulta es un string JSON o una consulta en otro formato + filter_dict, projection, collection_name, limit = self._parse_query(query) + + # Obtener la colección + if not collection_name: + raise ValueError("No se especificó el nombre de la colección en la consulta") + + collection = self.db[collection_name] + + # Ejecutar la consulta + if projection: + cursor = collection.find(filter_dict, projection).limit(limit or 100) + else: + cursor = collection.find(filter_dict).limit(limit or 100) + + # Convertir los resultados a formato serializable + results = [] + for doc in cursor: + processed_doc = self._process_document_for_serialization(doc) + results.append(processed_doc) + + return results + + except Exception as e: + # Intentar reconectar y reintentar una vez + try: + self.close() + if self.connect(): + print("Reconectando y reintentando consulta...") + + # Reintentar la consulta + filter_dict, projection, collection_name, limit = self._parse_query(query) + collection = self.db[collection_name] + + if projection: + cursor = collection.find(filter_dict, projection).limit(limit or 100) + else: + cursor = collection.find(filter_dict).limit(limit or 100) + + results = [] + for doc in cursor: + processed_doc = self._process_document_for_serialization(doc) + results.append(processed_doc) + + return results + except Exception as retry_error: + # Si falla el reintento, propagar el error original + raise Exception(f"Error al ejecutar consulta MongoDB: {str(e)}") + + # Si llegamos aquí, ha habido un error en el reintento + raise Exception(f"Error al ejecutar consulta MongoDB (después de reconexión): {str(e)}") + + def _parse_query(self, query: str) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]], str, Optional[int]]: + """ + Analyzes a query and extracts the necessary components. + + Args: + query: Query in string format + + Returns: + Tuple with (filter, projection, collection name, limit) + """ + # Intentar parsear como JSON + try: + query_dict = json.loads(query) + + # Extraer componentes de la consulta + filter_dict = query_dict.get("filter", {}) + projection = query_dict.get("projection") + collection_name = query_dict.get("collection") + limit = query_dict.get("limit") + + return filter_dict, projection, collection_name, limit + + except json.JSONDecodeError: + # Si no es JSON válido, intentar parsear el formato de consulta alternativo + collection_match = re.search(r'from\s+([a-zA-Z0-9_]+)', query, re.IGNORECASE) + collection_name = collection_match.group(1) if collection_match else None + + # Intentar extraer filtros + filter_match = re.search(r'where\s+(.+?)(?:limit|$)', query, re.IGNORECASE | re.DOTALL) + filter_str = filter_match.group(1).strip() if filter_match else "{}" + + # Intentar parsear los filtros como JSON + try: + filter_dict = json.loads(filter_str) + except json.JSONDecodeError: + # Si no se puede parsear, usar filtro vacío + filter_dict = {} + + # Extraer límite si existe + limit_match = re.search(r'limit\s+(\d+)', query, re.IGNORECASE) + limit = int(limit_match.group(1)) if limit_match else None + + return filter_dict, None, collection_name, limit + + def count_documents(self, collection_name: str, filter_dict: Optional[Dict[str, Any]] = None) -> int: + """ + Counts documents in a collection. + + Args: + collection_name: Name of the collection + filter_dict: Optional filter + + Returns: + Number of documents + """ + if not self.client and not self.connect(): + raise ConnectionError("No se pudo establecer conexión con MongoDB") + + try: + collection = self.db[collection_name] + return collection.count_documents(filter_dict or {}) + except Exception as e: + print(f"Error al contar documentos: {str(e)}") + return 0 + + def list_collections(self) -> List[str]: + """ + Returns a list of collections in the database. + + Returns: + List of collection names + """ + if not self.client and not self.connect(): + raise ConnectionError("No se pudo establecer conexión con MongoDB") + + try: + return self.db.list_collection_names() + except Exception as e: + print(f"Error al listar colecciones: {str(e)}") + return [] + + def close(self) -> None: + """Closes the MongoDB connection.""" + if self.client: + try: + self.client.close() + except: + pass + finally: + self.client = None + self.db = None + + def __del__(self): + """Destructor to ensure the connection is closed.""" + self.close() \ No newline at end of file diff --git a/corebrain/corebrain/db/connectors/sql.py b/corebrain/corebrain/db/connectors/sql.py new file mode 100644 index 0000000..82f49bf --- /dev/null +++ b/corebrain/corebrain/db/connectors/sql.py @@ -0,0 +1,598 @@ +""" +Conector para bases de datos SQL. +""" +import sqlite3 +import time +from typing import Dict, Any, List, Optional, Callable + +try: + import mysql.connector +except ImportError: + pass + +try: + import psycopg2 + import psycopg2.extras +except ImportError: + pass + +from corebrain.db.connector import DatabaseConnector + +class SQLConnector(DatabaseConnector): + """Optimized connector for SQL databases.""" + + def __init__(self, config: Dict[str, Any]): + """ + Initializes the SQL connector with the provided configuration. + + Args: + config: Dictionary with the connection configuration + """ + super().__init__(config) + self.conn = None + self.cursor = None + self.engine = config.get("engine", "").lower() + self.config = config + self.connection_timeout = 30 # segundos + + def connect(self) -> bool: + """ + Establishes a connection with optimized timeout. + + Returns: + True if the connection was successful, False otherwise + """ + try: + start_time = time.time() + + # Intentar la conexión con un límite de tiempo + while time.time() - start_time < self.connection_timeout: + try: + if self.engine == "sqlite": + if "connection_string" in self.config: + self.conn = sqlite3.connect(self.config["connection_string"], timeout=10.0) + else: + self.conn = sqlite3.connect(self.config.get("database", ""), timeout=10.0) + + # Configurar para que devuelva filas como diccionarios + self.conn.row_factory = sqlite3.Row + + elif self.engine == "mysql": + if "connection_string" in self.config: + self.conn = mysql.connector.connect( + connection_string=self.config["connection_string"], + connection_timeout=10 + ) + else: + self.conn = mysql.connector.connect( + host=self.config.get("host", "localhost"), + user=self.config.get("user", ""), + password=self.config.get("password", ""), + database=self.config.get("database", ""), + port=self.config.get("port", 3306), + connection_timeout=10 + ) + + elif self.engine == "postgresql": + # Determinar si usar cadena de conexión o parámetros + if "connection_string" in self.config: + # Agregar timeout a la cadena de conexión si no está presente + conn_str = self.config["connection_string"] + if "connect_timeout" not in conn_str: + if "?" in conn_str: + conn_str += "&connect_timeout=10" + else: + conn_str += "?connect_timeout=10" + + self.conn = psycopg2.connect(conn_str) + else: + self.conn = psycopg2.connect( + host=self.config.get("host", "localhost"), + user=self.config.get("user", ""), + password=self.config.get("password", ""), + dbname=self.config.get("database", ""), + port=self.config.get("port", 5432), + connect_timeout=10 + ) + + # Si llegamos aquí, la conexión fue exitosa + if self.conn: + # Verificar conexión con una consulta simple + cursor = self.conn.cursor() + cursor.execute("SELECT 1") + cursor.close() + return True + + except (sqlite3.Error, mysql.connector.Error, psycopg2.Error) as e: + # Si el error no es de timeout, propagar la excepción + if "timeout" not in str(e).lower() and "tiempo de espera" not in str(e).lower(): + raise + + # Si es un error de timeout, esperamos un poco y reintentamos + time.sleep(1.0) + + # Si llegamos aquí, se agotó el tiempo de espera + raise TimeoutError(f"No se pudo conectar a la base de datos en {self.connection_timeout} segundos") + + except Exception as e: + if self.conn: + try: + self.conn.close() + except: + pass + self.conn = None + + print(f"Error al conectar a la base de datos: {str(e)}") + return False + + def extract_schema(self, sample_limit: int = 5, table_limit: Optional[int] = None, + progress_callback: Optional[Callable] = None) -> Dict[str, Any]: + """ + Extracts the schema with limits and progress. + + Args: + sample_limit: Data sample limit per table + table_limit: Limit of tables to process (None for all) + progress_callback: Optional function to report progress + + Returns: + Dictionary with the database schema + """ + # Asegurar que estamos conectados + if not self.conn and not self.connect(): + return {"type": "sql", "tables": {}, "tables_list": []} + + # Inicializar esquema + schema = { + "type": "sql", + "engine": self.engine, + "database": self.config.get("database", ""), + "tables": {} + } + + # Seleccionar la función extractora según el motor + if self.engine == "sqlite": + return self._extract_sqlite_schema(sample_limit, table_limit, progress_callback) + elif self.engine == "mysql": + return self._extract_mysql_schema(sample_limit, table_limit, progress_callback) + elif self.engine == "postgresql": + return self._extract_postgresql_schema(sample_limit, table_limit, progress_callback) + else: + return schema # Esquema vacío si no se reconoce el motor + + def execute_query(self, query: str) -> List[Dict[str, Any]]: + """ + Executes an SQL query with improved error handling. + + Args: + query: SQL query to execute + + Returns: + List of resulting rows as dictionaries + """ + if not self.conn and not self.connect(): + raise ConnectionError("No se pudo establecer conexión con la base de datos") + + try: + # Ejecutar query según el motor + if self.engine == "sqlite": + return self._execute_sqlite_query(query) + elif self.engine == "mysql": + return self._execute_mysql_query(query) + elif self.engine == "postgresql": + return self._execute_postgresql_query(query) + else: + raise ValueError(f"Motor de base de datos no soportado: {self.engine}") + + except Exception as e: + # Intentar reconectar y reintentar una vez + try: + self.close() + if self.connect(): + print("Reconectando y reintentando consulta...") + + if self.engine == "sqlite": + return self._execute_sqlite_query(query) + elif self.engine == "mysql": + return self._execute_mysql_query(query) + elif self.engine == "postgresql": + return self._execute_postgresql_query(query) + + except Exception as retry_error: + # Si falla el reintento, propagar el error original + raise Exception(f"Error al ejecutar consulta: {str(e)}") + + # Si llegamos aquí sin retornar, ha habido un error en el reintento + raise Exception(f"Error al ejecutar consulta (después de reconexión): {str(e)}") + + def _execute_sqlite_query(self, query: str) -> List[Dict[str, Any]]: + """Executes a query in SQLite.""" + cursor = self.conn.cursor() + cursor.execute(query) + + # Convertir filas a diccionarios + columns = [desc[0] for desc in cursor.description] if cursor.description else [] + rows = cursor.fetchall() + result = [] + + for row in rows: + row_dict = {} + for i, column in enumerate(columns): + row_dict[column] = row[i] + result.append(row_dict) + + cursor.close() + return result + + def _execute_mysql_query(self, query: str) -> List[Dict[str, Any]]: + """Executes a query in MySQL.""" + cursor = self.conn.cursor(dictionary=True) + cursor.execute(query) + result = cursor.fetchall() + cursor.close() + return result + + def _execute_postgresql_query(self, query: str) -> List[Dict[str, Any]]: + """Executes a query in PostgreSQL.""" + cursor = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + cursor.execute(query) + results = [dict(row) for row in cursor.fetchall()] + cursor.close() + return results + + def _extract_sqlite_schema(self, sample_limit: int, table_limit: Optional[int], progress_callback: Optional[Callable]) -> Dict[str, Any]: + """ + Extracts specific schema for SQLite. + + Args: + sample_limit: Maximum number of sample rows per table + table_limit: Maximum number of tables to extract + progress_callback: Function to report progress + + Returns: + Dictionary with the database schema + """ + schema = { + "type": "sql", + "engine": "sqlite", + "database": self.config.get("database", ""), + "tables": {} + } + + try: + cursor = self.conn.cursor() + + # Obtener la lista de tablas + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;") + tables = [row[0] for row in cursor.fetchall()] + + # Limitar tablas si es necesario + if table_limit is not None and table_limit > 0: + tables = tables[:table_limit] + + # Procesar cada tabla + total_tables = len(tables) + for i, table_name in enumerate(tables): + # Reportar progreso si hay callback + if progress_callback: + progress_callback(i, total_tables, f"Procesando tabla {table_name}") + + # Extraer información de columnas + cursor.execute(f"PRAGMA table_info({table_name});") + columns = [{"name": col[1], "type": col[2]} for col in cursor.fetchall()] + + # Guardar información básica de la tabla + schema["tables"][table_name] = { + "columns": columns, + "sample_data": [] + } + + # Obtener muestra de datos + try: + cursor.execute(f"SELECT * FROM {table_name} LIMIT {sample_limit};") + + # Obtener nombres de columnas + col_names = [desc[0] for desc in cursor.description] + + # Procesar las filas + sample_data = [] + for row in cursor.fetchall(): + row_dict = {} + for j, value in enumerate(row): + # Convertir a string los valores que no son serializable directamente + if isinstance(value, (bytes, bytearray)): + row_dict[col_names[j]] = f"" + else: + row_dict[col_names[j]] = value + sample_data.append(row_dict) + + schema["tables"][table_name]["sample_data"] = sample_data + + except Exception as e: + print(f"Error al obtener muestra de datos para tabla {table_name}: {str(e)}") + + cursor.close() + + except Exception as e: + print(f"Error al extraer esquema SQLite: {str(e)}") + + # Crear la lista de tablas para compatibilidad + table_list = [] + for table_name, table_info in schema["tables"].items(): + table_data = {"name": table_name} + table_data.update(table_info) + table_list.append(table_data) + + schema["tables_list"] = table_list + return schema + + def _extract_mysql_schema(self, sample_limit: int, table_limit: Optional[int], progress_callback: Optional[Callable]) -> Dict[str, Any]: + """ + Extracts specific schema for MySQL. + + Args: + sample_limit: Maximum number of sample rows per table + table_limit: Maximum number of tables to extract + progress_callback: Function to report progress + + Returns: + Dictionary with the database schema + """ + schema = { + "type": "sql", + "engine": "mysql", + "database": self.config.get("database", ""), + "tables": {} + } + + try: + cursor = self.conn.cursor(dictionary=True) + + # Obtener la lista de tablas + cursor.execute("SHOW TABLES;") + tables_result = cursor.fetchall() + tables = [] + + # Extraer nombres de tablas (el formato puede variar según versión) + for row in tables_result: + if len(row) == 1: # Si es una lista simple + tables.extend(row.values()) + else: # Si tiene estructura compleja + for value in row.values(): + if isinstance(value, str): + tables.append(value) + break + + # Limitar tablas si es necesario + if table_limit is not None and table_limit > 0: + tables = tables[:table_limit] + + # Procesar cada tabla + total_tables = len(tables) + for i, table_name in enumerate(tables): + # Reportar progreso si hay callback + if progress_callback: + progress_callback(i, total_tables, f"Procesando tabla {table_name}") + + # Extraer información de columnas + cursor.execute(f"DESCRIBE `{table_name}`;") + columns = [{"name": col.get("Field"), "type": col.get("Type")} for col in cursor.fetchall()] + + # Guardar información básica de la tabla + schema["tables"][table_name] = { + "columns": columns, + "sample_data": [] + } + + # Obtener muestra de datos + try: + cursor.execute(f"SELECT * FROM `{table_name}` LIMIT {sample_limit};") + sample_data = cursor.fetchall() + + # Procesar valores que no son JSON serializable + processed_samples = [] + for row in sample_data: + processed_row = {} + for key, value in row.items(): + if isinstance(value, (bytes, bytearray)): + processed_row[key] = f"" + elif hasattr(value, 'isoformat'): # Para fechas y horas + processed_row[key] = value.isoformat() + else: + processed_row[key] = value + processed_samples.append(processed_row) + + schema["tables"][table_name]["sample_data"] = processed_samples + + except Exception as e: + print(f"Error al obtener muestra de datos para tabla {table_name}: {str(e)}") + + cursor.close() + + except Exception as e: + print(f"Error al extraer esquema MySQL: {str(e)}") + + # Crear la lista de tablas para compatibilidad + table_list = [] + for table_name, table_info in schema["tables"].items(): + table_data = {"name": table_name} + table_data.update(table_info) + table_list.append(table_data) + + schema["tables_list"] = table_list + return schema + + def _extract_postgresql_schema(self, sample_limit: int, table_limit: Optional[int], progress_callback: Optional[Callable]) -> Dict[str, Any]: + """ + Extracts specific schema for PostgreSQL with optimizations. + + Args: + sample_limit: Maximum number of sample rows per table + table_limit: Maximum number of tables to extract + progress_callback: Function to report progress + + Returns: + Dictionary with the database schema + """ + schema = { + "type": "sql", + "engine": "postgresql", + "database": self.config.get("database", ""), + "tables": {} + } + + try: + cursor = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + + # Estrategia 1: Buscar en todos los esquemas accesibles + cursor.execute(""" + SELECT table_schema, table_name + FROM information_schema.tables + WHERE table_schema NOT IN ('pg_catalog', 'information_schema') + AND table_type = 'BASE TABLE' + ORDER BY table_schema, table_name; + """) + tables = cursor.fetchall() + + # Si no se encontraron tablas, intentar estrategia alternativa + if not tables: + cursor.execute(""" + SELECT schemaname AS table_schema, tablename AS table_name + FROM pg_tables + WHERE schemaname NOT IN ('pg_catalog', 'information_schema') + ORDER BY schemaname, tablename; + """) + tables = cursor.fetchall() + + # Si aún no hay tablas, intentar buscar en esquemas específicos + if not tables: + cursor.execute(""" + SELECT DISTINCT table_schema + FROM information_schema.tables + ORDER BY table_schema; + """) + schemas = cursor.fetchall() + + # Intentar con esquemas que no sean del sistema + user_schemas = [s[0] for s in schemas if s[0] not in ('pg_catalog', 'information_schema')] + for schema_name in user_schemas: + cursor.execute(f""" + SELECT '{schema_name}' AS table_schema, table_name + FROM information_schema.tables + WHERE table_schema = '{schema_name}' + AND table_type = 'BASE TABLE'; + """) + schema_tables = cursor.fetchall() + if schema_tables: + tables.extend(schema_tables) + + # Limitar tablas si es necesario + if table_limit is not None and table_limit > 0: + tables = tables[:table_limit] + + # Procesar cada tabla + total_tables = len(tables) + for i, (schema_name, table_name) in enumerate(tables): + # Reportar progreso si hay callback + if progress_callback: + progress_callback(i, total_tables, f"Procesando tabla {schema_name}.{table_name}") + + # Determinar el nombre completo de la tabla + full_name = f"{schema_name}.{table_name}" if schema_name != 'public' else table_name + + # Extraer información de columnas + cursor.execute(f""" + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_schema = '{schema_name}' AND table_name = '{table_name}' + ORDER BY ordinal_position; + """) + + columns_data = cursor.fetchall() + if columns_data: + columns = [{"name": col[0], "type": col[1]} for col in columns_data] + schema["tables"][full_name] = {"columns": columns, "sample_data": []} + + # Obtener muestra de datos + try: + cursor.execute(f""" + SELECT * FROM "{schema_name}"."{table_name}" LIMIT {sample_limit}; + """) + rows = cursor.fetchall() + + # Obtener nombres de columnas + col_names = [desc[0] for desc in cursor.description] + + # Convertir filas a diccionarios + sample_data = [] + for row in rows: + row_dict = {} + for j, value in enumerate(row): + # Convertir a formato serializable + if hasattr(value, 'isoformat'): # Para fechas y horas + row_dict[col_names[j]] = value.isoformat() + elif isinstance(value, (bytes, bytearray)): + row_dict[col_names[j]] = f"" + else: + row_dict[col_names[j]] = str(value) if value is not None else None + sample_data.append(row_dict) + + schema["tables"][full_name]["sample_data"] = sample_data + + except Exception as e: + print(f"Error al obtener muestra de datos para tabla {full_name}: {str(e)}") + else: + # Registrar la tabla aunque no tenga columnas + schema["tables"][full_name] = {"columns": [], "sample_data": []} + + cursor.close() + + except Exception as e: + print(f"Error al extraer esquema PostgreSQL: {str(e)}") + + # Intento de recuperación para diagnosticar problemas + try: + if self.conn and self.conn.closed == 0: # 0 = conexión abierta + recovery_cursor = self.conn.cursor() + + # Verificar versión + recovery_cursor.execute("SELECT version();") + version = recovery_cursor.fetchone() + print(f"Versión PostgreSQL: {version[0] if version else 'Desconocida'}") + + # Verificar permisos + recovery_cursor.execute(""" + SELECT has_schema_privilege(current_user, 'public', 'USAGE') AS has_usage, + has_schema_privilege(current_user, 'public', 'CREATE') AS has_create; + """) + perms = recovery_cursor.fetchone() + if perms: + print(f"Permisos en esquema public: USAGE={perms[0]}, CREATE={perms[1]}") + + recovery_cursor.close() + except Exception as diag_err: + print(f"Error durante el diagnóstico: {str(diag_err)}") + + # Crear la lista de tablas para compatibilidad + table_list = [] + for table_name, table_info in schema["tables"].items(): + table_data = {"name": table_name} + table_data.update(table_info) + table_list.append(table_data) + + schema["tables_list"] = table_list + return schema + + def close(self) -> None: + """Closes the database connection.""" + if self.conn: + try: + self.conn.close() + except: + pass + finally: + self.conn = None + + def __del__(self): + """Destructor to ensure the connection is closed.""" + self.close() \ No newline at end of file diff --git a/corebrain/corebrain/db/engines.py b/corebrain/corebrain/db/engines.py new file mode 100644 index 0000000..51b51d6 --- /dev/null +++ b/corebrain/corebrain/db/engines.py @@ -0,0 +1,16 @@ +""" +Information about supported database engines. +""" +from typing import Dict, List + +def get_available_engines() -> Dict[str, List[str]]: + """ + Returns the available database engines by type. + + Returns: + Dict with DB types and a list of engines per type + """ + return { + "sql": ["sqlite", "mysql", "postgresql"], + "nosql": ["mongodb"] + } \ No newline at end of file diff --git a/corebrain/corebrain/db/factory.py b/corebrain/corebrain/db/factory.py new file mode 100644 index 0000000..c2c23bc --- /dev/null +++ b/corebrain/corebrain/db/factory.py @@ -0,0 +1,29 @@ +""" +Database connector factory. +""" +from typing import Dict, Any + +from corebrain.db.connector import DatabaseConnector +from corebrain.db.connectors.sql import SQLConnector +from corebrain.db.connectors.mongodb import MongoDBConnector + +def get_connector(db_config: Dict[str, Any], timeout: int = 10) -> DatabaseConnector: + """ + Database connector factory based on configuration. + + Args: + db_config: Database configuration + timeout: Timeout for DB operations + + Returns: + Instance of the appropriate connector + """ + db_type = db_config.get("type", "").lower() + engine = db_config.get("engine", "").lower() + + if db_type == "sql": + return SQLConnector(db_config, timeout) + elif db_type in ["nosql", "mongodb"] or engine == "mongodb": + return MongoDBConnector(db_config, timeout) + else: + raise ValueError(f"Tipo de base de datos no soportado: {db_type}") \ No newline at end of file diff --git a/corebrain/corebrain/db/interface.py b/corebrain/corebrain/db/interface.py new file mode 100644 index 0000000..d8373ff --- /dev/null +++ b/corebrain/corebrain/db/interface.py @@ -0,0 +1,36 @@ +""" +Abstract interfaces for database connections. +""" +from typing import Dict, Any, List, Optional, Protocol +from abc import ABC, abstractmethod + +from corebrain.core.common import ConfigDict, SchemaDict + +class DatabaseConnector(ABC): + """Abstract interface for database connectors.""" + + @abstractmethod + def connect(self, config: ConfigDict) -> Any: + """Establishes a connection with the database.""" + pass + + @abstractmethod + def extract_schema(self, connection: Any) -> SchemaDict: + """Extracts the database schema.""" + pass + + @abstractmethod + def execute_query(self, connection: Any, query: str) -> List[Dict[str, Any]]: + """Executes a query and returns results.""" + pass + + @abstractmethod + def close(self, connection: Any) -> None: + """Closes the connection.""" + pass + +# Posteriormente se podrían implementar conectores específicos: +# - SQLiteConnector +# - MySQLConnector +# - PostgresConnector +# - MongoDBConnector \ No newline at end of file diff --git a/corebrain/corebrain/db/schema/__init__.py b/corebrain/corebrain/db/schema/__init__.py new file mode 100644 index 0000000..388ee63 --- /dev/null +++ b/corebrain/corebrain/db/schema/__init__.py @@ -0,0 +1,11 @@ +""" +Components for extracting and optimizing database schemas. +""" +from .extractor import extract_schema +from .optimizer import SchemaOptimizer + +# Alias para compatibilidad con código existente +extract_db_schema = extract_schema +schemaOptimizer = SchemaOptimizer + +__all__ = ['extract_schema', 'extract_db_schema', 'schemaOptimizer'] \ No newline at end of file diff --git a/corebrain/corebrain/db/schema/extractor.py b/corebrain/corebrain/db/schema/extractor.py new file mode 100644 index 0000000..c361b83 --- /dev/null +++ b/corebrain/corebrain/db/schema/extractor.py @@ -0,0 +1,123 @@ +# db/schema/extractor.py (reemplaza la importación circular en db/schema.py) + +""" +Independent database schema extractor. +""" + +from typing import Dict, Any, Optional, Callable + +from corebrain.utils.logging import get_logger + +logger = get_logger(__name__) + +def extract_db_schema(db_config: Dict[str, Any], client_factory: Optional[Callable] = None) -> Dict[str, Any]: + """ + Extracts the database schema with dependency injection. + + Args: + db_config: Database configuration + client_factory: Optional function to create a client (avoids circular imports) + + Returns: + Dictionary with the database structure + """ + db_type = db_config.get("type", "").lower() + schema = { + "type": db_type, + "database": db_config.get("database", ""), + "tables": {}, + "tables_list": [] + } + + try: + # Si tenemos un cliente especializado, usarlo + if client_factory: + # La factoría crea un cliente y extrae el esquema + client = client_factory(db_config) + return client.extract_schema() + + # Extracción directa sin usar cliente de Corebrain + if db_type == "sql": + # Código para bases de datos SQL (sin dependencias circulares) + engine = db_config.get("engine", "").lower() + if engine == "sqlite": + # Extraer esquema SQLite + import sqlite3 + # (implementación...) + elif engine == "mysql": + # Extraer esquema MySQL + import mysql.connector + # (implementación...) + elif engine == "postgresql": + # Extraer esquema PostgreSQL + import psycopg2 + # (implementación...) + + elif db_type in ["nosql", "mongodb"]: + # Extraer esquema MongoDB + import pymongo + # (implementación...) + + # Convertir diccionario a lista para compatibilidad + table_list = [] + for table_name, table_info in schema["tables"].items(): + table_data = {"name": table_name} + table_data.update(table_info) + table_list.append(table_data) + + schema["tables_list"] = table_list + return schema + + except Exception as e: + logger.error(f"Error al extraer esquema: {str(e)}") + return {"type": db_type, "tables": {}, "tables_list": []} + + +def create_schema_from_corebrain() -> Callable: + """ + Creates an extraction function that uses Corebrain internally. + Loads dynamically to avoid circular imports. + + Returns: + Function that extracts schema using Corebrain + """ + def extract_with_corebrain(db_config: Dict[str, Any]) -> Dict[str, Any]: + # Importar dinámicamente para evitar circular + from corebrain.core.client import Corebrain + + # Crear cliente temporal solo para extraer el schema + try: + client = Corebrain( + api_token="temp_token", + db_config=db_config, + skip_verification=True + ) + schema = client.db_schema + client.close() + return schema + except Exception as e: + logger.error(f"Error al extraer schema con Corebrain: {str(e)}") + return {"type": db_config.get("type", ""), "tables": {}, "tables_list": []} + + return extract_with_corebrain + + +# Función pública expuesta +def extract_schema(db_config: Dict[str, Any], use_corebrain: bool = False) -> Dict[str, Any]: + """ + Public function that decides how to extract the schema. + + Args: + db_config: Database configuration + use_corebrain: If True, uses the Corebrain class for extraction + + Returns: + Database schema + """ + if use_corebrain: + # Intentar usar Corebrain si se solicita + factory = create_schema_from_corebrain() + return extract_db_schema(db_config, client_factory=factory) + else: + # Usar extracción directa sin dependencias circulares + return extract_db_schema(db_config) \ No newline at end of file diff --git a/corebrain/corebrain/db/schema/optimizer.py b/corebrain/corebrain/db/schema/optimizer.py new file mode 100644 index 0000000..c7840b0 --- /dev/null +++ b/corebrain/corebrain/db/schema/optimizer.py @@ -0,0 +1,157 @@ +""" +Components for database schema optimization. +""" +import re +from typing import Dict, Any, Optional + +from corebrain.utils.logging import get_logger + +logger = get_logger(__name__) + +class SchemaOptimizer: + """Optimizes the database schema to reduce context size.""" + + def __init__(self, max_tables: int = 10, max_columns_per_table: int = 15, max_samples: int = 2): + """ + Initializes the schema optimizer. + + Args: + max_tables: Maximum number of tables to include + max_columns_per_table: Maximum number of columns per table + max_samples: Maximum number of sample rows per table + """ + self.max_tables = max_tables + self.max_columns_per_table = max_columns_per_table + self.max_samples = max_samples + + # Tablas importantes que siempre deben incluirse si existen + self.priority_tables = set([ + "users", "customers", "products", "orders", "transactions", + "invoices", "accounts", "clients", "employees", "services" + ]) + + # Tablas típicamente menos importantes + self.low_priority_tables = set([ + "logs", "sessions", "tokens", "temp", "cache", "metrics", + "statistics", "audit", "history", "archives", "settings" + ]) + + def optimize_schema(self, db_schema: Dict[str, Any], query: str = None) -> Dict[str, Any]: + """ + Optimizes the schema to reduce its size. + + Args: + db_schema: Original database schema + query: User query (to prioritize relevant tables) + + Returns: + Optimized schema + """ + # Crear copia para no modificar el original + optimized_schema = { + "type": db_schema.get("type", ""), + "database": db_schema.get("database", ""), + "engine": db_schema.get("engine", ""), + "tables": {}, + "tables_list": [] + } + + # Determinar tablas relevantes para la consulta + query_relevant_tables = set() + if query: + # Extraer potenciales nombres de tablas de la consulta + normalized_query = query.lower() + + # Obtener nombres de todas las tablas + all_table_names = [ + name.lower() for name in db_schema.get("tables", {}).keys() + ] + + # Buscar menciones a tablas en la consulta + for table_name in all_table_names: + # Buscar el nombre exacto (como palabra completa) + if re.search(r'\b' + re.escape(table_name) + r'\b', normalized_query): + query_relevant_tables.add(table_name) + + # También buscar formas singulares/plurales simples + if table_name.endswith('s') and re.search(r'\b' + re.escape(table_name[:-1]) + r'\b', normalized_query): + query_relevant_tables.add(table_name) + elif not table_name.endswith('s') and re.search(r'\b' + re.escape(table_name + 's') + r'\b', normalized_query): + query_relevant_tables.add(table_name) + + # Priorizar tablas a incluir + table_scores = {} + for table_name in db_schema.get("tables", {}): + score = 0 + + # Tablas mencionadas en la consulta tienen máxima prioridad + if table_name.lower() in query_relevant_tables: + score += 100 + + # Tablas importantes + if table_name.lower() in self.priority_tables: + score += 50 + + # Tablas poco importantes + if table_name.lower() in self.low_priority_tables: + score -= 30 + + # Tablas con más columnas pueden ser más relevantes + table_info = db_schema["tables"].get(table_name, {}) + column_count = len(table_info.get("columns", [])) + score += min(column_count, 20) # Limitar a 20 puntos máximo + + # Guardar puntuación + table_scores[table_name] = score + + # Ordenar tablas por puntuación + sorted_tables = sorted(table_scores.items(), key=lambda x: x[1], reverse=True) + + # Limitar número de tablas + selected_tables = [name for name, _ in sorted_tables[:self.max_tables]] + + # Copiar tablas seleccionadas con optimizaciones + for table_name in selected_tables: + table_info = db_schema["tables"].get(table_name, {}) + + # Optimizar columnas + columns = table_info.get("columns", []) + if len(columns) > self.max_columns_per_table: + # Mantener las columnas más importantes (id, nombre, clave primaria, etc) + important_columns = [] + other_columns = [] + + for col in columns: + col_name = col.get("name", "").lower() + if col_name in ["id", "uuid", "name", "key", "code"] or "id" in col_name: + important_columns.append(col) + else: + other_columns.append(col) + + # Tomar las columnas importantes y completar con otras hasta el límite + optimized_columns = important_columns + remaining_slots = self.max_columns_per_table - len(optimized_columns) + if remaining_slots > 0: + optimized_columns.extend(other_columns[:remaining_slots]) + else: + optimized_columns = columns + + # Optimizar datos de muestra + sample_data = table_info.get("sample_data", []) + optimized_samples = sample_data[:self.max_samples] if sample_data else [] + + # Guardar tabla optimizada + optimized_schema["tables"][table_name] = { + "columns": optimized_columns, + "sample_data": optimized_samples + } + + # Añadir a la lista de tablas + optimized_schema["tables_list"].append({ + "name": table_name, + "columns": optimized_columns, + "sample_data": optimized_samples + }) + + return optimized_schema + diff --git a/corebrain/corebrain/db/schema_file.py b/corebrain/corebrain/db/schema_file.py new file mode 100644 index 0000000..c4dc8f7 --- /dev/null +++ b/corebrain/corebrain/db/schema_file.py @@ -0,0 +1,604 @@ +""" +Components for extracting and optimizing database schemas. +""" +import json + +from typing import Dict, Any, Optional + +def _print_colored(message: str, color: str) -> None: + """Simplified version of _print_colored that doesn't depend on cli.utils.""" + colors = { + "red": "\033[91m", + "green": "\033[92m", + "yellow": "\033[93m", + "blue": "\033[94m", + "default": "\033[0m" + } + color_code = colors.get(color, colors["default"]) + print(f"{color_code}{message}{colors['default']}") + +def extract_db_schema(db_config: Dict[str, Any]) -> Dict[str, Any]: + """ + Extracts the database schema directly without using the SDK. + + Args: + db_config: Database configuration + + Returns: + Dictionary with the database structure organized by tables/collections + """ + db_type = db_config["type"].lower() + schema = { + "type": db_type, + "database": db_config.get("database", ""), + "tables": {} # Cambiado a diccionario para facilitar el acceso directo a tablas por nombre + } + + try: + if db_type == "sql": + # Código para bases de datos SQL... + # [Se mantiene igual] + pass + + # Manejar tanto "nosql" como "mongodb" como tipos válidos + elif db_type == "nosql" or db_type == "mongodb": + import pymongo + + # Determinar el motor (si existe) + engine = db_config.get("engine", "").lower() + + # Si no se especifica el engine o es mongodb, proceder + if not engine or engine == "mongodb": + if "connection_string" in db_config: + client = pymongo.MongoClient(db_config["connection_string"]) + else: + # Diccionario de parámetros para MongoClient + mongo_params = { + "host": db_config.get("host", "localhost"), + "port": db_config.get("port", 27017) + } + + # Añadir credenciales solo si están presentes + if db_config.get("user"): + mongo_params["username"] = db_config["user"] + if db_config.get("password"): + mongo_params["password"] = db_config["password"] + + client = pymongo.MongoClient(**mongo_params) + + # Obtener la base de datos + db_name = db_config.get("database", "") + if not db_name: + _print_colored("⚠️ Nombre de base de datos no especificado", "yellow") + return schema + + try: + db = client[db_name] + collection_names = db.list_collection_names() + + # Procesar colecciones + for collection_name in collection_names: + collection = db[collection_name] + + # Obtener varios documentos de muestra + try: + sample_docs = list(collection.find().limit(5)) + + # Extraer estructura de campos a partir de los documentos + field_types = {} + + for doc in sample_docs: + for field, value in doc.items(): + if field != "_id": # Ignoramos el _id de MongoDB + # Actualizar el tipo si no existe o combinar si hay diferentes tipos + field_type = type(value).__name__ + if field not in field_types: + field_types[field] = field_type + elif field_types[field] != field_type: + field_types[field] = f"{field_types[field]}|{field_type}" + + # Convertir a formato esperado + fields = [{"name": field, "type": type_name} for field, type_name in field_types.items()] + + # Convertir documentos a formato serializable + sample_data = [] + for doc in sample_docs: + serialized_doc = {} + for key, value in doc.items(): + if key == "_id": + serialized_doc[key] = str(value) + elif isinstance(value, (dict, list)): + serialized_doc[key] = str(value) # Simplificar objetos anidados + else: + serialized_doc[key] = value + sample_data.append(serialized_doc) + + # Guardar información de la colección + schema["tables"][collection_name] = { + "fields": fields, + "sample_data": sample_data + } + except Exception as e: + _print_colored(f"Error al procesar colección {collection_name}: {str(e)}", "red") + schema["tables"][collection_name] = { + "fields": [], + "sample_data": [], + "error": str(e) + } + + except Exception as e: + _print_colored(f"Error al acceder a la base de datos MongoDB '{db_name}': {str(e)}", "red") + + finally: + # Cerrar la conexión + client.close() + else: + _print_colored(f"Motor de base de datos NoSQL no soportado: {engine}", "red") + + # Convertir el diccionario de tablas en una lista para mantener compatibilidad con el formato anterior + table_list = [] + for table_name, table_info in schema["tables"].items(): + table_data = {"name": table_name} + table_data.update(table_info) + table_list.append(table_data) + + # Guardar también la lista de tablas para mantener compatibilidad + schema["tables_list"] = table_list + + return schema + + except Exception as e: + _print_colored(f"Error al extraer el esquema de la base de datos: {str(e)}", "red") + # En caso de error, devolver un esquema vacío + return {"type": db_type, "tables": {}, "tables_list": []} + +def extract_db_schema_direct(db_config: Dict[str, Any]) -> Dict[str, Any]: + """ + Extracts the schema directly without using the Corebrain client. + This is a reduced version that doesn't require importing core. + """ + db_type = db_config["type"].lower() + schema = { + "type": db_type, + "database": db_config.get("database", ""), + "tables": {}, + "tables_list": [] # Lista inicialmente vacía + } + + try: + # [Implementación existente para extraer esquema sin usar Corebrain] + # ... + + return schema + except Exception as e: + _print_colored(f"Error al extraer esquema directamente: {str(e)}", "red") + return {"type": db_type, "tables": {}, "tables_list": []} + +def extract_schema_with_lazy_init(api_key: str, db_config: Dict[str, Any], api_url: Optional[str] = None) -> Dict[str, Any]: + """ + Extracts the schema using late import of the client. + + This function avoids the circular import issue by dynamically loading + the Corebrain client only when necessary. + """ + try: + # La importación se mueve aquí para evitar el problema de circular import + # Solo se ejecuta cuando realmente necesitamos crear el cliente + import importlib + core_module = importlib.import_module('core') + init_func = getattr(core_module, 'init') + + # Crear cliente con la configuración + api_url_to_use = api_url or "https://api.corebrain.com" + cb = init_func( + api_token=api_key, + db_config=db_config, + api_url=api_url_to_use, + skip_verification=True # No necesitamos verificar token para extraer schema + ) + + # Obtener el esquema y cerrar cliente + schema = cb.db_schema + cb.close() + + return schema + + except Exception as e: + _print_colored(f"Error al extraer esquema con cliente: {str(e)}", "red") + # Como alternativa, usar extracción directa sin cliente + return extract_db_schema_direct(db_config) +from typing import Dict, Any + +def test_connection(db_config: Dict[str, Any]) -> bool: + try: + if db_config["type"].lower() == "sql": + # Code to test SQL connection... + pass + elif db_config["type"].lower() in ["nosql", "mongodb"]: + import pymongo + + # Create MongoDB client + client = pymongo.MongoClient(db_config["connection_string"]) + client.admin.command('ping') # Test connection + + return True + else: + _print_colored("Unsupported database type.", "red") + return False + except Exception as e: + _print_colored(f"Failed to connect to the database: {str(e)}", "red") + return False + +def extract_schema_to_file(api_key: str, config_id: Optional[str] = None, output_file: Optional[str] = None, api_url: Optional[str] = None) -> bool: + """ + Extracts the database schema and saves it to a file. + + Args: + api_key: API Key to identify the configuration + config_id: Specific configuration ID (optional) + output_file: Path to the file where the schema will be saved + api_url: Optional API URL + + Returns: + True if extraction is successful, False otherwise + """ + try: + # Importación explícita con try-except para manejar errores + try: + from corebrain.config.manager import ConfigManager + except ImportError as e: + _print_colored(f"Error al importar ConfigManager: {e}", "red") + return False + + # Obtener las configuraciones disponibles + config_manager = ConfigManager() + configs = config_manager.list_configs(api_key) + + if not configs: + _print_colored("No hay configuraciones guardadas para esta API Key.", "yellow") + return False + + selected_config_id = config_id + + # Si no se especifica un config_id, mostrar lista para seleccionar + if not selected_config_id: + _print_colored("\n=== Configuraciones disponibles ===", "blue") + for i, conf_id in enumerate(configs, 1): + print(f"{i}. {conf_id}") + + try: + choice = int(input(f"\nSelecciona una configuración (1-{len(configs)}): ").strip()) + if 1 <= choice <= len(configs): + selected_config_id = configs[choice - 1] + else: + _print_colored("Opción inválida.", "red") + return False + except ValueError: + _print_colored("Por favor, introduce un número válido.", "red") + return False + + # Verificar que el config_id exista + if selected_config_id not in configs: + _print_colored(f"No se encontró la configuración con ID: {selected_config_id}", "red") + return False + + # Obtener la configuración seleccionada + db_config = config_manager.get_config(api_key, selected_config_id) + + if not db_config: + _print_colored(f"Error al obtener la configuración con ID: {selected_config_id}", "red") + return False + + _print_colored(f"\nExtrayendo esquema para configuración: {selected_config_id}", "blue") + print(f"Tipo: {db_config['type'].upper()}, Motor: {db_config.get('engine', 'No especificado').upper()}") + print(f"Base de datos: {db_config.get('database', 'No especificada')}") + + # Extraer el esquema de la base de datos + _print_colored("\nExtrayendo esquema de la base de datos...", "blue") + schema = extract_schema_with_lazy_init(api_key, db_config, api_url) + + # Verificar si se obtuvo un esquema válido + if not schema or not schema.get("tables"): + _print_colored("No se encontraron tablas/colecciones en la base de datos.", "yellow") + return False + + # Guardar el esquema en un archivo + output_path = output_file or "db_schema.json" + try: + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(schema, f, indent=2, default=str) + _print_colored(f"✅ Esquema extraído y guardado en: {output_path}", "green") + except Exception as e: + _print_colored(f"❌ Error al guardar el archivo: {str(e)}", "red") + return False + + # Mostrar un resumen de las tablas/colecciones encontradas + tables = schema.get("tables", {}) + _print_colored(f"\nResumen del esquema extraído: {len(tables)} tablas/colecciones", "green") + + for table_name in tables: + print(f"- {table_name}") + + return True + + except Exception as e: + _print_colored(f"❌ Error al extraer esquema: {str(e)}", "red") + return False + +def show_db_schema(api_token: str, config_id: Optional[str] = None, api_url: Optional[str] = None) -> None: + """ + Displays the schema of the configured database. + + Args: + api_token: API token + config_id: Specific configuration ID (optional) + api_url: Optional API URL + """ + try: + # Importación explícita con try-except para manejar errores + try: + from corebrain.config.manager import ConfigManager + except ImportError as e: + _print_colored(f"Error al importar ConfigManager: {e}", "red") + return False + + # Obtener las configuraciones disponibles + config_manager = ConfigManager() + configs = config_manager.list_configs(api_token) + + if not configs: + _print_colored("No hay configuraciones guardadas para este token.", "yellow") + return + + selected_config_id = config_id + + # Si no se especifica un config_id, mostrar lista para seleccionar + if not selected_config_id: + _print_colored("\n=== Configuraciones disponibles ===", "blue") + for i, conf_id in enumerate(configs, 1): + print(f"{i}. {conf_id}") + + try: + choice = int(input(f"\nSelecciona una configuración (1-{len(configs)}): ").strip()) + if 1 <= choice <= len(configs): + selected_config_id = configs[choice - 1] + else: + _print_colored("Opción inválida.", "red") + return + except ValueError: + _print_colored("Por favor, introduce un número válido.", "red") + return + + # Verificar que el config_id exista + if selected_config_id not in configs: + _print_colored(f"No se encontró la configuración con ID: {selected_config_id}", "red") + return + + if config_id and config_id in configs: + db_config = config_manager.get_config(api_token, config_id) + else: + # Obtener la configuración seleccionada + db_config = config_manager.get_config(api_token, selected_config_id) + + if not db_config: + _print_colored(f"Error al obtener la configuración con ID: {selected_config_id}", "red") + return + + _print_colored(f"\nObteniendo esquema para configuración: {selected_config_id}", "blue") + _print_colored("Tipo de base de datos:", "blue") + print(f" {db_config['type'].upper()}") + + if db_config.get('engine'): + _print_colored("Motor:", "blue") + print(f" {db_config['engine'].upper()}") + + _print_colored("Base de datos:", "blue") + print(f" {db_config.get('database', 'No especificada')}") + + # Extraer y mostrar el esquema + _print_colored("\nExtrayendo esquema de la base de datos...", "blue") + + # Intenta conectarse a la base de datos y extraer el esquema + try: + + # Creamos una instancia de Corebrain con la configuración seleccionada + """ + cb = init( + api_token=api_token, + config_id=selected_config_id, + api_url=api_url, + skip_verification=True # Skip verification for simplicity + ) + """ + + import importlib + core_module = importlib.import_module('core.client') + init_func = getattr(core_module, 'init') + + # Creamos una instancia de Corebrain con la configuración seleccionada + cb = init_func( + api_token=api_token, + config_id=config_id, + api_url=api_url, + skip_verification=True # Omitimos verificación para simplificar + ) + + # El esquema se extrae automáticamente al inicializar + schema = get_schema_with_dynamic_import( + api_token=api_token, + config_id=selected_config_id, + db_config=db_config, + api_url=api_url + ) + + # Si no hay esquema, intentamos extraerlo explícitamente + if not schema or not schema.get("tables"): + _print_colored("Intentando extraer esquema explícitamente...", "yellow") + schema = cb._extract_db_schema() + + # Cerramos la conexión + cb.close() + + except Exception as conn_error: + _print_colored(f"Error de conexión: {str(conn_error)}", "red") + print("Intentando método alternativo...") + + # Método alternativo: usar función extract_db_schema directamente + schema = extract_db_schema(db_config) + + # Verificar si se obtuvo un esquema válido + if not schema or not schema.get("tables"): + _print_colored("No se encontraron tablas/colecciones en la base de datos.", "yellow") + + # Información adicional para ayudar a diagnosticar el problema + print("\nInformación de depuración:") + print(f" Tipo de base de datos: {db_config.get('type', 'No especificado')}") + print(f" Motor: {db_config.get('engine', 'No especificado')}") + print(f" Host: {db_config.get('host', 'No especificado')}") + print(f" Puerto: {db_config.get('port', 'No especificado')}") + print(f" Base de datos: {db_config.get('database', 'No especificado')}") + + # Para PostgreSQL, sugerir verificar el esquema + if db_config.get('engine') == 'postgresql': + print("\nPara PostgreSQL, verifica que las tablas existan en el esquema 'public' o") + print("que tengas acceso a los esquemas donde están las tablas.") + print("Puedes verificar los esquemas disponibles con: SELECT DISTINCT table_schema FROM information_schema.tables;") + + return + + # Mostrar información del esquema + tables = schema.get("tables", {}) + + # Separar tablas SQL y colecciones NoSQL para mostrarlas apropiadamente + sql_tables = {} + nosql_collections = {} + + for name, info in tables.items(): + if "columns" in info: + sql_tables[name] = info + elif "fields" in info: + nosql_collections[name] = info + + # Mostrar tablas SQL + if sql_tables: + _print_colored(f"\nSe encontraron {len(sql_tables)} tablas SQL:", "green") + for table_name, table_info in sql_tables.items(): + _print_colored(f"\n=== Tabla: {table_name} ===", "bold") + + # Mostrar columnas + columns = table_info.get("columns", []) + if columns: + _print_colored("Columnas:", "blue") + for column in columns: + print(f" - {column['name']} ({column['type']})") + else: + _print_colored("No se encontraron columnas.", "yellow") + + # Mostrar muestra de datos si está disponible + sample_data = table_info.get("sample_data", []) + if sample_data: + _print_colored("\nMuestra de datos:", "blue") + for i, row in enumerate(sample_data[:2], 1): # Limitar a 2 filas para simplificar + print(f" Registro {i}: {row}") + + if len(sample_data) > 2: + print(f" ... ({len(sample_data) - 2} registros más)") + + # Mostrar colecciones NoSQL + if nosql_collections: + _print_colored(f"\nSe encontraron {len(nosql_collections)} colecciones NoSQL:", "green") + for coll_name, coll_info in nosql_collections.items(): + _print_colored(f"\n=== Colección: {coll_name} ===", "bold") + + # Mostrar campos + fields = coll_info.get("fields", []) + if fields: + _print_colored("Campos:", "blue") + for field in fields: + print(f" - {field['name']} ({field['type']})") + else: + _print_colored("No se encontraron campos.", "yellow") + + # Mostrar muestra de datos si está disponible + sample_data = coll_info.get("sample_data", []) + if sample_data: + _print_colored("\nMuestra de datos:", "blue") + for i, doc in enumerate(sample_data[:2], 1): # Limitar a 2 documentos + # Simplificar la visualización para documentos grandes + if isinstance(doc, dict) and len(doc) > 5: + simplified = {k: doc[k] for k in list(doc.keys())[:5]} + print(f" Documento {i}: {simplified} ... (y {len(doc) - 5} campos más)") + else: + print(f" Documento {i}: {doc}") + + if len(sample_data) > 2: + print(f" ... ({len(sample_data) - 2} documentos más)") + + _print_colored("\n✅ Esquema extraído correctamente!", "green") + + # Preguntar si quiere guardar el esquema en un archivo + save_option = input("\n¿Deseas guardar el esquema en un archivo? (s/n): ").strip().lower() + if save_option == "s": + filename = input("Nombre del archivo (por defecto: db_schema.json): ").strip() or "db_schema.json" + try: + with open(filename, 'w') as f: + json.dump(schema, f, indent=2, default=str) + _print_colored(f"\n✅ Esquema guardado en: {filename}", "green") + except Exception as e: + _print_colored(f"❌ Error al guardar el archivo: {str(e)}", "red") + + except Exception as e: + _print_colored(f"❌ Error al mostrar el esquema: {str(e)}", "red") + import traceback + traceback.print_exc() + + +def get_schema_with_dynamic_import(api_token: str, config_id: str, db_config: Dict[str, Any], api_url: Optional[str] = None) -> Dict[str, Any]: + """ + Retrieves the database schema using dynamic import. + + Args: + api_token: API token + config_id: Configuration ID + db_config: Database configuration + api_url: Optional API URL + + Returns: + Database schema + """ + try: + # Importación dinámica del módulo core + import importlib + core_module = importlib.import_module('core.client') + init_func = getattr(core_module, 'init') + + # Creamos una instancia de Corebrain con la configuración seleccionada + cb = init_func( + api_token=api_token, + config_id=config_id, + api_url=api_url, + skip_verification=True # Omitimos verificación para simplificar + ) + + # El esquema se extrae automáticamente al inicializar + schema = cb.db_schema + + # Si no hay esquema, intentamos extraerlo explícitamente + if not schema or not schema.get("tables"): + _print_colored("Intentando extraer esquema explícitamente...", "yellow") + schema = cb._extract_db_schema() + + # Cerramos la conexión + cb.close() + + return schema + + except ImportError: + # Si falla la importación dinámica, intentamos un enfoque alternativo + _print_colored("No se pudo importar el cliente. Usando método alternativo.", "yellow") + return extract_db_schema(db_config) + + except Exception as e: + _print_colored(f"Error al extraer esquema con cliente: {str(e)}", "red") + # Fallback a extracción directa + return extract_db_schema(db_config) diff --git a/corebrain/corebrain/lib/sso/__init__.py b/corebrain/corebrain/lib/sso/__init__.py new file mode 100644 index 0000000..033a277 --- /dev/null +++ b/corebrain/corebrain/lib/sso/__init__.py @@ -0,0 +1,4 @@ +from corebrain.lib.sso.auth import GlobodainSSOAuth +from corebrain.lib.sso.client import GlobodainSSOClient + +__all__ = ['GlobodainSSOAuth', 'GlobodainSSOClient'] \ No newline at end of file diff --git a/corebrain/corebrain/lib/sso/auth.py b/corebrain/corebrain/lib/sso/auth.py new file mode 100644 index 0000000..d065a20 --- /dev/null +++ b/corebrain/corebrain/lib/sso/auth.py @@ -0,0 +1,171 @@ +import requests +import logging +from urllib.parse import urlencode + +class GlobodainSSOAuth: + def __init__(self, config=None): + self.config = config or {} + self.logger = logging.getLogger(__name__) + + # Configuración por defecto + self.sso_url = self.config.get('GLOBODAIN_SSO_URL', 'http://localhost:3000/login') # URL del SSO + self.client_id = self.config.get('GLOBODAIN_CLIENT_ID', '') + self.client_secret = self.config.get('GLOBODAIN_CLIENT_SECRET', '') + self.redirect_uri = self.config.get('GLOBODAIN_REDIRECT_URI', '') + self.success_redirect = self.config.get('GLOBODAIN_SUCCESS_REDIRECT', 'https://sso.globodain.com/cli/success') + + def requires_auth(self, session_handler): + """ + Generic decorator that checks if the user is authenticated + + Args: + session_handler: Function that retrieves the current session object + + Returns: + A decorator function that can be applied to routes/views + """ + def decorator(func): + def wrapper(*args, **kwargs): + # Obtener la sesión actual usando el manejador proporcionado + session = session_handler() + + if 'user' not in session: + # Aquí retornamos información para que el framework redirija + return { + 'authenticated': False, + 'redirect_url': self.get_login_url() + } + return func(*args, **kwargs) + return wrapper + return decorator + + def get_login_url(self, state=None): + """ + Generates the URL to initiate SSO authentication + + Args: + state: Optional parameter to maintain state between requests + + Returns: + Full URL for SSO login initiation + """ + params = { + 'client_id': self.client_id, + 'redirect_uri': self.redirect_uri, + 'response_type': 'code', + } + + if state: + params['state'] = state + + return f"{self.sso_url}/api/auth/authorize?{urlencode(params)}" + + def verify_token(self, token): + """ + Verifies the token with the SSO server + + Args: + token: Access token to verify + + Returns: + Token data if valid, None otherwise + """ + try: + response = requests.post( + f"{self.sso_url}/api/auth/service-auth", + headers={'Authorization': f'Bearer {token}'}, + json={'service_id': self.client_id} + ) + if response.status_code == 200: + return response.json() + return None + except Exception as e: + self.logger.error(f"Error verificando token: {str(e)}") + return None + + def get_user_info(self, token): + """ + Retrieves user information using the token + + Args: + token: User access token + + Returns: + User profile information if the token is valid, None otherwise + """ + try: + response = requests.get( + f"{self.sso_url}/api/users/me/profile", + headers={'Authorization': f'Bearer {token}'} + ) + if response.status_code == 200: + return response.json() + return None + except Exception as e: + self.logger.error(f"Error obteniendo info de usuario: {str(e)}") + return None + + def exchange_code_for_token(self, code): + """ + Exchanges the authorization code for an access token + + Args: + code: Authorization code received from the SSO server + + Returns: + Access token data if the exchange is successful, None otherwise + """ + try: + response = requests.post( + f"{self.sso_url}/api/auth/token", + json={ + 'client_id': self.client_id, + 'client_secret': self.client_secret, + 'code': code, + 'grant_type': 'authorization_code', + 'redirect_uri': self.redirect_uri + } + ) + if response.status_code == 200: + return response.json() + return None + except Exception as e: + self.logger.error(f"Error intercambiando código: {str(e)}") + return None + + def handle_callback(self, code, session_handler, store_user_func=None): + """ + Handles the SSO callback by processing the received code + + Args: + code: Authorization code received + session_handler: Function that retrieves the current session object + store_user_func: Optional function to store user data elsewhere + + Returns: + Redirect URL after processing the code + """ + # Intercambiar código por token + token_data = self.exchange_code_for_token(code) + if not token_data: + # Error al obtener el token + return self.get_login_url() + + # Obtener información del usuario + user_info = self.get_user_info(token_data.get('access_token')) + if not user_info: + # Error al obtener información del usuario + return self.get_login_url() + + # Guardar información en la sesión + session = session_handler() + session['user'] = user_info + session['token'] = token_data + + # Si hay una función para almacenar el usuario, ejecutarla + if store_user_func and callable(store_user_func): + store_user_func(user_info, token_data) + + # Redirigir a la URL de éxito o a la URL guardada anteriormente + next_url = session.pop('next_url', self.success_redirect) + return next_url \ No newline at end of file diff --git a/corebrain/corebrain/lib/sso/client.py b/corebrain/corebrain/lib/sso/client.py new file mode 100644 index 0000000..0315085 --- /dev/null +++ b/corebrain/corebrain/lib/sso/client.py @@ -0,0 +1,194 @@ +# /auth/sso_client.py +import requests + +from typing import Dict, Any +from datetime import datetime, timedelta + +class GlobodainSSOClient: + """ + SDK client for Globodain services that connect to the central SSO + """ + + def __init__( + self, + sso_url: str, + client_id: str, + client_secret: str, + service_id: int, + redirect_uri: str + ): + """ + Initialize the SSO client + + Args: + sso_url: Base URL of the SSO service (e.g., https://sso.globodain.com) + client_id: Client ID of the service + client_secret: Client secret of the service + service_id: Numeric ID of the service on the SSO platform + redirect_uri: Redirect URI for OAuth + """ + self.sso_url = sso_url.rstrip('/') + self.client_id = client_id + self.client_secret = client_secret + self.service_id = service_id + self.redirect_uri = redirect_uri + self._token_cache = {} # Cache de tokens verificados + + + def get_login_url(self, provider: str = None) -> str: + """ + Get URL to initiate SSO login + + Args: + provider: OAuth provider (google, microsoft, github) or None for normal login + + Returns: + URL to redirect the user + """ + if provider: + return f"{self.sso_url}/api/auth/oauth/{provider}?service_id={self.service_id}" + else: + return f"{self.sso_url}/login?service_id={self.service_id}&redirect_uri={self.redirect_uri}" + + def verify_token(self, token: str) -> Dict[str, Any]: + """ + Verify an access token and retrieve user information + + Args: + token: JWT token to verify + + Returns: + User information if the token is valid + + Raises: + Exception: If the token is not valid + """ + # Verificar si ya tenemos información cacheada y válida del token + now = datetime.now() + if token in self._token_cache: + cache_data = self._token_cache[token] + if cache_data['expires_at'] > now: + return cache_data['user_info'] + else: + # Eliminar token expirado del caché + del self._token_cache[token] + + # Verificar token con el servicio SSO + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json" + } + + response = requests.post( + f"{self.sso_url}/api/auth/service-auth", + headers=headers, + json={"service_id": self.service_id} + ) + + if response.status_code != 200: + raise Exception(f"Token inválido: {response.text}") + + # Obtener información del usuario + user_response = requests.get( + f"{self.sso_url}/api/users/me", + headers=headers + ) + + if user_response.status_code != 200: + raise Exception(f"Error al obtener información del usuario: {user_response.text}") + + user_info = user_response.json() + + # Guardar en caché (15 minutos) + self._token_cache[token] = { + 'user_info': user_info, + 'expires_at': now + timedelta(minutes=15) + } + + return user_info + + def authenticate_service(self, token: str) -> Dict[str, Any]: + """ + Authenticate a token for use with this specific service + + Args: + token: JWT token obtained from the SSO + + Returns: + New service-specific token + + Raises: + Exception: If there is an authentication error + """ + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json" + } + + response = requests.post( + f"{self.sso_url}/api/auth/service-auth", + headers=headers, + json={"service_id": self.service_id} + ) + + if response.status_code != 200: + raise Exception(f"Error de autenticación: {response.text}") + + return response.json() + + def refresh_token(self, refresh_token: str) -> Dict[str, Any]: + """ + Renew an access token using a refresh token + + Args: + refresh_token: Refresh token + + Returns: + New access token + + Raises: + Exception: If there is an error renewing the token + """ + response = requests.post( + f"{self.sso_url}/api/auth/refresh", + json={"refresh_token": refresh_token} + ) + + if response.status_code != 200: + raise Exception(f"Error al renovar token: {response.text}") + + return response.json() + + def logout(self, refresh_token: str, access_token: str) -> bool: + """ + Log out (revoke refresh token) + + Args: + refresh_token: Refresh token to revoke + access_token: Valid access token + + Returns: + True if the logout was successful + + Raises: + Exception: If there is an error logging out + """ + headers = { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json" + } + + response = requests.post( + f"{self.sso_url}/api/auth/logout", + headers=headers, + json={"refresh_token": refresh_token} + ) + + if response.status_code != 200: + raise Exception(f"Error al cerrar sesión: {response.text}") + + # Limpiar cualquier token cacheado + if access_token in self._token_cache: + del self._token_cache[access_token] + + return True \ No newline at end of file diff --git a/corebrain/corebrain/network/__init__.py b/corebrain/corebrain/network/__init__.py new file mode 100644 index 0000000..aa079ff --- /dev/null +++ b/corebrain/corebrain/network/__init__.py @@ -0,0 +1,22 @@ +""" +Network components for Corebrain SDK. + +This package provides utilities and clients for communication +with the Corebrain API and other web services. +""" +from corebrain.network.client import ( + APIClient, + APIError, + APITimeoutError, + APIConnectionError, + APIAuthError +) + +# Exportación explícita de componentes públicos +__all__ = [ + 'APIClient', + 'APIError', + 'APITimeoutError', + 'APIConnectionError', + 'APIAuthError' +] \ No newline at end of file diff --git a/corebrain/corebrain/network/client.py b/corebrain/corebrain/network/client.py new file mode 100644 index 0000000..1176fb1 --- /dev/null +++ b/corebrain/corebrain/network/client.py @@ -0,0 +1,502 @@ +""" +HTTP client for communication with the Corebrain API. +""" +import time +import logging +import httpx + +from typing import Dict, Any, Optional, List +from urllib.parse import urljoin +from httpx import Response, ConnectError, ReadTimeout, WriteTimeout, PoolTimeout + +logger = logging.getLogger(__name__) +http_session = httpx.Client(timeout=10.0, verify=True) + +def __init__(self, verbose=False): + self.verbose = verbose + +class APIError(Exception): + """Generic error in the API.""" + def __init__(self, message: str, status_code: Optional[int] = None, + detail: Optional[str] = None, response: Optional[Response] = None): + self.message = message + self.status_code = status_code + self.detail = detail + self.response = response + super().__init__(message) + +class APITimeoutError(APIError): + """Timeout error in the API.""" + pass + +class APIConnectionError(APIError): + """Connection error to the API.""" + pass + +class APIAuthError(APIError): + """Authentication error in the API.""" + pass + +class APIClient: + """Optimized HTTP client for communication with the Corebrain API.""" + + # Constantes para manejo de reintentos y errores + MAX_RETRIES = 3 + RETRY_DELAY = 0.5 # segundos + RETRY_STATUS_CODES = [408, 429, 500, 502, 503, 504] + + def __init__(self, base_url: str, default_timeout: int = 10, + verify_ssl: bool = True, user_agent: Optional[str] = None): + """ + Initializes the API client with optimized configuration. + + Args: + base_url: Base URL for all requests + default_timeout: Default timeout in seconds + verify_ssl: Whether to verify the SSL certificate + user_agent: Custom user agent + """ + # Normalizar URL base para asegurar que termina con '/' + self.base_url = base_url if base_url.endswith('/') else base_url + '/' + self.default_timeout = default_timeout + self.verify_ssl = verify_ssl + + # Headers predeterminados + self.default_headers = { + 'User-Agent': user_agent or 'CorebrainSDK/1.0', + 'Accept': 'application/json', + 'Content-Type': 'application/json' + } + + # Crear sesión HTTP con límites y timeouts optimizados + self.session = httpx.Client( + timeout=httpx.Timeout(timeout=default_timeout), + verify=verify_ssl, + http2=True, # Usar HTTP/2 si está disponible + limits=httpx.Limits(max_connections=100, max_keepalive_connections=20) + ) + + # Estadísticas y métricas + self.request_count = 0 + self.error_count = 0 + self.total_request_time = 0 + + logger.debug(f"Cliente API inicializado con base_url={base_url}, timeout={default_timeout}s") + + def __del__(self): + """Ensure the session is closed when the client is deleted.""" + self.close() + + def close(self): + """Closes the HTTP session.""" + if hasattr(self, 'session') and self.session: + try: + self.session.close() + logger.debug("Sesión HTTP cerrada correctamente") + except Exception as e: + logger.warning(f"Error al cerrar sesión HTTP: {e}") + + def get_full_url(self, endpoint: str) -> str: + """ + Builds the full URL for an endpoint. + + Args: + endpoint: Relative path of the endpoint + + Returns: + Full URL + """ + # Eliminar '/' inicial si existe para evitar rutas duplicadas + endpoint = endpoint.lstrip('/') + return urljoin(self.base_url, endpoint) + + def prepare_headers(self, headers: Optional[Dict[str, str]] = None, + auth_token: Optional[str] = None) -> Dict[str, str]: + """ + Prepares the headers for a request. + + Args: + headers: Additional headers + auth_token: Authentication token + + Returns: + Combined headers + """ + # Comenzar con headers predeterminados + final_headers = self.default_headers.copy() + + # Añadir headers personalizados + if headers: + final_headers.update(headers) + + # Añadir token de autenticación si se proporciona + if auth_token: + final_headers['Authorization'] = f'Bearer {auth_token}' + + return final_headers + + def handle_response(self, response: Response) -> Response: + """ + Processes the response to handle common errors. + + Args: + response: HTTP response + + Returns: + The same response if there are no errors + + Raises: + APIError: If there are errors in the response + """ + status_code = response.status_code + + # Procesar errores según código de estado + if 400 <= status_code < 500: + error_detail = None + + # Intentar extraer detalles del error del cuerpo JSON + try: + json_data = response.json() + if isinstance(json_data, dict): + error_detail = ( + json_data.get('detail') or + json_data.get('message') or + json_data.get('error') + ) + except Exception: + # Si no podemos parsear JSON, usar el texto completo + error_detail = response.text[:200] + ('...' if len(response.text) > 200 else '') + + # Errores específicos según código + if status_code == 401: + msg = "Error de autenticación: token inválido o expirado" + logger.error(f"{msg} - {error_detail or ''}") + raise APIAuthError(msg, status_code, error_detail, response) + + elif status_code == 403: + msg = "Acceso prohibido: no tienes permisos suficientes" + logger.error(f"{msg} - {error_detail or ''}") + raise APIAuthError(msg, status_code, error_detail, response) + + elif status_code == 404: + msg = f"Recurso no encontrado: {response.url}" + logger.error(msg) + raise APIError(msg, status_code, error_detail, response) + + elif status_code == 429: + msg = "Demasiadas peticiones: límite de tasa excedido" + logger.warning(msg) + raise APIError(msg, status_code, error_detail, response) + + else: + msg = f"Error del cliente ({status_code}): {error_detail or 'sin detalles'}" + logger.error(msg) + raise APIError(msg, status_code, error_detail, response) + + elif 500 <= status_code < 600: + msg = f"Error del servidor ({status_code}): el servidor API encontró un error" + logger.error(msg) + raise APIError(msg, status_code, response.text[:200], response) + + return response + + def request(self, method: str, endpoint: str, *, + headers: Optional[Dict[str, str]] = None, + json: Optional[Any] = None, + data: Optional[Any] = None, + params: Optional[Dict[str, Any]] = None, + timeout: Optional[int] = None, + auth_token: Optional[str] = None, + retry: bool = True) -> Response: + """ + Makes an HTTP request with error handling and retries. + + Args: + method: HTTP method (GET, POST, etc.) + endpoint: Relative path of the endpoint + headers: Additional headers + json: Data to send as JSON + data: Data to send as form or bytes + params: Query string parameters + timeout: Timeout in seconds (overrides the default) + auth_token: Authentication token + retry: Whether to retry failed requests + + Returns: + Processed HTTP response + + Raises: + APIError: If there are errors in the request or response + APITimeoutError: If the request exceeds the timeout + APIConnectionError: If there are connection errors + """ + url = self.get_full_url(endpoint) + final_headers = self.prepare_headers(headers, auth_token) + + # Configurar timeout + request_timeout = timeout or self.default_timeout + + # Contador para reintentos + retries = 0 + last_error = None + + # Registrar inicio de la petición + start_time = time.time() + self.request_count += 1 + + while retries <= (self.MAX_RETRIES if retry else 0): + try: + if retries > 0: + # Esperar antes de reintentar con backoff exponencial + wait_time = self.RETRY_DELAY * (2 ** (retries - 1)) + logger.info(f"Reintentando petición ({retries}/{self.MAX_RETRIES}) a {url} después de {wait_time:.2f}s") + time.sleep(wait_time) + + # Realizar la petición + logger.debug(f"Enviando petición {method} a {url}") + response = self.session.request( + method=method, + url=url, + headers=final_headers, + json=json, + data=data, + params=params, + timeout=request_timeout + ) + + # Verificar si debemos reintentar por código de estado + if response.status_code in self.RETRY_STATUS_CODES and retry and retries < self.MAX_RETRIES: + logger.warning(f"Código de estado {response.status_code} recibido, reintentando") + retries += 1 + continue + + # Procesar la respuesta + processed_response = self.handle_response(response) + + # Registrar tiempo total + elapsed = time.time() - start_time + self.total_request_time += elapsed + logger.debug(f"Petición completada en {elapsed:.3f}s con estado {response.status_code}") + + return processed_response + + except (ConnectError, httpx.HTTPError) as e: + last_error = e + + # Decidir si reintentamos dependiendo del tipo de error + if isinstance(e, (ReadTimeout, WriteTimeout, PoolTimeout, ConnectError)) and retry and retries < self.MAX_RETRIES: + logger.warning(f"Error de conexión: {str(e)}, reintentando {retries+1}/{self.MAX_RETRIES}") + retries += 1 + continue + + # No más reintentos o error no recuperable + self.error_count += 1 + elapsed = time.time() - start_time + + if isinstance(e, (ReadTimeout, WriteTimeout, PoolTimeout)): + logger.error(f"Timeout en petición a {url} después de {elapsed:.3f}s: {str(e)}") + raise APITimeoutError(f"La petición a {endpoint} excedió el tiempo máximo de {request_timeout}s", + response=getattr(e, 'response', None)) + else: + logger.error(f"Error de conexión a {url} después de {elapsed:.3f}s: {str(e)}") + raise APIConnectionError(f"Error de conexión a {endpoint}: {str(e)}", + response=getattr(e, 'response', None)) + + except Exception as e: + # Error inesperado + self.error_count += 1 + elapsed = time.time() - start_time + logger.error(f"Error inesperado en petición a {url} después de {elapsed:.3f}s: {str(e)}") + raise APIError(f"Error inesperado en petición a {endpoint}: {str(e)}") + + # Si llegamos aquí es porque agotamos los reintentos + if last_error: + self.error_count += 1 + raise APIError(f"Petición a {endpoint} falló después de {retries} reintentos: {str(last_error)}") + + # Este punto nunca debería alcanzarse + raise APIError(f"Error inesperado en petición a {endpoint}") + + def get(self, endpoint: str, **kwargs) -> Response: + """Makes a GET request.""" + return self.request("GET", endpoint, **kwargs) + + def post(self, endpoint: str, **kwargs) -> Response: + """Makes a POST request.""" + return self.request("POST", endpoint, **kwargs) + + def put(self, endpoint: str, **kwargs) -> Response: + """Makes a PUT request.""" + return self.request("PUT", endpoint, **kwargs) + + def delete(self, endpoint: str, **kwargs) -> Response: + """Makes a DELETE request.""" + return self.request("DELETE", endpoint, **kwargs) + + def patch(self, endpoint: str, **kwargs) -> Response: + """Makes a PATCH request.""" + return self.request("PATCH", endpoint, **kwargs) + + def get_json(self, endpoint: str, **kwargs) -> Any: + """ + Makes a GET request and returns the JSON data. + + Args: + endpoint: Endpoint to query + **kwargs: Additional arguments for request() + + Returns: + Parsed JSON data + """ + response = self.get(endpoint, **kwargs) + try: + return response.json() + except Exception as e: + raise APIError(f"Error al parsear respuesta JSON: {str(e)}", response=response) + + def post_json(self, endpoint: str, **kwargs) -> Any: + """ + Makes a POST request and returns the JSON data. + + Args: + endpoint: Endpoint to query + **kwargs: Additional arguments for request() + + Returns: + Parsed JSON data + """ + response = self.post(endpoint, **kwargs) + try: + return response.json() + except Exception as e: + raise APIError(f"Error al parsear respuesta JSON: {str(e)}", response=response) + + # Métodos de alto nivel para operaciones comunes en la API de Corebrain + + def check_health(self, timeout: int = 5) -> bool: + """ + Checks if the API is available. + + Args: + timeout: Maximum wait time + + Returns: + True if the API is available + """ + try: + response = self.get("health", timeout=timeout, retry=False) + return response.status_code == 200 + except Exception: + return False + + def verify_token(self, token: str, timeout: int = 5) -> Dict[str, Any]: + """ + Verifies if a token is valid. + + Args: + token: Token to verify + timeout: Maximum wait time + + Returns: + User information if the token is valid + + Raises: + APIAuthError: If the token is invalid + """ + try: + response = self.get("api/auth/me", auth_token=token, timeout=timeout) + return response.json() + except APIAuthError: + raise + except Exception as e: + raise APIAuthError(f"Error al verificar token: {str(e)}") + + def get_api_keys(self, token: str) -> List[Dict[str, Any]]: + """ + Retrieves the available API keys for a user. + + Args: + token: Authentication token + + Returns: + List of API keys + """ + return self.get_json("api/auth/api-keys", auth_token=token) + + def update_api_key_metadata(self, token: str, api_key: str, metadata: Dict[str, Any]) -> Dict[str, Any]: + """ + Updates the metadata of an API key. + + Args: + token: Authentication token + api_key: API key ID + metadata: Metadata to update + + Returns: + Updated API key data + """ + data = {"metadata": metadata} + return self.put_json(f"api/auth/api-keys/{api_key}", auth_token=token, json=data) + + def query_database(self, token: str, question: str, db_schema: Dict[str, Any], + config_id: str, timeout: int = 30) -> Dict[str, Any]: + """ + Makes a natural language query. + + Args: + token: Authentication token + question: Natural language question + db_schema: Database schema + config_id: Configuration ID + timeout: Maximum wait time + + Returns: + Query result + """ + data = { + "question": question, + "db_schema": db_schema, + "config_id": config_id + } + return self.post_json("api/database/sdk/query", auth_token=token, json=data, timeout=timeout) + + def exchange_sso_token(self, sso_token: str, user_data: Dict[str, Any]) -> Dict[str, Any]: + """ + Exchanges an SSO token for an API token. + + Args: + sso_token: SSO token + user_data: User data + + Returns: + API token data + """ + headers = {"Authorization": f"Bearer {sso_token}"} + data = {"user_data": user_data} + return self.post_json("api/auth/sso/token", headers=headers, json=data) + + # Métodos para estadísticas y diagnóstico + + def get_stats(self) -> Dict[str, Any]: + """ + Retrieves client usage statistics. + + Returns: + Request statistics + """ + avg_time = self.total_request_time / max(1, self.request_count) + error_rate = (self.error_count / max(1, self.request_count)) * 100 + + return { + "request_count": self.request_count, + "error_count": self.error_count, + "error_rate": f"{error_rate:.2f}%", + "total_request_time": f"{self.total_request_time:.3f}s", + "average_request_time": f"{avg_time:.3f}s", + } + + def reset_stats(self) -> None: + """Resets the usage statistics.""" + self.request_count = 0 + self.error_count = 0 + self.total_request_time = 0 \ No newline at end of file diff --git a/corebrain/corebrain/sdk.py b/corebrain/corebrain/sdk.py new file mode 100644 index 0000000..7de1491 --- /dev/null +++ b/corebrain/corebrain/sdk.py @@ -0,0 +1,8 @@ +""" +Corebrain SDK for compatibility. +""" +from corebrain.config.manager import ConfigManager + +# Re-exportar elementos principales +list_configurations = ConfigManager().list_configs +remove_configuration = ConfigManager().remove_config \ No newline at end of file diff --git a/corebrain/corebrain/services/schema.py b/corebrain/corebrain/services/schema.py new file mode 100644 index 0000000..4155fb1 --- /dev/null +++ b/corebrain/corebrain/services/schema.py @@ -0,0 +1,31 @@ + +# Nuevo directorio: services/ +# Nuevo archivo: services/schema_service.py +""" +Services for managing database schemas. +""" +from typing import Dict, Any, Optional + +from corebrain.config.manager import ConfigManager +from corebrain.db.schema import extract_db_schema, SchemaOptimizer + +class SchemaService: + """Service for database schema operations.""" + + def __init__(self): + self.config_manager = ConfigManager() + self.schema_optimizer = SchemaOptimizer() + + def get_schema(self, api_token: str, config_id: str) -> Optional[Dict[str, Any]]: + """Retrieves the schema for a specific configuration.""" + config = self.config_manager.get_config(api_token, config_id) + if not config: + return None + + return extract_db_schema(config) + + def optimize_schema(self, schema: Dict[str, Any], query: str = None) -> Dict[str, Any]: + """Optimizes an existing schema.""" + return self.schema_optimizer.optimize_schema(schema, query) + + # Otros métodos de servicio... \ No newline at end of file diff --git a/corebrain/corebrain/utils/__init__.py b/corebrain/corebrain/utils/__init__.py new file mode 100644 index 0000000..3c89186 --- /dev/null +++ b/corebrain/corebrain/utils/__init__.py @@ -0,0 +1,66 @@ +""" +General utilities for Corebrain SDK. + +This package provides utilities shared by different +SDK components, such as serialization, encryption, and logging. +""" +import logging + +from corebrain.utils.serializer import serialize_to_json, JSONEncoder +from corebrain.utils.encrypter import ( + create_cipher, + generate_key, + derive_key_from_password, + ConfigEncrypter +) + +# Configuración de logging +logger = logging.getLogger('corebrain') + +def setup_logger(level=logging.INFO, + file_path=None, + format_string=None): + """ + Configures the main Corebrain logger. + + Args: + level: Logging level + file_path: Path to log file (optional) + format_string: Custom log format + """ + # Formato predeterminado + fmt = format_string or '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + formatter = logging.Formatter(fmt) + + # Handler de consola + console_handler = logging.StreamHandler() + console_handler.setFormatter(formatter) + + # Configurar logger principal + logger.setLevel(level) + logger.addHandler(console_handler) + + # Handler de archivo si se proporciona ruta + if file_path: + file_handler = logging.FileHandler(file_path) + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + + # Mensajes de diagnóstico + logger.debug(f"Logger configurado con nivel {logging.getLevelName(level)}") + if file_path: + logger.debug(f"Logs escritos a {file_path}") + + return logger + +# Exportación explícita de componentes públicos +__all__ = [ + 'serialize_to_json', + 'JSONEncoder', + 'create_cipher', + 'generate_key', + 'derive_key_from_password', + 'ConfigEncrypter', + 'setup_logger', + 'logger' +] \ No newline at end of file diff --git a/corebrain/corebrain/utils/encrypter.py b/corebrain/corebrain/utils/encrypter.py new file mode 100644 index 0000000..286a705 --- /dev/null +++ b/corebrain/corebrain/utils/encrypter.py @@ -0,0 +1,264 @@ +""" +Encryption utilities for Corebrain SDK. +""" +import os +import base64 +import logging + +from pathlib import Path +from typing import Optional, Union +from cryptography.fernet import Fernet, InvalidToken +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC + +logger = logging.getLogger(__name__) + +def derive_key_from_password(password: Union[str, bytes], salt: Optional[bytes] = None) -> bytes: + """ + Derives a secure encryption key from a password and salt. + + Args: + password: Password or passphrase + salt: Cryptographic salt (generated if not provided) + + Returns: + Derived key in bytes + """ + if isinstance(password, str): + password = password.encode() + + # Generar sal si no se proporciona + if salt is None: + salt = os.urandom(16) + + # Derivar clave usando PBKDF2 + kdf = PBKDF2HMAC( + algorithm=hashes.SHA256(), + length=32, + salt=salt, + iterations=100000 # Mayor número de iteraciones = mayor seguridad + ) + + key = kdf.derive(password) + return base64.urlsafe_b64encode(key) + +def generate_key() -> str: + """ + Generates a new random encryption key. + + Returns: + Encryption key in base64 format + """ + key = Fernet.generate_key() + return key.decode() + +def create_cipher(key: Optional[Union[str, bytes]] = None) -> Fernet: + """ + Creates a Fernet encryption object with the given key or generates a new one. + + Args: + key: Encryption key in base64 format or None to generate a new one + + Returns: + Fernet object for encryption/decryption + """ + if key is None: + key = Fernet.generate_key() + elif isinstance(key, str): + key = key.encode() + + return Fernet(key) + +class ConfigEncrypter: + """ + Encryption manager for configurations with key management. + """ + + def __init__(self, key_path: Optional[Union[str, Path]] = None): + """ + Initializes the encryptor with an optional key path. + + Args: + key_path: Path to the key file (will be created if it doesn't exist) + """ + self.key_path = Path(key_path) if key_path else None + self.cipher = None + self._init_cipher() + + def _init_cipher(self) -> None: + """Initializes the encryption object, creating or loading the key as needed.""" + key = None + + # Si hay ruta de clave, intentar cargar o crear + if self.key_path: + try: + if self.key_path.exists(): + with open(self.key_path, 'rb') as f: + key = f.read().strip() + logger.debug(f"Clave cargada desde {self.key_path}") + else: + # Crear directorio padre si no existe + self.key_path.parent.mkdir(parents=True, exist_ok=True) + + # Generar nueva clave + key = Fernet.generate_key() + + # Guardar clave + with open(self.key_path, 'wb') as f: + f.write(key) + + # Asegurar permisos restrictivos (solo el propietario puede leer) + try: + os.chmod(self.key_path, 0o600) + except Exception as e: + logger.warning(f"No se pudieron establecer permisos en archivo de clave: {e}") + + logger.debug(f"Nueva clave generada y guardada en {self.key_path}") + except Exception as e: + logger.error(f"Error al gestionar clave en {self.key_path}: {e}") + # En caso de error, generar clave efímera + key = None + + # Si no tenemos clave, generar una efímera + if not key: + key = Fernet.generate_key() + logger.debug("Usando clave efímera generada") + + self.cipher = Fernet(key) + + def encrypt(self, data: Union[str, bytes]) -> bytes: + """ + Encrypts data. + + Args: + data: Data to encrypt + + Returns: + Encrypted data in bytes + """ + if isinstance(data, str): + data = data.encode() + + try: + return self.cipher.encrypt(data) + except Exception as e: + logger.error(f"Error al cifrar datos: {e}") + raise + + def decrypt(self, encrypted_data: Union[str, bytes]) -> bytes: + """ + Decrypts data. + + Args: + encrypted_data: Encrypted data + + Returns: + Decrypted data in bytes + """ + if isinstance(encrypted_data, str): + encrypted_data = encrypted_data.encode() + + try: + return self.cipher.decrypt(encrypted_data) + except InvalidToken: + logger.error("Token inválido o datos corruptos") + raise ValueError("Los datos no pueden ser descifrados: token inválido o datos corruptos") + except Exception as e: + logger.error(f"Error al descifrar datos: {e}") + raise + + def encrypt_file(self, input_path: Union[str, Path], output_path: Optional[Union[str, Path]] = None) -> Path: + """ + Encrypts a complete file. + + Args: + input_path: Path to the file to encrypt + output_path: Path to save the encrypted file (if None, .enc is added) + + Returns: + Path of the encrypted file + """ + input_path = Path(input_path) + + if not output_path: + output_path = input_path.with_suffix(input_path.suffix + '.enc') + else: + output_path = Path(output_path) + + try: + with open(input_path, 'rb') as f: + data = f.read() + + encrypted_data = self.encrypt(data) + + with open(output_path, 'wb') as f: + f.write(encrypted_data) + + return output_path + except Exception as e: + logger.error(f"Error al cifrar archivo {input_path}: {e}") + raise + + def decrypt_file(self, input_path: Union[str, Path], output_path: Optional[Union[str, Path]] = None) -> Path: + """ + Decrypts a complete file. + + Args: + input_path: Path to the encrypted file + output_path: Path to save the decrypted file + + Returns: + Path of the decrypted file + """ + input_path = Path(input_path) + + if not output_path: + # Si termina en .enc, quitar esa extensión + if input_path.suffix == '.enc': + output_path = input_path.with_suffix('') + else: + output_path = input_path.with_suffix(input_path.suffix + '.dec') + else: + output_path = Path(output_path) + + try: + with open(input_path, 'rb') as f: + encrypted_data = f.read() + + decrypted_data = self.decrypt(encrypted_data) + + with open(output_path, 'wb') as f: + f.write(decrypted_data) + + return output_path + except Exception as e: + logger.error(f"Error al descifrar archivo {input_path}: {e}") + raise + + @staticmethod + def generate_key_file(key_path: Union[str, Path]) -> None: + """ + Generates and saves a new key to a file. + + Args: + key_path: Path to save the key + """ + key_path = Path(key_path) + + # Crear directorio padre si no existe + key_path.parent.mkdir(parents=True, exist_ok=True) + + # Generar clave + key = Fernet.generate_key() + + # Guardar clave + with open(key_path, 'wb') as f: + f.write(key) + + # Establecer permisos restrictivos + try: + os.chmod(key_path, 0o600) + except Exception as e: + logger.warning(f"No se pudieron establecer permisos en archivo de clave: {e}") + + logger.info(f"Nueva clave generada y guardada en {key_path}") \ No newline at end of file diff --git a/corebrain/corebrain/utils/logging.py b/corebrain/corebrain/utils/logging.py new file mode 100644 index 0000000..0ba559d --- /dev/null +++ b/corebrain/corebrain/utils/logging.py @@ -0,0 +1,243 @@ +""" +Logging utilities for Corebrain SDK. + +This module provides functions and classes to manage logging +within the SDK consistently. +""" +import logging +import sys +from datetime import datetime +from pathlib import Path +from typing import Optional, Any, Union + +# Niveles de logging personalizados +VERBOSE = 15 # Entre DEBUG e INFO + +# Configuración predeterminada +DEFAULT_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' +DEFAULT_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' +DEFAULT_LEVEL = logging.INFO +DEFAULT_LOG_DIR = Path.home() / ".corebrain" / "logs" + +# Colores para logging en terminal +LOG_COLORS = { + "DEBUG": "\033[94m", # Azul + "VERBOSE": "\033[96m", # Cian + "INFO": "\033[92m", # Verde + "WARNING": "\033[93m", # Amarillo + "ERROR": "\033[91m", # Rojo + "CRITICAL": "\033[95m", # Magenta + "RESET": "\033[0m" # Reset +} + +class VerboseLogger(logging.Logger): + """Custom logger with VERBOSE level.""" + + def verbose(self, msg: str, *args: Any, **kwargs: Any) -> None: + """ + Logs a message with VERBOSE level. + + Args: + msg: Message to log + *args: Arguments to format the message + **kwargs: Additional arguments for the logger + """ + return self.log(VERBOSE, msg, *args, **kwargs) + +class ColoredFormatter(logging.Formatter): + """Formatter that adds colors to log messages in the terminal.""" + + def __init__(self, fmt: str = DEFAULT_FORMAT, datefmt: str = DEFAULT_DATE_FORMAT, + use_colors: bool = True): + """ + Initializes the formatter. + + Args: + fmt: Message format + datefmt: Date format + use_colors: If True, uses colors in the terminal + """ + super().__init__(fmt, datefmt) + self.use_colors = use_colors and sys.stdout.isatty() + + def format(self, record: logging.LogRecord) -> str: + """ + Formats a log record with colors. + + Args: + record: Record to format + + Returns: + Formatted message + """ + levelname = record.levelname + message = super().format(record) + + if self.use_colors and levelname in LOG_COLORS: + return f"{LOG_COLORS[levelname]}{message}{LOG_COLORS['RESET']}" + return message + +def setup_logger(name: str = "corebrain", + level: int = DEFAULT_LEVEL, + file_path: Optional[Union[str, Path]] = None, + format_string: Optional[str] = None, + use_colors: bool = True, + propagate: bool = False) -> logging.Logger: + """ + Configures a logger with custom options. + + Args: + name: Logger name + level: Logging level + file_path: Path to the log file (optional) + format_string: Custom message format + use_colors: If True, uses colors in the terminal + propagate: If True, propagates messages to parent loggers + + Returns: + Configured logger + """ + # Registrar nivel personalizado VERBOSE + if not hasattr(logging, 'VERBOSE'): + logging.addLevelName(VERBOSE, 'VERBOSE') + + # Registrar clase de logger personalizada + logging.setLoggerClass(VerboseLogger) + + # Obtener o crear logger + logger = logging.getLogger(name) + + # Limpiar handlers existentes + for handler in logger.handlers[:]: + logger.removeHandler(handler) + + # Configurar nivel de logging + logger.setLevel(level) + logger.propagate = propagate + + # Formato predeterminado + fmt = format_string or DEFAULT_FORMAT + formatter = ColoredFormatter(fmt, use_colors=use_colors) + + # Handler de consola + console_handler = logging.StreamHandler() + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) + + # Handler de archivo si se proporciona ruta + if file_path: + # Asegurar que el directorio exista + file_path = Path(file_path) + file_path.parent.mkdir(parents=True, exist_ok=True) + + file_handler = logging.FileHandler(file_path) + # Para archivos, usar formateador sin colores + file_formatter = logging.Formatter(fmt) + file_handler.setFormatter(file_formatter) + logger.addHandler(file_handler) + + # Mensajes de diagnóstico + logger.debug(f"Logger '{name}' configurado con nivel {logging.getLevelName(level)}") + if file_path: + logger.debug(f"Logs escritos a {file_path}") + + return logger + +def get_logger(name: str, level: Optional[int] = None) -> logging.Logger: + """ + Retrieves an existing logger or creates a new one. + + Args: + name: Logger name + level: Optional logging level + + Returns: + Configured logger + """ + logger = logging.getLogger(name) + + # Si el logger no tiene handlers, configurarlo + if not logger.handlers: + # Determinar si es un logger secundario + if '.' in name: + # Es un sublogger, configurar para propagar a logger padre + logger.propagate = True + if level is not None: + logger.setLevel(level) + else: + # Es un logger principal, configurar completamente + logger = setup_logger(name, level or DEFAULT_LEVEL) + elif level is not None: + # Solo actualizar el nivel si se especifica + logger.setLevel(level) + + return logger + +def enable_file_logging(logger_name: str = "corebrain", + log_dir: Optional[Union[str, Path]] = None, + filename: Optional[str] = None) -> str: + """ + Enables file logging for an existing logger. + + Args: + logger_name: Logger name + log_dir: Directory for the logs (optional) + filename: Custom file name (optional) + + Returns: + Path to the log file + """ + logger = logging.getLogger(logger_name) + + # Determinar la ruta del archivo de log + log_dir = Path(log_dir) if log_dir else DEFAULT_LOG_DIR + log_dir.mkdir(parents=True, exist_ok=True) + + # Generar nombre de archivo si no se proporciona + if not filename: + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + filename = f"{logger_name}_{timestamp}.log" + + file_path = log_dir / filename + + # Verificar si ya hay un FileHandler + for handler in logger.handlers: + if isinstance(handler, logging.FileHandler): + logger.removeHandler(handler) + + # Agregar nuevo FileHandler + file_handler = logging.FileHandler(file_path) + formatter = logging.Formatter(DEFAULT_FORMAT) + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + + logger.info(f"Logging a archivo activado: {file_path}") + return str(file_path) + +def set_log_level(level: Union[int, str], + logger_name: Optional[str] = None) -> None: + """ + Sets the logging level for one or all loggers. + + Args: + level: Logging level (name or integer value) + logger_name: Specific logger name (if None, affects all) + """ + # Convertir nombre de nivel a valor si es necesario + if isinstance(level, str): + level = getattr(logging, level.upper(), logging.INFO) + + if logger_name: + # Afectar solo al logger especificado + logger = logging.getLogger(logger_name) + logger.setLevel(level) + logger.info(f"Nivel de log cambiado a {logging.getLevelName(level)}") + else: + # Afectar al logger raíz y a todos los loggers existentes + root = logging.getLogger() + root.setLevel(level) + + # También afectar a loggers específicos del SDK + for name in logging.root.manager.loggerDict: + if name.startswith("corebrain"): + logging.getLogger(name).setLevel(level) \ No newline at end of file diff --git a/corebrain/corebrain/utils/serializer.py b/corebrain/corebrain/utils/serializer.py new file mode 100644 index 0000000..c230c3e --- /dev/null +++ b/corebrain/corebrain/utils/serializer.py @@ -0,0 +1,33 @@ +""" +Serialization utilities for Corebrain SDK. +""" +import json + +from datetime import datetime, date, time +from bson import ObjectId +from decimal import Decimal + +class JSONEncoder(json.JSONEncoder): + """Custom JSON serializer for special types.""" + def default(self, obj): + # Objetos datetime + if isinstance(obj, (datetime, date, time)): + return obj.isoformat() + # Objetos timedelta + elif hasattr(obj, 'total_seconds'): # Para objetos timedelta + return obj.total_seconds() + # ObjectId de MongoDB + elif isinstance(obj, ObjectId): + return str(obj) + # Bytes o bytearray + elif isinstance(obj, (bytes, bytearray)): + return obj.hex() + # Decimal + elif isinstance(obj, Decimal): + return float(obj) + # Otros tipos + return super().default(obj) + +def serialize_to_json(obj): + """Serializes any object to JSON using the custom encoder""" + return json.dumps(obj, cls=JSONEncoder) \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp/.editorconfig b/corebrain/corebrain/wrappers/csharp/.editorconfig new file mode 100644 index 0000000..e4eb58c --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp/.editorconfig @@ -0,0 +1,432 @@ +# This file is the top-most EditorConfig file +root = true + +#All Files +[*] +charset = utf-8 +indent_style = space +indent_size = 2 +insert_final_newline = true +trim_trailing_whitespace = true +end_of_line = lf + +########################################## +# File Extension Settings +########################################## + +# Visual Studio Solution Files +[*.sln] +indent_style = tab + +# Visual Studio XML Project Files +[*.{csproj,vbproj,vcxproj.filters,proj,projitems,shproj}] +indent_size = 2 + +# XML Configuration Files +[*.{xml,config,props,targets,nuspec,resx,ruleset,vsixmanifest,vsct}] +indent_size = 2 + +# JSON Files +[*.{json,json5,webmanifest}] +indent_size = 2 + +# YAML Files +[*.{yml,yaml}] +indent_size = 2 + +# Markdown Files +[*.{md,mdx}] +trim_trailing_whitespace = false + +# Web Files +[*.{htm,html,js,jsm,ts,tsx,cjs,cts,ctsx,mjs,mts,mtsx,css,sass,scss,less,pcss,svg,vue}] +indent_size = 2 + +# Batch Files +[*.{cmd,bat}] +end_of_line = crlf + +# Bash Files +[*.sh] +end_of_line = lf + +# Makefiles +[Makefile] +indent_style = tab + +[{*_Generated.cs, *.g.cs, *.generated.cs}] +# Ignore a lack of documentation for generated code. Doesn't apply to builds, +# just to viewing generation output. +dotnet_diagnostic.CS1591.severity = none + +########################################## +# Default .NET Code Style Severities +########################################## + +[*.{cs,csx,cake,vb,vbx}] +# Default Severity for all .NET Code Style rules below +dotnet_analyzer_diagnostic.severity = warning + +########################################## +# Language Rules +########################################## + +# .NET Style Rules +[*.{cs,csx,cake,vb,vbx}] + +# "this." and "Me." qualifiers +dotnet_style_qualification_for_field = false +dotnet_style_qualification_for_property = false +dotnet_style_qualification_for_method = false +dotnet_style_qualification_for_event = false + +# Language keywords instead of framework type names for type references +dotnet_style_predefined_type_for_locals_parameters_members = true:warning +dotnet_style_predefined_type_for_member_access = true:warning + +# Modifier preferences +dotnet_style_require_accessibility_modifiers = always:warning +csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:warning +visual_basic_preferred_modifier_order = Partial,Default,Private,Protected,Public,Friend,NotOverridable,Overridable,MustOverride,Overloads,Overrides,MustInherit,NotInheritable,Static,Shared,Shadows,ReadOnly,WriteOnly,Dim,Const,WithEvents,Widening,Narrowing,Custom,Async:warning +dotnet_style_readonly_field = true:warning +dotnet_diagnostic.IDE0036.severity = warning + + +# Parentheses preferences +dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:warning +dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:warning +dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:warning +dotnet_style_parentheses_in_other_operators = never_if_unnecessary:warning + +# Expression-level preferences +dotnet_style_object_initializer = true:warning +dotnet_style_collection_initializer = true:warning +dotnet_style_explicit_tuple_names = true:warning +dotnet_style_prefer_inferred_tuple_names = true:warning +dotnet_style_prefer_inferred_anonymous_type_member_names = true:warning +dotnet_style_prefer_auto_properties = true:warning +dotnet_style_prefer_conditional_expression_over_assignment = true:suggestion +dotnet_diagnostic.IDE0045.severity = suggestion +dotnet_style_prefer_conditional_expression_over_return = true:suggestion +dotnet_diagnostic.IDE0046.severity = suggestion +dotnet_style_prefer_compound_assignment = true:warning +dotnet_style_prefer_simplified_interpolation = true:warning +dotnet_style_prefer_simplified_boolean_expressions = true:warning + +# Null-checking preferences +dotnet_style_coalesce_expression = true:warning +dotnet_style_null_propagation = true:warning +dotnet_style_prefer_is_null_check_over_reference_equality_method = true:warning + +# File header preferences +# Keep operators at end of line when wrapping. +dotnet_style_operator_placement_when_wrapping = end_of_line:warning +csharp_style_prefer_null_check_over_type_check = true:warning + +# Code block preferences +csharp_prefer_braces = true:warning +csharp_prefer_simple_using_statement = true:suggestion +dotnet_diagnostic.IDE0063.severity = suggestion + +# C# Style Rules +[*.{cs,csx,cake}] +# 'var' preferences +csharp_style_var_for_built_in_types = true:warning +csharp_style_var_when_type_is_apparent = true:warning +csharp_style_var_elsewhere = true:warning +# Expression-bodied members +csharp_style_expression_bodied_methods = true:warning +csharp_style_expression_bodied_constructors = false:warning +csharp_style_expression_bodied_operators = true:warning +csharp_style_expression_bodied_properties = true:warning +csharp_style_expression_bodied_indexers = true:warning +csharp_style_expression_bodied_accessors = true:warning +csharp_style_expression_bodied_lambdas = true:warning +csharp_style_expression_bodied_local_functions = true:warning +# Pattern matching preferences +csharp_style_pattern_matching_over_is_with_cast_check = true:warning +csharp_style_pattern_matching_over_as_with_null_check = true:warning +csharp_style_prefer_switch_expression = true:warning +csharp_style_prefer_pattern_matching = true:warning +csharp_style_prefer_not_pattern = true:warning +# Expression-level preferences +csharp_style_inlined_variable_declaration = true:warning +csharp_prefer_simple_default_expression = true:warning +csharp_style_pattern_local_over_anonymous_function = true:warning +csharp_style_deconstructed_variable_declaration = true:warning +csharp_style_prefer_index_operator = true:warning +csharp_style_prefer_range_operator = true:warning +csharp_style_implicit_object_creation_when_type_is_apparent = true:warning +# "Null" checking preferences +csharp_style_throw_expression = true:warning +csharp_style_conditional_delegate_call = true:warning +# Code block preferences +csharp_prefer_braces = true:warning +csharp_prefer_simple_using_statement = true:suggestion +dotnet_diagnostic.IDE0063.severity = suggestion +# 'using' directive preferences +csharp_using_directive_placement = inside_namespace:warning +# Modifier preferences +# Don't suggest making public methods static. Very annoying. +csharp_prefer_static_local_function = false +# Only suggest making private methods static (if they don't use instance data). +dotnet_code_quality.CA1822.api_surface = private + +########################################## +# Unnecessary Code Rules +########################################## + +# .NET Unnecessary code rules +[*.{cs,csx,cake,vb,vbx}] + +dotnet_code_quality_unused_parameters = non_public:suggestion +dotnet_remove_unnecessary_suppression_exclusions = none +dotnet_diagnostic.IDE0079.severity = warning + +# C# Unnecessary code rules +[*.{cs,csx,cake}] + + +# Don't remove method parameters that are unused. +dotnet_diagnostic.IDE0060.severity = none +dotnet_diagnostic.RCS1163.severity = none + +# Don't remove methods that are unused. +dotnet_diagnostic.IDE0051.severity = none +dotnet_diagnostic.RCS1213.severity = none + +# Use discard variable for unused expression values. +csharp_style_unused_value_expression_statement_preference = discard_variable + +# .NET formatting rules +[*.{cs,csx,cake,vb,vbx}] + +# Organize using directives +dotnet_sort_system_directives_first = true +dotnet_separate_import_directive_groups = false + +dotnet_sort_accessibility = true + +# Dotnet namespace options +# +# We don't care about namespaces matching folder structure. Games and apps +# are complicated and you are free to organize them however you like. Change +# this if you want to enforce it. +dotnet_style_namespace_match_folder = false +dotnet_diagnostic.IDE0130.severity = none + +# C# formatting rules +[*.{cs,csx,cake}] + +# Newline options +csharp_new_line_before_open_brace = none +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true +csharp_new_line_before_members_in_object_initializers = true +csharp_new_line_before_members_in_anonymous_types = true +csharp_new_line_between_query_expression_clauses = true + +# Indentation options +csharp_indent_switch_labels = true +csharp_indent_case_contents = true +csharp_indent_case_contents_when_block = true +csharp_indent_labels = no_change +csharp_indent_block_contents = true +csharp_indent_braces = false + +# Spacing options +csharp_space_after_cast = false +csharp_space_after_keywords_in_control_flow_statements = true +csharp_space_between_parentheses = false +csharp_space_before_colon_in_inheritance_clause = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_around_binary_operators = before_and_after +csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_between_method_declaration_empty_parameter_list_parentheses = false +csharp_space_between_method_declaration_name_and_open_parenthesis = false +csharp_space_between_method_call_parameter_list_parentheses = false +csharp_space_between_method_call_empty_parameter_list_parentheses = false +csharp_space_between_method_call_name_and_opening_parenthesis = false +csharp_space_after_comma = true +csharp_space_before_comma = false +csharp_space_after_dot = false +csharp_space_before_dot = false +csharp_space_after_semicolon_in_for_statement = true +csharp_space_before_semicolon_in_for_statement = false +csharp_space_around_declaration_statements = false +csharp_space_before_open_square_brackets = false +csharp_space_between_empty_square_brackets = false +csharp_space_between_square_brackets = false + +# Wrap options +csharp_preserve_single_line_statements = false +csharp_preserve_single_line_blocks = true + +# Namespace options +csharp_style_namespace_declarations = file_scoped:warning + +########################################## +# .NET Naming Rules +########################################## +[*.{cs,csx,cake,vb,vbx}] + +# Allow underscores in names. +dotnet_diagnostic.CA1707.severity = none + +# Styles +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +dotnet_naming_style.upper_case_style.capitalization = all_upper +dotnet_naming_style.upper_case_style.word_separator = _ + +dotnet_naming_style.camel_case_style.capitalization = camel_case + +dotnet_naming_style.camel_case_underscore_style.required_prefix = _ +dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case + +# Use uppercase for all constant fields. +dotnet_naming_rule.constants_uppercase.severity = suggestion +dotnet_naming_rule.constants_uppercase.symbols = constant_fields +dotnet_naming_rule.constants_uppercase.style = upper_case_style +dotnet_naming_symbols.constant_fields.applicable_kinds = field +dotnet_naming_symbols.constant_fields.applicable_accessibilities = * +dotnet_naming_symbols.constant_fields.required_modifiers = const + +# Non-public fields should be _camelCase +dotnet_naming_rule.non_public_fields_under_camel.severity = suggestion +dotnet_naming_rule.non_public_fields_under_camel.symbols = non_public_fields +dotnet_naming_rule.non_public_fields_under_camel.style = camel_case_underscore_style +dotnet_naming_symbols.non_public_fields.applicable_kinds = field +dotnet_naming_symbols.non_public_fields.required_modifiers = +dotnet_naming_symbols.non_public_fields.applicable_accessibilities = private,private_protected,internal,protected,protected_internal + +# Public fields should be PascalCase +dotnet_naming_rule.public_fields_pascal.severity = suggestion +dotnet_naming_rule.public_fields_pascal.symbols = public_fields +dotnet_naming_rule.public_fields_pascal.style = pascal_case_style +dotnet_naming_symbols.public_fields.applicable_kinds = field +dotnet_naming_symbols.public_fields.required_modifiers = +dotnet_naming_symbols.public_fields.applicable_accessibilities = public + +# Async methods should have "Async" suffix. +# Disabled because it makes tests too verbose. +# dotnet_naming_style.end_in_async.required_suffix = Async +# dotnet_naming_style.end_in_async.capitalization = pascal_case +# dotnet_naming_rule.methods_end_in_async.symbols = methods_async +# dotnet_naming_rule.methods_end_in_async.style = end_in_async +# dotnet_naming_rule.methods_end_in_async.severity = warning +# dotnet_naming_symbols.methods_async.applicable_kinds = method +# dotnet_naming_symbols.methods_async.required_modifiers = async +# dotnet_naming_symbols.methods_async.applicable_accessibilities = * + +########################################## +# Other Naming Rules +########################################## + +# All of the following must be PascalCase: +dotnet_naming_symbols.element_group.applicable_kinds = namespace, class, enum, struct, delegate, event, method, property +dotnet_naming_rule.element_rule.symbols = element_group +dotnet_naming_rule.element_rule.style = pascal_case_style +dotnet_naming_rule.element_rule.severity = warning + +# Interfaces use PascalCase and are prefixed with uppercase 'I' +# https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-classes-structs-and-interfaces +dotnet_naming_style.prefix_interface_with_i_style.capitalization = pascal_case +dotnet_naming_style.prefix_interface_with_i_style.required_prefix = I +dotnet_naming_symbols.interface_group.applicable_kinds = interface +dotnet_naming_rule.interface_rule.symbols = interface_group +dotnet_naming_rule.interface_rule.style = prefix_interface_with_i_style +dotnet_naming_rule.interface_rule.severity = warning + +# Generics Type Parameters use PascalCase and are prefixed with uppercase 'T' +# https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-classes-structs-and-interfaces +dotnet_naming_style.prefix_type_parameters_with_t_style.capitalization = pascal_case +dotnet_naming_style.prefix_type_parameters_with_t_style.required_prefix = T +dotnet_naming_symbols.type_parameter_group.applicable_kinds = type_parameter +dotnet_naming_rule.type_parameter_rule.symbols = type_parameter_group +dotnet_naming_rule.type_parameter_rule.style = prefix_type_parameters_with_t_style +dotnet_naming_rule.type_parameter_rule.severity = warning + +# Function parameters use camelCase +# https://docs.microsoft.com/dotnet/standard/design-guidelines/naming-parameters +dotnet_naming_symbols.parameters_group.applicable_kinds = parameter +dotnet_naming_rule.parameters_rule.symbols = parameters_group +dotnet_naming_rule.parameters_rule.style = camel_case_style +dotnet_naming_rule.parameters_rule.severity = warning + +# Anything not specified uses camel case. +dotnet_naming_rule.unspecified_naming.severity = warning +dotnet_naming_rule.unspecified_naming.symbols = unspecified +dotnet_naming_rule.unspecified_naming.style = camel_case_style +dotnet_naming_symbols.unspecified.applicable_kinds = * +dotnet_naming_symbols.unspecified.applicable_accessibilities = * + +########################################## +# Rule Overrides +########################################## + +roslyn_correctness.assembly_reference_validation = relaxed + +# Allow using keywords as names +# dotnet_diagnostic.CA1716.severity = none +# Don't require culture info for ToString() +dotnet_diagnostic.CA1304.severity = none +# Don't require a string comparison for comparing strings. +dotnet_diagnostic.CA1310.severity = none +# Don't require a string format specifier. +dotnet_diagnostic.CA1305.severity = none +# Allow protected fields. +dotnet_diagnostic.CA1051.severity = none +# Don't warn about checking values that are supposedly never null. Sometimes +# they are actually null. +dotnet_diagnostic.CS8073.severity = none +# Don't remove seemingly "unnecessary" assignments, as they often have +# intended side-effects. +dotnet_diagnostic.IDE0059.severity = none +# Switch/case should always have a default clause. Tell that to Roslynator. +dotnet_diagnostic.RCS1070.severity = none +# Tell roslynator not to eat unused parameters. +dotnet_diagnostic.RCS1163.severity = none +# Tell dotnet not to remove unused parameters. +dotnet_diagnostic.IDE0060.severity = none +# Tell roslynator not to remove `partial` modifiers. +dotnet_diagnostic.RCS1043.severity = none +# Tell roslynator not to make classes static so aggressively. +dotnet_diagnostic.RCS1102.severity = none +# Roslynator wants to make properties readonly all the time, so stop it. +# The developer knows best when it comes to contract definitions with Godot. +dotnet_diagnostic.RCS1170.severity = none +# Allow expression values to go unused, even without discard variable. +# Otherwise, using Moq would be way too verbose. +dotnet_diagnostic.IDE0058.severity = none +# Don't let roslynator turn every local variable into a const. +# If we did, we'd have to specify the types of local variables far more often, +# and this style prefers type inference. +dotnet_diagnostic.RCS1118.severity = none +# Enums don't need to declare explicit values. Everyone knows they start at 0. +dotnet_diagnostic.RCS1161.severity = none +# Allow unconstrained type parameter to be checked for null. +dotnet_diagnostic.RCS1165.severity = none +# Allow keyword-based names so that parameter names like `@event` can be used. +dotnet_diagnostic.CA1716.severity = none +# Allow me to use the word Collection if I want. +dotnet_diagnostic.CA1711.severity = none +# Not disposing of objects in a test is normal within Godot because of scene tree stuff. +dotnet_diagnostic.CA1001.severity = none +# No primary constructors — not supported well by tooling. +dotnet_diagnostic.IDE0290.severity = none +# Let me comment where I like +dotnet_diagnostic.RCS1181.severity = none +# Let me write dumb if checks, keeps it readable +dotnet_diagnostic.IDE0046.severity = none +# Don't make me use expression bodies for methods +dotnet_diagnostic.IDE0022.severity = none +# Don't use collection shorhand. +dotnet_diagnostic.IDE0300.severity = none +dotnet_diagnostic.IDE0028.severity = none +dotnet_diagnostic.IDE0305.severity = none +# Don't make me populate a switch expression redundantly +dotnet_diagnostic.IDE0072.severity = none +# Leave me alone about primary constructors +dotnet_diagnostic.IDE0290.severity = none diff --git a/corebrain/corebrain/wrappers/csharp/.gitignore b/corebrain/corebrain/wrappers/csharp/.gitignore new file mode 100644 index 0000000..d37d4fc --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp/.gitignore @@ -0,0 +1,417 @@ +# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,csharp +# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,csharp + +### Csharp ### +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Mono auto generated files +mono_crash.* + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUnit +*.VisualState.xml +TestResult.xml +nunit-*.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.log +*.tlog +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage*.json +coverage*.xml +coverage*.info + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# NuGet Symbol Packages +*.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx +*.appxbundle +*.appxupload + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio 6 auto-generated project file (contains which files were open etc.) +*.vbp + +# Visual Studio 6 workspace and project file (working project files containing files to include in project) +*.dsw +*.dsp + +# Visual Studio 6 technical files + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# Visual Studio History (VSHistory) files +.vshistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ + +# Fody - auto-generated XML schema +FodyWeavers.xsd + +# VS Code files for those working on multiple tools +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +.venv + +# Local History for Visual Studio Code +.history/ + +# Windows Installer files from build outputs +*.cab +*.msi +*.msix +*.msm +*.msp + +# JetBrains Rider +*.sln.iml + +### VisualStudioCode ### +!.vscode/*.code-snippets + +# Local History for Visual Studio Code + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,csharp \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp/.vscode/settings.json b/corebrain/corebrain/wrappers/csharp/.vscode/settings.json new file mode 100644 index 0000000..23c1fea --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "dotnet.defaultSolution": "CorebrainCS.sln" +} \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp/.vscode/tasks.json b/corebrain/corebrain/wrappers/csharp/.vscode/tasks.json new file mode 100644 index 0000000..f058896 --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp/.vscode/tasks.json @@ -0,0 +1,32 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Build Project", + "command": "dotnet", + "type": "process", + "args": [ + "build", + "${workspaceFolder}/CorebrainCS.Tests/CorebrainCS.Tests.csproj", + "--configuration", + "Release" + ], + "problemMatcher": "$msCompile", + "group": { + "kind": "build", + "isDefault": true + } + }, + { + "label": "Run Project", + "command": "dotnet", + "type": "process", + "args": [ + "run", + "--project", + "${workspaceFolder}/CorebrainCS.Tests/CorebrainCS.Tests.csproj" + ], + "dependsOn": ["Build Project"] + }, + ] +} \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/CorebrainCS.Tests.csproj b/corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/CorebrainCS.Tests.csproj new file mode 100644 index 0000000..6e20930 --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/CorebrainCS.Tests.csproj @@ -0,0 +1,14 @@ + + + + Exe + net9.0 + enable + enable + + + + + + + diff --git a/corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs b/corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs new file mode 100644 index 0000000..d58d0fa --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs @@ -0,0 +1,10 @@ +using CorebrainCS; + +Console.WriteLine("Hello, World!"); + +// For now it only works on windows +var corebrain = new CorebrainCS.CorebrainCS("../../../../venv/Scripts/python.exe", "../../../cli", false); + +Console.WriteLine(corebrain.Version()); + + diff --git a/corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/README.md b/corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/README.md new file mode 100644 index 0000000..9aa8159 --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/README.md @@ -0,0 +1,4 @@ +### Quick start + +* Create venv in the root directory and install all the dependencies. The instalation guide is in corebrain README.md +* Go to the CorebrainCS.Tests directory to see how the program runs and run `dotnet run` \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp/CorebrainCS.sln b/corebrain/corebrain/wrappers/csharp/CorebrainCS.sln new file mode 100644 index 0000000..e9542bb --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp/CorebrainCS.sln @@ -0,0 +1,48 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CorebrainCS", "CorebrainCS\CorebrainCS.csproj", "{152890AC-4B76-42F7-813B-CB7F3F902B9F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CorebrainCS.Tests", "CorebrainCS.Tests\CorebrainCS.Tests.csproj", "{664BB3EB-0364-4989-879A-D8CCDBCF6B89}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Debug|x64.ActiveCfg = Debug|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Debug|x64.Build.0 = Debug|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Debug|x86.ActiveCfg = Debug|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Debug|x86.Build.0 = Debug|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Release|Any CPU.Build.0 = Release|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Release|x64.ActiveCfg = Release|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Release|x64.Build.0 = Release|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Release|x86.ActiveCfg = Release|Any CPU + {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Release|x86.Build.0 = Release|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Debug|Any CPU.Build.0 = Debug|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Debug|x64.ActiveCfg = Debug|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Debug|x64.Build.0 = Debug|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Debug|x86.ActiveCfg = Debug|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Debug|x86.Build.0 = Debug|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Release|Any CPU.ActiveCfg = Release|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Release|Any CPU.Build.0 = Release|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Release|x64.ActiveCfg = Release|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Release|x64.Build.0 = Release|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Release|x86.ActiveCfg = Release|Any CPU + {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/corebrain/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs b/corebrain/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs new file mode 100644 index 0000000..c685d3f --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs @@ -0,0 +1,175 @@ +namespace CorebrainCS; + +using System; +using System.Diagnostics; + +/// +/// Creates the main corebrain interface. +/// +/// Path to the python which works with the corebrain cli, for example if you create the ./.venv you pass the path to the ./.venv python executable +/// Path to the corebrain cli script, if you installed it globally you just pass the `corebrain` path +/// +public class CorebrainCS(string pythonPath = "python", string scriptPath = "corebrain", bool verbose = false) { + private readonly string _pythonPath = Path.GetFullPath(pythonPath); + private readonly string _scriptPath = Path.GetFullPath(scriptPath); + private readonly bool _verbose = verbose; + + + public string Help() { + return ExecuteCommand("--help"); + } + + public string Version() { + return ExecuteCommand("--version"); + } + + public string Configure() { + return ExecuteCommand("--configure"); + } + + public string ListConfigs() { + return ExecuteCommand("--list-configs"); + } + + public string RemoveConfig() { + return ExecuteCommand("--remove-config"); + } + + public string ShowSchema() { + return ExecuteCommand("--show-schema"); + } + + public string ExtractSchema() { + return ExecuteCommand("--extract-schema"); + } + + public string ExtractSchemaToDefaultFile() { + return ExecuteCommand("--extract-schema --output-file test"); + } + + public string ConfigID() { + return ExecuteCommand("--extract-schema --config-id config"); + } + + public string SetToken(string token) { + return ExecuteCommand($"--token {token}"); + } + + public string ApiKey(string apikey) { + return ExecuteCommand($"--api-key {apikey}"); + } + + public string ApiUrl(string apiurl) { + if (string.IsNullOrWhiteSpace(apiurl)) { + throw new ArgumentException("API URL cannot be empty or whitespace", nameof(apiurl)); + } + + if (!Uri.TryCreate(apiurl, UriKind.Absolute, out var uriResult) || + (uriResult.Scheme != Uri.UriSchemeHttp && uriResult.Scheme != Uri.UriSchemeHttps)) { + throw new ArgumentException("Invalid API URL format. Must be a valid HTTP/HTTPS URL", nameof(apiurl)); + } + + var escapedUrl = apiurl.Replace("\"", "\\\""); + return ExecuteCommand($"--api-url \"{escapedUrl}\""); + } + public string SsoUrl(string ssoUrl) { + if (string.IsNullOrWhiteSpace(ssoUrl)) { + throw new ArgumentException("SSO URL cannot be empty or whitespace", nameof(ssoUrl)); + } + + if (!Uri.TryCreate(ssoUrl, UriKind.Absolute, out var uriResult) || + (uriResult.Scheme != Uri.UriSchemeHttp && uriResult.Scheme != Uri.UriSchemeHttps)) { + throw new ArgumentException("Invalid SSO URL format. Must be a valid HTTP/HTTPS URL", nameof(ssoUrl)); + } + + var escapedUrl = ssoUrl.Replace("\"", "\\\""); + return ExecuteCommand($"--sso-url \"{escapedUrl}\""); + } + public string Login(string username, string password){ + if (string.IsNullOrWhiteSpace(username)){ + throw new ArgumentException("Username cannot be empty or whitespace", nameof(username)); + } + + if (string.IsNullOrWhiteSpace(password)){ + throw new ArgumentException("Password cannot be empty or whitespace", nameof(password)); + } + + var escapedUsername = username.Replace("\"", "\\\""); + var escapedPassword = password.Replace("\"", "\\\""); + + return ExecuteCommand($"--login --username \"{escapedUsername}\" --password \"{escapedPassword}\""); + } + + public string LoginWithToken(string token) { + if (string.IsNullOrWhiteSpace(token)) { + throw new ArgumentException("Token cannot be empty or whitespace", nameof(token)); + } + + var escapedToken = token.Replace("\"", "\\\""); + return ExecuteCommand($"--login --token \"{escapedToken}\""); + } + + //When youre logged in use this function + public string TestAuth() { + return ExecuteCommand("--test-auth"); + } + + //Without beeing logged + public string TestAuth(string? apiUrl = null, string? token = null) { + var args = new List { "--test-auth" }; + + if (!string.IsNullOrEmpty(apiUrl)) { + if (!Uri.IsWellFormedUriString(apiUrl, UriKind.Absolute)) + throw new ArgumentException("Invalid API URL format", nameof(apiUrl)); + + args.Add($"--api-url \"{apiUrl}\""); + } + + if (!string.IsNullOrEmpty(token)) + args.Add($"--token \"{token}\""); + + return ExecuteCommand(string.Join(" ", args)); + } + public string ExecuteCommand(string arguments) + { + if (_verbose) + { + Console.WriteLine($"Executing: {_pythonPath} {_scriptPath} {arguments}"); + } + + var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = _pythonPath, + Arguments = $"\"{_scriptPath}\" {arguments}", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + process.Start(); + var output = process.StandardOutput.ReadToEnd(); + var error = process.StandardError.ReadToEnd(); + process.WaitForExit(); + + if (_verbose) + { + Console.WriteLine("Command output:"); + Console.WriteLine(output); + if (!string.IsNullOrEmpty(error)) + { + Console.WriteLine("Error output:\n" + error); + } + } + + if (!string.IsNullOrEmpty(error)) + { + throw new InvalidOperationException($"Python CLI error: {error}"); + } + + return output.Trim(); + } +} \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.csproj b/corebrain/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.csproj new file mode 100644 index 0000000..bf9d3be --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.csproj @@ -0,0 +1,7 @@ + + + net9.0 + enable + enable + + diff --git a/corebrain/corebrain/wrappers/csharp/LICENSE b/corebrain/corebrain/wrappers/csharp/LICENSE new file mode 100644 index 0000000..8e423f6 --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Oliwier Adamczyk + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/corebrain/corebrain/wrappers/csharp/README.md b/corebrain/corebrain/wrappers/csharp/README.md new file mode 100644 index 0000000..3d113c5 --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp/README.md @@ -0,0 +1,77 @@ +# CoreBrain-CS + +[![NuGet Version](https://img.shields.io/nuget/v/CorebrainCS.svg)](https://www.nuget.org/packages/CorebrainCS/) +[![Python Requirement](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/) + +A C# wrapper for the CoreBrain Python CLI tool, providing seamless integration between .NET applications and CoreBrain's cognitive computing capabilities. + +## Features + +- 🚀 Native C# interface for CoreBrain functions +- 🛠️ Supports both development and production workflows + +## Installation + +### Prerequisites + +- [.NET 8.0 SDK](https://dotnet.microsoft.com/download) +- [Python 3.8+](https://www.python.org/downloads/) + +## Corebrain installation + +See the main corebrain package installation on https://github.com/ceoweggo/Corebrain/blob/main/README.md#installation + +## Basic Usage + +```csharp +using CorebrainCS; + +// Initialize wrapper (auto-detects Python environment) +var corebrain = new CorebrainCS(); + +// Get version +Console.WriteLine($"CoreBrain version: {corebrain.Version()}"); +``` + +## Advanced Configuration + +```csharp +// Custom configuration +var corebrain = new CorebrainCS( + pythonPath: "path/to/python", // Custom python path + scriptPath: "path/to/cli", // Custom CoreBrain CLI path + verbose: true // Enable debug logging +); +``` + +## Common Commands + +| Command | C# Method | Description | +|---------|-----------|-------------| +| `--version` | `.Version()` | Get CoreBrain version | + + + +### File Structure + +``` +Corebrain-CS/ +├── CorebrainCS/ # C# wrapper library +├── CorebrainCLI/ # Example consumer app +├── corebrain/ # Embedded Python package +``` + +## License + +MIT License - See [LICENSE](LICENSE) for details. diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/.gitignore b/corebrain/corebrain/wrappers/csharp_cli_api/.gitignore new file mode 100644 index 0000000..5a8763b --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp_cli_api/.gitignore @@ -0,0 +1,548 @@ +# Created by https://www.toptal.com/developers/gitignore/api/csharp,dotnetcore,aspnetcore,visualstudiocode +# Edit at https://www.toptal.com/developers/gitignore?templates=csharp,dotnetcore,aspnetcore,visualstudiocode + +### ASPNETCore ### +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# DNX +project.lock.json +project.fragment.lock.json +artifacts/ + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# TODO: Comment the next line if you want to checkin your web deploy settings +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config +# NuGet v3's project.json files produces more ignoreable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +node_modules/ +orleans.codegen.cs + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush +.cr/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/ + +### Csharp ### +## +## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore + +# User-specific files +*.rsuser + +# User-specific files (MonoDevelop/Xamarin Studio) + +# Mono auto generated files +mono_crash.* + +# Build results +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +# Uncomment if you have tasks that create the project's static files in wwwroot + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results + +# NUnit +nunit-*.xml + +# Build Results of an ATL Project + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_h.h +*.iobj +*.ipdb +*_wpftmp.csproj +*.tlog + +# Chutzpah Test files + +# Visual C++ cache files + +# Visual Studio profiler + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace + +# Guidance Automation Toolkit + +# ReSharper is a .NET coding add-in + +# TeamCity is a build add-in + +# DotCover is a Code Coverage Tool + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage*.json +coverage*.xml +coverage*.info + +# Visual Studio code coverage results + +# NCrunch + +# MightyMoose + +# Web workbench (sass) + +# Installshield output folder + +# DocProject is a documentation generator add-in + +# Click-Once directory + +# Publish Web Output +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted + +# NuGet Packages +# NuGet Symbol Packages +*.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files + +# Microsoft Azure Build Output + +# Microsoft Azure Emulator + +# Windows Store app package directories and files +*.appx +*.appxbundle +*.appxupload + +# Visual Studio cache files +# files ending in .cache can be ignored +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) + +# RIA/Silverlight projects + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.ndf + +# Business Intelligence projects +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes + +# GhostDoc plugin setting file + +# Node.js Tools for Visual Studio + +# Visual Studio 6 build log + +# Visual Studio 6 workspace options file + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio 6 auto-generated project file (contains which files were open etc.) +*.vbp + +# Visual Studio 6 workspace and project file (working project files containing files to include in project) +*.dsw +*.dsp + +# Visual Studio 6 technical files + +# Visual Studio LightSwitch build output + +# Paket dependency manager + +# FAKE - F# Make + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# Visual Studio History (VSHistory) files +.vshistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ + +# Fody - auto-generated XML schema +FodyWeavers.xsd + +# VS Code files for those working on multiple tools +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# Local History for Visual Studio Code +.history/ + +# Windows Installer files from build outputs +*.cab +*.msi +*.msix +*.msm +*.msp + +# JetBrains Rider + +### DotnetCore ### +# .NET Core build folders +bin/ +obj/ + +# Common node modules locations +/node_modules +/wwwroot/node_modules + +### VisualStudioCode ### +!.vscode/*.code-snippets + +# Local History for Visual Studio Code + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +# End of https://www.toptal.com/developers/gitignore/api/csharp,dotnetcore,aspnetcore,visualstudiocode \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/.vscode/settings.json b/corebrain/corebrain/wrappers/csharp_cli_api/.vscode/settings.json new file mode 100644 index 0000000..8de99dd --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp_cli_api/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "dotnet.defaultSolution": "CorebrainCLIAPI.sln" +} \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/CorebrainCLIAPI.sln b/corebrain/corebrain/wrappers/csharp_cli_api/CorebrainCLIAPI.sln new file mode 100644 index 0000000..81bb4a4 --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp_cli_api/CorebrainCLIAPI.sln @@ -0,0 +1,50 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CorebrainCLIAPI", "src\CorebrainCLIAPI\CorebrainCLIAPI.csproj", "{3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Debug|x64.ActiveCfg = Debug|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Debug|x64.Build.0 = Debug|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Debug|x86.ActiveCfg = Debug|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Debug|x86.Build.0 = Debug|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Release|Any CPU.Build.0 = Release|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Release|x64.ActiveCfg = Release|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Release|x64.Build.0 = Release|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Release|x86.ActiveCfg = Release|Any CPU + {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Release|x86.Build.0 = Release|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Debug|x64.ActiveCfg = Debug|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Debug|x64.Build.0 = Debug|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Debug|x86.ActiveCfg = Debug|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Debug|x86.Build.0 = Debug|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Release|Any CPU.Build.0 = Release|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Release|x64.ActiveCfg = Release|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Release|x64.Build.0 = Release|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Release|x86.ActiveCfg = Release|Any CPU + {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {C5CF7B2F-DA16-24C6-929A-8AB8C4831AB0} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {1B7A4995-2D77-4398-BE28-B3B52C1E351B} = {C5CF7B2F-DA16-24C6-929A-8AB8C4831AB0} + EndGlobalSection +EndGlobal diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/README.md b/corebrain/corebrain/wrappers/csharp_cli_api/README.md new file mode 100644 index 0000000..41b69c8 --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp_cli_api/README.md @@ -0,0 +1,18 @@ +# Corebrain CLI API + +## Quick Start + +### Prerequisites + +- Python 3.8+ +- .NET 6.0+ +- Node.js 14+ +- Git + +### Installation + +1. Create **venv** in corebrain directory +2. Continue with installation provided here https://github.com/ceoweggo/Corebrain/blob/pre-release-v0.2.0/README.md#development-installation +3. If you changed the installation directory of venv or corebrain, change the paths in `CorebrainCLIAPI/appsettings.json` +4. Go to `src/CorebrainCLIAPI` +5. run `dotnet run` \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/src/.editorconfig b/corebrain/corebrain/wrappers/csharp_cli_api/src/.editorconfig new file mode 100644 index 0000000..0bcaf64 --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp_cli_api/src/.editorconfig @@ -0,0 +1,432 @@ +# This file is the top-most EditorConfig file +root = true + +#All Files +[*] +charset = utf-8 +indent_style = space +indent_size = 2 +insert_final_newline = true +trim_trailing_whitespace = true +end_of_line = lf + +########################################## +# File Extension Settings +########################################## + +# Visual Studio Solution Files +[*.sln] +indent_style = tab + +# Visual Studio XML Project Files +[*.{csproj,vbproj,vcxproj.filters,proj,projitems,shproj}] +indent_size = 2 + +# XML Configuration Files +[*.{xml,config,props,targets,nuspec,resx,ruleset,vsixmanifest,vsct}] +indent_size = 2 + +# JSON Files +[*.{json,json5,webmanifest}] +indent_size = 2 + +# YAML Files +[*.{yml,yaml}] +indent_size = 2 + +# Markdown Files +[*.{md,mdx}] +trim_trailing_whitespace = false + +# Web Files +[*.{htm,html,js,jsm,ts,tsx,cjs,cts,ctsx,mjs,mts,mtsx,css,sass,scss,less,pcss,svg,vue}] +indent_size = 2 + +# Batch Files +[*.{cmd,bat}] +end_of_line = crlf + +# Bash Files +[*.sh] +end_of_line = lf + +# Makefiles +[Makefile] +indent_style = tab + +[{*_Generated.cs, *.g.cs, *.generated.cs}] +# Ignore a lack of documentation for generated code. Doesn't apply to builds, +# just to viewing generation output. +dotnet_diagnostic.CS1591.severity = none + +########################################## +# Default .NET Code Style Severities +########################################## + +[*.{cs,csx,cake,vb,vbx}] +# Default Severity for all .NET Code Style rules below +dotnet_analyzer_diagnostic.severity = warning + +########################################## +# Language Rules +########################################## + +# .NET Style Rules +[*.{cs,csx,cake,vb,vbx}] + +# "this." and "Me." qualifiers +dotnet_style_qualification_for_field = false +dotnet_style_qualification_for_property = false +dotnet_style_qualification_for_method = false +dotnet_style_qualification_for_event = false + +# Language keywords instead of framework type names for type references +dotnet_style_predefined_type_for_locals_parameters_members = true:warning +dotnet_style_predefined_type_for_member_access = true:warning + +# Modifier preferences +dotnet_style_require_accessibility_modifiers = always:warning +csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:warning +visual_basic_preferred_modifier_order = Partial,Default,Private,Protected,Public,Friend,NotOverridable,Overridable,MustOverride,Overloads,Overrides,MustInherit,NotInheritable,Static,Shared,Shadows,ReadOnly,WriteOnly,Dim,Const,WithEvents,Widening,Narrowing,Custom,Async:warning +dotnet_style_readonly_field = true:warning +dotnet_diagnostic.IDE0036.severity = warning + + +# Parentheses preferences +dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:warning +dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:warning +dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:warning +dotnet_style_parentheses_in_other_operators = never_if_unnecessary:warning + +# Expression-level preferences +dotnet_style_object_initializer = true:warning +dotnet_style_collection_initializer = true:warning +dotnet_style_explicit_tuple_names = true:warning +dotnet_style_prefer_inferred_tuple_names = true:warning +dotnet_style_prefer_inferred_anonymous_type_member_names = true:warning +dotnet_style_prefer_auto_properties = true:warning +dotnet_style_prefer_conditional_expression_over_assignment = true:suggestion +dotnet_diagnostic.IDE0045.severity = suggestion +dotnet_style_prefer_conditional_expression_over_return = true:suggestion +dotnet_diagnostic.IDE0046.severity = suggestion +dotnet_style_prefer_compound_assignment = true:warning +dotnet_style_prefer_simplified_interpolation = true:warning +dotnet_style_prefer_simplified_boolean_expressions = true:warning + +# Null-checking preferences +dotnet_style_coalesce_expression = true:warning +dotnet_style_null_propagation = true:warning +dotnet_style_prefer_is_null_check_over_reference_equality_method = true:warning + +# File header preferences +# Keep operators at end of line when wrapping. +dotnet_style_operator_placement_when_wrapping = end_of_line:warning +csharp_style_prefer_null_check_over_type_check = true:warning + +# Code block preferences +csharp_prefer_braces = true:warning +csharp_prefer_simple_using_statement = true:suggestion +dotnet_diagnostic.IDE0063.severity = suggestion + +# C# Style Rules +[*.{cs,csx,cake}] +# 'var' preferences +csharp_style_var_for_built_in_types = true:warning +csharp_style_var_when_type_is_apparent = true:warning +csharp_style_var_elsewhere = true:warning +# Expression-bodied members +csharp_style_expression_bodied_methods = true:warning +csharp_style_expression_bodied_constructors = false:warning +csharp_style_expression_bodied_operators = true:warning +csharp_style_expression_bodied_properties = true:warning +csharp_style_expression_bodied_indexers = true:warning +csharp_style_expression_bodied_accessors = true:warning +csharp_style_expression_bodied_lambdas = true:warning +csharp_style_expression_bodied_local_functions = true:warning +# Pattern matching preferences +csharp_style_pattern_matching_over_is_with_cast_check = true:warning +csharp_style_pattern_matching_over_as_with_null_check = true:warning +csharp_style_prefer_switch_expression = true:warning +csharp_style_prefer_pattern_matching = true:warning +csharp_style_prefer_not_pattern = true:warning +# Expression-level preferences +csharp_style_inlined_variable_declaration = true:warning +csharp_prefer_simple_default_expression = true:warning +csharp_style_pattern_local_over_anonymous_function = true:warning +csharp_style_deconstructed_variable_declaration = true:warning +csharp_style_prefer_index_operator = true:warning +csharp_style_prefer_range_operator = true:warning +csharp_style_implicit_object_creation_when_type_is_apparent = true:warning +# "Null" checking preferences +csharp_style_throw_expression = true:warning +csharp_style_conditional_delegate_call = true:warning +# Code block preferences +csharp_prefer_braces = true:warning +csharp_prefer_simple_using_statement = true:suggestion +dotnet_diagnostic.IDE0063.severity = suggestion +# 'using' directive preferences +csharp_using_directive_placement = inside_namespace:warning +# Modifier preferences +# Don't suggest making public methods static. Very annoying. +csharp_prefer_static_local_function = false +# Only suggest making private methods static (if they don't use instance data). +dotnet_code_quality.CA1822.api_surface = private + +########################################## +# Unnecessary Code Rules +########################################## + +# .NET Unnecessary code rules +[*.{cs,csx,cake,vb,vbx}] + +dotnet_code_quality_unused_parameters = non_public:suggestion +dotnet_remove_unnecessary_suppression_exclusions = none +dotnet_diagnostic.IDE0079.severity = warning + +# C# Unnecessary code rules +[*.{cs,csx,cake}] + + +# Don't remove method parameters that are unused. +dotnet_diagnostic.IDE0060.severity = none +dotnet_diagnostic.RCS1163.severity = none + +# Don't remove methods that are unused. +dotnet_diagnostic.IDE0051.severity = none +dotnet_diagnostic.RCS1213.severity = none + +# Use discard variable for unused expression values. +csharp_style_unused_value_expression_statement_preference = discard_variable + +# .NET formatting rules +[*.{cs,csx,cake,vb,vbx}] + +# Organize using directives +dotnet_sort_system_directives_first = true +dotnet_separate_import_directive_groups = false + +dotnet_sort_accessibility = true + +# Dotnet namespace options +# +# We don't care about namespaces matching folder structure. Games and apps +# are complicated and you are free to organize them however you like. Change +# this if you want to enforce it. +dotnet_style_namespace_match_folder = false +dotnet_diagnostic.IDE0130.severity = none + +# C# formatting rules +[*.{cs,csx,cake}] + +# Newline options +csharp_new_line_before_open_brace = none +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true +csharp_new_line_before_members_in_object_initializers = true +csharp_new_line_before_members_in_anonymous_types = true +csharp_new_line_between_query_expression_clauses = true + +# Indentation options +csharp_indent_switch_labels = true +csharp_indent_case_contents = true +csharp_indent_case_contents_when_block = true +csharp_indent_labels = no_change +csharp_indent_block_contents = true +csharp_indent_braces = false + +# Spacing options +csharp_space_after_cast = false +csharp_space_after_keywords_in_control_flow_statements = true +csharp_space_between_parentheses = false +csharp_space_before_colon_in_inheritance_clause = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_around_binary_operators = before_and_after +csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_between_method_declaration_empty_parameter_list_parentheses = false +csharp_space_between_method_declaration_name_and_open_parenthesis = false +csharp_space_between_method_call_parameter_list_parentheses = false +csharp_space_between_method_call_empty_parameter_list_parentheses = false +csharp_space_between_method_call_name_and_opening_parenthesis = false +csharp_space_after_comma = true +csharp_space_before_comma = false +csharp_space_after_dot = false +csharp_space_before_dot = false +csharp_space_after_semicolon_in_for_statement = true +csharp_space_before_semicolon_in_for_statement = false +csharp_space_around_declaration_statements = false +csharp_space_before_open_square_brackets = false +csharp_space_between_empty_square_brackets = false +csharp_space_between_square_brackets = false + +# Wrap options +csharp_preserve_single_line_statements = false +csharp_preserve_single_line_blocks = true + +# Namespace options +csharp_style_namespace_declarations = file_scoped:warning + +########################################## +# .NET Naming Rules +########################################## +[*.{cs,csx,cake,vb,vbx}] + +# Allow underscores in names. +dotnet_diagnostic.CA1707.severity = none + +# Styles +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +dotnet_naming_style.upper_case_style.capitalization = all_upper +dotnet_naming_style.upper_case_style.word_separator = _ + +dotnet_naming_style.camel_case_style.capitalization = camel_case + +dotnet_naming_style.camel_case_underscore_style.required_prefix = _ +dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case + +# Use uppercase for all constant fields. +dotnet_naming_rule.constants_uppercase.severity = suggestion +dotnet_naming_rule.constants_uppercase.symbols = constant_fields +dotnet_naming_rule.constants_uppercase.style = upper_case_style +dotnet_naming_symbols.constant_fields.applicable_kinds = field +dotnet_naming_symbols.constant_fields.applicable_accessibilities = * +dotnet_naming_symbols.constant_fields.required_modifiers = const + +# Non-public fields should be _camelCase +dotnet_naming_rule.non_public_fields_under_camel.severity = suggestion +dotnet_naming_rule.non_public_fields_under_camel.symbols = non_public_fields +dotnet_naming_rule.non_public_fields_under_camel.style = camel_case_underscore_style +dotnet_naming_symbols.non_public_fields.applicable_kinds = field +dotnet_naming_symbols.non_public_fields.required_modifiers = +dotnet_naming_symbols.non_public_fields.applicable_accessibilities = private,private_protected,internal,protected,protected_internal + +# Public fields should be PascalCase +dotnet_naming_rule.public_fields_pascal.severity = suggestion +dotnet_naming_rule.public_fields_pascal.symbols = public_fields +dotnet_naming_rule.public_fields_pascal.style = pascal_case_style +dotnet_naming_symbols.public_fields.applicable_kinds = field +dotnet_naming_symbols.public_fields.required_modifiers = +dotnet_naming_symbols.public_fields.applicable_accessibilities = public + +# Async methods should have "Async" suffix. +# Disabled because it makes tests too verbose. +# dotnet_naming_style.end_in_async.required_suffix = Async +# dotnet_naming_style.end_in_async.capitalization = pascal_case +# dotnet_naming_rule.methods_end_in_async.symbols = methods_async +# dotnet_naming_rule.methods_end_in_async.style = end_in_async +# dotnet_naming_rule.methods_end_in_async.severity = warning +# dotnet_naming_symbols.methods_async.applicable_kinds = method +# dotnet_naming_symbols.methods_async.required_modifiers = async +# dotnet_naming_symbols.methods_async.applicable_accessibilities = * + +########################################## +# Other Naming Rules +########################################## + +# All of the following must be PascalCase: +dotnet_naming_symbols.element_group.applicable_kinds = namespace, class, enum, struct, delegate, event, method, property +dotnet_naming_rule.element_rule.symbols = element_group +dotnet_naming_rule.element_rule.style = pascal_case_style +dotnet_naming_rule.element_rule.severity = warning + +# Interfaces use PascalCase and are prefixed with uppercase 'I' +# https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-classes-structs-and-interfaces +dotnet_naming_style.prefix_interface_with_i_style.capitalization = pascal_case +dotnet_naming_style.prefix_interface_with_i_style.required_prefix = I +dotnet_naming_symbols.interface_group.applicable_kinds = interface +dotnet_naming_rule.interface_rule.symbols = interface_group +dotnet_naming_rule.interface_rule.style = prefix_interface_with_i_style +dotnet_naming_rule.interface_rule.severity = warning + +# Generics Type Parameters use PascalCase and are prefixed with uppercase 'T' +# https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-classes-structs-and-interfaces +dotnet_naming_style.prefix_type_parameters_with_t_style.capitalization = pascal_case +dotnet_naming_style.prefix_type_parameters_with_t_style.required_prefix = T +dotnet_naming_symbols.type_parameter_group.applicable_kinds = type_parameter +dotnet_naming_rule.type_parameter_rule.symbols = type_parameter_group +dotnet_naming_rule.type_parameter_rule.style = prefix_type_parameters_with_t_style +dotnet_naming_rule.type_parameter_rule.severity = warning + +# Function parameters use camelCase +# https://docs.microsoft.com/dotnet/standard/design-guidelines/naming-parameters +dotnet_naming_symbols.parameters_group.applicable_kinds = parameter +dotnet_naming_rule.parameters_rule.symbols = parameters_group +dotnet_naming_rule.parameters_rule.style = camel_case_style +dotnet_naming_rule.parameters_rule.severity = warning + +# Anything not specified uses camel case. +dotnet_naming_rule.unspecified_naming.severity = warning +dotnet_naming_rule.unspecified_naming.symbols = unspecified +dotnet_naming_rule.unspecified_naming.style = camel_case_style +dotnet_naming_symbols.unspecified.applicable_kinds = * +dotnet_naming_symbols.unspecified.applicable_accessibilities = * + +########################################## +# Rule Overrides +########################################## + +roslyn_correctness.assembly_reference_validation = relaxed + +# Allow using keywords as names +# dotnet_diagnostic.CA1716.severity = none +# Don't require culture info for ToString() +dotnet_diagnostic.CA1304.severity = none +# Don't require a string comparison for comparing strings. +dotnet_diagnostic.CA1310.severity = none +# Don't require a string format specifier. +dotnet_diagnostic.CA1305.severity = none +# Allow protected fields. +dotnet_diagnostic.CA1051.severity = none +# Don't warn about checking values that are supposedly never null. Sometimes +# they are actually null. +dotnet_diagnostic.CS8073.severity = none +# Don't remove seemingly "unnecessary" assignments, as they often have +# intended side-effects. +dotnet_diagnostic.IDE0059.severity = none +# Switch/case should always have a default clause. Tell that to Roslynator. +dotnet_diagnostic.RCS1070.severity = none +# Tell roslynator not to eat unused parameters. +dotnet_diagnostic.RCS1163.severity = none +# Tell dotnet not to remove unused parameters. +dotnet_diagnostic.IDE0060.severity = none +# Tell roslynator not to remove `partial` modifiers. +dotnet_diagnostic.RCS1043.severity = none +# Tell roslynator not to make classes static so aggressively. +dotnet_diagnostic.RCS1102.severity = none +# Roslynator wants to make properties readonly all the time, so stop it. +# The developer knows best when it comes to contract definitions with Godot. +dotnet_diagnostic.RCS1170.severity = none +# Allow expression values to go unused, even without discard variable. +# Otherwise, using Moq would be way too verbose. +dotnet_diagnostic.IDE0058.severity = none +# Don't let roslynator turn every local variable into a const. +# If we did, we'd have to specify the types of local variables far more often, +# and this style prefers type inference. +dotnet_diagnostic.RCS1118.severity = none +# Enums don't need to declare explicit values. Everyone knows they start at 0. +dotnet_diagnostic.RCS1161.severity = none +# Allow unconstrained type parameter to be checked for null. +dotnet_diagnostic.RCS1165.severity = none +# Allow keyword-based names so that parameter names like `@event` can be used. +dotnet_diagnostic.CA1716.severity = none +# Allow me to use the word Collection if I want. +dotnet_diagnostic.CA1711.severity = none +# Not disposing of objects in a test is normal within Godot because of scene tree stuff. +dotnet_diagnostic.CA1001.severity = none +# No primary constructors — not supported well by tooling. +dotnet_diagnostic.IDE0290.severity = none +# Let me comment where I like +dotnet_diagnostic.RCS1181.severity = none +# Let me write dumb if checks, keeps it readable +dotnet_diagnostic.IDE0046.severity = none +# Don't make me use expression bodies for methods +dotnet_diagnostic.IDE0022.severity = none +# Don't use collection shorhand. +dotnet_diagnostic.IDE0300.severity = none +dotnet_diagnostic.IDE0028.severity = none +dotnet_diagnostic.IDE0305.severity = none +# Don't make me populate a switch expression redundantly +dotnet_diagnostic.IDE0072.severity = none +# Leave me alone about primary constructors +dotnet_diagnostic.IDE0290.severity = none \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CommandController.cs b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CommandController.cs new file mode 100644 index 0000000..e0236d2 --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CommandController.cs @@ -0,0 +1,70 @@ +namespace CorebrainCLIAPI; + +using CorebrainCS; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Options; + +/// +/// Controller for executing Corebrain CLI commands +/// +[ApiController] +[Route("api/[controller]")] +[Produces("application/json")] +public class CommandController : ControllerBase { + private readonly CorebrainCS _corebrain; + + public CommandController(IOptions settings) { + var config = settings.Value; + _corebrain = new CorebrainCS( + config.PythonPath, + config.ScriptPath, + config.Verbose + ); + } + + /// + /// Executes a Corebrain CLI command + /// + /// + /// Sample request: + /// + /// POST /api/command + /// { + /// "arguments": "--help" + /// } + /// + /// + /// Command request containing the arguments + /// The output of the executed command + /// Returns the command output + /// If the arguments are empty + /// If there was an error executing the command + [HttpPost] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + [ProducesResponseType(StatusCodes.Status500InternalServerError)] + public IActionResult ExecuteCommand([FromBody] CommandRequest request) { + if (string.IsNullOrWhiteSpace(request.Arguments)) { + return BadRequest("Command arguments are required"); + } + + try { + var result = _corebrain.ExecuteCommand(request.Arguments); + return Ok(result); + } + catch (Exception ex) { + return StatusCode(500, $"Error executing command: {ex.Message}"); + } + } + + /// + /// Command request model + /// + public class CommandRequest { + /// + /// The arguments to pass to the Corebrain CLI + /// + /// --help + public required string Arguments { get; set; } + } +} diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainCLIAPI.csproj b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainCLIAPI.csproj new file mode 100644 index 0000000..279f7d0 --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainCLIAPI.csproj @@ -0,0 +1,20 @@ + + + + net9.0 + enable + enable + true + $(NoWarn);1591 + + + + + + + + + + + + diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainSettings.cs b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainSettings.cs new file mode 100644 index 0000000..82143b9 --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainSettings.cs @@ -0,0 +1,8 @@ +namespace CorebrainCLIAPI; + +public class CorebrainSettings +{ + public string PythonPath { get; set; } + public string ScriptPath { get; set; } + public bool Verbose { get; set; } = false; +} diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Program.cs b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Program.cs new file mode 100644 index 0000000..3ddd6a1 --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Program.cs @@ -0,0 +1,49 @@ +using System.Reflection; +using CorebrainCLIAPI; +using Microsoft.OpenApi.Models; + +var builder = WebApplication.CreateBuilder(args); + +// CORS policy to allow requests from the frontend +builder.Services.AddCors(options => options.AddPolicy("AllowFrontend", policy => + policy.WithOrigins("http://localhost:5173") + .AllowAnyMethod() + .AllowAnyHeader() +)); + +// Configure controllers and settings +builder.Services.AddControllers(); +builder.Services.Configure( + builder.Configuration.GetSection("CorebrainSettings")); + +// Swagger / OpenAPI +builder.Services.AddEndpointsApiExplorer(); +builder.Services.AddSwaggerGen(c => { + c.SwaggerDoc("v1", new OpenApiInfo { + Title = "Corebrain CLI API", + Version = "v1", + Description = "ASP.NET Core Web API for interfacing with Corebrain CLI commands" + }); + + var xmlFile = $"{Assembly.GetExecutingAssembly().GetName().Name}.xml"; + var xmlPath = Path.Combine(AppContext.BaseDirectory, xmlFile); + if (File.Exists(xmlPath)) { + c.IncludeXmlComments(xmlPath); + } +}); + +var app = builder.Build(); + +// Middleware pipeline +app.UseCors("AllowFrontend"); + +if (app.Environment.IsDevelopment()) { + app.UseSwagger(); + app.UseSwaggerUI(c => + c.SwaggerEndpoint("/swagger/v1/swagger.json", "Corebrain CLI API v1")); +} + +app.UseHttpsRedirection(); +app.UseAuthorization(); +app.MapControllers(); +app.Run(); diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Properties/launchSettings.json b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Properties/launchSettings.json new file mode 100644 index 0000000..f212316 --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Properties/launchSettings.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": false, + "applicationUrl": "http://localhost:5140", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": false, + "applicationUrl": "https://localhost:7261;http://localhost:5140", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.Development.json b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.Development.json new file mode 100644 index 0000000..0c208ae --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.json b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.json new file mode 100644 index 0000000..0ab3335 --- /dev/null +++ b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.json @@ -0,0 +1,14 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*", + "CorebrainSettings": { + "PythonPath": "../../../../../venv/Scripts/python.exe", + "ScriptPath": "../../../../cli", + "Verbose": false + } +} diff --git a/corebrain/docs/Makefile b/corebrain/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/corebrain/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/corebrain/docs/README.md b/corebrain/docs/README.md new file mode 100644 index 0000000..cd094fd --- /dev/null +++ b/corebrain/docs/README.md @@ -0,0 +1,13 @@ +### Generating docs + +Run in terminal: + +```bash + +.\docs\make.bat html + +``` + + + + diff --git a/corebrain/docs/make.bat b/corebrain/docs/make.bat new file mode 100644 index 0000000..dc1312a --- /dev/null +++ b/corebrain/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/corebrain/docs/source/_static/custom.css b/corebrain/docs/source/_static/custom.css new file mode 100644 index 0000000..e69de29 diff --git a/corebrain/docs/source/conf.py b/corebrain/docs/source/conf.py new file mode 100644 index 0000000..a59ab3a --- /dev/null +++ b/corebrain/docs/source/conf.py @@ -0,0 +1,25 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath('../..')) + +project = 'Corebrain Documentation' +copyright = '2025, Corebrain' +author = 'Corebrain' +release = '0.1' + +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', + 'sphinx.ext.viewcode', + 'sphinx_copybutton', + 'sphinx_design', +] + +templates_path = ['_templates'] +exclude_patterns = [] + +html_theme = 'furo' +html_css_files = ['custom.css'] +html_static_path = ['_static'] + diff --git a/corebrain/docs/source/corebrain.cli.auth.rst b/corebrain/docs/source/corebrain.cli.auth.rst new file mode 100644 index 0000000..85bb14a --- /dev/null +++ b/corebrain/docs/source/corebrain.cli.auth.rst @@ -0,0 +1,29 @@ +corebrain.cli.auth package +========================== + +Submodules +---------- + +corebrain.cli.auth.api\_keys module +----------------------------------- + +.. automodule:: corebrain.cli.auth.api_keys + :members: + :show-inheritance: + :undoc-members: + +corebrain.cli.auth.sso module +----------------------------- + +.. automodule:: corebrain.cli.auth.sso + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain.cli.auth + :members: + :show-inheritance: + :undoc-members: diff --git a/corebrain/docs/source/corebrain.cli.rst b/corebrain/docs/source/corebrain.cli.rst new file mode 100644 index 0000000..3fdb48b --- /dev/null +++ b/corebrain/docs/source/corebrain.cli.rst @@ -0,0 +1,53 @@ +corebrain.cli package +===================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + corebrain.cli.auth + +Submodules +---------- + +corebrain.cli.commands module +----------------------------- + +.. automodule:: corebrain.cli.commands + :members: + :show-inheritance: + :undoc-members: + +corebrain.cli.common module +--------------------------- + +.. automodule:: corebrain.cli.common + :members: + :show-inheritance: + :undoc-members: + +corebrain.cli.config module +--------------------------- + +.. automodule:: corebrain.cli.config + :members: + :show-inheritance: + :undoc-members: + +corebrain.cli.utils module +-------------------------- + +.. automodule:: corebrain.cli.utils + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain.cli + :members: + :show-inheritance: + :undoc-members: diff --git a/corebrain/docs/source/corebrain.config.rst b/corebrain/docs/source/corebrain.config.rst new file mode 100644 index 0000000..4168d30 --- /dev/null +++ b/corebrain/docs/source/corebrain.config.rst @@ -0,0 +1,21 @@ +corebrain.config package +======================== + +Submodules +---------- + +corebrain.config.manager module +------------------------------- + +.. automodule:: corebrain.config.manager + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain.config + :members: + :show-inheritance: + :undoc-members: diff --git a/corebrain/docs/source/corebrain.core.rst b/corebrain/docs/source/corebrain.core.rst new file mode 100644 index 0000000..0313ae3 --- /dev/null +++ b/corebrain/docs/source/corebrain.core.rst @@ -0,0 +1,45 @@ +corebrain.core package +====================== + +Submodules +---------- + +corebrain.core.client module +---------------------------- + +.. automodule:: corebrain.core.client + :members: + :show-inheritance: + :undoc-members: + +corebrain.core.common module +---------------------------- + +.. automodule:: corebrain.core.common + :members: + :show-inheritance: + :undoc-members: + +corebrain.core.query module +--------------------------- + +.. automodule:: corebrain.core.query + :members: + :show-inheritance: + :undoc-members: + +corebrain.core.test\_utils module +--------------------------------- + +.. automodule:: corebrain.core.test_utils + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain.core + :members: + :show-inheritance: + :undoc-members: diff --git a/corebrain/docs/source/corebrain.db.connectors.rst b/corebrain/docs/source/corebrain.db.connectors.rst new file mode 100644 index 0000000..d2710b3 --- /dev/null +++ b/corebrain/docs/source/corebrain.db.connectors.rst @@ -0,0 +1,29 @@ +corebrain.db.connectors package +=============================== + +Submodules +---------- + +corebrain.db.connectors.mongodb module +-------------------------------------- + +.. automodule:: corebrain.db.connectors.mongodb + :members: + :show-inheritance: + :undoc-members: + +corebrain.db.connectors.sql module +---------------------------------- + +.. automodule:: corebrain.db.connectors.sql + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain.db.connectors + :members: + :show-inheritance: + :undoc-members: diff --git a/corebrain/docs/source/corebrain.db.rst b/corebrain/docs/source/corebrain.db.rst new file mode 100644 index 0000000..751b1d4 --- /dev/null +++ b/corebrain/docs/source/corebrain.db.rst @@ -0,0 +1,62 @@ +corebrain.db package +==================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + corebrain.db.connectors + corebrain.db.schema + +Submodules +---------- + +corebrain.db.connector module +----------------------------- + +.. automodule:: corebrain.db.connector + :members: + :show-inheritance: + :undoc-members: + +corebrain.db.engines module +--------------------------- + +.. automodule:: corebrain.db.engines + :members: + :show-inheritance: + :undoc-members: + +corebrain.db.factory module +--------------------------- + +.. automodule:: corebrain.db.factory + :members: + :show-inheritance: + :undoc-members: + +corebrain.db.interface module +----------------------------- + +.. automodule:: corebrain.db.interface + :members: + :show-inheritance: + :undoc-members: + +corebrain.db.schema\_file module +-------------------------------- + +.. automodule:: corebrain.db.schema_file + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain.db + :members: + :show-inheritance: + :undoc-members: diff --git a/corebrain/docs/source/corebrain.db.schema.rst b/corebrain/docs/source/corebrain.db.schema.rst new file mode 100644 index 0000000..ccc435b --- /dev/null +++ b/corebrain/docs/source/corebrain.db.schema.rst @@ -0,0 +1,29 @@ +corebrain.db.schema package +=========================== + +Submodules +---------- + +corebrain.db.schema.extractor module +------------------------------------ + +.. automodule:: corebrain.db.schema.extractor + :members: + :show-inheritance: + :undoc-members: + +corebrain.db.schema.optimizer module +------------------------------------ + +.. automodule:: corebrain.db.schema.optimizer + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain.db.schema + :members: + :show-inheritance: + :undoc-members: diff --git a/corebrain/docs/source/corebrain.network.rst b/corebrain/docs/source/corebrain.network.rst new file mode 100644 index 0000000..3d94c4f --- /dev/null +++ b/corebrain/docs/source/corebrain.network.rst @@ -0,0 +1,21 @@ +corebrain.network package +========================= + +Submodules +---------- + +corebrain.network.client module +------------------------------- + +.. automodule:: corebrain.network.client + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain.network + :members: + :show-inheritance: + :undoc-members: diff --git a/corebrain/docs/source/corebrain.rst b/corebrain/docs/source/corebrain.rst new file mode 100644 index 0000000..fb44332 --- /dev/null +++ b/corebrain/docs/source/corebrain.rst @@ -0,0 +1,42 @@ +corebrain package +================= + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + corebrain.cli + corebrain.config + corebrain.core + corebrain.db + corebrain.network + corebrain.utils + +Submodules +---------- + +corebrain.cli module +-------------------- + +.. automodule:: corebrain.cli + :members: + :show-inheritance: + :undoc-members: + +corebrain.sdk module +-------------------- + +.. automodule:: corebrain.sdk + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain + :members: + :show-inheritance: + :undoc-members: diff --git a/corebrain/docs/source/corebrain.utils.rst b/corebrain/docs/source/corebrain.utils.rst new file mode 100644 index 0000000..4294529 --- /dev/null +++ b/corebrain/docs/source/corebrain.utils.rst @@ -0,0 +1,37 @@ +corebrain.utils package +======================= + +Submodules +---------- + +corebrain.utils.encrypter module +-------------------------------- + +.. automodule:: corebrain.utils.encrypter + :members: + :show-inheritance: + :undoc-members: + +corebrain.utils.logging module +------------------------------ + +.. automodule:: corebrain.utils.logging + :members: + :show-inheritance: + :undoc-members: + +corebrain.utils.serializer module +--------------------------------- + +.. automodule:: corebrain.utils.serializer + :members: + :show-inheritance: + :undoc-members: + +Module contents +--------------- + +.. automodule:: corebrain.utils + :members: + :show-inheritance: + :undoc-members: diff --git a/corebrain/docs/source/index.rst b/corebrain/docs/source/index.rst new file mode 100644 index 0000000..03ce071 --- /dev/null +++ b/corebrain/docs/source/index.rst @@ -0,0 +1,14 @@ +.. Documentation documentation master file, created by + sphinx-quickstart on Fri May 16 16:20:00 2025. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Corebrain's documentation! +=========================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + modules + diff --git a/corebrain/docs/source/modules.rst b/corebrain/docs/source/modules.rst new file mode 100644 index 0000000..7f3849e --- /dev/null +++ b/corebrain/docs/source/modules.rst @@ -0,0 +1,7 @@ +corebrain +========= + +.. toctree:: + :maxdepth: 4 + + corebrain diff --git a/corebrain/examples/add_config.py b/corebrain/examples/add_config.py new file mode 100644 index 0000000..963996a --- /dev/null +++ b/corebrain/examples/add_config.py @@ -0,0 +1,27 @@ +from corebrain import ConfigManager + +# Initialize config manager +config_manager = ConfigManager() + +# API key +api_key = "sk_bH8rnkIHCDF1BlRmgS9s6QAK" + +# Database configuration +db_config = { + "type": "sql", # or "mongodb" for MongoDB + "engine": "postgresql", # or "mysql", "sqlite", etc. + "host": "localhost", + "port": 5432, + "database": "your_database", + "user": "your_username", + "password": "your_password" +} + +# Add configuration +config_id = config_manager.add_config(api_key, db_config) +print(f"Configuration added with ID: {config_id}") + +# List all configurations +print("\nAvailable configurations:") +configs = config_manager.list_configs(api_key) +print(configs) \ No newline at end of file diff --git a/corebrain/examples/complex.py b/corebrain/examples/complex.py new file mode 100644 index 0000000..e66c21b --- /dev/null +++ b/corebrain/examples/complex.py @@ -0,0 +1,23 @@ +from corebrain import init + +api_key = "sk_bH8rnkIHCDF1BlRmgS9s6QAK" +#config_id = "c9913a04-a530-4ae3-a877-8e295be87f78" # MONGODB +config_id = "8bdba894-34a7-4453-b665-e640d11fd463" # POSTGRES + +# Initialize the SDK with API key and configuration ID +corebrain = init( + api_key=api_key, + config_id=config_id +) + +""" +Corebrain possible arguments (all optionals): + +- execute_query (bool) +- explain_results (bool) +- detail_level (string = "full") +""" + +result = corebrain.ask("Devuélveme 5 datos interesantes sobre mis usuarios", detail_level="full") + +print(result['explanation']) diff --git a/corebrain/examples/list_schema.py b/corebrain/examples/list_schema.py new file mode 100644 index 0000000..daeba01 --- /dev/null +++ b/corebrain/examples/list_schema.py @@ -0,0 +1,162 @@ +""" +Example script to list database schema and configuration details. +This helps diagnose issues with database connections and schema extraction. +""" +import os +import json +import logging +import psycopg2 +from corebrain import init + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +def verify_postgres_connection(db_config): + """Verify PostgreSQL connection and list tables directly""" + logger.info("\n=== Direct PostgreSQL Connection Test ===") + try: + # Create connection + conn = psycopg2.connect( + host=db_config.get("host", "localhost"), + user=db_config.get("user", ""), + password=db_config.get("password", ""), + dbname=db_config.get("database", ""), + port=db_config.get("port", 5432) + ) + + # Create cursor + cur = conn.cursor() + + # Test connection + cur.execute("SELECT version();") + version = cur.fetchone() + logger.info(f"PostgreSQL Version: {version[0]}") + + # List all schemas + cur.execute(""" + SELECT schema_name + FROM information_schema.schemata + WHERE schema_name NOT IN ('information_schema', 'pg_catalog'); + """) + schemas = cur.fetchall() + logger.info("\nAvailable Schemas:") + for schema in schemas: + logger.info(f" - {schema[0]}") + + # List all tables in public schema + cur.execute(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public'; + """) + tables = cur.fetchall() + logger.info("\nTables in public schema:") + for table in tables: + logger.info(f" - {table[0]}") + + # Get column info for each table + cur.execute(f""" + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = '{table[0]}'; + """) + columns = cur.fetchall() + logger.info(" Columns:") + for col in columns: + logger.info(f" - {col[0]}: {col[1]}") + + cur.close() + conn.close() + + except Exception as e: + logger.error(f"Error in direct PostgreSQL connection: {str(e)}", exc_info=True) + +def main(): + # Get API key from environment variable + api_key = "sk_bH8rnkIHCDF1BlRmgS9s6QAK" + if not api_key: + raise ValueError("Please set COREBRAIN_API_KEY environment variable") + + # Get config ID from environment variable + config_id = "8bdba894-34a7-4453-b665-e640d11fd463" + if not config_id: + raise ValueError("Please set COREBRAIN_CONFIG_ID environment variable") + + logger.info("Initializing Corebrain SDK...") + try: + corebrain = init( + api_key=api_key, + config_id=config_id, + skip_verification=True # Skip API key verification due to the error + ) + except Exception as e: + logger.error(f"Error initializing SDK: {str(e)}") + return + + # Print configuration details + logger.info("\n=== Configuration Details ===") + logger.info(f"Database Type: {corebrain.db_config.get('type')}") + logger.info(f"Database Engine: {corebrain.db_config.get('engine')}") + logger.info(f"Database Name: {corebrain.db_config.get('database')}") + logger.info(f"Config ID: {corebrain.config_id}") + + # Print full database configuration + logger.info("\n=== Full Database Configuration ===") + logger.info(json.dumps(corebrain.db_config, indent=2)) + + # If PostgreSQL, verify connection directly + if corebrain.db_config.get("type", "").lower() == "sql" and \ + corebrain.db_config.get("engine", "").lower() == "postgresql": + verify_postgres_connection(corebrain.db_config) + + # Extract and print schema + logger.info("\n=== Database Schema ===") + try: + schema = corebrain._extract_db_schema(detail_level="full") + + # Print schema summary + logger.info(f"Schema Type: {schema.get('type')}") + logger.info(f"Total Collections: {schema.get('total_collections', 0)}") + logger.info(f"Included Collections: {schema.get('included_collections', 0)}") + + # Print tables/collections + if schema.get("tables"): + logger.info("\n=== Tables/Collections ===") + for table_name, table_info in schema["tables"].items(): + logger.info(f"\nTable/Collection: {table_name}") + + # Print columns/fields + if "columns" in table_info: + logger.info("Columns:") + for col in table_info["columns"]: + logger.info(f" - {col['name']}: {col['type']}") + elif "fields" in table_info: + logger.info("Fields:") + for field in table_info["fields"]: + logger.info(f" - {field['name']}: {field['type']}") + + # Print document count if available + if "doc_count" in table_info: + logger.info(f"Document Count: {table_info['doc_count']}") + + # Print sample data if available + if "sample_data" in table_info and table_info["sample_data"]: + logger.info("Sample Data:") + for doc in table_info["sample_data"][:2]: # Show only first 2 documents + logger.info(f" {json.dumps(doc, indent=2)}") + else: + logger.warning("No tables/collections found in schema!") + + # Print raw schema for debugging + logger.info("\n=== Raw Schema ===") + logger.info(json.dumps(schema, indent=2)) + except Exception as e: + logger.error(f"Error extracting schema: {str(e)}", exc_info=True) + +if __name__ == "__main__": + try: + main() + except Exception as e: + logger.error(f"Error: {str(e)}", exc_info=True) \ No newline at end of file diff --git a/corebrain/examples/simple.py b/corebrain/examples/simple.py new file mode 100644 index 0000000..483c546 --- /dev/null +++ b/corebrain/examples/simple.py @@ -0,0 +1,15 @@ +from corebrain import init + +api_key = "sk_bH8rnkIHCDF1BlRmgS9s6QAK" +#config_id = "c9913a04-a530-4ae3-a877-8e295be87f78" MONGODB +config_id = "8bdba894-34a7-4453-b665-e640d11fd463" # POSTGRES + +# Initialize the SDK with API key and configuration ID +corebrain = init( + api_key=api_key, + config_id=config_id +) + +result = corebrain.ask("Analiza los usuarios y los servicios asociados a estos usuarios.") +print(result["explanation"]) + diff --git a/corebrain/health.py b/corebrain/health.py new file mode 100644 index 0000000..55ffd9d --- /dev/null +++ b/corebrain/health.py @@ -0,0 +1,47 @@ +# check_imports.py +import os +import importlib +import sys + +def check_imports(package_name, directory): + """ + Recursively checks imports into a directory. + """ + + for item in os.listdir(directory): + path = os.path.join(directory, item) + + # Ignore hidden folders or __pycache__ + if item.startswith('.') or item == '__pycache__': + continue + + if os.path.isdir(path): + + if os.path.exists(os.path.join(path, '__init__.py')): + subpackage = f"{package_name}.{item}" + try: + print(f"Verificating subpackage: {subpackage}") + importlib.import_module(subpackage) + check_imports(subpackage, path) + except Exception as e: + print(f"ERROR in {subpackage}: {e}") + + elif item.endswith('.py') and item != '__init__.py': + module_name = f"{package_name}.{item[:-3]}" # quitar .py + try: + print(f"Verificating module: {module_name}") + importlib.import_module(module_name) + except Exception as e: + print(f"ERROR in {module_name}: {e}") + +sys.path.insert(0, '.') + +# Verify all main modules +for pkg in ['corebrain']: + if os.path.exists(pkg): + try: + print(f"\Verificating pkg: {pkg}") + importlib.import_module(pkg) + check_imports(pkg, pkg) + except Exception as e: + print(f"ERROR in pkg {pkg}: {e}") \ No newline at end of file diff --git a/corebrain/pyproject.toml b/corebrain/pyproject.toml new file mode 100644 index 0000000..76f919d --- /dev/null +++ b/corebrain/pyproject.toml @@ -0,0 +1,85 @@ +[project] +name = "corebrain" +version = "0.1.0" +description = "SDK de Corebrain para consultas en lenguaje natural a bases de datos" +readme = "README.md" +authors = [ + {name = "Rubén Ayuso", email = "ruben@globodain.com"} +] +license = {text = "MIT"} +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] +requires-python = ">=3.8" +dependencies = [ + "httpx>=0.24.0", + "sqlalchemy>=2.0.0", + "pydantic>=2.0.0", + "cryptography>=40.0.0", + "python-dotenv>=1.0.0", + "typing-extensions>=4.4.0", + "requests>=2.28.0", + "asyncio>=3.4.3", + "psycopg2-binary>=2.9.0", # En lugar de psycopg2 para evitar problemas de compilación + "mysql-connector-python>=8.0.23", + "pymongo>=4.4.0", +] + +[project.optional-dependencies] +postgres = ["psycopg2-binary>=2.9.0"] +mongodb = ["pymongo>=4.4.0"] +mysql = ["mysql-connector-python>=8.0.23"] +all_db = [ + "psycopg2-binary>=2.9.0", + "pymongo>=4.4.0", + "mysql-connector-python>=8.0.23", +] +dev = [ + "pytest>=7.0.0", + "pytest-cov>=4.0.0", + "black>=23.0.0", + "isort>=5.12.0", + "mypy>=1.3.0", + "flake8>=6.0.0", + "sphinx>=8.2.3", + "furo>=2024.8.6", +] + + +[tool.setuptools] +packages = ["corebrain"] + +[project.urls] +"Homepage" = "https://github.com/ceoweggo/Corebrain" +"Bug Tracker" = "https://github.com/ceoweggo/Corebrain/issues" + +[project.scripts] +corebrain = "corebrain.cli.__main__:main" + +[tool.black] +line-length = 100 +target-version = ["py38"] +include = '\.pyi?$' + +[tool.isort] +profile = "black" +line_length = 100 + +[tool.mypy] +python_version = "3.8" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false +disallow_incomplete_defs = false + +[tool.pytest.ini_options] +minversion = "7.0" +testpaths = ["tests"] +pythonpath = ["."] \ No newline at end of file diff --git a/corebrain/setup.ps1 b/corebrain/setup.ps1 new file mode 100644 index 0000000..3d031a4 --- /dev/null +++ b/corebrain/setup.ps1 @@ -0,0 +1,5 @@ +python -m venv venv + +.\venv\Scripts\Activate.ps1 + +pip install -e ".[dev,all_db]" \ No newline at end of file diff --git a/corebrain/setup.py b/corebrain/setup.py new file mode 100644 index 0000000..b14bc71 --- /dev/null +++ b/corebrain/setup.py @@ -0,0 +1,38 @@ +""" +Installer configuration for Corebrain package. +""" + +from setuptools import setup, find_packages + +setup( + name="corebrain", + version="1.0.0", + description="SDK for natural language ask to DB", + author="Rubén Ayuso", + author_email="ruben@globodain.com", + packages=find_packages(), + install_requires=[ + "httpx>=0.23.0", + "pymongo>=4.3.0", + "psycopg2-binary>=2.9.5", + "mysql-connector-python>=8.0.31", + "sqlalchemy>=2.0.0", + "cryptography>=39.0.0", + "pydantic>=1.10.0", + ], + python_requires=">=3.8", + entry_points={ + "console_scripts": [ + "corebrain=corebrain.__main__:main", + ], + }, + classifiers=[ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + ], +) \ No newline at end of file diff --git a/corebrain/setup.sh b/corebrain/setup.sh new file mode 100644 index 0000000..d32b7c8 --- /dev/null +++ b/corebrain/setup.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Utwórz i aktywuj środowisko wirtualne +python3 -m venv venv +source venv/bin/activate + +pip install -e ".[dev,all_db]" \ No newline at end of file From 503c45950421ef1753a7cd718cd4dc6ac2e0a584 Mon Sep 17 00:00:00 2001 From: Pawel Wasilewski Date: Fri, 23 May 2025 11:24:41 +0200 Subject: [PATCH 38/81] minor fixes --- corebrain/corebrain/db/connectors/__init__.py | 19 +- corebrain/corebrain/db/connectors/nosql.py | 366 ++++++++++++++++++ 2 files changed, 377 insertions(+), 8 deletions(-) create mode 100644 corebrain/corebrain/db/connectors/nosql.py diff --git a/corebrain/corebrain/db/connectors/__init__.py b/corebrain/corebrain/db/connectors/__init__.py index 3db5c71..65acf56 100644 --- a/corebrain/corebrain/db/connectors/__init__.py +++ b/corebrain/corebrain/db/connectors/__init__.py @@ -5,7 +5,7 @@ from typing import Dict, Any from corebrain.db.connectors.sql import SQLConnector -from corebrain.db.connectors.mongodb import MongoDBConnector +from corebrain.db.connectors.nosql import NoSQLConnector def get_connector(db_config: Dict[str, Any]): """ @@ -18,11 +18,14 @@ def get_connector(db_config: Dict[str, Any]): Instance of the appropriate connector """ db_type = db_config.get("type", "").lower() + engine = db_config.get("engine", "").lower() + + match db_type: + case "sql": + return SQLConnector(db_config, engine) + case "nosql": + return NoSQLConnector(db_config, engine) + case _: + raise ValueError(f"Unsupported database type: {db_type}") - if db_type == "sql": - engine = db_config.get("engine", "").lower() - return SQLConnector(db_config, engine) - elif db_type == "nosql" or db_type == "mongodb": - return MongoDBConnector(db_config) - else: - raise ValueError(f"Tipo de base de datos no soportado: {db_type}") \ No newline at end of file + \ No newline at end of file diff --git a/corebrain/corebrain/db/connectors/nosql.py b/corebrain/corebrain/db/connectors/nosql.py new file mode 100644 index 0000000..c738946 --- /dev/null +++ b/corebrain/corebrain/db/connectors/nosql.py @@ -0,0 +1,366 @@ +''' +NoSQL Database Connector +This module provides a basic structure for connecting to a NoSQL database. +It includes methods for connecting, disconnecting, and executing queries. +''' + +import time +import json +import re + +from typing import Dict, Any, List, Optional, Callable, Tuple + +# Try'ies for imports DB's (for now only mongoDB) +try: + import pymongo + from pymongo.errors import ConnectionFailure, ServerSelectionTimeoutError + PYMONGO_IMPORTED = True +except ImportError: + PYMONGO_IMPORTED = False +# Whe nadding new DB type write a try to it from user + +from corebrain.db.connector import DatabaseConnector +class NoSQLConnector(DatabaseConnector): + ''' + NoSQL Database Connector + This class provides a basic structure for connecting to a NoSQL database. + It includes methods for connecting, disconnecting, and executing queries. + ''' + def __init__(self, config: Dict[str, Any]): + ''' + Initialize the NoSQL database connector. + Args: + engine (str): Name of the database. + config (dict): Configuration dictionary containing connection parameters. + ''' + super().__init__(config) + self.engine = config.get("engine", "").lower() + self.client = None + self.db = None + self.config = config + self.connection_timeout = 30 # seconds + + match self.engine: + case "mongodb": + if not PYMONGO_IMPORTED: + raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") + case _: + pass + + + def connect(self) -> bool: + ''' + Connection with NoSQL DB's + Args: + self.engine (str): Name of the database. + ''' + + match self.engine: + case "mongodb": + if not PYMONGO_IMPORTED: + raise ImportError("Pymongo is not installed. Please install it to use MongoDB connector.") + try: + start_time = time.time() + # Check if connection string is provided + if "connection_string" in self.config: + connection_string = self.config["connection_string"] + if "connectTimeoutMS=" not in connection_string: + if "?" in connection_string: + connection_string += "&connectTimeoutMS=10000" + else: + connection_string += "?connectTimeoutMS=10000" + self.client = pymongo.MongoClient(connection_string) + else: + + # Setup for MongoDB connection parameters + + mongo_params = { + "host": self.config.get("host", "localhost"), + "port": int(self.config.get("port", 27017)), + "connection_timeoutMS": 10000, + "serverSelectionTimeoutMS": 10000, + } + + # Required parameters + + if self.config.get("user"): + mongo_params["username"] = self.config["user"] + if self.config.get("password"): + mongo_params["password"] = self.config["password"] + + #Optional parameters + + if self.config.get("authSource"): + mongo_params["authSource"] = self.config["authSource"] + if self.config.get("authMechanism"): + mongo_params["authMechanism"] = self.config["authMechanism"] + + # Insert parameters for MongoDB + self.client = pymongo.MongoClient(**mongo_params) + # Ping test for DB connection + self.client.admin.command('ping') + + db_name = self.config.get("database", "") + + if not db_name: + db_names = self.client.list_database_names() + if not db_names: + raise ValueError("No database names found in the MongoDB server.") + system_dbs = ["admin", "local", "config"] + for name in db_names: + if name not in system_dbs: + db_name = name + break + if not db_name: + db_name = db_names[0] + print(f"Not specified database name. Using the first available database: {db_name}") + self.db = self.client[db_name] + return True + except (ConnectionFailure, ServerSelectionTimeoutError) as e: + if time.time() - start_time > self.connection_timeout: + print(f"Connection to MongoDB timed out after {self.connection_timeout} seconds.") + time.sleep(2) + self.close() + return False + # Add case when is needed new DB type + case _ : + raise ValueError(f"Unsupported NoSQL database: {self.engine}") + pass + + def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] = None, + progress_callback: Optional[Callable] = None) -> Dict[str, Any]: + ''' + Extract schema from the NoSQL database. + Args: + sample_limit (int): Number of samples to extract for schema inference. + collection_limit (int): Maximum number of collections to process. + progress_callback (Callable): Optional callback function for progress updates. + Returns: + Dict[str, Any]: Extracted schema information. + ''' + + match self.engine: + case "mongodb": + if not PYMONGO_IMPORTED: + raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") + if not self.client and not self.connect(): + return { + "type": "mongodb", + "tables": {}, + "tables_list": [] + } + schema = { + "type": "mongodb", + "database": self.db.name, + "tables": {}, # In MongoDB, tables are collections + } + try: + collections = self.db.list_collection_names() + if collection_limit is not None and collection_limit > 0: + collections = collections[:collection_limit] + total_collections = len(collections) + for i, collection_name in enumerate(collections): + if progress_callback: + progress_callback(i, total_collections, f"Processing collection: {collection_name}") + collection = self.db[collection_name] + + try: + doc_count = collection.count_documents({}) + if doc_count <= 0: + schema["tables"][collection_name] = { + "fields": [], + "sample_data": [], + "count": 0, + "empty": True + } + else: + sample_docs = list(collection.find().limit(sample_limit)) + fields = {} + sample_data = [] + + for doc in sample_docs: + self._extract_document_fields(doc, fields) + processed_doc = self._process_document_for_serialization(doc) + sample_data.append(processed_doc) + + formatted_fields = [{"name": field, "type": type_name} for field, type_name in fields.items()] + + schema["tables"][collection_name] = { + "fields": formatted_fields, + "sample_data": sample_data, + "count": doc_count, + } + except Exception as e: + print(f"Error processing collection {collection_name}: {e}") + schema["tables"][collection_name] = { + "fields": [], + "error": str(e) + } + # Convert the schema to a list of tables + table_list = [] + for collection_name, collection_info in schema["tables"].items(): + table_data = {"name": collection_name} + table_data.update(collection_info) + table_list.append(table_data) + schema["tables_list"] = table_list + return schema + except Exception as e: + print(f"Error extracting schema: {e}") + return { + "type": "mongodb", + "tables": {}, + "tabbles_list": [] + } + # Add case when is needed new DB type + case _ : + raise ValueError(f"Unsupported NoSQL database: {self.engine}") + def _extract_document_fields(self, doc: Dict[str, Any], fields: Dict[str, str], + prefix: str = "", max_depth: int = 3, current_depth: int = 0) -> None: + ''' + + Recursively extract fields from a document and determine their types. + Args: + doc (Dict[str, Any]): The document to extract fields from. + fields (Dict[str, str]): Dictionary to store field names and types. + prefix (str): Prefix for nested fields. + max_depth (int): Maximum depth for nested fields. + current_depth (int): Current depth in the recursion. + ''' + match self.engine: + case "mongodb": + if not PYMONGO_IMPORTED: + raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") + if current_depth >= max_depth: + return + for field, value in doc.items(): + if field == "_id": + field_type = "ObjectId" + elif isinstance(value, dict): + if value and current_depth < max_depth - 1: + self._extract_document_fields(value, fields, f"{prefix}{field}.", max_depth, current_depth + 1) + else: + field_type = f"array<{type(value[0]).__name__}>" + else: + field_type = "array" + else: + field_type = type(value).__name__ + + field_key = f"{prefix}{field}" + if field_key not in fields: + fields[field_key] = field_type + # Add case when is needed new DB type + case _ : + raise ValueError(f"Unsupported NoSQL database: {self.engine}") + + def _process_document_for_serialization(self, doc: Dict[str, Any]) -> Dict[str, Any]: + ''' + Proccesig a document for serialization of a JSON. + Args: + doc (Dict[str, Any]): The document to process. + Returns: + Procesed document + ''' + match self.engine: + case "mongodb": + if not PYMONGO_IMPORTED: + raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") + processed_doc = {} + for field, value in doc.items(): + if field == "_id": + processed_doc[field] = self._process_document_for_serialization(value) + elif isinstance(value, list): + processed_items = [] + for item in value: + if isinstance(item, dict): + processed_items.append(self._process_document_for_serialization(item)) + elif hasattr(item, "__str__"): + processed_items.append(str(item)) + else: + processed_items.append(item) + processed_doc[field] = processed_items + # Convert fetch to ISO + elif hasattr(value, 'isoformat'): + processed_doc[field] = value.isoformat() + # Convert data + else: + processed_doc[field] = value + return processed_doc + # Add case when is needed new DB type + case _ : + raise ValueError(f"Unsupported NoSQL database: {self.engine}") + + def execute_query(self, query: str) -> List[Dict[str, Any]]: + """ + Runs a NoSQL (or other) query with improved error handling + + Args: + query: A NoSQL (or other) query in JSON format or query language + + Returns: + List of resulting documents. + """ + + match self.engine: + case "nosql": + if not PYMONGO_IMPORTED: + raise ImportError("Pymongo is not installed. Please install it to use NoSQL connector.") + + if not self.client and not self.connect(): + raise ConnectionError("Couldn't estabilish a connection with NoSQL") + + try: + # Determine whether the query is a JSON string or a query in another format + filter_dict, projection, collection_name, limit = self._parse_query(query) + + # Get the collection + if not collection_name: + raise ValueError("Name of the colletion not specified in the query") + + collection = self.db[collection_name] + + # Execute the query + if projection: + cursor = collection.find(filter_dict, projection).limit(limit or 100) + else: + cursor = collection.find(filter_dict).limit(limit or 100) + + # Convert the results to a serializable format + results = [] + for doc in cursor: + processed_doc = self._process_document_for_serialization(doc) + results.append(processed_doc) + + return results + + except Exception as e: + # Reconnect and retry the query + try: + self.close() + if self.connect(): + print("Reconnecting and retrying the query...") + + # Retry the query + filter_dict, projection, collection_name, limit = self._parse_query(query) + collection = self.db[collection_name] + + if projection: + cursor = collection.find(filter_dict, projection).limit(limit or 100) + else: + cursor = collection.find(filter_dict).limit(limit or 100) + + results = [] + for doc in cursor: + processed_doc = self._process_document_for_serialization(doc) + results.append(processed_doc) + + return results + except Exception as retry_error: + # If retrying fails, show the original error + raise Exception(f"Failed to execute the NoSQL query: {str(e)}") + + # This code is will be executed if the retry fails + raise Exception(f"Failed to execute the NoSQL query (after the reconnection): {str(e)}") + + # Add case when is needed new DB type + case _ : + raise ValueError(f"Unsupported NoSQL database: {self.self.engine}") \ No newline at end of file From 518c6e6d0ac1ec32e33095d8412b3ac1f484ece7 Mon Sep 17 00:00:00 2001 From: KarixD2137 Date: Fri, 23 May 2025 12:29:10 +0200 Subject: [PATCH 39/81] Added CLI argument for API key creation - finally working --- corebrain/cli/auth/sso.py | 18 ++++++++++++++- corebrain/cli/commands.py | 47 ++++++++++++++++++++++++++++++++++----- corebrain/core/client.py | 18 --------------- 3 files changed, 58 insertions(+), 25 deletions(-) diff --git a/corebrain/cli/auth/sso.py b/corebrain/cli/auth/sso.py index cee535f..8c8e48a 100644 --- a/corebrain/cli/auth/sso.py +++ b/corebrain/cli/auth/sso.py @@ -8,6 +8,7 @@ import threading import urllib.parse import time +import json from typing import Tuple, Dict, Any, Optional @@ -449,4 +450,19 @@ def handler_factory(*args, **kwargs): server.server_close() except: # If there's any error closing the server, we ignore it - pass \ No newline at end of file + pass + +def save_api_token(api_token: str): + config_dir = os.path.join(os.path.expanduser("~"), ".corebrain") + os.makedirs(config_dir, exist_ok=True) + + token_path = os.path.join(config_dir, "token.json") + with open(token_path, "w") as f: + json.dump({"api_token": api_token}, f) + +def load_api_token() -> str: + token_path = os.path.join(os.path.expanduser("~"), ".corebrain", "token.json") + if os.path.exists(token_path): + with open(token_path, "r") as f: + return json.load(f).get("api_token") + return None \ No newline at end of file diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index 86e3c8f..952b9c8 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -12,14 +12,13 @@ from typing import Optional, List from corebrain.cli.common import DEFAULT_API_URL, DEFAULT_SSO_URL, DEFAULT_PORT, SSO_CLIENT_ID, SSO_CLIENT_SECRET -from corebrain.cli.auth.sso import authenticate_with_sso, authenticate_with_sso_and_api_key_request +from corebrain.cli.auth.sso import authenticate_with_sso, authenticate_with_sso_and_api_key_request, save_api_token, load_api_token from corebrain.cli.config import configure_sdk, get_api_credential from corebrain.cli.utils import print_colored from corebrain.config.manager import ConfigManager from corebrain.config.manager import export_config from corebrain.config.manager import validate_config from corebrain.lib.sso.auth import GlobodainSSOAuth -from corebrain.core.client import Corebrain def main_cli(argv: Optional[List[str]] = None) -> int: """ @@ -66,10 +65,9 @@ def main_cli(argv: Optional[List[str]] = None) -> int: parser.add_argument("--test-connection",action="store_true",help="Tests the connection to the Corebrain API using the provide credentials") parser.add_argument("--export-config",action="store_true",help="Exports the current configuration to a file") parser.add_argument("--gui", action="store_true", help="Check setup and launch the web interface") - parser.add_argument("--create-api-key", action="store_true", help="Create a new API Key") - parser.add_argument("--key-name", help="Name of the new API Key") - parser.add_argument("--key-level", choices=["read", "write", "admin"], default="read", help="Access level for the new API Key") + parser.add_argument("--key-name", help="Sets name of the new API Key") + parser.add_argument("--key-level", choices=["read", "write", "admin"], default="read", help="Specifies access level for the new API Key") args = parser.parse_args(argv) @@ -225,6 +223,8 @@ def authentication(): if api_token: # Save the general token for future use os.environ["COREBRAIN_API_TOKEN"] = api_token + save_api_token(api_token) + print("✅ API token saved.") if api_key: # Save the specific API key for future use @@ -435,15 +435,50 @@ def run_in_background_silent(cmd, cwd): print_colored(f"GUI: {url}", "cyan") webbrowser.open(url) + # Handles the CLI command to create a new API key using stored credentials (token from SSO) + if args.create_api_key: + + api_token = load_api_token() + if not api_token: + print_colored("❌ Missing valid API token. Please log in using --login.", "red") + return 1 + key_name = args.key_name or "default-key" + key_level = args.key_level or "read" + api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + payload = { + "name": key_name, + "access_level": key_level + } + headers = { + "Authorization": f"Bearer {api_token}", + "Content-Type": "application/json" + } + try: + response = requests.post( + f"{api_url}/api/auth/api-keys", + json=payload, + headers=headers + ) + if response.status_code == 200: + key_data = response.json() + print_colored("✅ API Key was created successfully:", "green") + print_colored(f"Name: {key_data['name']}", "blue") + print_colored(f"Key: {key_data['key']}", "blue") + else: + print_colored(f"❌ Error while creating API Key: {response.text}", "red") + return 1 + except Exception as e: + print_colored(f"❌ Exception occurred while creating API Key: {str(e)}", "red") + return 1 - + return 0 else: # If no option was specified, show help diff --git a/corebrain/core/client.py b/corebrain/core/client.py index ba341a5..e4ad986 100644 --- a/corebrain/core/client.py +++ b/corebrain/core/client.py @@ -1321,24 +1321,6 @@ def _execute_mongodb_query(self, query: Dict[str, Any]) -> List[Dict[str, Any]]: except Exception as e: raise CorebrainError(f"Error executing MongoDB query: {str(e)}") - # Sends request to Corebrain-API to create API key - def create_api_key(DEFAULT_API_URL: str, api_token: str, name: str, level: str = "read") -> dict: - """ - Create an API key using the backend API. - """ - url = f"{DEFAULT_API_URL}/api-keys" - headers = { - "Authorization": f"Bearer {api_token}", - "Content-Type": "application/json" - } - payload = { - "name": name, - "level": level - } - response = requests.post(url, headers=headers, json=payload) - response.raise_for_status() - return response.json() - def init( api_key: str = None, db_config: Dict = None, From e2d62f83f03456a728e476c572ffd3317e6c2c85 Mon Sep 17 00:00:00 2001 From: KarixD2137 Date: Fri, 23 May 2025 12:59:15 +0200 Subject: [PATCH 40/81] Added CLI argument for API key creation - finally working --- corebrain/cli/commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index 952b9c8..57cad09 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -448,6 +448,7 @@ def run_in_background_silent(cmd, cwd): api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + # Sending request to Corebrain-API payload = { "name": key_name, "access_level": key_level From f6da2feda3f29c75f8d5bf3c94598c5b29b3620e Mon Sep 17 00:00:00 2001 From: BartekPachniak Date: Fri, 23 May 2025 17:54:35 +0200 Subject: [PATCH 41/81] Complete commands Added --woami, --check-status, --task-id, --validate-config, --test-connection, --export-config. Completed all python commands --- .../csharp/CorebrainCS.Tests/Program.cs | 2 - .../csharp/CorebrainCS/CorebrainCS.cs | 139 ++++++++++++++++++ 2 files changed, 139 insertions(+), 2 deletions(-) diff --git a/corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs b/corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs index d58d0fa..d935860 100644 --- a/corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs +++ b/corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs @@ -6,5 +6,3 @@ var corebrain = new CorebrainCS.CorebrainCS("../../../../venv/Scripts/python.exe", "../../../cli", false); Console.WriteLine(corebrain.Version()); - - diff --git a/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs b/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs index c8b8fb2..4b1bb8d 100644 --- a/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs +++ b/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs @@ -130,6 +130,145 @@ public string TestAuth(string? apiUrl = null, string? token = null) { return ExecuteCommand(string.Join(" ", args)); } + +public string WoAmI() { + return ExecuteCommand("--woami"); +} + +public string CheckStatus() { + return ExecuteCommand("--check-status"); +} + +public string CheckStatus(string? apiUrl = null, string? token = null) { + var args = new List { "--check-status" }; + + if (!string.IsNullOrEmpty(apiUrl)) { + if (!Uri.IsWellFormedUriString(apiUrl, UriKind.Absolute)) + throw new ArgumentException("Invalid API URL format", nameof(apiUrl)); + + args.Add($"--api-url \"{apiUrl}\""); + } + + if (!string.IsNullOrEmpty(token)) + args.Add($"--token \"{token}\""); + + return ExecuteCommand(string.Join(" ", args)); +} + +public string TaskStatus(string taskId) { + if (string.IsNullOrWhiteSpace(taskId)) { + throw new ArgumentException("Task ID cannot be empty", nameof(taskId)); + } + + return ExecuteCommand($"--task-id {taskId}"); +} + +public string TaskStatus(string taskId, string? apiUrl = null, string? token = null) { + if (string.IsNullOrWhiteSpace(taskId)) { + throw new ArgumentException("Task ID cannot be empty", nameof(taskId)); + } + + var args = new List { $"--task-id {taskId}" }; + + if (!string.IsNullOrEmpty(apiUrl)) { + if (!Uri.IsWellFormedUriString(apiUrl, UriKind.Absolute)) + throw new ArgumentException("Invalid API URL format", nameof(apiUrl)); + + args.Add($"--api-url \"{apiUrl}\""); + } + + if (!string.IsNullOrEmpty(token)) + args.Add($"--token \"{token}\""); + + return ExecuteCommand(string.Join(" ", args)); +} + +public string ValidateConfig() { + return ExecuteCommand("--validate-config"); +} + +public string ValidateConfig(string configFilePath) { + if (string.IsNullOrWhiteSpace(configFilePath)) { + throw new ArgumentException("Config file path cannot be empty", nameof(configFilePath)); + } + + if (!File.Exists(configFilePath)) { + throw new FileNotFoundException("Config file not found", configFilePath); + } + + return ExecuteCommand($"--validate-config \"{configFilePath}\""); +} + + public string ValidateConfig(string? apiUrl = null, string? token = null) { + var args = new List { "--validate-config" }; + + if (!string.IsNullOrEmpty(apiUrl)) { + if (!Uri.IsWellFormedUriString(apiUrl, UriKind.Absolute)) + throw new ArgumentException("Invalid API URL format", nameof(apiUrl)); + + args.Add($"--api-url \"{apiUrl}\""); + } + + if (!string.IsNullOrEmpty(token)) + args.Add($"--token \"{token}\""); + + return ExecuteCommand(string.Join(" ", args)); + } +public string TestConnection() { + return ExecuteCommand("--test-connection"); +} + + public string TestConnection(string? apiUrl = null, string? token = null, bool fullDiagnostics = false) + { + var args = new List { "--test-connection" }; + + if (!string.IsNullOrEmpty(apiUrl)) + { + if (!Uri.IsWellFormedUriString(apiUrl, UriKind.Absolute)) + throw new ArgumentException("Invalid API URL format", nameof(apiUrl)); + + args.Add($"--api-url \"{apiUrl}\""); + } + + if (!string.IsNullOrEmpty(token)) + args.Add($"--token \"{token}\""); + + if (fullDiagnostics) + args.Add("--full"); + + return ExecuteCommand(string.Join(" ", args)); + } + +public string ExportConfig() { + return ExecuteCommand("--export-config"); +} + +public string ExportConfig(string outputDirectory, string? configId = null, bool overwrite = false) { + if (string.IsNullOrWhiteSpace(outputDirectory)) { + throw new ArgumentException("Output directory cannot be empty", nameof(outputDirectory)); + } + + if (!Directory.Exists(outputDirectory)) { + throw new DirectoryNotFoundException($"Directory not found: {outputDirectory}"); + } + + var args = new List { "--export-config" }; + + args.Add($"--output \"{outputDirectory}\""); + + if (!string.IsNullOrEmpty(configId)) { + args.Add($"--config-id \"{configId}\""); + } + + if (overwrite) { + args.Add("--overwrite"); + } + + + return ExecuteCommand(string.Join(" ", args)); +} + + public string ExecuteCommand(string arguments) { if (_verbose) From b54b59e9b04804ae45f78416885f5351acf65c8d Mon Sep 17 00:00:00 2001 From: Albert Stankowski Date: Fri, 23 May 2025 17:25:05 +0100 Subject: [PATCH 42/81] Add documentation to C# --- .../csharp/CorebrainCS/CorebrainCS.cs | 194 ++++++++++++++---- .../src/CorebrainCLIAPI/CorebrainSettings.cs | 16 ++ 2 files changed, 167 insertions(+), 43 deletions(-) diff --git a/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs b/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs index c8b8fb2..52d3e2e 100644 --- a/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs +++ b/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs @@ -14,84 +14,166 @@ public class CorebrainCS(string pythonPath = "python", string scriptPath = "core private readonly string _scriptPath = Path.GetFullPath(scriptPath); private readonly bool _verbose = verbose; - + /// + /// Executes the CLI with the "--help" flag. + /// + /// Help text from the Corebrain CLI. public string Help() { return ExecuteCommand("--help"); } - public string Version() { + /// + /// Executes the CLI with the "--version" flag. + /// + /// The version of the Corebrain CLI. + public string Version() + { return ExecuteCommand("--version"); } - public string Configure() { + /// + /// Starts interactive configuration of the Corebrain CLI. + /// + /// Output from the configuration process. + public string Configure() + { return ExecuteCommand("--configure"); } - public string ListConfigs() { + /// + /// Lists all stored configurations. + /// + /// Configuration listing. + public string ListConfigs() + { return ExecuteCommand("--list-configs"); } - public string RemoveConfig() { + /// + /// Removes a stored configuration. + /// + /// Result of the remove operation. + public string RemoveConfig() + { return ExecuteCommand("--remove-config"); } - public string ShowSchema() { + /// + /// Shows the schema used by Corebrain. + /// + /// The current schema as a string. + public string ShowSchema() + { return ExecuteCommand("--show-schema"); } - public string ExtractSchema() { + /// + /// Extracts the current schema to the console. + /// + /// The extracted schema. + public string ExtractSchema() + { return ExecuteCommand("--extract-schema"); } - public string ExtractSchemaToDefaultFile() { + /// + /// Extracts schema and saves it to a default file ("test"). + /// + /// CLI output from the extract operation. + public string ExtractSchemaToDefaultFile() + { return ExecuteCommand("--extract-schema --output-file test"); } - - public string ConfigID() { + + /// + /// Extracts schema with a specific configuration ID. + /// + /// CLI output from the operation. + public string ConfigID() + { return ExecuteCommand("--extract-schema --config-id config"); } - public string SetToken(string token) { + /// + /// Sets the authentication token. + /// + /// Authentication token. + /// Result of setting the token. + public string SetToken(string token) + { return ExecuteCommand($"--token {token}"); } - public string ApiKey(string apikey) { + /// + /// Sets the API key. + /// + /// API key string. + /// Result of setting the API key. + public string ApiKey(string apikey) + { return ExecuteCommand($"--api-key {apikey}"); } - - public string ApiUrl(string apiurl) { - if (string.IsNullOrWhiteSpace(apiurl)) { + + /// + /// Sets the API URL. + /// + /// A valid HTTP or HTTPS URL. + /// CLI output after setting the URL. + public string ApiUrl(string apiurl) + { + if (string.IsNullOrWhiteSpace(apiurl)) + { throw new ArgumentException("API URL cannot be empty or whitespace", nameof(apiurl)); } if (!Uri.TryCreate(apiurl, UriKind.Absolute, out var uriResult) || - (uriResult.Scheme != Uri.UriSchemeHttp && uriResult.Scheme != Uri.UriSchemeHttps)) { + (uriResult.Scheme != Uri.UriSchemeHttp && uriResult.Scheme != Uri.UriSchemeHttps)) + { throw new ArgumentException("Invalid API URL format. Must be a valid HTTP/HTTPS URL", nameof(apiurl)); } var escapedUrl = apiurl.Replace("\"", "\\\""); return ExecuteCommand($"--api-url \"{escapedUrl}\""); } - public string SsoUrl(string ssoUrl) { - if (string.IsNullOrWhiteSpace(ssoUrl)) { - throw new ArgumentException("SSO URL cannot be empty or whitespace", nameof(ssoUrl)); + + /// + /// Sets the Single Sign-On (SSO) URL. + /// + /// A valid SSO URL. + /// CLI output after setting the SSO URL. + public string SsoUrl(string ssoUrl) + { + if (string.IsNullOrWhiteSpace(ssoUrl)) + { + throw new ArgumentException("SSO URL cannot be empty or whitespace", nameof(ssoUrl)); } if (!Uri.TryCreate(ssoUrl, UriKind.Absolute, out var uriResult) || - (uriResult.Scheme != Uri.UriSchemeHttp && uriResult.Scheme != Uri.UriSchemeHttps)) { - throw new ArgumentException("Invalid SSO URL format. Must be a valid HTTP/HTTPS URL", nameof(ssoUrl)); + (uriResult.Scheme != Uri.UriSchemeHttp && uriResult.Scheme != Uri.UriSchemeHttps)) + { + throw new ArgumentException("Invalid SSO URL format. Must be a valid HTTP/HTTPS URL", nameof(ssoUrl)); } var escapedUrl = ssoUrl.Replace("\"", "\\\""); return ExecuteCommand($"--sso-url \"{escapedUrl}\""); } - public string Login(string username, string password){ - if (string.IsNullOrWhiteSpace(username)){ - throw new ArgumentException("Username cannot be empty or whitespace", nameof(username)); + + /// + /// Logs in using username and password. + /// + /// User's username. + /// User's password. + /// CLI output from login attempt. + public string Login(string username, string password) + { + if (string.IsNullOrWhiteSpace(username)) + { + throw new ArgumentException("Username cannot be empty or whitespace", nameof(username)); } - if (string.IsNullOrWhiteSpace(password)){ - throw new ArgumentException("Password cannot be empty or whitespace", nameof(password)); + if (string.IsNullOrWhiteSpace(password)) + { + throw new ArgumentException("Password cannot be empty or whitespace", nameof(password)); } var escapedUsername = username.Replace("\"", "\\\""); @@ -100,36 +182,62 @@ public string Login(string username, string password){ return ExecuteCommand($"--login --username \"{escapedUsername}\" --password \"{escapedPassword}\""); } - public string LoginWithToken(string token) { - if (string.IsNullOrWhiteSpace(token)) { - throw new ArgumentException("Token cannot be empty or whitespace", nameof(token)); - } + /// + /// Logs in using an authentication token. + /// + /// Authentication token. + /// CLI output from login attempt. + public string LoginWithToken(string token) + { + if (string.IsNullOrWhiteSpace(token)) + { + throw new ArgumentException("Token cannot be empty or whitespace", nameof(token)); + } - var escapedToken = token.Replace("\"", "\\\""); - return ExecuteCommand($"--login --token \"{escapedToken}\""); + var escapedToken = token.Replace("\"", "\\\""); + return ExecuteCommand($"--login --token \"{escapedToken}\""); } //When youre logged in use this function - public string TestAuth() { + /// + /// Tests authentication status for the currently logged-in user. + /// + /// CLI output from the authentication test. + public string TestAuth() + { return ExecuteCommand("--test-auth"); } //Without beeing logged - public string TestAuth(string? apiUrl = null, string? token = null) { + /// + /// Tests authentication status using provided token and/or API URL. + /// + /// Optional API URL to use for the test. + /// Optional token to use for the test. + /// CLI output from the authentication test. + public string TestAuth(string? apiUrl = null, string? token = null) + { var args = new List { "--test-auth" }; - - if (!string.IsNullOrEmpty(apiUrl)) { - if (!Uri.IsWellFormedUriString(apiUrl, UriKind.Absolute)) - throw new ArgumentException("Invalid API URL format", nameof(apiUrl)); - - args.Add($"--api-url \"{apiUrl}\""); - } - + + if (!string.IsNullOrEmpty(apiUrl)) + { + if (!Uri.IsWellFormedUriString(apiUrl, UriKind.Absolute)) + throw new ArgumentException("Invalid API URL format", nameof(apiUrl)); + + args.Add($"--api-url \"{apiUrl}\""); + } + if (!string.IsNullOrEmpty(token)) - args.Add($"--token \"{token}\""); + args.Add($"--token \"{token}\""); return ExecuteCommand(string.Join(" ", args)); } + /// + /// Executes the given CLI command arguments using the configured Python and script paths. + /// + /// Command-line arguments for the Corebrain CLI. + /// Standard output from the executed command. + /// Thrown if there is an error in the CLI output. public string ExecuteCommand(string arguments) { if (_verbose) diff --git a/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainSettings.cs b/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainSettings.cs index 82143b9..6de197e 100644 --- a/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainSettings.cs +++ b/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainSettings.cs @@ -1,8 +1,24 @@ namespace CorebrainCLIAPI; +/// +/// Represents the configuration settings for the Corebrain CLI wrapper. +/// public class CorebrainSettings { + + /// + /// Gets or sets the path to the Python executable (e.g., "./.venv/Scripts/python"). + /// public string PythonPath { get; set; } + + /// + /// Gets or sets the path to the Corebrain CLI script or the command name if installed globally (e.g., "corebrain"). + /// public string ScriptPath { get; set; } + + /// + /// Gets or sets a value indicating whether verbose logging is enabled. + /// Default is false. + /// public bool Verbose { get; set; } = false; } From 6ebc98ce4ea9104de8446ebd5fe4e4bac541dac2 Mon Sep 17 00:00:00 2001 From: Pawel Wasilewski Date: Mon, 26 May 2025 09:23:22 +0200 Subject: [PATCH 43/81] restructurezation of NoSQL --- corebrain/corebrain/db/connectors/nosql.py | 2 +- corebrain/corebrain/db/factory.py | 6 +- corebrain/corebrain/db/schema_file.py | 164 +++++++-------------- corebrain/db/connectors/nosql.py | 24 ++- 4 files changed, 84 insertions(+), 112 deletions(-) diff --git a/corebrain/corebrain/db/connectors/nosql.py b/corebrain/corebrain/db/connectors/nosql.py index c738946..0a153f4 100644 --- a/corebrain/corebrain/db/connectors/nosql.py +++ b/corebrain/corebrain/db/connectors/nosql.py @@ -17,7 +17,7 @@ PYMONGO_IMPORTED = True except ImportError: PYMONGO_IMPORTED = False -# Whe nadding new DB type write a try to it from user +# When adding new DB type write a try to it from user from corebrain.db.connector import DatabaseConnector class NoSQLConnector(DatabaseConnector): diff --git a/corebrain/corebrain/db/factory.py b/corebrain/corebrain/db/factory.py index c2c23bc..092f827 100644 --- a/corebrain/corebrain/db/factory.py +++ b/corebrain/corebrain/db/factory.py @@ -5,7 +5,7 @@ from corebrain.db.connector import DatabaseConnector from corebrain.db.connectors.sql import SQLConnector -from corebrain.db.connectors.mongodb import MongoDBConnector +from corebrain.db.connectors.nosql import NoSQLConnector def get_connector(db_config: Dict[str, Any], timeout: int = 10) -> DatabaseConnector: """ @@ -23,7 +23,7 @@ def get_connector(db_config: Dict[str, Any], timeout: int = 10) -> DatabaseConne if db_type == "sql": return SQLConnector(db_config, timeout) - elif db_type in ["nosql", "mongodb"] or engine == "mongodb": - return MongoDBConnector(db_config, timeout) + elif db_type == "nosql": + return NoSQLConnector(db_config, timeout) else: raise ValueError(f"Tipo de base de datos no soportado: {db_type}") \ No newline at end of file diff --git a/corebrain/corebrain/db/schema_file.py b/corebrain/corebrain/db/schema_file.py index c4dc8f7..3c1edcd 100644 --- a/corebrain/corebrain/db/schema_file.py +++ b/corebrain/corebrain/db/schema_file.py @@ -40,117 +40,67 @@ def extract_db_schema(db_config: Dict[str, Any]) -> Dict[str, Any]: # [Se mantiene igual] pass - # Manejar tanto "nosql" como "mongodb" como tipos válidos - elif db_type == "nosql" or db_type == "mongodb": - import pymongo + # Manejar tanto "nosql" como tipos válidos + elif db_type == "nosql": + match db_config.get("engine", "").lower(): + case "mongodb": + try: + import pymongo + PYMONGO_IMPORTED = True + except ImportError: + PYMONGO_IMPORTED = False + if not PYMONGO_IMPORTED: + raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") + # Defying the engine + engine = db_config.get("engine", "").lower() + + if engine == "mongodb": + if "connection_string" in db_config: + client = pymongo.MongoClient(db_config["connection_string"]) + else: + # Diccionario de parámetros para MongoClient + mongo_params = { + "host": db_config.get("host", "localhost"), + "port": db_config.get("port", 27017) + } + + # Añadir credenciales solo si están presentes + if db_config.get("user"): + mongo_params["username"] = db_config["user"] + if db_config.get("password"): + mongo_params["password"] = db_config["password"] + + client = pymongo.MongoClient(**mongo_params) + db_name = db_config.get("database","") + + if not db_name: + _print_colored("⚠️ Database is not specified", "yellow") + return schema + try: + db = client[db_name] + collection_names = db.list_collection_names() + + # Process collection + + for collection_name in collection_names: + collection = db[collection_name] + + try: + sample_docs = list(collection.find().lkmit(5)) + + field_types = {} + + + + except Exception as e: + except Exception as e: + + - # Determinar el motor (si existe) - engine = db_config.get("engine", "").lower() - # Si no se especifica el engine o es mongodb, proceder - if not engine or engine == "mongodb": - if "connection_string" in db_config: - client = pymongo.MongoClient(db_config["connection_string"]) - else: - # Diccionario de parámetros para MongoClient - mongo_params = { - "host": db_config.get("host", "localhost"), - "port": db_config.get("port", 27017) - } - - # Añadir credenciales solo si están presentes - if db_config.get("user"): - mongo_params["username"] = db_config["user"] - if db_config.get("password"): - mongo_params["password"] = db_config["password"] + - client = pymongo.MongoClient(**mongo_params) - - # Obtener la base de datos - db_name = db_config.get("database", "") - if not db_name: - _print_colored("⚠️ Nombre de base de datos no especificado", "yellow") - return schema - - try: - db = client[db_name] - collection_names = db.list_collection_names() - # Procesar colecciones - for collection_name in collection_names: - collection = db[collection_name] - - # Obtener varios documentos de muestra - try: - sample_docs = list(collection.find().limit(5)) - - # Extraer estructura de campos a partir de los documentos - field_types = {} - - for doc in sample_docs: - for field, value in doc.items(): - if field != "_id": # Ignoramos el _id de MongoDB - # Actualizar el tipo si no existe o combinar si hay diferentes tipos - field_type = type(value).__name__ - if field not in field_types: - field_types[field] = field_type - elif field_types[field] != field_type: - field_types[field] = f"{field_types[field]}|{field_type}" - - # Convertir a formato esperado - fields = [{"name": field, "type": type_name} for field, type_name in field_types.items()] - - # Convertir documentos a formato serializable - sample_data = [] - for doc in sample_docs: - serialized_doc = {} - for key, value in doc.items(): - if key == "_id": - serialized_doc[key] = str(value) - elif isinstance(value, (dict, list)): - serialized_doc[key] = str(value) # Simplificar objetos anidados - else: - serialized_doc[key] = value - sample_data.append(serialized_doc) - - # Guardar información de la colección - schema["tables"][collection_name] = { - "fields": fields, - "sample_data": sample_data - } - except Exception as e: - _print_colored(f"Error al procesar colección {collection_name}: {str(e)}", "red") - schema["tables"][collection_name] = { - "fields": [], - "sample_data": [], - "error": str(e) - } - - except Exception as e: - _print_colored(f"Error al acceder a la base de datos MongoDB '{db_name}': {str(e)}", "red") - - finally: - # Cerrar la conexión - client.close() - else: - _print_colored(f"Motor de base de datos NoSQL no soportado: {engine}", "red") - - # Convertir el diccionario de tablas en una lista para mantener compatibilidad con el formato anterior - table_list = [] - for table_name, table_info in schema["tables"].items(): - table_data = {"name": table_name} - table_data.update(table_info) - table_list.append(table_data) - - # Guardar también la lista de tablas para mantener compatibilidad - schema["tables_list"] = table_list - - return schema - - except Exception as e: - _print_colored(f"Error al extraer el esquema de la base de datos: {str(e)}", "red") - # En caso de error, devolver un esquema vacío - return {"type": db_type, "tables": {}, "tables_list": []} def extract_db_schema_direct(db_config: Dict[str, Any]) -> Dict[str, Any]: """ diff --git a/corebrain/db/connectors/nosql.py b/corebrain/db/connectors/nosql.py index c738946..00bb3e5 100644 --- a/corebrain/db/connectors/nosql.py +++ b/corebrain/db/connectors/nosql.py @@ -34,12 +34,19 @@ def __init__(self, config: Dict[str, Any]): config (dict): Configuration dictionary containing connection parameters. ''' super().__init__(config) + + self.conn = None + self.cursor = None + self.engine = config.get("engine", "").lower() + self.config = config + self.connection_timeout = 30 # seconds + ''' self.engine = config.get("engine", "").lower() self.client = None self.db = None self.config = config self.connection_timeout = 30 # seconds - + ''' match self.engine: case "mongodb": if not PYMONGO_IMPORTED: @@ -55,6 +62,20 @@ def connect(self) -> bool: self.engine (str): Name of the database. ''' + try: + + start_time = time.time() + + while time.time() - start_time < self.connection_timeout: + try: + if self.engine == "mongodb": + if not PYMONGO_IMPORTED: + raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") + if not + + + + ''' match self.engine: case "mongodb": if not PYMONGO_IMPORTED: @@ -126,6 +147,7 @@ def connect(self) -> bool: case _ : raise ValueError(f"Unsupported NoSQL database: {self.engine}") pass + ''' def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] = None, progress_callback: Optional[Callable] = None) -> Dict[str, Any]: From 1c09708011c8dac55cb94d665935070b275c8d2d Mon Sep 17 00:00:00 2001 From: palstr Date: Mon, 26 May 2025 10:00:09 +0200 Subject: [PATCH 44/81] NoSQL Restructurization --- corebrain/db/factory.py | 9 ++++++--- corebrain/db/schema/extractor.py | 11 ++++++++--- corebrain/db/schema_file.py | 9 ++++++--- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/corebrain/db/factory.py b/corebrain/db/factory.py index c2c23bc..cc28296 100644 --- a/corebrain/db/factory.py +++ b/corebrain/db/factory.py @@ -23,7 +23,10 @@ def get_connector(db_config: Dict[str, Any], timeout: int = 10) -> DatabaseConne if db_type == "sql": return SQLConnector(db_config, timeout) - elif db_type in ["nosql", "mongodb"] or engine == "mongodb": - return MongoDBConnector(db_config, timeout) + elif db_type == "nosql": + if engine == "mongodb": + return MongoDBConnector(db_config, timeout) + else: + raise ValueError(f"Unsupported NoSQL engine: {engine}") else: - raise ValueError(f"Tipo de base de datos no soportado: {db_type}") \ No newline at end of file + raise ValueError(f"Unsupported database type: {db_type}") \ No newline at end of file diff --git a/corebrain/db/schema/extractor.py b/corebrain/db/schema/extractor.py index c361b83..8349ce8 100644 --- a/corebrain/db/schema/extractor.py +++ b/corebrain/db/schema/extractor.py @@ -53,10 +53,15 @@ def extract_db_schema(db_config: Dict[str, Any], client_factory: Optional[Callab import psycopg2 # (implementación...) - elif db_type in ["nosql", "mongodb"]: + elif db_type == "nosql": + engine = db_config.get("engine", "").lower() + if engine == "mongodb": # Extraer esquema MongoDB - import pymongo - # (implementación...) + try: + import pymongo + except ImportError: + logger.error("pymongo in not installed. Use 'pip install pymongo'.") + # (implementación...) # Convertir diccionario a lista para compatibilidad table_list = [] diff --git a/corebrain/db/schema_file.py b/corebrain/db/schema_file.py index c4dc8f7..2e20b9f 100644 --- a/corebrain/db/schema_file.py +++ b/corebrain/db/schema_file.py @@ -41,7 +41,7 @@ def extract_db_schema(db_config: Dict[str, Any]) -> Dict[str, Any]: pass # Manejar tanto "nosql" como "mongodb" como tipos válidos - elif db_type == "nosql" or db_type == "mongodb": + elif db_type == "nosql": import pymongo # Determinar el motor (si existe) @@ -214,8 +214,11 @@ def test_connection(db_config: Dict[str, Any]) -> bool: if db_config["type"].lower() == "sql": # Code to test SQL connection... pass - elif db_config["type"].lower() in ["nosql", "mongodb"]: - import pymongo + elif db_config["type"].lower() == "nosql": + if db_config["engine"].lower() == "mongodb": + import pymongo + else: + raise ValueError(f"Unsupported NoSQL engine: {db_config['engine']}") # Create MongoDB client client = pymongo.MongoClient(db_config["connection_string"]) From 82a6489f320f304cc6d38aa94550243fb3d82b99 Mon Sep 17 00:00:00 2001 From: Pawel Wasilewski Date: Mon, 26 May 2025 10:14:17 +0200 Subject: [PATCH 45/81] update --- corebrain/db/connectors/nosql.py | 167 +++++++++++------- .../db/connectors/subconnectors/mongodb.py | 67 +++++++ 2 files changed, 166 insertions(+), 68 deletions(-) create mode 100644 corebrain/db/connectors/subconnectors/mongodb.py diff --git a/corebrain/db/connectors/nosql.py b/corebrain/db/connectors/nosql.py index 00bb3e5..0f77f76 100644 --- a/corebrain/db/connectors/nosql.py +++ b/corebrain/db/connectors/nosql.py @@ -36,7 +36,6 @@ def __init__(self, config: Dict[str, Any]): super().__init__(config) self.conn = None - self.cursor = None self.engine = config.get("engine", "").lower() self.config = config self.connection_timeout = 30 # seconds @@ -61,7 +60,6 @@ def connect(self) -> bool: Args: self.engine (str): Name of the database. ''' - try: start_time = time.time() @@ -69,85 +67,91 @@ def connect(self) -> bool: while time.time() - start_time < self.connection_timeout: try: if self.engine == "mongodb": + # Checking if pymongo is imported if not PYMONGO_IMPORTED: raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") - if not - - - - ''' - match self.engine: - case "mongodb": - if not PYMONGO_IMPORTED: - raise ImportError("Pymongo is not installed. Please install it to use MongoDB connector.") - try: - start_time = time.time() - # Check if connection string is provided - if "connection_string" in self.config: - connection_string = self.config["connection_string"] - if "connectTimeoutMS=" not in connection_string: - if "?" in connection_string: - connection_string += "&connectTimeoutMS=10000" - else: - connection_string += "?connectTimeoutMS=10000" - self.client = pymongo.MongoClient(connection_string) - else: - - # Setup for MongoDB connection parameters - - mongo_params = { - "host": self.config.get("host", "localhost"), - "port": int(self.config.get("port", 27017)), - "connection_timeoutMS": 10000, - "serverSelectionTimeoutMS": 10000, - } + + # Construction of the MongoDB connection + if "connection_string" in self.config: - # Required parameters + # Check if connection string is provided + connection_string = self.config["connection_string"] - if self.config.get("user"): - mongo_params["username"] = self.config["user"] - if self.config.get("password"): - mongo_params["password"] = self.config["password"] + if "connectTimeoutMS=" not in connection_string: + if "?" in connection_string: + connection_string += "&connectTimeoutMS=10000" + else: + connection_string += "?connectTimeoutMS=10000" + # Connecting to MongoDB using the connection string + self.client = pymongo.MongoClient(connection_string) - #Optional parameters + else: + # Setup for MongoDB connection parameters + mongo_params = { + "host": self.config.get("host", "localhost"), + "port": int(self.config.get("port", 27017)), + # 10000 = 10 seconds + "connectTimeoutMS": 10000, + "serverSelectionTimeoutMS": 10000, + } - if self.config.get("authSource"): - mongo_params["authSource"] = self.config["authSource"] - if self.config.get("authMechanism"): - mongo_params["authMechanism"] = self.config["authMechanism"] + # Required parameters + if self.config.get("user"): + mongo_params["username"] = self.config["user"] + if self.config.get("password"): + mongo_params["password"] = self.config["password"] - # Insert parameters for MongoDB - self.client = pymongo.MongoClient(**mongo_params) - # Ping test for DB connection - self.client.admin.command('ping') + # Optional parameters + if self.config.get("authSource"): + mongo_params["authSource"] = self.config["authSource"] + if self.config.get("authMechanism"): + mongo_params["authMechanism"] = self.config["authMechanism"] - db_name = self.config.get("database", "") + # Insert parameters for MongoDB + self.client = pymongo.MongoClient(**mongo_params) + # + # If adding new db add thru self.engine variable + # + else: + raise ValueError(f"Unsupported NoSQL database: {self.engine}") + if self.conn: + if self.engine == "mongodb": + # Testing connection for MongoDB + self.client.admin.command('ping') - if not db_name: - db_names = self.client.list_database_names() - if not db_names: - raise ValueError("No database names found in the MongoDB server.") - system_dbs = ["admin", "local", "config"] - for name in db_names: - if name not in system_dbs: - db_name = name - break - if not db_name: - db_name = db_names[0] - print(f"Not specified database name. Using the first available database: {db_name}") - self.db = self.client[db_name] - return True + # If connection is successful, set the database + db_name = self.config.get("database", "") + if not db_name: + # If database name is not specified, use the first available database + db_names = self.client.list_database_names() + if not db_names: + raise ValueError("No database names found in the MongoDB server.") + # Exclude system database (MongoDB) from the list + system_dbs = ["admin", "local", "config"] + for name in db_names: + if name not in system_dbs: + db_name = name + break + if not db_name: + db_name = db_names[0] + print(f"Not specified database name. Using the first available database: {db_name}") + # Connect to the specified database + self.db = self.client[db_name] + return True + else: + # If the engine is not Supported, raise an error + raise ValueError(f"Unsupported NoSQL database: {self.engine}") except (ConnectionFailure, ServerSelectionTimeoutError) as e: + # If connection fails, check if timeout is reached if time.time() - start_time > self.connection_timeout: - print(f"Connection to MongoDB timed out after {self.connection_timeout} seconds.") - time.sleep(2) + print(f"Connection to {self.engine} timed out after {self.connection_timeout} seconds.") + time.sleep(2) self.close() return False - # Add case when is needed new DB type - case _ : - raise ValueError(f"Unsupported NoSQL database: {self.engine}") - pass - ''' + except Exception as e: + # If cannot connect to the database, print the error + print(f"Error connecting to {self.engine}: {e}") + return False def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] = None, progress_callback: Optional[Callable] = None) -> Dict[str, Any]: @@ -160,7 +164,33 @@ def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] Returns: Dict[str, Any]: Extracted schema information. ''' + if not self.client and not self.connect(): + return { + "type": self.engine, + "tables": {}, + "tables_list": [] + } + + schema = { + "type": self.engine, + "database": self.db.name, + "tables": {}, # Depends on DB + } + match self.engine: + case "mongodb": + if not PYMONGO_IMPORTED: + raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") + try: + import corebrain.db.connectors.subconnectors.mongodb as mongodb_subconnector + return mongodb_subconnector.extract_schema(self, sample_limit, collection_limit, progress_callback) + except ImportError: + raise ImportError("Failed to import MongoDB subconnector. Please ensure it is installed correctly.") + # If adding new db add thru self.engine variable + # Add case when is needed new DB type + case _: + return schema + ''' match self.engine: case "mongodb": if not PYMONGO_IMPORTED: @@ -236,6 +266,7 @@ def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] # Add case when is needed new DB type case _ : raise ValueError(f"Unsupported NoSQL database: {self.engine}") + ''' def _extract_document_fields(self, doc: Dict[str, Any], fields: Dict[str, str], prefix: str = "", max_depth: int = 3, current_depth: int = 0) -> None: ''' diff --git a/corebrain/db/connectors/subconnectors/mongodb.py b/corebrain/db/connectors/subconnectors/mongodb.py new file mode 100644 index 0000000..0409006 --- /dev/null +++ b/corebrain/db/connectors/subconnectors/mongodb.py @@ -0,0 +1,67 @@ +def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] = None, + progress_callback: Optional[Callable] = None) -> Dict[str, Any]: + ''' + extract schema for MongoDB collections + Args: + sample_limit (int): Number of samples to extract from each collection. + collection_limit (Optional[int]): Maximum number of collections to process. + progress_callback (Optional[Callable]): Function to call for progress updates. + ''' + + try: + collections = self.db.list_collection_names() + if collection_limit is not None and collection_limit > 0: + collections = collections[:collection_limit] + total_collections = len(collections) + for i, collection_name in enumerate(collections): + if progress_callback: + progress_callback(i, total_collections, f"Processing collection: {collection_name}") + collection = self.db[collection_name] + + try: + doc_count = collection.count_documents({}) + if doc_count <= 0: + schema["tables"][collection_name] = { + "fields": [], + "sample_data": [], + "count": 0, + "empty": True + } + else: + sample_docs = list(collection.find().limit(sample_limit)) + fields = {} + sample_data = [] + + for doc in sample_docs: + self._extract_document_fields(doc, fields) + processed_doc = self._process_document_for_serialization(doc) + sample_data.append(processed_doc) + + formatted_fields = [{"name": field, "type": type_name} for field, type_name in fields.items()] + + schema["tables"][collection_name] = { + "fields": formatted_fields, + "sample_data": sample_data, + "count": doc_count, + } + except Exception as e: + print(f"Error processing collection {collection_name}: {e}") + schema["tables"][collection_name] = { + "fields": [], + "error": str(e) + } + # Convert the schema to a list of tables + table_list = [] + for collection_name, collection_info in schema["tables"].items(): + table_data = {"name": collection_name} + table_data.update(collection_info) + table_list.append(table_data) + schema["tables_list"] = table_list + return schema + except Exception as e: + print(f"Error extracting schema: {e}") + return { + "type": "mongodb", + "tables": {}, + "tabbles_list": [] + } \ No newline at end of file From 768d6f1a9cf70f09d64d845e5ed61fbc6ee0ccbb Mon Sep 17 00:00:00 2001 From: palstr Date: Mon, 26 May 2025 10:22:45 +0200 Subject: [PATCH 46/81] Fixed Indentation Problem --- .../db/connectors/subconnectors/mongodb.py | 106 +++++++++--------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/corebrain/db/connectors/subconnectors/mongodb.py b/corebrain/db/connectors/subconnectors/mongodb.py index 0409006..001be63 100644 --- a/corebrain/db/connectors/subconnectors/mongodb.py +++ b/corebrain/db/connectors/subconnectors/mongodb.py @@ -8,60 +8,60 @@ def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] progress_callback (Optional[Callable]): Function to call for progress updates. ''' - try: - collections = self.db.list_collection_names() - if collection_limit is not None and collection_limit > 0: - collections = collections[:collection_limit] - total_collections = len(collections) - for i, collection_name in enumerate(collections): - if progress_callback: - progress_callback(i, total_collections, f"Processing collection: {collection_name}") - collection = self.db[collection_name] + try: + collections = self.db.list_collection_names() + if collection_limit is not None and collection_limit > 0: + collections = collections[:collection_limit] + total_collections = len(collections) + for i, collection_name in enumerate(collections): + if progress_callback: + progress_callback(i, total_collections, f"Processing collection: {collection_name}") + collection = self.db[collection_name] - try: - doc_count = collection.count_documents({}) - if doc_count <= 0: - schema["tables"][collection_name] = { - "fields": [], - "sample_data": [], - "count": 0, - "empty": True - } - else: - sample_docs = list(collection.find().limit(sample_limit)) - fields = {} - sample_data = [] + try: + doc_count = collection.count_documents({}) + if doc_count <= 0: + schema["tables"][collection_name] = { + "fields": [], + "sample_data": [], + "count": 0, + "empty": True + } + else: + sample_docs = list(collection.find().limit(sample_limit)) + fields = {} + sample_data = [] - for doc in sample_docs: - self._extract_document_fields(doc, fields) - processed_doc = self._process_document_for_serialization(doc) - sample_data.append(processed_doc) + for doc in sample_docs: + self._extract_document_fields(doc, fields) + processed_doc = self._process_document_for_serialization(doc) + sample_data.append(processed_doc) - formatted_fields = [{"name": field, "type": type_name} for field, type_name in fields.items()] + formatted_fields = [{"name": field, "type": type_name} for field, type_name in fields.items()] - schema["tables"][collection_name] = { - "fields": formatted_fields, - "sample_data": sample_data, - "count": doc_count, - } - except Exception as e: - print(f"Error processing collection {collection_name}: {e}") - schema["tables"][collection_name] = { - "fields": [], - "error": str(e) - } - # Convert the schema to a list of tables - table_list = [] - for collection_name, collection_info in schema["tables"].items(): - table_data = {"name": collection_name} - table_data.update(collection_info) - table_list.append(table_data) - schema["tables_list"] = table_list - return schema - except Exception as e: - print(f"Error extracting schema: {e}") - return { - "type": "mongodb", - "tables": {}, - "tabbles_list": [] - } \ No newline at end of file + schema["tables"][collection_name] = { + "fields": formatted_fields, + "sample_data": sample_data, + "count": doc_count, + } + except Exception as e: + print(f"Error processing collection {collection_name}: {e}") + schema["tables"][collection_name] = { + "fields": [], + "error": str(e) + } + # Convert the schema to a list of tables + table_list = [] + for collection_name, collection_info in schema["tables"].items(): + table_data = {"name": collection_name} + table_data.update(collection_info) + table_list.append(table_data) + schema["tables_list"] = table_list + return schema + except Exception as e: + print(f"Error extracting schema: {e}") + return { + "type": "mongodb", + "tables": {}, + "tabbles_list": [] + } \ No newline at end of file From bdad98ff3cfa3db5e83899e752b36746b954c90b Mon Sep 17 00:00:00 2001 From: Pawel Wasilewski Date: Mon, 26 May 2025 11:17:29 +0200 Subject: [PATCH 47/81] completed restructurization of nosql.py and created subconnectors/nosql/mongodb.py subconnector for request --- corebrain/db/connectors/nosql.py | 281 ++++-------------- .../db/connectors/subconnectors/mongodb.py | 67 ----- .../connectors/subconnectors/nosql/mongodb.py | 208 +++++++++++++ 3 files changed, 261 insertions(+), 295 deletions(-) delete mode 100644 corebrain/db/connectors/subconnectors/mongodb.py create mode 100644 corebrain/db/connectors/subconnectors/nosql/mongodb.py diff --git a/corebrain/db/connectors/nosql.py b/corebrain/db/connectors/nosql.py index 0f77f76..21e1819 100644 --- a/corebrain/db/connectors/nosql.py +++ b/corebrain/db/connectors/nosql.py @@ -17,7 +17,14 @@ PYMONGO_IMPORTED = True except ImportError: PYMONGO_IMPORTED = False -# Whe nadding new DB type write a try to it from user + # Whe nadding new DB type write a try to it from user + +try: + import corebrain.db.connectors.subconnectors.nosql.mongodb as mongodb_subconnector + MONGO_MODULES = True +except ImportError: + MONGO_MODULES = False + from corebrain.db.connector import DatabaseConnector class NoSQLConnector(DatabaseConnector): @@ -170,178 +177,24 @@ def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] "tables": {}, "tables_list": [] } - - schema = { - "type": self.engine, - "database": self.db.name, - "tables": {}, # Depends on DB - } - match self.engine: case "mongodb": + if not PYMONGO_IMPORTED: raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") - try: - import corebrain.db.connectors.subconnectors.mongodb as mongodb_subconnector - return mongodb_subconnector.extract_schema(self, sample_limit, collection_limit, progress_callback) - except ImportError: - raise ImportError("Failed to import MongoDB subconnector. Please ensure it is installed correctly.") + if not MONGO_MODULES: + raise ImportError("MongoDB subconnector modules are not available. Please check your installation.") + # Use the MongoDB subconnector to extract schema + return mongodb_subconnector.extract_schema(self, sample_limit, collection_limit, progress_callback) # If adding new db add thru self.engine variable # Add case when is needed new DB type case _: - return schema - ''' - match self.engine: - case "mongodb": - if not PYMONGO_IMPORTED: - raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") - if not self.client and not self.connect(): - return { - "type": "mongodb", - "tables": {}, - "tables_list": [] - } - schema = { - "type": "mongodb", + return { + "type": self.engine, "database": self.db.name, - "tables": {}, # In MongoDB, tables are collections + "tables": {}, # Depends on DB } - try: - collections = self.db.list_collection_names() - if collection_limit is not None and collection_limit > 0: - collections = collections[:collection_limit] - total_collections = len(collections) - for i, collection_name in enumerate(collections): - if progress_callback: - progress_callback(i, total_collections, f"Processing collection: {collection_name}") - collection = self.db[collection_name] - - try: - doc_count = collection.count_documents({}) - if doc_count <= 0: - schema["tables"][collection_name] = { - "fields": [], - "sample_data": [], - "count": 0, - "empty": True - } - else: - sample_docs = list(collection.find().limit(sample_limit)) - fields = {} - sample_data = [] - - for doc in sample_docs: - self._extract_document_fields(doc, fields) - processed_doc = self._process_document_for_serialization(doc) - sample_data.append(processed_doc) - - formatted_fields = [{"name": field, "type": type_name} for field, type_name in fields.items()] - - schema["tables"][collection_name] = { - "fields": formatted_fields, - "sample_data": sample_data, - "count": doc_count, - } - except Exception as e: - print(f"Error processing collection {collection_name}: {e}") - schema["tables"][collection_name] = { - "fields": [], - "error": str(e) - } - # Convert the schema to a list of tables - table_list = [] - for collection_name, collection_info in schema["tables"].items(): - table_data = {"name": collection_name} - table_data.update(collection_info) - table_list.append(table_data) - schema["tables_list"] = table_list - return schema - except Exception as e: - print(f"Error extracting schema: {e}") - return { - "type": "mongodb", - "tables": {}, - "tabbles_list": [] - } - # Add case when is needed new DB type - case _ : - raise ValueError(f"Unsupported NoSQL database: {self.engine}") - ''' - def _extract_document_fields(self, doc: Dict[str, Any], fields: Dict[str, str], - prefix: str = "", max_depth: int = 3, current_depth: int = 0) -> None: - ''' - - Recursively extract fields from a document and determine their types. - Args: - doc (Dict[str, Any]): The document to extract fields from. - fields (Dict[str, str]): Dictionary to store field names and types. - prefix (str): Prefix for nested fields. - max_depth (int): Maximum depth for nested fields. - current_depth (int): Current depth in the recursion. - ''' - match self.engine: - case "mongodb": - if not PYMONGO_IMPORTED: - raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") - if current_depth >= max_depth: - return - for field, value in doc.items(): - if field == "_id": - field_type = "ObjectId" - elif isinstance(value, dict): - if value and current_depth < max_depth - 1: - self._extract_document_fields(value, fields, f"{prefix}{field}.", max_depth, current_depth + 1) - else: - field_type = f"array<{type(value[0]).__name__}>" - else: - field_type = "array" - else: - field_type = type(value).__name__ - - field_key = f"{prefix}{field}" - if field_key not in fields: - fields[field_key] = field_type - # Add case when is needed new DB type - case _ : - raise ValueError(f"Unsupported NoSQL database: {self.engine}") - - def _process_document_for_serialization(self, doc: Dict[str, Any]) -> Dict[str, Any]: - ''' - Proccesig a document for serialization of a JSON. - Args: - doc (Dict[str, Any]): The document to process. - Returns: - Procesed document - ''' - match self.engine: - case "mongodb": - if not PYMONGO_IMPORTED: - raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") - processed_doc = {} - for field, value in doc.items(): - if field == "_id": - processed_doc[field] = self._process_document_for_serialization(value) - elif isinstance(value, list): - processed_items = [] - for item in value: - if isinstance(item, dict): - processed_items.append(self._process_document_for_serialization(item)) - elif hasattr(item, "__str__"): - processed_items.append(str(item)) - else: - processed_items.append(item) - processed_doc[field] = processed_items - # Convert fetch to ISO - elif hasattr(value, 'isoformat'): - processed_doc[field] = value.isoformat() - # Convert data - else: - processed_doc[field] = value - return processed_doc - # Add case when is needed new DB type - case _ : - raise ValueError(f"Unsupported NoSQL database: {self.engine}") - + def execute_query(self, query: str) -> List[Dict[str, Any]]: """ Runs a NoSQL (or other) query with improved error handling @@ -352,68 +205,40 @@ def execute_query(self, query: str) -> List[Dict[str, Any]]: Returns: List of resulting documents. """ - - match self.engine: - case "nosql": + if not self.conn and not self.connect(): + raise ConnectionError("Couldn't establish a connection with NoSQL database.") + + try: + if self.engine == "mongodb": if not PYMONGO_IMPORTED: - raise ImportError("Pymongo is not installed. Please install it to use NoSQL connector.") - - if not self.client and not self.connect(): - raise ConnectionError("Couldn't estabilish a connection with NoSQL") - - try: - # Determine whether the query is a JSON string or a query in another format - filter_dict, projection, collection_name, limit = self._parse_query(query) - - # Get the collection - if not collection_name: - raise ValueError("Name of the colletion not specified in the query") - - collection = self.db[collection_name] - - # Execute the query - if projection: - cursor = collection.find(filter_dict, projection).limit(limit or 100) - else: - cursor = collection.find(filter_dict).limit(limit or 100) - - # Convert the results to a serializable format - results = [] - for doc in cursor: - processed_doc = self._process_document_for_serialization(doc) - results.append(processed_doc) - - return results - - except Exception as e: - # Reconnect and retry the query - try: - self.close() - if self.connect(): - print("Reconnecting and retrying the query...") - - # Retry the query - filter_dict, projection, collection_name, limit = self._parse_query(query) - collection = self.db[collection_name] - - if projection: - cursor = collection.find(filter_dict, projection).limit(limit or 100) - else: - cursor = collection.find(filter_dict).limit(limit or 100) - - results = [] - for doc in cursor: - processed_doc = self._process_document_for_serialization(doc) - results.append(processed_doc) - - return results - except Exception as retry_error: - # If retrying fails, show the original error - raise Exception(f"Failed to execute the NoSQL query: {str(e)}") - - # This code is will be executed if the retry fails - raise Exception(f"Failed to execute the NoSQL query (after the reconnection): {str(e)}") - - # Add case when is needed new DB type - case _ : - raise ValueError(f"Unsupported NoSQL database: {self.self.engine}") \ No newline at end of file + raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") + if not MONGO_MODULES: + raise ImportError("MongoDB subconnector modules are not available. Please check your installation.") + # Use the MongoDB subconnector to execute the query + return mongodb_subconnector.execute_query(self, query) + except Exception as e: + try: + # Attempt to reconnect and retry the query + self.close() + if self.connect(): + print("Reconnecting and retrying the query...") + return mongodb_subconnector.execute_query(self, query) + except Exception as retry_error: + # If retrying fails, show the original error + raise Exception(f"Failed to execute the NoSQL query: {str(e)}") + def close(self) -> None: + """ + Close the connection to the NoSQL database. + """ + if self.client: + self.client.close() + self.client = None + self.db = None + print(f"Connection to {self.engine} closed.") + else: + print(f"No active connection to {self.engine} to close.") + def __del__(self): + """ + Destructor to ensure the connection is closed when the object is deleted. + """ + self.close() \ No newline at end of file diff --git a/corebrain/db/connectors/subconnectors/mongodb.py b/corebrain/db/connectors/subconnectors/mongodb.py deleted file mode 100644 index 0409006..0000000 --- a/corebrain/db/connectors/subconnectors/mongodb.py +++ /dev/null @@ -1,67 +0,0 @@ -def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] = None, - progress_callback: Optional[Callable] = None) -> Dict[str, Any]: - ''' - extract schema for MongoDB collections - Args: - sample_limit (int): Number of samples to extract from each collection. - collection_limit (Optional[int]): Maximum number of collections to process. - progress_callback (Optional[Callable]): Function to call for progress updates. - ''' - - try: - collections = self.db.list_collection_names() - if collection_limit is not None and collection_limit > 0: - collections = collections[:collection_limit] - total_collections = len(collections) - for i, collection_name in enumerate(collections): - if progress_callback: - progress_callback(i, total_collections, f"Processing collection: {collection_name}") - collection = self.db[collection_name] - - try: - doc_count = collection.count_documents({}) - if doc_count <= 0: - schema["tables"][collection_name] = { - "fields": [], - "sample_data": [], - "count": 0, - "empty": True - } - else: - sample_docs = list(collection.find().limit(sample_limit)) - fields = {} - sample_data = [] - - for doc in sample_docs: - self._extract_document_fields(doc, fields) - processed_doc = self._process_document_for_serialization(doc) - sample_data.append(processed_doc) - - formatted_fields = [{"name": field, "type": type_name} for field, type_name in fields.items()] - - schema["tables"][collection_name] = { - "fields": formatted_fields, - "sample_data": sample_data, - "count": doc_count, - } - except Exception as e: - print(f"Error processing collection {collection_name}: {e}") - schema["tables"][collection_name] = { - "fields": [], - "error": str(e) - } - # Convert the schema to a list of tables - table_list = [] - for collection_name, collection_info in schema["tables"].items(): - table_data = {"name": collection_name} - table_data.update(collection_info) - table_list.append(table_data) - schema["tables_list"] = table_list - return schema - except Exception as e: - print(f"Error extracting schema: {e}") - return { - "type": "mongodb", - "tables": {}, - "tabbles_list": [] - } \ No newline at end of file diff --git a/corebrain/db/connectors/subconnectors/nosql/mongodb.py b/corebrain/db/connectors/subconnectors/nosql/mongodb.py new file mode 100644 index 0000000..4874331 --- /dev/null +++ b/corebrain/db/connectors/subconnectors/nosql/mongodb.py @@ -0,0 +1,208 @@ +import time +import json +import re + +from typing import Dict, Any, List, Optional, Callable, Tuple +from corebrain.db.connectors.nosql import PYMONGO_IMPORTED + +def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] = None, + progress_callback: Optional[Callable] = None) -> Dict[str, Any]: + ''' + extract schema for MongoDB collections + Args: + sample_limit (int): Number of samples to extract from each collection. + collection_limit (Optional[int]): Maximum number of collections to process. + progress_callback (Optional[Callable]): Function to call for progress updates. + ''' + schema = { + "type": self.engine, + "database": self.db.name, + "tables": {}, # Depends on DB + } + + try: + collections = self.db.list_collection_names() + if collection_limit is not None and collection_limit > 0: + collections = collections[:collection_limit] + total_collections = len(collections) + for i, collection_name in enumerate(collections): + if progress_callback: + progress_callback(i, total_collections, f"Processing collection: {collection_name}") + collection = self.db[collection_name] + + try: + doc_count = collection.count_documents({}) + if doc_count <= 0: + schema["tables"][collection_name] = { + "fields": [], + "sample_data": [], + "count": 0, + "empty": True + } + else: + sample_docs = list(collection.find().limit(sample_limit)) + fields = {} + sample_data = [] + + for doc in sample_docs: + self._extract_document_fields(doc, fields) + processed_doc = self._process_document_for_serialization(doc) + sample_data.append(processed_doc) + + formatted_fields = [{"name": field, "type": type_name} for field, type_name in fields.items()] + + schema["tables"][collection_name] = { + "fields": formatted_fields, + "sample_data": sample_data, + "count": doc_count, + } + except Exception as e: + print(f"Error processing collection {collection_name}: {e}") + schema["tables"][collection_name] = { + "fields": [], + "error": str(e) + } + # Convert the schema to a list of tables + table_list = [] + for collection_name, collection_info in schema["tables"].items(): + table_data = {"name": collection_name} + table_data.update(collection_info) + table_list.append(table_data) + schema["tables_list"] = table_list + return schema + except Exception as e: + print(f"Error extracting schema: {e}") + return { + "type": "mongodb", + "tables": {}, + "tabbles_list": [] + } + +def _extract_document_fields(self, doc: Dict[str, Any], fields: Dict[str, str], + prefix: str = "", max_depth: int = 3, current_depth: int = 0) -> None: + ''' + Recursively extract fields from a document and determine their types. + Args: + doc (Dict[str, Any]): The document to extract fields from. + fields (Dict[str, str]): Dictionary to store field names and types. + prefix (str): Prefix for nested fields. + max_depth (int): Maximum depth for nested fields. + current_depth (int): Current depth in the recursion. + ''' + if not PYMONGO_IMPORTED: + raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") + if current_depth >= max_depth: + return + for field, value in doc.items(): + if field == "_id": + field_type = "ObjectId" + elif isinstance(value, dict): + if value and current_depth < max_depth - 1: + self._extract_document_fields(value, fields, f"{prefix}{field}.", max_depth, current_depth + 1) + continue + else: + field_type = f"object" + elif isinstance(value, list): + if value and isinstance(value[0], dict) and current_depth < max_depth - 1: + self._extract_document_fields(value[0], fields, f"{prefix}{field}[].", max_depth, current_depth + 1) + field_type = f"array" + elif value: + field_type = f"array<{type(value[0]).__name__}>" + else: + field_type = "array" + else: + field_type = type(value).__name__ + + field_key = f"{prefix}{field}" + if field_key not in fields: + fields[field_key] = field_type + +def _process_document_for_serialization(self, doc: Dict[str, Any]) -> Dict[str, Any]: + ''' + Proccesig a document for serialization of a JSON. + Args: + doc (Dict[str, Any]): The document to process. + Returns: + Procesed document + ''' + processed_doc = {} + for field, value in doc.items(): + if field == "_id": + processed_doc[field] = self._process_document_for_serialization(value) + elif isinstance(value, list): + processed_items = [] + for item in value: + if isinstance(item, dict): + processed_items.append(self._process_document_for_serialization(item)) + elif hasattr(item, "__str__"): + processed_items.append(str(item)) + else: + processed_items.append(item) + processed_doc[field] = processed_items + # Convert fetch to ISO + elif hasattr(value, 'isoformat'): + processed_doc[field] = value.isoformat() + # Convert data + else: + processed_doc[field] = value + return processed_doc + +def execute_query(self, query: str) -> List[Dict[str, Any]]: + ''' + Execute a query on the MongoDB database. + Args: + query (str): The query to execute, in JSON format. + Returns: + List[Dict[str, Any]]: The results of the query. + ''' + + if not self.client and not self.connect(): + raise ConnectionError("Couldn't establish a connection with NoSQL database.") + try: + # Check if the query is a valid JSON string + filter_dict, projection, collection_name, limit = self._parse_query(query) + + # Get the collection + if not collection_name: + raise ValueError("Name of the collection not specified in the query") + + collection = self.db[collection_name] + + # Execute the query + if projection: + cursor = collection.find(filter_dict, projection).limit(limit or 100) + else: + cursor = collection.find(filter_dict).limit(limit or 100) + + # Convert the results to a serializable format + results = [] + for doc in cursor: + processed_doc = self._process_document_for_serialization(doc) + results.append(processed_doc) + + return results + except Exception as e: + # Reconnect and retry the query + try: + self.close() + if self.connect(): + print("Reconnecting and retrying the query...") + + # Retry the query + filter_dict, projection, collection_name, limit = self._parse_query(query) + collection = self.db[collection_name] + + if projection: + cursor = collection.find(filter_dict, projection).limit(limit or 100) + else: + cursor = collection.find(filter_dict).limit(limit or 100) + + results = [] + for doc in cursor: + processed_doc = self._process_document_for_serialization(doc) + results.append(processed_doc) + return results + except Exception as retry_error: + # If retrying fails, show the original error + raise Exception(f"Failed to execute the NoSQL query: {str(e)}") + \ No newline at end of file From 7944d3257b45ad2e4816e07f6ff46ff989ecf7d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Ayuso?= Date: Mon, 26 May 2025 11:41:06 +0200 Subject: [PATCH 48/81] --show-schema, --remove-config and --extract-schema will be move into --list-configs --- corebrain/cli/commands.py | 30 ++-- corebrain/cli/common.py | 2 +- corebrain/core/__init__.py | 2 +- corebrain/db/__init__.py | 2 +- corebrain/db/connectors/sql.py | 36 ++-- corebrain/db/schema/__init__.py | 2 +- corebrain/db/schema/optimizer.py | 50 +++--- corebrain/db/schema_file.py | 300 +++++++++++++++---------------- corebrain/lib/sso/auth.py | 4 +- corebrain/lib/sso/client.py | 12 +- corebrain/network/__init__.py | 2 +- corebrain/network/client.py | 104 +++++------ corebrain/services/schema.py | 4 - corebrain/utils/__init__.py | 17 +- corebrain/utils/encrypter.py | 24 +-- 15 files changed, 292 insertions(+), 299 deletions(-) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index 96eac44..7248db6 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -47,9 +47,6 @@ def main_cli(argv: Optional[List[str]] = None) -> int: parser.add_argument("--create-user", action="store_true", help="Create an user and API Key by default") parser.add_argument("--configure", action="store_true", help="Configure the Corebrain SDK") parser.add_argument("--list-configs", action="store_true", help="List available configurations") - parser.add_argument("--remove-config", action="store_true", help="Remove a configuration") - parser.add_argument("--show-schema", action="store_true", help="Show the schema of the configured database") - parser.add_argument("--extract-schema", action="store_true", help="Extract the database schema and save it to a file") parser.add_argument("--output-file", help="File to save the extracted schema") parser.add_argument("--config-id", help="Specific configuration ID to use") parser.add_argument("--token", help="Corebrain API token (any type)") @@ -75,7 +72,7 @@ def authentication(): if sso_token: try: print_colored("✅ Returning SSO Token.", "green") - print_colored(f"{sso_user}", "blue") + print_colored(f"{sso_token}", "blue") print_colored("✅ Returning User data.", "green") print_colored(f"{sso_user}", "blue") return sso_token, sso_user @@ -88,7 +85,17 @@ def authentication(): print_colored("❌ Could not authenticate with SSO.", "red") return None, None - # Made by Lukasz + + # Show version + if args.version: + try: + from importlib.metadata import version + sdk_version = version("corebrain") + print(f"Corebrain SDK version {sdk_version}") + except Exception: + print(f"Corebrain SDK version {__version__}") + return 0 + if args.export_config: export_config(args.export_config) # --> config/manager.py --> export_config @@ -100,16 +107,7 @@ def authentication(): return validate_config(args.config_id) - # Show version - if args.version: - try: - from importlib.metadata import version - sdk_version = version("corebrain") - print(f"Corebrain SDK version {sdk_version}") - except Exception: - print(f"Corebrain SDK version {__version__}") - return 0 - + # Create an user and API Key by default if args.authentication: authentication() @@ -304,7 +302,7 @@ def authentication(): return 1 # Operations that require credentials: configure, list, remove or show schema - if args.configure or args.list_configs or args.remove_config or args.show_schema or args.extract_schema: + if args.configure or args.list_configs: # Get URLs api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL diff --git a/corebrain/cli/common.py b/corebrain/cli/common.py index 7799dcf..8599bee 100644 --- a/corebrain/cli/common.py +++ b/corebrain/cli/common.py @@ -2,7 +2,7 @@ Default values for SSO and API connection """ -DEFAULT_API_URL = "http://localhost:5000" +DEFAULT_API_URL = "http://localhost:1000" # Use 5000 in Windows / 1000 in MacOS by default #DEFAULT_SSO_URL = "http://localhost:3000" # localhost DEFAULT_SSO_URL = "https://sso.globodain.com" # remote DEFAULT_PORT = 8765 diff --git a/corebrain/core/__init__.py b/corebrain/core/__init__.py index a5fb552..d10b67f 100644 --- a/corebrain/core/__init__.py +++ b/corebrain/core/__init__.py @@ -8,7 +8,7 @@ from corebrain.core.query import QueryCache, QueryAnalyzer, QueryTemplate from corebrain.core.test_utils import test_natural_language_query, generate_test_question_from_schema -# Exportación explícita de componentes públicos +# Export public componentes __all__ = [ 'Corebrain', 'init', diff --git a/corebrain/db/__init__.py b/corebrain/db/__init__.py index 6ac390d..b60cf2e 100644 --- a/corebrain/db/__init__.py +++ b/corebrain/db/__init__.py @@ -13,7 +13,7 @@ from corebrain.db.schema.optimizer import SchemaOptimizer from corebrain.db.schema.extractor import extract_db_schema -# Exportación explícita de componentes públicos +# Export public components __all__ = [ 'DatabaseConnector', 'get_connector', diff --git a/corebrain/db/connectors/sql.py b/corebrain/db/connectors/sql.py index 82f49bf..7f457a0 100644 --- a/corebrain/db/connectors/sql.py +++ b/corebrain/db/connectors/sql.py @@ -105,14 +105,14 @@ def connect(self) -> bool: except (sqlite3.Error, mysql.connector.Error, psycopg2.Error) as e: # Si el error no es de timeout, propagar la excepción - if "timeout" not in str(e).lower() and "tiempo de espera" not in str(e).lower(): + if "timeout" not in str(e).lower() and "wait timeout" not in str(e).lower(): raise # Si es un error de timeout, esperamos un poco y reintentamos time.sleep(1.0) # Si llegamos aquí, se agotó el tiempo de espera - raise TimeoutError(f"No se pudo conectar a la base de datos en {self.connection_timeout} segundos") + raise TimeoutError(f"Could not connect to the database in {self.connection_timeout} seconds") except Exception as e: if self.conn: @@ -122,7 +122,7 @@ def connect(self) -> bool: pass self.conn = None - print(f"Error al conectar a la base de datos: {str(e)}") + print(f"Error connecting to the database: {str(e)}") return False def extract_schema(self, sample_limit: int = 5, table_limit: Optional[int] = None, @@ -182,14 +182,14 @@ def execute_query(self, query: str) -> List[Dict[str, Any]]: elif self.engine == "postgresql": return self._execute_postgresql_query(query) else: - raise ValueError(f"Motor de base de datos no soportado: {self.engine}") + raise ValueError(f"Database engine not supported: {self.engine}") except Exception as e: # Intentar reconectar y reintentar una vez try: self.close() if self.connect(): - print("Reconectando y reintentando consulta...") + print("Reconnecting and retrying query...") if self.engine == "sqlite": return self._execute_sqlite_query(query) @@ -200,10 +200,10 @@ def execute_query(self, query: str) -> List[Dict[str, Any]]: except Exception as retry_error: # Si falla el reintento, propagar el error original - raise Exception(f"Error al ejecutar consulta: {str(e)}") + raise Exception(f"Error executing query: {str(e)}") # Si llegamos aquí sin retornar, ha habido un error en el reintento - raise Exception(f"Error al ejecutar consulta (después de reconexión): {str(e)}") + raise Exception(f"Error executing query (after reconnection): {str(e)}") def _execute_sqlite_query(self, query: str) -> List[Dict[str, Any]]: """Executes a query in SQLite.""" @@ -275,7 +275,7 @@ def _extract_sqlite_schema(self, sample_limit: int, table_limit: Optional[int], for i, table_name in enumerate(tables): # Reportar progreso si hay callback if progress_callback: - progress_callback(i, total_tables, f"Procesando tabla {table_name}") + progress_callback(i, total_tables, f"Processing table {table_name}") # Extraer información de columnas cursor.execute(f"PRAGMA table_info({table_name});") @@ -309,12 +309,12 @@ def _extract_sqlite_schema(self, sample_limit: int, table_limit: Optional[int], schema["tables"][table_name]["sample_data"] = sample_data except Exception as e: - print(f"Error al obtener muestra de datos para tabla {table_name}: {str(e)}") + print(f"Error getting sample data for table {table_name}: {str(e)}") # TODO: Translate to English cursor.close() except Exception as e: - print(f"Error al extraer esquema SQLite: {str(e)}") + print(f"Error extracting SQLite schema: {str(e)}") # TODO: Translate to English # Crear la lista de tablas para compatibilidad table_list = [] @@ -372,7 +372,7 @@ def _extract_mysql_schema(self, sample_limit: int, table_limit: Optional[int], p for i, table_name in enumerate(tables): # Reportar progreso si hay callback if progress_callback: - progress_callback(i, total_tables, f"Procesando tabla {table_name}") + progress_callback(i, total_tables, f"Processing table {table_name}") # Extraer información de columnas cursor.execute(f"DESCRIBE `{table_name}`;") @@ -405,12 +405,12 @@ def _extract_mysql_schema(self, sample_limit: int, table_limit: Optional[int], p schema["tables"][table_name]["sample_data"] = processed_samples except Exception as e: - print(f"Error al obtener muestra de datos para tabla {table_name}: {str(e)}") + print(f"Error getting sample data for table {table_name}: {str(e)}") # TODO: Translate to English cursor.close() except Exception as e: - print(f"Error al extraer esquema MySQL: {str(e)}") + print(f"Error extracting MySQL schema: {str(e)}") # TODO: Translate to English # Crear la lista de tablas para compatibilidad table_list = [] @@ -540,7 +540,7 @@ def _extract_postgresql_schema(self, sample_limit: int, table_limit: Optional[in schema["tables"][full_name]["sample_data"] = sample_data except Exception as e: - print(f"Error al obtener muestra de datos para tabla {full_name}: {str(e)}") + print(f"Error getting sample data for table {full_name}: {str(e)}") # TODO: Translate to English else: # Registrar la tabla aunque no tenga columnas schema["tables"][full_name] = {"columns": [], "sample_data": []} @@ -548,7 +548,7 @@ def _extract_postgresql_schema(self, sample_limit: int, table_limit: Optional[in cursor.close() except Exception as e: - print(f"Error al extraer esquema PostgreSQL: {str(e)}") + print(f"Error extracting PostgreSQL schema: {str(e)}") # TODO: Translate to English # Intento de recuperación para diagnosticar problemas try: @@ -558,7 +558,7 @@ def _extract_postgresql_schema(self, sample_limit: int, table_limit: Optional[in # Verificar versión recovery_cursor.execute("SELECT version();") version = recovery_cursor.fetchone() - print(f"Versión PostgreSQL: {version[0] if version else 'Desconocida'}") + print(f"PostgreSQL version: {version[0] if version else 'Unknown'}") # Verificar permisos recovery_cursor.execute(""" @@ -567,11 +567,11 @@ def _extract_postgresql_schema(self, sample_limit: int, table_limit: Optional[in """) perms = recovery_cursor.fetchone() if perms: - print(f"Permisos en esquema public: USAGE={perms[0]}, CREATE={perms[1]}") + print(f"Permissions in public schema: USAGE={perms[0]}, CREATE={perms[1]}") # TODO: Translate to English recovery_cursor.close() except Exception as diag_err: - print(f"Error durante el diagnóstico: {str(diag_err)}") + print(f"Error during diagnosis: {str(diag_err)}") # TODO: Translate to English # Crear la lista de tablas para compatibilidad table_list = [] diff --git a/corebrain/db/schema/__init__.py b/corebrain/db/schema/__init__.py index 388ee63..a620024 100644 --- a/corebrain/db/schema/__init__.py +++ b/corebrain/db/schema/__init__.py @@ -4,7 +4,7 @@ from .extractor import extract_schema from .optimizer import SchemaOptimizer -# Alias para compatibilidad con código existente +# Alias for compatibility with existing code extract_db_schema = extract_schema schemaOptimizer = SchemaOptimizer diff --git a/corebrain/db/schema/optimizer.py b/corebrain/db/schema/optimizer.py index c7840b0..1d5751a 100644 --- a/corebrain/db/schema/optimizer.py +++ b/corebrain/db/schema/optimizer.py @@ -24,13 +24,13 @@ def __init__(self, max_tables: int = 10, max_columns_per_table: int = 15, max_sa self.max_columns_per_table = max_columns_per_table self.max_samples = max_samples - # Tablas importantes que siempre deben incluirse si existen + # Tables that are always important self.priority_tables = set([ "users", "customers", "products", "orders", "transactions", "invoices", "accounts", "clients", "employees", "services" ]) - # Tablas típicamente menos importantes + # Tables that are typically less important self.low_priority_tables = set([ "logs", "sessions", "tokens", "temp", "cache", "metrics", "statistics", "audit", "history", "archives", "settings" @@ -47,7 +47,7 @@ def optimize_schema(self, db_schema: Dict[str, Any], query: str = None) -> Dict[ Returns: Optimized schema """ - # Crear copia para no modificar el original + # Create a copy to not modify the original optimized_schema = { "type": db_schema.get("type", ""), "database": db_schema.get("database", ""), @@ -56,68 +56,68 @@ def optimize_schema(self, db_schema: Dict[str, Any], query: str = None) -> Dict[ "tables_list": [] } - # Determinar tablas relevantes para la consulta + # Determine relevant tables for the query query_relevant_tables = set() if query: - # Extraer potenciales nombres de tablas de la consulta + # Extract potential table names from the query normalized_query = query.lower() - # Obtener nombres de todas las tablas + # Get all table names all_table_names = [ name.lower() for name in db_schema.get("tables", {}).keys() ] - # Buscar menciones a tablas en la consulta + # Search for table mentions in the query for table_name in all_table_names: - # Buscar el nombre exacto (como palabra completa) + # Search for the exact name (as a whole word) if re.search(r'\b' + re.escape(table_name) + r'\b', normalized_query): query_relevant_tables.add(table_name) - # También buscar formas singulares/plurales simples + # Also search for singular/plural simple forms if table_name.endswith('s') and re.search(r'\b' + re.escape(table_name[:-1]) + r'\b', normalized_query): query_relevant_tables.add(table_name) elif not table_name.endswith('s') and re.search(r'\b' + re.escape(table_name + 's') + r'\b', normalized_query): query_relevant_tables.add(table_name) - # Priorizar tablas a incluir + # Prioritize tables to include table_scores = {} for table_name in db_schema.get("tables", {}): score = 0 - # Tablas mencionadas en la consulta tienen máxima prioridad + # Tables mentioned in the query have maximum priority if table_name.lower() in query_relevant_tables: score += 100 - # Tablas importantes + # Important tables if table_name.lower() in self.priority_tables: score += 50 - # Tablas poco importantes + # Less important tables if table_name.lower() in self.low_priority_tables: score -= 30 - # Tablas con más columnas pueden ser más relevantes + # Tables with more columns may be more relevant table_info = db_schema["tables"].get(table_name, {}) column_count = len(table_info.get("columns", [])) - score += min(column_count, 20) # Limitar a 20 puntos máximo + score += min(column_count, 20) # Limit to 20 points maximum - # Guardar puntuación + # Save score table_scores[table_name] = score - # Ordenar tablas por puntuación + # Sort tables by score sorted_tables = sorted(table_scores.items(), key=lambda x: x[1], reverse=True) - # Limitar número de tablas + # Limit number of tables selected_tables = [name for name, _ in sorted_tables[:self.max_tables]] - # Copiar tablas seleccionadas con optimizaciones + # Copy selected tables with optimizations for table_name in selected_tables: table_info = db_schema["tables"].get(table_name, {}) - # Optimizar columnas + # Optimize columns columns = table_info.get("columns", []) if len(columns) > self.max_columns_per_table: - # Mantener las columnas más importantes (id, nombre, clave primaria, etc) + # Keep most important columns (id, name, primary key, etc) important_columns = [] other_columns = [] @@ -128,7 +128,7 @@ def optimize_schema(self, db_schema: Dict[str, Any], query: str = None) -> Dict[ else: other_columns.append(col) - # Tomar las columnas importantes y completar con otras hasta el límite + # Take the most important columns and complete with others up to the limit optimized_columns = important_columns remaining_slots = self.max_columns_per_table - len(optimized_columns) if remaining_slots > 0: @@ -136,17 +136,17 @@ def optimize_schema(self, db_schema: Dict[str, Any], query: str = None) -> Dict[ else: optimized_columns = columns - # Optimizar datos de muestra + # Optimize sample data sample_data = table_info.get("sample_data", []) optimized_samples = sample_data[:self.max_samples] if sample_data else [] - # Guardar tabla optimizada + # Save optimized table optimized_schema["tables"][table_name] = { "columns": optimized_columns, "sample_data": optimized_samples } - # Añadir a la lista de tablas + # Add to the list of tables optimized_schema["tables_list"].append({ "name": table_name, "columns": optimized_columns, diff --git a/corebrain/db/schema_file.py b/corebrain/db/schema_file.py index c4dc8f7..a18480f 100644 --- a/corebrain/db/schema_file.py +++ b/corebrain/db/schema_file.py @@ -31,34 +31,34 @@ def extract_db_schema(db_config: Dict[str, Any]) -> Dict[str, Any]: schema = { "type": db_type, "database": db_config.get("database", ""), - "tables": {} # Cambiado a diccionario para facilitar el acceso directo a tablas por nombre + "tables": {} # Changed to dictionary for direct table access by name } try: if db_type == "sql": - # Código para bases de datos SQL... - # [Se mantiene igual] + # Code for SQL databases... + # [Kept the same] pass - # Manejar tanto "nosql" como "mongodb" como tipos válidos + # Handle both "nosql" and "mongodb" as valid types elif db_type == "nosql" or db_type == "mongodb": import pymongo - # Determinar el motor (si existe) + # Determine the engine (if it exists) engine = db_config.get("engine", "").lower() - # Si no se especifica el engine o es mongodb, proceder + # If the engine is not specified or is mongodb, proceed if not engine or engine == "mongodb": if "connection_string" in db_config: client = pymongo.MongoClient(db_config["connection_string"]) else: - # Diccionario de parámetros para MongoClient + # Dictionary of parameters for MongoClient mongo_params = { "host": db_config.get("host", "localhost"), "port": db_config.get("port", 27017) } - # Añadir credenciales solo si están presentes + # Add credentials only if they are present if db_config.get("user"): mongo_params["username"] = db_config["user"] if db_config.get("password"): @@ -66,41 +66,41 @@ def extract_db_schema(db_config: Dict[str, Any]) -> Dict[str, Any]: client = pymongo.MongoClient(**mongo_params) - # Obtener la base de datos + # Get the database db_name = db_config.get("database", "") if not db_name: - _print_colored("⚠️ Nombre de base de datos no especificado", "yellow") + _print_colored("⚠️ Database name not specified", "yellow") return schema try: db = client[db_name] collection_names = db.list_collection_names() - # Procesar colecciones + # Process collections for collection_name in collection_names: collection = db[collection_name] - # Obtener varios documentos de muestra + # Get several sample documents try: sample_docs = list(collection.find().limit(5)) - # Extraer estructura de campos a partir de los documentos + # Extract field structure from documents field_types = {} for doc in sample_docs: for field, value in doc.items(): - if field != "_id": # Ignoramos el _id de MongoDB - # Actualizar el tipo si no existe o combinar si hay diferentes tipos + if field != "_id": # Ignore the _id of MongoDB + # Update the type if it doesn't exist or combine if there are different types field_type = type(value).__name__ if field not in field_types: field_types[field] = field_type elif field_types[field] != field_type: field_types[field] = f"{field_types[field]}|{field_type}" - # Convertir a formato esperado + # Convert to expected format fields = [{"name": field, "type": type_name} for field, type_name in field_types.items()] - # Convertir documentos a formato serializable + # Convert documents to serializable format sample_data = [] for doc in sample_docs: serialized_doc = {} @@ -108,18 +108,18 @@ def extract_db_schema(db_config: Dict[str, Any]) -> Dict[str, Any]: if key == "_id": serialized_doc[key] = str(value) elif isinstance(value, (dict, list)): - serialized_doc[key] = str(value) # Simplificar objetos anidados + serialized_doc[key] = str(value) # Simplify nested objects else: serialized_doc[key] = value sample_data.append(serialized_doc) - # Guardar información de la colección + # Save collection information schema["tables"][collection_name] = { "fields": fields, "sample_data": sample_data } except Exception as e: - _print_colored(f"Error al procesar colección {collection_name}: {str(e)}", "red") + _print_colored(f"Error processing collection {collection_name}: {str(e)}", "red") schema["tables"][collection_name] = { "fields": [], "sample_data": [], @@ -127,29 +127,29 @@ def extract_db_schema(db_config: Dict[str, Any]) -> Dict[str, Any]: } except Exception as e: - _print_colored(f"Error al acceder a la base de datos MongoDB '{db_name}': {str(e)}", "red") + _print_colored(f"Error accessing MongoDB database '{db_name}': {str(e)}", "red") finally: - # Cerrar la conexión + # Close the connection client.close() else: - _print_colored(f"Motor de base de datos NoSQL no soportado: {engine}", "red") + _print_colored(f"Database engine not supported: {engine}", "red") - # Convertir el diccionario de tablas en una lista para mantener compatibilidad con el formato anterior + # Convert the dictionary of tables to a list to maintain compatibility with the previous format table_list = [] for table_name, table_info in schema["tables"].items(): table_data = {"name": table_name} table_data.update(table_info) table_list.append(table_data) - # Guardar también la lista de tablas para mantener compatibilidad + # Save the list of tables to maintain compatibility schema["tables_list"] = table_list return schema except Exception as e: - _print_colored(f"Error al extraer el esquema de la base de datos: {str(e)}", "red") - # En caso de error, devolver un esquema vacío + _print_colored(f"Error extracting database schema: {str(e)}", "red") + # In case of error, return an empty schema return {"type": db_type, "tables": {}, "tables_list": []} def extract_db_schema_direct(db_config: Dict[str, Any]) -> Dict[str, Any]: @@ -162,16 +162,16 @@ def extract_db_schema_direct(db_config: Dict[str, Any]) -> Dict[str, Any]: "type": db_type, "database": db_config.get("database", ""), "tables": {}, - "tables_list": [] # Lista inicialmente vacía + "tables_list": [] # Initially empty list } try: - # [Implementación existente para extraer esquema sin usar Corebrain] - # ... + # [Existing implementation to extract schema without using Corebrain] + # Add function to extract schema without using Corebrain return schema except Exception as e: - _print_colored(f"Error al extraer esquema directamente: {str(e)}", "red") + _print_colored(f"Error extracting schema directly: {str(e)}", "red") return {"type": db_type, "tables": {}, "tables_list": []} def extract_schema_with_lazy_init(api_key: str, db_config: Dict[str, Any], api_url: Optional[str] = None) -> Dict[str, Any]: @@ -182,30 +182,30 @@ def extract_schema_with_lazy_init(api_key: str, db_config: Dict[str, Any], api_u the Corebrain client only when necessary. """ try: - # La importación se mueve aquí para evitar el problema de circular import - # Solo se ejecuta cuando realmente necesitamos crear el cliente + # The import is moved here to avoid the circular import problem + # It is only executed when we really need to create the client import importlib core_module = importlib.import_module('core') init_func = getattr(core_module, 'init') - # Crear cliente con la configuración - api_url_to_use = api_url or "https://api.corebrain.com" + # Create client with the configuration + api_url_to_use = api_url or "https://api.etedata.com" cb = init_func( api_token=api_key, db_config=db_config, api_url=api_url_to_use, - skip_verification=True # No necesitamos verificar token para extraer schema + skip_verification=True # We don't need to verify token to extract schema ) - # Obtener el esquema y cerrar cliente + # Get the schema and close the client schema = cb.db_schema cb.close() return schema except Exception as e: - _print_colored(f"Error al extraer esquema con cliente: {str(e)}", "red") - # Como alternativa, usar extracción directa sin cliente + _print_colored(f"Error extracting schema with client: {str(e)}", "red") + # As an alternative, use direct extraction without client return extract_db_schema_direct(db_config) from typing import Dict, Any @@ -247,74 +247,74 @@ def extract_schema_to_file(api_key: str, config_id: Optional[str] = None, output try: from corebrain.config.manager import ConfigManager except ImportError as e: - _print_colored(f"Error al importar ConfigManager: {e}", "red") + _print_colored(f"Error importing ConfigManager: {e}", "red") return False - # Obtener las configuraciones disponibles + # Get the available configurations config_manager = ConfigManager() configs = config_manager.list_configs(api_key) if not configs: - _print_colored("No hay configuraciones guardadas para esta API Key.", "yellow") + _print_colored("No configurations saved for this API Key.", "yellow") return False selected_config_id = config_id - # Si no se especifica un config_id, mostrar lista para seleccionar + # If no config_id is specified, show list to select if not selected_config_id: - _print_colored("\n=== Configuraciones disponibles ===", "blue") + _print_colored("\n=== Available configurations ===", "blue") for i, conf_id in enumerate(configs, 1): print(f"{i}. {conf_id}") try: - choice = int(input(f"\nSelecciona una configuración (1-{len(configs)}): ").strip()) + choice = int(input(f"\nSelect a configuration (1-{len(configs)}): ").strip()) if 1 <= choice <= len(configs): selected_config_id = configs[choice - 1] else: - _print_colored("Opción inválida.", "red") + _print_colored("Invalid option.", "red") return False except ValueError: - _print_colored("Por favor, introduce un número válido.", "red") + _print_colored("Please enter a valid number.", "red") return False - # Verificar que el config_id exista + # Verify that the config_id exists if selected_config_id not in configs: - _print_colored(f"No se encontró la configuración con ID: {selected_config_id}", "red") + _print_colored(f"No configuration found with ID: {selected_config_id}", "red") return False - # Obtener la configuración seleccionada + # Get the selected configuration db_config = config_manager.get_config(api_key, selected_config_id) if not db_config: - _print_colored(f"Error al obtener la configuración con ID: {selected_config_id}", "red") + _print_colored(f"Error getting configuration with ID: {selected_config_id}", "red") return False - _print_colored(f"\nExtrayendo esquema para configuración: {selected_config_id}", "blue") - print(f"Tipo: {db_config['type'].upper()}, Motor: {db_config.get('engine', 'No especificado').upper()}") - print(f"Base de datos: {db_config.get('database', 'No especificada')}") + _print_colored(f"\nExtracting schema for configuration: {selected_config_id}", "blue") + print(f"Type: {db_config['type'].upper()}, Engine: {db_config.get('engine', 'No specified').upper()}") + print(f"Database: {db_config.get('database', 'No specified')}") - # Extraer el esquema de la base de datos - _print_colored("\nExtrayendo esquema de la base de datos...", "blue") + # Extract the schema from the database + _print_colored("\nExtracting schema from the database...", "blue") schema = extract_schema_with_lazy_init(api_key, db_config, api_url) - # Verificar si se obtuvo un esquema válido + # Verify if a valid schema was obtained if not schema or not schema.get("tables"): - _print_colored("No se encontraron tablas/colecciones en la base de datos.", "yellow") + _print_colored("No tables/collections found in the database.", "yellow") return False - # Guardar el esquema en un archivo + # Save the schema in a file output_path = output_file or "db_schema.json" try: with open(output_path, 'w', encoding='utf-8') as f: json.dump(schema, f, indent=2, default=str) - _print_colored(f"✅ Esquema extraído y guardado en: {output_path}", "green") + _print_colored(f"✅ Schema extracted and saved in: {output_path}", "green") except Exception as e: - _print_colored(f"❌ Error al guardar el archivo: {str(e)}", "red") + _print_colored(f"❌ Error saving the file: {str(e)}", "red") return False - # Mostrar un resumen de las tablas/colecciones encontradas + # Show a summary of the tables/collections found tables = schema.get("tables", {}) - _print_colored(f"\nResumen del esquema extraído: {len(tables)} tablas/colecciones", "green") + _print_colored(f"\nSummary of the extracted schema: {len(tables)} tables/collections", "green") for table_name in tables: print(f"- {table_name}") @@ -322,7 +322,7 @@ def extract_schema_to_file(api_key: str, config_id: Optional[str] = None, output return True except Exception as e: - _print_colored(f"❌ Error al extraer esquema: {str(e)}", "red") + _print_colored(f"❌ Error extracting schema: {str(e)}", "red") return False def show_db_schema(api_token: str, config_id: Optional[str] = None, api_url: Optional[str] = None) -> None: @@ -335,73 +335,73 @@ def show_db_schema(api_token: str, config_id: Optional[str] = None, api_url: Opt api_url: Optional API URL """ try: - # Importación explícita con try-except para manejar errores + # Explicit import with try-except to handle errors try: from corebrain.config.manager import ConfigManager except ImportError as e: - _print_colored(f"Error al importar ConfigManager: {e}", "red") + _print_colored(f"Error importing ConfigManager: {e}", "red") return False - # Obtener las configuraciones disponibles + # Get the available configurations config_manager = ConfigManager() configs = config_manager.list_configs(api_token) if not configs: - _print_colored("No hay configuraciones guardadas para este token.", "yellow") + _print_colored("No configurations saved for this token.", "yellow") return selected_config_id = config_id - # Si no se especifica un config_id, mostrar lista para seleccionar + # If no config_id is specified, show list to select if not selected_config_id: - _print_colored("\n=== Configuraciones disponibles ===", "blue") + _print_colored("\n=== Available configurations ===", "blue") for i, conf_id in enumerate(configs, 1): print(f"{i}. {conf_id}") try: - choice = int(input(f"\nSelecciona una configuración (1-{len(configs)}): ").strip()) - if 1 <= choice <= len(configs): + choice = int(input(f"\nSelect a configuration (1-{len(configs)}): ").strip()) + if 1 <= choice <= len(configs): selected_config_id = configs[choice - 1] else: - _print_colored("Opción inválida.", "red") + _print_colored("Invalid option.", "red") return except ValueError: - _print_colored("Por favor, introduce un número válido.", "red") + _print_colored("Please enter a valid number.", "red") return - # Verificar que el config_id exista + # Verify that the config_id exists if selected_config_id not in configs: - _print_colored(f"No se encontró la configuración con ID: {selected_config_id}", "red") + _print_colored(f"No configuration found with ID: {selected_config_id}", "red") return if config_id and config_id in configs: db_config = config_manager.get_config(api_token, config_id) else: - # Obtener la configuración seleccionada + # Get the selected configuration db_config = config_manager.get_config(api_token, selected_config_id) if not db_config: - _print_colored(f"Error al obtener la configuración con ID: {selected_config_id}", "red") + _print_colored(f"Error getting configuration with ID: {selected_config_id}", "red") return - _print_colored(f"\nObteniendo esquema para configuración: {selected_config_id}", "blue") - _print_colored("Tipo de base de datos:", "blue") + _print_colored(f"\nGetting schema for configuration: {selected_config_id}", "blue") + _print_colored("Database type:", "blue") print(f" {db_config['type'].upper()}") if db_config.get('engine'): _print_colored("Motor:", "blue") print(f" {db_config['engine'].upper()}") - _print_colored("Base de datos:", "blue") - print(f" {db_config.get('database', 'No especificada')}") + _print_colored("Database:", "blue") + print(f" {db_config.get('database', 'No specified')}") - # Extraer y mostrar el esquema - _print_colored("\nExtrayendo esquema de la base de datos...", "blue") + # Extract and show the schema + _print_colored("\nExtracting schema from the database...", "blue") - # Intenta conectarse a la base de datos y extraer el esquema + # Try to connect to the database and extract the schema try: - # Creamos una instancia de Corebrain con la configuración seleccionada + # Create a Corebrain instance with the selected configuration """ cb = init( api_token=api_token, @@ -415,15 +415,15 @@ def show_db_schema(api_token: str, config_id: Optional[str] = None, api_url: Opt core_module = importlib.import_module('core.client') init_func = getattr(core_module, 'init') - # Creamos una instancia de Corebrain con la configuración seleccionada + # Create a Corebrain instance with the selected configuration cb = init_func( api_token=api_token, config_id=config_id, api_url=api_url, - skip_verification=True # Omitimos verificación para simplificar + skip_verification=True # Skip verification for simplicity ) - # El esquema se extrae automáticamente al inicializar + # The schema is extracted automatically when initializing schema = get_schema_with_dynamic_import( api_token=api_token, config_id=selected_config_id, @@ -431,45 +431,45 @@ def show_db_schema(api_token: str, config_id: Optional[str] = None, api_url: Opt api_url=api_url ) - # Si no hay esquema, intentamos extraerlo explícitamente + # If there is no schema, try to extract it explicitly if not schema or not schema.get("tables"): - _print_colored("Intentando extraer esquema explícitamente...", "yellow") + _print_colored("Trying to extract schema explicitly...", "yellow") schema = cb._extract_db_schema() - # Cerramos la conexión + # Close the connection cb.close() except Exception as conn_error: - _print_colored(f"Error de conexión: {str(conn_error)}", "red") - print("Intentando método alternativo...") + _print_colored(f"Connection error: {str(conn_error)}", "red") + print("Trying alternative method...") - # Método alternativo: usar función extract_db_schema directamente + # Alternative method: use extract_db_schema directly schema = extract_db_schema(db_config) - # Verificar si se obtuvo un esquema válido + # Verify if a valid schema was obtained if not schema or not schema.get("tables"): - _print_colored("No se encontraron tablas/colecciones en la base de datos.", "yellow") + _print_colored("No tables/collections found in the database.", "yellow") - # Información adicional para ayudar a diagnosticar el problema - print("\nInformación de depuración:") - print(f" Tipo de base de datos: {db_config.get('type', 'No especificado')}") - print(f" Motor: {db_config.get('engine', 'No especificado')}") - print(f" Host: {db_config.get('host', 'No especificado')}") - print(f" Puerto: {db_config.get('port', 'No especificado')}") - print(f" Base de datos: {db_config.get('database', 'No especificado')}") + # Additional information to help diagnose the problem + print("\nDebug information:") + print(f" Database type: {db_config.get('type', 'No specified')}") + print(f" Engine: {db_config.get('engine', 'No specified')}") + print(f" Host: {db_config.get('host', 'No specified')}") + print(f" Port: {db_config.get('port', 'No specified')}") + print(f" Database: {db_config.get('database', 'No specified')}") - # Para PostgreSQL, sugerir verificar el esquema + # For PostgreSQL, suggest verifying the schema if db_config.get('engine') == 'postgresql': - print("\nPara PostgreSQL, verifica que las tablas existan en el esquema 'public' o") - print("que tengas acceso a los esquemas donde están las tablas.") - print("Puedes verificar los esquemas disponibles con: SELECT DISTINCT table_schema FROM information_schema.tables;") + print("\nFor PostgreSQL, verify that the tables exist in the 'public' schema or") + print("that you have access to the schemas where the tables are.") + print("You can verify the available schemas with: SELECT DISTINCT table_schema FROM information_schema.tables;") return - # Mostrar información del esquema + # Show schema information tables = schema.get("tables", {}) - # Separar tablas SQL y colecciones NoSQL para mostrarlas apropiadamente + # Separate SQL tables and NoSQL collections to show them appropriately sql_tables = {} nosql_collections = {} @@ -479,76 +479,76 @@ def show_db_schema(api_token: str, config_id: Optional[str] = None, api_url: Opt elif "fields" in info: nosql_collections[name] = info - # Mostrar tablas SQL + # Show SQL tables if sql_tables: - _print_colored(f"\nSe encontraron {len(sql_tables)} tablas SQL:", "green") + _print_colored(f"\nFound {len(sql_tables)} SQL tables:", "green") for table_name, table_info in sql_tables.items(): - _print_colored(f"\n=== Tabla: {table_name} ===", "bold") + _print_colored(f"\n=== Table: {table_name} ===", "bold") - # Mostrar columnas + # Show columns columns = table_info.get("columns", []) if columns: - _print_colored("Columnas:", "blue") + _print_colored("Columns:", "blue") for column in columns: print(f" - {column['name']} ({column['type']})") else: - _print_colored("No se encontraron columnas.", "yellow") + _print_colored("No columns found.", "yellow") - # Mostrar muestra de datos si está disponible + # Show sample data if available sample_data = table_info.get("sample_data", []) if sample_data: - _print_colored("\nMuestra de datos:", "blue") + _print_colored("\nData sample:", "blue") for i, row in enumerate(sample_data[:2], 1): # Limitar a 2 filas para simplificar - print(f" Registro {i}: {row}") + print(f" Record {i}: {row}") if len(sample_data) > 2: - print(f" ... ({len(sample_data) - 2} registros más)") + print(f" ... ({len(sample_data) - 2} more records)") - # Mostrar colecciones NoSQL + # Show NoSQL collections if nosql_collections: - _print_colored(f"\nSe encontraron {len(nosql_collections)} colecciones NoSQL:", "green") + _print_colored(f"\nFound {len(nosql_collections)} NoSQL collections:", "green") for coll_name, coll_info in nosql_collections.items(): - _print_colored(f"\n=== Colección: {coll_name} ===", "bold") + _print_colored(f"\n=== Collection: {coll_name} ===", "bold") - # Mostrar campos + # Show fields fields = coll_info.get("fields", []) if fields: - _print_colored("Campos:", "blue") + _print_colored("Fields:", "blue") for field in fields: print(f" - {field['name']} ({field['type']})") else: - _print_colored("No se encontraron campos.", "yellow") + _print_colored("No fields found.", "yellow") - # Mostrar muestra de datos si está disponible + # Show sample data if available sample_data = coll_info.get("sample_data", []) if sample_data: - _print_colored("\nMuestra de datos:", "blue") - for i, doc in enumerate(sample_data[:2], 1): # Limitar a 2 documentos - # Simplificar la visualización para documentos grandes + _print_colored("\nData sample:", "blue") + for i, doc in enumerate(sample_data[:2], 1): # Limit to 2 documents + # Simplify the visualization for large documents if isinstance(doc, dict) and len(doc) > 5: simplified = {k: doc[k] for k in list(doc.keys())[:5]} - print(f" Documento {i}: {simplified} ... (y {len(doc) - 5} campos más)") + print(f" Document {i}: {simplified} ... (and {len(doc) - 5} more fields)") else: - print(f" Documento {i}: {doc}") + print(f" Document {i}: {doc}") if len(sample_data) > 2: - print(f" ... ({len(sample_data) - 2} documentos más)") + print(f" ... ({len(sample_data) - 2} more documents)") - _print_colored("\n✅ Esquema extraído correctamente!", "green") + _print_colored("\n✅ Schema extracted correctly!", "green") - # Preguntar si quiere guardar el esquema en un archivo - save_option = input("\n¿Deseas guardar el esquema en un archivo? (s/n): ").strip().lower() + # Ask if you want to save the schema in a file + save_option = input("\nDo you want to save the schema in a file? (s/n): ").strip().lower() if save_option == "s": - filename = input("Nombre del archivo (por defecto: db_schema.json): ").strip() or "db_schema.json" + filename = input("File name (default: db_schema.json): ").strip() or "db_schema.json" try: with open(filename, 'w') as f: json.dump(schema, f, indent=2, default=str) - _print_colored(f"\n✅ Esquema guardado en: {filename}", "green") + _print_colored(f"\n✅ Schema saved in: {filename}", "green") except Exception as e: - _print_colored(f"❌ Error al guardar el archivo: {str(e)}", "red") + _print_colored(f"❌ Error saving the file: {str(e)}", "red") except Exception as e: - _print_colored(f"❌ Error al mostrar el esquema: {str(e)}", "red") + _print_colored(f"❌ Error showing the schema: {str(e)}", "red") import traceback traceback.print_exc() @@ -572,33 +572,33 @@ def get_schema_with_dynamic_import(api_token: str, config_id: str, db_config: Di core_module = importlib.import_module('core.client') init_func = getattr(core_module, 'init') - # Creamos una instancia de Corebrain con la configuración seleccionada + # Create a Corebrain instance with the selected configuration cb = init_func( api_token=api_token, config_id=config_id, api_url=api_url, - skip_verification=True # Omitimos verificación para simplificar + skip_verification=True # Skip verification for simplicity ) - # El esquema se extrae automáticamente al inicializar + # The schema is extracted automatically when initializing schema = cb.db_schema - # Si no hay esquema, intentamos extraerlo explícitamente + # If there is no schema, try to extract it explicitly if not schema or not schema.get("tables"): - _print_colored("Intentando extraer esquema explícitamente...", "yellow") + _print_colored("Trying to extract schema explicitly...", "yellow") schema = cb._extract_db_schema() - # Cerramos la conexión + # Close the connection cb.close() return schema except ImportError: - # Si falla la importación dinámica, intentamos un enfoque alternativo - _print_colored("No se pudo importar el cliente. Usando método alternativo.", "yellow") + # If dynamic import fails, try an alternative approach + _print_colored("Could not import the client. Using alternative method.", "yellow") return extract_db_schema(db_config) except Exception as e: - _print_colored(f"Error al extraer esquema con cliente: {str(e)}", "red") - # Fallback a extracción directa + _print_colored(f"Error extracting schema with client: {str(e)}", "red") + # Fallback to direct extraction return extract_db_schema(db_config) diff --git a/corebrain/lib/sso/auth.py b/corebrain/lib/sso/auth.py index d065a20..0f3b568 100644 --- a/corebrain/lib/sso/auth.py +++ b/corebrain/lib/sso/auth.py @@ -7,8 +7,8 @@ def __init__(self, config=None): self.config = config or {} self.logger = logging.getLogger(__name__) - # Configuración por defecto - self.sso_url = self.config.get('GLOBODAIN_SSO_URL', 'http://localhost:3000/login') # URL del SSO + # Default configuration + self.sso_url = self.config.get('GLOBODAIN_SSO_URL', 'https://sso.globodain.com/login') # URL del SSO self.client_id = self.config.get('GLOBODAIN_CLIENT_ID', '') self.client_secret = self.config.get('GLOBODAIN_CLIENT_SECRET', '') self.redirect_uri = self.config.get('GLOBODAIN_REDIRECT_URI', '') diff --git a/corebrain/lib/sso/client.py b/corebrain/lib/sso/client.py index 0315085..94e2155 100644 --- a/corebrain/lib/sso/client.py +++ b/corebrain/lib/sso/client.py @@ -32,7 +32,7 @@ def __init__( self.client_secret = client_secret self.service_id = service_id self.redirect_uri = redirect_uri - self._token_cache = {} # Cache de tokens verificados + self._token_cache = {} # Cache of verified tokens def get_login_url(self, provider: str = None) -> str: @@ -63,17 +63,17 @@ def verify_token(self, token: str) -> Dict[str, Any]: Raises: Exception: If the token is not valid """ - # Verificar si ya tenemos información cacheada y válida del token + # Check if we have cached and valid token information now = datetime.now() if token in self._token_cache: cache_data = self._token_cache[token] if cache_data['expires_at'] > now: return cache_data['user_info'] else: - # Eliminar token expirado del caché + # Remove expired token from cache del self._token_cache[token] - # Verificar token con el servicio SSO + # Verify token with the SSO service headers = { "Authorization": f"Bearer {token}", "Content-Type": "application/json" @@ -88,7 +88,7 @@ def verify_token(self, token: str) -> Dict[str, Any]: if response.status_code != 200: raise Exception(f"Token inválido: {response.text}") - # Obtener información del usuario + # Get user information user_response = requests.get( f"{self.sso_url}/api/users/me", headers=headers @@ -99,7 +99,7 @@ def verify_token(self, token: str) -> Dict[str, Any]: user_info = user_response.json() - # Guardar en caché (15 minutos) + # Save in cache (15 minutes) self._token_cache[token] = { 'user_info': user_info, 'expires_at': now + timedelta(minutes=15) diff --git a/corebrain/network/__init__.py b/corebrain/network/__init__.py index aa079ff..2d7d1ba 100644 --- a/corebrain/network/__init__.py +++ b/corebrain/network/__init__.py @@ -12,7 +12,7 @@ APIAuthError ) -# Exportación explícita de componentes públicos +# Explicit export of public components __all__ = [ 'APIClient', 'APIError', diff --git a/corebrain/network/client.py b/corebrain/network/client.py index 1176fb1..cc9c481 100644 --- a/corebrain/network/client.py +++ b/corebrain/network/client.py @@ -40,7 +40,7 @@ class APIAuthError(APIError): class APIClient: """Optimized HTTP client for communication with the Corebrain API.""" - # Constantes para manejo de reintentos y errores + # Constants for retry handling and errors MAX_RETRIES = 3 RETRY_DELAY = 0.5 # segundos RETRY_STATUS_CODES = [408, 429, 500, 502, 503, 504] @@ -56,32 +56,32 @@ def __init__(self, base_url: str, default_timeout: int = 10, verify_ssl: Whether to verify the SSL certificate user_agent: Custom user agent """ - # Normalizar URL base para asegurar que termina con '/' + # Normalize base URL to ensure it ends with '/' self.base_url = base_url if base_url.endswith('/') else base_url + '/' self.default_timeout = default_timeout self.verify_ssl = verify_ssl - # Headers predeterminados + # Default headers self.default_headers = { 'User-Agent': user_agent or 'CorebrainSDK/1.0', 'Accept': 'application/json', 'Content-Type': 'application/json' } - # Crear sesión HTTP con límites y timeouts optimizados + # Create HTTP session with optimized limits and timeouts self.session = httpx.Client( timeout=httpx.Timeout(timeout=default_timeout), verify=verify_ssl, - http2=True, # Usar HTTP/2 si está disponible + http2=True, # Use HTTP/2 if available limits=httpx.Limits(max_connections=100, max_keepalive_connections=20) ) - # Estadísticas y métricas + # Statistics and metrics self.request_count = 0 self.error_count = 0 self.total_request_time = 0 - logger.debug(f"Cliente API inicializado con base_url={base_url}, timeout={default_timeout}s") + logger.debug(f"API Client initialized with base_url={base_url}, timeout={default_timeout}s") def __del__(self): """Ensure the session is closed when the client is deleted.""" @@ -92,9 +92,9 @@ def close(self): if hasattr(self, 'session') and self.session: try: self.session.close() - logger.debug("Sesión HTTP cerrada correctamente") + logger.debug("HTTP session closed correctly") except Exception as e: - logger.warning(f"Error al cerrar sesión HTTP: {e}") + logger.warning(f"Error closing HTTP session: {e}") def get_full_url(self, endpoint: str) -> str: """ @@ -106,7 +106,7 @@ def get_full_url(self, endpoint: str) -> str: Returns: Full URL """ - # Eliminar '/' inicial si existe para evitar rutas duplicadas + # Remove '/' if it exists at the beginning to avoid duplicate paths endpoint = endpoint.lstrip('/') return urljoin(self.base_url, endpoint) @@ -122,14 +122,14 @@ def prepare_headers(self, headers: Optional[Dict[str, str]] = None, Returns: Combined headers """ - # Comenzar con headers predeterminados + # Start with default headers final_headers = self.default_headers.copy() - # Añadir headers personalizados + # Add custom headers if headers: final_headers.update(headers) - # Añadir token de autenticación si se proporciona + # Add authentication token if provided if auth_token: final_headers['Authorization'] = f'Bearer {auth_token}' @@ -150,11 +150,11 @@ def handle_response(self, response: Response) -> Response: """ status_code = response.status_code - # Procesar errores según código de estado + # Process errors according to status code if 400 <= status_code < 500: error_detail = None - # Intentar extraer detalles del error del cuerpo JSON + # Try to extract error details from JSON body try: json_data = response.json() if isinstance(json_data, dict): @@ -164,37 +164,37 @@ def handle_response(self, response: Response) -> Response: json_data.get('error') ) except Exception: - # Si no podemos parsear JSON, usar el texto completo + # If we can't parse JSON, use the full text error_detail = response.text[:200] + ('...' if len(response.text) > 200 else '') - # Errores específicos según código + # Specific errors according to code if status_code == 401: - msg = "Error de autenticación: token inválido o expirado" + msg = "Authentication error: invalid or expired token" logger.error(f"{msg} - {error_detail or ''}") raise APIAuthError(msg, status_code, error_detail, response) elif status_code == 403: - msg = "Acceso prohibido: no tienes permisos suficientes" + msg = "Access denied: you don't have enough permissions" logger.error(f"{msg} - {error_detail or ''}") raise APIAuthError(msg, status_code, error_detail, response) elif status_code == 404: - msg = f"Recurso no encontrado: {response.url}" + msg = f"Resource not found: {response.url}" logger.error(msg) raise APIError(msg, status_code, error_detail, response) elif status_code == 429: - msg = "Demasiadas peticiones: límite de tasa excedido" + msg = "Too many requests: rate limit exceeded" logger.warning(msg) raise APIError(msg, status_code, error_detail, response) else: - msg = f"Error del cliente ({status_code}): {error_detail or 'sin detalles'}" + msg = f"Client error ({status_code}): {error_detail or 'no details'}" logger.error(msg) raise APIError(msg, status_code, error_detail, response) elif 500 <= status_code < 600: - msg = f"Error del servidor ({status_code}): el servidor API encontró un error" + msg = f"Server error ({status_code}): the API server found an error" logger.error(msg) raise APIError(msg, status_code, response.text[:200], response) @@ -233,27 +233,27 @@ def request(self, method: str, endpoint: str, *, url = self.get_full_url(endpoint) final_headers = self.prepare_headers(headers, auth_token) - # Configurar timeout + # Set timeout request_timeout = timeout or self.default_timeout - # Contador para reintentos + # Retry counter retries = 0 last_error = None - # Registrar inicio de la petición + # Register start of request start_time = time.time() self.request_count += 1 while retries <= (self.MAX_RETRIES if retry else 0): try: if retries > 0: - # Esperar antes de reintentar con backoff exponencial + # Wait before retrying with exponential backoff wait_time = self.RETRY_DELAY * (2 ** (retries - 1)) - logger.info(f"Reintentando petición ({retries}/{self.MAX_RETRIES}) a {url} después de {wait_time:.2f}s") + logger.info(f"Retrying request ({retries}/{self.MAX_RETRIES}) to {url} after {wait_time:.2f}s") time.sleep(wait_time) - # Realizar la petición - logger.debug(f"Enviando petición {method} a {url}") + # Make the request + logger.debug(f"Sending {method} request to {url}") response = self.session.request( method=method, url=url, @@ -264,16 +264,16 @@ def request(self, method: str, endpoint: str, *, timeout=request_timeout ) - # Verificar si debemos reintentar por código de estado + # Check if we should retry by status code if response.status_code in self.RETRY_STATUS_CODES and retry and retries < self.MAX_RETRIES: - logger.warning(f"Código de estado {response.status_code} recibido, reintentando") + logger.warning(f"Status code {response.status_code} received, retrying") retries += 1 continue - # Procesar la respuesta + # Process the response processed_response = self.handle_response(response) - # Registrar tiempo total + # Register total time elapsed = time.time() - start_time self.total_request_time += elapsed logger.debug(f"Petición completada en {elapsed:.3f}s con estado {response.status_code}") @@ -283,9 +283,9 @@ def request(self, method: str, endpoint: str, *, except (ConnectError, httpx.HTTPError) as e: last_error = e - # Decidir si reintentamos dependiendo del tipo de error + # Decide if we should retry depending on the error type if isinstance(e, (ReadTimeout, WriteTimeout, PoolTimeout, ConnectError)) and retry and retries < self.MAX_RETRIES: - logger.warning(f"Error de conexión: {str(e)}, reintentando {retries+1}/{self.MAX_RETRIES}") + logger.warning(f"Connection error: {str(e)}, retrying {retries+1}/{self.MAX_RETRIES}") retries += 1 continue @@ -294,28 +294,28 @@ def request(self, method: str, endpoint: str, *, elapsed = time.time() - start_time if isinstance(e, (ReadTimeout, WriteTimeout, PoolTimeout)): - logger.error(f"Timeout en petición a {url} después de {elapsed:.3f}s: {str(e)}") - raise APITimeoutError(f"La petición a {endpoint} excedió el tiempo máximo de {request_timeout}s", + logger.error(f"Timeout in request to {url} after {elapsed:.3f}s: {str(e)}") + raise APITimeoutError(f"Request to {endpoint} exceeded the maximum time of {request_timeout}s", response=getattr(e, 'response', None)) else: - logger.error(f"Error de conexión a {url} después de {elapsed:.3f}s: {str(e)}") - raise APIConnectionError(f"Error de conexión a {endpoint}: {str(e)}", + logger.error(f"Connection error to {url} after {elapsed:.3f}s: {str(e)}") + raise APIConnectionError(f"Connection error to {endpoint}: {str(e)}", response=getattr(e, 'response', None)) except Exception as e: - # Error inesperado + # Unexpected error self.error_count += 1 elapsed = time.time() - start_time - logger.error(f"Error inesperado en petición a {url} después de {elapsed:.3f}s: {str(e)}") - raise APIError(f"Error inesperado en petición a {endpoint}: {str(e)}") + logger.error(f"Unexpected error in request to {url} after {elapsed:.3f}s: {str(e)}") + raise APIError(f"Unexpected error in request to {endpoint}: {str(e)}") - # Si llegamos aquí es porque agotamos los reintentos + # If we get here, we have exhausted the retries if last_error: self.error_count += 1 - raise APIError(f"Petición a {endpoint} falló después de {retries} reintentos: {str(last_error)}") + raise APIError(f"Request to {endpoint} failed after {retries} retries: {str(last_error)}") - # Este punto nunca debería alcanzarse - raise APIError(f"Error inesperado en petición a {endpoint}") + # This point should never be reached + raise APIError(f"Unexpected error in request to {endpoint}") def get(self, endpoint: str, **kwargs) -> Response: """Makes a GET request.""" @@ -352,7 +352,7 @@ def get_json(self, endpoint: str, **kwargs) -> Any: try: return response.json() except Exception as e: - raise APIError(f"Error al parsear respuesta JSON: {str(e)}", response=response) + raise APIError(f"Error parsing JSON response: {str(e)}", response=response) def post_json(self, endpoint: str, **kwargs) -> Any: """ @@ -369,9 +369,9 @@ def post_json(self, endpoint: str, **kwargs) -> Any: try: return response.json() except Exception as e: - raise APIError(f"Error al parsear respuesta JSON: {str(e)}", response=response) + raise APIError(f"Error parsing JSON response: {str(e)}", response=response) - # Métodos de alto nivel para operaciones comunes en la API de Corebrain + # High-level methods for common operations in the Corebrain API def check_health(self, timeout: int = 5) -> bool: """ @@ -409,7 +409,7 @@ def verify_token(self, token: str, timeout: int = 5) -> Dict[str, Any]: except APIAuthError: raise except Exception as e: - raise APIAuthError(f"Error al verificar token: {str(e)}") + raise APIAuthError(f"Error verifying token: {str(e)}") def get_api_keys(self, token: str) -> List[Dict[str, Any]]: """ @@ -475,7 +475,7 @@ def exchange_sso_token(self, sso_token: str, user_data: Dict[str, Any]) -> Dict[ data = {"user_data": user_data} return self.post_json("api/auth/sso/token", headers=headers, json=data) - # Métodos para estadísticas y diagnóstico + # Methods for statistics and diagnostics def get_stats(self) -> Dict[str, Any]: """ diff --git a/corebrain/services/schema.py b/corebrain/services/schema.py index 4155fb1..917a788 100644 --- a/corebrain/services/schema.py +++ b/corebrain/services/schema.py @@ -1,6 +1,4 @@ -# Nuevo directorio: services/ -# Nuevo archivo: services/schema_service.py """ Services for managing database schemas. """ @@ -27,5 +25,3 @@ def get_schema(self, api_token: str, config_id: str) -> Optional[Dict[str, Any]] def optimize_schema(self, schema: Dict[str, Any], query: str = None) -> Dict[str, Any]: """Optimizes an existing schema.""" return self.schema_optimizer.optimize_schema(schema, query) - - # Otros métodos de servicio... \ No newline at end of file diff --git a/corebrain/utils/__init__.py b/corebrain/utils/__init__.py index 3c89186..df10d4c 100644 --- a/corebrain/utils/__init__.py +++ b/corebrain/utils/__init__.py @@ -14,7 +14,6 @@ ConfigEncrypter ) -# Configuración de logging logger = logging.getLogger('corebrain') def setup_logger(level=logging.INFO, @@ -28,32 +27,32 @@ def setup_logger(level=logging.INFO, file_path: Path to log file (optional) format_string: Custom log format """ - # Formato predeterminado + # Default format fmt = format_string or '%(asctime)s - %(name)s - %(levelname)s - %(message)s' formatter = logging.Formatter(fmt) - # Handler de consola + # Console handler console_handler = logging.StreamHandler() console_handler.setFormatter(formatter) - # Configurar logger principal + # Configure main logger logger.setLevel(level) logger.addHandler(console_handler) - # Handler de archivo si se proporciona ruta + # File handler if path is provided if file_path: file_handler = logging.FileHandler(file_path) file_handler.setFormatter(formatter) logger.addHandler(file_handler) - # Mensajes de diagnóstico - logger.debug(f"Logger configurado con nivel {logging.getLevelName(level)}") + # Diagnostic messages + logger.debug(f"Logger configured with level {logging.getLevelName(level)}") if file_path: - logger.debug(f"Logs escritos a {file_path}") + logger.debug(f"Logs written to {file_path}") return logger -# Exportación explícita de componentes públicos +# Explicit export of public components __all__ = [ 'serialize_to_json', 'JSONEncoder', diff --git a/corebrain/utils/encrypter.py b/corebrain/utils/encrypter.py index 286a705..6cfbd3d 100644 --- a/corebrain/utils/encrypter.py +++ b/corebrain/utils/encrypter.py @@ -27,16 +27,16 @@ def derive_key_from_password(password: Union[str, bytes], salt: Optional[bytes] if isinstance(password, str): password = password.encode() - # Generar sal si no se proporciona + # Generate salt if not provided if salt is None: salt = os.urandom(16) - # Derivar clave usando PBKDF2 + # Derive key using PBKDF2 kdf = PBKDF2HMAC( algorithm=hashes.SHA256(), length=32, salt=salt, - iterations=100000 # Mayor número de iteraciones = mayor seguridad + iterations=100000 # Higher number of iterations = higher security ) key = kdf.derive(password) @@ -89,7 +89,7 @@ def _init_cipher(self) -> None: """Initializes the encryption object, creating or loading the key as needed.""" key = None - # Si hay ruta de clave, intentar cargar o crear + # If there is a key path, try to load or create if self.key_path: try: if self.key_path.exists(): @@ -97,17 +97,17 @@ def _init_cipher(self) -> None: key = f.read().strip() logger.debug(f"Clave cargada desde {self.key_path}") else: - # Crear directorio padre si no existe + # Create parent directory if it doesn't exist self.key_path.parent.mkdir(parents=True, exist_ok=True) - # Generar nueva clave + # Generate new key key = Fernet.generate_key() - # Guardar clave + # Save key with open(self.key_path, 'wb') as f: f.write(key) - # Asegurar permisos restrictivos (solo el propietario puede leer) + # Ensure restrictive permissions (only the owner can read) try: os.chmod(self.key_path, 0o600) except Exception as e: @@ -116,13 +116,13 @@ def _init_cipher(self) -> None: logger.debug(f"Nueva clave generada y guardada en {self.key_path}") except Exception as e: logger.error(f"Error al gestionar clave en {self.key_path}: {e}") - # En caso de error, generar clave efímera + # If there is an error, generate a temporary key key = None - # Si no tenemos clave, generar una efímera + # If we don't have a key, generate a temporary key if not key: key = Fernet.generate_key() - logger.debug("Usando clave efímera generada") + logger.debug("Using generated temporary key") self.cipher = Fernet(key) @@ -142,7 +142,7 @@ def encrypt(self, data: Union[str, bytes]) -> bytes: try: return self.cipher.encrypt(data) except Exception as e: - logger.error(f"Error al cifrar datos: {e}") + logger.error(f"Error encrypting data: {e}") raise def decrypt(self, encrypted_data: Union[str, bytes]) -> bytes: From 04a0abab4f56c76785841b9134ffd587b12d9b68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Ayuso?= Date: Mon, 26 May 2025 11:44:54 +0200 Subject: [PATCH 49/81] Some comments traslated to English. Version changed to 0.2.0 --- corebrain/__init__.py | 3 --- corebrain/cli/commands.py | 2 +- pyproject.toml | 6 +++--- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/corebrain/__init__.py b/corebrain/__init__.py index 6819487..d1107b1 100644 --- a/corebrain/__init__.py +++ b/corebrain/__init__.py @@ -27,9 +27,6 @@ '__version__' ] -# Variable de versión -__version__ = "1.0.0" - def init(api_key: str, config_id: str, skip_verification: bool = False) -> Corebrain: """ Initialize the Corebrain SDK with the provided API key and configuration. diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index 7248db6..6cc3e05 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -32,7 +32,7 @@ def main_cli(argv: Optional[List[str]] = None) -> int: """ # Package version - __version__ = "0.1.0" + __version__ = "0.2.0" try: print_colored("Corebrain CLI started. Version ", __version__, "blue") diff --git a/pyproject.toml b/pyproject.toml index 76f919d..59511c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "corebrain" -version = "0.1.0" -description = "SDK de Corebrain para consultas en lenguaje natural a bases de datos" +version = "0.2.0" +description = "Corebrain SDK for querys to DB in natural language" readme = "README.md" authors = [ {name = "Rubén Ayuso", email = "ruben@globodain.com"} @@ -27,7 +27,7 @@ dependencies = [ "typing-extensions>=4.4.0", "requests>=2.28.0", "asyncio>=3.4.3", - "psycopg2-binary>=2.9.0", # En lugar de psycopg2 para evitar problemas de compilación + "psycopg2-binary>=2.9.0", # Instead of psycopg2 to avoid compilation errors "mysql-connector-python>=8.0.23", "pymongo>=4.4.0", ] From 3a8290111b8b1524965902353f6b5469316d6452 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Ayuso?= Date: Mon, 26 May 2025 11:46:49 +0200 Subject: [PATCH 50/81] --output-file and --config-id commands will be moved into --list-configs. --- corebrain/cli/commands.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index 6cc3e05..b8f28be 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -47,8 +47,6 @@ def main_cli(argv: Optional[List[str]] = None) -> int: parser.add_argument("--create-user", action="store_true", help="Create an user and API Key by default") parser.add_argument("--configure", action="store_true", help="Configure the Corebrain SDK") parser.add_argument("--list-configs", action="store_true", help="List available configurations") - parser.add_argument("--output-file", help="File to save the extracted schema") - parser.add_argument("--config-id", help="Specific configuration ID to use") parser.add_argument("--token", help="Corebrain API token (any type)") parser.add_argument("--api-key", help="Specific API Key for Corebrain") parser.add_argument("--api-url", help="Corebrain API URL") From af77c1600b2523d2cc0ab72a7fd4eeca86c9a7ad Mon Sep 17 00:00:00 2001 From: Pawel Wasilewski Date: Mon, 26 May 2025 12:06:08 +0200 Subject: [PATCH 51/81] minor fixes in _process_codument_for_serialization --- corebrain/cli/common.py | 2 +- corebrain/corebrain/db/__init__.py | 2 +- corebrain/db/__init__.py | 2 +- corebrain/db/connectors/{mongodb.py => mongodb_LEGACY.py} | 0 corebrain/db/factory.py | 2 +- corebrain/examples/complex.py | 4 ++-- 6 files changed, 6 insertions(+), 6 deletions(-) rename corebrain/db/connectors/{mongodb.py => mongodb_LEGACY.py} (100%) diff --git a/corebrain/cli/common.py b/corebrain/cli/common.py index d083f32..7799dcf 100644 --- a/corebrain/cli/common.py +++ b/corebrain/cli/common.py @@ -2,7 +2,7 @@ Default values for SSO and API connection """ -DEFAULT_API_URL = "http://localhost:8000" +DEFAULT_API_URL = "http://localhost:5000" #DEFAULT_SSO_URL = "http://localhost:3000" # localhost DEFAULT_SSO_URL = "https://sso.globodain.com" # remote DEFAULT_PORT = 8765 diff --git a/corebrain/corebrain/db/__init__.py b/corebrain/corebrain/db/__init__.py index 6ac390d..743c519 100644 --- a/corebrain/corebrain/db/__init__.py +++ b/corebrain/corebrain/db/__init__.py @@ -8,7 +8,7 @@ from corebrain.db.factory import get_connector from corebrain.db.engines import get_available_engines from corebrain.db.connectors.sql import SQLConnector -from corebrain.db.connectors.mongodb import MongoDBConnector +from corebrain.db.connectors.mongodb_LEGACY import MongoDBConnector from corebrain.db.schema_file import get_schema_with_dynamic_import from corebrain.db.schema.optimizer import SchemaOptimizer from corebrain.db.schema.extractor import extract_db_schema diff --git a/corebrain/db/__init__.py b/corebrain/db/__init__.py index 6ac390d..743c519 100644 --- a/corebrain/db/__init__.py +++ b/corebrain/db/__init__.py @@ -8,7 +8,7 @@ from corebrain.db.factory import get_connector from corebrain.db.engines import get_available_engines from corebrain.db.connectors.sql import SQLConnector -from corebrain.db.connectors.mongodb import MongoDBConnector +from corebrain.db.connectors.mongodb_LEGACY import MongoDBConnector from corebrain.db.schema_file import get_schema_with_dynamic_import from corebrain.db.schema.optimizer import SchemaOptimizer from corebrain.db.schema.extractor import extract_db_schema diff --git a/corebrain/db/connectors/mongodb.py b/corebrain/db/connectors/mongodb_LEGACY.py similarity index 100% rename from corebrain/db/connectors/mongodb.py rename to corebrain/db/connectors/mongodb_LEGACY.py diff --git a/corebrain/db/factory.py b/corebrain/db/factory.py index cc28296..125c0fe 100644 --- a/corebrain/db/factory.py +++ b/corebrain/db/factory.py @@ -5,7 +5,7 @@ from corebrain.db.connector import DatabaseConnector from corebrain.db.connectors.sql import SQLConnector -from corebrain.db.connectors.mongodb import MongoDBConnector +from corebrain.db.connectors.mongodb_LEGACY import MongoDBConnector def get_connector(db_config: Dict[str, Any], timeout: int = 10) -> DatabaseConnector: """ diff --git a/corebrain/examples/complex.py b/corebrain/examples/complex.py index e66c21b..41ccc4e 100644 --- a/corebrain/examples/complex.py +++ b/corebrain/examples/complex.py @@ -1,8 +1,8 @@ from corebrain import init -api_key = "sk_bH8rnkIHCDF1BlRmgS9s6QAK" +api_key = "sk_NPNLbEAjxQm86u6OX97An5ev" #config_id = "c9913a04-a530-4ae3-a877-8e295be87f78" # MONGODB -config_id = "8bdba894-34a7-4453-b665-e640d11fd463" # POSTGRES +config_id = "be43981c-0015-4ba4-9861-f12e82f6805e" # POSTGRES # Initialize the SDK with API key and configuration ID corebrain = init( From b6311ea465024ec45eba2d1eaa2081cb2643ed04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Ayuso?= Date: Mon, 26 May 2025 12:14:45 +0200 Subject: [PATCH 52/81] --check-status command was modified to check every requirements in the system to work with SDK is ok --- corebrain/cli/commands.py | 256 ++++++++++++++++++++++++++++++++------ 1 file changed, 215 insertions(+), 41 deletions(-) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index b8f28be..048676b 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -47,6 +47,7 @@ def main_cli(argv: Optional[List[str]] = None) -> int: parser.add_argument("--create-user", action="store_true", help="Create an user and API Key by default") parser.add_argument("--configure", action="store_true", help="Configure the Corebrain SDK") parser.add_argument("--list-configs", action="store_true", help="List available configurations") + parser.add_argument("--token", help="Corebrain API token (any type)") parser.add_argument("--api-key", help="Specific API Key for Corebrain") parser.add_argument("--api-url", help="Corebrain API URL") @@ -86,6 +87,9 @@ def authentication(): # Show version if args.version: + """ + Show the library version. + """ try: from importlib.metadata import version sdk_version = version("corebrain") @@ -94,6 +98,216 @@ def authentication(): print(f"Corebrain SDK version {__version__}") return 0 + if args.check_status: + """ + If you're in development mode: + + Check that all requirements for developing code and performing tests or other functions are accessible: + - Check that the API Server is runned + - Check that the Redis is runned on port 6379 + - Check that the SSO Server is active (sso.globodain.com) + - Check that MongoDB is runned on port 27017 + - Check that the all libraries are installed: + + httpx>=0.23.0 + pymongo>=4.3.0 + psycopg2-binary>=2.9.5 + mysql-connector-python>=8.0.31 + sqlalchemy>=2.0.0 + cryptography>=39.0.0 + pydantic>=1.10.0 + + + If you're in production mode: + + Check that the API Server is active (api.etedata.com) + Check that the SSO Server is active (sso.globodain.com) + Check that the all libraries are installed: + + httpx>=0.23.0 + pymongo>=4.3.0 + psycopg2-binary>=2.9.5 + mysql-connector-python>=8.0.31 + sqlalchemy>=2.0.0 + cryptography>=39.0.0 + pydantic>=1.10.0 + + """ + + import socket + import subprocess + import importlib.util + + def check_port(host, port, service_name): + """Check if a service is running on a specific port""" + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + result = sock.connect_ex((host, port)) + sock.close() + if result == 0: + print_colored(f"✅ {service_name} is running on {host}:{port}", "green") + return True + else: + print_colored(f"❌ {service_name} is not accessible on {host}:{port}", "red") + return False + except Exception as e: + print_colored(f"❌ Error checking {service_name}: {str(e)}", "red") + return False + + def check_url(url, service_name): + """Check if a service is accessible via HTTP""" + try: + response = requests.get(url, timeout=10) + if response.status_code < 500: + print_colored(f"✅ {service_name} is accessible at {url}", "green") + return True + else: + print_colored(f"❌ {service_name} returned status {response.status_code} at {url}", "red") + return False + except Exception as e: + print_colored(f"❌ {service_name} is not accessible at {url}: {str(e)}", "red") + return False + + def check_library(library_name, min_version): + """Check if a library is installed with minimum version""" + # Mapping of PyPI package names to import names + package_import_mapping = { + 'psycopg2-binary': 'psycopg2', + 'mysql-connector-python': 'mysql.connector', + 'httpx': 'httpx', + 'pymongo': 'pymongo', + 'sqlalchemy': 'sqlalchemy', + 'cryptography': 'cryptography', + 'pydantic': 'pydantic' + } + + package_name = library_name.split('>=')[0] + import_name = package_import_mapping.get(package_name, package_name) + + try: + # Check if the module can be imported + if '.' in import_name: + # For modules like mysql.connector + parts = import_name.split('.') + spec = importlib.util.find_spec(parts[0]) + if spec is None: + print_colored(f"❌ {package_name} is not installed", "red") + return False + # Try to import the full module path + try: + __import__(import_name) + except ImportError: + print_colored(f"❌ {package_name} is not installed", "red") + return False + else: + spec = importlib.util.find_spec(import_name) + if spec is None: + print_colored(f"❌ {package_name} is not installed", "red") + return False + + # Try to get version using different methods + try: + from importlib.metadata import version + # Try with the package name first + try: + installed_version = version(package_name) + except: + # If that fails, try with common alternative names + alternative_names = { + 'psycopg2-binary': ['psycopg2', 'psycopg2-binary'], + 'mysql-connector-python': ['mysql-connector-python', 'mysql-connector'] + } + + installed_version = None + for alt_name in alternative_names.get(package_name, [package_name]): + try: + installed_version = version(alt_name) + break + except: + continue + + if installed_version is None: + raise Exception("Version not found") + + print_colored(f"✅ {package_name} {installed_version} is installed", "green") + return True + + except Exception: + # If version check fails, at least we know the module can be imported + print_colored(f"✅ {package_name} is installed (version check failed)", "yellow") + return True + + except Exception as e: + print_colored(f"❌ Error checking {package_name}: {str(e)}", "red") + return False + + # Determine if in development or production mode + api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + is_development = "localhost" in api_url or "127.0.0.1" in api_url or api_url == DEFAULT_API_URL + + print_colored("🔍 Checking system status...", "blue") + print_colored(f"Mode: {'Development' if is_development else 'Production'}", "blue") + print_colored(f"API URL: {api_url}", "blue") + print() + + all_checks_passed = True + + # Required libraries for both modes + required_libraries = [ + "httpx>=0.23.0", + "pymongo>=4.3.0", + "psycopg2-binary>=2.9.5", + "mysql-connector-python>=8.0.31", + "sqlalchemy>=2.0.0", + "cryptography>=39.0.0", + "pydantic>=1.10.0" + ] + + # Check libraries + print_colored("📚 Checking required libraries:", "blue") + for library in required_libraries: + if not check_library(library, library.split('>=')[1] if '>=' in library else None): + all_checks_passed = False + print() + + # Check services based on mode + if is_development: + print_colored("🔧 Development mode - Checking local services:", "blue") + + # Check local API server + if not check_url(api_url, "API Server"): + all_checks_passed = False + + # Check Redis + if not check_port("localhost", 6379, "Redis"): + all_checks_passed = False + + # Check MongoDB + if not check_port("localhost", 27017, "MongoDB"): + all_checks_passed = False + + else: + print_colored("🌐 Production mode - Checking remote services:", "blue") + + # Check production API server + if not check_url("https://api.etedata.com", "API Server (Production)"): + all_checks_passed = False + + # Check SSO service for both modes + sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + if not check_url(sso_url, "SSO Server"): + all_checks_passed = False + + print() + if all_checks_passed: + print_colored("✅ All system checks passed!", "green") + return 0 + else: + print_colored("❌ Some system checks failed. Please review the issues above.", "red") + return 1 + + if args.export_config: export_config(args.export_config) # --> config/manager.py --> export_config @@ -235,47 +449,7 @@ def authentication(): print_colored("You can create an API Key in the Corebrain dashboard.", "yellow") return 1 - if args.check_status: - if not args.task_id: - print_colored("❌ Please provide a task ID using --task-id", "red") - return 1 - - # Get URLs - api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL - sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL - - # Prioritize api_key if explicitly provided - token_arg = args.api_key if args.api_key else args.token - - # Get API credentials - api_key, user_data, api_token = get_api_credential(token_arg, sso_url) - - if not api_key: - print_colored("❌ API Key is required to check task status. Use --api-key or login via --login", "red") - return 1 - - try: - task_id = args.task_id - headers = { - "Authorization": f"Bearer {api_key}", - "Accept": "application/json" - } - url = f"{api_url}/tasks/{task_id}/status" - response = requests.get(url, headers=headers) - - if response.status_code == 404: - print_colored(f"❌ Task with ID '{task_id}' not found.", "red") - return 1 - - response.raise_for_status() - data = response.json() - status = data.get("status", "unknown") - - print_colored(f"✅ Task '{task_id}' status: {status}", "green") - return 0 - except Exception as e: - print_colored(f"❌ Failed to check status: {str(e)}", "red") - return 1 + if args.woami: try: From 6eced9fd146097325c08327ddb8d530d3a65d51b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Ayuso?= Date: Mon, 26 May 2025 12:17:42 +0200 Subject: [PATCH 53/81] Manager.py in Config was translated to English --- corebrain/config/manager.py | 38 ++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/corebrain/config/manager.py b/corebrain/config/manager.py index 0fa9f65..6682e88 100644 --- a/corebrain/config/manager.py +++ b/corebrain/config/manager.py @@ -86,11 +86,11 @@ def _ensure_config_dir(self) -> None: """Ensures that the configuration directory exists.""" try: self.CONFIG_DIR.mkdir(parents=True, exist_ok=True) - logger.debug(f"Directorio de configuración asegurado: {self.CONFIG_DIR}") - _print_colored(f"Directorio de configuración asegurado: {self.CONFIG_DIR}", "blue") + logger.debug(f"Configuration directory ensured: {self.CONFIG_DIR}") + _print_colored(f"Configuration directory ensured: {self.CONFIG_DIR}", "blue") except Exception as e: - logger.error(f"Error al crear directorio de configuración: {str(e)}") - _print_colored(f"Error al crear directorio de configuración: {str(e)}", "red") + logger.error(f"Error creating configuration directory: {str(e)}") + _print_colored(f"Error creating configuration directory: {str(e)}", "red") def _load_secret_key(self) -> None: """Loads or generates the secret key to encrypt sensitive data.""" @@ -99,14 +99,14 @@ def _load_secret_key(self) -> None: key = Fernet.generate_key() with open(self.SECRET_KEY_FILE, 'wb') as key_file: key_file.write(key) - _print_colored(f"Nueva clave secreta generada en: {self.SECRET_KEY_FILE}", "green") + _print_colored(f"New secret key generated in: {self.SECRET_KEY_FILE}", "green") with open(self.SECRET_KEY_FILE, 'rb') as key_file: self.secret_key = key_file.read() self.cipher = Fernet(self.secret_key) except Exception as e: - _print_colored(f"Error al cargar/generar clave secreta: {str(e)}", "red") + _print_colored(f"Error loading/generating secret key: {str(e)}", "red") # Fallback a una clave temporal (menos segura pero funcional) self.secret_key = Fernet.generate_key() self.cipher = Fernet(self.secret_key) @@ -114,7 +114,7 @@ def _load_secret_key(self) -> None: def _load_configs(self) -> Dict[str, Dict[str, Any]]: """Loads the saved configurations.""" if not self.CONFIG_FILE.exists(): - _print_colored(f"Archivo de configuración no encontrado: {self.CONFIG_FILE}", "yellow") + _print_colored(f"Configuration file not found: {self.CONFIG_FILE}", "yellow") return {} try: @@ -122,7 +122,7 @@ def _load_configs(self) -> Dict[str, Dict[str, Any]]: encrypted_data = f.read() if not encrypted_data: - _print_colored("Archivo de configuración vacío", "yellow") + _print_colored("Configuration file is empty", "yellow") return {} try: @@ -131,17 +131,17 @@ def _load_configs(self) -> Dict[str, Dict[str, Any]]: configs = json.loads(decrypted_data) except Exception as e: # Si falla el descifrado, intentar cargar como JSON plano - logger.warning(f"Error al descifrar configuración: {e}") + logger.warning(f"Error decrypting configuration: {e}") configs = json.loads(encrypted_data) if isinstance(configs, str): configs = json.loads(configs) - _print_colored(f"Configuración cargada", "green") + _print_colored(f"Configuration loaded", "green") self.configs = configs return configs except Exception as e: - _print_colored(f"Error al cargar configuraciones: {str(e)}", "red") + _print_colored(f"Error loading configurations: {str(e)}", "red") return {} def _save_configs(self) -> None: @@ -153,9 +153,9 @@ def _save_configs(self) -> None: with open(self.CONFIG_FILE, 'w') as f: f.write(encrypted_data) - _print_colored(f"Configuraciones guardadas en: {self.CONFIG_FILE}", "green") + _print_colored(f"Configurations saved in: {self.CONFIG_FILE}", "green") except Exception as e: - _print_colored(f"Error al guardar configuraciones: {str(e)}", "red") + _print_colored(f"Error saving configurations: {str(e)}", "red") def add_config(self, api_key: str, db_config: Dict[str, Any], config_id: Optional[str] = None) -> str: """ @@ -173,15 +173,15 @@ def add_config(self, api_key: str, db_config: Dict[str, Any], config_id: Optiona config_id = str(uuid.uuid4()) db_config["config_id"] = config_id - # Crear o actualizar la entrada para este token + # Create or update the entry for this token if api_key not in self.configs: self.configs[api_key] = {} - # Añadir la configuración + # Add the configuration self.configs[api_key][config_id] = db_config self._save_configs() - _print_colored(f"Configuración agregada: {config_id} para la API Key: {api_key[:8]}...", "green") + _print_colored(f"Configuration added: {config_id} for API Key: {api_key[:8]}...", "green") return config_id def get_config(self, api_key_selected: str, config_id: str) -> Optional[Dict[str, Any]]: @@ -223,13 +223,13 @@ def remove_config(self, api_key_selected: str, config_id: str) -> bool: if api_key_selected in self.configs and config_id in self.configs[api_key_selected]: del self.configs[api_key_selected][config_id] - # Si no quedan configuraciones para este token, eliminar la entrada + # If there are no configurations for this token, delete the entry if not self.configs[api_key_selected]: del self.configs[api_key_selected] self._save_configs() - _print_colored(f"Configuración {config_id} eliminada para API Key: {api_key_selected[:8]}...", "green") + _print_colored(f"Configuration {config_id} removed for API Key: {api_key_selected[:8]}...", "green") return True - _print_colored(f"Configuración {config_id} no encontrada para API Key: {api_key_selected[:8]}...", "yellow") + _print_colored(f"Configuration {config_id} not found for API Key: {api_key_selected[:8]}...", "yellow") return False \ No newline at end of file From d77c62d32d84dddb82ce617cb3961ee49a5d2e5f Mon Sep 17 00:00:00 2001 From: palstr Date: Mon, 26 May 2025 12:21:00 +0200 Subject: [PATCH 54/81] Replaced MongoDBConnector with NoSQLConnector --- corebrain/corebrain/db/__init__.py | 4 ++-- corebrain/corebrain/db/connectors/mongodb.py | 2 +- corebrain/corebrain/db/interface.py | 2 +- corebrain/db/__init__.py | 4 ++-- corebrain/db/connectors/mongodb.py | 2 +- corebrain/db/factory.py | 4 ++-- corebrain/db/interface.py | 2 +- pyproject.toml | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/corebrain/corebrain/db/__init__.py b/corebrain/corebrain/db/__init__.py index 6ac390d..dd53f0e 100644 --- a/corebrain/corebrain/db/__init__.py +++ b/corebrain/corebrain/db/__init__.py @@ -8,7 +8,7 @@ from corebrain.db.factory import get_connector from corebrain.db.engines import get_available_engines from corebrain.db.connectors.sql import SQLConnector -from corebrain.db.connectors.mongodb import MongoDBConnector +from corebrain.db.connectors.nosql import NoSQLConnector from corebrain.db.schema_file import get_schema_with_dynamic_import from corebrain.db.schema.optimizer import SchemaOptimizer from corebrain.db.schema.extractor import extract_db_schema @@ -19,7 +19,7 @@ 'get_connector', 'get_available_engines', 'SQLConnector', - 'MongoDBConnector', + 'NoSQLConnector', 'SchemaOptimizer', 'extract_db_schema', 'get_schema_with_dynamic_import' diff --git a/corebrain/corebrain/db/connectors/mongodb.py b/corebrain/corebrain/db/connectors/mongodb.py index 1b98575..925d410 100644 --- a/corebrain/corebrain/db/connectors/mongodb.py +++ b/corebrain/corebrain/db/connectors/mongodb.py @@ -17,7 +17,7 @@ from corebrain.db.connector import DatabaseConnector -class MongoDBConnector(DatabaseConnector): +class NoSQLConenctor(DatabaseConnector): """Optimized connector for MongoDB.""" def __init__(self, config: Dict[str, Any]): diff --git a/corebrain/corebrain/db/interface.py b/corebrain/corebrain/db/interface.py index d8373ff..25fd433 100644 --- a/corebrain/corebrain/db/interface.py +++ b/corebrain/corebrain/db/interface.py @@ -33,4 +33,4 @@ def close(self, connection: Any) -> None: # - SQLiteConnector # - MySQLConnector # - PostgresConnector -# - MongoDBConnector \ No newline at end of file +# - NoSQLConnector \ No newline at end of file diff --git a/corebrain/db/__init__.py b/corebrain/db/__init__.py index 6ac390d..dd53f0e 100644 --- a/corebrain/db/__init__.py +++ b/corebrain/db/__init__.py @@ -8,7 +8,7 @@ from corebrain.db.factory import get_connector from corebrain.db.engines import get_available_engines from corebrain.db.connectors.sql import SQLConnector -from corebrain.db.connectors.mongodb import MongoDBConnector +from corebrain.db.connectors.nosql import NoSQLConnector from corebrain.db.schema_file import get_schema_with_dynamic_import from corebrain.db.schema.optimizer import SchemaOptimizer from corebrain.db.schema.extractor import extract_db_schema @@ -19,7 +19,7 @@ 'get_connector', 'get_available_engines', 'SQLConnector', - 'MongoDBConnector', + 'NoSQLConnector', 'SchemaOptimizer', 'extract_db_schema', 'get_schema_with_dynamic_import' diff --git a/corebrain/db/connectors/mongodb.py b/corebrain/db/connectors/mongodb.py index 1b98575..bff3da7 100644 --- a/corebrain/db/connectors/mongodb.py +++ b/corebrain/db/connectors/mongodb.py @@ -17,7 +17,7 @@ from corebrain.db.connector import DatabaseConnector -class MongoDBConnector(DatabaseConnector): +class NoSQLConnector(DatabaseConnector): """Optimized connector for MongoDB.""" def __init__(self, config: Dict[str, Any]): diff --git a/corebrain/db/factory.py b/corebrain/db/factory.py index cc28296..851704f 100644 --- a/corebrain/db/factory.py +++ b/corebrain/db/factory.py @@ -5,7 +5,7 @@ from corebrain.db.connector import DatabaseConnector from corebrain.db.connectors.sql import SQLConnector -from corebrain.db.connectors.mongodb import MongoDBConnector +from corebrain.db.connectors.nosql import NoSQLConnector def get_connector(db_config: Dict[str, Any], timeout: int = 10) -> DatabaseConnector: """ @@ -25,7 +25,7 @@ def get_connector(db_config: Dict[str, Any], timeout: int = 10) -> DatabaseConne return SQLConnector(db_config, timeout) elif db_type == "nosql": if engine == "mongodb": - return MongoDBConnector(db_config, timeout) + return NoSQLConnector(db_config, timeout) else: raise ValueError(f"Unsupported NoSQL engine: {engine}") else: diff --git a/corebrain/db/interface.py b/corebrain/db/interface.py index d8373ff..25fd433 100644 --- a/corebrain/db/interface.py +++ b/corebrain/db/interface.py @@ -33,4 +33,4 @@ def close(self, connection: Any) -> None: # - SQLiteConnector # - MySQLConnector # - PostgresConnector -# - MongoDBConnector \ No newline at end of file +# - NoSQLConnector \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 76f919d..07ea4f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ [project.optional-dependencies] postgres = ["psycopg2-binary>=2.9.0"] -mongodb = ["pymongo>=4.4.0"] + = ["pymongo>=4.4.0"] mysql = ["mysql-connector-python>=8.0.23"] all_db = [ "psycopg2-binary>=2.9.0", From 6baf62bc82246a6db2396402a646f07857301a0d Mon Sep 17 00:00:00 2001 From: Pawel Wasilewski Date: Mon, 26 May 2025 12:21:32 +0200 Subject: [PATCH 55/81] minor fixes --- corebrain/db/connectors/mongodb_LEGACY.py | 474 ------------------ .../connectors/subconnectors/nosql/mongodb.py | 2 +- 2 files changed, 1 insertion(+), 475 deletions(-) delete mode 100644 corebrain/db/connectors/mongodb_LEGACY.py diff --git a/corebrain/db/connectors/mongodb_LEGACY.py b/corebrain/db/connectors/mongodb_LEGACY.py deleted file mode 100644 index 1b98575..0000000 --- a/corebrain/db/connectors/mongodb_LEGACY.py +++ /dev/null @@ -1,474 +0,0 @@ -""" -Connector for MongoDB databases. -""" - -import time -import json -import re - -from typing import Dict, Any, List, Optional, Callable, Tuple - -try: - import pymongo - from pymongo.errors import ConnectionFailure, ServerSelectionTimeoutError - PYMONGO_AVAILABLE = True -except ImportError: - PYMONGO_AVAILABLE = False - -from corebrain.db.connector import DatabaseConnector - -class MongoDBConnector(DatabaseConnector): - """Optimized connector for MongoDB.""" - - def __init__(self, config: Dict[str, Any]): - """ - Initializes the MongoDB connector with the provided configuration. - - Args: - config: Dictionary with the connection configuration - """ - super().__init__(config) - self.client = None - self.db = None - self.config = config - self.connection_timeout = 30 # segundos - - if not PYMONGO_AVAILABLE: - print("Advertencia: pymongo no está instalado. Instálalo con 'pip install pymongo'") - - def connect(self) -> bool: - """ - Establishes a connection with optimized timeout. - - Returns: - True if the connection was successful, False otherwise - """ - if not PYMONGO_AVAILABLE: - raise ImportError("pymongo no está instalado. Instálalo con 'pip install pymongo'") - - try: - start_time = time.time() - - # Construir los parámetros de conexión - if "connection_string" in self.config: - connection_string = self.config["connection_string"] - # Añadir timeout a la cadena de conexión si no está presente - if "connectTimeoutMS=" not in connection_string: - if "?" in connection_string: - connection_string += "&connectTimeoutMS=10000" # 10 segundos - else: - connection_string += "?connectTimeoutMS=10000" - - # Crear cliente MongoDB con la cadena de conexión - self.client = pymongo.MongoClient(connection_string) - else: - # Diccionario de parámetros para MongoClient - mongo_params = { - "host": self.config.get("host", "localhost"), - "port": int(self.config.get("port", 27017)), - "connectTimeoutMS": 10000, # 10 segundos - "serverSelectionTimeoutMS": 10000 - } - - # Añadir credenciales solo si están presentes - if self.config.get("user"): - mongo_params["username"] = self.config.get("user") - if self.config.get("password"): - mongo_params["password"] = self.config.get("password") - - # Opcionalmente añadir opciones de autenticación - if self.config.get("auth_source"): - mongo_params["authSource"] = self.config.get("auth_source") - if self.config.get("auth_mechanism"): - mongo_params["authMechanism"] = self.config.get("auth_mechanism") - - # Crear cliente MongoDB con parámetros - self.client = pymongo.MongoClient(**mongo_params) - - # Verificar que la conexión funciona - self.client.admin.command('ping') - - # Seleccionar la base de datos - db_name = self.config.get("database", "") - if not db_name: - # Si no hay base de datos especificada, listar las disponibles - db_names = self.client.list_database_names() - if not db_names: - raise ValueError("No se encontraron bases de datos disponibles") - - # Seleccionar la primera que no sea de sistema - system_dbs = ["admin", "local", "config"] - for name in db_names: - if name not in system_dbs: - db_name = name - break - - # Si no encontramos ninguna que no sea de sistema, usar la primera - if not db_name: - db_name = db_names[0] - - print(f"No se especificó base de datos. Usando '{db_name}'") - - # Guardar la referencia a la base de datos - self.db = self.client[db_name] - return True - - except (ConnectionFailure, ServerSelectionTimeoutError) as e: - # Si es un error de timeout, reintentar - if time.time() - start_time < self.connection_timeout: - print(f"Timeout al conectar a MongoDB: {str(e)}. Reintentando...") - time.sleep(2) # Esperar antes de reintentar - return self.connect() - else: - print(f"Error de conexión a MongoDB después de {self.connection_timeout}s: {str(e)}") - self.close() - return False - except Exception as e: - print(f"Error al conectar a MongoDB: {str(e)}") - self.close() - return False - - def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] = None, - progress_callback: Optional[Callable] = None) -> Dict[str, Any]: - """ - Extracts the schema with limits and progress to improve performance. - - Args: - sample_limit: Maximum number of sample documents per collection - collection_limit: Limit of collections to process (None for all) - progress_callback: Optional function to report progress - - Returns: - Dictionary with the database schema - """ - # Asegurar que estamos conectados - if not self.client and not self.connect(): - return {"type": "mongodb", "tables": {}, "tables_list": []} - - # Inicializar el esquema - schema = { - "type": "mongodb", - "database": self.db.name, - "tables": {} # En MongoDB, las "tablas" son colecciones - } - - try: - # Obtener la lista de colecciones - collections = self.db.list_collection_names() - - # Limitar colecciones si es necesario - if collection_limit is not None and collection_limit > 0: - collections = collections[:collection_limit] - - # Procesar cada colección - total_collections = len(collections) - for i, collection_name in enumerate(collections): - # Reportar progreso si hay callback - if progress_callback: - progress_callback(i, total_collections, f"Procesando colección {collection_name}") - - collection = self.db[collection_name] - - try: - # Contar documentos - doc_count = collection.count_documents({}) - - if doc_count > 0: - # Obtener muestra de documentos - sample_docs = list(collection.find().limit(sample_limit)) - - # Extraer campos y sus tipos - fields = {} - for doc in sample_docs: - self._extract_document_fields(doc, fields) - - # Convertir a formato esperado - formatted_fields = [{"name": field, "type": type_name} for field, type_name in fields.items()] - - # Procesar documentos para sample_data - sample_data = [] - for doc in sample_docs: - processed_doc = self._process_document_for_serialization(doc) - sample_data.append(processed_doc) - - # Guardar en el esquema - schema["tables"][collection_name] = { - "fields": formatted_fields, - "sample_data": sample_data, - "count": doc_count - } - else: - # Colección vacía - schema["tables"][collection_name] = { - "fields": [], - "sample_data": [], - "count": 0, - "empty": True - } - - except Exception as e: - print(f"Error al procesar colección {collection_name}: {str(e)}") - schema["tables"][collection_name] = { - "fields": [], - "error": str(e) - } - - # Crear la lista de tablas/colecciones para compatibilidad - table_list = [] - for collection_name, collection_info in schema["tables"].items(): - table_data = {"name": collection_name} - table_data.update(collection_info) - table_list.append(table_data) - - # Guardar también la lista de tablas para compatibilidad - schema["tables_list"] = table_list - - return schema - - except Exception as e: - print(f"Error al extraer el esquema MongoDB: {str(e)}") - return {"type": "mongodb", "tables": {}, "tables_list": []} - - def _extract_document_fields(self, doc: Dict[str, Any], fields: Dict[str, str], - prefix: str = "", max_depth: int = 3, current_depth: int = 0) -> None: - """ - Recursively extracts fields and types from a MongoDB document. - - Args: - doc: Document to analyze - fields: Dictionary to store fields and types - prefix: Prefix for nested fields - max_depth: Maximum depth for nested fields - current_depth: Current depth - """ - if current_depth >= max_depth: - return - - for field, value in doc.items(): - # Para _id y otros campos especiales - if field == "_id": - field_type = "ObjectId" - elif isinstance(value, dict): - if current_depth < max_depth - 1: - # Recursión para campos anidados - self._extract_document_fields(value, fields, - f"{prefix}{field}.", max_depth, current_depth + 1) - field_type = "object" - elif isinstance(value, list): - if value and current_depth < max_depth - 1: - # Si tenemos elementos en la lista, analizar el primero - if isinstance(value[0], dict): - self._extract_document_fields(value[0], fields, - f"{prefix}{field}[].", max_depth, current_depth + 1) - else: - # Para listas de tipos primitivos - field_type = f"array<{type(value[0]).__name__}>" - else: - field_type = "array" - else: - field_type = type(value).__name__ - - # Guardar el tipo del campo actual - field_key = f"{prefix}{field}" - if field_key not in fields: - fields[field_key] = field_type - - def _process_document_for_serialization(self, doc: Dict[str, Any]) -> Dict[str, Any]: - """ - Processes a document to be JSON serializable. - - Args: - doc: Document to process - - Returns: - Processed document - """ - processed_doc = {} - for field, value in doc.items(): - # Convertir ObjectId a string - if field == "_id": - processed_doc[field] = str(value) - # Manejar objetos anidados - elif isinstance(value, dict): - processed_doc[field] = self._process_document_for_serialization(value) - # Manejar arrays - elif isinstance(value, list): - processed_items = [] - for item in value: - if isinstance(item, dict): - processed_items.append(self._process_document_for_serialization(item)) - elif hasattr(item, "__str__"): - processed_items.append(str(item)) - else: - processed_items.append(item) - processed_doc[field] = processed_items - # Convertir fechas a ISO - elif hasattr(value, 'isoformat'): - processed_doc[field] = value.isoformat() - # Otros tipos de datos - else: - processed_doc[field] = value - - return processed_doc - - def execute_query(self, query: str) -> List[Dict[str, Any]]: - """ - Executes a MongoDB query with improved error handling. - - Args: - query: MongoDB query in JSON format or query language - - Returns: - List of resulting documents - """ - if not self.client and not self.connect(): - raise ConnectionError("No se pudo establecer conexión con MongoDB") - - try: - # Determinar si la consulta es un string JSON o una consulta en otro formato - filter_dict, projection, collection_name, limit = self._parse_query(query) - - # Obtener la colección - if not collection_name: - raise ValueError("No se especificó el nombre de la colección en la consulta") - - collection = self.db[collection_name] - - # Ejecutar la consulta - if projection: - cursor = collection.find(filter_dict, projection).limit(limit or 100) - else: - cursor = collection.find(filter_dict).limit(limit or 100) - - # Convertir los resultados a formato serializable - results = [] - for doc in cursor: - processed_doc = self._process_document_for_serialization(doc) - results.append(processed_doc) - - return results - - except Exception as e: - # Intentar reconectar y reintentar una vez - try: - self.close() - if self.connect(): - print("Reconectando y reintentando consulta...") - - # Reintentar la consulta - filter_dict, projection, collection_name, limit = self._parse_query(query) - collection = self.db[collection_name] - - if projection: - cursor = collection.find(filter_dict, projection).limit(limit or 100) - else: - cursor = collection.find(filter_dict).limit(limit or 100) - - results = [] - for doc in cursor: - processed_doc = self._process_document_for_serialization(doc) - results.append(processed_doc) - - return results - except Exception as retry_error: - # Si falla el reintento, propagar el error original - raise Exception(f"Error al ejecutar consulta MongoDB: {str(e)}") - - # Si llegamos aquí, ha habido un error en el reintento - raise Exception(f"Error al ejecutar consulta MongoDB (después de reconexión): {str(e)}") - - def _parse_query(self, query: str) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]], str, Optional[int]]: - """ - Analyzes a query and extracts the necessary components. - - Args: - query: Query in string format - - Returns: - Tuple with (filter, projection, collection name, limit) - """ - # Intentar parsear como JSON - try: - query_dict = json.loads(query) - - # Extraer componentes de la consulta - filter_dict = query_dict.get("filter", {}) - projection = query_dict.get("projection") - collection_name = query_dict.get("collection") - limit = query_dict.get("limit") - - return filter_dict, projection, collection_name, limit - - except json.JSONDecodeError: - # Si no es JSON válido, intentar parsear el formato de consulta alternativo - collection_match = re.search(r'from\s+([a-zA-Z0-9_]+)', query, re.IGNORECASE) - collection_name = collection_match.group(1) if collection_match else None - - # Intentar extraer filtros - filter_match = re.search(r'where\s+(.+?)(?:limit|$)', query, re.IGNORECASE | re.DOTALL) - filter_str = filter_match.group(1).strip() if filter_match else "{}" - - # Intentar parsear los filtros como JSON - try: - filter_dict = json.loads(filter_str) - except json.JSONDecodeError: - # Si no se puede parsear, usar filtro vacío - filter_dict = {} - - # Extraer límite si existe - limit_match = re.search(r'limit\s+(\d+)', query, re.IGNORECASE) - limit = int(limit_match.group(1)) if limit_match else None - - return filter_dict, None, collection_name, limit - - def count_documents(self, collection_name: str, filter_dict: Optional[Dict[str, Any]] = None) -> int: - """ - Counts documents in a collection. - - Args: - collection_name: Name of the collection - filter_dict: Optional filter - - Returns: - Number of documents - """ - if not self.client and not self.connect(): - raise ConnectionError("No se pudo establecer conexión con MongoDB") - - try: - collection = self.db[collection_name] - return collection.count_documents(filter_dict or {}) - except Exception as e: - print(f"Error al contar documentos: {str(e)}") - return 0 - - def list_collections(self) -> List[str]: - """ - Returns a list of collections in the database. - - Returns: - List of collection names - """ - if not self.client and not self.connect(): - raise ConnectionError("No se pudo establecer conexión con MongoDB") - - try: - return self.db.list_collection_names() - except Exception as e: - print(f"Error al listar colecciones: {str(e)}") - return [] - - def close(self) -> None: - """Closes the MongoDB connection.""" - if self.client: - try: - self.client.close() - except: - pass - finally: - self.client = None - self.db = None - - def __del__(self): - """Destructor to ensure the connection is closed.""" - self.close() \ No newline at end of file diff --git a/corebrain/db/connectors/subconnectors/nosql/mongodb.py b/corebrain/db/connectors/subconnectors/nosql/mongodb.py index 4874331..57dd992 100644 --- a/corebrain/db/connectors/subconnectors/nosql/mongodb.py +++ b/corebrain/db/connectors/subconnectors/nosql/mongodb.py @@ -128,7 +128,7 @@ def _process_document_for_serialization(self, doc: Dict[str, Any]) -> Dict[str, processed_doc = {} for field, value in doc.items(): if field == "_id": - processed_doc[field] = self._process_document_for_serialization(value) + processed_doc[field] = str(value) elif isinstance(value, list): processed_items = [] for item in value: From abecc235b3e87a6d31e25d9043b29c7bdcc62a61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Ayuso?= Date: Mon, 26 May 2025 12:52:27 +0200 Subject: [PATCH 56/81] Show shcema added into configure or list configs arguments. Pending to adapt it yet --- corebrain/cli/commands.py | 70 ++++++++++++++++++------------------- corebrain/config/manager.py | 15 +------- 2 files changed, 35 insertions(+), 50 deletions(-) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index 048676b..a162d06 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -307,17 +307,43 @@ def check_library(library_name, min_version): print_colored("❌ Some system checks failed. Please review the issues above.", "red") return 1 + if args.configure or args.list_configs or args.show_schema: + """ + Configure, list or show schema of the configured database. + Reuse the same autehntication code for configure, list and show schema. + """ + # Get URLs + api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + + # Prioritize api_key if explicitly provided + token_arg = args.api_key if args.api_key else args.token + + # Get API credentials + api_key, user_data, api_token = get_api_credential(token_arg, sso_url) + + if not api_key: + print_colored("Error: An API Key is required. You can generate one at dashboard.corebrain.com", "red") + print_colored("Or use the 'corebrain --login' command to login via SSO.", "blue") + return 1 + + from corebrain.db.schema_file import show_db_schema, extract_schema_to_file + + # Execute the selected operation + if args.configure: + configure_sdk(api_token, api_key, api_url, sso_url, user_data) + elif args.list_configs: + ConfigManager.list_configs(api_key, api_url) + elif args.remove_config: + ConfigManager.remove_config(api_key, api_url) + elif args.show_schema: + show_db_schema(api_key, args.config_id, api_url) + if args.export_config: export_config(args.export_config) # --> config/manager.py --> export_config - if args.validate_config: - if not args.config_id: - print_colored("Error: --config-id is required for validation", "red") - return 1 - return validate_config(args.config_id) - # Create an user and API Key by default @@ -473,36 +499,8 @@ def check_library(library_name, min_version): print_colored(f"❌ Error when downloading data about user {str(e)}", "red") return 1 - # Operations that require credentials: configure, list, remove or show schema - if args.configure or args.list_configs: - # Get URLs - api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL - sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL - - # Prioritize api_key if explicitly provided - token_arg = args.api_key if args.api_key else args.token - - # Get API credentials - api_key, user_data, api_token = get_api_credential(token_arg, sso_url) - - if not api_key: - print_colored("Error: An API Key is required. You can generate one at dashboard.corebrain.com", "red") - print_colored("Or use the 'corebrain --login' command to login via SSO.", "blue") - return 1 - - from corebrain.db.schema_file import show_db_schema, extract_schema_to_file - - # Execute the selected operation - if args.configure: - configure_sdk(api_token, api_key, api_url, sso_url, user_data) - elif args.list_configs: - ConfigManager.list_configs(api_key, api_url) - elif args.remove_config: - ConfigManager.remove_config(api_key, api_url) - elif args.show_schema: - show_db_schema(api_key, args.config_id, api_url) - elif args.extract_schema: - extract_schema_to_file(api_key, args.config_id, args.output_file, api_url) + + if args.test_connection: # Test connection to the Corebrain API diff --git a/corebrain/config/manager.py b/corebrain/config/manager.py index 6682e88..8b645ef 100644 --- a/corebrain/config/manager.py +++ b/corebrain/config/manager.py @@ -4,6 +4,7 @@ import json import uuid +import os from pathlib import Path from typing import Dict, Any, List, Optional from cryptography.fernet import Fernet @@ -40,20 +41,6 @@ def export_config(filepath="config.json"): with open(filepath, "w") as f: json.dump(config, f, indent=4) print(f"Configuration exported to {filepath}") - -# Validates that a configuration with the given ID exists. -def validate_config(config_id: str): - # The API key under which configs are stored - api_key = os.environ.get("COREBRAIN_API_KEY", "") - manager = ConfigManager() - cfg = manager.get_config(api_key, config_id) - - if cfg: - print(f"✅ Configuration '{config_id}' is present and valid.") - return 0 - else: - print(f"❌ Configuration '{config_id}' not found.") - return 1 # Función para imprimir mensajes coloreados def _print_colored(message: str, color: str) -> None: From 206813c10b928298b6ce5bc2074a7b84014d7794 Mon Sep 17 00:00:00 2001 From: KarixD2137 Date: Mon, 26 May 2025 12:57:38 +0200 Subject: [PATCH 57/81] Merging with pre-release-verison --- corebrain/cli/auth/sso.py | 1 + corebrain/cli/commands.py | 6 +----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/corebrain/cli/auth/sso.py b/corebrain/cli/auth/sso.py index 8c8e48a..4f97ce4 100644 --- a/corebrain/cli/auth/sso.py +++ b/corebrain/cli/auth/sso.py @@ -465,4 +465,5 @@ def load_api_token() -> str: if os.path.exists(token_path): with open(token_path, "r") as f: return json.load(f).get("api_token") + return None \ No newline at end of file diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index 57cad09..8199106 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -437,11 +437,7 @@ def run_in_background_silent(cmd, cwd): # Handles the CLI command to create a new API key using stored credentials (token from SSO) if args.create_api_key: - - api_token = load_api_token() - if not api_token: - print_colored("❌ Missing valid API token. Please log in using --login.", "red") - return 1 + sso_token, sso_user = authentication() # Authentica use with SSO key_name = args.key_name or "default-key" key_level = args.key_level or "read" From 74200fc59bf55bfab261e4ab94b75a605c01d915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Ayuso?= Date: Mon, 26 May 2025 15:14:54 +0200 Subject: [PATCH 58/81] args.remove_config deleted from args.configure in cli/commands --- corebrain/cli/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index a162d06..cea4909 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -85,7 +85,7 @@ def authentication(): return None, None - # Show version + if args.version: """ Show the library version. From b11adac137c8a8059c7cb4fe1c9f8ec099857471 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Ayuso?= Date: Mon, 26 May 2025 15:30:03 +0200 Subject: [PATCH 59/81] Some comments added and commands deleted --- corebrain/cli/commands.py | 600 +++++++++++++++++++++++++++++++++++++- 1 file changed, 591 insertions(+), 9 deletions(-) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index cea4909..dd6a4a6 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -17,7 +17,6 @@ from corebrain.cli.utils import print_colored from corebrain.config.manager import ConfigManager from corebrain.config.manager import export_config -from corebrain.config.manager import validate_config from corebrain.lib.sso.auth import GlobodainSSOAuth def main_cli(argv: Optional[List[str]] = None) -> int: @@ -61,6 +60,9 @@ def main_cli(argv: Optional[List[str]] = None) -> int: parser.add_argument("--test-connection",action="store_true",help="Tests the connection to the Corebrain API using the provide credentials") parser.add_argument("--export-config",action="store_true",help="Exports the current configuration to a file") parser.add_argument("--gui", action="store_true", help="Check setup and launch the web interface") + parser.add_argument("--config-id", help="Configuration ID for operations that require it") + parser.add_argument("--output-file", help="Output file path for export operations") + parser.add_argument("--show-schema", action="store_true", help="Display database schema for a configuration") args = parser.parse_args(argv) @@ -88,7 +90,16 @@ def authentication(): if args.version: """ - Show the library version. + Display the current version of the Corebrain SDK. + + This command shows the version of the installed Corebrain SDK package. + It attempts to get the version from the package metadata first, and if that fails, + it falls back to the hardcoded version in the CLI module. + + Usage: corebrain --version + + Example output: + Corebrain SDK version 0.2.0 """ try: from importlib.metadata import version @@ -328,29 +339,422 @@ def check_library(library_name, min_version): print_colored("Or use the 'corebrain --login' command to login via SSO.", "blue") return 1 - from corebrain.db.schema_file import show_db_schema, extract_schema_to_file + from corebrain.db.schema_file import show_db_schema # Execute the selected operation if args.configure: + """ + Launch the comprehensive SDK configuration wizard. + + This is the main configuration command that guides you through setting up + a complete database connection for use with the Corebrain SDK. The wizard + walks you through each step of the configuration process interactively. + + Configuration phases: + 1. Authentication verification (already completed) + 2. Database type selection (SQL or MongoDB) + 3. Database engine selection (PostgreSQL, MySQL, SQLite, etc.) + 4. Connection parameters input (host, port, credentials, etc.) + 5. Database connection testing and validation + 6. Schema accessibility configuration (excluded tables/collections) + 7. Configuration saving and server synchronization + 8. Optional natural language query testing + + Usage: corebrain --configure [--api-key ] [--api-url ] [--sso-url ] + + Interactive prompts guide you through: + - Database type (sql/mongodb) + - Engine selection (postgresql, mysql, sqlite, etc.) + - Connection details (host, port, database name) + - Authentication credentials (username, password) + - Connection string (alternative to individual parameters) + - Table/collection exclusions for security + - Configuration naming and saving + + Supported databases: + SQL: + - PostgreSQL (local and remote) + - MySQL/MariaDB (local and remote) + - SQLite (file-based and in-memory) + + NoSQL: + - MongoDB (local and remote, with or without authentication) + + Security features: + - Encrypted local storage of configurations + - Secure credential handling + - Table/collection access control + - Server synchronization with encrypted transport + + After successful configuration: + - Configuration is saved locally with encryption + - Synchronization with Corebrain API server + - Ready to use with SDK (init function) + - Available for natural language queries + + Example usage after configuration: + ```python + from corebrain import init + + client = init( + api_key="your_api_key", + config_id="generated_config_id" + ) + + result = client.ask("How many users are in the database?") + ``` + + Prerequisites: + - Valid API key (obtain via --login or --api-key) + - Network access to target database + - Appropriate database permissions for schema reading + - Internet connectivity for API synchronization + """ configure_sdk(api_token, api_key, api_url, sso_url, user_data) + elif args.list_configs: + """ + List and manage all saved database configurations for your API key. + + This command provides an interactive interface to view and manage all + database configurations associated with your API key. It serves as a + central hub for configuration management operations. + + Main features: + - View all saved configurations with details + - Interactive selection and management + - Multiple management operations per configuration + - Safe deletion with confirmation prompts + - Configuration validation and testing + - Import/export capabilities + + Usage: corebrain --list-configs [--api-key ] + + Available operations for each configuration: + 1. Show Schema: Display detailed database structure + - Tables/collections list + - Column details and types + - Indexes and relationships + - Safe read-only operation + + 2. Validate Config: Comprehensive configuration validation + - Structure and format verification + - Database connectivity testing + - Permission and access verification + - Error reporting and diagnostics + + 3. Remove Config: Safe configuration deletion + - Confirmation prompts + - Local storage cleanup + - Server synchronization + - Irreversible operation warning + + 4. Modify Config: Update existing configuration + - Interactive parameter editing + - Connection parameter updates + - Excluded tables management + - Automatic validation after changes + + 5. Export Config: Backup configuration to file + - JSON format export + - Credential handling options + - Shareable format creation + - Backup and migration support + + 6. Import Config: Load configuration from file + - JSON file import + - Validation before saving + - Conflict resolution + - Batch import support + + 7. Configure New: Launch configuration wizard + - Full setup process + - Database connection setup + - Testing and validation + - Save new configuration + + Information displayed for each configuration: + - Configuration ID (unique identifier) + - Database type and engine + - Connection details (host, database name) + - Creation and last modified dates + - Validation status + - Usage statistics + + Interactive navigation: + - Arrow keys or numbers for selection + - Enter to confirm operations + - ESC or 'q' to exit + - Help available with '?' key + + Security considerations: + - Configurations stored with encryption + - Sensitive data masked in display + - Secure credential handling + - Server synchronization with HTTPS + + Use cases: + - Review existing database connections + - Maintain multiple database configurations + - Troubleshoot connection issues + - Backup and restore configurations + - Share configurations between environments + - Clean up unused configurations + + Prerequisites: + - Valid API key for authentication + - Internet connectivity for server operations + - Appropriate permissions for configuration management + + Note: This command provides a safe environment for configuration + management with confirmation prompts for destructive operations. + """ ConfigManager.list_configs(api_key, api_url) - elif args.remove_config: - ConfigManager.remove_config(api_key, api_url) + elif args.show_schema: + """ + Display the schema of a configured database without connecting through the SDK. + + This command allows you to explore the structure of a database by showing + detailed information about tables, columns, indexes, and relationships. + It's useful for understanding the database structure before writing queries. + + The command can work in two ways: + 1. With a saved configuration (using --config-id) + 2. By prompting you to select from available configurations + + Usage: corebrain --show-schema [--config-id ] + + Information displayed: + - Database type and engine + - List of all tables/collections + - Column details (name, type, constraints) + - Primary keys and foreign keys + - Indexes and their properties + - Table relationships and dependencies + + Supported databases: + - SQL: PostgreSQL, MySQL, SQLite + - NoSQL: MongoDB + + Note: This command only reads schema information and doesn't modify + the database in any way. It's safe to run on production databases. + """ show_db_schema(api_key, args.config_id, api_url) - - if args.export_config: - export_config(args.export_config) - # --> config/manager.py --> export_config + # Handle validate-config and export-config commands + if args.validate_config: + """ + Validate a saved configuration without executing any operations. + + This command performs comprehensive validation of a database configuration + to ensure it's correctly formatted and all required parameters are present. + It checks the configuration syntax, required fields, and optionally tests + the database connection. + + Validation checks performed: + 1. Configuration format and structure + 2. Required fields presence (type, engine, credentials) + 3. Field value validity (ports, hostnames, database names) + 4. Database connection test (optional) + 5. Authentication and permissions verification + + Usage: corebrain --validate-config --config-id [--api-key ] + + Validation levels: + - Structure: Validates configuration format and required fields + - Connection: Tests actual database connectivity + - Permissions: Verifies database access permissions + - Schema: Checks if the database schema can be read + + Exit codes: + - 0: Configuration is valid + - 1: Configuration has errors + + Use cases: + - Verify configuration before deployment + - Troubleshoot connection issues + - Validate imported configurations + - Check configuration after database changes + + Note: This command requires a valid API key to access saved configurations. + """ + if not args.config_id: + print_colored("Error: --config-id is required for validation", "red") + return 1 + + # Get credentials + api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + token_arg = args.api_key if args.api_key else args.token + api_key, user_data, api_token = get_api_credential(token_arg, sso_url) + + if not api_key: + print_colored("Error: An API Key is required. Use --api-key or login via --login", "red") + return 1 + + # Validate the configuration + try: + config_manager = ConfigManager() + config = config_manager.get_config(api_key, args.config_id) + + if not config: + print_colored(f"Configuration with ID '{args.config_id}' not found", "red") + return 1 + + print_colored(f"✅ Validating configuration: {args.config_id}", "blue") + + # Create a temporary Corebrain instance to validate + from corebrain.core.client import Corebrain + try: + temp_client = Corebrain( + api_key=api_key, + db_config=config, + skip_verification=True + ) + print_colored("✅ Configuration validation passed!", "green") + print_colored(f"Database type: {config.get('type', 'Unknown')}", "blue") + print_colored(f"Engine: {config.get('engine', 'Unknown')}", "blue") + return 0 + except Exception as validation_error: + print_colored(f"❌ Configuration validation failed: {str(validation_error)}", "red") + return 1 + + except Exception as e: + print_colored(f"❌ Error during validation: {str(e)}", "red") + return 1 + if args.export_config: + """ + Export a saved configuration to a JSON file. + + This command exports a database configuration from the local storage + to a JSON file that can be shared, backed up, or imported on another system. + The exported file contains all connection parameters and settings needed + to recreate the configuration. + + The export process: + 1. Retrieves the specified configuration from local storage + 2. Decrypts sensitive information (if encrypted) + 3. Formats the configuration as readable JSON + 4. Saves to the specified output file + 5. Optionally removes sensitive data for sharing + + Usage: corebrain --export-config --config-id [--output-file ] [--api-key ] + + Options: + --config-id: ID of the configuration to export (required) + --output-file: Path for the exported file (default: config_.json) + --remove-credentials: Remove sensitive data for sharing (optional) + --pretty-print: Format JSON with indentation for readability + + Exported data includes: + - Database connection parameters + - Engine and type information + - Configuration metadata + - Excluded tables/collections list + - Custom settings and preferences + + Security considerations: + - Exported files may contain sensitive credentials + - Use --remove-credentials flag when sharing configurations + - Store exported files in secure locations + - Consider encrypting exported files for transmission + + Use cases: + - Backup configurations before changes + - Share configurations between team members + - Migrate configurations to different environments + - Create configuration templates + - Document database connection settings + """ + if not args.config_id: + print_colored("Error: --config-id is required for export", "red") + return 1 + + # Get credentials + api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + token_arg = args.api_key if args.api_key else args.token + api_key, user_data, api_token = get_api_credential(token_arg, sso_url) + + if not api_key: + print_colored("Error: An API Key is required. Use --api-key or login via --login", "red") + return 1 + + # Export the configuration + try: + config_manager = ConfigManager() + config = config_manager.get_config(api_key, args.config_id) + + if not config: + print_colored(f"Configuration with ID '{args.config_id}' not found", "red") + return 1 + + # Generate output filename if not provided + output_file = getattr(args, 'output_file', None) or f"config_{args.config_id}.json" + + # Export to file + import json + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(config, f, indent=2, default=str) + + print_colored(f"✅ Configuration exported to: {output_file}", "green") + return 0 + + except Exception as e: + print_colored(f"❌ Error exporting configuration: {str(e)}", "red") + return 1 # Create an user and API Key by default if args.authentication: + """ + Perform SSO authentication and display the obtained tokens and user data. + + This command initiates the SSO (Single Sign-On) authentication flow through the browser. + It opens a browser window for the user to authenticate with their Globodain SSO credentials + and returns the authentication token and user information. + + This is primarily used for testing authentication or when you need to see the raw + authentication data. For normal usage, prefer --login which also obtains API keys. + + Usage: corebrain --authentication [--sso-url ] + + Returns: + - SSO authentication token + - User profile data (name, email, etc.) + + Note: This command only authenticates but doesn't save credentials for future use. + """ authentication() if args.create_user: + """ + Create a new user account and generate an associated API Key. + + This command performs a complete user registration process: + 1. Authenticates the user through SSO (Single Sign-On) + 2. Creates a new user account in the Corebrain system using SSO data + 3. Automatically generates an API Key for the new user + + The user can choose to use their SSO password or create a new password + specifically for their Corebrain account. If using SSO password fails, + a random secure password will be generated. + + Usage: corebrain --create-user [--api-url ] [--sso-url ] + + Interactive prompts: + - SSO authentication (browser-based) + - Password choice (use SSO password or create new) + - Password confirmation (if creating new) + + Requirements: + - Valid Globodain SSO account + - Internet connection for API communication + + On success: Creates user account and displays confirmation + On failure: Shows specific error message + """ sso_token, sso_user = authentication() # Authentica use with SSO if sso_token and sso_user: @@ -413,6 +817,31 @@ def check_library(library_name, min_version): # Test SSO authentication if args.test_auth: + """ + Test the SSO (Single Sign-On) authentication system. + + This command performs a comprehensive test of the SSO authentication flow + without saving any credentials or performing any actual operations. It's useful + for diagnosing authentication issues and verifying that the SSO system is working. + + The test process: + 1. Configures the SSO authentication client + 2. Generates a login URL + 3. Opens the browser for user authentication + 4. Waits for user to complete the authentication process + 5. Reports success or failure + + Usage: corebrain --test-auth [--sso-url ] + + What it tests: + - SSO server connectivity + - Client configuration validity + - Authentication flow completion + - Browser integration + + Note: This is a diagnostic tool and doesn't save any authentication data. + For actual login, use --login instead. + """ sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL print_colored("Testing SSO authentication...", "blue") @@ -450,6 +879,34 @@ def check_library(library_name, min_version): # Login via SSO if args.login: + """ + Login via SSO and obtain API credentials for SDK usage. + + This is the primary authentication command for normal SDK usage. It performs + a complete authentication and credential acquisition process: + + 1. Opens browser for SSO authentication + 2. Exchanges SSO token for Corebrain API token + 3. Fetches available API keys for the user + 4. Allows user to select which API key to use + 5. Saves credentials in environment variables for immediate use + + Usage: corebrain --login [--sso-url ] [--configure] + + What it provides: + - API Token: For general authentication with Corebrain services + - API Key: For specific SDK operations and database access + - User Data: Profile information from SSO + + Environment variables set: + - COREBRAIN_API_TOKEN: General API authentication token + - COREBRAIN_API_KEY: Specific API key for SDK operations + + Optional: If --configure is also specified, it will automatically launch + the configuration wizard after successful login. + + This is the recommended way to authenticate for first-time users. + """ sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL api_key, user_data, api_token = authenticate_with_sso_and_api_key_request(sso_url) @@ -478,6 +935,38 @@ def check_library(library_name, min_version): if args.woami: + """ + Display information about the currently authenticated user. + + This command shows detailed information about the user associated with the + current authentication credentials. It's similar to the Unix 'whoami' command + but for the Corebrain system. + + The command attempts to retrieve user data using the following credential sources + (in order of priority): + 1. API key provided via --api-key argument + 2. Token provided via --token argument + 3. COREBRAIN_API_KEY environment variable + 4. COREBRAIN_API_TOKEN environment variable + 5. SSO authentication (if no other credentials found) + + Usage: corebrain --woami [--api-key ] [--token ] [--sso-url ] + + Information displayed: + - User ID and email + - Name and profile details + - Account creation and last login dates + - Associated roles and permissions + - Any other profile metadata from SSO + + Use cases: + - Verify which user account is currently active + - Debug authentication issues + - Check user permissions and profile data + - Confirm successful login + + Note: Requires valid authentication credentials to work. + """ try: #downloading user data sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL @@ -503,6 +992,47 @@ def check_library(library_name, min_version): if args.test_connection: + """ + Test the connection to the Corebrain API using the provided credentials. + + This command verifies that the SDK can successfully connect to and authenticate + with the Corebrain API server. It's useful for diagnosing connectivity issues, + credential problems, or API server availability. + + The test process: + 1. Retrieves API credentials from various sources + 2. Attempts to connect to the specified API endpoint + 3. Performs authentication verification + 4. Reports connection status and any errors + + Usage: corebrain --test-connection [--token ] [--api-url ] [--sso-url ] + + Credential sources (in order of priority): + 1. Token provided via --token argument + 2. COREBRAIN_API_KEY environment variable + 3. COREBRAIN_API_TOKEN environment variable + 4. SSO authentication (if no credentials found) + + What it tests: + - Network connectivity to API server + - API server availability and responsiveness + - Credential validity and authentication + - SSL/TLS connection (for HTTPS endpoints) + + Success indicators: + - ✅ Successfully connected to Corebrain API + + Failure indicators: + - Connection timeouts or network errors + - Invalid or expired credentials + - API server errors or maintenance + + Use cases: + - Troubleshoot connection issues + - Verify API credentials before starting work + - Check API server status + - Validate network connectivity in restricted environments + """ # Test connection to the Corebrain API api_url = args.api_url or os.environ.get("COREBRAIN_API_URL", DEFAULT_API_URL) sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL", DEFAULT_SSO_URL) @@ -534,6 +1064,58 @@ def check_library(library_name, min_version): if args.gui: + """ + Check setup and launch the web-based graphical user interface. + + This command sets up and launches a complete web-based GUI for the Corebrain SDK, + providing a user-friendly alternative to the command-line interface. The GUI includes + both frontend and backend components and integrates with the Corebrain API. + + Components launched: + 1. React Frontend (client) - User interface running on port 5173 + 2. Express Backend (server) - API server for the frontend + 3. Corebrain API wrapper (C#) - Additional API integration + + Setup process: + 1. Validates required directory structure + 2. Installs Node.js dependencies if not present + 3. Configures development tools (Vite, TypeScript) + 4. Starts all services concurrently + 5. Opens browser to the GUI automatically + + Usage: corebrain --gui + + Directory structure required: + - CLI-UI/client/ (React frontend) + - CLI-UI/server/ (Express backend) + - wrappers/csharp_cli_api/ (C# API wrapper) + + Dependencies installed automatically: + Frontend (React): + - Standard React dependencies + - History library for routing + - Vite for development and building + - Concurrently for running multiple processes + + Backend (Express): + - Standard Express dependencies + - TypeScript development tools + - ts-node-dev for hot reloading + + Access points: + - Frontend GUI: http://localhost:5173/ + - Backend API: Usually http://localhost:3000/ + - C# API wrapper: Usually http://localhost:5000/ + + Use cases: + - Visual configuration of database connections + - Interactive query building and testing + - Graphical schema exploration + - User-friendly alternative to CLI commands + - Debugging and development interface + + Note: Requires Node.js, npm, and .NET runtime to be installed on the system. + """ import subprocess from pathlib import Path From b00c49510ec20e30a79b45b49f8c54449d5e2e3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Ayuso?= Date: Mon, 26 May 2025 16:45:03 +0200 Subject: [PATCH 60/81] Test connection will be moved into --list-configs to test the configuration selected. Commands were organized and comments added. --- corebrain/cli/commands.py | 711 ++++++++++++++++---------------------- 1 file changed, 291 insertions(+), 420 deletions(-) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index dd6a4a6..674d1f4 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -16,7 +16,6 @@ from corebrain.cli.config import configure_sdk, get_api_credential from corebrain.cli.utils import print_colored from corebrain.config.manager import ConfigManager -from corebrain.config.manager import export_config from corebrain.lib.sso.auth import GlobodainSSOAuth def main_cli(argv: Optional[List[str]] = None) -> int: @@ -39,36 +38,9 @@ def main_cli(argv: Optional[List[str]] = None) -> int: if argv is None: argv = sys.argv[1:] - # Argument parser configuration - parser = argparse.ArgumentParser(description="Corebrain SDK CLI") - parser.add_argument("--version", action="store_true", help="Show SDK version") - parser.add_argument("--authentication", action="store_true", help="Authenticate with SSO") - parser.add_argument("--create-user", action="store_true", help="Create an user and API Key by default") - parser.add_argument("--configure", action="store_true", help="Configure the Corebrain SDK") - parser.add_argument("--list-configs", action="store_true", help="List available configurations") - - parser.add_argument("--token", help="Corebrain API token (any type)") - parser.add_argument("--api-key", help="Specific API Key for Corebrain") - parser.add_argument("--api-url", help="Corebrain API URL") - parser.add_argument("--sso-url", help="Globodain SSO service URL") - parser.add_argument("--login", action="store_true", help="Login via SSO") - parser.add_argument("--test-auth", action="store_true", help="Test SSO authentication system") - parser.add_argument("--woami",action="store_true",help="Display information about the current user") - parser.add_argument("--check-status",action="store_true",help="Checks status of task") - parser.add_argument("--task-id",help="ID of the task to check status for") - parser.add_argument("--validate-config",action="store_true",help="Validates the selected configuration without executing any operations") - parser.add_argument("--test-connection",action="store_true",help="Tests the connection to the Corebrain API using the provide credentials") - parser.add_argument("--export-config",action="store_true",help="Exports the current configuration to a file") - parser.add_argument("--gui", action="store_true", help="Check setup and launch the web interface") - parser.add_argument("--config-id", help="Configuration ID for operations that require it") - parser.add_argument("--output-file", help="Output file path for export operations") - parser.add_argument("--show-schema", action="store_true", help="Display database schema for a configuration") - - - args = parser.parse_args(argv) - + # Functions def authentication(): - sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + sso_url = os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL sso_token, sso_user = authenticate_with_sso(sso_url) if sso_token: try: @@ -85,9 +57,50 @@ def authentication(): else: print_colored("❌ Could not authenticate with SSO.", "red") return None, None + + def authentication_with_api_key_return(): + sso_url = os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + api_key_selected, user_data, api_token = authenticate_with_sso_and_api_key_request(sso_url) + + if sso_token: + try: + print_colored("✅ User authenticated and SDK is now connected to API.", "green") + print_colored("✅ Returning User data.", "green") + print_colored(f"{user_data}", "blue") + return api_key_selected, user_data, api_token + + except Exception as e: + print_colored("❌ Could not return SSO Token or SSO User data.", "red") + return api_key_selected, user_data, api_token + + else: + print_colored("❌ Could not authenticate with SSO.", "red") + return None, None, None + + # Argument parser configuration + parser = argparse.ArgumentParser(description="Corebrain SDK CLI") + + # Arguments for development + parser.add_argument("--version", action="store_true", help="Show SDK version") + parser.add_argument("--check-status",action="store_true",help="Checks status of task") + parser.add_argument("--authentication", action="store_true", help="Authenticate with SSO") + parser.add_argument("--test-auth", action="store_true", help="Test SSO authentication system") + + # Arguments to use the SDK + parser.add_argument("--create-user", action="store_true", help="Create an user and API Key by default") + parser.add_argument("--configure", action="store_true", help="Configure the Corebrain SDK") + parser.add_argument("--list-configs", action="store_true", help="List available configurations") + parser.add_argument("--show-schema", action="store_true", help="Display database schema for a configuration") + parser.add_argument("--woami",action="store_true",help="Display information about the current user") + parser.add_argument("--gui", action="store_true", help="Check setup and launch the web interface") + args = parser.parse_args(argv) + # Common variables + api_url = os.environ.get("COREBRAIN_API_URL", DEFAULT_API_URL) + sso_url = os.environ.get("COREBRAIN_SSO_URL", DEFAULT_SSO_URL) + ## ** For development ** ## if args.version: """ Display the current version of the Corebrain SDK. @@ -254,7 +267,7 @@ def check_library(library_name, min_version): return False # Determine if in development or production mode - api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + api_url = os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL is_development = "localhost" in api_url or "127.0.0.1" in api_url or api_url == DEFAULT_API_URL print_colored("🔍 Checking system status...", "blue") @@ -306,7 +319,7 @@ def check_library(library_name, min_version): all_checks_passed = False # Check SSO service for both modes - sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + sso_url = os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL if not check_url(sso_url, "SSO Server"): all_checks_passed = False @@ -318,25 +331,190 @@ def check_library(library_name, min_version): print_colored("❌ Some system checks failed. Please review the issues above.", "red") return 1 + if args.authentication: + """ + Perform SSO authentication and display the obtained tokens and user data. + + This command initiates the SSO (Single Sign-On) authentication flow through the browser. + It opens a browser window for the user to authenticate with their Globodain SSO credentials + and returns the authentication token and user information. + + This is primarily used for testing authentication or when you need to see the raw + authentication data. For normal usage, prefer --login which also obtains API keys. + + Usage: corebrain --authentication [--sso-url ] + + Returns: + - SSO authentication token + - User profile data (name, email, etc.) + + Note: This command only authenticates but doesn't save credentials for future use. + """ + authentication() + + if args.test_auth: + """ + Test the SSO (Single Sign-On) authentication system. + + This command performs a comprehensive test of the SSO authentication flow + without saving any credentials or performing any actual operations. It's useful + for diagnosing authentication issues and verifying that the SSO system is working. + + The test process: + 1. Configures the SSO authentication client + 2. Generates a login URL + 3. Opens the browser for user authentication + 4. Waits for user to complete the authentication process + 5. Reports success or failure + + Usage: corebrain --test-auth [--sso-url ] + + What it tests: + - SSO server connectivity + - Client configuration validity + - Authentication flow completion + - Browser integration + + Note: This is a diagnostic tool and doesn't save any authentication data. + For actual login, use --login instead. + """ + sso_url = os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + + print_colored("Testing SSO authentication...", "blue") + + # Authentication configuration + auth_config = { + 'GLOBODAIN_SSO_URL': sso_url, + 'GLOBODAIN_CLIENT_ID': SSO_CLIENT_ID, + 'GLOBODAIN_CLIENT_SECRET': SSO_CLIENT_SECRET, + 'GLOBODAIN_REDIRECT_URI': f"http://localhost:{DEFAULT_PORT}/auth/sso/callback", + 'GLOBODAIN_SUCCESS_REDIRECT': f"http://localhost:{DEFAULT_PORT}/auth/sso/callback" + } + + try: + # Instantiate authentication client + sso_auth = GlobodainSSOAuth(config=auth_config) + + # Get login URL + login_url = sso_auth.get_login_url() + + print_colored(f"Login URL: {login_url}", "blue") + print_colored("Opening browser for login...", "blue") + + # Open browser + webbrowser.open(login_url) + + print_colored("Please complete the login process in the browser.", "blue") + input("\nPress Enter when you've completed the process or to cancel...") + + print_colored("✅ SSO authentication test completed!", "green") + return 0 + except Exception as e: + print_colored(f"❌ Error during test: {str(e)}", "red") + return 1 + + + ## ** SDK ** ## + + if args.create_user: + """ + Create a new user account and generate an associated API Key. + + This command performs a complete user registration process: + 1. Authenticates the user through SSO (Single Sign-On) + 2. Creates a new user account in the Corebrain system using SSO data + 3. Automatically generates an API Key for the new user + + The user can choose to use their SSO password or create a new password + specifically for their Corebrain account. If using SSO password fails, + a random secure password will be generated. + + Usage: corebrain --create-user [--api-url ] [--sso-url ] + + Interactive prompts: + - SSO authentication (browser-based) + - Password choice (use SSO password or create new) + - Password confirmation (if creating new) + + Requirements: + - Valid Globodain SSO account + - Internet connection for API communication + + On success: Creates user account and displays confirmation + On failure: Shows specific error message + """ + sso_token, sso_user = authentication() # Authentica use with SSO + + if sso_token and sso_user: + print_colored("✅ Enter to create an user and API Key.", "green") + + # Get API URL from environment or use default + api_url = os.environ.get("COREBRAIN_API_URL", DEFAULT_API_URL) + + """ + Create user data with SSO information. + If the user wants to use a different password than their SSO account, + they can specify it here. + """ + # Ask if user wants to use SSO password or create a new one + use_sso_password = input("Do you want to use your SSO password? (y/n): ").lower().strip() == 'y' + + if use_sso_password: + random_password = ''.join(random.choices(string.ascii_letters + string.digits, k=12)) + password = sso_user.get("password", random_password) + else: + while True: + password = input("Enter new password: ").strip() + if len(password) >= 8: + break + print_colored("Password must be at least 8 characters long", "yellow") + + user_data = { + "email": sso_user["email"], + "name": f"{sso_user['first_name']} {sso_user['last_name']}", + "password": password + } + + try: + # Make the API request + response = requests.post( + f"{api_url}/api/auth/users", + json=user_data, + headers={ + "Authorization": f"Bearer {sso_token}", + "Content-Type": "application/json" + } + ) + + # Check if the request was successful + print("response API: ", response) + if response.status_code == 200: + print_colored("✅ User and API Key created successfully!", "green") + return 0 + else: + print_colored(f"❌ Error creating user: {response.text}", "red") + return 1 + + except requests.exceptions.RequestException as e: + print_colored(f"❌ Error connecting to API: {str(e)}", "red") + return 1 + + else: + print_colored("❌ Could not create the user or the API KEY.", "red") + return 1 + if args.configure or args.list_configs or args.show_schema: """ Configure, list or show schema of the configured database. - Reuse the same autehntication code for configure, list and show schema. + Reuse the same authentication code for configure, list and show schema. """ - # Get URLs - api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL - sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL - - # Prioritize api_key if explicitly provided - token_arg = args.api_key if args.api_key else args.token - # Get API credentials - api_key, user_data, api_token = get_api_credential(token_arg, sso_url) + api_key_selected, user_data, api_token = authentication_with_api_key_return() # Authentica use with SSO - if not api_key: - print_colored("Error: An API Key is required. You can generate one at dashboard.corebrain.com", "red") - print_colored("Or use the 'corebrain --login' command to login via SSO.", "blue") + if not api_key_selected: + print_colored("Error: An API Key is required. You can generate one at dashboard.etedata.com", "red") + print_colored("Or use the 'corebrain --create-api-key' command to create a new one using CLI.", "blue") return 1 from corebrain.db.schema_file import show_db_schema @@ -410,7 +588,7 @@ def check_library(library_name, min_version): - Appropriate database permissions for schema reading - Internet connectivity for API synchronization """ - configure_sdk(api_token, api_key, api_url, sso_url, user_data) + configure_sdk(api_token, api_key_selected, api_url, sso_url, user_data) elif args.list_configs: """ @@ -542,8 +720,12 @@ def check_library(library_name, min_version): """ show_db_schema(api_key, args.config_id, api_url) + + + # ** move to the config manager --> inside of the command --list-configs ** + # Handle validate-config and export-config commands - if args.validate_config: + #if args.validate_config: """ Validate a saved configuration without executing any operations. @@ -579,52 +761,52 @@ def check_library(library_name, min_version): Note: This command requires a valid API key to access saved configurations. """ - if not args.config_id: - print_colored("Error: --config-id is required for validation", "red") - return 1 + # if not args.config_id: + # print_colored("Error: --config-id is required for validation", "red") + # return 1 # Get credentials - api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL - sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL - token_arg = args.api_key if args.api_key else args.token - api_key, user_data, api_token = get_api_credential(token_arg, sso_url) + # api_url = os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + # sso_url = os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + # token_arg = args.api_key if args.api_key else args.token + # api_key, user_data, api_token = get_api_credential(token_arg, sso_url) - if not api_key: - print_colored("Error: An API Key is required. Use --api-key or login via --login", "red") - return 1 + # if not api_key: + # print_colored("Error: An API Key is required. Use --api-key or login via --login", "red") + # return 1 # Validate the configuration - try: - config_manager = ConfigManager() - config = config_manager.get_config(api_key, args.config_id) + # try: + # config_manager = ConfigManager() + # config = config_manager.get_config(api_key, args.config_id) - if not config: - print_colored(f"Configuration with ID '{args.config_id}' not found", "red") - return 1 + # if not config: + # print_colored(f"Configuration with ID '{args.config_id}' not found", "red") + # return 1 - print_colored(f"✅ Validating configuration: {args.config_id}", "blue") + # print_colored(f"✅ Validating configuration: {args.config_id}", "blue") # Create a temporary Corebrain instance to validate - from corebrain.core.client import Corebrain - try: - temp_client = Corebrain( - api_key=api_key, - db_config=config, - skip_verification=True - ) - print_colored("✅ Configuration validation passed!", "green") - print_colored(f"Database type: {config.get('type', 'Unknown')}", "blue") - print_colored(f"Engine: {config.get('engine', 'Unknown')}", "blue") - return 0 - except Exception as validation_error: - print_colored(f"❌ Configuration validation failed: {str(validation_error)}", "red") - return 1 + # from corebrain.core.client import Corebrain + # try: + # temp_client = Corebrain( + # api_key=api_key, + # db_config=config, + # skip_verification=True + # ) + # print_colored("✅ Configuration validation passed!", "green") + # print_colored(f"Database type: {config.get('type', 'Unknown')}", "blue") + # print_colored(f"Engine: {config.get('engine', 'Unknown')}", "blue") + # return 0 + # except Exception as validation_error: + # print_colored(f"❌ Configuration validation failed: {str(validation_error)}", "red") + # return 1 - except Exception as e: - print_colored(f"❌ Error during validation: {str(e)}", "red") - return 1 + # except Exception as e: + # print_colored(f"❌ Error during validation: {str(e)}", "red") + # return 1 - if args.export_config: + #if args.export_config: """ Export a saved configuration to a JSON file. @@ -668,271 +850,44 @@ def check_library(library_name, min_version): - Create configuration templates - Document database connection settings """ - if not args.config_id: - print_colored("Error: --config-id is required for export", "red") - return 1 + # if not args.config_id: + # print_colored("Error: --config-id is required for export", "red") + # return 1 # Get credentials - api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL - sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL - token_arg = args.api_key if args.api_key else args.token - api_key, user_data, api_token = get_api_credential(token_arg, sso_url) + # api_url = os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + # sso_url = os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + # token_arg = args.api_key if args.api_key else args.token + # api_key, user_data, api_token = get_api_credential(token_arg, sso_url) - if not api_key: - print_colored("Error: An API Key is required. Use --api-key or login via --login", "red") - return 1 + # if not api_key: + # print_colored("Error: An API Key is required. Use --api-key or login via --login", "red") + # return 1 # Export the configuration - try: - config_manager = ConfigManager() - config = config_manager.get_config(api_key, args.config_id) + # try: + # config_manager = ConfigManager() + # config = config_manager.get_config(api_key, args.config_id) - if not config: - print_colored(f"Configuration with ID '{args.config_id}' not found", "red") - return 1 + # if not config: + # print_colored(f"Configuration with ID '{args.config_id}' not found", "red") + # return 1 # Generate output filename if not provided - output_file = getattr(args, 'output_file', None) or f"config_{args.config_id}.json" + # output_file = getattr(args, 'output_file', None) or f"config_{args.config_id}.json" # Export to file - import json - with open(output_file, 'w', encoding='utf-8') as f: - json.dump(config, f, indent=2, default=str) + # import json + # with open(output_file, 'w', encoding='utf-8') as f: + # json.dump(config, f, indent=2, default=str) - print_colored(f"✅ Configuration exported to: {output_file}", "green") - return 0 + # print_colored(f"✅ Configuration exported to: {output_file}", "green") + # return 0 - except Exception as e: - print_colored(f"❌ Error exporting configuration: {str(e)}", "red") - return 1 + # except Exception as e: + # print_colored(f"❌ Error exporting configuration: {str(e)}", "red") + # return 1 - # Create an user and API Key by default - if args.authentication: - """ - Perform SSO authentication and display the obtained tokens and user data. - - This command initiates the SSO (Single Sign-On) authentication flow through the browser. - It opens a browser window for the user to authenticate with their Globodain SSO credentials - and returns the authentication token and user information. - - This is primarily used for testing authentication or when you need to see the raw - authentication data. For normal usage, prefer --login which also obtains API keys. - - Usage: corebrain --authentication [--sso-url ] - - Returns: - - SSO authentication token - - User profile data (name, email, etc.) - - Note: This command only authenticates but doesn't save credentials for future use. - """ - authentication() - - if args.create_user: - """ - Create a new user account and generate an associated API Key. - - This command performs a complete user registration process: - 1. Authenticates the user through SSO (Single Sign-On) - 2. Creates a new user account in the Corebrain system using SSO data - 3. Automatically generates an API Key for the new user - - The user can choose to use their SSO password or create a new password - specifically for their Corebrain account. If using SSO password fails, - a random secure password will be generated. - - Usage: corebrain --create-user [--api-url ] [--sso-url ] - - Interactive prompts: - - SSO authentication (browser-based) - - Password choice (use SSO password or create new) - - Password confirmation (if creating new) - - Requirements: - - Valid Globodain SSO account - - Internet connection for API communication - - On success: Creates user account and displays confirmation - On failure: Shows specific error message - """ - sso_token, sso_user = authentication() # Authentica use with SSO - - if sso_token and sso_user: - print_colored("✅ Enter to create an user and API Key.", "green") - - # Get API URL from environment or use default - api_url = os.environ.get("COREBRAIN_API_URL", DEFAULT_API_URL) - - """ - Create user data with SSO information. - If the user wants to use a different password than their SSO account, - they can specify it here. - """ - # Ask if user wants to use SSO password or create a new one - use_sso_password = input("Do you want to use your SSO password? (y/n): ").lower().strip() == 'y' - - if use_sso_password: - random_password = ''.join(random.choices(string.ascii_letters + string.digits, k=12)) - password = sso_user.get("password", random_password) - else: - while True: - password = input("Enter new password: ").strip() - if len(password) >= 8: - break - print_colored("Password must be at least 8 characters long", "yellow") - - user_data = { - "email": sso_user["email"], - "name": f"{sso_user['first_name']} {sso_user['last_name']}", - "password": password - } - - try: - # Make the API request - response = requests.post( - f"{api_url}/api/auth/users", - json=user_data, - headers={ - "Authorization": f"Bearer {sso_token}", - "Content-Type": "application/json" - } - ) - - # Check if the request was successful - print("response API: ", response) - if response.status_code == 200: - print_colored("✅ User and API Key created successfully!", "green") - return 0 - else: - print_colored(f"❌ Error creating user: {response.text}", "red") - return 1 - - except requests.exceptions.RequestException as e: - print_colored(f"❌ Error connecting to API: {str(e)}", "red") - return 1 - - else: - print_colored("❌ Could not create the user or the API KEY.", "red") - return 1 - - # Test SSO authentication - if args.test_auth: - """ - Test the SSO (Single Sign-On) authentication system. - - This command performs a comprehensive test of the SSO authentication flow - without saving any credentials or performing any actual operations. It's useful - for diagnosing authentication issues and verifying that the SSO system is working. - - The test process: - 1. Configures the SSO authentication client - 2. Generates a login URL - 3. Opens the browser for user authentication - 4. Waits for user to complete the authentication process - 5. Reports success or failure - - Usage: corebrain --test-auth [--sso-url ] - - What it tests: - - SSO server connectivity - - Client configuration validity - - Authentication flow completion - - Browser integration - - Note: This is a diagnostic tool and doesn't save any authentication data. - For actual login, use --login instead. - """ - sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL - - print_colored("Testing SSO authentication...", "blue") - - # Authentication configuration - auth_config = { - 'GLOBODAIN_SSO_URL': sso_url, - 'GLOBODAIN_CLIENT_ID': SSO_CLIENT_ID, - 'GLOBODAIN_CLIENT_SECRET': SSO_CLIENT_SECRET, - 'GLOBODAIN_REDIRECT_URI': f"http://localhost:{DEFAULT_PORT}/auth/sso/callback", - 'GLOBODAIN_SUCCESS_REDIRECT': f"http://localhost:{DEFAULT_PORT}/auth/sso/callback" - } - - try: - # Instantiate authentication client - sso_auth = GlobodainSSOAuth(config=auth_config) - - # Get login URL - login_url = sso_auth.get_login_url() - - print_colored(f"Login URL: {login_url}", "blue") - print_colored("Opening browser for login...", "blue") - - # Open browser - webbrowser.open(login_url) - - print_colored("Please complete the login process in the browser.", "blue") - input("\nPress Enter when you've completed the process or to cancel...") - - print_colored("✅ SSO authentication test completed!", "green") - return 0 - except Exception as e: - print_colored(f"❌ Error during test: {str(e)}", "red") - return 1 - - # Login via SSO - if args.login: - """ - Login via SSO and obtain API credentials for SDK usage. - - This is the primary authentication command for normal SDK usage. It performs - a complete authentication and credential acquisition process: - - 1. Opens browser for SSO authentication - 2. Exchanges SSO token for Corebrain API token - 3. Fetches available API keys for the user - 4. Allows user to select which API key to use - 5. Saves credentials in environment variables for immediate use - - Usage: corebrain --login [--sso-url ] [--configure] - - What it provides: - - API Token: For general authentication with Corebrain services - - API Key: For specific SDK operations and database access - - User Data: Profile information from SSO - - Environment variables set: - - COREBRAIN_API_TOKEN: General API authentication token - - COREBRAIN_API_KEY: Specific API key for SDK operations - - Optional: If --configure is also specified, it will automatically launch - the configuration wizard after successful login. - - This is the recommended way to authenticate for first-time users. - """ - sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL - api_key, user_data, api_token = authenticate_with_sso_and_api_key_request(sso_url) - - if api_token: - # Save the general token for future use - os.environ["COREBRAIN_API_TOKEN"] = api_token - - if api_key: - # Save the specific API key for future use - os.environ["COREBRAIN_API_KEY"] = api_key - print_colored("✅ API Key successfully saved. You can use the SDK now.", "green") - - # If configuration was also requested, continue with the process - if args.configure: - api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL - configure_sdk(api_token, api_key, api_url, sso_url, user_data) - - return 0 - else: - print_colored("❌ Could not obtain an API Key via SSO.", "red") - if api_token: - print_colored("A general API token was obtained, but not a specific API Key.", "yellow") - print_colored("You can create an API Key in the Corebrain dashboard.", "yellow") - return 1 - - if args.woami: """ @@ -969,7 +924,7 @@ def check_library(library_name, min_version): """ try: #downloading user data - sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + sso_url = os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL token_arg = args.api_key if args.api_key else args.token #using saved data about user @@ -987,81 +942,6 @@ def check_library(library_name, min_version): except Exception as e: print_colored(f"❌ Error when downloading data about user {str(e)}", "red") return 1 - - - - - if args.test_connection: - """ - Test the connection to the Corebrain API using the provided credentials. - - This command verifies that the SDK can successfully connect to and authenticate - with the Corebrain API server. It's useful for diagnosing connectivity issues, - credential problems, or API server availability. - - The test process: - 1. Retrieves API credentials from various sources - 2. Attempts to connect to the specified API endpoint - 3. Performs authentication verification - 4. Reports connection status and any errors - - Usage: corebrain --test-connection [--token ] [--api-url ] [--sso-url ] - - Credential sources (in order of priority): - 1. Token provided via --token argument - 2. COREBRAIN_API_KEY environment variable - 3. COREBRAIN_API_TOKEN environment variable - 4. SSO authentication (if no credentials found) - - What it tests: - - Network connectivity to API server - - API server availability and responsiveness - - Credential validity and authentication - - SSL/TLS connection (for HTTPS endpoints) - - Success indicators: - - ✅ Successfully connected to Corebrain API - - Failure indicators: - - Connection timeouts or network errors - - Invalid or expired credentials - - API server errors or maintenance - - Use cases: - - Troubleshoot connection issues - - Verify API credentials before starting work - - Check API server status - - Validate network connectivity in restricted environments - """ - # Test connection to the Corebrain API - api_url = args.api_url or os.environ.get("COREBRAIN_API_URL", DEFAULT_API_URL) - sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL", DEFAULT_SSO_URL) - - try: - # Retrieve API credentials - api_key, user_data, api_token = get_api_credential(args.token, sso_url) - except Exception as e: - print_colored(f"Error while retrieving API credentials: {e}", "red") - return 1 - - if not api_key: - print_colored( - "Error: An API key is required. You can generate one at dashboard.corebrain.com.", - "red" - ) - return 1 - - try: - # Test the connection - from corebrain.db.schema_file import test_connection - test_connection(api_key, api_url) - print_colored("Successfully connected to Corebrain API.", "green") - except Exception as e: - print_colored(f"Failed to connect to Corebrain API: {e}", "red") - return 1 - - - if args.gui: """ @@ -1179,16 +1059,7 @@ def run_in_background_silent(cmd, cwd): url = "http://localhost:5173/" print_colored(f"GUI: {url}", "cyan") webbrowser.open(url) - - - - - - - - - - + else: # If no option was specified, show help @@ -1200,4 +1071,4 @@ def run_in_background_silent(cmd, cwd): print_colored(f"Error: {str(e)}", "red") import traceback traceback.print_exc() - return 1 \ No newline at end of file + return 1 From 671dfe2b5e4a68e5108555b62a65c596dda2d0cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Ayuso?= Date: Mon, 26 May 2025 16:48:41 +0200 Subject: [PATCH 61/81] People working on list configs. ** Pending pull request. This command is not working yet; --gui has some errors. --- corebrain/cli/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index 674d1f4..5a0dc10 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -84,7 +84,7 @@ def authentication_with_api_key_return(): parser.add_argument("--version", action="store_true", help="Show SDK version") parser.add_argument("--check-status",action="store_true",help="Checks status of task") parser.add_argument("--authentication", action="store_true", help="Authenticate with SSO") - parser.add_argument("--test-auth", action="store_true", help="Test SSO authentication system") + parser.add_argument("--test-auth", action="store_true", help="Test SSO authentication system") # Is this command really useful? # Arguments to use the SDK parser.add_argument("--create-user", action="store_true", help="Create an user and API Key by default") From c6a632aaf65dedae3ac941a53e15c577818e2af0 Mon Sep 17 00:00:00 2001 From: BartekPachniak Date: Mon, 26 May 2025 17:35:38 +0200 Subject: [PATCH 62/81] Remove commands and add new Removed old unnecessary commands and added new ones, also added descriptions what the function does --- .../csharp/CorebrainCS/CorebrainCS.cs | 290 +++++------------- 1 file changed, 80 insertions(+), 210 deletions(-) diff --git a/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs b/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs index 4b1bb8d..622e3b3 100644 --- a/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs +++ b/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs @@ -2,6 +2,7 @@ using System; using System.Diagnostics; +using System.Collections.Generic; /// /// Creates the main corebrain interface. @@ -9,267 +10,136 @@ /// Path to the python which works with the corebrain cli, for example if you create the ./.venv you pass the path to the ./.venv python executable /// Path to the corebrain cli script, if you installed it globally you just pass the `corebrain` path /// -public class CorebrainCS(string pythonPath = "python", string scriptPath = "corebrain", bool verbose = false) { +public class CorebrainCS(string pythonPath = "python", string scriptPath = "corebrain", bool verbose = false) +{ private readonly string _pythonPath = Path.GetFullPath(pythonPath); private readonly string _scriptPath = Path.GetFullPath(scriptPath); private readonly bool _verbose = verbose; - public string Help() { - return ExecuteCommand("--help"); - } - - public string Version() { - return ExecuteCommand("--version"); - } - - public string Configure() { - return ExecuteCommand("--configure"); - } + /// Shows help message with all available commands - public string ListConfigs() { - return ExecuteCommand("--list-configs"); - } - - public string RemoveConfig() { - return ExecuteCommand("--remove-config"); - } - - public string ShowSchema() { - return ExecuteCommand("--show-schema"); + public string Help() + { + return ExecuteCommand("--help"); } - public string ExtractSchema() { - return ExecuteCommand("--extract-schema"); - } - public string ExtractSchemaToDefaultFile() { - return ExecuteCommand("--extract-schema --output-file test"); - } + /// Shows the current version of the Corebrain SDK - public string ConfigID() { - return ExecuteCommand("--extract-schema --config-id config"); + public string Version() + { + return ExecuteCommand("--version"); } - public string SetToken(string token) { - return ExecuteCommand($"--token {token}"); - } + /// Checks system status including: + /// - API Server status + /// - Redis status + /// - SSO Server status + /// - MongoDB status + /// - Required libraries installation - public string ApiKey(string apikey) { - return ExecuteCommand($"--api-key {apikey}"); + public string CheckStatus() + { + return ExecuteCommand("--check-status"); } - public string ApiUrl(string apiurl) { - if (string.IsNullOrWhiteSpace(apiurl)) { - throw new ArgumentException("API URL cannot be empty or whitespace", nameof(apiurl)); - } - - if (!Uri.TryCreate(apiurl, UriKind.Absolute, out var uriResult) || - (uriResult.Scheme != Uri.UriSchemeHttp && uriResult.Scheme != Uri.UriSchemeHttps)) { - throw new ArgumentException("Invalid API URL format. Must be a valid HTTP/HTTPS URL", nameof(apiurl)); - } - var escapedUrl = apiurl.Replace("\"", "\\\""); - return ExecuteCommand($"--api-url \"{escapedUrl}\""); - } - public string SsoUrl(string ssoUrl) { - if (string.IsNullOrWhiteSpace(ssoUrl)) { - throw new ArgumentException("SSO URL cannot be empty or whitespace", nameof(ssoUrl)); - } + /// Checks system status with optional API URL and token parameters - if (!Uri.TryCreate(ssoUrl, UriKind.Absolute, out var uriResult) || - (uriResult.Scheme != Uri.UriSchemeHttp && uriResult.Scheme != Uri.UriSchemeHttps)) { - throw new ArgumentException("Invalid SSO URL format. Must be a valid HTTP/HTTPS URL", nameof(ssoUrl)); - } + public string CheckStatus(string? apiUrl = null, string? token = null) + { + var args = new List { "--check-status" }; - var escapedUrl = ssoUrl.Replace("\"", "\\\""); - return ExecuteCommand($"--sso-url \"{escapedUrl}\""); - } - public string Login(string username, string password){ - if (string.IsNullOrWhiteSpace(username)){ - throw new ArgumentException("Username cannot be empty or whitespace", nameof(username)); - } + if (!string.IsNullOrEmpty(apiUrl)) + { + if (!Uri.IsWellFormedUriString(apiUrl, UriKind.Absolute)) + throw new ArgumentException("Invalid API URL format", nameof(apiUrl)); - if (string.IsNullOrWhiteSpace(password)){ - throw new ArgumentException("Password cannot be empty or whitespace", nameof(password)); + args.Add($"--api-url \"{apiUrl}\""); } - var escapedUsername = username.Replace("\"", "\\\""); - var escapedPassword = password.Replace("\"", "\\\""); - - return ExecuteCommand($"--login --username \"{escapedUsername}\" --password \"{escapedPassword}\""); - } - - public string LoginWithToken(string token) { - if (string.IsNullOrWhiteSpace(token)) { - throw new ArgumentException("Token cannot be empty or whitespace", nameof(token)); - } - - var escapedToken = token.Replace("\"", "\\\""); - return ExecuteCommand($"--login --token \"{escapedToken}\""); - } - - //When youre logged in use this function - public string TestAuth() { - return ExecuteCommand("--test-auth"); - } - - //Without beeing logged - public string TestAuth(string? apiUrl = null, string? token = null) { - var args = new List { "--test-auth" }; - - if (!string.IsNullOrEmpty(apiUrl)) { - if (!Uri.IsWellFormedUriString(apiUrl, UriKind.Absolute)) - throw new ArgumentException("Invalid API URL format", nameof(apiUrl)); - - args.Add($"--api-url \"{apiUrl}\""); - } - if (!string.IsNullOrEmpty(token)) - args.Add($"--token \"{token}\""); + args.Add($"--token \"{token}\""); return ExecuteCommand(string.Join(" ", args)); } -public string WoAmI() { - return ExecuteCommand("--woami"); -} - -public string CheckStatus() { - return ExecuteCommand("--check-status"); -} + /// Authenticates with SSO using username and password -public string CheckStatus(string? apiUrl = null, string? token = null) { - var args = new List { "--check-status" }; - - if (!string.IsNullOrEmpty(apiUrl)) { - if (!Uri.IsWellFormedUriString(apiUrl, UriKind.Absolute)) - throw new ArgumentException("Invalid API URL format", nameof(apiUrl)); - - args.Add($"--api-url \"{apiUrl}\""); + public string Authentication(string username, string password) + { + if (string.IsNullOrWhiteSpace(username)) + { + throw new ArgumentException("Username cannot be empty or whitespace", nameof(username)); } - - if (!string.IsNullOrEmpty(token)) - args.Add($"--token \"{token}\""); - return ExecuteCommand(string.Join(" ", args)); -} - -public string TaskStatus(string taskId) { - if (string.IsNullOrWhiteSpace(taskId)) { - throw new ArgumentException("Task ID cannot be empty", nameof(taskId)); + if (string.IsNullOrWhiteSpace(password)) + { + throw new ArgumentException("Password cannot be empty or whitespace", nameof(password)); } - return ExecuteCommand($"--task-id {taskId}"); -} + var escapedUsername = username.Replace("\"", "\\\""); + var escapedPassword = password.Replace("\"", "\\\""); -public string TaskStatus(string taskId, string? apiUrl = null, string? token = null) { - if (string.IsNullOrWhiteSpace(taskId)) { - throw new ArgumentException("Task ID cannot be empty", nameof(taskId)); - } + return ExecuteCommand($"--authentication --username \"{escapedUsername}\" --password \"{escapedPassword}\""); + } - var args = new List { $"--task-id {taskId}" }; + /// Authenticates with SSO using a token - if (!string.IsNullOrEmpty(apiUrl)) { - if (!Uri.IsWellFormedUriString(apiUrl, UriKind.Absolute)) - throw new ArgumentException("Invalid API URL format", nameof(apiUrl)); - - args.Add($"--api-url \"{apiUrl}\""); + public string AuthenticationWithToken(string token) + { + if (string.IsNullOrWhiteSpace(token)) + { + throw new ArgumentException("Token cannot be empty or whitespace", nameof(token)); } - if (!string.IsNullOrEmpty(token)) - args.Add($"--token \"{token}\""); - - return ExecuteCommand(string.Join(" ", args)); -} - -public string ValidateConfig() { - return ExecuteCommand("--validate-config"); -} + var escapedToken = token.Replace("\"", "\\\""); + return ExecuteCommand($"--authentication --token \"{escapedToken}\""); + } -public string ValidateConfig(string configFilePath) { - if (string.IsNullOrWhiteSpace(configFilePath)) { - throw new ArgumentException("Config file path cannot be empty", nameof(configFilePath)); - } - if (!File.Exists(configFilePath)) { - throw new FileNotFoundException("Config file not found", configFilePath); - } + /// Creates a new user account and generates an associated API Key - return ExecuteCommand($"--validate-config \"{configFilePath}\""); -} + public string CreateUser() + { + return ExecuteCommand("--create-user"); + } - public string ValidateConfig(string? apiUrl = null, string? token = null) { - var args = new List { "--validate-config" }; - if (!string.IsNullOrEmpty(apiUrl)) { - if (!Uri.IsWellFormedUriString(apiUrl, UriKind.Absolute)) - throw new ArgumentException("Invalid API URL format", nameof(apiUrl)); + /// Launches the configuration wizard for setting up database connections - args.Add($"--api-url \"{apiUrl}\""); - } + public string Configure() + { + return ExecuteCommand("--configure"); + } - if (!string.IsNullOrEmpty(token)) - args.Add($"--token \"{token}\""); + /// Lists all available database configurations - return ExecuteCommand(string.Join(" ", args)); + public string ListConfigs() + { + return ExecuteCommand("--list-configs"); } -public string TestConnection() { - return ExecuteCommand("--test-connection"); -} - public string TestConnection(string? apiUrl = null, string? token = null, bool fullDiagnostics = false) + /// Displays the database schema for a configured database + public string ShowSchema() { - var args = new List { "--test-connection" }; - - if (!string.IsNullOrEmpty(apiUrl)) - { - if (!Uri.IsWellFormedUriString(apiUrl, UriKind.Absolute)) - throw new ArgumentException("Invalid API URL format", nameof(apiUrl)); - - args.Add($"--api-url \"{apiUrl}\""); - } - - if (!string.IsNullOrEmpty(token)) - args.Add($"--token \"{token}\""); - - if (fullDiagnostics) - args.Add("--full"); - - return ExecuteCommand(string.Join(" ", args)); + return ExecuteCommand("--show-schema"); } -public string ExportConfig() { - return ExecuteCommand("--export-config"); -} - -public string ExportConfig(string outputDirectory, string? configId = null, bool overwrite = false) { - if (string.IsNullOrWhiteSpace(outputDirectory)) { - throw new ArgumentException("Output directory cannot be empty", nameof(outputDirectory)); - } - - if (!Directory.Exists(outputDirectory)) { - throw new DirectoryNotFoundException($"Directory not found: {outputDirectory}"); - } - - var args = new List { "--export-config" }; - - args.Add($"--output \"{outputDirectory}\""); - - if (!string.IsNullOrEmpty(configId)) { - args.Add($"--config-id \"{configId}\""); - } - - if (overwrite) { - args.Add("--overwrite"); - } - - - return ExecuteCommand(string.Join(" ", args)); -} + /// Displays information about the currently authenticated user + public string WhoAmI() + { + return ExecuteCommand("--woami"); + } + /// Launches the web-based graphical user interface + public string Gui() + { + return ExecuteCommand("--gui"); + } - public string ExecuteCommand(string arguments) + private string ExecuteCommand(string arguments) { if (_verbose) { From 4048444fdab3faf41fdc07d090aa2acb4a4f820c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Ayuso?= Date: Mon, 26 May 2025 17:36:21 +0200 Subject: [PATCH 63/81] mongodb.py connector moved into NoSQL folder inside of connectors --- corebrain/db/connectors/{subconnectors/nosql => NoSQL}/mongodb.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename corebrain/db/connectors/{subconnectors/nosql => NoSQL}/mongodb.py (100%) diff --git a/corebrain/db/connectors/subconnectors/nosql/mongodb.py b/corebrain/db/connectors/NoSQL/mongodb.py similarity index 100% rename from corebrain/db/connectors/subconnectors/nosql/mongodb.py rename to corebrain/db/connectors/NoSQL/mongodb.py From 3ec27ac1aa129919408be341b9ed2d554b56ae21 Mon Sep 17 00:00:00 2001 From: Albert Stankowski Date: Mon, 26 May 2025 16:47:42 +0100 Subject: [PATCH 64/81] Add translations to commands --- corebrain/db/connectors/mongodb.py | 114 +++++++++++++------------- corebrain/db/connectors/sql.py | 126 ++++++++++++++--------------- corebrain/db/interface.py | 2 +- corebrain/db/schema/extractor.py | 32 ++++---- corebrain/lib/sso/auth.py | 18 ++--- corebrain/lib/sso/client.py | 2 +- corebrain/network/client.py | 2 +- corebrain/utils/encrypter.py | 10 +-- corebrain/utils/logging.py | 54 ++++++------- corebrain/utils/serializer.py | 12 +-- 10 files changed, 186 insertions(+), 186 deletions(-) diff --git a/corebrain/db/connectors/mongodb.py b/corebrain/db/connectors/mongodb.py index 1b98575..815902f 100644 --- a/corebrain/db/connectors/mongodb.py +++ b/corebrain/db/connectors/mongodb.py @@ -31,7 +31,7 @@ def __init__(self, config: Dict[str, Any]): self.client = None self.db = None self.config = config - self.connection_timeout = 30 # segundos + self.connection_timeout = 30 # seconds if not PYMONGO_AVAILABLE: print("Advertencia: pymongo no está instalado. Instálalo con 'pip install pymongo'") @@ -49,24 +49,24 @@ def connect(self) -> bool: try: start_time = time.time() - # Construir los parámetros de conexión + # Build the connection parameters if "connection_string" in self.config: connection_string = self.config["connection_string"] - # Añadir timeout a la cadena de conexión si no está presente + # Add timeout to the connection string if not present if "connectTimeoutMS=" not in connection_string: if "?" in connection_string: - connection_string += "&connectTimeoutMS=10000" # 10 segundos + connection_string += "&connectTimeoutMS=10000" # 10 seconds else: connection_string += "?connectTimeoutMS=10000" - # Crear cliente MongoDB con la cadena de conexión + # Create MongoDB client with connection string self.client = pymongo.MongoClient(connection_string) else: - # Diccionario de parámetros para MongoClient + # Parameter dictionary for MongoClient mongo_params = { "host": self.config.get("host", "localhost"), "port": int(self.config.get("port", 27017)), - "connectTimeoutMS": 10000, # 10 segundos + "connectTimeoutMS": 10000, # 10 seconds "serverSelectionTimeoutMS": 10000 } @@ -76,48 +76,48 @@ def connect(self) -> bool: if self.config.get("password"): mongo_params["password"] = self.config.get("password") - # Opcionalmente añadir opciones de autenticación + # Optionally add authentication options if self.config.get("auth_source"): mongo_params["authSource"] = self.config.get("auth_source") if self.config.get("auth_mechanism"): mongo_params["authMechanism"] = self.config.get("auth_mechanism") - # Crear cliente MongoDB con parámetros + # Create MongoDB client with parameters self.client = pymongo.MongoClient(**mongo_params) - # Verificar que la conexión funciona + # Verify that the connection works self.client.admin.command('ping') - # Seleccionar la base de datos + # Select the database db_name = self.config.get("database", "") if not db_name: - # Si no hay base de datos especificada, listar las disponibles + # If no database is specified, list the available ones db_names = self.client.list_database_names() if not db_names: raise ValueError("No se encontraron bases de datos disponibles") - # Seleccionar la primera que no sea de sistema + # Select the first non-system one system_dbs = ["admin", "local", "config"] for name in db_names: if name not in system_dbs: db_name = name break - # Si no encontramos ninguna que no sea de sistema, usar la primera + # If we don't find any non-system ones, use the first one if not db_name: db_name = db_names[0] print(f"No se especificó base de datos. Usando '{db_name}'") - # Guardar la referencia a la base de datos + # Save the reference to the database self.db = self.client[db_name] return True except (ConnectionFailure, ServerSelectionTimeoutError) as e: - # Si es un error de timeout, reintentar + # If it is a timeout error, retry if time.time() - start_time < self.connection_timeout: print(f"Timeout al conectar a MongoDB: {str(e)}. Reintentando...") - time.sleep(2) # Esperar antes de reintentar + time.sleep(2) # Wait before retrying return self.connect() else: print(f"Error de conexión a MongoDB después de {self.connection_timeout}s: {str(e)}") @@ -141,64 +141,64 @@ def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] Returns: Dictionary with the database schema """ - # Asegurar que estamos conectados + # Ensure we are connected if not self.client and not self.connect(): return {"type": "mongodb", "tables": {}, "tables_list": []} - # Inicializar el esquema + # Initialize the schema schema = { "type": "mongodb", "database": self.db.name, - "tables": {} # En MongoDB, las "tablas" son colecciones + "tables": {} # In MongoDB, "tables" are collections } try: - # Obtener la lista de colecciones + # Get the list of collections collections = self.db.list_collection_names() # Limitar colecciones si es necesario if collection_limit is not None and collection_limit > 0: collections = collections[:collection_limit] - # Procesar cada colección + # Process each collection total_collections = len(collections) for i, collection_name in enumerate(collections): - # Reportar progreso si hay callback + # Report progress if there is a callback if progress_callback: progress_callback(i, total_collections, f"Procesando colección {collection_name}") collection = self.db[collection_name] try: - # Contar documentos + # Count documents doc_count = collection.count_documents({}) if doc_count > 0: - # Obtener muestra de documentos + # Get sample documents sample_docs = list(collection.find().limit(sample_limit)) - # Extraer campos y sus tipos + # Extract fields and their types fields = {} for doc in sample_docs: self._extract_document_fields(doc, fields) - # Convertir a formato esperado + # Convert to expected format formatted_fields = [{"name": field, "type": type_name} for field, type_name in fields.items()] - # Procesar documentos para sample_data + # Process documents for sample_data sample_data = [] for doc in sample_docs: processed_doc = self._process_document_for_serialization(doc) sample_data.append(processed_doc) - # Guardar en el esquema + # Save to outline schema["tables"][collection_name] = { "fields": formatted_fields, "sample_data": sample_data, "count": doc_count } else: - # Colección vacía + # Empty collection schema["tables"][collection_name] = { "fields": [], "sample_data": [], @@ -213,14 +213,14 @@ def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] "error": str(e) } - # Crear la lista de tablas/colecciones para compatibilidad + # Create the list of tables/collections for compatibility table_list = [] for collection_name, collection_info in schema["tables"].items(): table_data = {"name": collection_name} table_data.update(collection_info) table_list.append(table_data) - # Guardar también la lista de tablas para compatibilidad + # Also save the list of tables for compatibility schema["tables_list"] = table_list return schema @@ -245,30 +245,30 @@ def _extract_document_fields(self, doc: Dict[str, Any], fields: Dict[str, str], return for field, value in doc.items(): - # Para _id y otros campos especiales + # For _id and other special fields if field == "_id": field_type = "ObjectId" elif isinstance(value, dict): if current_depth < max_depth - 1: - # Recursión para campos anidados + # Recursion for nested fields self._extract_document_fields(value, fields, f"{prefix}{field}.", max_depth, current_depth + 1) field_type = "object" elif isinstance(value, list): if value and current_depth < max_depth - 1: - # Si tenemos elementos en la lista, analizar el primero + # If we have elements in the list, analyze the first one if isinstance(value[0], dict): self._extract_document_fields(value[0], fields, f"{prefix}{field}[].", max_depth, current_depth + 1) else: - # Para listas de tipos primitivos + # For lists of primitive types field_type = f"array<{type(value[0]).__name__}>" else: field_type = "array" else: field_type = type(value).__name__ - # Guardar el tipo del campo actual + # Save the current field type field_key = f"{prefix}{field}" if field_key not in fields: fields[field_key] = field_type @@ -285,13 +285,13 @@ def _process_document_for_serialization(self, doc: Dict[str, Any]) -> Dict[str, """ processed_doc = {} for field, value in doc.items(): - # Convertir ObjectId a string + # Convert ObjectId to string if field == "_id": processed_doc[field] = str(value) - # Manejar objetos anidados + # Handling nested objects elif isinstance(value, dict): processed_doc[field] = self._process_document_for_serialization(value) - # Manejar arrays + # Handling arrays elif isinstance(value, list): processed_items = [] for item in value: @@ -302,10 +302,10 @@ def _process_document_for_serialization(self, doc: Dict[str, Any]) -> Dict[str, else: processed_items.append(item) processed_doc[field] = processed_items - # Convertir fechas a ISO + # Convert dates to ISO elif hasattr(value, 'isoformat'): processed_doc[field] = value.isoformat() - # Otros tipos de datos + # Other types of data else: processed_doc[field] = value @@ -325,22 +325,22 @@ def execute_query(self, query: str) -> List[Dict[str, Any]]: raise ConnectionError("No se pudo establecer conexión con MongoDB") try: - # Determinar si la consulta es un string JSON o una consulta en otro formato + # Determine whether the query is a JSON string or a query in another format filter_dict, projection, collection_name, limit = self._parse_query(query) - # Obtener la colección + # Get the collection if not collection_name: raise ValueError("No se especificó el nombre de la colección en la consulta") collection = self.db[collection_name] - # Ejecutar la consulta + # Run the query if projection: cursor = collection.find(filter_dict, projection).limit(limit or 100) else: cursor = collection.find(filter_dict).limit(limit or 100) - # Convertir los resultados a formato serializable + # Convert results to serializable format results = [] for doc in cursor: processed_doc = self._process_document_for_serialization(doc) @@ -349,13 +349,13 @@ def execute_query(self, query: str) -> List[Dict[str, Any]]: return results except Exception as e: - # Intentar reconectar y reintentar una vez + # Try to reconnect and try again once try: self.close() if self.connect(): print("Reconectando y reintentando consulta...") - # Reintentar la consulta + # Retry the query filter_dict, projection, collection_name, limit = self._parse_query(query) collection = self.db[collection_name] @@ -371,10 +371,10 @@ def execute_query(self, query: str) -> List[Dict[str, Any]]: return results except Exception as retry_error: - # Si falla el reintento, propagar el error original + # If the retry fails, propagate the original error raise Exception(f"Error al ejecutar consulta MongoDB: {str(e)}") - # Si llegamos aquí, ha habido un error en el reintento + # If we get here, there was an error in the retry. raise Exception(f"Error al ejecutar consulta MongoDB (después de reconexión): {str(e)}") def _parse_query(self, query: str) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]], str, Optional[int]]: @@ -387,11 +387,11 @@ def _parse_query(self, query: str) -> Tuple[Dict[str, Any], Optional[Dict[str, A Returns: Tuple with (filter, projection, collection name, limit) """ - # Intentar parsear como JSON + # Trying to parse as JSON try: query_dict = json.loads(query) - # Extraer componentes de la consulta + # Extract components from the query filter_dict = query_dict.get("filter", {}) projection = query_dict.get("projection") collection_name = query_dict.get("collection") @@ -400,22 +400,22 @@ def _parse_query(self, query: str) -> Tuple[Dict[str, Any], Optional[Dict[str, A return filter_dict, projection, collection_name, limit except json.JSONDecodeError: - # Si no es JSON válido, intentar parsear el formato de consulta alternativo + # If not valid JSON, attempt to parse the alternative query format collection_match = re.search(r'from\s+([a-zA-Z0-9_]+)', query, re.IGNORECASE) collection_name = collection_match.group(1) if collection_match else None - # Intentar extraer filtros + # Try to extract filters filter_match = re.search(r'where\s+(.+?)(?:limit|$)', query, re.IGNORECASE | re.DOTALL) filter_str = filter_match.group(1).strip() if filter_match else "{}" - # Intentar parsear los filtros como JSON + # Try to parse the filters as JSON try: filter_dict = json.loads(filter_str) except json.JSONDecodeError: - # Si no se puede parsear, usar filtro vacío + # If parsing is not possible, use empty filter filter_dict = {} - # Extraer límite si existe + # Extract limit if it exists limit_match = re.search(r'limit\s+(\d+)', query, re.IGNORECASE) limit = int(limit_match.group(1)) if limit_match else None diff --git a/corebrain/db/connectors/sql.py b/corebrain/db/connectors/sql.py index 7f457a0..4b9bcab 100644 --- a/corebrain/db/connectors/sql.py +++ b/corebrain/db/connectors/sql.py @@ -33,7 +33,7 @@ def __init__(self, config: Dict[str, Any]): self.cursor = None self.engine = config.get("engine", "").lower() self.config = config - self.connection_timeout = 30 # segundos + self.connection_timeout = 30 # seconds def connect(self) -> bool: """ @@ -45,7 +45,7 @@ def connect(self) -> bool: try: start_time = time.time() - # Intentar la conexión con un límite de tiempo + # Attempt to connect with a time limit while time.time() - start_time < self.connection_timeout: try: if self.engine == "sqlite": @@ -54,7 +54,7 @@ def connect(self) -> bool: else: self.conn = sqlite3.connect(self.config.get("database", ""), timeout=10.0) - # Configurar para que devuelva filas como diccionarios + # Configure to return rows as dictionaries self.conn.row_factory = sqlite3.Row elif self.engine == "mysql": @@ -74,9 +74,9 @@ def connect(self) -> bool: ) elif self.engine == "postgresql": - # Determinar si usar cadena de conexión o parámetros + # Determine whether to use connection string or parameters if "connection_string" in self.config: - # Agregar timeout a la cadena de conexión si no está presente + # Add timeout to the connection string if not present conn_str = self.config["connection_string"] if "connect_timeout" not in conn_str: if "?" in conn_str: @@ -95,23 +95,23 @@ def connect(self) -> bool: connect_timeout=10 ) - # Si llegamos aquí, la conexión fue exitosa + # If we get here, the connection was successful. if self.conn: - # Verificar conexión con una consulta simple + # Check connection with a simple query cursor = self.conn.cursor() cursor.execute("SELECT 1") cursor.close() return True except (sqlite3.Error, mysql.connector.Error, psycopg2.Error) as e: - # Si el error no es de timeout, propagar la excepción + # If the error is not a timeout, propagate the exception if "timeout" not in str(e).lower() and "wait timeout" not in str(e).lower(): raise - # Si es un error de timeout, esperamos un poco y reintentamos + # If it is a timeout error, we wait a bit and try again. time.sleep(1.0) - # Si llegamos aquí, se agotó el tiempo de espera + # If we get here, the wait time is up. raise TimeoutError(f"Could not connect to the database in {self.connection_timeout} seconds") except Exception as e: @@ -138,11 +138,11 @@ def extract_schema(self, sample_limit: int = 5, table_limit: Optional[int] = Non Returns: Dictionary with the database schema """ - # Asegurar que estamos conectados + # Ensure we are connected if not self.conn and not self.connect(): return {"type": "sql", "tables": {}, "tables_list": []} - # Inicializar esquema + # Initialize schema schema = { "type": "sql", "engine": self.engine, @@ -150,7 +150,7 @@ def extract_schema(self, sample_limit: int = 5, table_limit: Optional[int] = Non "tables": {} } - # Seleccionar la función extractora según el motor + # Select the extractor function according to the motor if self.engine == "sqlite": return self._extract_sqlite_schema(sample_limit, table_limit, progress_callback) elif self.engine == "mysql": @@ -158,7 +158,7 @@ def extract_schema(self, sample_limit: int = 5, table_limit: Optional[int] = Non elif self.engine == "postgresql": return self._extract_postgresql_schema(sample_limit, table_limit, progress_callback) else: - return schema # Esquema vacío si no se reconoce el motor + return schema # Empty diagram if the engine is not recognized def execute_query(self, query: str) -> List[Dict[str, Any]]: """ @@ -174,7 +174,7 @@ def execute_query(self, query: str) -> List[Dict[str, Any]]: raise ConnectionError("No se pudo establecer conexión con la base de datos") try: - # Ejecutar query según el motor + # Execute query according to the engine if self.engine == "sqlite": return self._execute_sqlite_query(query) elif self.engine == "mysql": @@ -185,7 +185,7 @@ def execute_query(self, query: str) -> List[Dict[str, Any]]: raise ValueError(f"Database engine not supported: {self.engine}") except Exception as e: - # Intentar reconectar y reintentar una vez + # Try to reconnect and try again once try: self.close() if self.connect(): @@ -199,10 +199,10 @@ def execute_query(self, query: str) -> List[Dict[str, Any]]: return self._execute_postgresql_query(query) except Exception as retry_error: - # Si falla el reintento, propagar el error original + # If the retry fails, propagate the original error raise Exception(f"Error executing query: {str(e)}") - # Si llegamos aquí sin retornar, ha habido un error en el reintento + # If we get here without returning, there was an error in the retry. raise Exception(f"Error executing query (after reconnection): {str(e)}") def _execute_sqlite_query(self, query: str) -> List[Dict[str, Any]]: @@ -210,7 +210,7 @@ def _execute_sqlite_query(self, query: str) -> List[Dict[str, Any]]: cursor = self.conn.cursor() cursor.execute(query) - # Convertir filas a diccionarios + # Convert rows to dictionaries columns = [desc[0] for desc in cursor.description] if cursor.description else [] rows = cursor.fetchall() result = [] @@ -262,44 +262,44 @@ def _extract_sqlite_schema(self, sample_limit: int, table_limit: Optional[int], try: cursor = self.conn.cursor() - # Obtener la lista de tablas + # Get the list of tables cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;") tables = [row[0] for row in cursor.fetchall()] - # Limitar tablas si es necesario + # Limit tables if necessary if table_limit is not None and table_limit > 0: tables = tables[:table_limit] - # Procesar cada tabla + # Process each table total_tables = len(tables) for i, table_name in enumerate(tables): - # Reportar progreso si hay callback + # Report progress if there is a callback if progress_callback: progress_callback(i, total_tables, f"Processing table {table_name}") - # Extraer información de columnas + # Extract information from columns cursor.execute(f"PRAGMA table_info({table_name});") columns = [{"name": col[1], "type": col[2]} for col in cursor.fetchall()] - # Guardar información básica de la tabla + # Save basic table information schema["tables"][table_name] = { "columns": columns, "sample_data": [] } - # Obtener muestra de datos + # Get data sample try: cursor.execute(f"SELECT * FROM {table_name} LIMIT {sample_limit};") - # Obtener nombres de columnas + # Get column names col_names = [desc[0] for desc in cursor.description] - # Procesar las filas + # Process the rows sample_data = [] for row in cursor.fetchall(): row_dict = {} for j, value in enumerate(row): - # Convertir a string los valores que no son serializable directamente + # Convert values ​​that are not directly serializable to string if isinstance(value, (bytes, bytearray)): row_dict[col_names[j]] = f"" else: @@ -316,7 +316,7 @@ def _extract_sqlite_schema(self, sample_limit: int, table_limit: Optional[int], except Exception as e: print(f"Error extracting SQLite schema: {str(e)}") # TODO: Translate to English - # Crear la lista de tablas para compatibilidad + # Create the list of tables for compatibility table_list = [] for table_name, table_info in schema["tables"].items(): table_data = {"name": table_name} @@ -348,55 +348,55 @@ def _extract_mysql_schema(self, sample_limit: int, table_limit: Optional[int], p try: cursor = self.conn.cursor(dictionary=True) - # Obtener la lista de tablas + # Get the list of tables cursor.execute("SHOW TABLES;") tables_result = cursor.fetchall() tables = [] - # Extraer nombres de tablas (el formato puede variar según versión) + # Extract table names (format may vary depending on version) for row in tables_result: - if len(row) == 1: # Si es una lista simple + if len(row) == 1: # If it is a simple list tables.extend(row.values()) - else: # Si tiene estructura compleja + else: # If it has a complex structure for value in row.values(): if isinstance(value, str): tables.append(value) break - # Limitar tablas si es necesario + # Limit tables if necessary if table_limit is not None and table_limit > 0: tables = tables[:table_limit] - # Procesar cada tabla + # Process each table total_tables = len(tables) for i, table_name in enumerate(tables): - # Reportar progreso si hay callback + # Report progress if there is a callback if progress_callback: progress_callback(i, total_tables, f"Processing table {table_name}") - # Extraer información de columnas + # Extract information from columns cursor.execute(f"DESCRIBE `{table_name}`;") columns = [{"name": col.get("Field"), "type": col.get("Type")} for col in cursor.fetchall()] - # Guardar información básica de la tabla + # Save basic table information schema["tables"][table_name] = { "columns": columns, "sample_data": [] } - # Obtener muestra de datos + # Get data sample try: cursor.execute(f"SELECT * FROM `{table_name}` LIMIT {sample_limit};") sample_data = cursor.fetchall() - # Procesar valores que no son JSON serializable + # Process values ​​that are not JSON serializable processed_samples = [] for row in sample_data: processed_row = {} for key, value in row.items(): if isinstance(value, (bytes, bytearray)): processed_row[key] = f"" - elif hasattr(value, 'isoformat'): # Para fechas y horas + elif hasattr(value, 'isoformat'): # For dates and times processed_row[key] = value.isoformat() else: processed_row[key] = value @@ -412,7 +412,7 @@ def _extract_mysql_schema(self, sample_limit: int, table_limit: Optional[int], p except Exception as e: print(f"Error extracting MySQL schema: {str(e)}") # TODO: Translate to English - # Crear la lista de tablas para compatibilidad + # Create the list of tables for compatibility table_list = [] for table_name, table_info in schema["tables"].items(): table_data = {"name": table_name} @@ -444,7 +444,7 @@ def _extract_postgresql_schema(self, sample_limit: int, table_limit: Optional[in try: cursor = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) - # Estrategia 1: Buscar en todos los esquemas accesibles + # Strategy 1: Search all accessible schemas cursor.execute(""" SELECT table_schema, table_name FROM information_schema.tables @@ -454,7 +454,7 @@ def _extract_postgresql_schema(self, sample_limit: int, table_limit: Optional[in """) tables = cursor.fetchall() - # Si no se encontraron tablas, intentar estrategia alternativa + # If no tables were found, try alternative strategy if not tables: cursor.execute(""" SELECT schemaname AS table_schema, tablename AS table_name @@ -464,7 +464,7 @@ def _extract_postgresql_schema(self, sample_limit: int, table_limit: Optional[in """) tables = cursor.fetchall() - # Si aún no hay tablas, intentar buscar en esquemas específicos + # If there are no tables yet, try searching in specific schemas if not tables: cursor.execute(""" SELECT DISTINCT table_schema @@ -473,7 +473,7 @@ def _extract_postgresql_schema(self, sample_limit: int, table_limit: Optional[in """) schemas = cursor.fetchall() - # Intentar con esquemas que no sean del sistema + # Try non-system schemes user_schemas = [s[0] for s in schemas if s[0] not in ('pg_catalog', 'information_schema')] for schema_name in user_schemas: cursor.execute(f""" @@ -486,21 +486,21 @@ def _extract_postgresql_schema(self, sample_limit: int, table_limit: Optional[in if schema_tables: tables.extend(schema_tables) - # Limitar tablas si es necesario + # Limit tables if necessary if table_limit is not None and table_limit > 0: tables = tables[:table_limit] - # Procesar cada tabla + # Process each table total_tables = len(tables) for i, (schema_name, table_name) in enumerate(tables): - # Reportar progreso si hay callback + # Report progress if there is a callback if progress_callback: progress_callback(i, total_tables, f"Procesando tabla {schema_name}.{table_name}") - # Determinar el nombre completo de la tabla + # Determine the full name of the table full_name = f"{schema_name}.{table_name}" if schema_name != 'public' else table_name - # Extraer información de columnas + # Extract information from columns cursor.execute(f""" SELECT column_name, data_type FROM information_schema.columns @@ -513,23 +513,23 @@ def _extract_postgresql_schema(self, sample_limit: int, table_limit: Optional[in columns = [{"name": col[0], "type": col[1]} for col in columns_data] schema["tables"][full_name] = {"columns": columns, "sample_data": []} - # Obtener muestra de datos + # Get data sample try: cursor.execute(f""" SELECT * FROM "{schema_name}"."{table_name}" LIMIT {sample_limit}; """) rows = cursor.fetchall() - # Obtener nombres de columnas + # Get column names col_names = [desc[0] for desc in cursor.description] - # Convertir filas a diccionarios + # Convert rows to dictionaries sample_data = [] for row in rows: row_dict = {} for j, value in enumerate(row): - # Convertir a formato serializable - if hasattr(value, 'isoformat'): # Para fechas y horas + # Convert to serializable format + if hasattr(value, 'isoformat'): # For dates and times row_dict[col_names[j]] = value.isoformat() elif isinstance(value, (bytes, bytearray)): row_dict[col_names[j]] = f"" @@ -542,7 +542,7 @@ def _extract_postgresql_schema(self, sample_limit: int, table_limit: Optional[in except Exception as e: print(f"Error getting sample data for table {full_name}: {str(e)}") # TODO: Translate to English else: - # Registrar la tabla aunque no tenga columnas + # Register the table even if it has no columns schema["tables"][full_name] = {"columns": [], "sample_data": []} cursor.close() @@ -550,17 +550,17 @@ def _extract_postgresql_schema(self, sample_limit: int, table_limit: Optional[in except Exception as e: print(f"Error extracting PostgreSQL schema: {str(e)}") # TODO: Translate to English - # Intento de recuperación para diagnosticar problemas + # Recovery attempt to diagnose problems try: - if self.conn and self.conn.closed == 0: # 0 = conexión abierta + if self.conn and self.conn.closed == 0: # 0 = open connection recovery_cursor = self.conn.cursor() - # Verificar versión + # Check version recovery_cursor.execute("SELECT version();") version = recovery_cursor.fetchone() print(f"PostgreSQL version: {version[0] if version else 'Unknown'}") - # Verificar permisos + # Check permissions recovery_cursor.execute(""" SELECT has_schema_privilege(current_user, 'public', 'USAGE') AS has_usage, has_schema_privilege(current_user, 'public', 'CREATE') AS has_create; @@ -573,7 +573,7 @@ def _extract_postgresql_schema(self, sample_limit: int, table_limit: Optional[in except Exception as diag_err: print(f"Error during diagnosis: {str(diag_err)}") # TODO: Translate to English - # Crear la lista de tablas para compatibilidad + # Create the list of tables for compatibility table_list = [] for table_name, table_info in schema["tables"].items(): table_data = {"name": table_name} diff --git a/corebrain/db/interface.py b/corebrain/db/interface.py index d8373ff..1043e59 100644 --- a/corebrain/db/interface.py +++ b/corebrain/db/interface.py @@ -29,7 +29,7 @@ def close(self, connection: Any) -> None: """Closes the connection.""" pass -# Posteriormente se podrían implementar conectores específicos: +# Specific connectors could be implemented later: # - SQLiteConnector # - MySQLConnector # - PostgresConnector diff --git a/corebrain/db/schema/extractor.py b/corebrain/db/schema/extractor.py index c361b83..1dff31c 100644 --- a/corebrain/db/schema/extractor.py +++ b/corebrain/db/schema/extractor.py @@ -1,4 +1,4 @@ -# db/schema/extractor.py (reemplaza la importación circular en db/schema.py) +# db/schema/extractor.py (replaces circular import in db/schema.py) """ Independent database schema extractor. @@ -30,35 +30,35 @@ def extract_db_schema(db_config: Dict[str, Any], client_factory: Optional[Callab } try: - # Si tenemos un cliente especializado, usarlo + # If we have a specialized client, use it if client_factory: - # La factoría crea un cliente y extrae el esquema + # The factory creates a client and extracts the schema client = client_factory(db_config) return client.extract_schema() - # Extracción directa sin usar cliente de Corebrain + # Direct extraction without using Corebrain client if db_type == "sql": - # Código para bases de datos SQL (sin dependencias circulares) + # Code for SQL databases (without circular dependencies) engine = db_config.get("engine", "").lower() if engine == "sqlite": - # Extraer esquema SQLite + # Extract SQLite schema import sqlite3 - # (implementación...) + # (implementation...) elif engine == "mysql": # Extraer esquema MySQL import mysql.connector - # (implementación...) + # (implementation...) elif engine == "postgresql": # Extraer esquema PostgreSQL import psycopg2 # (implementación...) elif db_type in ["nosql", "mongodb"]: - # Extraer esquema MongoDB + # Extract MongoDB schema import pymongo - # (implementación...) + # (implementation...) - # Convertir diccionario a lista para compatibilidad + # Convert dictionary to list for compatibility table_list = [] for table_name, table_info in schema["tables"].items(): table_data = {"name": table_name} @@ -82,10 +82,10 @@ def create_schema_from_corebrain() -> Callable: Function that extracts schema using Corebrain """ def extract_with_corebrain(db_config: Dict[str, Any]) -> Dict[str, Any]: - # Importar dinámicamente para evitar circular + # Import dynamically to avoid circularity from corebrain.core.client import Corebrain - # Crear cliente temporal solo para extraer el schema + # Create temporary client just to extract the schema try: client = Corebrain( api_token="temp_token", @@ -102,7 +102,7 @@ def extract_with_corebrain(db_config: Dict[str, Any]) -> Dict[str, Any]: return extract_with_corebrain -# Función pública expuesta +# Public function exposed def extract_schema(db_config: Dict[str, Any], use_corebrain: bool = False) -> Dict[str, Any]: """ Public function that decides how to extract the schema. @@ -115,9 +115,9 @@ def extract_schema(db_config: Dict[str, Any], use_corebrain: bool = False) -> Di Database schema """ if use_corebrain: - # Intentar usar Corebrain si se solicita + # Attempt to use Corebrain if requested factory = create_schema_from_corebrain() return extract_db_schema(db_config, client_factory=factory) else: - # Usar extracción directa sin dependencias circulares + # Use direct extraction without circular dependencies return extract_db_schema(db_config) \ No newline at end of file diff --git a/corebrain/lib/sso/auth.py b/corebrain/lib/sso/auth.py index 0f3b568..3e5dd37 100644 --- a/corebrain/lib/sso/auth.py +++ b/corebrain/lib/sso/auth.py @@ -26,11 +26,11 @@ def requires_auth(self, session_handler): """ def decorator(func): def wrapper(*args, **kwargs): - # Obtener la sesión actual usando el manejador proporcionado + # Get the current session using the provided handler session = session_handler() if 'user' not in session: - # Aquí retornamos información para que el framework redirija + # Here we return information for the framework to redirect return { 'authenticated': False, 'redirect_url': self.get_login_url() @@ -145,27 +145,27 @@ def handle_callback(self, code, session_handler, store_user_func=None): Returns: Redirect URL after processing the code """ - # Intercambiar código por token + # Exchange code for token token_data = self.exchange_code_for_token(code) if not token_data: - # Error al obtener el token + # Error getting token return self.get_login_url() - # Obtener información del usuario + # Get user information user_info = self.get_user_info(token_data.get('access_token')) if not user_info: - # Error al obtener información del usuario + # Error getting user information return self.get_login_url() - # Guardar información en la sesión + # Save information in the session session = session_handler() session['user'] = user_info session['token'] = token_data - # Si hay una función para almacenar el usuario, ejecutarla + # If there is a function to store the user, execute it if store_user_func and callable(store_user_func): store_user_func(user_info, token_data) - # Redirigir a la URL de éxito o a la URL guardada anteriormente + # Redirect to the success URL or previously saved URL next_url = session.pop('next_url', self.success_redirect) return next_url \ No newline at end of file diff --git a/corebrain/lib/sso/client.py b/corebrain/lib/sso/client.py index 94e2155..e3089e3 100644 --- a/corebrain/lib/sso/client.py +++ b/corebrain/lib/sso/client.py @@ -187,7 +187,7 @@ def logout(self, refresh_token: str, access_token: str) -> bool: if response.status_code != 200: raise Exception(f"Error al cerrar sesión: {response.text}") - # Limpiar cualquier token cacheado + # Clear any cached tokens if access_token in self._token_cache: del self._token_cache[access_token] diff --git a/corebrain/network/client.py b/corebrain/network/client.py index cc9c481..347b4b6 100644 --- a/corebrain/network/client.py +++ b/corebrain/network/client.py @@ -289,7 +289,7 @@ def request(self, method: str, endpoint: str, *, retries += 1 continue - # No más reintentos o error no recuperable + # No more retries or unrecoverable errors self.error_count += 1 elapsed = time.time() - start_time diff --git a/corebrain/utils/encrypter.py b/corebrain/utils/encrypter.py index 6cfbd3d..6fe2a1b 100644 --- a/corebrain/utils/encrypter.py +++ b/corebrain/utils/encrypter.py @@ -213,7 +213,7 @@ def decrypt_file(self, input_path: Union[str, Path], output_path: Optional[Union input_path = Path(input_path) if not output_path: - # Si termina en .enc, quitar esa extensión + # If it ends in .enc, remove that extension if input_path.suffix == '.enc': output_path = input_path.with_suffix('') else: @@ -245,17 +245,17 @@ def generate_key_file(key_path: Union[str, Path]) -> None: """ key_path = Path(key_path) - # Crear directorio padre si no existe + # Create parent directory if it does not exist key_path.parent.mkdir(parents=True, exist_ok=True) - # Generar clave + # Generate key key = Fernet.generate_key() - # Guardar clave + # Save key with open(key_path, 'wb') as f: f.write(key) - # Establecer permisos restrictivos + # Set restrictive permissions try: os.chmod(key_path, 0o600) except Exception as e: diff --git a/corebrain/utils/logging.py b/corebrain/utils/logging.py index 0ba559d..1f39f69 100644 --- a/corebrain/utils/logging.py +++ b/corebrain/utils/logging.py @@ -10,16 +10,16 @@ from pathlib import Path from typing import Optional, Any, Union -# Niveles de logging personalizados +# Custom logging levels VERBOSE = 15 # Entre DEBUG e INFO -# Configuración predeterminada +# Default settings DEFAULT_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' DEFAULT_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' DEFAULT_LEVEL = logging.INFO DEFAULT_LOG_DIR = Path.home() / ".corebrain" / "logs" -# Colores para logging en terminal +# Colors for logging in terminal LOG_COLORS = { "DEBUG": "\033[94m", # Azul "VERBOSE": "\033[96m", # Cian @@ -97,46 +97,46 @@ def setup_logger(name: str = "corebrain", Returns: Configured logger """ - # Registrar nivel personalizado VERBOSE + # Register custom level VERBOSE. if not hasattr(logging, 'VERBOSE'): logging.addLevelName(VERBOSE, 'VERBOSE') - # Registrar clase de logger personalizada + # Register custom logger class. logging.setLoggerClass(VerboseLogger) - # Obtener o crear logger + # Get or create logger. logger = logging.getLogger(name) - # Limpiar handlers existentes + # Clear existing handlers. for handler in logger.handlers[:]: logger.removeHandler(handler) - # Configurar nivel de logging + # Configure logging level. logger.setLevel(level) logger.propagate = propagate - # Formato predeterminado + # Default format. fmt = format_string or DEFAULT_FORMAT formatter = ColoredFormatter(fmt, use_colors=use_colors) - # Handler de consola + # Console handler console_handler = logging.StreamHandler() console_handler.setFormatter(formatter) logger.addHandler(console_handler) - # Handler de archivo si se proporciona ruta + # File handler if path is provided if file_path: - # Asegurar que el directorio exista + # Ensure that the directory exists file_path = Path(file_path) file_path.parent.mkdir(parents=True, exist_ok=True) file_handler = logging.FileHandler(file_path) - # Para archivos, usar formateador sin colores + # For files, use colorless formatter file_formatter = logging.Formatter(fmt) file_handler.setFormatter(file_formatter) logger.addHandler(file_handler) - # Mensajes de diagnóstico + # Diagnostic messages logger.debug(f"Logger '{name}' configurado con nivel {logging.getLevelName(level)}") if file_path: logger.debug(f"Logs escritos a {file_path}") @@ -156,19 +156,19 @@ def get_logger(name: str, level: Optional[int] = None) -> logging.Logger: """ logger = logging.getLogger(name) - # Si el logger no tiene handlers, configurarlo + # If the logger does not have handlers, configure it if not logger.handlers: - # Determinar si es un logger secundario + # Determine if it is a secondary logger if '.' in name: - # Es un sublogger, configurar para propagar a logger padre + # It is a sublogger, configure to propagate to parent logger logger.propagate = True if level is not None: logger.setLevel(level) else: - # Es un logger principal, configurar completamente + # It is a main logger, fully configure logger = setup_logger(name, level or DEFAULT_LEVEL) elif level is not None: - # Solo actualizar el nivel si se especifica + # Only update level if specified logger.setLevel(level) return logger @@ -189,23 +189,23 @@ def enable_file_logging(logger_name: str = "corebrain", """ logger = logging.getLogger(logger_name) - # Determinar la ruta del archivo de log + # Determine the path of the log file log_dir = Path(log_dir) if log_dir else DEFAULT_LOG_DIR log_dir.mkdir(parents=True, exist_ok=True) - # Generar nombre de archivo si no se proporciona + # Generate filename if not provided if not filename: timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') filename = f"{logger_name}_{timestamp}.log" file_path = log_dir / filename - # Verificar si ya hay un FileHandler + # Check if a FileHandler already exists for handler in logger.handlers: if isinstance(handler, logging.FileHandler): logger.removeHandler(handler) - # Agregar nuevo FileHandler + # Add new FileHandler file_handler = logging.FileHandler(file_path) formatter = logging.Formatter(DEFAULT_FORMAT) file_handler.setFormatter(formatter) @@ -223,21 +223,21 @@ def set_log_level(level: Union[int, str], level: Logging level (name or integer value) logger_name: Specific logger name (if None, affects all) """ - # Convertir nombre de nivel a valor si es necesario + # Convert level name to value if necessary if isinstance(level, str): level = getattr(logging, level.upper(), logging.INFO) if logger_name: - # Afectar solo al logger especificado + # Affect only the specified logger logger = logging.getLogger(logger_name) logger.setLevel(level) logger.info(f"Nivel de log cambiado a {logging.getLevelName(level)}") else: - # Afectar al logger raíz y a todos los loggers existentes + # Affect the root logger and all existing loggers root = logging.getLogger() root.setLevel(level) - # También afectar a loggers específicos del SDK + # Also affect SDK-specific loggers for name in logging.root.manager.loggerDict: if name.startswith("corebrain"): logging.getLogger(name).setLevel(level) \ No newline at end of file diff --git a/corebrain/utils/serializer.py b/corebrain/utils/serializer.py index c230c3e..d9eac5a 100644 --- a/corebrain/utils/serializer.py +++ b/corebrain/utils/serializer.py @@ -10,22 +10,22 @@ class JSONEncoder(json.JSONEncoder): """Custom JSON serializer for special types.""" def default(self, obj): - # Objetos datetime + # Datetime objects if isinstance(obj, (datetime, date, time)): return obj.isoformat() - # Objetos timedelta - elif hasattr(obj, 'total_seconds'): # Para objetos timedelta + # Timedelta objects + elif hasattr(obj, 'total_seconds'): # For timedelta objects return obj.total_seconds() - # ObjectId de MongoDB + # MongoDB ObjectId elif isinstance(obj, ObjectId): return str(obj) - # Bytes o bytearray + # Bytes or bytearray elif isinstance(obj, (bytes, bytearray)): return obj.hex() # Decimal elif isinstance(obj, Decimal): return float(obj) - # Otros tipos + # Other types return super().default(obj) def serialize_to_json(obj): From 1d73b714e463207ec44bc414efdf2ac23d5e782e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Ayuso?= Date: Mon, 26 May 2025 18:02:07 +0200 Subject: [PATCH 65/81] FIles translated to English. --- corebrain/corebrain/__init__.py | 5 - corebrain/corebrain/config/manager.py | 46 +-- corebrain/corebrain/core/query.py | 90 ++--- corebrain/corebrain/core/test_utils.py | 72 ++-- corebrain/corebrain/db/connectors/mongodb.py | 88 ++--- corebrain/corebrain/db/connectors/sql.py | 50 +-- corebrain/corebrain/db/schema/extractor.py | 42 +-- corebrain/corebrain/db/schema_file.py | 326 ++++++++++--------- corebrain/corebrain/lib/sso/auth.py | 28 +- corebrain/corebrain/lib/sso/client.py | 24 +- corebrain/corebrain/network/__init__.py | 1 - corebrain/corebrain/network/client.py | 104 +++--- corebrain/corebrain/sdk.py | 1 - corebrain/corebrain/services/schema.py | 3 +- corebrain/corebrain/utils/__init__.py | 6 +- corebrain/corebrain/utils/encrypter.py | 30 +- corebrain/corebrain/utils/logging.py | 56 ++-- corebrain/corebrain/utils/serializer.py | 10 +- corebrain/db/connectors/nosql.py | 2 +- corebrain/db/interface.py | 8 +- corebrain/db/schema_file.py | 2 - 21 files changed, 494 insertions(+), 500 deletions(-) diff --git a/corebrain/corebrain/__init__.py b/corebrain/corebrain/__init__.py index 6819487..151df33 100644 --- a/corebrain/corebrain/__init__.py +++ b/corebrain/corebrain/__init__.py @@ -7,16 +7,13 @@ import logging from typing import Dict, Any, List, Optional -# Configuración básica de logging logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) -# Importaciones seguras (sin dependencias circulares) from corebrain.db.engines import get_available_engines from corebrain.core.client import Corebrain from corebrain.config.manager import ConfigManager -# Exportación explícita de componentes públicos __all__ = [ 'init', 'extract_db_schema', @@ -27,7 +24,6 @@ '__version__' ] -# Variable de versión __version__ = "1.0.0" def init(api_key: str, config_id: str, skip_verification: bool = False) -> Corebrain: @@ -43,7 +39,6 @@ def init(api_key: str, config_id: str, skip_verification: bool = False) -> Coreb """ return Corebrain(api_key=api_key, config_id=config_id, skip_verification=skip_verification) -# Funciones de conveniencia a nivel de paquete def list_configurations(api_key: str) -> List[str]: """ Lists the available configurations for an API key. diff --git a/corebrain/corebrain/config/manager.py b/corebrain/corebrain/config/manager.py index 0fa9f65..3cc6dd6 100644 --- a/corebrain/corebrain/config/manager.py +++ b/corebrain/corebrain/config/manager.py @@ -55,7 +55,7 @@ def validate_config(config_id: str): print(f"❌ Configuration '{config_id}' not found.") return 1 -# Función para imprimir mensajes coloreados +# Function to print colored messages def _print_colored(message: str, color: str) -> None: """Simplified version of _print_colored that does not depend on cli.utils.""" colors = { @@ -86,11 +86,11 @@ def _ensure_config_dir(self) -> None: """Ensures that the configuration directory exists.""" try: self.CONFIG_DIR.mkdir(parents=True, exist_ok=True) - logger.debug(f"Directorio de configuración asegurado: {self.CONFIG_DIR}") - _print_colored(f"Directorio de configuración asegurado: {self.CONFIG_DIR}", "blue") + logger.debug(f"Configuration directory secured: {self.CONFIG_DIR}") + _print_colored(f"Configuration directory secured: {self.CONFIG_DIR}", "blue") except Exception as e: - logger.error(f"Error al crear directorio de configuración: {str(e)}") - _print_colored(f"Error al crear directorio de configuración: {str(e)}", "red") + logger.error(f"Error creating configuration directory: {str(e)}") + _print_colored(f"Error creating configuration directory: {str(e)}", "red") def _load_secret_key(self) -> None: """Loads or generates the secret key to encrypt sensitive data.""" @@ -99,22 +99,22 @@ def _load_secret_key(self) -> None: key = Fernet.generate_key() with open(self.SECRET_KEY_FILE, 'wb') as key_file: key_file.write(key) - _print_colored(f"Nueva clave secreta generada en: {self.SECRET_KEY_FILE}", "green") + _print_colored(f"New secret key generated at: {self.SECRET_KEY_FILE}", "green") with open(self.SECRET_KEY_FILE, 'rb') as key_file: self.secret_key = key_file.read() self.cipher = Fernet(self.secret_key) except Exception as e: - _print_colored(f"Error al cargar/generar clave secreta: {str(e)}", "red") - # Fallback a una clave temporal (menos segura pero funcional) + _print_colored(f"Error loading/generating secret key: {str(e)}", "red") + # Fallback to temporary key (less secure but functional) self.secret_key = Fernet.generate_key() self.cipher = Fernet(self.secret_key) def _load_configs(self) -> Dict[str, Dict[str, Any]]: """Loads the saved configurations.""" if not self.CONFIG_FILE.exists(): - _print_colored(f"Archivo de configuración no encontrado: {self.CONFIG_FILE}", "yellow") + _print_colored(f"Configuration file not found: {self.CONFIG_FILE}", "yellow") return {} try: @@ -122,26 +122,26 @@ def _load_configs(self) -> Dict[str, Dict[str, Any]]: encrypted_data = f.read() if not encrypted_data: - _print_colored("Archivo de configuración vacío", "yellow") + _print_colored("Empty configuration file", "yellow") return {} try: - # Intentar descifrar los datos + # Try to decrypt the data decrypted_data = self.cipher.decrypt(encrypted_data.encode()).decode() configs = json.loads(decrypted_data) except Exception as e: - # Si falla el descifrado, intentar cargar como JSON plano - logger.warning(f"Error al descifrar configuración: {e}") + # If decryption fails, try to load as plain JSON + logger.warning(f"Error decrypting configuration: {e}") configs = json.loads(encrypted_data) if isinstance(configs, str): configs = json.loads(configs) - _print_colored(f"Configuración cargada", "green") + _print_colored(f"Configuration loaded", "green") self.configs = configs return configs except Exception as e: - _print_colored(f"Error al cargar configuraciones: {str(e)}", "red") + _print_colored(f"Error loading configurations: {str(e)}", "red") return {} def _save_configs(self) -> None: @@ -153,9 +153,9 @@ def _save_configs(self) -> None: with open(self.CONFIG_FILE, 'w') as f: f.write(encrypted_data) - _print_colored(f"Configuraciones guardadas en: {self.CONFIG_FILE}", "green") + _print_colored(f"Configurations saved to: {self.CONFIG_FILE}", "green") except Exception as e: - _print_colored(f"Error al guardar configuraciones: {str(e)}", "red") + _print_colored(f"Error saving configurations: {str(e)}", "red") def add_config(self, api_key: str, db_config: Dict[str, Any], config_id: Optional[str] = None) -> str: """ @@ -173,15 +173,15 @@ def add_config(self, api_key: str, db_config: Dict[str, Any], config_id: Optiona config_id = str(uuid.uuid4()) db_config["config_id"] = config_id - # Crear o actualizar la entrada para este token + # Create or update the entry for this token if api_key not in self.configs: self.configs[api_key] = {} - # Añadir la configuración + # Add the configuration self.configs[api_key][config_id] = db_config self._save_configs() - _print_colored(f"Configuración agregada: {config_id} para la API Key: {api_key[:8]}...", "green") + _print_colored(f"Configuration added: {config_id} for API Key: {api_key[:8]}...", "green") return config_id def get_config(self, api_key_selected: str, config_id: str) -> Optional[Dict[str, Any]]: @@ -223,13 +223,13 @@ def remove_config(self, api_key_selected: str, config_id: str) -> bool: if api_key_selected in self.configs and config_id in self.configs[api_key_selected]: del self.configs[api_key_selected][config_id] - # Si no quedan configuraciones para este token, eliminar la entrada + # If no configurations remain for this token, remove the entry if not self.configs[api_key_selected]: del self.configs[api_key_selected] self._save_configs() - _print_colored(f"Configuración {config_id} eliminada para API Key: {api_key_selected[:8]}...", "green") + _print_colored(f"Configuration {config_id} removed for API Key: {api_key_selected[:8]}...", "green") return True - _print_colored(f"Configuración {config_id} no encontrada para API Key: {api_key_selected[:8]}...", "yellow") + _print_colored(f"Configuration {config_id} not found for API Key: {api_key_selected[:8]}...", "yellow") return False \ No newline at end of file diff --git a/corebrain/corebrain/core/query.py b/corebrain/corebrain/core/query.py index b23eddb..9c678d2 100644 --- a/corebrain/corebrain/core/query.py +++ b/corebrain/corebrain/core/query.py @@ -40,21 +40,21 @@ def __init__(self, cache_dir: str = None, ttl: int = 86400, memory_limit: int = else: self.cache_dir = Path.home() / ".corebrain_cache" - # Crear directorio de caché si no existe + # Create cache directory if it doesn't exist self.cache_dir.mkdir(parents=True, exist_ok=True) - # Inicializar base de datos SQLite para metadatos + # Initialize SQLite database for metadata self.db_path = self.cache_dir / "cache_metadata.db" self._init_db() - print_colored(f"Caché inicializado en {self.cache_dir}", "blue") + print_colored(f"Cache initialized at {self.cache_dir}", "blue") def _init_db(self): """Initializes the SQLite database for cache metadata.""" conn = sqlite3.connect(str(self.db_path)) cursor = conn.cursor() - # Crear tabla de metadatos si no existe + # Create metadata table if it doesn't exist cursor.execute(''' CREATE TABLE IF NOT EXISTS cache_metadata ( query_hash TEXT PRIMARY KEY, @@ -71,21 +71,21 @@ def _init_db(self): def _get_hash(self, query: str, config_id: str, collection_name: Optional[str] = None) -> str: """Generates a unique hash for the query.""" - # Normalizar la consulta (eliminar espacios extra, convertir a minúsculas) + # Normalize the query (remove extra spaces, convert to lowercase) normalized_query = re.sub(r'\s+', ' ', query.lower().strip()) - # Crear string compuesto para el hash + # Create composite string for the hash hash_input = f"{normalized_query}|{config_id}" if collection_name: hash_input += f"|{collection_name}" - # Generar el hash + # Generate the hash return hashlib.md5(hash_input.encode()).hexdigest() def _get_cache_path(self, query_hash: str) -> Path: """Gets the cache file path for a given hash.""" - # Usar los primeros caracteres del hash para crear subdirectorios - # Esto evita tener demasiados archivos en un solo directorio + # Use the first characters of the hash to create subdirectories + # This avoids having too many files in a single directory subdir = query_hash[:2] cache_subdir = self.cache_dir / subdir cache_subdir.mkdir(exist_ok=True) @@ -99,12 +99,12 @@ def _update_metadata(self, query_hash: str, query: str, config_id: str): now = datetime.now().isoformat() - # Verificar si el hash ya existe + # Check if the hash already exists cursor.execute("SELECT hit_count FROM cache_metadata WHERE query_hash = ?", (query_hash,)) result = cursor.fetchone() if result: - # Actualizar entrada existente + # Update existing entry hit_count = result[0] + 1 cursor.execute(''' UPDATE cache_metadata @@ -112,7 +112,7 @@ def _update_metadata(self, query_hash: str, query: str, config_id: str): WHERE query_hash = ? ''', (now, hit_count, query_hash)) else: - # Insertar nueva entrada + # Insert new entry cursor.execute(''' INSERT INTO cache_metadata (query_hash, query, config_id, created_at, last_accessed, hit_count) VALUES (?, ?, ?, ?, ?, 1) @@ -124,12 +124,12 @@ def _update_metadata(self, query_hash: str, query: str, config_id: str): def _update_memory_lru(self, query_hash: str): """Updates the LRU (Least Recently Used) list for the in-memory cache.""" if query_hash in self.memory_lru: - # Mover al final (más recientemente usado) + # Move to the end (most recently used) self.memory_lru.remove(query_hash) self.memory_lru.append(query_hash) - # Si excedemos el límite, eliminar el elemento menos usado recientemente + # If we exceed the limit, remove the least recently used element if len(self.memory_lru) > self.memory_limit: oldest_hash = self.memory_lru.pop(0) if oldest_hash in self.memory_cache: @@ -184,11 +184,11 @@ def get(self, query: str, config_id: str, collection_name: Optional[str] = None) print_colored(f"Cache hit (disk): {query[:30]}...", "green") return result except Exception as e: - print_colored(f"Error al cargar caché: {str(e)}", "red") - # Si hay error al cargar, eliminar el archivo corrupto + print_colored(f"Error loading cache: {str(e)}", "red") + # If there's an error loading, remove the corrupted file cache_path.unlink(missing_ok=True) else: - # Archivo expirado, eliminarlo + # Expired file, delete it cache_path.unlink(missing_ok=True) return None @@ -205,23 +205,23 @@ def set(self, query: str, config_id: str, result: Dict[str, Any], collection_nam """ query_hash = self._get_hash(query, config_id, collection_name) - # 1. Guardar en caché de memoria + # 1. Save to memory cache self.memory_cache[query_hash] = result self.memory_timestamps[query_hash] = time.time() self._update_memory_lru(query_hash) - # 2. Guardar en caché persistente + # 2. Save to persistent cache try: cache_path = self._get_cache_path(query_hash) with open(cache_path, 'wb') as f: pickle.dump(result, f) - # 3. Actualizar metadatos + # 3. Update metadata self._update_metadata(query_hash, query, config_id) print_colored(f"Cached: {query[:30]}...", "green") except Exception as e: - print_colored(f"Error al guardar en caché: {str(e)}", "red") + print_colored(f"Error saving to cache: {str(e)}", "red") def clear(self, older_than: int = None): """ @@ -230,7 +230,7 @@ def clear(self, older_than: int = None): Args: older_than: Only clear entries older than this number of seconds """ - # Limpiar caché en memoria + # Clear memory cache if older_than: current_time = time.time() keys_to_remove = [ @@ -250,15 +250,15 @@ def clear(self, older_than: int = None): self.memory_timestamps.clear() self.memory_lru.clear() - # Limpiar caché en disco + # Clear disk cache if older_than: cutoff_time = time.time() - older_than - # Usar la base de datos para encontrar archivos antiguos + # Use the database to find old files conn = sqlite3.connect(str(self.db_path)) cursor = conn.cursor() - # Convertir cutoff_time a formato ISO + # Convert cutoff_time to ISO format cutoff_datetime = datetime.fromtimestamp(cutoff_time).isoformat() cursor.execute( @@ -268,13 +268,13 @@ def clear(self, older_than: int = None): old_hashes = [row[0] for row in cursor.fetchall()] - # Eliminar archivos antiguos + # Delete old files for query_hash in old_hashes: cache_path = self._get_cache_path(query_hash) if cache_path.exists(): cache_path.unlink() - # Eliminar de la base de datos + # Delete from database cursor.execute( "DELETE FROM cache_metadata WHERE query_hash = ?", (query_hash,) @@ -283,13 +283,13 @@ def clear(self, older_than: int = None): conn.commit() conn.close() else: - # Eliminar todos los archivos de caché + # Delete all cache files for subdir in self.cache_dir.iterdir(): if subdir.is_dir(): for cache_file in subdir.glob("*.cache"): cache_file.unlink() - # Reiniciar la base de datos + # Reset the database conn = sqlite3.connect(str(self.db_path)) cursor = conn.cursor() cursor.execute("DELETE FROM cache_metadata") @@ -298,27 +298,27 @@ def clear(self, older_than: int = None): def get_stats(self) -> Dict[str, Any]: """Gets cache statistics.""" - # Contar archivos en disco + # Count files on disk disk_count = 0 for subdir in self.cache_dir.iterdir(): if subdir.is_dir(): disk_count += len(list(subdir.glob("*.cache"))) - # Obtener estadísticas de la base de datos + # Get database statistics conn = sqlite3.connect(str(self.db_path)) cursor = conn.cursor() - # Total de entradas + # Total entries cursor.execute("SELECT COUNT(*) FROM cache_metadata") total_entries = cursor.fetchone()[0] - # Consultas más frecuentes + # Most frequent queries cursor.execute( "SELECT query, hit_count FROM cache_metadata ORDER BY hit_count DESC LIMIT 5" ) top_queries = cursor.fetchall() - # Edad promedio + # Average age cursor.execute( "SELECT AVG(strftime('%s', 'now') - strftime('%s', created_at)) FROM cache_metadata" ) @@ -361,27 +361,27 @@ def __init__(self, pattern: str, description: str, self.db_type = db_type self.applicable_tables = applicable_tables or [] - # Compilar expresión regular para el patrón + # Compile regular expression for the pattern self.regex = self._compile_pattern(pattern) def _compile_pattern(self, pattern: str) -> re.Pattern: """Compiles the pattern into a regular expression.""" - # Reemplazar marcadores especiales con grupos de captura + # Replace special markers with capture groups regex_pattern = pattern - # {table} se convierte en grupo de captura para el nombre de tabla + # {table} becomes capture group for table name regex_pattern = regex_pattern.replace("{table}", r"(\w+)") - # {field} se convierte en grupo de captura para el nombre de campo + # {field} becomes capture group for field name regex_pattern = regex_pattern.replace("{field}", r"(\w+)") - # {value} se convierte en grupo de captura para un valor + # {value} becomes capture group for a value regex_pattern = regex_pattern.replace("{value}", r"([^,.\s]+)") - # {number} se convierte en grupo de captura para un número + # {number} becomes capture group for a number regex_pattern = regex_pattern.replace("{number}", r"(\d+)") - # Hacer coincidir el patrón completo + # Make the pattern match the entire string regex_pattern = f"^{regex_pattern}$" return re.compile(regex_pattern, re.IGNORECASE) @@ -413,22 +413,22 @@ def generate_query(self, params: List[str], db_schema: Dict[str, Any]) -> Option Generated query or None if it cannot be generated """ if self.generator_func: - # Usar función personalizada + # Use custom function return self.generator_func(params, db_schema) if not self.sql_template: return None - # Intentar aplicar la plantilla SQL con los parámetros + # Try applying the SQL template with the parameters try: sql_query = self.sql_template - # Reemplazar parámetros en la plantilla + # Replace parameters in the template for i, param in enumerate(params): placeholder = f"${i+1}" sql_query = sql_query.replace(placeholder, param) - # Verificar si hay algún parámetro sin reemplazar + # Check if there are any parameters left unreplaced if "$" in sql_query: return None diff --git a/corebrain/corebrain/core/test_utils.py b/corebrain/corebrain/core/test_utils.py index 5e334d9..8fead19 100644 --- a/corebrain/corebrain/core/test_utils.py +++ b/corebrain/corebrain/core/test_utils.py @@ -20,25 +20,25 @@ def generate_test_question_from_schema(schema: Dict[str, Any]) -> str: Generated test question """ if not schema or not schema.get("tables"): - return "¿Cuáles son las tablas disponibles?" + return "What are the available tables?" tables = schema["tables"] if not tables: - return "¿Cuáles son las tablas disponibles?" + return "What are the available tables?" - # Seleccionar una tabla aleatoria + # Select a random table table = random.choice(tables) table_name = table["name"] - # Determinar el tipo de pregunta + # Determine the type of question question_types = [ - f"¿Cuántos registros hay en la tabla {table_name}?", - f"Muestra los primeros 5 registros de {table_name}", - f"¿Cuáles son los campos de la tabla {table_name}?", + f"How many records are in the {table_name} table?", + f"Show the first 5 records from {table_name}", + f"What are the fields in the {table_name} table?", ] - # Obtener columnas según la estructura (SQL vs NoSQL) + # Get columns according to structure (SQL vs NoSQL) columns = [] if "columns" in table and table["columns"]: columns = table["columns"] @@ -46,13 +46,13 @@ def generate_test_question_from_schema(schema: Dict[str, Any]) -> str: columns = table["fields"] if columns: - # Si tenemos información de columnas/campos + # If we have column/field information column_name = columns[0]["name"] if columns else "id" - # Añadir preguntas específicas con columnas + # Add specific questions with columns question_types.extend([ - f"¿Cuál es el valor máximo de {column_name} en {table_name}?", - f"¿Cuáles son los valores únicos de {column_name} en {table_name}?", + f"What is the maximum value of {column_name} in {table_name}?", + f"What are the unique values of {column_name} in {table_name}?", ]) return random.choice(question_types) @@ -71,18 +71,18 @@ def test_natural_language_query(api_token: str, db_config: Dict[str, Any], api_u True if the test is successful, False otherwise """ try: - print_colored("\nRealizando prueba de consulta en lenguaje natural...", "blue") + print_colored("\nPerforming natural language query test...", "blue") - # Importación dinámica para evitar circular imports + # Dynamic import to avoid circular imports from db.schema_file import extract_db_schema - # Generar una pregunta de prueba basada en el esquema extraído directamente + # Generate a test question based on the directly extracted schema schema = extract_db_schema(db_config) - print("REcoge esquema: ", schema) + print("Retrieved schema: ", schema) question = generate_test_question_from_schema(schema) - print(f"Pregunta de prueba: {question}") + print(f"Test question: {question}") - # Preparar los datos para la petición + # Prepare the data for the request api_url = api_url or DEFAULT_API_URL if not api_url.startswith(("http://", "https://")): api_url = "https://" + api_url @@ -90,26 +90,26 @@ def test_natural_language_query(api_token: str, db_config: Dict[str, Any], api_u if api_url.endswith('/'): api_url = api_url[:-1] - # Construir endpoint para la consulta + # Build endpoint for the query endpoint = f"{api_url}/api/database/sdk/query" - # Datos para la consulta + # Data for the query request_data = { "question": question, "db_schema": schema, "config_id": db_config["config_id"] } - # Realizar la petición al API + # Make the API request headers = { "Authorization": f"Bearer {api_token}", "Content-Type": "application/json" } - timeout = 15.0 # Tiempo máximo de espera reducido + timeout = 15.0 # Reduced maximum wait time try: - print_colored("Enviando consulta al API...", "blue") + print_colored("Sending query to API...", "blue") response = http_session.post( endpoint, headers=headers, @@ -117,25 +117,25 @@ def test_natural_language_query(api_token: str, db_config: Dict[str, Any], api_u timeout=timeout ) - # Verificar la respuesta + # Verify the response if response.status_code == 200: result = response.json() - # Verificar si hay explicación en el resultado + # Check if there's an explanation in the result if "explanation" in result: - print_colored("\nRespuesta:", "green") + print_colored("\nResponse:", "green") print(result["explanation"]) - print_colored("\n✅ Prueba de consulta exitosa!", "green") + print_colored("\n✅ Query test successful!", "green") return True else: - # Si no hay explicación pero la API responde, puede ser un formato diferente - print_colored("\nRespuesta recibida del API (formato diferente al esperado):", "yellow") + # If there's no explanation but the API responds, it may be a different format + print_colored("\nResponse received from API (different format than expected):", "yellow") print(json.dumps(result, indent=2)) - print_colored("\n⚠️ La API respondió, pero con un formato diferente al esperado.", "yellow") + print_colored("\n⚠️ The API responded, but with a different format than expected.", "yellow") return True else: - print_colored(f"❌ Error en la respuesta: Código {response.status_code}", "red") + print_colored(f"❌ Error in response: Code {response.status_code}", "red") try: error_data = response.json() print(json.dumps(error_data, indent=2)) @@ -144,14 +144,14 @@ def test_natural_language_query(api_token: str, db_config: Dict[str, Any], api_u return False except http_session.TimeoutException: - print_colored("⚠️ Timeout al realizar la consulta. El API puede estar ocupado o no disponible.", "yellow") - print_colored("Esto no afecta a la configuración guardada.", "yellow") + print_colored("⚠️ Timeout while performing query. The API may be busy or unavailable.", "yellow") + print_colored("This does not affect the saved configuration.", "yellow") return False except http_session.RequestError as e: - print_colored(f"⚠️ Error de conexión: {str(e)}", "yellow") - print_colored("Verifica la URL de la API y tu conexión a internet.", "yellow") + print_colored(f"⚠️ Connection error: {str(e)}", "yellow") + print_colored("Check the API URL and your internet connection.", "yellow") return False except Exception as e: - print_colored(f"❌ Error al realizar la consulta: {str(e)}", "red") + print_colored(f"❌ Error performing query: {str(e)}", "red") return False \ No newline at end of file diff --git a/corebrain/corebrain/db/connectors/mongodb.py b/corebrain/corebrain/db/connectors/mongodb.py index 925d410..c62c335 100644 --- a/corebrain/corebrain/db/connectors/mongodb.py +++ b/corebrain/corebrain/db/connectors/mongodb.py @@ -31,10 +31,10 @@ def __init__(self, config: Dict[str, Any]): self.client = None self.db = None self.config = config - self.connection_timeout = 30 # segundos + self.connection_timeout = 30 # seconds if not PYMONGO_AVAILABLE: - print("Advertencia: pymongo no está instalado. Instálalo con 'pip install pymongo'") + print("Warning: pymongo is not installed. Install it with 'pip install pymongo'") def connect(self) -> bool: """ @@ -44,87 +44,87 @@ def connect(self) -> bool: True if the connection was successful, False otherwise """ if not PYMONGO_AVAILABLE: - raise ImportError("pymongo no está instalado. Instálalo con 'pip install pymongo'") + raise ImportError("pymongo is not installed. Install it with 'pip install pymongo'") try: start_time = time.time() - # Construir los parámetros de conexión + # Build connection parameters if "connection_string" in self.config: connection_string = self.config["connection_string"] - # Añadir timeout a la cadena de conexión si no está presente + # Add timeout to connection string if not present if "connectTimeoutMS=" not in connection_string: if "?" in connection_string: - connection_string += "&connectTimeoutMS=10000" # 10 segundos + connection_string += "&connectTimeoutMS=10000" # 10 seconds else: connection_string += "?connectTimeoutMS=10000" - # Crear cliente MongoDB con la cadena de conexión + # Create MongoDB client with connection string self.client = pymongo.MongoClient(connection_string) else: - # Diccionario de parámetros para MongoClient + # Dictionary of parameters for MongoClient mongo_params = { "host": self.config.get("host", "localhost"), "port": int(self.config.get("port", 27017)), - "connectTimeoutMS": 10000, # 10 segundos + "connectTimeoutMS": 10000, # 10 seconds "serverSelectionTimeoutMS": 10000 } - # Añadir credenciales solo si están presentes + # Add credentials only if present if self.config.get("user"): mongo_params["username"] = self.config.get("user") if self.config.get("password"): mongo_params["password"] = self.config.get("password") - # Opcionalmente añadir opciones de autenticación + # Optionally add authentication options if self.config.get("auth_source"): mongo_params["authSource"] = self.config.get("auth_source") if self.config.get("auth_mechanism"): mongo_params["authMechanism"] = self.config.get("auth_mechanism") - # Crear cliente MongoDB con parámetros + # Create MongoDB client with parameters self.client = pymongo.MongoClient(**mongo_params) - # Verificar que la conexión funciona + # Verify that the connection works self.client.admin.command('ping') - # Seleccionar la base de datos + # Select the database db_name = self.config.get("database", "") if not db_name: - # Si no hay base de datos especificada, listar las disponibles + # If no database is specified, list available ones db_names = self.client.list_database_names() if not db_names: - raise ValueError("No se encontraron bases de datos disponibles") + raise ValueError("No available databases found") - # Seleccionar la primera que no sea de sistema + # Select the first one that is not a system database system_dbs = ["admin", "local", "config"] for name in db_names: if name not in system_dbs: db_name = name break - # Si no encontramos ninguna que no sea de sistema, usar la primera + # If we don't find any non-system database, use the first one if not db_name: db_name = db_names[0] - print(f"No se especificó base de datos. Usando '{db_name}'") + print(f"No database specified. Using '{db_name}'") - # Guardar la referencia a la base de datos + # Save the reference to the database self.db = self.client[db_name] return True except (ConnectionFailure, ServerSelectionTimeoutError) as e: - # Si es un error de timeout, reintentar + # If it's a timeout error, retry if time.time() - start_time < self.connection_timeout: - print(f"Timeout al conectar a MongoDB: {str(e)}. Reintentando...") - time.sleep(2) # Esperar antes de reintentar + print(f"Timeout connecting to MongoDB: {str(e)}. Retrying...") + time.sleep(2) # Wait before retrying return self.connect() else: - print(f"Error de conexión a MongoDB después de {self.connection_timeout}s: {str(e)}") + print(f"MongoDB connection error after {self.connection_timeout}s: {str(e)}") self.close() return False except Exception as e: - print(f"Error al conectar a MongoDB: {str(e)}") + print(f"Error connecting to MongoDB: {str(e)}") self.close() return False @@ -141,64 +141,64 @@ def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] Returns: Dictionary with the database schema """ - # Asegurar que estamos conectados + # Ensure we are connected if not self.client and not self.connect(): return {"type": "mongodb", "tables": {}, "tables_list": []} - # Inicializar el esquema + # Initialize the schema schema = { "type": "mongodb", "database": self.db.name, - "tables": {} # En MongoDB, las "tablas" son colecciones + "tables": {} # In MongoDB, "tables" are collections } try: - # Obtener la lista de colecciones + # Get the list of collections collections = self.db.list_collection_names() - # Limitar colecciones si es necesario + # Limit collections if necessary if collection_limit is not None and collection_limit > 0: collections = collections[:collection_limit] - # Procesar cada colección + # Process each collection total_collections = len(collections) for i, collection_name in enumerate(collections): - # Reportar progreso si hay callback + # Report progress if there's a callback if progress_callback: - progress_callback(i, total_collections, f"Procesando colección {collection_name}") + progress_callback(i, total_collections, f"Processing collection {collection_name}") collection = self.db[collection_name] try: - # Contar documentos + # Count documents doc_count = collection.count_documents({}) if doc_count > 0: - # Obtener muestra de documentos + # Get sample documents sample_docs = list(collection.find().limit(sample_limit)) - # Extraer campos y sus tipos + # Extract fields and their types fields = {} for doc in sample_docs: self._extract_document_fields(doc, fields) - # Convertir a formato esperado + # Convert to expected format formatted_fields = [{"name": field, "type": type_name} for field, type_name in fields.items()] - # Procesar documentos para sample_data + # Process documents for sample_data sample_data = [] for doc in sample_docs: processed_doc = self._process_document_for_serialization(doc) sample_data.append(processed_doc) - # Guardar en el esquema + # Save to schema schema["tables"][collection_name] = { "fields": formatted_fields, "sample_data": sample_data, "count": doc_count } else: - # Colección vacía + # Empty collection schema["tables"][collection_name] = { "fields": [], "sample_data": [], @@ -207,26 +207,26 @@ def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] } except Exception as e: - print(f"Error al procesar colección {collection_name}: {str(e)}") + print(f"Error processing collection {collection_name}: {str(e)}") schema["tables"][collection_name] = { "fields": [], "error": str(e) } - # Crear la lista de tablas/colecciones para compatibilidad + # Create the list of tables/collections for compatibility table_list = [] for collection_name, collection_info in schema["tables"].items(): table_data = {"name": collection_name} table_data.update(collection_info) table_list.append(table_data) - # Guardar también la lista de tablas para compatibilidad + # Also save the list of tables for compatibility schema["tables_list"] = table_list return schema except Exception as e: - print(f"Error al extraer el esquema MongoDB: {str(e)}") + print(f"Error extracting MongoDB schema: {str(e)}") return {"type": "mongodb", "tables": {}, "tables_list": []} def _extract_document_fields(self, doc: Dict[str, Any], fields: Dict[str, str], diff --git a/corebrain/corebrain/db/connectors/sql.py b/corebrain/corebrain/db/connectors/sql.py index 82f49bf..d9190b9 100644 --- a/corebrain/corebrain/db/connectors/sql.py +++ b/corebrain/corebrain/db/connectors/sql.py @@ -33,7 +33,7 @@ def __init__(self, config: Dict[str, Any]): self.cursor = None self.engine = config.get("engine", "").lower() self.config = config - self.connection_timeout = 30 # segundos + self.connection_timeout = 30 # seconds def connect(self) -> bool: """ @@ -45,7 +45,7 @@ def connect(self) -> bool: try: start_time = time.time() - # Intentar la conexión con un límite de tiempo + # Try the connection with a time limit while time.time() - start_time < self.connection_timeout: try: if self.engine == "sqlite": @@ -54,7 +54,7 @@ def connect(self) -> bool: else: self.conn = sqlite3.connect(self.config.get("database", ""), timeout=10.0) - # Configurar para que devuelva filas como diccionarios + # Configure to return rows as dictionaries self.conn.row_factory = sqlite3.Row elif self.engine == "mysql": @@ -74,9 +74,9 @@ def connect(self) -> bool: ) elif self.engine == "postgresql": - # Determinar si usar cadena de conexión o parámetros + # Determine whether to use connection string or parameters if "connection_string" in self.config: - # Agregar timeout a la cadena de conexión si no está presente + # Add timeout to connection string if not present conn_str = self.config["connection_string"] if "connect_timeout" not in conn_str: if "?" in conn_str: @@ -95,24 +95,24 @@ def connect(self) -> bool: connect_timeout=10 ) - # Si llegamos aquí, la conexión fue exitosa + # If we get here, the connection was successful if self.conn: - # Verificar conexión con una consulta simple + # Verify connection with a simple query cursor = self.conn.cursor() cursor.execute("SELECT 1") cursor.close() return True except (sqlite3.Error, mysql.connector.Error, psycopg2.Error) as e: - # Si el error no es de timeout, propagar la excepción + # If the error is not a timeout, propagate the exception if "timeout" not in str(e).lower() and "tiempo de espera" not in str(e).lower(): raise - # Si es un error de timeout, esperamos un poco y reintentamos + # If it's a timeout error, wait a bit and retry time.sleep(1.0) - # Si llegamos aquí, se agotó el tiempo de espera - raise TimeoutError(f"No se pudo conectar a la base de datos en {self.connection_timeout} segundos") + # If we get here, the timeout was exceeded + raise TimeoutError(f"Could not connect to database in {self.connection_timeout} seconds") except Exception as e: if self.conn: @@ -122,7 +122,7 @@ def connect(self) -> bool: pass self.conn = None - print(f"Error al conectar a la base de datos: {str(e)}") + print(f"Error connecting to database: {str(e)}") return False def extract_schema(self, sample_limit: int = 5, table_limit: Optional[int] = None, @@ -138,11 +138,11 @@ def extract_schema(self, sample_limit: int = 5, table_limit: Optional[int] = Non Returns: Dictionary with the database schema """ - # Asegurar que estamos conectados + # Ensure we are connected if not self.conn and not self.connect(): return {"type": "sql", "tables": {}, "tables_list": []} - # Inicializar esquema + # Initialize schema schema = { "type": "sql", "engine": self.engine, @@ -150,7 +150,7 @@ def extract_schema(self, sample_limit: int = 5, table_limit: Optional[int] = Non "tables": {} } - # Seleccionar la función extractora según el motor + # Select the extractor function according to the engine if self.engine == "sqlite": return self._extract_sqlite_schema(sample_limit, table_limit, progress_callback) elif self.engine == "mysql": @@ -158,7 +158,7 @@ def extract_schema(self, sample_limit: int = 5, table_limit: Optional[int] = Non elif self.engine == "postgresql": return self._extract_postgresql_schema(sample_limit, table_limit, progress_callback) else: - return schema # Esquema vacío si no se reconoce el motor + return schema # Empty schema if engine is not recognized def execute_query(self, query: str) -> List[Dict[str, Any]]: """ @@ -171,10 +171,10 @@ def execute_query(self, query: str) -> List[Dict[str, Any]]: List of resulting rows as dictionaries """ if not self.conn and not self.connect(): - raise ConnectionError("No se pudo establecer conexión con la base de datos") + raise ConnectionError("Could not establish connection to database") try: - # Ejecutar query según el motor + # Execute query according to engine if self.engine == "sqlite": return self._execute_sqlite_query(query) elif self.engine == "mysql": @@ -182,14 +182,14 @@ def execute_query(self, query: str) -> List[Dict[str, Any]]: elif self.engine == "postgresql": return self._execute_postgresql_query(query) else: - raise ValueError(f"Motor de base de datos no soportado: {self.engine}") + raise ValueError(f"Unsupported database engine: {self.engine}") except Exception as e: - # Intentar reconectar y reintentar una vez + # Try to reconnect and retry once try: self.close() if self.connect(): - print("Reconectando y reintentando consulta...") + print("Reconnecting and retrying query...") if self.engine == "sqlite": return self._execute_sqlite_query(query) @@ -199,11 +199,11 @@ def execute_query(self, query: str) -> List[Dict[str, Any]]: return self._execute_postgresql_query(query) except Exception as retry_error: - # Si falla el reintento, propagar el error original - raise Exception(f"Error al ejecutar consulta: {str(e)}") + # If retry fails, propagate the original error + raise Exception(f"Error executing query: {str(e)}") - # Si llegamos aquí sin retornar, ha habido un error en el reintento - raise Exception(f"Error al ejecutar consulta (después de reconexión): {str(e)}") + # If we get here without returning, there was an error in the retry + raise Exception(f"Error executing query (after reconnection): {str(e)}") def _execute_sqlite_query(self, query: str) -> List[Dict[str, Any]]: """Executes a query in SQLite.""" diff --git a/corebrain/corebrain/db/schema/extractor.py b/corebrain/corebrain/db/schema/extractor.py index c361b83..4aeabcf 100644 --- a/corebrain/corebrain/db/schema/extractor.py +++ b/corebrain/corebrain/db/schema/extractor.py @@ -1,4 +1,4 @@ -# db/schema/extractor.py (reemplaza la importación circular en db/schema.py) +# db/schema/extractor.py (replaces the circular import in db/schema.py) """ Independent database schema extractor. @@ -30,35 +30,35 @@ def extract_db_schema(db_config: Dict[str, Any], client_factory: Optional[Callab } try: - # Si tenemos un cliente especializado, usarlo + # If we have a specialized client, use it if client_factory: - # La factoría crea un cliente y extrae el esquema + # The factory creates a client and extracts the schema client = client_factory(db_config) return client.extract_schema() - # Extracción directa sin usar cliente de Corebrain + # Direct extraction without using Corebrain client if db_type == "sql": - # Código para bases de datos SQL (sin dependencias circulares) + # Code for SQL databases (without circular dependencies) engine = db_config.get("engine", "").lower() if engine == "sqlite": - # Extraer esquema SQLite + # Extract SQLite schema import sqlite3 - # (implementación...) + # (implementation...) elif engine == "mysql": - # Extraer esquema MySQL + # Extract MySQL schema import mysql.connector - # (implementación...) + # (implementation...) elif engine == "postgresql": - # Extraer esquema PostgreSQL + # Extract PostgreSQL schema import psycopg2 - # (implementación...) + # (implementation...) elif db_type in ["nosql", "mongodb"]: - # Extraer esquema MongoDB + # Extract MongoDB schema import pymongo - # (implementación...) + # (implementation...) - # Convertir diccionario a lista para compatibilidad + # Convert dictionary to list for compatibility table_list = [] for table_name, table_info in schema["tables"].items(): table_data = {"name": table_name} @@ -69,7 +69,7 @@ def extract_db_schema(db_config: Dict[str, Any], client_factory: Optional[Callab return schema except Exception as e: - logger.error(f"Error al extraer esquema: {str(e)}") + logger.error(f"Error extracting schema: {str(e)}") return {"type": db_type, "tables": {}, "tables_list": []} @@ -82,10 +82,10 @@ def create_schema_from_corebrain() -> Callable: Function that extracts schema using Corebrain """ def extract_with_corebrain(db_config: Dict[str, Any]) -> Dict[str, Any]: - # Importar dinámicamente para evitar circular + # Import dynamically to avoid circular dependencies from corebrain.core.client import Corebrain - # Crear cliente temporal solo para extraer el schema + # Create temporary client only to extract the schema try: client = Corebrain( api_token="temp_token", @@ -96,13 +96,13 @@ def extract_with_corebrain(db_config: Dict[str, Any]) -> Dict[str, Any]: client.close() return schema except Exception as e: - logger.error(f"Error al extraer schema con Corebrain: {str(e)}") + logger.error(f"Error extracting schema with Corebrain: {str(e)}") return {"type": db_config.get("type", ""), "tables": {}, "tables_list": []} return extract_with_corebrain -# Función pública expuesta +# Public function exposed def extract_schema(db_config: Dict[str, Any], use_corebrain: bool = False) -> Dict[str, Any]: """ Public function that decides how to extract the schema. @@ -115,9 +115,9 @@ def extract_schema(db_config: Dict[str, Any], use_corebrain: bool = False) -> Di Database schema """ if use_corebrain: - # Intentar usar Corebrain si se solicita + # Try to use Corebrain if requested factory = create_schema_from_corebrain() return extract_db_schema(db_config, client_factory=factory) else: - # Usar extracción directa sin dependencias circulares + # Use direct extraction without circular dependencies return extract_db_schema(db_config) \ No newline at end of file diff --git a/corebrain/corebrain/db/schema_file.py b/corebrain/corebrain/db/schema_file.py index 3c1edcd..0f75ebc 100644 --- a/corebrain/corebrain/db/schema_file.py +++ b/corebrain/corebrain/db/schema_file.py @@ -31,16 +31,16 @@ def extract_db_schema(db_config: Dict[str, Any]) -> Dict[str, Any]: schema = { "type": db_type, "database": db_config.get("database", ""), - "tables": {} # Cambiado a diccionario para facilitar el acceso directo a tablas por nombre + "tables": {} # Changed to dictionary to facilitate direct access to tables by name } try: if db_type == "sql": - # Código para bases de datos SQL... - # [Se mantiene igual] + # Code for SQL databases... + # [Keeps the same] pass - # Manejar tanto "nosql" como tipos válidos + # Handle both "nosql" and valid types elif db_type == "nosql": match db_config.get("engine", "").lower(): case "mongodb": @@ -51,56 +51,67 @@ def extract_db_schema(db_config: Dict[str, Any]) -> Dict[str, Any]: PYMONGO_IMPORTED = False if not PYMONGO_IMPORTED: raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") - # Defying the engine + # Defining the engine engine = db_config.get("engine", "").lower() if engine == "mongodb": if "connection_string" in db_config: client = pymongo.MongoClient(db_config["connection_string"]) else: - # Diccionario de parámetros para MongoClient + # Dictionary of parameters for MongoClient mongo_params = { "host": db_config.get("host", "localhost"), "port": db_config.get("port", 27017) } - # Añadir credenciales solo si están presentes + # Add credentials only if they are present if db_config.get("user"): mongo_params["username"] = db_config["user"] if db_config.get("password"): mongo_params["password"] = db_config["password"] client = pymongo.MongoClient(**mongo_params) - db_name = db_config.get("database","") + + db_name = db_config.get("database","") - if not db_name: - _print_colored("⚠️ Database is not specified", "yellow") - return schema - try: - db = client[db_name] - collection_names = db.list_collection_names() + if not db_name: + _print_colored("⚠️ Database is not specified", "yellow") + return schema + + try: + db = client[db_name] + collection_names = db.list_collection_names() - # Process collection + # Process collection + for collection_name in collection_names: + collection = db[collection_name] - for collection_name in collection_names: - collection = db[collection_name] + try: + sample_docs = list(collection.find().limit(5)) - try: - sample_docs = list(collection.find().lkmit(5)) + field_types = {} + # Process sample documents to infer field types + for doc in sample_docs: + for field, value in doc.items(): + if field not in field_types: + field_types[field] = type(value).__name__ + + # Add collection to schema + schema["tables"][collection_name] = { + "fields": [{"name": field, "type": field_type} for field, field_type in field_types.items()], + "sample_data": sample_docs + } - field_types = {} - - - - except Exception as e: - except Exception as e: - - - - - - - + except Exception as e: + _print_colored(f"Error processing collection {collection_name}: {str(e)}", "red") + + except Exception as e: + _print_colored(f"Error accessing database: {str(e)}", "red") + + except Exception as e: + _print_colored(f"Error extracting schema: {str(e)}", "red") + + return schema def extract_db_schema_direct(db_config: Dict[str, Any]) -> Dict[str, Any]: """ @@ -112,16 +123,16 @@ def extract_db_schema_direct(db_config: Dict[str, Any]) -> Dict[str, Any]: "type": db_type, "database": db_config.get("database", ""), "tables": {}, - "tables_list": [] # Lista inicialmente vacía + "tables_list": [] # Initially empty list } try: - # [Implementación existente para extraer esquema sin usar Corebrain] + # [Existing implementation to extract schema without using Corebrain] # ... return schema except Exception as e: - _print_colored(f"Error al extraer esquema directamente: {str(e)}", "red") + _print_colored(f"Error extracting schema directly: {str(e)}", "red") return {"type": db_type, "tables": {}, "tables_list": []} def extract_schema_with_lazy_init(api_key: str, db_config: Dict[str, Any], api_url: Optional[str] = None) -> Dict[str, Any]: @@ -132,31 +143,32 @@ def extract_schema_with_lazy_init(api_key: str, db_config: Dict[str, Any], api_u the Corebrain client only when necessary. """ try: - # La importación se mueve aquí para evitar el problema de circular import - # Solo se ejecuta cuando realmente necesitamos crear el cliente + # The import is moved here to avoid the circular import problem + # It is only executed when we really need to create the client import importlib core_module = importlib.import_module('core') init_func = getattr(core_module, 'init') - # Crear cliente con la configuración + # Create client with the configuration api_url_to_use = api_url or "https://api.corebrain.com" cb = init_func( api_token=api_key, db_config=db_config, api_url=api_url_to_use, - skip_verification=True # No necesitamos verificar token para extraer schema + skip_verification=True # We don't need to verify token to extract schema ) - # Obtener el esquema y cerrar cliente + # Get the schema and close the client schema = cb.db_schema cb.close() return schema except Exception as e: - _print_colored(f"Error al extraer esquema con cliente: {str(e)}", "red") - # Como alternativa, usar extracción directa sin cliente + _print_colored(f"Error extracting schema with client: {str(e)}", "red") + # As an alternative, use direct extraction without client return extract_db_schema_direct(db_config) + from typing import Dict, Any def test_connection(db_config: Dict[str, Any]) -> bool: @@ -193,78 +205,78 @@ def extract_schema_to_file(api_key: str, config_id: Optional[str] = None, output True if extraction is successful, False otherwise """ try: - # Importación explícita con try-except para manejar errores + # Explicit import with try-except to handle errors try: from corebrain.config.manager import ConfigManager except ImportError as e: - _print_colored(f"Error al importar ConfigManager: {e}", "red") + _print_colored(f"Error importing ConfigManager: {e}", "red") return False - # Obtener las configuraciones disponibles + # Get the available configurations config_manager = ConfigManager() configs = config_manager.list_configs(api_key) if not configs: - _print_colored("No hay configuraciones guardadas para esta API Key.", "yellow") + _print_colored("No configurations saved for this API Key.", "yellow") return False selected_config_id = config_id - # Si no se especifica un config_id, mostrar lista para seleccionar + # If no config_id is specified, show list to select if not selected_config_id: - _print_colored("\n=== Configuraciones disponibles ===", "blue") + _print_colored("\n=== Available configurations ===", "blue") for i, conf_id in enumerate(configs, 1): print(f"{i}. {conf_id}") try: - choice = int(input(f"\nSelecciona una configuración (1-{len(configs)}): ").strip()) + choice = int(input(f"\nSelect a configuration (1-{len(configs)}): ").strip()) if 1 <= choice <= len(configs): selected_config_id = configs[choice - 1] else: - _print_colored("Opción inválida.", "red") + _print_colored("Invalid option.", "red") return False except ValueError: - _print_colored("Por favor, introduce un número válido.", "red") + _print_colored("Please enter a valid number.", "red") return False - # Verificar que el config_id exista + # Verify that the config_id exists if selected_config_id not in configs: - _print_colored(f"No se encontró la configuración con ID: {selected_config_id}", "red") + _print_colored(f"Configuration with ID not found: {selected_config_id}", "red") return False - # Obtener la configuración seleccionada + # Get the selected configuration db_config = config_manager.get_config(api_key, selected_config_id) if not db_config: - _print_colored(f"Error al obtener la configuración con ID: {selected_config_id}", "red") + _print_colored(f"Error getting configuration with ID: {selected_config_id}", "red") return False - _print_colored(f"\nExtrayendo esquema para configuración: {selected_config_id}", "blue") - print(f"Tipo: {db_config['type'].upper()}, Motor: {db_config.get('engine', 'No especificado').upper()}") - print(f"Base de datos: {db_config.get('database', 'No especificada')}") + _print_colored(f"\nExtracting schema for configuration: {selected_config_id}", "blue") + print(f"Type: {db_config['type'].upper()}, Engine: {db_config.get('engine', 'Not specified').upper()}") + print(f"Database: {db_config.get('database', 'Not specified')}") - # Extraer el esquema de la base de datos - _print_colored("\nExtrayendo esquema de la base de datos...", "blue") + # Extract the database schema + _print_colored("\nExtracting database schema...", "blue") schema = extract_schema_with_lazy_init(api_key, db_config, api_url) - # Verificar si se obtuvo un esquema válido + # Verify if a valid schema was obtained if not schema or not schema.get("tables"): - _print_colored("No se encontraron tablas/colecciones en la base de datos.", "yellow") + _print_colored("No tables/collections found in the database.", "yellow") return False - # Guardar el esquema en un archivo + # Save the schema to a file output_path = output_file or "db_schema.json" try: with open(output_path, 'w', encoding='utf-8') as f: json.dump(schema, f, indent=2, default=str) - _print_colored(f"✅ Esquema extraído y guardado en: {output_path}", "green") + _print_colored(f"✅ Schema extracted and saved in: {output_path}", "green") except Exception as e: - _print_colored(f"❌ Error al guardar el archivo: {str(e)}", "red") + _print_colored(f"❌ Error saving the file: {str(e)}", "red") return False - # Mostrar un resumen de las tablas/colecciones encontradas + # Show a summary of the tables/collections found tables = schema.get("tables", {}) - _print_colored(f"\nResumen del esquema extraído: {len(tables)} tablas/colecciones", "green") + _print_colored(f"\nExtracted schema summary: {len(tables)} tables/collections", "green") for table_name in tables: print(f"- {table_name}") @@ -272,7 +284,7 @@ def extract_schema_to_file(api_key: str, config_id: Optional[str] = None, output return True except Exception as e: - _print_colored(f"❌ Error al extraer esquema: {str(e)}", "red") + _print_colored(f"❌ Error extracting schema: {str(e)}", "red") return False def show_db_schema(api_token: str, config_id: Optional[str] = None, api_url: Optional[str] = None) -> None: @@ -285,73 +297,73 @@ def show_db_schema(api_token: str, config_id: Optional[str] = None, api_url: Opt api_url: Optional API URL """ try: - # Importación explícita con try-except para manejar errores + # Explicit import with try-except to handle errors try: from corebrain.config.manager import ConfigManager except ImportError as e: - _print_colored(f"Error al importar ConfigManager: {e}", "red") + _print_colored(f"Error importing ConfigManager: {e}", "red") return False - # Obtener las configuraciones disponibles + # Get the available configurations config_manager = ConfigManager() configs = config_manager.list_configs(api_token) if not configs: - _print_colored("No hay configuraciones guardadas para este token.", "yellow") + _print_colored("No configurations saved for this token.", "yellow") return selected_config_id = config_id - # Si no se especifica un config_id, mostrar lista para seleccionar + # If no config_id is specified, show list to select if not selected_config_id: - _print_colored("\n=== Configuraciones disponibles ===", "blue") + _print_colored("\n=== Available configurations ===", "blue") for i, conf_id in enumerate(configs, 1): print(f"{i}. {conf_id}") try: - choice = int(input(f"\nSelecciona una configuración (1-{len(configs)}): ").strip()) + choice = int(input(f"\nSelect a configuration (1-{len(configs)}): ").strip()) if 1 <= choice <= len(configs): selected_config_id = configs[choice - 1] else: - _print_colored("Opción inválida.", "red") + _print_colored("Invalid option.", "red") return except ValueError: - _print_colored("Por favor, introduce un número válido.", "red") + _print_colored("Please enter a valid number.", "red") return - # Verificar que el config_id exista + # Verify that the config_id exists if selected_config_id not in configs: - _print_colored(f"No se encontró la configuración con ID: {selected_config_id}", "red") + _print_colored(f"Configuration with ID not found: {selected_config_id}", "red") return if config_id and config_id in configs: db_config = config_manager.get_config(api_token, config_id) else: - # Obtener la configuración seleccionada + # Get the selected configuration db_config = config_manager.get_config(api_token, selected_config_id) if not db_config: - _print_colored(f"Error al obtener la configuración con ID: {selected_config_id}", "red") + _print_colored(f"Error getting configuration with ID: {selected_config_id}", "red") return - _print_colored(f"\nObteniendo esquema para configuración: {selected_config_id}", "blue") - _print_colored("Tipo de base de datos:", "blue") + _print_colored(f"\nGetting schema for configuration: {selected_config_id}", "blue") + _print_colored("Database type:", "blue") print(f" {db_config['type'].upper()}") if db_config.get('engine'): - _print_colored("Motor:", "blue") + _print_colored("Engine:", "blue") print(f" {db_config['engine'].upper()}") - _print_colored("Base de datos:", "blue") - print(f" {db_config.get('database', 'No especificada')}") + _print_colored("Database:", "blue") + print(f" {db_config.get('database', 'Not specified')}") - # Extraer y mostrar el esquema - _print_colored("\nExtrayendo esquema de la base de datos...", "blue") + # Extract and show the schema + _print_colored("\nExtracting database schema...", "blue") - # Intenta conectarse a la base de datos y extraer el esquema + # Try to connect to the database and extract the schema try: - # Creamos una instancia de Corebrain con la configuración seleccionada + # Create a Corebrain instance with the selected configuration """ cb = init( api_token=api_token, @@ -365,15 +377,15 @@ def show_db_schema(api_token: str, config_id: Optional[str] = None, api_url: Opt core_module = importlib.import_module('core.client') init_func = getattr(core_module, 'init') - # Creamos una instancia de Corebrain con la configuración seleccionada + # Create a Corebrain instance with the selected configuration cb = init_func( api_token=api_token, config_id=config_id, api_url=api_url, - skip_verification=True # Omitimos verificación para simplificar + skip_verification=True # Skip verification for simplicity ) - # El esquema se extrae automáticamente al inicializar + # The schema is automatically extracted when initializing schema = get_schema_with_dynamic_import( api_token=api_token, config_id=selected_config_id, @@ -381,45 +393,45 @@ def show_db_schema(api_token: str, config_id: Optional[str] = None, api_url: Opt api_url=api_url ) - # Si no hay esquema, intentamos extraerlo explícitamente + # If there's no schema, we try to extract it explicitly if not schema or not schema.get("tables"): - _print_colored("Intentando extraer esquema explícitamente...", "yellow") + _print_colored("Trying to extract schema explicitly...", "yellow") schema = cb._extract_db_schema() - # Cerramos la conexión + # Close the connection cb.close() except Exception as conn_error: - _print_colored(f"Error de conexión: {str(conn_error)}", "red") - print("Intentando método alternativo...") + _print_colored(f"Connection error: {str(conn_error)}", "red") + print("Trying alternative method...") - # Método alternativo: usar función extract_db_schema directamente + # Alternative method: use extract_db_schema function directly schema = extract_db_schema(db_config) - # Verificar si se obtuvo un esquema válido + # Verify if a valid schema was obtained if not schema or not schema.get("tables"): - _print_colored("No se encontraron tablas/colecciones en la base de datos.", "yellow") + _print_colored("No tables/collections found in the database.", "yellow") - # Información adicional para ayudar a diagnosticar el problema - print("\nInformación de depuración:") - print(f" Tipo de base de datos: {db_config.get('type', 'No especificado')}") - print(f" Motor: {db_config.get('engine', 'No especificado')}") - print(f" Host: {db_config.get('host', 'No especificado')}") - print(f" Puerto: {db_config.get('port', 'No especificado')}") - print(f" Base de datos: {db_config.get('database', 'No especificado')}") + # Additional information to help diagnose the problem + print("\nDebug information:") + print(f" Database type: {db_config.get('type', 'Not specified')}") + print(f" Engine: {db_config.get('engine', 'Not specified')}") + print(f" Host: {db_config.get('host', 'Not specified')}") + print(f" Port: {db_config.get('port', 'Not specified')}") + print(f" Database: {db_config.get('database', 'Not specified')}") - # Para PostgreSQL, sugerir verificar el esquema + # For PostgreSQL, suggest checking the schema if db_config.get('engine') == 'postgresql': - print("\nPara PostgreSQL, verifica que las tablas existan en el esquema 'public' o") - print("que tengas acceso a los esquemas donde están las tablas.") - print("Puedes verificar los esquemas disponibles con: SELECT DISTINCT table_schema FROM information_schema.tables;") + print("\nFor PostgreSQL, verify that tables exist in the 'public' schema or") + print("that you have access to the schemas where the tables are located.") + print("You can check available schemas with: SELECT DISTINCT table_schema FROM information_schema.tables;") return - # Mostrar información del esquema + # Show schema information tables = schema.get("tables", {}) - # Separar tablas SQL y colecciones NoSQL para mostrarlas apropiadamente + # Separate SQL tables and NoSQL collections to display them appropriately sql_tables = {} nosql_collections = {} @@ -429,76 +441,76 @@ def show_db_schema(api_token: str, config_id: Optional[str] = None, api_url: Opt elif "fields" in info: nosql_collections[name] = info - # Mostrar tablas SQL + # Show SQL tables if sql_tables: - _print_colored(f"\nSe encontraron {len(sql_tables)} tablas SQL:", "green") + _print_colored(f"\n{len(sql_tables)} SQL tables found:", "green") for table_name, table_info in sql_tables.items(): - _print_colored(f"\n=== Tabla: {table_name} ===", "bold") + _print_colored(f"\n=== Table: {table_name} ===", "bold") - # Mostrar columnas + # Show columns columns = table_info.get("columns", []) if columns: - _print_colored("Columnas:", "blue") + _print_colored("Columns:", "blue") for column in columns: print(f" - {column['name']} ({column['type']})") else: - _print_colored("No se encontraron columnas.", "yellow") + _print_colored("No columns found.", "yellow") - # Mostrar muestra de datos si está disponible + # Show sample data if available sample_data = table_info.get("sample_data", []) if sample_data: - _print_colored("\nMuestra de datos:", "blue") - for i, row in enumerate(sample_data[:2], 1): # Limitar a 2 filas para simplificar - print(f" Registro {i}: {row}") + _print_colored("\nSample data:", "blue") + for i, row in enumerate(sample_data[:2], 1): # Limit to 2 rows for simplicity + print(f" Record {i}: {row}") if len(sample_data) > 2: - print(f" ... ({len(sample_data) - 2} registros más)") + print(f" ... ({len(sample_data) - 2} more records)") - # Mostrar colecciones NoSQL + # Show NoSQL collections if nosql_collections: - _print_colored(f"\nSe encontraron {len(nosql_collections)} colecciones NoSQL:", "green") + _print_colored(f"\n{len(nosql_collections)} NoSQL collections found:", "green") for coll_name, coll_info in nosql_collections.items(): - _print_colored(f"\n=== Colección: {coll_name} ===", "bold") + _print_colored(f"\n=== Collection: {coll_name} ===", "bold") - # Mostrar campos + # Show fields fields = coll_info.get("fields", []) if fields: - _print_colored("Campos:", "blue") + _print_colored("Fields:", "blue") for field in fields: print(f" - {field['name']} ({field['type']})") else: - _print_colored("No se encontraron campos.", "yellow") + _print_colored("No fields found.", "yellow") - # Mostrar muestra de datos si está disponible + # Show sample data if available sample_data = coll_info.get("sample_data", []) if sample_data: - _print_colored("\nMuestra de datos:", "blue") - for i, doc in enumerate(sample_data[:2], 1): # Limitar a 2 documentos - # Simplificar la visualización para documentos grandes + _print_colored("\nSample data:", "blue") + for i, doc in enumerate(sample_data[:2], 1): # Limit to 2 documents + # Simplify visualization for large documents if isinstance(doc, dict) and len(doc) > 5: simplified = {k: doc[k] for k in list(doc.keys())[:5]} - print(f" Documento {i}: {simplified} ... (y {len(doc) - 5} campos más)") + print(f" Document {i}: {simplified} ... (and {len(doc) - 5} more fields)") else: - print(f" Documento {i}: {doc}") + print(f" Document {i}: {doc}") if len(sample_data) > 2: - print(f" ... ({len(sample_data) - 2} documentos más)") + print(f" ... ({len(sample_data) - 2} more documents)") - _print_colored("\n✅ Esquema extraído correctamente!", "green") + _print_colored("\n✅ Schema extracted successfully!", "green") - # Preguntar si quiere guardar el esquema en un archivo - save_option = input("\n¿Deseas guardar el esquema en un archivo? (s/n): ").strip().lower() - if save_option == "s": - filename = input("Nombre del archivo (por defecto: db_schema.json): ").strip() or "db_schema.json" + # Ask if they want to save the schema to a file + save_option = input("\nDo you want to save the schema to a file? (y/n): ").strip().lower() + if save_option == "y": + filename = input("File name (default: db_schema.json): ").strip() or "db_schema.json" try: with open(filename, 'w') as f: json.dump(schema, f, indent=2, default=str) - _print_colored(f"\n✅ Esquema guardado en: {filename}", "green") + _print_colored(f"\n✅ Schema saved in: {filename}", "green") except Exception as e: - _print_colored(f"❌ Error al guardar el archivo: {str(e)}", "red") + _print_colored(f"❌ Error saving the file: {str(e)}", "red") except Exception as e: - _print_colored(f"❌ Error al mostrar el esquema: {str(e)}", "red") + _print_colored(f"❌ Error showing schema: {str(e)}", "red") import traceback traceback.print_exc() @@ -517,38 +529,38 @@ def get_schema_with_dynamic_import(api_token: str, config_id: str, db_config: Di Database schema """ try: - # Importación dinámica del módulo core + # Dynamic import of the core module import importlib core_module = importlib.import_module('core.client') init_func = getattr(core_module, 'init') - # Creamos una instancia de Corebrain con la configuración seleccionada + # Create a Corebrain instance with the selected configuration cb = init_func( api_token=api_token, config_id=config_id, api_url=api_url, - skip_verification=True # Omitimos verificación para simplificar + skip_verification=True # Skip verification for simplicity ) - # El esquema se extrae automáticamente al inicializar + # The schema is automatically extracted when initializing schema = cb.db_schema - # Si no hay esquema, intentamos extraerlo explícitamente + # If there's no schema, we try to extract it explicitly if not schema or not schema.get("tables"): - _print_colored("Intentando extraer esquema explícitamente...", "yellow") + _print_colored("Trying to extract schema explicitly...", "yellow") schema = cb._extract_db_schema() - # Cerramos la conexión + # Close the connection cb.close() return schema except ImportError: - # Si falla la importación dinámica, intentamos un enfoque alternativo - _print_colored("No se pudo importar el cliente. Usando método alternativo.", "yellow") + # If dynamic import fails, we try an alternative approach + _print_colored("Could not import client. Using alternative method.", "yellow") return extract_db_schema(db_config) except Exception as e: - _print_colored(f"Error al extraer esquema con cliente: {str(e)}", "red") - # Fallback a extracción directa + _print_colored(f"Error extracting schema with client: {str(e)}", "red") + # Fallback to direct extraction return extract_db_schema(db_config) diff --git a/corebrain/corebrain/lib/sso/auth.py b/corebrain/corebrain/lib/sso/auth.py index d065a20..a83736e 100644 --- a/corebrain/corebrain/lib/sso/auth.py +++ b/corebrain/corebrain/lib/sso/auth.py @@ -7,8 +7,8 @@ def __init__(self, config=None): self.config = config or {} self.logger = logging.getLogger(__name__) - # Configuración por defecto - self.sso_url = self.config.get('GLOBODAIN_SSO_URL', 'http://localhost:3000/login') # URL del SSO + # Default configuration + self.sso_url = self.config.get('GLOBODAIN_SSO_URL', 'http://localhost:3000/login') # SSO URL self.client_id = self.config.get('GLOBODAIN_CLIENT_ID', '') self.client_secret = self.config.get('GLOBODAIN_CLIENT_SECRET', '') self.redirect_uri = self.config.get('GLOBODAIN_REDIRECT_URI', '') @@ -26,11 +26,11 @@ def requires_auth(self, session_handler): """ def decorator(func): def wrapper(*args, **kwargs): - # Obtener la sesión actual usando el manejador proporcionado + # Get the current session using the provided handler session = session_handler() if 'user' not in session: - # Aquí retornamos información para que el framework redirija + # Here we return information for the framework to redirect return { 'authenticated': False, 'redirect_url': self.get_login_url() @@ -80,7 +80,7 @@ def verify_token(self, token): return response.json() return None except Exception as e: - self.logger.error(f"Error verificando token: {str(e)}") + self.logger.error(f"Error verifying token: {str(e)}") return None def get_user_info(self, token): @@ -102,7 +102,7 @@ def get_user_info(self, token): return response.json() return None except Exception as e: - self.logger.error(f"Error obteniendo info de usuario: {str(e)}") + self.logger.error(f"Error getting user info: {str(e)}") return None def exchange_code_for_token(self, code): @@ -130,7 +130,7 @@ def exchange_code_for_token(self, code): return response.json() return None except Exception as e: - self.logger.error(f"Error intercambiando código: {str(e)}") + self.logger.error(f"Error exchanging code: {str(e)}") return None def handle_callback(self, code, session_handler, store_user_func=None): @@ -145,27 +145,27 @@ def handle_callback(self, code, session_handler, store_user_func=None): Returns: Redirect URL after processing the code """ - # Intercambiar código por token + # Exchange code for token token_data = self.exchange_code_for_token(code) if not token_data: - # Error al obtener el token + # Error getting the token return self.get_login_url() - # Obtener información del usuario + # Get user information user_info = self.get_user_info(token_data.get('access_token')) if not user_info: - # Error al obtener información del usuario + # Error getting user information return self.get_login_url() - # Guardar información en la sesión + # Save information in the session session = session_handler() session['user'] = user_info session['token'] = token_data - # Si hay una función para almacenar el usuario, ejecutarla + # If there is a function to store the user, execute it if store_user_func and callable(store_user_func): store_user_func(user_info, token_data) - # Redirigir a la URL de éxito o a la URL guardada anteriormente + # Redirect to the success URL or the previously saved URL next_url = session.pop('next_url', self.success_redirect) return next_url \ No newline at end of file diff --git a/corebrain/corebrain/lib/sso/client.py b/corebrain/corebrain/lib/sso/client.py index 0315085..d47e1ac 100644 --- a/corebrain/corebrain/lib/sso/client.py +++ b/corebrain/corebrain/lib/sso/client.py @@ -32,7 +32,7 @@ def __init__( self.client_secret = client_secret self.service_id = service_id self.redirect_uri = redirect_uri - self._token_cache = {} # Cache de tokens verificados + self._token_cache = {} # Verified tokens cache def get_login_url(self, provider: str = None) -> str: @@ -63,17 +63,17 @@ def verify_token(self, token: str) -> Dict[str, Any]: Raises: Exception: If the token is not valid """ - # Verificar si ya tenemos información cacheada y válida del token + # Check if we have cached and valid token information now = datetime.now() if token in self._token_cache: cache_data = self._token_cache[token] if cache_data['expires_at'] > now: return cache_data['user_info'] else: - # Eliminar token expirado del caché + # Delete expired token from cache del self._token_cache[token] - # Verificar token con el servicio SSO + # Verify token with the SSO service headers = { "Authorization": f"Bearer {token}", "Content-Type": "application/json" @@ -86,20 +86,20 @@ def verify_token(self, token: str) -> Dict[str, Any]: ) if response.status_code != 200: - raise Exception(f"Token inválido: {response.text}") + raise Exception(f"Invalid token: {response.text}") - # Obtener información del usuario + # Get user information user_response = requests.get( f"{self.sso_url}/api/users/me", headers=headers ) if user_response.status_code != 200: - raise Exception(f"Error al obtener información del usuario: {user_response.text}") + raise Exception(f"Error getting user information: {user_response.text}") user_info = user_response.json() - # Guardar en caché (15 minutos) + # Save in cache (15 minutes) self._token_cache[token] = { 'user_info': user_info, 'expires_at': now + timedelta(minutes=15) @@ -132,7 +132,7 @@ def authenticate_service(self, token: str) -> Dict[str, Any]: ) if response.status_code != 200: - raise Exception(f"Error de autenticación: {response.text}") + raise Exception(f"Authentication error: {response.text}") return response.json() @@ -155,7 +155,7 @@ def refresh_token(self, refresh_token: str) -> Dict[str, Any]: ) if response.status_code != 200: - raise Exception(f"Error al renovar token: {response.text}") + raise Exception(f"Error renewing token: {response.text}") return response.json() @@ -185,9 +185,9 @@ def logout(self, refresh_token: str, access_token: str) -> bool: ) if response.status_code != 200: - raise Exception(f"Error al cerrar sesión: {response.text}") + raise Exception(f"Error logging out: {response.text}") - # Limpiar cualquier token cacheado + # Clean any cached token if access_token in self._token_cache: del self._token_cache[access_token] diff --git a/corebrain/corebrain/network/__init__.py b/corebrain/corebrain/network/__init__.py index aa079ff..20328ee 100644 --- a/corebrain/corebrain/network/__init__.py +++ b/corebrain/corebrain/network/__init__.py @@ -12,7 +12,6 @@ APIAuthError ) -# Exportación explícita de componentes públicos __all__ = [ 'APIClient', 'APIError', diff --git a/corebrain/corebrain/network/client.py b/corebrain/corebrain/network/client.py index 1176fb1..8875f2c 100644 --- a/corebrain/corebrain/network/client.py +++ b/corebrain/corebrain/network/client.py @@ -40,9 +40,9 @@ class APIAuthError(APIError): class APIClient: """Optimized HTTP client for communication with the Corebrain API.""" - # Constantes para manejo de reintentos y errores + # Constants for retry handling and errors MAX_RETRIES = 3 - RETRY_DELAY = 0.5 # segundos + RETRY_DELAY = 0.5 # seconds RETRY_STATUS_CODES = [408, 429, 500, 502, 503, 504] def __init__(self, base_url: str, default_timeout: int = 10, @@ -56,27 +56,27 @@ def __init__(self, base_url: str, default_timeout: int = 10, verify_ssl: Whether to verify the SSL certificate user_agent: Custom user agent """ - # Normalizar URL base para asegurar que termina con '/' + # Normalize base URL to ensure it ends with '/' self.base_url = base_url if base_url.endswith('/') else base_url + '/' self.default_timeout = default_timeout self.verify_ssl = verify_ssl - # Headers predeterminados + # Default headers self.default_headers = { 'User-Agent': user_agent or 'CorebrainSDK/1.0', 'Accept': 'application/json', 'Content-Type': 'application/json' } - # Crear sesión HTTP con límites y timeouts optimizados + # Create HTTP session with optimized limits and timeouts self.session = httpx.Client( timeout=httpx.Timeout(timeout=default_timeout), verify=verify_ssl, - http2=True, # Usar HTTP/2 si está disponible + http2=True, # Use HTTP/2 if available limits=httpx.Limits(max_connections=100, max_keepalive_connections=20) ) - # Estadísticas y métricas + # Statistics and metrics self.request_count = 0 self.error_count = 0 self.total_request_time = 0 @@ -106,7 +106,7 @@ def get_full_url(self, endpoint: str) -> str: Returns: Full URL """ - # Eliminar '/' inicial si existe para evitar rutas duplicadas + # Remove '/' if it exists to avoid duplicate paths endpoint = endpoint.lstrip('/') return urljoin(self.base_url, endpoint) @@ -122,14 +122,14 @@ def prepare_headers(self, headers: Optional[Dict[str, str]] = None, Returns: Combined headers """ - # Comenzar con headers predeterminados + # Start with default headers final_headers = self.default_headers.copy() - # Añadir headers personalizados + # Add custom headers if headers: final_headers.update(headers) - # Añadir token de autenticación si se proporciona + # Add authentication token if provided if auth_token: final_headers['Authorization'] = f'Bearer {auth_token}' @@ -150,11 +150,11 @@ def handle_response(self, response: Response) -> Response: """ status_code = response.status_code - # Procesar errores según código de estado + # Process errors according to status code if 400 <= status_code < 500: error_detail = None - # Intentar extraer detalles del error del cuerpo JSON + # Try to extract error details from JSON body try: json_data = response.json() if isinstance(json_data, dict): @@ -164,37 +164,37 @@ def handle_response(self, response: Response) -> Response: json_data.get('error') ) except Exception: - # Si no podemos parsear JSON, usar el texto completo + # If we can't parse JSON, use the full text error_detail = response.text[:200] + ('...' if len(response.text) > 200 else '') - # Errores específicos según código + # Specific errors according to status code if status_code == 401: - msg = "Error de autenticación: token inválido o expirado" + msg = "Authentication error: invalid or expired token" logger.error(f"{msg} - {error_detail or ''}") raise APIAuthError(msg, status_code, error_detail, response) elif status_code == 403: - msg = "Acceso prohibido: no tienes permisos suficientes" + msg = "Access denied: you don't have enough permissions" logger.error(f"{msg} - {error_detail or ''}") raise APIAuthError(msg, status_code, error_detail, response) elif status_code == 404: - msg = f"Recurso no encontrado: {response.url}" + msg = f"Resource not found: {response.url}" logger.error(msg) raise APIError(msg, status_code, error_detail, response) elif status_code == 429: - msg = "Demasiadas peticiones: límite de tasa excedido" + msg = "Too many requests: rate limit exceeded" logger.warning(msg) raise APIError(msg, status_code, error_detail, response) else: - msg = f"Error del cliente ({status_code}): {error_detail or 'sin detalles'}" + msg = f"Client error ({status_code}): {error_detail or 'no details'}" logger.error(msg) raise APIError(msg, status_code, error_detail, response) elif 500 <= status_code < 600: - msg = f"Error del servidor ({status_code}): el servidor API encontró un error" + msg = f"Server error ({status_code}): the API server found an error" logger.error(msg) raise APIError(msg, status_code, response.text[:200], response) @@ -233,27 +233,27 @@ def request(self, method: str, endpoint: str, *, url = self.get_full_url(endpoint) final_headers = self.prepare_headers(headers, auth_token) - # Configurar timeout + # Configure timeout request_timeout = timeout or self.default_timeout - # Contador para reintentos + # Counter for retries retries = 0 last_error = None - # Registrar inicio de la petición + # Register start of request start_time = time.time() self.request_count += 1 while retries <= (self.MAX_RETRIES if retry else 0): try: if retries > 0: - # Esperar antes de reintentar con backoff exponencial + # Wait before retrying with exponential backoff wait_time = self.RETRY_DELAY * (2 ** (retries - 1)) - logger.info(f"Reintentando petición ({retries}/{self.MAX_RETRIES}) a {url} después de {wait_time:.2f}s") + logger.info(f"Retrying request ({retries}/{self.MAX_RETRIES}) to {url} after {wait_time:.2f}s") time.sleep(wait_time) - # Realizar la petición - logger.debug(f"Enviando petición {method} a {url}") + # Make the request + logger.debug(f"Sending request {method} to {url}") response = self.session.request( method=method, url=url, @@ -264,58 +264,58 @@ def request(self, method: str, endpoint: str, *, timeout=request_timeout ) - # Verificar si debemos reintentar por código de estado + # Check if we should retry by status code if response.status_code in self.RETRY_STATUS_CODES and retry and retries < self.MAX_RETRIES: - logger.warning(f"Código de estado {response.status_code} recibido, reintentando") + logger.warning(f"Status code {response.status_code} received, retrying") retries += 1 continue - # Procesar la respuesta + # Process the response processed_response = self.handle_response(response) - # Registrar tiempo total + # Register total time elapsed = time.time() - start_time self.total_request_time += elapsed - logger.debug(f"Petición completada en {elapsed:.3f}s con estado {response.status_code}") + logger.debug(f"Request completed in {elapsed:.3f}s with status {response.status_code}") return processed_response except (ConnectError, httpx.HTTPError) as e: last_error = e - # Decidir si reintentamos dependiendo del tipo de error + # Decide if we should retry depending on the type of error if isinstance(e, (ReadTimeout, WriteTimeout, PoolTimeout, ConnectError)) and retry and retries < self.MAX_RETRIES: - logger.warning(f"Error de conexión: {str(e)}, reintentando {retries+1}/{self.MAX_RETRIES}") + logger.warning(f"Connection error: {str(e)}, retrying {retries+1}/{self.MAX_RETRIES}") retries += 1 continue - # No más reintentos o error no recuperable + # No more retries or unrecoverable error self.error_count += 1 elapsed = time.time() - start_time if isinstance(e, (ReadTimeout, WriteTimeout, PoolTimeout)): - logger.error(f"Timeout en petición a {url} después de {elapsed:.3f}s: {str(e)}") - raise APITimeoutError(f"La petición a {endpoint} excedió el tiempo máximo de {request_timeout}s", + logger.error(f"Timeout in request to {url} after {elapsed:.3f}s: {str(e)}") + raise APITimeoutError(f"The request to {endpoint} exceeded the maximum time of {request_timeout}s", response=getattr(e, 'response', None)) else: - logger.error(f"Error de conexión a {url} después de {elapsed:.3f}s: {str(e)}") - raise APIConnectionError(f"Error de conexión a {endpoint}: {str(e)}", + logger.error(f"Connection error to {url} after {elapsed:.3f}s: {str(e)}") + raise APIConnectionError(f"Connection error to {endpoint}: {str(e)}", response=getattr(e, 'response', None)) except Exception as e: - # Error inesperado + # Unexpected error self.error_count += 1 elapsed = time.time() - start_time - logger.error(f"Error inesperado en petición a {url} después de {elapsed:.3f}s: {str(e)}") - raise APIError(f"Error inesperado en petición a {endpoint}: {str(e)}") + logger.error(f"Unexpected error in request to {url} after {elapsed:.3f}s: {str(e)}") + raise APIError(f"Unexpected error in request to {endpoint}: {str(e)}") - # Si llegamos aquí es porque agotamos los reintentos + # If we get here, we have exhausted the retries if last_error: self.error_count += 1 - raise APIError(f"Petición a {endpoint} falló después de {retries} reintentos: {str(last_error)}") + raise APIError(f"Request to {endpoint} failed after {retries} retries: {str(last_error)}") - # Este punto nunca debería alcanzarse - raise APIError(f"Error inesperado en petición a {endpoint}") + # This point should never be reached + raise APIError(f"Unexpected error in request to {endpoint}") def get(self, endpoint: str, **kwargs) -> Response: """Makes a GET request.""" @@ -352,7 +352,7 @@ def get_json(self, endpoint: str, **kwargs) -> Any: try: return response.json() except Exception as e: - raise APIError(f"Error al parsear respuesta JSON: {str(e)}", response=response) + raise APIError(f"Error parsing JSON response: {str(e)}", response=response) def post_json(self, endpoint: str, **kwargs) -> Any: """ @@ -369,9 +369,9 @@ def post_json(self, endpoint: str, **kwargs) -> Any: try: return response.json() except Exception as e: - raise APIError(f"Error al parsear respuesta JSON: {str(e)}", response=response) + raise APIError(f"Error parsing JSON response: {str(e)}", response=response) - # Métodos de alto nivel para operaciones comunes en la API de Corebrain + # High-level methods for common operations in the Corebrain API def check_health(self, timeout: int = 5) -> bool: """ @@ -409,7 +409,7 @@ def verify_token(self, token: str, timeout: int = 5) -> Dict[str, Any]: except APIAuthError: raise except Exception as e: - raise APIAuthError(f"Error al verificar token: {str(e)}") + raise APIAuthError(f"Error verifying token: {str(e)}") def get_api_keys(self, token: str) -> List[Dict[str, Any]]: """ @@ -475,7 +475,7 @@ def exchange_sso_token(self, sso_token: str, user_data: Dict[str, Any]) -> Dict[ data = {"user_data": user_data} return self.post_json("api/auth/sso/token", headers=headers, json=data) - # Métodos para estadísticas y diagnóstico + # Methods for statistics and diagnostics def get_stats(self) -> Dict[str, Any]: """ diff --git a/corebrain/corebrain/sdk.py b/corebrain/corebrain/sdk.py index 7de1491..b08226c 100644 --- a/corebrain/corebrain/sdk.py +++ b/corebrain/corebrain/sdk.py @@ -3,6 +3,5 @@ """ from corebrain.config.manager import ConfigManager -# Re-exportar elementos principales list_configurations = ConfigManager().list_configs remove_configuration = ConfigManager().remove_config \ No newline at end of file diff --git a/corebrain/corebrain/services/schema.py b/corebrain/corebrain/services/schema.py index 4155fb1..d18beea 100644 --- a/corebrain/corebrain/services/schema.py +++ b/corebrain/corebrain/services/schema.py @@ -27,5 +27,4 @@ def get_schema(self, api_token: str, config_id: str) -> Optional[Dict[str, Any]] def optimize_schema(self, schema: Dict[str, Any], query: str = None) -> Dict[str, Any]: """Optimizes an existing schema.""" return self.schema_optimizer.optimize_schema(schema, query) - - # Otros métodos de servicio... \ No newline at end of file + \ No newline at end of file diff --git a/corebrain/corebrain/utils/__init__.py b/corebrain/corebrain/utils/__init__.py index 3c89186..e264aff 100644 --- a/corebrain/corebrain/utils/__init__.py +++ b/corebrain/corebrain/utils/__init__.py @@ -14,7 +14,6 @@ ConfigEncrypter ) -# Configuración de logging logger = logging.getLogger('corebrain') def setup_logger(level=logging.INFO, @@ -40,20 +39,19 @@ def setup_logger(level=logging.INFO, logger.setLevel(level) logger.addHandler(console_handler) - # Handler de archivo si se proporciona ruta + # File handler if path is provided if file_path: file_handler = logging.FileHandler(file_path) file_handler.setFormatter(formatter) logger.addHandler(file_handler) - # Mensajes de diagnóstico logger.debug(f"Logger configurado con nivel {logging.getLevelName(level)}") if file_path: logger.debug(f"Logs escritos a {file_path}") return logger -# Exportación explícita de componentes públicos + # Exportación explícita de componentes públicos __all__ = [ 'serialize_to_json', 'JSONEncoder', diff --git a/corebrain/corebrain/utils/encrypter.py b/corebrain/corebrain/utils/encrypter.py index 286a705..1d18abd 100644 --- a/corebrain/corebrain/utils/encrypter.py +++ b/corebrain/corebrain/utils/encrypter.py @@ -27,16 +27,16 @@ def derive_key_from_password(password: Union[str, bytes], salt: Optional[bytes] if isinstance(password, str): password = password.encode() - # Generar sal si no se proporciona + # Generate salt if not provided if salt is None: salt = os.urandom(16) - # Derivar clave usando PBKDF2 + # Derive key using PBKDF2 kdf = PBKDF2HMAC( algorithm=hashes.SHA256(), length=32, salt=salt, - iterations=100000 # Mayor número de iteraciones = mayor seguridad + iterations=100000 # Higher number of iterations = higher security ) key = kdf.derive(password) @@ -89,7 +89,7 @@ def _init_cipher(self) -> None: """Initializes the encryption object, creating or loading the key as needed.""" key = None - # Si hay ruta de clave, intentar cargar o crear + # If there is a key path, try to load or create if self.key_path: try: if self.key_path.exists(): @@ -97,17 +97,17 @@ def _init_cipher(self) -> None: key = f.read().strip() logger.debug(f"Clave cargada desde {self.key_path}") else: - # Crear directorio padre si no existe + # Create parent directory if it doesn't exist self.key_path.parent.mkdir(parents=True, exist_ok=True) - # Generar nueva clave + # Generate new key key = Fernet.generate_key() - # Guardar clave + # Save key with open(self.key_path, 'wb') as f: f.write(key) - # Asegurar permisos restrictivos (solo el propietario puede leer) + # Ensure restrictive permissions (only the owner can read) try: os.chmod(self.key_path, 0o600) except Exception as e: @@ -116,10 +116,10 @@ def _init_cipher(self) -> None: logger.debug(f"Nueva clave generada y guardada en {self.key_path}") except Exception as e: logger.error(f"Error al gestionar clave en {self.key_path}: {e}") - # En caso de error, generar clave efímera + # If there is an error, generate a temporary key key = None - # Si no tenemos clave, generar una efímera + # If we don't have a key, generate a temporary key if not key: key = Fernet.generate_key() logger.debug("Usando clave efímera generada") @@ -213,7 +213,7 @@ def decrypt_file(self, input_path: Union[str, Path], output_path: Optional[Union input_path = Path(input_path) if not output_path: - # Si termina en .enc, quitar esa extensión + # If it ends in .enc, remove that extension if input_path.suffix == '.enc': output_path = input_path.with_suffix('') else: @@ -245,17 +245,17 @@ def generate_key_file(key_path: Union[str, Path]) -> None: """ key_path = Path(key_path) - # Crear directorio padre si no existe + # Create parent directory if it doesn't exist key_path.parent.mkdir(parents=True, exist_ok=True) - # Generar clave + # Generate key key = Fernet.generate_key() - # Guardar clave + # Save key with open(key_path, 'wb') as f: f.write(key) - # Establecer permisos restrictivos + # Set restrictive permissions try: os.chmod(key_path, 0o600) except Exception as e: diff --git a/corebrain/corebrain/utils/logging.py b/corebrain/corebrain/utils/logging.py index 0ba559d..6b7c5e7 100644 --- a/corebrain/corebrain/utils/logging.py +++ b/corebrain/corebrain/utils/logging.py @@ -10,16 +10,16 @@ from pathlib import Path from typing import Optional, Any, Union -# Niveles de logging personalizados -VERBOSE = 15 # Entre DEBUG e INFO +# Custom logging levels +VERBOSE = 15 # Between DEBUG and INFO -# Configuración predeterminada +# Default configuration DEFAULT_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' DEFAULT_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' DEFAULT_LEVEL = logging.INFO DEFAULT_LOG_DIR = Path.home() / ".corebrain" / "logs" -# Colores para logging en terminal +# Colores de logging en terminal LOG_COLORS = { "DEBUG": "\033[94m", # Azul "VERBOSE": "\033[96m", # Cian @@ -97,46 +97,46 @@ def setup_logger(name: str = "corebrain", Returns: Configured logger """ - # Registrar nivel personalizado VERBOSE + # Register custom VERBOSE level if not hasattr(logging, 'VERBOSE'): logging.addLevelName(VERBOSE, 'VERBOSE') - # Registrar clase de logger personalizada + # Register custom logger class logging.setLoggerClass(VerboseLogger) - # Obtener o crear logger + # Get or create logger logger = logging.getLogger(name) - # Limpiar handlers existentes + # Clean existing handlers for handler in logger.handlers[:]: logger.removeHandler(handler) - # Configurar nivel de logging + # Configure logging level logger.setLevel(level) logger.propagate = propagate - # Formato predeterminado + # Default format fmt = format_string or DEFAULT_FORMAT formatter = ColoredFormatter(fmt, use_colors=use_colors) - # Handler de consola + # Console handler console_handler = logging.StreamHandler() console_handler.setFormatter(formatter) logger.addHandler(console_handler) - # Handler de archivo si se proporciona ruta + # File handler if path is provided if file_path: - # Asegurar que el directorio exista + # Ensure directory exists file_path = Path(file_path) file_path.parent.mkdir(parents=True, exist_ok=True) file_handler = logging.FileHandler(file_path) - # Para archivos, usar formateador sin colores + # For files, use formatter without colors file_formatter = logging.Formatter(fmt) file_handler.setFormatter(file_formatter) logger.addHandler(file_handler) - # Mensajes de diagnóstico + # Diagnostic messages logger.debug(f"Logger '{name}' configurado con nivel {logging.getLevelName(level)}") if file_path: logger.debug(f"Logs escritos a {file_path}") @@ -156,19 +156,19 @@ def get_logger(name: str, level: Optional[int] = None) -> logging.Logger: """ logger = logging.getLogger(name) - # Si el logger no tiene handlers, configurarlo + # If logger has no handlers, configure it if not logger.handlers: - # Determinar si es un logger secundario + # Determine if it is a secondary logger if '.' in name: - # Es un sublogger, configurar para propagar a logger padre + # It is a sublogger, configure to propagate to parent logger logger.propagate = True if level is not None: logger.setLevel(level) else: - # Es un logger principal, configurar completamente + # It is a main logger, configure completely logger = setup_logger(name, level or DEFAULT_LEVEL) elif level is not None: - # Solo actualizar el nivel si se especifica + # Update level only if specified logger.setLevel(level) return logger @@ -189,23 +189,23 @@ def enable_file_logging(logger_name: str = "corebrain", """ logger = logging.getLogger(logger_name) - # Determinar la ruta del archivo de log + # Determine the log file path log_dir = Path(log_dir) if log_dir else DEFAULT_LOG_DIR log_dir.mkdir(parents=True, exist_ok=True) - # Generar nombre de archivo si no se proporciona + # Generate file name if not provided if not filename: timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') filename = f"{logger_name}_{timestamp}.log" file_path = log_dir / filename - # Verificar si ya hay un FileHandler + # Check if there is already a FileHandler for handler in logger.handlers: if isinstance(handler, logging.FileHandler): logger.removeHandler(handler) - # Agregar nuevo FileHandler + # Add new FileHandler file_handler = logging.FileHandler(file_path) formatter = logging.Formatter(DEFAULT_FORMAT) file_handler.setFormatter(formatter) @@ -223,21 +223,21 @@ def set_log_level(level: Union[int, str], level: Logging level (name or integer value) logger_name: Specific logger name (if None, affects all) """ - # Convertir nombre de nivel a valor si es necesario + # Convert level name to value if necessary if isinstance(level, str): level = getattr(logging, level.upper(), logging.INFO) if logger_name: - # Afectar solo al logger especificado + # Affect only the specified logger logger = logging.getLogger(logger_name) logger.setLevel(level) logger.info(f"Nivel de log cambiado a {logging.getLevelName(level)}") else: - # Afectar al logger raíz y a todos los loggers existentes + # Affect the root logger and all existing loggers root = logging.getLogger() root.setLevel(level) - # También afectar a loggers específicos del SDK + # Also affect specific SDK loggers for name in logging.root.manager.loggerDict: if name.startswith("corebrain"): logging.getLogger(name).setLevel(level) \ No newline at end of file diff --git a/corebrain/corebrain/utils/serializer.py b/corebrain/corebrain/utils/serializer.py index c230c3e..6652e02 100644 --- a/corebrain/corebrain/utils/serializer.py +++ b/corebrain/corebrain/utils/serializer.py @@ -10,13 +10,13 @@ class JSONEncoder(json.JSONEncoder): """Custom JSON serializer for special types.""" def default(self, obj): - # Objetos datetime + # Objects datetime if isinstance(obj, (datetime, date, time)): return obj.isoformat() - # Objetos timedelta - elif hasattr(obj, 'total_seconds'): # Para objetos timedelta + # Objects timedelta + elif hasattr(obj, 'total_seconds'): # For objects timedelta return obj.total_seconds() - # ObjectId de MongoDB + # ObjectId from MongoDB elif isinstance(obj, ObjectId): return str(obj) # Bytes o bytearray @@ -25,7 +25,7 @@ def default(self, obj): # Decimal elif isinstance(obj, Decimal): return float(obj) - # Otros tipos + # Other types return super().default(obj) def serialize_to_json(obj): diff --git a/corebrain/db/connectors/nosql.py b/corebrain/db/connectors/nosql.py index 21e1819..95e0093 100644 --- a/corebrain/db/connectors/nosql.py +++ b/corebrain/db/connectors/nosql.py @@ -20,7 +20,7 @@ # Whe nadding new DB type write a try to it from user try: - import corebrain.db.connectors.subconnectors.nosql.mongodb as mongodb_subconnector + import corebrain.db.connectors.NoSQL.mongodb as mongodb_subconnector MONGO_MODULES = True except ImportError: MONGO_MODULES = False diff --git a/corebrain/db/interface.py b/corebrain/db/interface.py index 25fd433..8e08d67 100644 --- a/corebrain/db/interface.py +++ b/corebrain/db/interface.py @@ -27,10 +27,4 @@ def execute_query(self, connection: Any, query: str) -> List[Dict[str, Any]]: @abstractmethod def close(self, connection: Any) -> None: """Closes the connection.""" - pass - -# Posteriormente se podrían implementar conectores específicos: -# - SQLiteConnector -# - MySQLConnector -# - PostgresConnector -# - NoSQLConnector \ No newline at end of file + pass \ No newline at end of file diff --git a/corebrain/db/schema_file.py b/corebrain/db/schema_file.py index 2ae76d6..c5979c3 100644 --- a/corebrain/db/schema_file.py +++ b/corebrain/db/schema_file.py @@ -36,8 +36,6 @@ def extract_db_schema(db_config: Dict[str, Any]) -> Dict[str, Any]: try: if db_type == "sql": - # Code for SQL databases... - # [Kept the same] pass # Handle both "nosql" and "mongodb" as valid types From 3fac1ed8db43282d1ebe7d2bdee28cccd1328355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Ayuso?= Date: Mon, 26 May 2025 18:26:52 +0200 Subject: [PATCH 66/81] Duplicated files and wrong structure organized fixed. Error in --configure command fixed. --- corebrain/.gitignore | 178 --- corebrain/.gitmodules | 3 - corebrain/CONTRIBUTING.md | 147 -- corebrain/LICENSE | 21 - corebrain/README.md | 203 --- corebrain/cli/commands.py | 4 +- corebrain/corebrain/__init__.py | 81 - corebrain/corebrain/cli.py | 8 - corebrain/corebrain/cli/__init__.py | 57 - corebrain/corebrain/cli/__main__.py | 12 - corebrain/corebrain/cli/auth/__init__.py | 22 - corebrain/corebrain/cli/auth/api_keys.py | 299 ---- corebrain/corebrain/cli/auth/sso.py | 452 ------ corebrain/corebrain/cli/commands.py | 453 ------ corebrain/corebrain/cli/common.py | 15 - corebrain/corebrain/cli/config.py | 489 ------ corebrain/corebrain/cli/utils.py | 595 ------- corebrain/corebrain/config/__init__.py | 10 - corebrain/corebrain/config/manager.py | 235 --- corebrain/corebrain/core/__init__.py | 20 - corebrain/corebrain/core/client.py | 1364 ----------------- corebrain/corebrain/core/common.py | 225 --- corebrain/corebrain/core/query.py | 1037 ------------- corebrain/corebrain/core/test_utils.py | 157 -- corebrain/corebrain/db/__init__.py | 26 - corebrain/corebrain/db/connector.py | 33 - corebrain/corebrain/db/connectors/__init__.py | 31 - corebrain/corebrain/db/connectors/mongodb.py | 474 ------ corebrain/corebrain/db/connectors/nosql.py | 366 ----- corebrain/corebrain/db/connectors/sql.py | 598 -------- corebrain/corebrain/db/engines.py | 16 - corebrain/corebrain/db/factory.py | 29 - corebrain/corebrain/db/interface.py | 36 - corebrain/corebrain/db/schema/__init__.py | 11 - corebrain/corebrain/db/schema/extractor.py | 123 -- corebrain/corebrain/db/schema/optimizer.py | 157 -- corebrain/corebrain/db/schema_file.py | 566 ------- corebrain/corebrain/lib/sso/__init__.py | 4 - corebrain/corebrain/lib/sso/auth.py | 171 --- corebrain/corebrain/lib/sso/client.py | 194 --- corebrain/corebrain/network/__init__.py | 21 - corebrain/corebrain/network/client.py | 502 ------ corebrain/corebrain/sdk.py | 7 - corebrain/corebrain/services/schema.py | 30 - corebrain/corebrain/utils/__init__.py | 64 - corebrain/corebrain/utils/encrypter.py | 264 ---- corebrain/corebrain/utils/logging.py | 243 --- corebrain/corebrain/utils/serializer.py | 33 - .../corebrain/wrappers/csharp/.editorconfig | 432 ------ .../corebrain/wrappers/csharp/.gitignore | 417 ----- .../wrappers/csharp/.vscode/settings.json | 3 - .../wrappers/csharp/.vscode/tasks.json | 32 - .../CorebrainCS.Tests.csproj | 14 - .../csharp/CorebrainCS.Tests/Program.cs | 10 - .../csharp/CorebrainCS.Tests/README.md | 4 - .../corebrain/wrappers/csharp/CorebrainCS.sln | 48 - .../csharp/CorebrainCS/CorebrainCS.cs | 175 --- .../csharp/CorebrainCS/CorebrainCS.csproj | 7 - corebrain/corebrain/wrappers/csharp/LICENSE | 21 - corebrain/corebrain/wrappers/csharp/README.md | 77 - .../wrappers/csharp_cli_api/.gitignore | 548 ------- .../csharp_cli_api/.vscode/settings.json | 3 - .../csharp_cli_api/CorebrainCLIAPI.sln | 50 - .../wrappers/csharp_cli_api/README.md | 18 - .../wrappers/csharp_cli_api/src/.editorconfig | 432 ------ .../src/CorebrainCLIAPI/CommandController.cs | 70 - .../CorebrainCLIAPI/CorebrainCLIAPI.csproj | 20 - .../src/CorebrainCLIAPI/CorebrainSettings.cs | 8 - .../src/CorebrainCLIAPI/Program.cs | 49 - .../Properties/launchSettings.json | 23 - .../appsettings.Development.json | 8 - .../src/CorebrainCLIAPI/appsettings.json | 14 - corebrain/docs/Makefile | 20 - corebrain/docs/README.md | 13 - corebrain/docs/make.bat | 35 - corebrain/docs/source/_static/custom.css | 0 corebrain/docs/source/conf.py | 25 - corebrain/docs/source/corebrain.cli.auth.rst | 29 - corebrain/docs/source/corebrain.cli.rst | 53 - corebrain/docs/source/corebrain.config.rst | 21 - corebrain/docs/source/corebrain.core.rst | 45 - .../docs/source/corebrain.db.connectors.rst | 29 - corebrain/docs/source/corebrain.db.rst | 62 - corebrain/docs/source/corebrain.db.schema.rst | 29 - corebrain/docs/source/corebrain.network.rst | 21 - corebrain/docs/source/corebrain.rst | 42 - corebrain/docs/source/corebrain.utils.rst | 37 - corebrain/docs/source/index.rst | 14 - corebrain/docs/source/modules.rst | 7 - corebrain/examples/add_config.py | 27 - corebrain/examples/complex.py | 23 - corebrain/examples/list_schema.py | 162 -- corebrain/examples/simple.py | 15 - corebrain/health.py | 47 - corebrain/pyproject.toml | 85 - corebrain/setup.ps1 | 5 - corebrain/setup.py | 38 - corebrain/setup.sh | 6 - examples/complex.py | 6 +- 99 files changed, 5 insertions(+), 13460 deletions(-) delete mode 100644 corebrain/.gitignore delete mode 100644 corebrain/.gitmodules delete mode 100644 corebrain/CONTRIBUTING.md delete mode 100644 corebrain/LICENSE delete mode 100644 corebrain/README.md delete mode 100644 corebrain/corebrain/__init__.py delete mode 100644 corebrain/corebrain/cli.py delete mode 100644 corebrain/corebrain/cli/__init__.py delete mode 100644 corebrain/corebrain/cli/__main__.py delete mode 100644 corebrain/corebrain/cli/auth/__init__.py delete mode 100644 corebrain/corebrain/cli/auth/api_keys.py delete mode 100644 corebrain/corebrain/cli/auth/sso.py delete mode 100644 corebrain/corebrain/cli/commands.py delete mode 100644 corebrain/corebrain/cli/common.py delete mode 100644 corebrain/corebrain/cli/config.py delete mode 100644 corebrain/corebrain/cli/utils.py delete mode 100644 corebrain/corebrain/config/__init__.py delete mode 100644 corebrain/corebrain/config/manager.py delete mode 100644 corebrain/corebrain/core/__init__.py delete mode 100644 corebrain/corebrain/core/client.py delete mode 100644 corebrain/corebrain/core/common.py delete mode 100644 corebrain/corebrain/core/query.py delete mode 100644 corebrain/corebrain/core/test_utils.py delete mode 100644 corebrain/corebrain/db/__init__.py delete mode 100644 corebrain/corebrain/db/connector.py delete mode 100644 corebrain/corebrain/db/connectors/__init__.py delete mode 100644 corebrain/corebrain/db/connectors/mongodb.py delete mode 100644 corebrain/corebrain/db/connectors/nosql.py delete mode 100644 corebrain/corebrain/db/connectors/sql.py delete mode 100644 corebrain/corebrain/db/engines.py delete mode 100644 corebrain/corebrain/db/factory.py delete mode 100644 corebrain/corebrain/db/interface.py delete mode 100644 corebrain/corebrain/db/schema/__init__.py delete mode 100644 corebrain/corebrain/db/schema/extractor.py delete mode 100644 corebrain/corebrain/db/schema/optimizer.py delete mode 100644 corebrain/corebrain/db/schema_file.py delete mode 100644 corebrain/corebrain/lib/sso/__init__.py delete mode 100644 corebrain/corebrain/lib/sso/auth.py delete mode 100644 corebrain/corebrain/lib/sso/client.py delete mode 100644 corebrain/corebrain/network/__init__.py delete mode 100644 corebrain/corebrain/network/client.py delete mode 100644 corebrain/corebrain/sdk.py delete mode 100644 corebrain/corebrain/services/schema.py delete mode 100644 corebrain/corebrain/utils/__init__.py delete mode 100644 corebrain/corebrain/utils/encrypter.py delete mode 100644 corebrain/corebrain/utils/logging.py delete mode 100644 corebrain/corebrain/utils/serializer.py delete mode 100644 corebrain/corebrain/wrappers/csharp/.editorconfig delete mode 100644 corebrain/corebrain/wrappers/csharp/.gitignore delete mode 100644 corebrain/corebrain/wrappers/csharp/.vscode/settings.json delete mode 100644 corebrain/corebrain/wrappers/csharp/.vscode/tasks.json delete mode 100644 corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/CorebrainCS.Tests.csproj delete mode 100644 corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs delete mode 100644 corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/README.md delete mode 100644 corebrain/corebrain/wrappers/csharp/CorebrainCS.sln delete mode 100644 corebrain/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs delete mode 100644 corebrain/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.csproj delete mode 100644 corebrain/corebrain/wrappers/csharp/LICENSE delete mode 100644 corebrain/corebrain/wrappers/csharp/README.md delete mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/.gitignore delete mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/.vscode/settings.json delete mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/CorebrainCLIAPI.sln delete mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/README.md delete mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/src/.editorconfig delete mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CommandController.cs delete mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainCLIAPI.csproj delete mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainSettings.cs delete mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Program.cs delete mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Properties/launchSettings.json delete mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.Development.json delete mode 100644 corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.json delete mode 100644 corebrain/docs/Makefile delete mode 100644 corebrain/docs/README.md delete mode 100644 corebrain/docs/make.bat delete mode 100644 corebrain/docs/source/_static/custom.css delete mode 100644 corebrain/docs/source/conf.py delete mode 100644 corebrain/docs/source/corebrain.cli.auth.rst delete mode 100644 corebrain/docs/source/corebrain.cli.rst delete mode 100644 corebrain/docs/source/corebrain.config.rst delete mode 100644 corebrain/docs/source/corebrain.core.rst delete mode 100644 corebrain/docs/source/corebrain.db.connectors.rst delete mode 100644 corebrain/docs/source/corebrain.db.rst delete mode 100644 corebrain/docs/source/corebrain.db.schema.rst delete mode 100644 corebrain/docs/source/corebrain.network.rst delete mode 100644 corebrain/docs/source/corebrain.rst delete mode 100644 corebrain/docs/source/corebrain.utils.rst delete mode 100644 corebrain/docs/source/index.rst delete mode 100644 corebrain/docs/source/modules.rst delete mode 100644 corebrain/examples/add_config.py delete mode 100644 corebrain/examples/complex.py delete mode 100644 corebrain/examples/list_schema.py delete mode 100644 corebrain/examples/simple.py delete mode 100644 corebrain/health.py delete mode 100644 corebrain/pyproject.toml delete mode 100644 corebrain/setup.ps1 delete mode 100644 corebrain/setup.py delete mode 100644 corebrain/setup.sh diff --git a/corebrain/.gitignore b/corebrain/.gitignore deleted file mode 100644 index cf98110..0000000 --- a/corebrain/.gitignore +++ /dev/null @@ -1,178 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class -venv/ -.tofix/ -README-no-valid.md -requirements.txt - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -#lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# UV -# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -#uv.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/latest/usage/project/#working-with-version-control -.pdm.toml -.pdm-python -.pdm-build/ - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ - -# Ruff stuff: -.ruff_cache/ - -# PyPI configuration file -.pypirc diff --git a/corebrain/.gitmodules b/corebrain/.gitmodules deleted file mode 100644 index a034212..0000000 --- a/corebrain/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "corebrain/CLI-UI"] - path = corebrain/CLI-UI - url = https://github.com/Luki20091/CLI-UI.git diff --git a/corebrain/CONTRIBUTING.md b/corebrain/CONTRIBUTING.md deleted file mode 100644 index 47e6927..0000000 --- a/corebrain/CONTRIBUTING.md +++ /dev/null @@ -1,147 +0,0 @@ -# How to Contribute to Corebrain SDK - -Thank you for your interest in contributing to CoreBrain SDK! This document provides guidelines for contributing to the project. - -## Code of Conduct - -By participating in this project, you commit to maintaining a respectful and collaborative environment. - -## How to Contribute - -### Reporting Bugs - -1. Verify that the bug hasn't already been reported in the [issues](https://github.com/ceoweggo/Corebrain/issues) -2. Use the bug template to create a new issue -3. Include as much detail as possible: steps to reproduce, environment, versions, etc. -4. If possible, include a minimal example that reproduces the problem - -### Suggesting Improvements - -1. Check the [issues](https://github.com/ceoweggo/Corebrain/issues) to see if it has already been suggested -2. Use the feature template to create a new issue -3. Clearly describe the improvement and justify its value - -### Submitting Changes - -1. Fork the repository -2. Create a branch for your change (`git checkout -b feature/amazing-feature`) -3. Make your changes following the code conventions -4. Write tests for your changes -5. Ensure all tests pass -6. Commit your changes (`git commit -m 'Add amazing feature'`) -7. Push your branch (`git push origin feature/amazing-feature`) -8. Open a Pull Request - -## Development Environment - -### Installation for Development - -```bash -# Clone the repository -git clone https://github.com/ceoweggo/Corebrain.git -cd sdk - -# Create virtual environment -python -m venv venv -source venv/bin/activate # On Windows: venv\Scripts\activate - -# Install for development -pip install -e ".[dev]" -``` - -### Project Structure - -``` -v1/ -├── corebrain/ # Main package -│ ├── __init__.py -│ ├── _pycache_/ -│ ├── cli/ # Command-line interface -│ ├── config/ # Configuration management -│ ├── core/ # Core functionality -│ ├── db/ # Database interactions -│ ├── lib/ # Library components -│ └── SSO/ # Globodain SSO Authentication -│ ├── network/ # Network functionality -│ ├── services/ # Service implementations -│ ├── utils/ # Utility functions -│ ├── cli.py # CLI entry point -│ └── sdk.py # SDK entry point -├── corebrain.egg-info/ # Package metadata -├── docs/ # Documentation -├── examples/ # Usage examples -├── screenshots/ # Project screenshots -├── venv/ # Virtual environment (not to be committed) -├── .github/ # GitHub files directory -├── _pycache_/ # Python cache files -├── .tofix/ # Files to be fixed -├── .gitignore # Git ignore rules -├── CONTRIBUTING.md # Contribution guidelines -├── health.py # Health check script -├── LICENSE # License information -├── pyproject.toml # Project configuration -├── README-no-valid.md # Outdated README -├── README.md # Project overview -├── requirements.txt # Production dependencies -└── setup.py # Package setup -``` - -### Running Tests - -```bash -# Run all tests -pytest - -# Run specific test file -pytest tests/test_specific.py - -# Run tests with coverage -pytest --cov=corebrain -``` - -## Coding Standards - -### Style Guide - -- We follow [PEP 8](https://www.python.org/dev/peps/pep-0008/) for Python code -- Use 4 spaces for indentation -- Maximum line length is 88 characters -- Use descriptive variable and function names - -### Documentation - -- All modules, classes, and functions should have docstrings -- Follow the [Google docstring format](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings) -- Keep documentation up-to-date with code changes - -### Commit Messages - -- Use clear, concise commit messages -- Start with a verb in the present tense (e.g., "Add feature" not "Added feature") -- Reference issue numbers when applicable (e.g., "Fix #123: Resolve memory leak") - -## Pull Request Process - -1. Update documentation if necessary -2. Add or update tests as needed -3. Ensure CI checks pass -4. Request a review from maintainers -5. Address review feedback -6. Maintainers will merge your PR once approved - -## Release Process - -Our maintainers follow semantic versioning (MAJOR.MINOR.PATCH): -- MAJOR version for incompatible API changes -- MINOR version for backward-compatible functionality -- PATCH version for backward-compatible bug fixes - -## Getting Help - -If you need help with anything: -- Join our [Discord community](https://discord.gg/m2AXjPn2yV) -- Join our [Whatsapp Channel](https://whatsapp.com/channel/0029Vap43Vy5EjxvR4rncQ1I) -- Ask questions in the GitHub Discussions -- Contact the maintainers at ruben@globodain.com - -Thank you for contributing to Corebrain SDK! \ No newline at end of file diff --git a/corebrain/LICENSE b/corebrain/LICENSE deleted file mode 100644 index 30ba189..0000000 --- a/corebrain/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2025 Rubén Ayuso - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/corebrain/README.md b/corebrain/README.md deleted file mode 100644 index 8d22293..0000000 --- a/corebrain/README.md +++ /dev/null @@ -1,203 +0,0 @@ -# Corebrain SDK - -![CI Status](https://github.com/ceoweggo/Corebrain/workflows/Corebrain%20SDK%20CI/CD/badge.svg) -[![PyPI version](https://badge.fury.io/py/corebrain.svg)](https://badge.fury.io/py/corebrain) -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) - -SDK for natural language queries to relational and non-relational databases. Enables interaction with databases using natural language questions. - -## ✨ Features - -- **Natural Language Queries**: Transforms human language questions into database queries (SQL/NoSQL) -- **Multi-Database Support**: Compatible with SQLite, MySQL, PostgreSQL, and MongoDB -- **Unified Interface**: Consistent API across different database types -- **Built-in CLI**: Interact with your databases directly from the terminal -- **Strong Security**: Robust authentication and secure credential management -- **Highly Extensible**: Designed for easy integration with new engines and features -- **Comprehensive Documentation**: Usage examples, API reference, and step-by-step guides - -## 📋 Requirements - -- Python 3.8+ -- Specific dependencies based on the database engine: - - **SQLite**: Included in Python - - **PostgreSQL**: `psycopg2-binary` - - **MySQL**: `mysql-connector-python` - - **MongoDB**: `pymongo` - -## 🔧 Installation - -### From PyPI (recommended) - -```bash -# Minimal installation -pip install corebrain - -### From source code - -```bash -git clone https://github.com/ceoweggo/Corebrain.git -pip install -e . -``` - -## 🚀 Quick Start Guide - -### Initialization - -# > **⚠️ IMPORTANT:** -# > * If you don't have an existing configuration, first run `corebrain --configure` -# > * If you need to generate a new API key, use `corebrain --create` -# > * Never share your API key in public repositories. Use environment variables instead. - - -```python -from corebrain import init - -# Initialize with a previously saved configuration -corebrain = init( - api_key="your_api_key", - config_id="your_config_id" -) -``` - -### Making Natural Language Queries - -```python -# Simple query -result = client.ask("How many active users are there?") -print(result["explanation"]) # Natural language explanation -print(result["query"]) # Generated SQL/NoSQL query -print(result["results"]) # Query results - -# Query with additional parameters -result = client.ask( - "Show the last 5 orders", - collection_name="orders", - limit=5, - filters={"status": "completed"} -) - -# Iterate over the results -for item in result["results"]: - print(item) -``` - -### Getting the Database Schema - -```python -# Get the complete schema -schema = client.db_schema - -# List all tables/collections -tables = client.list_collections_name() -print(tables) -``` - -### Closing the Connection - -```python -# It's recommended to close the connection when finished -client.close() - -# Or use the with context -with init(api_key="your_api_key", config_id="your_config_id") as client: - result = client.ask("How many users are there?") - print(result["explanation"]) -``` - -## 🖥️ Command Line Interface Usage - -### Configure Connection - -```bash -# Init configuration -corebrain --configure -``` - -### Display Database Schema - -```bash -# Show complete schema -corebrain --show-schema -``` - -### List Configurations - -```bash -# List all configurations -corebrain --list-configs -``` - -## 📝 Advanced Documentation - -### Configuration Management - -```python -from corebrain import list_configurations, remove_configuration, get_config - -# List all configurations -configs = list_configurations(api_token="your_api_token") -print(configs) - -# Get details of a configuration -config = get_config(api_token="your_api_token", config_id="your_config_id") -print(config) - -# Remove a configuration -removed = remove_configuration(api_token="your_api_token", config_id="your_config_id") -print(f"Configuration removed: {removed}") -``` - -## 🧪 Testing and Development - -### Development Installation - -```bash -# Clone the repository -git clone https://github.com/ceoweggo/Corebrain.git -cd corebrain - -# Install in development mode with extra tools -pip install -e ".[dev,all_db]" -``` - -### Verifying Style and Typing - -```bash -# Check style with flake8 -flake8 . - -# Check typing with mypy -mypy core db cli utils - -# Format code with black -black . -``` - -### Continuous Integration and Deployment (CI/CD) - -The project uses GitHub Actions to automate: - -1. **Testing**: Runs tests on multiple Python versions (3.8-3.11) -2. **Quality Verification**: Checks style, typing, and formatting -3. **Coverage**: Generates code coverage reports -4. **Automatic Publication**: Publishes new versions to PyPI when tags are created -5. **Docker Images**: Builds and publishes Docker images with each version - -You can see the complete configuration in `.github/workflows/ci.yml`. - -## 🛠️ Contributions - -Contributions are welcome! To contribute: - -1. Fork the repository -2. Create a branch for your feature (`git checkout -b feature/amazing-feature`) -3. Commit your changes (`git commit -m 'Add some amazing feature'`) -4. Push to the branch (`git push origin feature/amazing-feature`) -5. Open a Pull Request - -Please make sure your changes pass all tests and comply with the style guidelines. - -## 📄 License - -Distributed under the MIT License. See `LICENSE` for more information. \ No newline at end of file diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index 5a0dc10..f3314dc 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -16,7 +16,7 @@ from corebrain.cli.config import configure_sdk, get_api_credential from corebrain.cli.utils import print_colored from corebrain.config.manager import ConfigManager -from corebrain.lib.sso.auth import GlobodainSSOAuth +from corebrain.lib.sso.auth import GlobodainSSOAuth def main_cli(argv: Optional[List[str]] = None) -> int: """ @@ -62,7 +62,7 @@ def authentication_with_api_key_return(): sso_url = os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL api_key_selected, user_data, api_token = authenticate_with_sso_and_api_key_request(sso_url) - if sso_token: + if api_token: try: print_colored("✅ User authenticated and SDK is now connected to API.", "green") print_colored("✅ Returning User data.", "green") diff --git a/corebrain/corebrain/__init__.py b/corebrain/corebrain/__init__.py deleted file mode 100644 index 151df33..0000000 --- a/corebrain/corebrain/__init__.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -Corebrain SDK. - -This package provides a Python SDK for interacting with the Corebrain API -and enables natural language queries to relational and non-relational databases. -""" -import logging -from typing import Dict, Any, List, Optional - -logger = logging.getLogger(__name__) -logger.addHandler(logging.NullHandler()) - -from corebrain.db.engines import get_available_engines -from corebrain.core.client import Corebrain -from corebrain.config.manager import ConfigManager - -__all__ = [ - 'init', - 'extract_db_schema', - 'list_configurations', - 'remove_configuration', - 'get_available_engines', - 'get_config', - '__version__' -] - -__version__ = "1.0.0" - -def init(api_key: str, config_id: str, skip_verification: bool = False) -> Corebrain: - """ - Initialize the Corebrain SDK with the provided API key and configuration. - - Args: - api_key: API Key de Corebrain - config_id: ID de la configuración a usar - - Returns: - Instancia de Corebrain configurada - """ - return Corebrain(api_key=api_key, config_id=config_id, skip_verification=skip_verification) - -def list_configurations(api_key: str) -> List[str]: - """ - Lists the available configurations for an API key. - - Args: - api_key: Corebrain API Key - - Returns: - List of available configuration IDs - """ - config_manager = ConfigManager() - return config_manager.list_configs(api_key) - -def remove_configuration(api_key: str, config_id: str) -> bool: - """ - Deletes a specific configuration. - - Args: - api_key: Corebrain API Key - config_id: ID of the configuration to delete - - Returns: - True if deleted successfully, False otherwise - """ - config_manager = ConfigManager() - return config_manager.remove_config(api_key, config_id) - -def get_config(api_key: str, config_id: str) -> Optional[Dict[str, Any]]: - """ - Retrieves a specific configuration. - - Args: - api_key: Corebrain API Key - config_id: ID of the configuration to retrieve - - Returns: - Dictionary with the configuration or None if it does not exist - """ - config_manager = ConfigManager() - return config_manager.get_config(api_key, config_id) \ No newline at end of file diff --git a/corebrain/corebrain/cli.py b/corebrain/corebrain/cli.py deleted file mode 100644 index 7e17025..0000000 --- a/corebrain/corebrain/cli.py +++ /dev/null @@ -1,8 +0,0 @@ -""" -Entry point for the Corebrain CLI for compatibility. -""" -from corebrain.cli.__main__ import main - -if __name__ == "__main__": - import sys - sys.exit(main()) \ No newline at end of file diff --git a/corebrain/corebrain/cli/__init__.py b/corebrain/corebrain/cli/__init__.py deleted file mode 100644 index 53672d1..0000000 --- a/corebrain/corebrain/cli/__init__.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -Command-line interface for the Corebrain SDK. - -This module provides a command-line interface to configure -and use the Corebrain SDK for natural language queries to databases. -""" -import sys -from typing import Optional, List - -# Importar componentes principales para CLI -from corebrain.cli.commands import main_cli -from corebrain.cli.utils import print_colored, ProgressTracker, get_free_port -from corebrain.cli.config import ( - configure_sdk, - get_db_type, - get_db_engine, - get_connection_params, - test_database_connection, - select_excluded_tables -) -from corebrain.cli.auth import ( - authenticate_with_sso, - fetch_api_keys, - exchange_sso_token_for_api_token, - verify_api_token -) - - -# Exportación explícita de componentes públicos -__all__ = [ - 'main_cli', - 'run_cli', - 'print_colored', - 'ProgressTracker', - 'get_free_port', - 'configure_sdk', - 'authenticate_with_sso', - 'fetch_api_keys', - 'exchange_sso_token_for_api_token', - 'verify_api_token' -] - -# Función de conveniencia para ejecutar CLI -def run_cli(argv: Optional[List[str]] = None) -> int: - """ - Run the CLI with the provided arguments. - - Args: - argv: List of arguments (use sys.argv if None) - - Returns: - Exit code - """ - if argv is None: - argv = sys.argv[1:] - - return main_cli(argv) \ No newline at end of file diff --git a/corebrain/corebrain/cli/__main__.py b/corebrain/corebrain/cli/__main__.py deleted file mode 100644 index db91155..0000000 --- a/corebrain/corebrain/cli/__main__.py +++ /dev/null @@ -1,12 +0,0 @@ -""" -Entry point to run the CLI as a module. -""" -import sys -from corebrain.cli.commands import main_cli - -def main(): - """Main function for the entry point in pyproject.toml.""" - return main_cli() - -if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file diff --git a/corebrain/corebrain/cli/auth/__init__.py b/corebrain/corebrain/cli/auth/__init__.py deleted file mode 100644 index 873572d..0000000 --- a/corebrain/corebrain/cli/auth/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -Authentication modules for the Corebrain CLI. - -This package provides functionality for authentication, -token management, and API keys in the Corebrain CLI. -""" -from corebrain.cli.auth.sso import authenticate_with_sso, TokenHandler -from corebrain.cli.auth.api_keys import ( - fetch_api_keys, - exchange_sso_token_for_api_token, - verify_api_token, - get_api_key_id_from_token -) -# Exportación explícita de componentes públicos -__all__ = [ - 'authenticate_with_sso', - 'TokenHandler', - 'fetch_api_keys', - 'exchange_sso_token_for_api_token', - 'verify_api_token', - 'get_api_key_id_from_token' -] \ No newline at end of file diff --git a/corebrain/corebrain/cli/auth/api_keys.py b/corebrain/corebrain/cli/auth/api_keys.py deleted file mode 100644 index 5be72f0..0000000 --- a/corebrain/corebrain/cli/auth/api_keys.py +++ /dev/null @@ -1,299 +0,0 @@ -""" -API Keys Management for the CLI. -""" -import uuid -import httpx - -from typing import Optional, Dict, Any, Tuple - -from corebrain.cli.utils import print_colored -from corebrain.network.client import http_session -from corebrain.core.client import Corebrain - -def verify_api_token(token: str, api_url: Optional[str] = None, user_data: Optional[Dict[str, Any]] = None) -> Tuple[bool, Optional[Dict[str, Any]]]: - """ - Verifies if an API token is valid. - - Args: - token (str): API token to verify. - api_url (str, optional): API URL. Defaults to None. - user_data (dict, optional): User data. Defaults to None. - - Returns: - tuple: (validity (bool), user information (dict)) if valid, else (False, None). - """ - try: - # Create a temporary SDK instance to verify the token - config = {"type": "test", "config_id": str(uuid.uuid4())} - kwargs = {"api_token": token, "db_config": config} - - if user_data: - kwargs["user_data"] = user_data - - if api_url: - kwargs["api_url"] = api_url - - sdk = Corebrain(**kwargs) - return True, sdk.user_info - except Exception as e: - print_colored(f"Error verifying API token: {str(e)}", "red") - return False, None - -def fetch_api_keys(api_url: str, api_token: str, user_data: Dict[str, Any]) -> Optional[str]: - """ - Retrieves the available API keys for the user and allows selecting one. - - Args: - api_url: Base URL of the Corebrain API - api_token: API token (exchanged from SSO token) - user_data: User data - - Returns: - Selected API key or None if none is selected - """ - if not user_data or 'id' not in user_data: - print_colored("Could not identify the user to retrieve their API keys.", "yellow") - return None - - try: - # Ensure protocol in URL - if not api_url.startswith(("http://", "https://")): - api_url = "https://" + api_url - - # Remove trailing slash if it exists - if api_url.endswith('/'): - api_url = api_url[:-1] - - # Build endpoint to get API keys - endpoint = f"{api_url}/api/auth/api-keys" - - print_colored(f"Requesting user's API keys...", "blue") - - # Configure client with timeout and error handling - headers = { - "Authorization": f"Bearer {api_token}", - "Content-Type": "application/json" - } - - response = http_session.get(endpoint, headers=headers) - - # Verify response - if response.status_code == 200: - try: - api_keys_data = response.json() - # Verify response format - if not isinstance(api_keys_data, (list, dict)): - print_colored(f"Unexpected response format: {type(api_keys_data)}", "yellow") - return None - - # Handle both direct list and dictionary with list - api_keys = api_keys_data if isinstance(api_keys_data, list) else api_keys_data.get("data", []) - - if not api_keys: - print_colored("No API keys available for this user.", "yellow") - return None - - print_colored(f"\nFound {len(api_keys)} API keys", "green") - print_colored("\n=== Available API Keys ===", "blue") - - # Show available API keys - for i, key_info in enumerate(api_keys, 1): - key_id = key_info.get('id', 'No ID') - key_value = key_info.get('key', 'No value') - key_name = key_info.get('name', 'No name') - key_active = key_info.get('active') - - # Show status with color - status_color = "green" if key_active == True else "red" - status_text = "Active" if key_active == True else "Inactive" - - print(f"{i}. {key_name} - {print_colored(status_text, status_color, return_str=True)} (Value: {key_value})") - - # Ask user to select an API key - while True: - try: - choice = input(f"\nSelect an API key (1-{len(api_keys)}) or press Enter to cancel: ").strip() - - # Allow canceling and using API token - if not choice: - print_colored("No API key selected.", "yellow") - return None - - choice_num = int(choice) - if 1 <= choice_num <= len(api_keys): - selected_key = api_keys[choice_num - 1] - - # Verify if the key is active - if selected_key.get('active') != True: - print_colored("⚠️ The selected API key is not active. Select another one.", "yellow") - continue - - # Get information of the selected key - key_name = selected_key.get('name', 'Unknown') - key_value = selected_key.get('key', None) - - if not key_value: - print_colored("⚠️ The selected API key does not have a valid value.", "yellow") - continue - - print_colored(f"✅ You selected: {key_name}", "green") - print_colored("Wait while we assign the API key to your SDK...", "yellow") - - return key_value - else: - print_colored("Invalid option. Try again.", "red") - except ValueError: - print_colored("Please enter a valid number.", "red") - except Exception as e: - print_colored(f"Error processing JSON response: {str(e)}", "red") - return None - else: - # Handle error by status code - error_message = f"Error retrieving API keys: {response.status_code}" - - try: - error_data = response.json() - if "message" in error_data: - error_message += f" - {error_data['message']}" - elif "detail" in error_data: - error_message += f" - {error_data['detail']}" - except: - # If we can't parse JSON, use the full text - error_message += f" - {response.text[:100]}..." - - print_colored(error_message, "red") - - # Try to identify common problems - if response.status_code == 401: - print_colored("The authentication token has expired or is invalid.", "yellow") - elif response.status_code == 403: - print_colored("You don't have permissions to access the API keys.", "yellow") - elif response.status_code == 404: - print_colored("The API keys endpoint doesn't exist. Verify the API URL.", "yellow") - elif response.status_code >= 500: - print_colored("Server error. Try again later.", "yellow") - - return None - - except httpx.RequestError as e: - print_colored(f"Connection error: {str(e)}", "red") - print_colored("Verify the API URL and your internet connection.", "yellow") - return None - except Exception as e: - print_colored(f"Unexpected error retrieving API keys: {str(e)}", "red") - return None - -def get_api_key_id_from_token(sso_token: str, api_token: str, api_url: str) -> Optional[str]: - """ - Gets the ID of an API key from its token. - - Args: - sso_token: SSO token - api_token: API token - api_url: API URL - - Returns: - API key ID or None if it cannot be obtained - """ - try: - # Endpoint to get information of the current user - endpoint = f"{api_url}/api/auth/api-keys/{api_token}" - - headers = { - "Authorization": f"Bearer {api_token}", - "Content-Type": "application/json" - } - - response = httpx.get( - endpoint, - headers=headers - ) - - print("API keys response: ", response.json()) - - if response.status_code == 200: - key_data = response.json() - key_id = key_data.get("id") - return key_id - else: - print_colored("⚠️ Could not find the API key ID", "yellow") - return None - - except Exception as e: - print_colored(f"Error getting API key ID: {str(e)}", "red") - return None - -def exchange_sso_token_for_api_token(api_url: str, sso_token: str, user_data: Dict[str, Any]) -> Optional[str]: - """ - Exchanges a Globodain SSO token for a Corebrain API token. - - Args: - api_url: Base URL of the Corebrain API - sso_token: Globodain SSO token - user_data: User data - - Returns: - API token or None if it fails - """ - try: - # Ensure protocol in URL - if not api_url.startswith(("http://", "https://")): - api_url = "https://" + api_url - - # Remove trailing slash if it exists - if api_url.endswith('/'): - api_url = api_url[:-1] - - # Endpoint to exchange token - endpoint = f"{api_url}/api/auth/sso/token" - - print_colored(f"Exchanging SSO token for API token...", "blue") - - # Configure client with timeout and error handling - headers = { - 'Authorization': f'Bearer {sso_token}', - 'Content-Type': 'application/json' - } - body = { - "user_data": user_data - } - - response = http_session.post(endpoint, json=body, headers=headers) - - if response.status_code == 200: - try: - token_data = response.json() - api_token = token_data.get("access_token") - - if not api_token: - print_colored("The response does not contain a valid API token", "red") - return None - - print_colored("✅ API token successfully obtained", "green") - return api_token - except Exception as e: - print_colored(f"Error processing JSON response: {str(e)}", "red") - return None - else: - # Handle error by status code - error_message = f"Error exchanging token: {response.status_code}" - - try: - error_data = response.json() - if "message" in error_data: - error_message += f" - {error_data['message']}" - elif "detail" in error_data: - error_message += f" - {error_data['detail']}" - except: - # If we can't parse JSON, use the full text - error_message += f" - {response.text[:100]}..." - - print_colored(error_message, "red") - return None - - except httpx.RequestError as e: - print_colored(f"Connection error: {str(e)}", "red") - return None - except Exception as e: - print_colored(f"Unexpected error exchanging token: {str(e)}", "red") - return None \ No newline at end of file diff --git a/corebrain/corebrain/cli/auth/sso.py b/corebrain/corebrain/cli/auth/sso.py deleted file mode 100644 index cee535f..0000000 --- a/corebrain/corebrain/cli/auth/sso.py +++ /dev/null @@ -1,452 +0,0 @@ -""" -SSO Authentication for the CLI. -""" -import os -import webbrowser -import http.server -import socketserver -import threading -import urllib.parse -import time - -from typing import Tuple, Dict, Any, Optional - -from corebrain.cli.common import DEFAULT_API_URL, DEFAULT_SSO_URL, DEFAULT_PORT, SSO_CLIENT_ID, SSO_CLIENT_SECRET -from corebrain.cli.utils import print_colored -from corebrain.lib.sso.auth import GlobodainSSOAuth - -class TokenHandler(http.server.SimpleHTTPRequestHandler): - """ - Handler for the local HTTP server that processes the SSO authentication callback. - """ - def __init__(self, *args, **kwargs): - self.sso_auth = kwargs.pop('sso_auth', None) - self.result = kwargs.pop('result', {}) - self.session_data = kwargs.pop('session_data', {}) - self.auth_completed = kwargs.pop('auth_completed', None) - super().__init__(*args, **kwargs) - - def do_GET(self): - # Parse the URL to get the parameters - parsed_path = urllib.parse.urlparse(self.path) - - # Check if it's the callback path - if parsed_path.path == "/auth/sso/callback": - query = urllib.parse.parse_qs(parsed_path.query) - - if "code" in query: - code = query["code"][0] - - try: - # Exchange code for token using the sso_auth object - token_data = self.sso_auth.exchange_code_for_token(code) - - if not token_data: - raise ValueError("Could not obtain the token") - - # Save token in the result and session - access_token = token_data.get('access_token') - if not access_token: - raise ValueError("The received token does not contain an access_token") - - # Updated: save as sso_token for clarity - self.result["sso_token"] = access_token - self.session_data['sso_token'] = token_data - - # Get user information - user_info = self.sso_auth.get_user_info(access_token) - if user_info: - self.session_data['user'] = user_info - # Extract email to identify the user - if 'email' in user_info: - self.session_data['email'] = user_info['email'] - - # Signal that authentication has completed - self.auth_completed.set() - - # Send a success response to the browser - self.send_response(200) - self.send_header("Content-type", "text/html") - self.end_headers() - success_html = """ - - - Corebrain - Authentication Completed - - - -
-

Authentication Completed

-

You have successfully logged in to Corebrain CLI.

-

You can close this window and return to the terminal.

-
- - - """ - self.wfile.write(success_html.encode()) - except Exception as e: - # If there's an error, show error message - self.send_response(400) - self.send_header("Content-type", "text/html") - self.end_headers() - error_html = f""" - - - Corebrain - Authentication Error - - - -
-

Authentication Error

-

Error: {str(e)}

-

Please close this window and try again.

-
- - - """ - self.wfile.write(error_html.encode()) - else: - # If there's no code, it's an error - self.send_response(400) - self.send_header("Content-type", "text/html") - self.end_headers() - error_html = """ - - - Corebrain - Authentication Error - - - -
-

Authentication Error

-

Could not complete the authentication process.

-

Please close this window and try again.

-
- - - """ - self.wfile.write(error_html.encode()) - else: - # For any other path, show a 404 error - self.send_response(404) - self.end_headers() - self.wfile.write(b"Not Found") - - def log_message(self, format, *args): - # Silence server logs - return - -def authenticate_with_sso(sso_url: str) -> Tuple[Optional[str], Optional[Dict[str, Any]], Optional[str]]: - """ - Initiates an SSO authentication flow through the browser and uses the callback system. - - Args: - sso_url: Base URL of the SSO service - - Returns: - Tuple with (api_key, user_data, api_token) or (None, None, None) if it fails - - api_key: Selected API key to use with the SDK - - user_data: Authenticated user data - - api_token: API token obtained from SSO for general authentication - """ - - # Token to store the result - result = {"sso_token": None} # Renamed for clarity - auth_completed = threading.Event() - session_data = {} - - # Find an available port - #port = get_free_port(DEFAULT_PORT) - - # SSO client configuration - auth_config = { - 'GLOBODAIN_SSO_URL': sso_url or DEFAULT_SSO_URL, - 'GLOBODAIN_CLIENT_ID': SSO_CLIENT_ID, - 'GLOBODAIN_CLIENT_SECRET': SSO_CLIENT_SECRET, - 'GLOBODAIN_REDIRECT_URI': f"http://localhost:{DEFAULT_PORT}/auth/sso/callback", - 'GLOBODAIN_SUCCESS_REDIRECT': 'https://sso.globodain.com/cli/success' - } - - sso_auth = GlobodainSSOAuth(config=auth_config) - - # Factory to create TokenHandler instances with the desired parameters - def handler_factory(*args, **kwargs): - return TokenHandler( - *args, - sso_auth=sso_auth, - result=result, - session_data=session_data, - auth_completed=auth_completed, - **kwargs - ) - - # Start server in the background - server = socketserver.TCPServer(("", DEFAULT_PORT), handler_factory) - server_thread = threading.Thread(target=server.serve_forever) - server_thread.daemon = True - server_thread.start() - - try: - # Build complete URL with protocol if missing - if sso_url and not sso_url.startswith(("http://", "https://")): - sso_url = "https://" + sso_url - - # URL to start the SSO flow - login_url = sso_auth.get_login_url() - auth_url = login_url - - print_colored(f"Opening browser for SSO authentication...", "blue") - print_colored(f"If the browser doesn't open automatically, visit:", "blue") - print_colored(f"{auth_url}", "bold") - - # Try to open the browser - if not webbrowser.open(auth_url): - print_colored("Could not open the browser automatically.", "yellow") - print_colored(f"Please copy and paste the following URL into your browser:", "yellow") - print_colored(f"{auth_url}", "bold") - - # Tell the user to wait - print_colored("\nWaiting for you to complete authentication in the browser...", "blue") - - # Wait for authentication to complete (with timeout) - timeout_seconds = 60 - start_time = time.time() - - # We use a loop with better feedback - while not auth_completed.is_set() and (time.time() - start_time < timeout_seconds): - elapsed = int(time.time() - start_time) - if elapsed % 5 == 0: # Every 5 seconds we show a message - remaining = timeout_seconds - elapsed - #print_colored(f"Waiting for authentication... ({remaining}s remaining)", "yellow") - - # Check every 0.5 seconds for better reactivity - auth_completed.wait(0.5) - - # Verify if authentication was completed - if auth_completed.is_set(): - print_colored("✅ SSO authentication completed successfully!", "green") - return result["sso_token"], session_data['user'] - else: - print_colored(f"❌ Could not complete SSO authentication in {timeout_seconds} seconds.", "red") - print_colored("You can try again or use a token manually.", "yellow") - return None, None, None - except Exception as e: - print_colored(f"❌ Error during SSO authentication: {str(e)}", "red") - return None, None, None - finally: - # Stop the server - try: - server.shutdown() - server.server_close() - except: - # If there's any error closing the server, we ignore it - pass - -def authenticate_with_sso_and_api_key_request(sso_url: str) -> Tuple[Optional[str], Optional[Dict[str, Any]], Optional[str]]: - """ - Initiates an SSO authentication flow through the browser and uses the callback system. - - Args: - sso_url: Base URL of the SSO service - - Returns: - Tuple with (api_key, user_data, api_token) or (None, None, None) if it fails - - api_key: Selected API key to use with the SDK - - user_data: Authenticated user data - - api_token: API token obtained from SSO for general authentication - """ - # Import inside the function to avoid circular dependencies - from corebrain.cli.auth.api_keys import fetch_api_keys, exchange_sso_token_for_api_token - - # Token to store the result - result = {"sso_token": None} # Renamed for clarity - auth_completed = threading.Event() - session_data = {} - - # Find an available port - #port = get_free_port(DEFAULT_PORT) - - # SSO client configuration - auth_config = { - 'GLOBODAIN_SSO_URL': sso_url or DEFAULT_SSO_URL, - 'GLOBODAIN_CLIENT_ID': SSO_CLIENT_ID, - 'GLOBODAIN_CLIENT_SECRET': SSO_CLIENT_SECRET, - 'GLOBODAIN_REDIRECT_URI': f"http://localhost:{DEFAULT_PORT}/auth/sso/callback", - 'GLOBODAIN_SUCCESS_REDIRECT': 'https://sso.globodain.com/cli/success' - } - - sso_auth = GlobodainSSOAuth(config=auth_config) - - # Factory to create TokenHandler instances with the desired parameters - def handler_factory(*args, **kwargs): - return TokenHandler( - *args, - sso_auth=sso_auth, - result=result, - session_data=session_data, - auth_completed=auth_completed, - **kwargs - ) - - # Start server in the background - server = socketserver.TCPServer(("", DEFAULT_PORT), handler_factory) - server_thread = threading.Thread(target=server.serve_forever) - server_thread.daemon = True - server_thread.start() - - try: - # Build complete URL with protocol if missing - if sso_url and not sso_url.startswith(("http://", "https://")): - sso_url = "https://" + sso_url - - # URL to start the SSO flow - login_url = sso_auth.get_login_url() - auth_url = login_url - - print_colored(f"Opening browser for SSO authentication...", "blue") - print_colored(f"If the browser doesn't open automatically, visit:", "blue") - print_colored(f"{auth_url}", "bold") - - # Try to open the browser - if not webbrowser.open(auth_url): - print_colored("Could not open the browser automatically.", "yellow") - print_colored(f"Please copy and paste the following URL into your browser:", "yellow") - print_colored(f"{auth_url}", "bold") - - # Tell the user to wait - print_colored("\nWaiting for you to complete authentication in the browser...", "blue") - - # Wait for authentication to complete (with timeout) - timeout_seconds = 60 - start_time = time.time() - - # We use a loop with better feedback - while not auth_completed.is_set() and (time.time() - start_time < timeout_seconds): - elapsed = int(time.time() - start_time) - if elapsed % 5 == 0: # Every 5 seconds we show a message - remaining = timeout_seconds - elapsed - #print_colored(f"Waiting for authentication... ({remaining}s remaining)", "yellow") - - # Check every 0.5 seconds for better reactivity - auth_completed.wait(0.5) - - # Verify if authentication was completed - if auth_completed.is_set(): - user_data = None - if 'user' in session_data: - user_data = session_data['user'] - - print_colored("✅ SSO authentication completed successfully!", "green") - - # Get and select an API key - api_url = os.environ.get("COREBRAIN_API_URL", DEFAULT_API_URL) - - # Now we use the SSO token to get an API token and then the API keys - # First we verify that we have a token - if result["sso_token"]: - api_token = exchange_sso_token_for_api_token(api_url, result["sso_token"], user_data) - - if not api_token: - print_colored("⚠️ Could not obtain an API Token with the SSO Token", "yellow") - return None, None, None - - # Now that we have the API Token, we get the available API Keys - api_key_selected = fetch_api_keys(api_url, api_token, user_data) - - if api_key_selected: - # We return the selected api_key - return api_key_selected, user_data, api_token - else: - print_colored("⚠️ Could not obtain an API Key. Create a new one using the command", "yellow") - return None, user_data, api_token - else: - print_colored("❌ No valid token was obtained during authentication.", "red") - return None, None, None - - # We don't have a token or user data - print_colored("❌ Authentication did not produce a valid token.", "red") - return None, None, None - else: - print_colored(f"❌ Could not complete SSO authentication in {timeout_seconds} seconds.", "red") - print_colored("You can try again or use a token manually.", "yellow") - return None, None, None - except Exception as e: - print_colored(f"❌ Error during SSO authentication: {str(e)}", "red") - return None, None, None - finally: - # Stop the server - try: - server.shutdown() - server.server_close() - except: - # If there's any error closing the server, we ignore it - pass \ No newline at end of file diff --git a/corebrain/corebrain/cli/commands.py b/corebrain/corebrain/cli/commands.py deleted file mode 100644 index 96eac44..0000000 --- a/corebrain/corebrain/cli/commands.py +++ /dev/null @@ -1,453 +0,0 @@ -""" -Main commands for the Corebrain CLI. -""" -import argparse -import os -import sys -import webbrowser -import requests -import random -import string - -from typing import Optional, List - -from corebrain.cli.common import DEFAULT_API_URL, DEFAULT_SSO_URL, DEFAULT_PORT, SSO_CLIENT_ID, SSO_CLIENT_SECRET -from corebrain.cli.auth.sso import authenticate_with_sso, authenticate_with_sso_and_api_key_request -from corebrain.cli.config import configure_sdk, get_api_credential -from corebrain.cli.utils import print_colored -from corebrain.config.manager import ConfigManager -from corebrain.config.manager import export_config -from corebrain.config.manager import validate_config -from corebrain.lib.sso.auth import GlobodainSSOAuth - -def main_cli(argv: Optional[List[str]] = None) -> int: - """ - Main entry point for the Corebrain CLI. - - Args: - argv: List of command line arguments (defaults to sys.argv[1:]) - - Returns: - Exit code (0 for success, other value for error) - """ - - # Package version - __version__ = "0.1.0" - - try: - print_colored("Corebrain CLI started. Version ", __version__, "blue") - - if argv is None: - argv = sys.argv[1:] - - # Argument parser configuration - parser = argparse.ArgumentParser(description="Corebrain SDK CLI") - parser.add_argument("--version", action="store_true", help="Show SDK version") - parser.add_argument("--authentication", action="store_true", help="Authenticate with SSO") - parser.add_argument("--create-user", action="store_true", help="Create an user and API Key by default") - parser.add_argument("--configure", action="store_true", help="Configure the Corebrain SDK") - parser.add_argument("--list-configs", action="store_true", help="List available configurations") - parser.add_argument("--remove-config", action="store_true", help="Remove a configuration") - parser.add_argument("--show-schema", action="store_true", help="Show the schema of the configured database") - parser.add_argument("--extract-schema", action="store_true", help="Extract the database schema and save it to a file") - parser.add_argument("--output-file", help="File to save the extracted schema") - parser.add_argument("--config-id", help="Specific configuration ID to use") - parser.add_argument("--token", help="Corebrain API token (any type)") - parser.add_argument("--api-key", help="Specific API Key for Corebrain") - parser.add_argument("--api-url", help="Corebrain API URL") - parser.add_argument("--sso-url", help="Globodain SSO service URL") - parser.add_argument("--login", action="store_true", help="Login via SSO") - parser.add_argument("--test-auth", action="store_true", help="Test SSO authentication system") - parser.add_argument("--woami",action="store_true",help="Display information about the current user") - parser.add_argument("--check-status",action="store_true",help="Checks status of task") - parser.add_argument("--task-id",help="ID of the task to check status for") - parser.add_argument("--validate-config",action="store_true",help="Validates the selected configuration without executing any operations") - parser.add_argument("--test-connection",action="store_true",help="Tests the connection to the Corebrain API using the provide credentials") - parser.add_argument("--export-config",action="store_true",help="Exports the current configuration to a file") - parser.add_argument("--gui", action="store_true", help="Check setup and launch the web interface") - - - args = parser.parse_args(argv) - - def authentication(): - sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL - sso_token, sso_user = authenticate_with_sso(sso_url) - if sso_token: - try: - print_colored("✅ Returning SSO Token.", "green") - print_colored(f"{sso_user}", "blue") - print_colored("✅ Returning User data.", "green") - print_colored(f"{sso_user}", "blue") - return sso_token, sso_user - - except Exception as e: - print_colored("❌ Could not return SSO Token or SSO User data.", "red") - return sso_token, sso_user - - else: - print_colored("❌ Could not authenticate with SSO.", "red") - return None, None - - # Made by Lukasz - if args.export_config: - export_config(args.export_config) - # --> config/manager.py --> export_config - - if args.validate_config: - if not args.config_id: - print_colored("Error: --config-id is required for validation", "red") - return 1 - return validate_config(args.config_id) - - - # Show version - if args.version: - try: - from importlib.metadata import version - sdk_version = version("corebrain") - print(f"Corebrain SDK version {sdk_version}") - except Exception: - print(f"Corebrain SDK version {__version__}") - return 0 - - # Create an user and API Key by default - if args.authentication: - authentication() - - if args.create_user: - sso_token, sso_user = authentication() # Authentica use with SSO - - if sso_token and sso_user: - print_colored("✅ Enter to create an user and API Key.", "green") - - # Get API URL from environment or use default - api_url = os.environ.get("COREBRAIN_API_URL", DEFAULT_API_URL) - - """ - Create user data with SSO information. - If the user wants to use a different password than their SSO account, - they can specify it here. - """ - # Ask if user wants to use SSO password or create a new one - use_sso_password = input("Do you want to use your SSO password? (y/n): ").lower().strip() == 'y' - - if use_sso_password: - random_password = ''.join(random.choices(string.ascii_letters + string.digits, k=12)) - password = sso_user.get("password", random_password) - else: - while True: - password = input("Enter new password: ").strip() - if len(password) >= 8: - break - print_colored("Password must be at least 8 characters long", "yellow") - - user_data = { - "email": sso_user["email"], - "name": f"{sso_user['first_name']} {sso_user['last_name']}", - "password": password - } - - try: - # Make the API request - response = requests.post( - f"{api_url}/api/auth/users", - json=user_data, - headers={ - "Authorization": f"Bearer {sso_token}", - "Content-Type": "application/json" - } - ) - - # Check if the request was successful - print("response API: ", response) - if response.status_code == 200: - print_colored("✅ User and API Key created successfully!", "green") - return 0 - else: - print_colored(f"❌ Error creating user: {response.text}", "red") - return 1 - - except requests.exceptions.RequestException as e: - print_colored(f"❌ Error connecting to API: {str(e)}", "red") - return 1 - - else: - print_colored("❌ Could not create the user or the API KEY.", "red") - return 1 - - # Test SSO authentication - if args.test_auth: - sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL - - print_colored("Testing SSO authentication...", "blue") - - # Authentication configuration - auth_config = { - 'GLOBODAIN_SSO_URL': sso_url, - 'GLOBODAIN_CLIENT_ID': SSO_CLIENT_ID, - 'GLOBODAIN_CLIENT_SECRET': SSO_CLIENT_SECRET, - 'GLOBODAIN_REDIRECT_URI': f"http://localhost:{DEFAULT_PORT}/auth/sso/callback", - 'GLOBODAIN_SUCCESS_REDIRECT': f"http://localhost:{DEFAULT_PORT}/auth/sso/callback" - } - - try: - # Instantiate authentication client - sso_auth = GlobodainSSOAuth(config=auth_config) - - # Get login URL - login_url = sso_auth.get_login_url() - - print_colored(f"Login URL: {login_url}", "blue") - print_colored("Opening browser for login...", "blue") - - # Open browser - webbrowser.open(login_url) - - print_colored("Please complete the login process in the browser.", "blue") - input("\nPress Enter when you've completed the process or to cancel...") - - print_colored("✅ SSO authentication test completed!", "green") - return 0 - except Exception as e: - print_colored(f"❌ Error during test: {str(e)}", "red") - return 1 - - # Login via SSO - if args.login: - sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL - api_key, user_data, api_token = authenticate_with_sso_and_api_key_request(sso_url) - - if api_token: - # Save the general token for future use - os.environ["COREBRAIN_API_TOKEN"] = api_token - - if api_key: - # Save the specific API key for future use - os.environ["COREBRAIN_API_KEY"] = api_key - print_colored("✅ API Key successfully saved. You can use the SDK now.", "green") - - # If configuration was also requested, continue with the process - if args.configure: - api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL - configure_sdk(api_token, api_key, api_url, sso_url, user_data) - - return 0 - else: - print_colored("❌ Could not obtain an API Key via SSO.", "red") - if api_token: - print_colored("A general API token was obtained, but not a specific API Key.", "yellow") - print_colored("You can create an API Key in the Corebrain dashboard.", "yellow") - return 1 - - if args.check_status: - if not args.task_id: - print_colored("❌ Please provide a task ID using --task-id", "red") - return 1 - - # Get URLs - api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL - sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL - - # Prioritize api_key if explicitly provided - token_arg = args.api_key if args.api_key else args.token - - # Get API credentials - api_key, user_data, api_token = get_api_credential(token_arg, sso_url) - - if not api_key: - print_colored("❌ API Key is required to check task status. Use --api-key or login via --login", "red") - return 1 - - try: - task_id = args.task_id - headers = { - "Authorization": f"Bearer {api_key}", - "Accept": "application/json" - } - url = f"{api_url}/tasks/{task_id}/status" - response = requests.get(url, headers=headers) - - if response.status_code == 404: - print_colored(f"❌ Task with ID '{task_id}' not found.", "red") - return 1 - - response.raise_for_status() - data = response.json() - status = data.get("status", "unknown") - - print_colored(f"✅ Task '{task_id}' status: {status}", "green") - return 0 - except Exception as e: - print_colored(f"❌ Failed to check status: {str(e)}", "red") - return 1 - - if args.woami: - try: - #downloading user data - sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL - token_arg = args.api_key if args.api_key else args.token - - #using saved data about user - api_key, user_data, api_token = get_api_credential(token_arg, sso_url) - #printing user data - if user_data: - print_colored("User Data:", "blue") - for k, v in user_data.items(): - print(f"{k}: {v}") - else: - print_colored("❌ Can't find data about user, be sure that you are logged into --login.", "red") - return 1 - - return 0 - except Exception as e: - print_colored(f"❌ Error when downloading data about user {str(e)}", "red") - return 1 - - # Operations that require credentials: configure, list, remove or show schema - if args.configure or args.list_configs or args.remove_config or args.show_schema or args.extract_schema: - # Get URLs - api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL - sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL - - # Prioritize api_key if explicitly provided - token_arg = args.api_key if args.api_key else args.token - - # Get API credentials - api_key, user_data, api_token = get_api_credential(token_arg, sso_url) - - if not api_key: - print_colored("Error: An API Key is required. You can generate one at dashboard.corebrain.com", "red") - print_colored("Or use the 'corebrain --login' command to login via SSO.", "blue") - return 1 - - from corebrain.db.schema_file import show_db_schema, extract_schema_to_file - - # Execute the selected operation - if args.configure: - configure_sdk(api_token, api_key, api_url, sso_url, user_data) - elif args.list_configs: - ConfigManager.list_configs(api_key, api_url) - elif args.remove_config: - ConfigManager.remove_config(api_key, api_url) - elif args.show_schema: - show_db_schema(api_key, args.config_id, api_url) - elif args.extract_schema: - extract_schema_to_file(api_key, args.config_id, args.output_file, api_url) - - if args.test_connection: - # Test connection to the Corebrain API - api_url = args.api_url or os.environ.get("COREBRAIN_API_URL", DEFAULT_API_URL) - sso_url = args.sso_url or os.environ.get("COREBRAIN_SSO_URL", DEFAULT_SSO_URL) - - try: - # Retrieve API credentials - api_key, user_data, api_token = get_api_credential(args.token, sso_url) - except Exception as e: - print_colored(f"Error while retrieving API credentials: {e}", "red") - return 1 - - if not api_key: - print_colored( - "Error: An API key is required. You can generate one at dashboard.corebrain.com.", - "red" - ) - return 1 - - try: - # Test the connection - from corebrain.db.schema_file import test_connection - test_connection(api_key, api_url) - print_colored("Successfully connected to Corebrain API.", "green") - except Exception as e: - print_colored(f"Failed to connect to Corebrain API: {e}", "red") - return 1 - - - - - if args.gui: - import subprocess - from pathlib import Path - - def run_cmd(cmd, cwd=None): - print_colored(f"▶ {cmd}", "yellow") - subprocess.run(cmd, shell=True, cwd=cwd, check=True) - - print("Checking GUI setup...") - - commands_path = Path(__file__).resolve() - corebrain_root = commands_path.parents[1] - - cli_ui_path = corebrain_root / "CLI-UI" - client_path = cli_ui_path / "client" - server_path = cli_ui_path / "server" - api_path = corebrain_root / "wrappers" / "csharp_cli_api" - - # Path validation - if not client_path.exists(): - print_colored(f"Folder {client_path} does not exist!", "red") - sys.exit(1) - if not server_path.exists(): - print_colored(f"Folder {server_path} does not exist!", "red") - sys.exit(1) - if not api_path.exists(): - print_colored(f"Folder {api_path} does not exist!", "red") - sys.exit(1) - - # Setup client - if not (client_path / "node_modules").exists(): - print_colored("Installing frontend (React) dependencies...", "cyan") - run_cmd("npm install", cwd=client_path) - run_cmd("npm install history", cwd=client_path) - run_cmd("npm install --save-dev vite", cwd=client_path) - run_cmd("npm install concurrently --save-dev", cwd=client_path) - - # Setup server - if not (server_path / "node_modules").exists(): - print_colored("Installing backend (Express) dependencies...", "cyan") - run_cmd("npm install", cwd=server_path) - run_cmd("npm install --save-dev ts-node-dev", cwd=server_path) - - # Start GUI: CLI UI + Corebrain API - print("Starting GUI (CLI-UI + Corebrain API)...") - - def run_in_background_silent(cmd, cwd): - return subprocess.Popen( - cmd, - cwd=cwd, - shell=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL - ) - - run_in_background_silent("dotnet run", cwd=api_path) - run_in_background_silent( - 'npx concurrently "npm --prefix server run dev" "npm --prefix client run dev"', - cwd=cli_ui_path - ) - - url = "http://localhost:5173/" - print_colored(f"GUI: {url}", "cyan") - webbrowser.open(url) - - - - - - - - - - - - else: - # If no option was specified, show help - parser.print_help() - print_colored("\nTip: Use 'corebrain --login' to login via SSO.", "blue") - - return 0 - except Exception as e: - print_colored(f"Error: {str(e)}", "red") - import traceback - traceback.print_exc() - return 1 \ No newline at end of file diff --git a/corebrain/corebrain/cli/common.py b/corebrain/corebrain/cli/common.py deleted file mode 100644 index 7799dcf..0000000 --- a/corebrain/corebrain/cli/common.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -Default values for SSO and API connection -""" - -DEFAULT_API_URL = "http://localhost:5000" -#DEFAULT_SSO_URL = "http://localhost:3000" # localhost -DEFAULT_SSO_URL = "https://sso.globodain.com" # remote -DEFAULT_PORT = 8765 -DEFAULT_TIMEOUT = 10 -#SSO_CLIENT_ID = '401dca6e-3f3b-4458-b3ef-f87eaae0398d' # localhost -#SSO_CLIENT_SECRET = 'f9d315ea-5a65-4e3f-be35-b27a933dfb5b' # localhost -SSO_CLIENT_ID = '63d767e9-5a06-4890-a194-8608ae29d426' # remote -SSO_CLIENT_SECRET = '06cf39f6-ca93-466e-955e-cb6ea0a02d4d' # remote -SSO_REDIRECT_URI = 'http://localhost:8765/oauth/callback' -SSO_SERVICE_ID = 2 \ No newline at end of file diff --git a/corebrain/corebrain/cli/config.py b/corebrain/corebrain/cli/config.py deleted file mode 100644 index cf8a175..0000000 --- a/corebrain/corebrain/cli/config.py +++ /dev/null @@ -1,489 +0,0 @@ -""" -Configuration functions for the CLI. -""" -import json -import uuid -import getpass -import os -from typing import Dict, Any, List, Optional, Tuple -from datetime import datetime - -from corebrain.cli.common import DEFAULT_API_URL, DEFAULT_SSO_URL -from corebrain.cli.auth.sso import authenticate_with_sso, authenticate_with_sso_and_api_key_request -from corebrain.cli.utils import print_colored, ProgressTracker -from corebrain.db.engines import get_available_engines -from corebrain.config.manager import ConfigManager -from corebrain.network.client import http_session -from corebrain.core.test_utils import test_natural_language_query -from corebrain.db.schema_file import extract_db_schema - -def get_api_credential(args_token: Optional[str] = None, sso_url: Optional[str] = None) -> Tuple[Optional[str], Optional[Dict[str, Any]], Optional[str]]: - """ - Obtains the API credential (API key), trying several methods in order: - 1. Token provided as argument - 2. Environment variable - 3. SSO authentication - 4. Manual user input - - Args: - args_token: Token provided as argument - sso_url: SSO service URL - - Returns: - Tuple with (api_key, user_data, api_token) or (None, None, None) if couldn't be obtained - - api_key: API key to use with SDK - - user_data: User data - - api_token: API token for general authentication - """ - # 1. Check if provided as argument - if args_token: - print_colored("Using token provided as argument.", "blue") - # Assume the provided token is directly an API key - return args_token, None, args_token - - # 2. Check environment variable for API key - env_api_key = os.environ.get("COREBRAIN_API_KEY") - if env_api_key: - print_colored("Using API key from COREBRAIN_API_KEY environment variable.", "blue") - return env_api_key, None, env_api_key - - # 3. Check environment variable for API token - env_api_token = os.environ.get("COREBRAIN_API_TOKEN") - if env_api_token: - print_colored("Using API token from COREBRAIN_API_TOKEN environment variable.", "blue") - # Note: Here we return the same value as api_key and api_token - # because we have no way to obtain a specific api_key - return env_api_token, None, env_api_token - - # 4. Try SSO authentication - print_colored("Attempting authentication via SSO...", "blue") - api_key, user_data, api_token = authenticate_with_sso_and_api_key_request(sso_url or DEFAULT_SSO_URL) - print("Exit from authenticate_with_sso: ", datetime.now()) - if api_key: - # Save for future use - os.environ["COREBRAIN_API_KEY"] = api_key - os.environ["COREBRAIN_API_TOKEN"] = api_token - return api_key, user_data, api_token - - # 5. Request manually - print_colored("\nCouldn't complete SSO authentication.", "yellow") - print_colored("You can directly enter an API key:", "blue") - manual_input = input("Enter your Corebrain API key: ").strip() - if manual_input: - # Assume manual input is an API key - return manual_input, None, manual_input - - # If we got here, we couldn't get a credential - return None, None, None - -def get_db_type() -> str: - """ - Prompts the user to select a database type. - - Returns: - Selected database type - """ - print_colored("\n=== Select the database type ===", "blue") - print("1. SQL (SQLite, MySQL, PostgreSQL)") - print("2. NoSQL (MongoDB)") - - while True: - try: - choice = int(input("\nSelect an option (1-2): ").strip()) - if choice == 1: - return "sql" - elif choice == 2: - return "nosql" - else: - print_colored("Invalid option. Try again.", "red") - except ValueError: - print_colored("Please enter a number.", "red") - -def get_db_engine(db_type: str) -> str: - """ - Prompts the user to select a database engine. - - Args: - db_type: Selected database type - - Returns: - Selected database engine - """ - engines = get_available_engines() - - if db_type == "sql": - available_engines = engines["sql"] - print_colored("\n=== Select the SQL engine ===", "blue") - for i, engine in enumerate(available_engines, 1): - print(f"{i}. {engine.capitalize()}") - - while True: - try: - choice = int(input(f"\nSelect an option (1-{len(available_engines)}): ").strip()) - if 1 <= choice <= len(available_engines): - return available_engines[choice - 1] - else: - print_colored("Invalid option. Try again.", "red") - except ValueError: - print_colored("Please enter a number.", "red") - else: - # For NoSQL, we only have MongoDB for now - return "mongodb" - -def get_connection_params(db_type: str, engine: str) -> Dict[str, Any]: - """ - Prompts for connection parameters according to the database type and engine. - - Args: - db_type: Database type - engine: Database engine - - Returns: - Dictionary with connection parameters - """ - params = {"type": db_type, "engine": engine} - - # Specific parameters by type and engine - if db_type == "sql": - if engine == "sqlite": - path = input("\nPath to SQLite database file: ").strip() - params["database"] = path - else: - # MySQL or PostgreSQL - print_colored("\n=== Connection Parameters ===", "blue") - params["host"] = input("Host (default: localhost): ").strip() or "localhost" - - if engine == "mysql": - params["port"] = int(input("Port (default: 3306): ").strip() or "3306") - else: # PostgreSQL - params["port"] = int(input("Port (default: 5432): ").strip() or "5432") - - params["user"] = input("User: ").strip() - params["password"] = getpass.getpass("Password: ") - params["database"] = input("Database name: ").strip() - else: - # MongoDB - print_colored("\n=== MongoDB Connection Parameters ===", "blue") - use_connection_string = input("Use connection string? (y/n): ").strip().lower() == "y" - - if use_connection_string: - params["connection_string"] = input("MongoDB connection string: ").strip() - else: - params["host"] = input("Host (default: localhost): ").strip() or "localhost" - params["port"] = int(input("Port (default: 27017): ").strip() or "27017") - - use_auth = input("Use authentication? (y/n): ").strip().lower() == "y" - if use_auth: - params["user"] = input("User: ").strip() - params["password"] = getpass.getpass("Password: ") - - params["database"] = input("Database name: ").strip() - - # Add configuration ID - params["config_id"] = str(uuid.uuid4()) - params["excluded_tables"] = [] - - return params - -def test_database_connection(api_token: str, db_config: Dict[str, Any], api_url: Optional[str] = None, user_data: Optional[Dict[str, Any]] = None) -> bool: - """ - Tests the database connection without verifying the API token. - - Args: - api_token: API token - db_config: Database configuration - api_url: Optional API URL - user_data: User data - - Returns: - True if connection is successful, False otherwise - """ - try: - print_colored("\nTesting database connection...", "blue") - - db_type = db_config["type"].lower() - engine = db_config.get("engine", "").lower() - - if db_type == "sql": - if engine == "sqlite": - import sqlite3 - conn = sqlite3.connect(db_config.get("database", "")) - cursor = conn.cursor() - cursor.execute("SELECT 1") - cursor.close() - conn.close() - - elif engine == "mysql": - import mysql.connector - if "connection_string" in db_config: - conn = mysql.connector.connect(connection_string=db_config["connection_string"]) - else: - conn = mysql.connector.connect( - host=db_config.get("host", "localhost"), - user=db_config.get("user", ""), - password=db_config.get("password", ""), - database=db_config.get("database", ""), - port=db_config.get("port", 3306) - ) - cursor = conn.cursor() - cursor.execute("SELECT 1") - cursor.close() - conn.close() - - elif engine == "postgresql": - import psycopg2 - if "connection_string" in db_config: - conn = psycopg2.connect(db_config["connection_string"]) - else: - conn = psycopg2.connect( - host=db_config.get("host", "localhost"), - user=db_config.get("user", ""), - password=db_config.get("password", ""), - dbname=db_config.get("database", ""), - port=db_config.get("port", 5432) - ) - cursor = conn.cursor() - cursor.execute("SELECT 1") - cursor.close() - conn.close() - - elif db_type == "nosql" and engine == "mongodb": - import pymongo - if "connection_string" in db_config: - client = pymongo.MongoClient(db_config["connection_string"]) - else: - client = pymongo.MongoClient( - host=db_config.get("host", "localhost"), - port=db_config.get("port", 27017), - username=db_config.get("user"), - password=db_config.get("password") - ) - - # Verify connection by trying to access the database - db = client[db_config["database"]] - # List collections to verify we can access - _ = db.list_collection_names() - client.close() - - # If we got here, the connection was successful - print_colored("✅ Database connection successful!", "green") - return True - except Exception as e: - print_colored(f"❌ Error connecting to the database: {str(e)}", "red") - return False - -def select_excluded_tables(api_token: str, db_config: Dict[str, Any], api_url: Optional[str] = None, user_data: Optional[Dict[str, Any]] = None) -> List[str]: - - """ - Allows the user to select tables/collections to exclude. - - Args: - api_token: API token - db_config: Database configuration - api_url: Optional API URL - user_data: User data - - Returns: - List of excluded tables/collections - """ - print_colored("\nRetrieving database schema...", "blue") - - # Get the database schema directly - schema = extract_db_schema(db_config) - - if not schema or not schema.get("tables"): - print_colored("No tables/collections found.", "yellow") - return [] - - print_colored("\n=== Tables/Collections found ===", "blue") - print("Mark with 'n' the tables that should NOT be accessible (y for accessible)") - - # Use the tables list instead of the dictionary - tables_list = schema.get("tables_list", []) - excluded_tables = [] - - if not tables_list: - # If there's no table list, convert the tables dictionary to a list - tables = schema.get("tables", {}) - for table_name in tables: - choice = input(f"{table_name} (accessible? y/n): ").strip().lower() - if choice == "n": - excluded_tables.append(table_name) - else: - # If there's a table list, use it directly - for i, table in enumerate(tables_list, 1): - table_name = table["name"] - choice = input(f"{i}. {table_name} (accessible? y/n): ").strip().lower() - if choice == "n": - excluded_tables.append(table_name) - - print_colored(f"\n{len(excluded_tables)} tables/collections have been excluded", "green") - return excluded_tables - -def save_configuration(sso_token: str, api_key: str, db_config: Dict[str, Any], api_url: Optional[str] = None) -> bool: - """ - Saves the configuration locally and syncs it with the API server. - - Args: - sso_token: SSO authentication token - api_key: API Key to identify the configuration - db_config: Database configuration - api_url: Optional API URL - - Returns: - True if saved correctly, False otherwise - """ - config_id = db_config.get("config_id") - if not config_id: - config_id = str(uuid.uuid4()) - db_config["config_id"] = config_id - - print_colored(f"\nSaving configuration with ID: {config_id}...", "blue") - - try: - config_manager = ConfigManager() - config_manager.add_config(api_key, db_config, config_id) - - # 2. Verify that the configuration was saved locally - saved_config = config_manager.get_config(api_key, config_id) - if not saved_config: - print_colored("⚠️ Could not verify local saving of configuration", "yellow") - else: - print_colored("✅ Configuration saved locally successfully", "green") - - # 3. Try to sync with the server - try: - if api_url: - print_colored("Syncing configuration with server...", "blue") - - # Prepare URL - if not api_url.startswith(("http://", "https://")): - api_url = "https://" + api_url - - if api_url.endswith('/'): - api_url = api_url[:-1] - - # Endpoint to update API key - endpoint = f"{api_url}/api/auth/api-keys/{api_key}" - - # Create ApiKeyUpdate object according to your model - update_data = { - "metadata": { - "config_id": config_id, - "db_config": db_config, - "corebrain_sdk": { - "version": "1.0.0", - "updated_at": datetime.now().isoformat() - } - } - } - - print_colored(f"Updating API key with ID: {api_key}", "blue") - - # Send to server - headers = { - "Authorization": f"Bearer {sso_token}", - "Content-Type": "application/json" - } - - response = http_session.put( - endpoint, - headers=headers, - json=update_data, - timeout=5.0 - ) - - if response.status_code in [200, 201, 204]: - print_colored("✅ Configuration successfully synced with server", "green") - else: - print_colored(f"⚠️ Error syncing with server (Code: {response.status_code})", "yellow") - print_colored(f"Response: {response.text[:200]}...", "yellow") - - except Exception as e: - print_colored(f"⚠️ Error syncing with server: {str(e)}", "yellow") - print_colored("The configuration is still saved locally", "green") - - return True - - except Exception as e: - print_colored(f"❌ Error saving configuration: {str(e)}", "red") - return False - -def configure_sdk(api_token: str, api_key: str, api_url: Optional[str] = None, sso_url: Optional[str] = None, user_data: Optional[Dict[str, Any]] = None) -> None: - """ - Configures the Corebrain SDK with a step-by-step wizard. - - Args: - api_token: API token for general authentication (obtained from SSO) - api_key: Specific API key selected to use with the SDK - api_url: Corebrain API URL - sso_url: Globodain SSO service URL - user_data: User data obtained from SSO - """ - # Ensure default values for URLs - api_url = api_url or DEFAULT_API_URL - sso_url = sso_url or DEFAULT_SSO_URL - - print_colored("\n=== COREBRAIN SDK CONFIGURATION WIZARD ===", "bold") - - # PHASE 1-3: Already completed - User authentication - - # PHASE 4: Select database type - print_colored("\n2. Selecting database type...", "blue") - db_type = get_db_type() - - # PHASE 4: Select database engine - print_colored("\n3. Selecting database engine...", "blue") - engine = get_db_engine(db_type) - - # PHASE 5: Configure connection parameters - print_colored("\n4. Configuring connection parameters...", "blue") - db_config = get_connection_params(db_type, engine) - - # PHASE 5: Verify database connection - print_colored("\n5. Verifying database connection...", "blue") - if not test_database_connection(api_key, db_config, api_url, user_data): - print_colored("❌ Configuration not completed due to connection errors.", "red") - return - - # PHASE 6: Define non-accessible tables/collections - print_colored("\n6. Defining non-accessible tables/collections...", "blue") - excluded_tables = select_excluded_tables(api_key, db_config, api_url, user_data) - db_config["excluded_tables"] = excluded_tables - - # PHASE 7: Save configuration - print_colored("\n7. Saving configuration...", "blue") - config_id = db_config["config_id"] - - # Save the configuration - if not save_configuration(api_token, api_key, db_config, api_url): - print_colored("❌ Error saving configuration.", "red") - return - - """ # * --> Deactivated - # PHASE 8: Test natural language query (optional depending on API status) - try: - print_colored("\n8. Testing natural language query...", "blue") - test_natural_language_query(api_key, db_config, api_url, user_data) - except Exception as e: - print_colored(f"⚠️ Could not perform the query test: {str(e)}", "yellow") - print_colored("This does not affect the saved configuration.", "yellow") - """ - - # Final message - print_colored("\n✅ Configuration completed successfully!", "green") - print_colored(f"\nYou can use this SDK in your code with:", "blue") - print(f""" - from corebrain import init - - # Initialize the SDK with API key and configuration ID - corebrain = init( - api_key="{api_key}", - config_id="{config_id}" - ) - - # Perform natural language queries - result = corebrain.ask("Your question in natural language") - print(result["explanation"]) - """ - ) \ No newline at end of file diff --git a/corebrain/corebrain/cli/utils.py b/corebrain/corebrain/cli/utils.py deleted file mode 100644 index 6c0ccac..0000000 --- a/corebrain/corebrain/cli/utils.py +++ /dev/null @@ -1,595 +0,0 @@ -""" -Utilities for the Corebrain CLI. - -This module provides utility functions and classes for the -Corebrain command-line interface. -""" -import sys -import time -import socket -import random -import logging -import threading -import socketserver - -from typing import Optional, Dict, Any, List, Union -from pathlib import Path - -from corebrain.cli.common import DEFAULT_PORT, DEFAULT_TIMEOUT - -logger = logging.getLogger(__name__) - -# Terminal color definitions -COLORS = { - "default": "\033[0m", - "bold": "\033[1m", - "green": "\033[92m", - "red": "\033[91m", - "yellow": "\033[93m", - "blue": "\033[94m", - "magenta": "\033[95m", - "cyan": "\033[96m", - "white": "\033[97m", - "black": "\033[30m", - "bg_green": "\033[42m", - "bg_red": "\033[41m", - "bg_yellow": "\033[43m", - "bg_blue": "\033[44m", -} - -def print_colored(text: str, color: str = "default", return_str: bool = False) -> Optional[str]: - """ - Prints colored text in the terminal or returns the colored text. - - Args: - text: Text to color - color: Color to use (default, green, red, yellow, blue, bold, etc.) - return_str: If True, returns the colored text instead of printing it - - Returns: - If return_str is True, returns the colored text, otherwise None - """ - try: - # Get color code - start_color = COLORS.get(color, COLORS["default"]) - end_color = COLORS["default"] - - # Compose colored text - colored_text = f"{start_color}{text}{end_color}" - - # Return or print - if return_str: - return colored_text - else: - print(colored_text) - return None - except Exception as e: - # If there's an error with colors (e.g., terminal that doesn't support them) - logger.debug(f"Error using colors: {e}") - if return_str: - return text - else: - print(text) - return None - -def format_table(data: List[Dict[str, Any]], columns: Optional[List[str]] = None, - max_width: int = 80) -> str: - """ - Formats data as a text table for display in the terminal. - - Args: - data: List of dictionaries with the data - columns: List of columns to display (if None, uses all columns) - max_width: Maximum width of the table - - Returns: - Table formatted as text - """ - if not data: - return "No data to display" - - # Determine columns to display - if not columns: - # Use all columns from the first element - columns = list(data[0].keys()) - - # Get the maximum width for each column - widths = {col: len(col) for col in columns} - for row in data: - for col in columns: - if col in row: - val = str(row[col]) - widths[col] = max(widths[col], min(len(val), 30)) # Limit to 30 characters - - # Adjust widths if they exceed the maximum - total_width = sum(widths.values()) + (3 * len(columns)) - 1 - if total_width > max_width: - # Reduce proportionally - ratio = max_width / total_width - for col in widths: - widths[col] = max(8, int(widths[col] * ratio)) - - # Header - header = " | ".join(col.ljust(widths[col]) for col in columns) - separator = "-+-".join("-" * widths[col] for col in columns) - - # Rows - rows = [] - for row in data: - row_str = " | ".join( - str(row.get(col, "")).ljust(widths[col])[:widths[col]] - for col in columns - ) - rows.append(row_str) - - # Compose table - return "\n".join([header, separator] + rows) - -def get_free_port(start_port: int = DEFAULT_PORT) -> int: - """ - Finds an available port, starting with the suggested port. - - Args: - start_port: Initial port to try - - Returns: - Available port - """ - try: - # Try with the suggested port first - with socketserver.TCPServer(("", start_port), None) as _: - return start_port # The port is available - except OSError: - # If the suggested port is busy, look for a free one - for _ in range(10): # Try 10 times - # Choose a random port between 8000 and 9000 - port = random.randint(8000, 9000) - try: - with socketserver.TCPServer(("", port), None) as _: - return port # Port available - except OSError: - continue # Try with another port - - # If we can't find a free port, use a default high one - return 10000 + random.randint(0, 1000) - -def is_port_in_use(port: int) -> bool: - """ - Checks if a port is in use. - - Args: - port: Port number to check - - Returns: - True if the port is in use - """ - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - return s.connect_ex(('localhost', port)) == 0 - -def is_interactive() -> bool: - """ - Determines if the current session is interactive. - - Returns: - True if the session is interactive - """ - return sys.stdin.isatty() and sys.stdout.isatty() - -def confirm_action(message: str, default: bool = False) -> bool: - """ - Asks the user for confirmation of an action. - - Args: - message: Confirmation message - default: Default value if the user just presses Enter - - Returns: - True if the user confirms, False otherwise - """ - if not is_interactive(): - return default - - default_text = "Y/n" if default else "y/N" - response = input(f"{message} ({default_text}): ").strip().lower() - - if not response: - return default - - return response.startswith('y') - -def get_input_with_default(prompt: str, default: Optional[str] = None) -> str: - """ - Requests input from the user with a default value. - - Args: - prompt: Request message - default: Default value - - Returns: - Value entered by the user or default value - """ - if default: - full_prompt = f"{prompt} (default: {default}): " - else: - full_prompt = f"{prompt}: " - - response = input(full_prompt).strip() - - return response if response else (default or "") - -def get_password_input(prompt: str = "Password") -> str: - """ - Requests a password from the user without displaying it on screen. - - Args: - prompt: Request message - - Returns: - Password entered - """ - import getpass - return getpass.getpass(f"{prompt}: ") - -def truncate_text(text: str, max_length: int = 100, suffix: str = "...") -> str: - """ - Truncates text if it exceeds the maximum length. - - Args: - text: Text to truncate - max_length: Maximum length - suffix: Suffix to add if the text is truncated - - Returns: - Truncated text if necessary - """ - if not text or len(text) <= max_length: - return text - - return text[:max_length - len(suffix)] + suffix - -def ensure_dir(path: Union[str, Path]) -> Path: - """ - Ensures that a directory exists, creating it if necessary. - - Args: - path: Directory path - - Returns: - Path object of the directory - """ - path_obj = Path(path) - path_obj.mkdir(parents=True, exist_ok=True) - return path_obj - -class ProgressTracker: - """ - Displays progress of CLI operations with colors and timing. - """ - - def __init__(self, verbose: bool = False, spinner: bool = True): - """ - Initializes the progress tracker. - - Args: - verbose: Whether to show detailed information - spinner: Whether to show an animated spinner - """ - self.verbose = verbose - self.use_spinner = spinner and is_interactive() - self.start_time = None - self.current_task = None - self.total = None - self.steps = 0 - self.spinner_thread = None - self.stop_spinner = threading.Event() - self.last_update_time = 0 - self.update_interval = 0.2 # Seconds between updates - - def _run_spinner(self): - """Displays an animated spinner in the console.""" - spinner_chars = "|/-\\" - idx = 0 - - while not self.stop_spinner.is_set(): - if self.current_task: - elapsed = time.time() - self.start_time - status = f"{self.steps}/{self.total}" if self.total else f"step {self.steps}" - sys.stdout.write(f"\r{COLORS['blue']}[{spinner_chars[idx]}] {self.current_task} ({status}, {elapsed:.1f}s){COLORS['default']} ") - sys.stdout.flush() - idx = (idx + 1) % len(spinner_chars) - time.sleep(0.1) - - def start(self, task: str, total: Optional[int] = None) -> None: - """ - Starts tracking a task. - - Args: - task: Task description - total: Total number of steps (optional) - """ - self.reset() # Ensure there's no previous task - - self.current_task = task - self.total = total - self.start_time = time.time() - self.steps = 0 - self.last_update_time = self.start_time - - # Show initial message - print_colored(f"▶ {task}...", "blue") - - # Start spinner if enabled - if self.use_spinner: - self.stop_spinner.clear() - self.spinner_thread = threading.Thread(target=self._run_spinner) - self.spinner_thread.daemon = True - self.spinner_thread.start() - - def update(self, message: Optional[str] = None, increment: int = 1) -> None: - """ - Updates progress with optional message. - - Args: - message: Progress message - increment: Step increment - """ - if not self.start_time: - return # No active task - - self.steps += increment - current_time = time.time() - - # Limit update frequency to avoid saturating the output - if (current_time - self.last_update_time < self.update_interval) and not message: - return - - self.last_update_time = current_time - - # If there's an active spinner, temporarily stop it to show the message - if self.use_spinner and self.spinner_thread and self.spinner_thread.is_alive(): - sys.stdout.write("\r" + " " * 80 + "\r") # Clear current line - sys.stdout.flush() - - if message or self.verbose: - elapsed = current_time - self.start_time - status = f"{self.steps}/{self.total}" if self.total else f"step {self.steps}" - - if message: - print_colored(f" • {message} ({status}, {elapsed:.1f}s)", "blue") - elif self.verbose: - print_colored(f" • Progress: {status}, {elapsed:.1f}s", "blue") - - def finish(self, message: Optional[str] = None) -> None: - """ - Finishes a task with success message. - - Args: - message: Final message - """ - if not self.start_time: - return # No active task - - # Stop spinner if active - self._stop_spinner() - - elapsed = time.time() - self.start_time - msg = message or f"{self.current_task} completed" - print_colored(f"✅ {msg} in {elapsed:.2f}s", "green") - - self.reset() - - def fail(self, message: Optional[str] = None) -> None: - """ - Marks a task as failed. - - Args: - message: Error message - """ - if not self.start_time: - return # No active task - - # Stop spinner if active - self._stop_spinner() - - elapsed = time.time() - self.start_time - msg = message or f"{self.current_task} failed" - print_colored(f"❌ {msg} after {elapsed:.2f}s", "red") - - self.reset() - - def _stop_spinner(self) -> None: - """Stops the spinner if active.""" - if self.use_spinner and self.spinner_thread and self.spinner_thread.is_alive(): - self.stop_spinner.set() - self.spinner_thread.join(timeout=0.5) - - # Clear spinner line - sys.stdout.write("\r" + " " * 80 + "\r") - sys.stdout.flush() - - def reset(self) -> None: - """Resets the tracker.""" - self._stop_spinner() - self.start_time = None - self.current_task = None - self.total = None - self.steps = 0 - self.spinner_thread = None - -class CliConfig: - """ - Manages the CLI configuration. - """ - - def __init__(self, config_dir: Optional[Union[str, Path]] = None): - """ - Initializes the CLI configuration. - - Args: - config_dir: Directory for configuration files - """ - if config_dir: - self.config_dir = Path(config_dir) - else: - self.config_dir = Path.home() / ".corebrain" / "cli" - - self.config_file = self.config_dir / "config.json" - self.config = self._load_config() - - def _load_config(self) -> Dict[str, Any]: - """ - Loads configuration from file. - - Returns: - Loaded configuration - """ - if not self.config_file.exists(): - return self._create_default_config() - - try: - import json - with open(self.config_file, 'r') as f: - return json.load(f) - except Exception as e: - logger.warning(f"Error loading configuration: {e}") - return self._create_default_config() - - def _create_default_config(self) -> Dict[str, Any]: - """ - Creates a default configuration. - - Returns: - Default configuration - """ - from corebrain.cli.common import DEFAULT_API_URL, DEFAULT_SSO_URL - - config = { - "api_url": DEFAULT_API_URL, - "sso_url": DEFAULT_SSO_URL, - "verbose": False, - "timeout": DEFAULT_TIMEOUT, - "last_used": { - "api_key": None, - "config_id": None - }, - "ui": { - "use_colors": True, - "use_spinner": True, - "verbose": False - } - } - - # Ensure the directory exists - ensure_dir(self.config_dir) - - # Save default configuration - try: - import json - with open(self.config_file, 'w') as f: - json.dump(config, f, indent=2) - except Exception as e: - logger.warning(f"Error saving configuration: {e}") - - return config - - def save(self) -> bool: - """ - Saves current configuration. - - Returns: - True if saved correctly - """ - try: - # Ensure the directory exists - ensure_dir(self.config_dir) - - import json - with open(self.config_file, 'w') as f: - json.dump(self.config, f, indent=2) - return True - except Exception as e: - logger.error(f"Error saving configuration: {e}") - return False - - def get(self, key: str, default: Any = None) -> Any: - """ - Gets a configuration value. - - Args: - key: Configuration key - default: Default value - - Returns: - Configuration value - """ - # Support for nested keys with dots - if "." in key: - parts = key.split(".") - current = self.config - for part in parts: - if part not in current: - return default - current = current[part] - return current - - return self.config.get(key, default) - - def set(self, key: str, value: Any) -> bool: - """ - Sets a configuration value. - - Args: - key: Configuration key - value: Value to set - - Returns: - True if set correctly - """ - # Support for nested keys with dots - if "." in key: - parts = key.split(".") - current = self.config - for part in parts[:-1]: - if part not in current: - current[part] = {} - current = current[part] - current[parts[-1]] = value - else: - self.config[key] = value - - return self.save() - - def update(self, config_dict: Dict[str, Any]) -> bool: - """ - Updates configuration with a dictionary. - - Args: - config_dict: Configuration dictionary - - Returns: - True if updated correctly - """ - self.config.update(config_dict) - return self.save() - - def update_last_used(self, api_key: Optional[str] = None, config_id: Optional[str] = None) -> bool: - """ - Updates the last used configuration. - - Args: - api_key: API key used - config_id: Configuration ID used - - Returns: - True if updated correctly - """ - if not self.config.get("last_used"): - self.config["last_used"] = {} - - if api_key: - self.config["last_used"]["api_key"] = api_key - - if config_id: - self.config["last_used"]["config_id"] = config_id - - return self.save() \ No newline at end of file diff --git a/corebrain/corebrain/config/__init__.py b/corebrain/corebrain/config/__init__.py deleted file mode 100644 index 7149948..0000000 --- a/corebrain/corebrain/config/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -""" -Configuration management for the Corebrain SDK. - -This package provides functionality to manage database connection configurations -and SDK preferences. -""" -from .manager import ConfigManager - -# Exportación explícita de componentes públicos -__all__ = ['ConfigManager'] \ No newline at end of file diff --git a/corebrain/corebrain/config/manager.py b/corebrain/corebrain/config/manager.py deleted file mode 100644 index 3cc6dd6..0000000 --- a/corebrain/corebrain/config/manager.py +++ /dev/null @@ -1,235 +0,0 @@ -""" -Configuration manager for the Corebrain SDK. -""" - -import json -import uuid -from pathlib import Path -from typing import Dict, Any, List, Optional -from cryptography.fernet import Fernet -from corebrain.utils.serializer import serialize_to_json -from corebrain.core.common import logger - -# Made by Lukasz -# get data from pyproject.toml -def load_project_metadata(): - pyproject_path = Path(__file__).resolve().parent.parent / "pyproject.toml" - try: - with open(pyproject_path, "rb") as f: - data = tomli.load(f) - return data.get("project", {}) - except (FileNotFoundError, tomli.TOMLDecodeError) as e: - print(f"Warning: Could not load project metadata: {e}") - return {} - -# Made by Lukasz -# get the name, version, etc. -def get_config(): - metadata = load_project_metadata() # ^ - return { - "model": metadata.get("name", "unknown"), - "version": metadata.get("version", "0.0.0"), - "debug": False, - "logging": {"level": "info"} - } - -# Made by Lukasz -# export config to file -def export_config(filepath="config.json"): - config = get_config() # ^ - with open(filepath, "w") as f: - json.dump(config, f, indent=4) - print(f"Configuration exported to {filepath}") - -# Validates that a configuration with the given ID exists. -def validate_config(config_id: str): - # The API key under which configs are stored - api_key = os.environ.get("COREBRAIN_API_KEY", "") - manager = ConfigManager() - cfg = manager.get_config(api_key, config_id) - - if cfg: - print(f"✅ Configuration '{config_id}' is present and valid.") - return 0 - else: - print(f"❌ Configuration '{config_id}' not found.") - return 1 - -# Function to print colored messages -def _print_colored(message: str, color: str) -> None: - """Simplified version of _print_colored that does not depend on cli.utils.""" - colors = { - "red": "\033[91m", - "green": "\033[92m", - "yellow": "\033[93m", - "blue": "\033[94m", - "default": "\033[0m" - } - color_code = colors.get(color, colors["default"]) - print(f"{color_code}{message}{colors['default']}") - -class ConfigManager: - """SDK configuration manager with improved security and performance.""" - - CONFIG_DIR = Path.home() / ".corebrain" - CONFIG_FILE = CONFIG_DIR / "config.json" - SECRET_KEY_FILE = CONFIG_DIR / "secret.key" - - def __init__(self): - self.configs = {} - self.cipher = None - self._ensure_config_dir() - self._load_secret_key() - self._load_configs() - - def _ensure_config_dir(self) -> None: - """Ensures that the configuration directory exists.""" - try: - self.CONFIG_DIR.mkdir(parents=True, exist_ok=True) - logger.debug(f"Configuration directory secured: {self.CONFIG_DIR}") - _print_colored(f"Configuration directory secured: {self.CONFIG_DIR}", "blue") - except Exception as e: - logger.error(f"Error creating configuration directory: {str(e)}") - _print_colored(f"Error creating configuration directory: {str(e)}", "red") - - def _load_secret_key(self) -> None: - """Loads or generates the secret key to encrypt sensitive data.""" - try: - if not self.SECRET_KEY_FILE.exists(): - key = Fernet.generate_key() - with open(self.SECRET_KEY_FILE, 'wb') as key_file: - key_file.write(key) - _print_colored(f"New secret key generated at: {self.SECRET_KEY_FILE}", "green") - - with open(self.SECRET_KEY_FILE, 'rb') as key_file: - self.secret_key = key_file.read() - - self.cipher = Fernet(self.secret_key) - except Exception as e: - _print_colored(f"Error loading/generating secret key: {str(e)}", "red") - # Fallback to temporary key (less secure but functional) - self.secret_key = Fernet.generate_key() - self.cipher = Fernet(self.secret_key) - - def _load_configs(self) -> Dict[str, Dict[str, Any]]: - """Loads the saved configurations.""" - if not self.CONFIG_FILE.exists(): - _print_colored(f"Configuration file not found: {self.CONFIG_FILE}", "yellow") - return {} - - try: - with open(self.CONFIG_FILE, 'r') as f: - encrypted_data = f.read() - - if not encrypted_data: - _print_colored("Empty configuration file", "yellow") - return {} - - try: - # Try to decrypt the data - decrypted_data = self.cipher.decrypt(encrypted_data.encode()).decode() - configs = json.loads(decrypted_data) - except Exception as e: - # If decryption fails, try to load as plain JSON - logger.warning(f"Error decrypting configuration: {e}") - configs = json.loads(encrypted_data) - - if isinstance(configs, str): - configs = json.loads(configs) - - _print_colored(f"Configuration loaded", "green") - self.configs = configs - return configs - except Exception as e: - _print_colored(f"Error loading configurations: {str(e)}", "red") - return {} - - def _save_configs(self) -> None: - """Saves the current configurations.""" - try: - configs_json = serialize_to_json(self.configs) - encrypted_data = self.cipher.encrypt(json.dumps(configs_json).encode()).decode() - - with open(self.CONFIG_FILE, 'w') as f: - f.write(encrypted_data) - - _print_colored(f"Configurations saved to: {self.CONFIG_FILE}", "green") - except Exception as e: - _print_colored(f"Error saving configurations: {str(e)}", "red") - - def add_config(self, api_key: str, db_config: Dict[str, Any], config_id: Optional[str] = None) -> str: - """ - Adds a new configuration. - - Args: - api_key: Selected API Key - db_config: Database configuration - config_id: Optional ID for the configuration (one is generated if not provided) - - Returns: - Configuration ID - """ - if not config_id: - config_id = str(uuid.uuid4()) - db_config["config_id"] = config_id - - # Create or update the entry for this token - if api_key not in self.configs: - self.configs[api_key] = {} - - # Add the configuration - self.configs[api_key][config_id] = db_config - self._save_configs() - - _print_colored(f"Configuration added: {config_id} for API Key: {api_key[:8]}...", "green") - return config_id - - def get_config(self, api_key_selected: str, config_id: str) -> Optional[Dict[str, Any]]: - """ - Retrieves a specific configuration. - - Args: - api_key_selected: Selected API Key - config_id: Configuration ID - - Returns: - Configuration or None if it does not exist - """ - return self.configs.get(api_key_selected, {}).get(config_id) - - def list_configs(self, api_key_selected: str) -> List[str]: - """ - Lists the available configuration IDs for an API Key. - - Args: - api_key_selected: Selected API Key - - Returns: - List of configuration IDs - """ - return list(self.configs.get(api_key_selected, {}).keys()) - - def remove_config(self, api_key_selected: str, config_id: str) -> bool: - """ - Deletes a configuration. - - Args: - api_key_selected: Selected API Key - config_id: Configuration ID - - Returns: - True if deleted successfully, False otherwise - """ - if api_key_selected in self.configs and config_id in self.configs[api_key_selected]: - del self.configs[api_key_selected][config_id] - - # If no configurations remain for this token, remove the entry - if not self.configs[api_key_selected]: - del self.configs[api_key_selected] - - self._save_configs() - _print_colored(f"Configuration {config_id} removed for API Key: {api_key_selected[:8]}...", "green") - return True - - _print_colored(f"Configuration {config_id} not found for API Key: {api_key_selected[:8]}...", "yellow") - return False \ No newline at end of file diff --git a/corebrain/corebrain/core/__init__.py b/corebrain/corebrain/core/__init__.py deleted file mode 100644 index a5fb552..0000000 --- a/corebrain/corebrain/core/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Corebrain SDK main components. - -This package contains the core components of the SDK, -including the main client and schema handling. -""" -from corebrain.core.client import Corebrain, init -from corebrain.core.query import QueryCache, QueryAnalyzer, QueryTemplate -from corebrain.core.test_utils import test_natural_language_query, generate_test_question_from_schema - -# Exportación explícita de componentes públicos -__all__ = [ - 'Corebrain', - 'init', - 'QueryCache', - 'QueryAnalyzer', - 'QueryTemplate', - 'test_natural_language_query', - 'generate_test_question_from_schema' -] \ No newline at end of file diff --git a/corebrain/corebrain/core/client.py b/corebrain/corebrain/core/client.py deleted file mode 100644 index 9b65225..0000000 --- a/corebrain/corebrain/core/client.py +++ /dev/null @@ -1,1364 +0,0 @@ -""" -Corebrain SDK Main Client. - -This module provides the main interface to interact with the Corebrain API -and enables natural language queries to relational and non-relational databases. -""" -import uuid -import re -import logging -import requests -import httpx -import sqlite3 -import mysql.connector -import psycopg2 -import pymongo -import json -from typing import Dict, Any, List, Optional -from sqlalchemy import create_engine, inspect -from pathlib import Path -from datetime import datetime - -from corebrain.core.common import logger, CorebrainError - -class Corebrain: - """ - Main client for the Corebrain SDK for natural language database queries. - - This class provides a unified interface to interact with different types of databases - (SQL and NoSQL) using natural language. It manages the connection, schema extraction, - and query processing through the Corebrain API. - - Attributes: - api_key (str): Authentication key for the Corebrain API. - db_config (Dict[str, Any]): Database connection configuration. - config_id (str): Unique identifier for the current configuration. - api_url (str): Base URL for the Corebrain API. - user_info (Dict[str, Any]): Information about the authenticated user. - db_connection: Active database connection. - db_schema (Dict[str, Any]): Extracted database schema. - - Examples: - SQLite initialization: - ```python - from corebrain import init - - # Connect to a SQLite database - client = init( - api_key="your_api_key", - db_config={ - "type": "sql", - "engine": "sqlite", - "database": "my_database.db" - } - ) - - # Make a query - result = client.ask("How many registered users are there?") - print(result["explanation"]) - ``` - - PostgreSQL initialization: - ```python - # Connect to PostgreSQL - client = init( - api_key="your_api_key", - db_config={ - "type": "sql", - "engine": "postgresql", - "host": "localhost", - "port": 5432, - "user": "postgres", - "password": "your_password", - "database": "my_database" - } - ) - ``` - - MongoDB initialization: - ```python - # Connect to MongoDB - client = init( - api_key="your_api_key", - db_config={ - "type": "mongodb", - "host": "localhost", - "port": 27017, - "database": "my_database" - } - ) - ``` - """ - - def __init__( - self, - api_key: str, - db_config: Optional[Dict[str, Any]] = None, - config_id: Optional[str] = None, - user_data: Optional[Dict[str, Any]] = None, - api_url: str = "http://localhost:5000", - skip_verification: bool = False - ): - """ - Initialize the Corebrain SDK client. - - Args: - api_key (str): Required API key for authentication with the Corebrain service. - Can be generated from the dashboard at https://dashboard.corebrain.com. - - db_config (Dict[str, Any], optional): Database configuration to query. - This parameter is required if config_id is not provided. Must contain at least: - - "type": Database type ("sql" or "mongodb") - - For SQL: "engine" ("sqlite", "postgresql", "mysql") - - Specific connection parameters depending on type and engine - - Example for SQLite: - ``` - { - "type": "sql", - "engine": "sqlite", - "database": "path/to/database.db" - } - ``` - - Example for PostgreSQL: - ``` - { - "type": "sql", - "engine": "postgresql", - "host": "localhost", - "port": 5432, - "user": "postgres", - "password": "password", - "database": "db_name" - } - ``` - - config_id (str, optional): Identifier for a previously saved configuration. - If provided, this configuration will be used instead of db_config. - Useful for maintaining persistent configurations between sessions. - - user_data (Dict[str, Any], optional): Additional user information for verification. - Can contain data like "email" for more precise token validation. - - api_url (str, optional): Base URL for the Corebrain API. - Defaults to "http://localhost:5000" for local development. - In production, it is typically "https://api.corebrain.com". - - skip_verification (bool, optional): If True, skips token verification with the server. - Useful in offline environments or for local testing. - Defaults to False. - - Raises: - ValueError: If required parameters are missing or if the configuration is invalid. - CorebrainError: If there are issues with the API connection or database. - - Example: - ```python - from corebrain import Corebrain - - # Basic initialization with SQLite - client = Corebrain( - api_key="your_api_key", - db_config={ - "type": "sql", - "engine": "sqlite", - "database": "my_db.db" - } - ) - ``` - """ - self.api_key = api_key - self.user_data = user_data - self.api_url = api_url.rstrip('/') - self.db_connection = None - self.db_schema = None - - # Import ConfigManager dynamically to avoid circular dependency - try: - from corebrain.config.manager import ConfigManager - self.config_manager = ConfigManager() - except ImportError as e: - logger.error(f"Error importing ConfigManager: {e}") - raise CorebrainError(f"Could not load configuration manager: {e}") - - # Determine which configuration to use - if config_id: - saved_config = self.config_manager.get_config(api_key, config_id) - if not saved_config: - # Try to load from old format - old_config = self._load_old_config(api_key, config_id) - if old_config: - self.db_config = old_config - self.config_id = config_id - # Save in new format - self.config_manager.add_config(api_key, old_config, config_id) - else: - raise ValueError(f"Configuration with ID {config_id} not found for the provided key") - else: - self.db_config = saved_config - self.config_id = config_id - elif db_config: - self.db_config = db_config - - # Generate config ID if it doesn't exist - if "config_id" in db_config: - self.config_id = db_config["config_id"] - else: - self.config_id = str(uuid.uuid4()) - db_config["config_id"] = self.config_id - - # Save the configuration - self.config_manager.add_config(api_key, db_config, self.config_id) - else: - raise ValueError("db_config or config_id must be provided") - - # Validate configuration - self._validate_config() - - # Verify the API token (only if necessary) - if not skip_verification: - self._verify_api_token() - else: - # Initialize user_info with basic information if not verifying - self.user_info = {"token": api_key} - - # Connect to the database - self._connect_to_database() - - # Extract database schema - self.db_schema = self._extract_db_schema() - - self.metadata = { - "config_id": self.config_id, - "api_key": api_key, - "db_config": self.db_config - } - - def _load_old_config(self, api_key: str, config_id: str) -> Optional[Dict[str, Any]]: - """ - Try to load configuration from old format. - - Args: - api_key: API key - config_id: Configuration ID - - Returns: - Configuration dictionary if found, None otherwise - """ - try: - # Try to load from old config file - old_config_path = Path.home() / ".corebrain" / "config.json" - if old_config_path.exists(): - with open(old_config_path, 'r') as f: - old_configs = json.load(f) - if api_key in old_configs and config_id in old_configs[api_key]: - return old_configs[api_key][config_id] - except Exception as e: - logger.warning(f"Error loading old config: {e}") - return None - - def _validate_config(self) -> None: - """ - Validate the provided configuration. - - This internal function verifies that the database configuration - contains all necessary fields according to the specified database type. - - Raises: - ValueError: If the database configuration is invalid or incomplete. - """ - if not self.api_key: - raise ValueError("API key is required. Generate one at dashboard.corebrain.com") - - if not self.db_config: - raise ValueError("Database configuration is required") - - if "type" not in self.db_config: - raise ValueError("Database type is required in db_config") - - if "connection_string" not in self.db_config and self.db_config["type"] != "sqlite_memory": - if self.db_config["type"] == "sql": - if "engine" not in self.db_config: - raise ValueError("Database engine is required for 'sql' type") - - # Verify alternative configuration for SQL engines - if self.db_config["engine"] == "mysql" or self.db_config["engine"] == "postgresql": - if not ("host" in self.db_config and "user" in self.db_config and - "password" in self.db_config and "database" in self.db_config): - raise ValueError("host, user, password, and database are required for MySQL/PostgreSQL") - elif self.db_config["engine"] == "sqlite": - if "database" not in self.db_config: - raise ValueError("database field is required for SQLite") - elif self.db_config["type"] == "mongodb": - if "database" not in self.db_config: - raise ValueError("database field is required for MongoDB") - - if "connection_string" not in self.db_config: - if not ("host" in self.db_config and "port" in self.db_config): - raise ValueError("host and port are required for MongoDB without connection_string") - - def _verify_api_token(self) -> None: - """ - Verify the API token with the server. - - This internal function sends a request to the Corebrain server - to validate that the provided API token is valid. - If the user provided additional information (like email), - it will be used for more precise verification. - - The verification results are stored in self.user_info. - - Raises: - ValueError: If the API token is invalid. - """ - try: - # Use the user's email for verification if available - if self.user_data and 'email' in self.user_data: - endpoint = f"{self.api_url}/api/auth/users/{self.user_data['email']}" - - response = httpx.get( - endpoint, - headers={"X-API-Key": self.api_key}, - timeout=10.0 - ) - - if response.status_code != 200: - raise ValueError(f"Invalid API token. Error code: {response.status_code}") - - # Store user information - self.user_info = response.json() - else: - # If no email, do a simple verification with a generic endpoint - endpoint = f"{self.api_url}/api/auth/verify" - - try: - response = httpx.get( - endpoint, - headers={"X-API-Key": self.api_key}, - timeout=5.0 - ) - - if response.status_code == 200: - self.user_info = response.json() - else: - # If it fails, just store basic information - self.user_info = {"token": self.api_key} - except Exception as e: - # If there's a connection error, don't fail, just store basic info - logger.warning(f"Could not verify token: {str(e)}") - self.user_info = {"token": self.api_key} - - except httpx.RequestError as e: - # Connection error shouldn't be fatal if we already have a configuration - logger.warning(f"Error connecting to API: {str(e)}") - self.user_info = {"token": self.api_key} - except Exception as e: - # Other errors are logged but not fatal - logger.warning(f"Error in token verification: {str(e)}") - self.user_info = {"token": self.api_key} - - def _connect_to_database(self) -> None: - """ - Establish a connection to the database according to the configuration. - - This internal function creates a database connection using the parameters - defined in self.db_config. It supports various database types: - - SQLite (file or in-memory) - - PostgreSQL - - MySQL - - MongoDB - - The connection is stored in self.db_connection for later use. - - Raises: - CorebrainError: If the connection to the database cannot be established. - NotImplementedError: If the database type is not supported. - """ - db_type = self.db_config["type"].lower() - - try: - if db_type == "sql": - engine = self.db_config.get("engine", "").lower() - - if engine == "sqlite": - database = self.db_config.get("database", "") - if database: - self.db_connection = sqlite3.connect(database) - else: - self.db_connection = sqlite3.connect(self.db_config.get("connection_string", "")) - - elif engine == "mysql": - if "connection_string" in self.db_config: - self.db_connection = mysql.connector.connect( - connection_string=self.db_config["connection_string"] - ) - else: - self.db_connection = mysql.connector.connect( - host=self.db_config.get("host", "localhost"), - user=self.db_config.get("user", ""), - password=self.db_config.get("password", ""), - database=self.db_config.get("database", ""), - port=self.db_config.get("port", 3306) - ) - - elif engine == "postgresql": - if "connection_string" in self.db_config: - self.db_connection = psycopg2.connect(self.db_config["connection_string"]) - else: - self.db_connection = psycopg2.connect( - host=self.db_config.get("host", "localhost"), - user=self.db_config.get("user", ""), - password=self.db_config.get("password", ""), - dbname=self.db_config.get("database", ""), - port=self.db_config.get("port", 5432) - ) - - else: - # Use SQLAlchemy for other engines - self.db_connection = create_engine(self.db_config["connection_string"]) - - # Improved code for MongoDB - elif db_type == "nosql" or db_type == "mongodb": - # If engine is mongodb or the type is directly mongodb - engine = self.db_config.get("engine", "").lower() - if not engine or engine == "mongodb": - # Create connection parameters - mongo_params = {} - - if "connection_string" in self.db_config: - # Save the MongoDB client to be able to close it correctly later - self.mongo_client = pymongo.MongoClient(self.db_config["connection_string"]) - else: - # Configure host and port - mongo_params["host"] = self.db_config.get("host", "localhost") - if "port" in self.db_config: - mongo_params["port"] = self.db_config.get("port") - - # Add credentials if available - if "user" in self.db_config and self.db_config["user"]: - mongo_params["username"] = self.db_config["user"] - if "password" in self.db_config and self.db_config["password"]: - mongo_params["password"] = self.db_config["password"] - - # Create MongoDB client - self.mongo_client = pymongo.MongoClient(**mongo_params) - - # Get the database - db_name = self.db_config.get("database", "") - if db_name: - # Save reference to the database - self.db_connection = self.mongo_client[db_name] - else: - # If there's no database name, use 'admin' as fallback - logger.warning("Database name not specified for MongoDB, using 'admin'") - self.db_connection = self.mongo_client["admin"] - else: - raise ValueError(f"Unsupported NoSQL database engine: {engine}") - - elif db_type == "sqlite_memory": - self.db_connection = sqlite3.connect(":memory:") - - else: - raise ValueError(f"Unsupported database type: {db_type}. Valid types: 'sql', 'nosql', 'mongodb'") - - except Exception as e: - logger.error(f"Error connecting to database: {str(e)}") - raise ConnectionError(f"Error connecting to database: {str(e)}") - - def _extract_db_schema(self, detail_level: str = "full", specific_collections: List[str] = None) -> Dict[str, Any]: - """ - Extracts the database schema to provide context to the AI. - - Returns: - Dictionary with the database structure organized by tables/collections - """ - logger.info(f"Extrayendo esquema de base de datos. Tipo: {self.db_config['type']}, Motor: {self.db_config.get('engine')}") - - db_type = self.db_config["type"].lower() - schema = { - "type": db_type, - "database": self.db_config.get("database", ""), - "tables": {}, - "total_collections": 0, # Añadir contador total - "included_collections": 0 # Contador de incluidas - } - excluded_tables = set(self.db_config.get("excluded_tables", [])) - logger.info(f"Tablas excluidas: {excluded_tables}") - - try: - if db_type == "sql": - engine = self.db_config.get("engine", "").lower() - logger.info(f"Procesando base de datos SQL con motor: {engine}") - - if engine in ["sqlite", "mysql", "postgresql"]: - cursor = self.db_connection.cursor() - - if engine == "sqlite": - logger.info("Obteniendo tablas de SQLite") - # Obtener listado de tablas - cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") - tables = cursor.fetchall() - logger.info(f"Tablas encontradas en SQLite: {tables}") - - elif engine == "mysql": - logger.info("Obteniendo tablas de MySQL") - cursor.execute("SHOW TABLES;") - tables = cursor.fetchall() - logger.info(f"Tablas encontradas en MySQL: {tables}") - - elif engine == "postgresql": - logger.info("Obteniendo tablas de PostgreSQL") - cursor.execute(""" - SELECT table_name FROM information_schema.tables - WHERE table_schema = 'public'; - """) - tables = cursor.fetchall() - logger.info(f"Tablas encontradas en PostgreSQL: {tables}") - - # Procesar las tablas encontradas - for table in tables: - table_name = table[0] - logger.info(f"Procesando tabla: {table_name}") - - # Saltar tablas excluidas - if table_name in excluded_tables: - logger.info(f"Saltando tabla excluida: {table_name}") - continue - - try: - # Obtener información de columnas según el motor - if engine == "sqlite": - cursor.execute(f"PRAGMA table_info({table_name});") - elif engine == "mysql": - cursor.execute(f"DESCRIBE {table_name};") - elif engine == "postgresql": - cursor.execute(f""" - SELECT column_name, data_type - FROM information_schema.columns - WHERE table_name = '{table_name}'; - """) - - columns = cursor.fetchall() - logger.info(f"Columnas encontradas para {table_name}: {columns}") - - # Estructura de columnas según el motor - if engine == "sqlite": - column_info = [{"name": col[1], "type": col[2]} for col in columns] - elif engine == "mysql": - column_info = [{"name": col[0], "type": col[1]} for col in columns] - elif engine == "postgresql": - column_info = [{"name": col[0], "type": col[1]} for col in columns] - - # Guardar información de la tabla - schema["tables"][table_name] = { - "columns": column_info, - "sample_data": [] # No obtenemos datos de muestra por defecto - } - - except Exception as e: - logger.error(f"Error procesando tabla {table_name}: {str(e)}") - - else: - # Usando SQLAlchemy - logger.info("Usando SQLAlchemy para obtener el esquema") - inspector = inspect(self.db_connection) - table_names = inspector.get_table_names() - logger.info(f"Tablas encontradas con SQLAlchemy: {table_names}") - - for table_name in table_names: - if table_name in excluded_tables: - logger.info(f"Saltando tabla excluida: {table_name}") - continue - - try: - columns = inspector.get_columns(table_name) - column_info = [{"name": col["name"], "type": str(col["type"])} for col in columns] - - schema["tables"][table_name] = { - "columns": column_info, - "sample_data": [] - } - except Exception as e: - logger.error(f"Error procesando tabla {table_name} con SQLAlchemy: {str(e)}") - - elif db_type in ["nosql", "mongodb"]: - logger.info("Procesando base de datos MongoDB") - if not hasattr(self, 'db_connection') or self.db_connection is None: - logger.error("La conexión a MongoDB no está disponible") - return schema - - try: - collection_names = [] - try: - collection_names = self.db_connection.list_collection_names() - schema["total_collections"] = len(collection_names) - logger.info(f"Colecciones encontradas en MongoDB: {collection_names}") - except Exception as e: - logger.error(f"Error al obtener colecciones MongoDB: {str(e)}") - return schema - - # Si solo queremos los nombres - if detail_level == "names_only": - schema["collection_names"] = collection_names - return schema - - # Procesar cada colección - for collection_name in collection_names: - if collection_name in excluded_tables: - logger.info(f"Saltando colección excluida: {collection_name}") - continue - - try: - collection = self.db_connection[collection_name] - # Obtener un documento para inferir estructura - first_doc = collection.find_one() - - if first_doc: - fields = [] - for field, value in first_doc.items(): - if field != "_id": - field_type = type(value).__name__ - fields.append({"name": field, "type": field_type}) - - schema["tables"][collection_name] = { - "fields": fields, - "doc_count": collection.estimated_document_count() - } - logger.info(f"Procesada colección {collection_name} con {len(fields)} campos") - else: - logger.info(f"Colección {collection_name} está vacía") - schema["tables"][collection_name] = { - "fields": [], - "doc_count": 0 - } - except Exception as e: - logger.error(f"Error procesando colección {collection_name}: {str(e)}") - - except Exception as e: - logger.error(f"Error general procesando MongoDB: {str(e)}") - - # Convertir el diccionario de tablas en una lista - table_list = [] - for table_name, table_info in schema["tables"].items(): - table_data = {"name": table_name} - table_data.update(table_info) - table_list.append(table_data) - - schema["tables_list"] = table_list - logger.info(f"Esquema final - Tablas encontradas: {len(schema['tables'])}") - logger.info(f"Nombres de tablas: {list(schema['tables'].keys())}") - - return schema - - except Exception as e: - logger.error(f"Error al extraer el esquema de la base de datos: {str(e)}") - return {"type": db_type, "tables": {}, "tables_list": []} - - def list_collections_name(self) -> List[str]: - """ - Returns a list of the available collections or tables in the database. - - Returns: - List of collections or tables - """ - print("Excluded tables: ", self.db_schema.get("excluded_tables", [])) - return self.db_schema.get("tables", []) - - def ask(self, question: str, **kwargs) -> Dict: - """ - Perform a natural language query to the database. - - Args: - question: The natural language question - **kwargs: Additional parameters: - - collection_name: For MongoDB, the collection to query - - limit: Maximum number of results - - detail_level: Schema detail level ("names_only", "structure", "full") - - auto_select: Whether to automatically select collections - - max_collections: Maximum number of collections to include - - execute_query: Whether to execute the query (True by default) - - explain_results: Whether to generate an explanation of results (True by default) - - Returns: - Dictionary with the query results and explanation - """ - try: - # Verificar opciones de comportamiento - execute_query = kwargs.get("execute_query", True) - explain_results = kwargs.get("explain_results", True) - - # Obtener esquema con el nivel de detalle apropiado - detail_level = kwargs.get("detail_level", "full") - schema = self._extract_db_schema(detail_level=detail_level) - - # Validar que el esquema tiene tablas/colecciones - if not schema.get("tables"): - print("Error: No se encontraron tablas/colecciones en la base de datos") - return {"error": True, "explanation": "No se encontraron tablas/colecciones en la base de datos"} - - # Obtener nombres de tablas disponibles para validación - available_tables = set() - if isinstance(schema.get("tables"), dict): - available_tables.update(schema["tables"].keys()) - elif isinstance(schema.get("tables_list"), list): - available_tables.update(table["name"] for table in schema["tables_list"]) - - # Preparar datos de la solicitud con información de esquema mejorada - request_data = { - "question": question, - "db_schema": schema, - "config_id": self.config_id, - "metadata": { - "type": self.db_config["type"].lower(), - "engine": self.db_config.get("engine", "").lower(), - "database": self.db_config.get("database", ""), - "available_tables": list(available_tables), - "collections": list(available_tables) - } - } - - # Añadir configuración de la base de datos al request - # Esto permite a la API ejecutar directamente las consultas si es necesario - if execute_query: - request_data["db_config"] = self.db_config - - # Añadir datos de usuario si están disponibles - if self.user_data: - request_data["user_data"] = self.user_data - - # Preparar headers para la solicitud - headers = { - "X-API-Key": self.api_key, - "Content-Type": "application/json" - } - - # Determinar el endpoint adecuado según el modo de ejecución - if execute_query: - # Usar el endpoint de ejecución completa - endpoint = f"{self.api_url}/api/database/sdk/query" - else: - # Usar el endpoint de solo generación de consulta - endpoint = f"{self.api_url}/api/database/generate" - - # Realizar solicitud a la API - response = httpx.post( - endpoint, - headers=headers, - content=json.dumps(request_data, default=str), - timeout=60.0 - ) - - # Verificar respuesta - if response.status_code != 200: - error_msg = f"Error {response.status_code} al realizar la consulta" - try: - error_data = response.json() - if isinstance(error_data, dict): - error_msg += f": {error_data.get('detail', error_data.get('message', response.text))}" - except: - error_msg += f": {response.text}" - return {"error": True, "explanation": error_msg} - - # Procesar respuesta de la API - api_response = response.json() - - # Verificar si la API reportó un error - if api_response.get("error", False): - return api_response - - # Verificar si se generó una consulta válida - if "query" not in api_response: - return { - "error": True, - "explanation": "La API no generó una consulta válida." - } - - # Si se debe ejecutar la consulta pero la API no lo hizo - # (esto ocurriría solo en caso de cambios de configuración o fallbacks) - if execute_query and "result" not in api_response: - try: - # Preparar la consulta para ejecución local - query_type = self.db_config.get("engine", "").lower() if self.db_config["type"].lower() == "sql" else self.db_config["type"].lower() - query_value = api_response["query"] - - # Para SQL, asegurarse de que la consulta es un string - if query_type in ["sqlite", "mysql", "postgresql"]: - if isinstance(query_value, dict): - sql_candidate = query_value.get("sql") or query_value.get("query") - if isinstance(sql_candidate, str): - query_value = sql_candidate - else: - raise CorebrainError(f"La consulta SQL generada no es un string: {query_value}") - - # Preparar la consulta con el formato adecuado - query_to_execute = { - "type": query_type, - "query": query_value - } - - # Para MongoDB, añadir información específica - if query_type in ["nosql", "mongodb"]: - # Obtener nombre de colección - collection_name = None - if isinstance(api_response["query"], dict): - collection_name = api_response["query"].get("collection") - if not collection_name and "collection_name" in kwargs: - collection_name = kwargs["collection_name"] - if not collection_name and "collection" in self.db_config: - collection_name = self.db_config["collection"] - if not collection_name and available_tables: - collection_name = list(available_tables)[0] - - # Validar nombre de colección - if not collection_name: - raise CorebrainError("No se especificó colección y no se encontraron colecciones en el esquema") - if not isinstance(collection_name, str) or not collection_name.strip(): - raise CorebrainError("Nombre de colección inválido: debe ser un string no vacío") - - # Añadir colección a la consulta - query_to_execute["collection"] = collection_name - - # Añadir tipo de operación - if isinstance(api_response["query"], dict): - query_to_execute["operation"] = api_response["query"].get("operation", "find") - - # Añadir límite si se especifica - if "limit" in kwargs: - query_to_execute["limit"] = kwargs["limit"] - - # Ejecutar la consulta - start_time = datetime.now() - query_result = self._execute_query(query_to_execute) - query_time_ms = int((datetime.now() - start_time).total_seconds() * 1000) - - # Actualizar la respuesta con los resultados - api_response["result"] = { - "data": query_result, - "count": len(query_result) if isinstance(query_result, list) else 1, - "query_time_ms": query_time_ms, - "has_more": False - } - - # Si se debe generar explicación pero la API no lo hizo - if explain_results and ( - "explanation" not in api_response or - not isinstance(api_response.get("explanation"), str) or - len(str(api_response.get("explanation", ""))) < 15 # Detectar explicaciones numéricas o muy cortas - ): - # Preparar datos para obtener explicación - explanation_data = { - "question": question, - "query": api_response["query"], - "result": query_result, - "query_time_ms": query_time_ms, - "config_id": self.config_id, - "metadata": { - "collections_used": [query_to_execute.get("collection")] if query_to_execute.get("collection") else [], - "execution_time_ms": query_time_ms, - "available_tables": list(available_tables) - } - } - - try: - # Obtener explicación de la API - explanation_response = httpx.post( - f"{self.api_url}/api/database/sdk/query/explain", - headers=headers, - content=json.dumps(explanation_data, default=str), - timeout=30.0 - ) - - if explanation_response.status_code == 200: - explanation_result = explanation_response.json() - api_response["explanation"] = explanation_result.get("explanation", "No se pudo generar una explicación.") - else: - api_response["explanation"] = self._generate_fallback_explanation(query_to_execute, query_result) - except Exception as explain_error: - logger.error(f"Error al obtener explicación: {str(explain_error)}") - api_response["explanation"] = self._generate_fallback_explanation(query_to_execute, query_result) - - except Exception as e: - error_msg = f"Error al ejecutar la consulta: {str(e)}" - logger.error(error_msg) - return { - "error": True, - "explanation": error_msg, - "query": api_response.get("query", {}), - "metadata": { - "available_tables": list(available_tables) - } - } - - # Verificar si la explicación es un número (probablemente el tiempo de ejecución) y corregirlo - if "explanation" in api_response and not isinstance(api_response["explanation"], str): - # Si la explicación es un número, reemplazarla con una explicación generada - try: - is_sql = False - if "query" in api_response: - if isinstance(api_response["query"], dict) and "sql" in api_response["query"]: - is_sql = True - - if "result" in api_response: - result_data = api_response["result"] - if isinstance(result_data, dict) and "data" in result_data: - result_data = result_data["data"] - - if is_sql: - sql_query = api_response["query"].get("sql", "") - api_response["explanation"] = self._generate_sql_explanation(sql_query, result_data) - else: - # Para MongoDB o genérico - api_response["explanation"] = self._generate_generic_explanation(api_response["query"], result_data) - else: - api_response["explanation"] = "La consulta se ha ejecutado correctamente." - except Exception as exp_fix_error: - logger.error(f"Error al corregir explicación: {str(exp_fix_error)}") - api_response["explanation"] = "La consulta se ha ejecutado correctamente." - - # Preparar la respuesta final - result = { - "question": question, - "query": api_response["query"], - "config_id": self.config_id, - "metadata": { - "available_tables": list(available_tables) - } - } - - # Añadir resultados si están disponibles - if "result" in api_response: - if isinstance(api_response["result"], dict) and "data" in api_response["result"]: - result["result"] = api_response["result"] - else: - result["result"] = { - "data": api_response["result"], - "count": len(api_response["result"]) if isinstance(api_response["result"], list) else 1, - "query_time_ms": api_response.get("query_time_ms", 0), - "has_more": False - } - - # Añadir explicación si está disponible - if "explanation" in api_response: - result["explanation"] = api_response["explanation"] - - return result - - except httpx.TimeoutException: - return {"error": True, "explanation": "Tiempo de espera agotado al conectar con el servidor."} - - except httpx.RequestError as e: - return {"error": True, "explanation": f"Error de conexión con el servidor: {str(e)}"} - - except Exception as e: - import traceback - error_details = traceback.format_exc() - logger.error(f"Error inesperado en ask(): {error_details}") - return {"error": True, "explanation": f"Error inesperado: {str(e)}"} - - def _generate_fallback_explanation(self, query, results): - """ - Generates a fallback explanation when the explanation generation fails. - - Args: - query: The executed query - results: The obtained results - - Returns: - Generated explanation - """ - # Determinar si es SQL o MongoDB - if isinstance(query, dict): - query_type = query.get("type", "").lower() - - if query_type in ["sqlite", "mysql", "postgresql"]: - return self._generate_sql_explanation(query.get("query", ""), results) - elif query_type in ["nosql", "mongodb"]: - return self._generate_mongodb_explanation(query, results) - - # Fallback genérico - result_count = len(results) if isinstance(results, list) else (1 if results else 0) - return f"La consulta devolvió {result_count} resultados." - - def _generate_sql_explanation(self, sql_query, results): - """ - Generates a simple explanation for SQL queries. - - Args: - sql_query: The executed SQL query - results: The obtained results - - Returns: - Generated explanation - """ - sql_lower = sql_query.lower() if isinstance(sql_query, str) else "" - result_count = len(results) if isinstance(results, list) else (1 if results else 0) - - # Extraer nombres de tablas si es posible - tables = [] - from_match = re.search(r'from\s+([a-zA-Z0-9_]+)', sql_lower) - if from_match: - tables.append(from_match.group(1)) - - join_matches = re.findall(r'join\s+([a-zA-Z0-9_]+)', sql_lower) - if join_matches: - tables.extend(join_matches) - - # Detectar tipo de consulta - if "select" in sql_lower: - if "join" in sql_lower: - if len(tables) > 1: - if "where" in sql_lower: - return f"Se encontraron {result_count} registros que cumplen con los criterios especificados, relacionando información de las tablas {', '.join(tables)}." - else: - return f"Se obtuvieron {result_count} registros relacionando información de las tablas {', '.join(tables)}." - else: - return f"Se obtuvieron {result_count} registros relacionando datos entre tablas." - - elif "where" in sql_lower: - return f"Se encontraron {result_count} registros que cumplen con los criterios de búsqueda." - - else: - return f"La consulta devolvió {result_count} registros de la base de datos." - - # Para otros tipos de consultas (INSERT, UPDATE, DELETE) - if "insert" in sql_lower: - return "Se insertaron correctamente los datos en la base de datos." - elif "update" in sql_lower: - return "Se actualizaron correctamente los datos en la base de datos." - elif "delete" in sql_lower: - return "Se eliminaron correctamente los datos de la base de datos." - - # Fallback genérico - return f"La consulta SQL se ejecutó correctamente y devolvió {result_count} resultados." - - - def _generate_mongodb_explanation(self, query, results): - """ - Generates a simple explanation for MongoDB queries. - - Args: - query: The executed MongoDB query - results: The obtained results - - Returns: - Generated explanation - """ - collection = query.get("collection", "la colección") - operation = query.get("operation", "find") - result_count = len(results) if isinstance(results, list) else (1 if results else 0) - - # Generar explicación según la operación - if operation == "find": - return f"Se encontraron {result_count} documentos en la colección {collection} que coinciden con los criterios de búsqueda." - elif operation == "findOne": - if result_count > 0: - return f"Se encontró el documento solicitado en la colección {collection}." - else: - return f"No se encontró ningún documento en la colección {collection} que coincida con los criterios." - elif operation == "aggregate": - return f"La agregación en la colección {collection} devolvió {result_count} resultados." - elif operation == "insertOne": - return f"Se ha insertado correctamente un nuevo documento en la colección {collection}." - elif operation == "updateOne": - return f"Se ha actualizado correctamente un documento en la colección {collection}." - elif operation == "deleteOne": - return f"Se ha eliminado correctamente un documento de la colección {collection}." - - # Fallback genérico - return f"La operación {operation} se ejecutó correctamente en la colección {collection} y devolvió {result_count} resultados." - - - def _generate_generic_explanation(self, query, results): - """ - Generates a generic explanation when the query type cannot be determined. - - Args: - query: The executed query - results: The obtained results - - Returns: - Generated explanation - """ - result_count = len(results) if isinstance(results, list) else (1 if results else 0) - - if result_count == 0: - return "La consulta no devolvió ningún resultado." - elif result_count == 1: - return "La consulta devolvió 1 resultado." - else: - return f"La consulta devolvió {result_count} resultados." - - - def close(self) -> None: - """ - Close the database connection and release resources. - - This method should be called when the client is no longer needed to - ensure proper cleanup of resources. - """ - if self.db_connection: - db_type = self.db_config["type"].lower() - - try: - if db_type == "sql": - engine = self.db_config.get("engine", "").lower() - if engine in ["sqlite", "mysql", "postgresql"]: - self.db_connection.close() - else: - # SQLAlchemy engine - self.db_connection.dispose() - - elif db_type == "nosql" or db_type == "mongodb": - # For MongoDB, we close the client - if hasattr(self, 'mongo_client') and self.mongo_client: - self.mongo_client.close() - - elif db_type == "sqlite_memory": - self.db_connection.close() - - except Exception as e: - logger.warning(f"Error closing database connection: {str(e)}") - - self.db_connection = None - logger.info("Database connection closed") - - def _execute_query(self, query: Dict[str, Any]) -> List[Dict[str, Any]]: - """ - Execute a query based on its type. - - Args: - query: Dictionary containing query information - - Returns: - List of dictionaries containing query results - """ - query_type = query.get("type", "").lower() - - if query_type in ["sqlite", "mysql", "postgresql"]: - return self._execute_sql_query(query) - elif query_type in ["nosql", "mongodb"]: - return self._execute_mongodb_query(query) - else: - raise CorebrainError(f"Unsupported query type: {query_type}") - - def _execute_sql_query(self, query: Dict[str, Any]) -> List[Dict[str, Any]]: - """ - Execute a SQL query. - - Args: - query: Dictionary containing SQL query information - - Returns: - List of dictionaries containing query results - """ - query_type = query.get("type", "").lower() - - if query_type in ["sqlite", "mysql", "postgresql"]: - sql_query = query.get("query", "") - if not sql_query: - raise CorebrainError("No SQL query provided") - - engine = self.db_config.get("engine", "").lower() - - if engine == "sqlite": - return self._execute_sqlite_query(sql_query) - elif engine == "mysql": - return self._execute_mysql_query(sql_query) - elif engine == "postgresql": - return self._execute_postgresql_query(sql_query) - else: - raise CorebrainError(f"Unsupported SQL engine: {engine}") - - else: - raise CorebrainError(f"Unsupported SQL query type: {query_type}") - - def _execute_sqlite_query(self, sql_query: str) -> List[Dict[str, Any]]: - """ - Execute a SQLite query. - - Args: - sql_query (str): SQL query to execute - - Returns: - List[Dict[str, Any]]: List of results as dictionaries - """ - cursor = self.db_connection.cursor() - cursor.execute(sql_query) - - # Get column names - columns = [description[0] for description in cursor.description] - - # Convert results to list of dictionaries - results = [] - for row in cursor.fetchall(): - result = {} - for i, value in enumerate(row): - # Convert datetime objects to strings - if hasattr(value, 'isoformat'): - result[columns[i]] = value.isoformat() - else: - result[columns[i]] = value - results.append(result) - - return results - - def _execute_mysql_query(self, sql_query: str) -> List[Dict[str, Any]]: - """ - Execute a MySQL query. - - Args: - sql_query (str): SQL query to execute - - Returns: - List[Dict[str, Any]]: List of results as dictionaries - """ - cursor = self.db_connection.cursor(dictionary=True) - cursor.execute(sql_query) - - # Convert results to list of dictionaries - results = [] - for row in cursor.fetchall(): - result = {} - for key, value in row.items(): - # Convert datetime objects to strings - if hasattr(value, 'isoformat'): - result[key] = value.isoformat() - else: - result[key] = value - results.append(result) - - return results - - def _execute_postgresql_query(self, sql_query: str) -> List[Dict[str, Any]]: - """ - Execute a PostgreSQL query. - - Args: - sql_query (str): SQL query to execute - - Returns: - List[Dict[str, Any]]: List of results as dictionaries - """ - cursor = self.db_connection.cursor() - cursor.execute(sql_query) - - # Get column names - columns = [description[0] for description in cursor.description] - - # Convert results to list of dictionaries - results = [] - for row in cursor.fetchall(): - result = {} - for i, value in enumerate(row): - # Convert datetime objects to strings - if hasattr(value, 'isoformat'): - result[columns[i]] = value.isoformat() - else: - result[columns[i]] = value - results.append(result) - - return results - - def _execute_mongodb_query(self, query: Dict[str, Any]) -> List[Dict[str, Any]]: - """ - Execute a MongoDB query. - - Args: - query: Dictionary containing MongoDB query information - - Returns: - List of dictionaries containing query results - """ - try: - # Get collection name from query or use default - collection_name = query.get("collection") - if not collection_name: - raise CorebrainError("No collection specified for MongoDB query") - - # Get MongoDB collection - collection = self.mongo_client[self.db_config.get("database", "")][collection_name] - - # Execute query based on operation type - operation = query.get("operation", "find") - - if operation == "find": - # Handle find operation - cursor = collection.find( - query.get("query", {}), - projection=query.get("projection"), - sort=query.get("sort"), - limit=query.get("limit", 10), - skip=query.get("skip", 0) - ) - results = list(cursor) - - elif operation == "aggregate": - # Handle aggregate operation - pipeline = query.get("pipeline", []) - cursor = collection.aggregate(pipeline) - results = list(cursor) - - else: - raise CorebrainError(f"Unsupported MongoDB operation: {operation}") - - # Convert results to dictionaries and handle datetime serialization - serialized_results = [] - for doc in results: - # Convert ObjectId to string - if "_id" in doc: - doc["_id"] = str(doc["_id"]) - - # Handle datetime objects - for key, value in doc.items(): - if hasattr(value, 'isoformat'): - doc[key] = value.isoformat() - - serialized_results.append(doc) - - return serialized_results - - except Exception as e: - raise CorebrainError(f"Error executing MongoDB query: {str(e)}") - - -def init( - api_key: str = None, - db_config: Dict = None, - config_id: str = None, - user_data: Dict = None, - api_url: str = None, - skip_verification: bool = False -) -> Corebrain: - """ - Initialize and return a Corebrain client instance. - - This function creates a new Corebrain SDK client with the provided configuration. - It's a convenient factory function that wraps the Corebrain class initialization. - - Args: - api_key (str, optional): Corebrain API key. If not provided, it will attempt - to read from the COREBRAIN_API_KEY environment variable. - db_config (Dict, optional): Database configuration dictionary. If not provided, - it will attempt to read from the COREBRAIN_DB_CONFIG environment variable - (expected in JSON format). - config_id (str, optional): Configuration ID for saving/loading configurations. - user_data (Dict, optional): Optional user data for personalization. - api_url (str, optional): Corebrain API URL. Defaults to the production API. - skip_verification (bool, optional): Skip API token verification. Default False. - - Returns: - Corebrain: An initialized Corebrain client instance. - - Example: - >>> client = init(api_key="your_api_key", db_config={"type": "sql", "engine": "sqlite", "database": "example.db"}) - """ - return Corebrain( - api_key=api_key, - db_config=db_config, - config_id=config_id, - user_data=user_data, - api_url=api_url, - skip_verification=skip_verification - ) - diff --git a/corebrain/corebrain/core/common.py b/corebrain/corebrain/core/common.py deleted file mode 100644 index 3d75c8e..0000000 --- a/corebrain/corebrain/core/common.py +++ /dev/null @@ -1,225 +0,0 @@ -""" -Core functionalities shared across the Corebrain SDK. - -This module contains common elements used throughout the SDK, including: -- Logging system configuration -- Common type definitions and aliases -- Custom exceptions for better error handling -- Component registry system for dependency management - -These elements provide a common foundation for implementing -the rest of the SDK modules, ensuring consistency and facilitating -maintenance. -""" -import logging -from typing import Dict, Any, Optional, List, Callable, TypeVar, Union - -# Global logging configuration -logger = logging.getLogger("corebrain") -logger.addHandler(logging.NullHandler()) - -# Type aliases to improve readability and maintenance -ConfigDict = Dict[str, Any] -""" -Type representing a configuration as a key-value dictionary. - -Example: -```python -config: ConfigDict = { - "type": "sql", - "engine": "postgresql", - "host": "localhost", - "port": 5432, - "user": "postgres", - "password": "password", - "database": "mydatabase" -} -``` -""" - -SchemaDict = Dict[str, Any] -""" -Type representing a database schema as a dictionary. - -Example: -```python -schema: SchemaDict = { - "tables": [ - { - "name": "users", - "columns": [ - {"name": "id", "type": "INTEGER", "primary_key": True}, - {"name": "name", "type": "TEXT"}, - {"name": "email", "type": "TEXT"} - ] - } - ] -} -``` -""" - -# Generic component for typing -T = TypeVar('T') - -# SDK exceptions -class CorebrainError(Exception): - """ - Base exception for all Corebrain SDK errors. - - All other specific exceptions inherit from this class, - allowing you to catch any SDK error with a single - except block. - - Example: - ```python - try: - result = client.ask("How many users are there?") - except CorebrainError as e: - print(f"Corebrain error: {e}") - ``` - """ - pass - -class ConfigError(CorebrainError): - """ - Error related to SDK configuration. - - Raised when there are issues with the provided configuration, - such as invalid credentials, missing parameters, or incorrect formats. - - Example: - ```python - try: - client = init(api_key="invalid_key", db_config={}) - except ConfigError as e: - print(f"Configuration error: {e}") - ``` - """ - pass - -class DatabaseError(CorebrainError): - """ - Error related to database connection or query. - - Raised when there are problems connecting to the database, - executing queries, or extracting schema information. - - Example: - ```python - try: - result = client.ask("select * from a_table_that_does_not_exist") - except DatabaseError as e: - print(f"Database error: {e}") - ``` - """ - pass - -class APIError(CorebrainError): - """ - Error related to communication with the Corebrain API. - - Raised when there are issues in communicating with the service, - such as network errors, authentication failures, or unexpected responses. - - Example: - ```python - try: - result = client.ask("How many users are there?") - except APIError as e: - print(f"API error: {e}") - if e.status_code == 401: - print("Please verify your API key") - ``` - """ - def __init__(self, message: str, status_code: Optional[int] = None, response: Optional[Dict[str, Any]] = None): - """ - Initialize an APIError exception. - - Args: - message: Descriptive error message - status_code: Optional HTTP status code (e.g., 401, 404, 500) - response: Server response content if available - """ - self.status_code = status_code - self.response = response - super().__init__(message) - -# Component registry (to avoid circular imports) -_registry: Dict[str, Any] = {} - -def register_component(name: str, component: Any) -> None: - """ - Register a component in the global registry. - - This mechanism resolves circular dependencies between modules - by providing a way to access components without importing them directly. - - Args: - name: Unique name to identify the component - component: The component to register (can be any object) - - Example: - ```python - # In the module that defines the component - from core.common import register_component - - class DatabaseConnector: - def connect(self): - pass - - # Register the component - connector = DatabaseConnector() - register_component("db_connector", connector) - ``` - """ - _registry[name] = component - -def get_component(name: str) -> Any: - """ - Get a component from the global registry. - - Args: - name: Name of the component to retrieve - - Returns: - The registered component or None if it doesn't exist - - Example: - ```python - # In another module that needs to use the component - from core.common import get_component - - # Get the component - connector = get_component("db_connector") - if connector: - connector.connect() - ``` - """ - return _registry.get(name) - -def safely_get_component(name: str, default: Optional[T] = None) -> Union[Any, T]: - """ - Safely get a component from the global registry. - - If the component doesn't exist, it returns the provided default - value instead of None. - - Args: - name: Name of the component to retrieve - default: Default value to return if the component doesn't exist - - Returns: - The registered component or the default value - - Example: - ```python - # In another module - from core.common import safely_get_component - - # Get the component with a default value - connector = safely_get_component("db_connector", MyDefaultConnector()) - connector.connect() # Guaranteed not to be None - ``` - """ - component = _registry.get(name) - return component if component is not None else default \ No newline at end of file diff --git a/corebrain/corebrain/core/query.py b/corebrain/corebrain/core/query.py deleted file mode 100644 index 9c678d2..0000000 --- a/corebrain/corebrain/core/query.py +++ /dev/null @@ -1,1037 +0,0 @@ -""" -Components for query handling and analysis. -""" -import os -import json -import time -import re -import sqlite3 -import pickle -import hashlib - -from typing import Dict, Any, List, Optional, Tuple, Callable -from datetime import datetime -from pathlib import Path - -from corebrain.cli.utils import print_colored - -class QueryCache: - """Multilevel cache system for queries.""" - - def __init__(self, cache_dir: str = None, ttl: int = 86400, memory_limit: int = 100): - """ - Initializes the cache system. - - Args: - cache_dir: Directory for persistent cache - ttl: Time-to-live of the cache in seconds (default: 24 hours) - memory_limit: Memory cache entry limit - """ - # Caché en memoria (más rápido, pero volátil) - self.memory_cache = {} - self.memory_timestamps = {} - self.memory_limit = memory_limit - self.memory_lru = [] # Lista para seguimiento de menos usados recientemente - - # Caché persistente (más lento, pero permanente) - self.ttl = ttl - if cache_dir: - self.cache_dir = Path(cache_dir) - else: - self.cache_dir = Path.home() / ".corebrain_cache" - - # Create cache directory if it doesn't exist - self.cache_dir.mkdir(parents=True, exist_ok=True) - - # Initialize SQLite database for metadata - self.db_path = self.cache_dir / "cache_metadata.db" - self._init_db() - - print_colored(f"Cache initialized at {self.cache_dir}", "blue") - - def _init_db(self): - """Initializes the SQLite database for cache metadata.""" - conn = sqlite3.connect(str(self.db_path)) - cursor = conn.cursor() - - # Create metadata table if it doesn't exist - cursor.execute(''' - CREATE TABLE IF NOT EXISTS cache_metadata ( - query_hash TEXT PRIMARY KEY, - query TEXT, - config_id TEXT, - created_at TIMESTAMP, - last_accessed TIMESTAMP, - hit_count INTEGER DEFAULT 1 - ) - ''') - - conn.commit() - conn.close() - - def _get_hash(self, query: str, config_id: str, collection_name: Optional[str] = None) -> str: - """Generates a unique hash for the query.""" - # Normalize the query (remove extra spaces, convert to lowercase) - normalized_query = re.sub(r'\s+', ' ', query.lower().strip()) - - # Create composite string for the hash - hash_input = f"{normalized_query}|{config_id}" - if collection_name: - hash_input += f"|{collection_name}" - - # Generate the hash - return hashlib.md5(hash_input.encode()).hexdigest() - - def _get_cache_path(self, query_hash: str) -> Path: - """Gets the cache file path for a given hash.""" - # Use the first characters of the hash to create subdirectories - # This avoids having too many files in a single directory - subdir = query_hash[:2] - cache_subdir = self.cache_dir / subdir - cache_subdir.mkdir(exist_ok=True) - - return cache_subdir / f"{query_hash}.cache" - - def _update_metadata(self, query_hash: str, query: str, config_id: str): - """Updates the metadata in the database.""" - conn = sqlite3.connect(str(self.db_path)) - cursor = conn.cursor() - - now = datetime.now().isoformat() - - # Check if the hash already exists - cursor.execute("SELECT hit_count FROM cache_metadata WHERE query_hash = ?", (query_hash,)) - result = cursor.fetchone() - - if result: - # Update existing entry - hit_count = result[0] + 1 - cursor.execute(''' - UPDATE cache_metadata - SET last_accessed = ?, hit_count = ? - WHERE query_hash = ? - ''', (now, hit_count, query_hash)) - else: - # Insert new entry - cursor.execute(''' - INSERT INTO cache_metadata (query_hash, query, config_id, created_at, last_accessed, hit_count) - VALUES (?, ?, ?, ?, ?, 1) - ''', (query_hash, query, config_id, now, now)) - - conn.commit() - conn.close() - - def _update_memory_lru(self, query_hash: str): - """Updates the LRU (Least Recently Used) list for the in-memory cache.""" - if query_hash in self.memory_lru: - # Move to the end (most recently used) - self.memory_lru.remove(query_hash) - - self.memory_lru.append(query_hash) - - # If we exceed the limit, remove the least recently used element - if len(self.memory_lru) > self.memory_limit: - oldest_hash = self.memory_lru.pop(0) - if oldest_hash in self.memory_cache: - del self.memory_cache[oldest_hash] - del self.memory_timestamps[oldest_hash] - - def get(self, query: str, config_id: str, collection_name: Optional[str] = None) -> Optional[Dict[str, Any]]: - """ - Retrieves a cached result if it exists and has not expired. - - Args: - query: Natural language query - config_id: Database configuration ID - collection_name: Name of the collection/table (optional) - - Returns: - Cached result or None if it does not exist or has expired - """ - query_hash = self._get_hash(query, config_id, collection_name) - - # 1. Verificar caché en memoria (más rápido) - if query_hash in self.memory_cache: - timestamp = self.memory_timestamps[query_hash] - if (time.time() - timestamp) < self.ttl: - self._update_memory_lru(query_hash) - self._update_metadata(query_hash, query, config_id) - print_colored(f"Cache hit (memory): {query[:30]}...", "green") - return self.memory_cache[query_hash] - else: - # Expirado en memoria - del self.memory_cache[query_hash] - del self.memory_timestamps[query_hash] - if query_hash in self.memory_lru: - self.memory_lru.remove(query_hash) - - # 2. Verificar caché en disco - cache_path = self._get_cache_path(query_hash) - if cache_path.exists(): - # Verificar edad del archivo - file_age = time.time() - cache_path.stat().st_mtime - if file_age < self.ttl: - try: - with open(cache_path, 'rb') as f: - result = pickle.load(f) - - # Guardar también en caché de memoria - self.memory_cache[query_hash] = result - self.memory_timestamps[query_hash] = time.time() - self._update_memory_lru(query_hash) - self._update_metadata(query_hash, query, config_id) - - print_colored(f"Cache hit (disk): {query[:30]}...", "green") - return result - except Exception as e: - print_colored(f"Error loading cache: {str(e)}", "red") - # If there's an error loading, remove the corrupted file - cache_path.unlink(missing_ok=True) - else: - # Expired file, delete it - cache_path.unlink(missing_ok=True) - - return None - - def set(self, query: str, config_id: str, result: Dict[str, Any], collection_name: Optional[str] = None): - """ - Saves a result in the cache. - - Args: - query: Natural language query - config_id: Configuration ID - result: Result to cache - collection_name: Name of the collection/table (optional) - """ - query_hash = self._get_hash(query, config_id, collection_name) - - # 1. Save to memory cache - self.memory_cache[query_hash] = result - self.memory_timestamps[query_hash] = time.time() - self._update_memory_lru(query_hash) - - # 2. Save to persistent cache - try: - cache_path = self._get_cache_path(query_hash) - with open(cache_path, 'wb') as f: - pickle.dump(result, f) - - # 3. Update metadata - self._update_metadata(query_hash, query, config_id) - - print_colored(f"Cached: {query[:30]}...", "green") - except Exception as e: - print_colored(f"Error saving to cache: {str(e)}", "red") - - def clear(self, older_than: int = None): - """ - Clears the cache. - - Args: - older_than: Only clear entries older than this number of seconds - """ - # Clear memory cache - if older_than: - current_time = time.time() - keys_to_remove = [ - k for k, timestamp in self.memory_timestamps.items() - if (current_time - timestamp) > older_than - ] - - for k in keys_to_remove: - if k in self.memory_cache: - del self.memory_cache[k] - if k in self.memory_timestamps: - del self.memory_timestamps[k] - if k in self.memory_lru: - self.memory_lru.remove(k) - else: - self.memory_cache.clear() - self.memory_timestamps.clear() - self.memory_lru.clear() - - # Clear disk cache - if older_than: - cutoff_time = time.time() - older_than - - # Use the database to find old files - conn = sqlite3.connect(str(self.db_path)) - cursor = conn.cursor() - - # Convert cutoff_time to ISO format - cutoff_datetime = datetime.fromtimestamp(cutoff_time).isoformat() - - cursor.execute( - "SELECT query_hash FROM cache_metadata WHERE last_accessed < ?", - (cutoff_datetime,) - ) - - old_hashes = [row[0] for row in cursor.fetchall()] - - # Delete old files - for query_hash in old_hashes: - cache_path = self._get_cache_path(query_hash) - if cache_path.exists(): - cache_path.unlink() - - # Delete from database - cursor.execute( - "DELETE FROM cache_metadata WHERE query_hash = ?", - (query_hash,) - ) - - conn.commit() - conn.close() - else: - # Delete all cache files - for subdir in self.cache_dir.iterdir(): - if subdir.is_dir(): - for cache_file in subdir.glob("*.cache"): - cache_file.unlink() - - # Reset the database - conn = sqlite3.connect(str(self.db_path)) - cursor = conn.cursor() - cursor.execute("DELETE FROM cache_metadata") - conn.commit() - conn.close() - - def get_stats(self) -> Dict[str, Any]: - """Gets cache statistics.""" - # Count files on disk - disk_count = 0 - for subdir in self.cache_dir.iterdir(): - if subdir.is_dir(): - disk_count += len(list(subdir.glob("*.cache"))) - - # Get database statistics - conn = sqlite3.connect(str(self.db_path)) - cursor = conn.cursor() - - # Total entries - cursor.execute("SELECT COUNT(*) FROM cache_metadata") - total_entries = cursor.fetchone()[0] - - # Most frequent queries - cursor.execute( - "SELECT query, hit_count FROM cache_metadata ORDER BY hit_count DESC LIMIT 5" - ) - top_queries = cursor.fetchall() - - # Average age - cursor.execute( - "SELECT AVG(strftime('%s', 'now') - strftime('%s', created_at)) FROM cache_metadata" - ) - avg_age = cursor.fetchone()[0] - - conn.close() - - return { - "memory_cache_size": len(self.memory_cache), - "disk_cache_size": disk_count, - "total_entries": total_entries, - "top_queries": top_queries, - "average_age_seconds": avg_age, - "cache_directory": str(self.cache_dir) - } - -class QueryTemplate: - """Predefined query template for common patterns.""" - - def __init__(self, pattern: str, description: str, - sql_template: Optional[str] = None, - generator_func: Optional[Callable] = None, - db_type: str = "sql", - applicable_tables: Optional[List[str]] = None): - """ - Initializes a query template. - - Args: - pattern: Natural language pattern that matches this template - description: Description of the template - sql_template: SQL template with placeholders for parameters - generator_func: Alternative function to generate the query - db_type: Database type (sql, mongodb) - applicable_tables: List of tables to which this template applies - """ - self.pattern = pattern - self.description = description - self.sql_template = sql_template - self.generator_func = generator_func - self.db_type = db_type - self.applicable_tables = applicable_tables or [] - - # Compile regular expression for the pattern - self.regex = self._compile_pattern(pattern) - - def _compile_pattern(self, pattern: str) -> re.Pattern: - """Compiles the pattern into a regular expression.""" - # Replace special markers with capture groups - regex_pattern = pattern - - # {table} becomes capture group for table name - regex_pattern = regex_pattern.replace("{table}", r"(\w+)") - - # {field} becomes capture group for field name - regex_pattern = regex_pattern.replace("{field}", r"(\w+)") - - # {value} becomes capture group for a value - regex_pattern = regex_pattern.replace("{value}", r"([^,.\s]+)") - - # {number} becomes capture group for a number - regex_pattern = regex_pattern.replace("{number}", r"(\d+)") - - # Make the pattern match the entire string - regex_pattern = f"^{regex_pattern}$" - - return re.compile(regex_pattern, re.IGNORECASE) - - def matches(self, query: str) -> Tuple[bool, List[str]]: - """ - Checks if a query matches this template. - - Args: - query: Query to check - - Returns: - Tuple of (match, [captured parameters]) - """ - match = self.regex.match(query) - if match: - return True, list(match.groups()) - return False, [] - - def generate_query(self, params: List[str], db_schema: Dict[str, Any]) -> Optional[Dict[str, Any]]: - """ - Generates a query from the captured parameters. - - Args: - params: Captured parameters from the pattern - db_schema: Database schema - - Returns: - Generated query or None if it cannot be generated - """ - if self.generator_func: - # Use custom function - return self.generator_func(params, db_schema) - - if not self.sql_template: - return None - - # Try applying the SQL template with the parameters - try: - sql_query = self.sql_template - - # Replace parameters in the template - for i, param in enumerate(params): - placeholder = f"${i+1}" - sql_query = sql_query.replace(placeholder, param) - - # Check if there are any parameters left unreplaced - if "$" in sql_query: - return None - - return {"sql": sql_query} - except Exception: - return None - -class QueryAnalyzer: - """Analyzes query patterns to suggest optimizations.""" - - def __init__(self, query_log_path: str = None, template_path: str = None): - """ - Initializes the query analyzer. - - Args: - query_log_path: Path to the query log file - template_path: Path to the template file - """ - self.query_log_path = query_log_path or os.path.join( - Path.home(), ".corebrain_cache", "query_log.db" - ) - - self.template_path = template_path or os.path.join( - Path.home(), ".corebrain_cache", "templates.json" - ) - - # Inicializar base de datos - self._init_db() - - # Plantillas predefinidas para consultas comunes - self.templates = self._load_default_templates() - - # Cargar plantillas personalizadas - self._load_custom_templates() - - # Plantillas comunes para identificar patrones - self.common_patterns = [ - r"muestra\s+(?:todos\s+)?los\s+(\w+)", - r"lista\s+(?:de\s+)?(?:todos\s+)?los\s+(\w+)", - r"busca\s+(\w+)\s+donde", - r"cu[aá]ntos\s+(\w+)\s+hay", - r"total\s+de\s+(\w+)" - ] - - def _init_db(self): - """Initializes the database for query logging.""" - # Asegurar que el directorio existe - os.makedirs(os.path.dirname(self.query_log_path), exist_ok=True) - - conn = sqlite3.connect(self.query_log_path) - cursor = conn.cursor() - - # Crear tabla de registro si no existe - cursor.execute(''' - CREATE TABLE IF NOT EXISTS query_log ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - query TEXT, - config_id TEXT, - collection_name TEXT, - timestamp TIMESTAMP, - execution_time REAL, - cost REAL, - result_count INTEGER, - pattern TEXT - ) - ''') - - # Crear tabla de patrones detectados - cursor.execute(''' - CREATE TABLE IF NOT EXISTS query_patterns ( - pattern TEXT PRIMARY KEY, - count INTEGER, - avg_execution_time REAL, - avg_cost REAL, - last_updated TIMESTAMP - ) - ''') - - conn.commit() - conn.close() - - def _load_default_templates(self) -> List[QueryTemplate]: - """Carga las plantillas predefinidas para consultas comunes.""" - templates = [] - - # Listar todos los registros de una tabla - templates.append( - QueryTemplate( - pattern="muestra todos los {table}", - description="Listar todos los registros de una tabla", - sql_template="SELECT * FROM $1 LIMIT 100", - db_type="sql" - ) - ) - - # Contar registros - templates.append( - QueryTemplate( - pattern="cuántos {table} hay", - description="Contar registros en una tabla", - sql_template="SELECT COUNT(*) FROM $1", - db_type="sql" - ) - ) - - # Buscar por ID - templates.append( - QueryTemplate( - pattern="busca el {table} con id {value}", - description="Buscar registro por ID", - sql_template="SELECT * FROM $1 WHERE id = $2", - db_type="sql" - ) - ) - - # Listar ordenados - templates.append( - QueryTemplate( - pattern="lista los {table} ordenados por {field}", - description="Listar registros ordenados por campo", - sql_template="SELECT * FROM $1 ORDER BY $2 LIMIT 100", - db_type="sql" - ) - ) - - # Buscar por email - templates.append( - QueryTemplate( - pattern="busca el usuario con email {value}", - description="Buscar usuario por email", - sql_template="SELECT * FROM users WHERE email = '$2'", - db_type="sql" - ) - ) - - # Contar por campo - templates.append( - QueryTemplate( - pattern="cuántos {table} hay por {field}", - description="Contar registros agrupados por campo", - sql_template="SELECT $2, COUNT(*) FROM $1 GROUP BY $2", - db_type="sql" - ) - ) - - # Contar usuarios activos - templates.append( - QueryTemplate( - pattern="cuántos usuarios activos hay", - description="Contar usuarios activos", - sql_template="SELECT COUNT(*) FROM users WHERE is_active = TRUE", - db_type="sql", - applicable_tables=["users"] - ) - ) - - # Listar usuarios por fecha de registro - templates.append( - QueryTemplate( - pattern="usuarios registrados en los últimos {number} días", - description="Listar usuarios recientes", - sql_template=""" - SELECT * FROM users - WHERE created_at >= datetime('now', '-$2 days') - ORDER BY created_at DESC - LIMIT 100 - """, - db_type="sql", - applicable_tables=["users"] - ) - ) - - # Buscar empresas - templates.append( - QueryTemplate( - pattern="usuarios que tienen empresa", - description="Buscar usuarios con empresa asignada", - sql_template=""" - SELECT u.* FROM users u - INNER JOIN businesses b ON u.id = b.owner_id - WHERE u.is_business = TRUE - LIMIT 100 - """, - db_type="sql", - applicable_tables=["users", "businesses"] - ) - ) - - # Buscar negocios - templates.append( - QueryTemplate( - pattern="busca negocios en {value}", - description="Buscar negocios por ubicación", - sql_template=""" - SELECT * FROM businesses - WHERE address_city LIKE '%$2%' OR address_province LIKE '%$2%' - LIMIT 100 - """, - db_type="sql", - applicable_tables=["businesses"] - ) - ) - - # MongoDB: Listar documentos - templates.append( - QueryTemplate( - pattern="muestra todos los documentos de {table}", - description="Listar documentos en una colección", - db_type="mongodb", - generator_func=lambda params, schema: { - "collection": params[0], - "operation": "find", - "query": {}, - "limit": 100 - } - ) - ) - - return templates - - def _load_custom_templates(self): - """Loads custom templates from the file.""" - if not os.path.exists(self.template_path): - return - - try: - with open(self.template_path, 'r') as f: - custom_templates = json.load(f) - - for template_data in custom_templates: - # Crear plantilla desde datos JSON - template = QueryTemplate( - pattern=template_data.get("pattern", ""), - description=template_data.get("description", ""), - sql_template=template_data.get("sql_template"), - db_type=template_data.get("db_type", "sql"), - applicable_tables=template_data.get("applicable_tables", []) - ) - - self.templates.append(template) - - except Exception as e: - print_colored(f"Error al cargar plantillas personalizadas: {str(e)}", "red") - - def save_custom_template(self, template: QueryTemplate) -> bool: - """ - Saves a custom template. - - Args: - template: Template to save - - Returns: - True if saved successfully - """ - # Cargar plantillas existentes - custom_templates = [] - if os.path.exists(self.template_path): - try: - with open(self.template_path, 'r') as f: - custom_templates = json.load(f) - except: - custom_templates = [] - - # Convertir plantilla a diccionario - template_data = { - "pattern": template.pattern, - "description": template.description, - "sql_template": template.sql_template, - "db_type": template.db_type, - "applicable_tables": template.applicable_tables - } - - # Verificar si ya existe una plantilla con el mismo patrón - for i, existing in enumerate(custom_templates): - if existing.get("pattern") == template.pattern: - # Actualizar existente - custom_templates[i] = template_data - break - else: - # Agregar nueva - custom_templates.append(template_data) - - # Guardar plantillas - try: - with open(self.template_path, 'w') as f: - json.dump(custom_templates, f, indent=2) - - # Actualizar lista de plantillas - self.templates.append(template) - - return True - except Exception as e: - print_colored(f"Error al guardar plantilla personalizada: {str(e)}", "red") - return False - - def find_matching_template(self, query: str, db_schema: Dict[str, Any]) -> Optional[Tuple[QueryTemplate, List[str]]]: - """ - Searches for a template that matches the query. - - Args: - query: Natural language query - db_schema: Database schema - - Returns: - Tuple of (template, parameters) or None if no match is found - """ - for template in self.templates: - matches, params = template.matches(query) - if matches: - # Verificar si la plantilla es aplicable a las tablas existentes - if template.applicable_tables: - available_tables = set(db_schema.get("tables", {}).keys()) - if not any(table in available_tables for table in template.applicable_tables): - continue - - return template, params - - return None - - def log_query(self, query: str, config_id: str, collection_name: str = None, - execution_time: float = 0, cost: float = 0.09, result_count: int = 0): - """ - Registers a query for analysis. - - Args: - query: Natural language query - config_id: Configuration ID - collection_name: Name of the collection/table - execution_time: Execution time in seconds - cost: Estimated cost of the query - result_count: Number of results obtained - """ - # Detectar patrón - pattern = self._detect_pattern(query) - - # Registrar en la base de datos - conn = sqlite3.connect(self.query_log_path) - cursor = conn.cursor() - - cursor.execute(''' - INSERT INTO query_log (query, config_id, collection_name, timestamp, execution_time, cost, result_count, pattern) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ''', ( - query, config_id, collection_name, datetime.now().isoformat(), - execution_time, cost, result_count, pattern - )) - - # Actualizar estadísticas de patrones - if pattern: - cursor.execute( - "SELECT count, avg_execution_time, avg_cost FROM query_patterns WHERE pattern = ?", - (pattern,) - ) - result = cursor.fetchone() - - if result: - # Actualizar patrón existente - count, avg_exec_time, avg_cost = result - new_count = count + 1 - new_avg_exec_time = (avg_exec_time * count + execution_time) / new_count - new_avg_cost = (avg_cost * count + cost) / new_count - - cursor.execute(''' - UPDATE query_patterns - SET count = ?, avg_execution_time = ?, avg_cost = ?, last_updated = ? - WHERE pattern = ? - ''', (new_count, new_avg_exec_time, new_avg_cost, datetime.now().isoformat(), pattern)) - else: - # Insertar nuevo patrón - cursor.execute(''' - INSERT INTO query_patterns (pattern, count, avg_execution_time, avg_cost, last_updated) - VALUES (?, 1, ?, ?, ?) - ''', (pattern, execution_time, cost, datetime.now().isoformat())) - - conn.commit() - conn.close() - - def _detect_pattern(self, query: str) -> Optional[str]: - """ - Detects a pattern in the query. - - Args: - query: Query to analyze - - Returns: - Detected pattern or None - """ - normalized_query = query.lower() - - # Comprobar patrones predefinidos - for pattern in self.common_patterns: - match = re.search(pattern, normalized_query) - if match: - # Devolver el patrón con comodines - entity = match.group(1) - return pattern.replace(r'(\w+)', f"{entity}") - - # Si no se detecta ningún patrón predefinido, intentar generalizar - words = normalized_query.split() - if len(words) < 3: - return None - - # Intentar generalizar consultas simples - if "mostrar" in words or "muestra" in words or "listar" in words or "lista" in words: - for i, word in enumerate(words): - if word in ["de", "los", "las", "todos", "todas"]: - if i+1 < len(words): - return f"lista_de_{words[i+1]}" - - return None - - def get_common_patterns(self, limit: int = 5) -> List[Dict[str, Any]]: - """ - Retrieves the most common query patterns. - - Args: - limit: Maximum number of patterns to return - - Returns: - List of the most common patterns - """ - conn = sqlite3.connect(self.query_log_path) - cursor = conn.cursor() - - cursor.execute(''' - SELECT pattern, count, avg_execution_time, avg_cost - FROM query_patterns - ORDER BY count DESC - LIMIT ? - ''', (limit,)) - - patterns = [] - for row in cursor.fetchall(): - pattern, count, avg_time, avg_cost = row - patterns.append({ - "pattern": pattern, - "count": count, - "avg_execution_time": avg_time, - "avg_cost": avg_cost, - "estimated_monthly_cost": round(avg_cost * count * 30 / 7, 2) # Estimación mensual - }) - - conn.close() - return patterns - - def suggest_new_template(self, query: str, sql_query: str) -> Optional[QueryTemplate]: - """ - Suggests a new template based on a successful query. - - Args: - query: Natural language query - sql_query: Generated SQL query - - Returns: - Suggested template or None - """ - # Detectar patrón - pattern = self._detect_pattern(query) - if not pattern: - return None - - # Generalizar la consulta SQL - generalized_sql = sql_query - - # Reemplazar valores específicos con marcadores - # Esto es una simplificación, idealmente se usaría un parser SQL - tokens = query.lower().split() - - # Identificar posibles valores a parametrizar - for i, token in enumerate(tokens): - if token.isdigit(): - # Reemplazar números - generalized_sql = re.sub(r'\b' + re.escape(token) + r'\b', '$1', generalized_sql) - pattern = pattern.replace(token, "{number}") - elif '@' in token and '.' in token: - # Reemplazar emails - generalized_sql = re.sub(r'\b' + re.escape(token) + r'\b', '$1', generalized_sql) - pattern = pattern.replace(token, "{value}") - elif token.startswith('"') or token.startswith("'"): - # Reemplazar strings - value = token.strip('"\'') - if len(value) > 2: # Evitar reemplazar strings muy cortos - generalized_sql = re.sub(r'[\'"]' + re.escape(value) + r'[\'"]', "'$1'", generalized_sql) - pattern = pattern.replace(token, "{value}") - - # Crear plantilla - return QueryTemplate( - pattern=pattern, - description=f"Plantilla generada automáticamente para: {pattern}", - sql_template=generalized_sql, - db_type="sql" - ) - - def get_optimization_suggestions(self) -> List[Dict[str, Any]]: - """ - Generates suggestions to optimize queries. - - Returns: - List of optimization suggestions - """ - suggestions = [] - - # Calcular estadísticas generales - conn = sqlite3.connect(self.query_log_path) - cursor = conn.cursor() - - # Total de consultas y costo en los últimos 30 días - cursor.execute(''' - SELECT COUNT(*) as query_count, SUM(cost) as total_cost - FROM query_log - WHERE timestamp > datetime('now', '-30 day') - ''') - - row = cursor.fetchone() - if row: - query_count, total_cost = row - - if query_count and query_count > 100: - # Si hay muchas consultas en total, sugerir plan de volumen - suggestions.append({ - "type": "volume_plan", - "query_count": query_count, - "total_cost": round(total_cost, 2) if total_cost else 0, - "suggestion": f"Considerar negociar un plan por volumen. Actualmente ~{query_count} consultas/mes." - }) - - # Sugerir ajustar el TTL del caché según frecuencia - avg_queries_per_day = query_count / 30 - suggested_ttl = max(3600, min(86400 * 3, 86400 * (100 / avg_queries_per_day))) - - suggestions.append({ - "type": "cache_adjustment", - "current_rate": f"{avg_queries_per_day:.1f} consultas/día", - "suggestion": f"Ajustar TTL del caché a {suggested_ttl/3600:.1f} horas basado en su patrón de uso" - }) - - # Obtener patrones comunes - common_patterns = self.get_common_patterns(10) - - for pattern in common_patterns: - if pattern["count"] >= 5: - # Si un patrón se repite mucho, sugerir precompilación - suggestions.append({ - "type": "precompile", - "pattern": pattern["pattern"], - "count": pattern["count"], - "estimated_savings": round(pattern["avg_cost"] * pattern["count"] * 0.9, 2), # 90% de ahorro - "suggestion": f"Crear una plantilla SQL para consultas del tipo '{pattern['pattern']}'" - }) - - # Si un patrón es costoso pero poco frecuente - if pattern["avg_cost"] > 0.1 and pattern["count"] < 5: - suggestions.append({ - "type": "analyze", - "pattern": pattern["pattern"], - "avg_cost": pattern["avg_cost"], - "suggestion": f"Revisar manualmente consultas del tipo '{pattern['pattern']}' para optimizar" - }) - - # Buscar períodos con alta carga para ajustar parámetros - cursor.execute(''' - SELECT strftime('%Y-%m-%d %H', timestamp) as hour, COUNT(*) as count, SUM(cost) as total_cost - FROM query_log - WHERE timestamp > datetime('now', '-7 day') - GROUP BY hour - ORDER BY count DESC - LIMIT 5 - ''') - - for row in cursor.fetchall(): - hour, count, total_cost = row - if count > 20: # Si hay más de 20 consultas en una hora - suggestions.append({ - "type": "load_balancing", - "hour": hour, - "query_count": count, - "total_cost": round(total_cost, 2), - "suggestion": f"Alta carga de consultas detectada el {hour} ({count} consultas). Considerar técnicas de agrupación." - }) - - # Buscar consultas redundantes (misma consulta en corto tiempo) - cursor.execute(''' - SELECT query, COUNT(*) as count - FROM query_log - WHERE timestamp > datetime('now', '-1 day') - GROUP BY query - HAVING COUNT(*) > 3 - ORDER BY COUNT(*) DESC - LIMIT 5 - ''') - - for row in cursor.fetchall(): - query, count = row - suggestions.append({ - "type": "redundant", - "query": query, - "count": count, - "estimated_savings": round(0.09 * (count - 1), 2), # Ahorro por no repetir - "suggestion": f"Implementar caché para la consulta '{query[:50]}...' que se repitió {count} veces" - }) - - conn.close() - return suggestions - - - \ No newline at end of file diff --git a/corebrain/corebrain/core/test_utils.py b/corebrain/corebrain/core/test_utils.py deleted file mode 100644 index 8fead19..0000000 --- a/corebrain/corebrain/core/test_utils.py +++ /dev/null @@ -1,157 +0,0 @@ -""" -Utilities for testing and validating components. -""" -import json -import random -from typing import Dict, Any, Optional - -from corebrain.cli.utils import print_colored -from corebrain.cli.common import DEFAULT_API_URL -from corebrain.network.client import http_session - -def generate_test_question_from_schema(schema: Dict[str, Any]) -> str: - """ - Generates a test question based on the database schema. - - Args: - schema: Database schema - - Returns: - Generated test question - """ - if not schema or not schema.get("tables"): - return "What are the available tables?" - - tables = schema["tables"] - - if not tables: - return "What are the available tables?" - - # Select a random table - table = random.choice(tables) - table_name = table["name"] - - # Determine the type of question - question_types = [ - f"How many records are in the {table_name} table?", - f"Show the first 5 records from {table_name}", - f"What are the fields in the {table_name} table?", - ] - - # Get columns according to structure (SQL vs NoSQL) - columns = [] - if "columns" in table and table["columns"]: - columns = table["columns"] - elif "fields" in table and table["fields"]: - columns = table["fields"] - - if columns: - # If we have column/field information - column_name = columns[0]["name"] if columns else "id" - - # Add specific questions with columns - question_types.extend([ - f"What is the maximum value of {column_name} in {table_name}?", - f"What are the unique values of {column_name} in {table_name}?", - ]) - - return random.choice(question_types) - -def test_natural_language_query(api_token: str, db_config: Dict[str, Any], api_url: Optional[str] = None, user_data: Optional[Dict[str, Any]] = None) -> bool: - """ - Tests a natural language query. - - Args: - api_token: API token - db_config: Database configuration - api_url: Optional API URL - user_data: User data - - Returns: - True if the test is successful, False otherwise - """ - try: - print_colored("\nPerforming natural language query test...", "blue") - - # Dynamic import to avoid circular imports - from db.schema_file import extract_db_schema - - # Generate a test question based on the directly extracted schema - schema = extract_db_schema(db_config) - print("Retrieved schema: ", schema) - question = generate_test_question_from_schema(schema) - print(f"Test question: {question}") - - # Prepare the data for the request - api_url = api_url or DEFAULT_API_URL - if not api_url.startswith(("http://", "https://")): - api_url = "https://" + api_url - - if api_url.endswith('/'): - api_url = api_url[:-1] - - # Build endpoint for the query - endpoint = f"{api_url}/api/database/sdk/query" - - # Data for the query - request_data = { - "question": question, - "db_schema": schema, - "config_id": db_config["config_id"] - } - - # Make the API request - headers = { - "Authorization": f"Bearer {api_token}", - "Content-Type": "application/json" - } - - timeout = 15.0 # Reduced maximum wait time - - try: - print_colored("Sending query to API...", "blue") - response = http_session.post( - endpoint, - headers=headers, - json=request_data, - timeout=timeout - ) - - # Verify the response - if response.status_code == 200: - result = response.json() - - # Check if there's an explanation in the result - if "explanation" in result: - print_colored("\nResponse:", "green") - print(result["explanation"]) - - print_colored("\n✅ Query test successful!", "green") - return True - else: - # If there's no explanation but the API responds, it may be a different format - print_colored("\nResponse received from API (different format than expected):", "yellow") - print(json.dumps(result, indent=2)) - print_colored("\n⚠️ The API responded, but with a different format than expected.", "yellow") - return True - else: - print_colored(f"❌ Error in response: Code {response.status_code}", "red") - try: - error_data = response.json() - print(json.dumps(error_data, indent=2)) - except: - print(response.text[:500]) - return False - - except http_session.TimeoutException: - print_colored("⚠️ Timeout while performing query. The API may be busy or unavailable.", "yellow") - print_colored("This does not affect the saved configuration.", "yellow") - return False - except http_session.RequestError as e: - print_colored(f"⚠️ Connection error: {str(e)}", "yellow") - print_colored("Check the API URL and your internet connection.", "yellow") - return False - - except Exception as e: - print_colored(f"❌ Error performing query: {str(e)}", "red") - return False \ No newline at end of file diff --git a/corebrain/corebrain/db/__init__.py b/corebrain/corebrain/db/__init__.py deleted file mode 100644 index dd53f0e..0000000 --- a/corebrain/corebrain/db/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Database connectors for Corebrain SDK. - -This package provides connectors for different types and -database engines supported by Corebrain. -""" -from corebrain.db.connector import DatabaseConnector -from corebrain.db.factory import get_connector -from corebrain.db.engines import get_available_engines -from corebrain.db.connectors.sql import SQLConnector -from corebrain.db.connectors.nosql import NoSQLConnector -from corebrain.db.schema_file import get_schema_with_dynamic_import -from corebrain.db.schema.optimizer import SchemaOptimizer -from corebrain.db.schema.extractor import extract_db_schema - -# Exportación explícita de componentes públicos -__all__ = [ - 'DatabaseConnector', - 'get_connector', - 'get_available_engines', - 'SQLConnector', - 'NoSQLConnector', - 'SchemaOptimizer', - 'extract_db_schema', - 'get_schema_with_dynamic_import' -] \ No newline at end of file diff --git a/corebrain/corebrain/db/connector.py b/corebrain/corebrain/db/connector.py deleted file mode 100644 index 886a2a9..0000000 --- a/corebrain/corebrain/db/connector.py +++ /dev/null @@ -1,33 +0,0 @@ -""" -Base connectors for different types of databases. -""" -from typing import Dict, Any, List, Optional, Callable - -class DatabaseConnector: - """Base class for all database connectors.""" - - def __init__(self, config: Dict[str, Any], timeout: int = 10): - self.config = config - self.timeout = timeout - self.connection = None - - def connect(self): - """Establishes a connection to the database.""" - raise NotImplementedError - - def extract_schema(self, sample_limit: int = 5, table_limit: Optional[int] = None, - progress_callback: Optional[Callable] = None) -> Dict[str, Any]: - """Extracts the database schema.""" - raise NotImplementedError - - def execute_query(self, query: str) -> List[Dict[str, Any]]: - """Executes a query on the database.""" - raise NotImplementedError - - def close(self): - """Closes the connection.""" - if self.connection: - try: - self.connection.close() - except: - pass \ No newline at end of file diff --git a/corebrain/corebrain/db/connectors/__init__.py b/corebrain/corebrain/db/connectors/__init__.py deleted file mode 100644 index 65acf56..0000000 --- a/corebrain/corebrain/db/connectors/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Database connectors for different engines. -""" - -from typing import Dict, Any - -from corebrain.db.connectors.sql import SQLConnector -from corebrain.db.connectors.nosql import NoSQLConnector - -def get_connector(db_config: Dict[str, Any]): - """ - Gets the appropriate connector based on the database configuration. - - Args: - db_config: Database configuration - - Returns: - Instance of the appropriate connector - """ - db_type = db_config.get("type", "").lower() - engine = db_config.get("engine", "").lower() - - match db_type: - case "sql": - return SQLConnector(db_config, engine) - case "nosql": - return NoSQLConnector(db_config, engine) - case _: - raise ValueError(f"Unsupported database type: {db_type}") - - \ No newline at end of file diff --git a/corebrain/corebrain/db/connectors/mongodb.py b/corebrain/corebrain/db/connectors/mongodb.py deleted file mode 100644 index 0fbfdf9..0000000 --- a/corebrain/corebrain/db/connectors/mongodb.py +++ /dev/null @@ -1,474 +0,0 @@ -""" -Connector for MongoDB databases. -""" - -import time -import json -import re - -from typing import Dict, Any, List, Optional, Callable, Tuple - -try: - import pymongo - from pymongo.errors import ConnectionFailure, ServerSelectionTimeoutError - PYMONGO_AVAILABLE = True -except ImportError: - PYMONGO_AVAILABLE = False - -from corebrain.db.connector import DatabaseConnector - -class NoSQLConenctor(DatabaseConnector): - """Optimized connector for MongoDB.""" - - def __init__(self, config: Dict[str, Any]): - """ - Initializes the MongoDB connector with the provided configuration. - - Args: - config: Dictionary with the connection configuration - """ - super().__init__(config) - self.client = None - self.db = None - self.config = config - self.connection_timeout = 30 # seconds - - if not PYMONGO_AVAILABLE: - print("Warning: pymongo is not installed. Install it with 'pip install pymongo'") - - def connect(self) -> bool: - """ - Establishes a connection with optimized timeout. - - Returns: - True if the connection was successful, False otherwise - """ - if not PYMONGO_AVAILABLE: - raise ImportError("pymongo is not installed. Install it with 'pip install pymongo'") - - try: - start_time = time.time() - - # Build connection parameters - if "connection_string" in self.config: - connection_string = self.config["connection_string"] - # Add timeout to connection string if not present - if "connectTimeoutMS=" not in connection_string: - if "?" in connection_string: - connection_string += "&connectTimeoutMS=10000" # 10 seconds - else: - connection_string += "?connectTimeoutMS=10000" - - # Create MongoDB client with connection string - self.client = pymongo.MongoClient(connection_string) - else: - # Dictionary of parameters for MongoClient - mongo_params = { - "host": self.config.get("host", "localhost"), - "port": int(self.config.get("port", 27017)), - "connectTimeoutMS": 10000, # 10 seconds - "serverSelectionTimeoutMS": 10000 - } - - # Add credentials only if present - if self.config.get("user"): - mongo_params["username"] = self.config.get("user") - if self.config.get("password"): - mongo_params["password"] = self.config.get("password") - - # Optionally add authentication options - if self.config.get("auth_source"): - mongo_params["authSource"] = self.config.get("auth_source") - if self.config.get("auth_mechanism"): - mongo_params["authMechanism"] = self.config.get("auth_mechanism") - - # Create MongoDB client with parameters - self.client = pymongo.MongoClient(**mongo_params) - - # Verify that the connection works - self.client.admin.command('ping') - - # Select the database - db_name = self.config.get("database", "") - if not db_name: - # If no database is specified, list available ones - db_names = self.client.list_database_names() - if not db_names: - raise ValueError("No available databases found") - - # Select the first one that is not a system database - system_dbs = ["admin", "local", "config"] - for name in db_names: - if name not in system_dbs: - db_name = name - break - - # If we don't find any non-system database, use the first one - if not db_name: - db_name = db_names[0] - - print(f"No database specified. Using '{db_name}'") - - # Save the reference to the database - self.db = self.client[db_name] - return True - - except (ConnectionFailure, ServerSelectionTimeoutError) as e: - # If it's a timeout error, retry - if time.time() - start_time < self.connection_timeout: - print(f"Timeout connecting to MongoDB: {str(e)}. Retrying...") - time.sleep(2) # Wait before retrying - return self.connect() - else: - print(f"MongoDB connection error after {self.connection_timeout}s: {str(e)}") - self.close() - return False - except Exception as e: - print(f"Error connecting to MongoDB: {str(e)}") - self.close() - return False - - def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] = None, - progress_callback: Optional[Callable] = None) -> Dict[str, Any]: - """ - Extracts the schema with limits and progress to improve performance. - - Args: - sample_limit: Maximum number of sample documents per collection - collection_limit: Limit of collections to process (None for all) - progress_callback: Optional function to report progress - - Returns: - Dictionary with the database schema - """ - # Ensure we are connected - if not self.client and not self.connect(): - return {"type": "mongodb", "tables": {}, "tables_list": []} - - # Initialize the schema - schema = { - "type": "mongodb", - "database": self.db.name, - "tables": {} # In MongoDB, "tables" are collections - } - - try: - # Get the list of collections - collections = self.db.list_collection_names() - - # Limit collections if necessary - if collection_limit is not None and collection_limit > 0: - collections = collections[:collection_limit] - - # Process each collection - total_collections = len(collections) - for i, collection_name in enumerate(collections): - # Report progress if there is a callback - if progress_callback: - progress_callback(i, total_collections, f"Processing collection {collection_name}") - - collection = self.db[collection_name] - - try: - # Count documents - doc_count = collection.count_documents({}) - - if doc_count > 0: - # Get sample documents - sample_docs = list(collection.find().limit(sample_limit)) - - # Extract fields and their types - fields = {} - for doc in sample_docs: - self._extract_document_fields(doc, fields) - - # Convert to expected format - formatted_fields = [{"name": field, "type": type_name} for field, type_name in fields.items()] - - # Process documents for sample_data - sample_data = [] - for doc in sample_docs: - processed_doc = self._process_document_for_serialization(doc) - sample_data.append(processed_doc) - - # Save to outline - schema["tables"][collection_name] = { - "fields": formatted_fields, - "sample_data": sample_data, - "count": doc_count - } - else: - # Empty collection - schema["tables"][collection_name] = { - "fields": [], - "sample_data": [], - "count": 0, - "empty": True - } - - except Exception as e: - print(f"Error processing collection {collection_name}: {str(e)}") - schema["tables"][collection_name] = { - "fields": [], - "error": str(e) - } - - # Create the list of tables/collections for compatibility - table_list = [] - for collection_name, collection_info in schema["tables"].items(): - table_data = {"name": collection_name} - table_data.update(collection_info) - table_list.append(table_data) - - # Also save the list of tables for compatibility - schema["tables_list"] = table_list - - return schema - - except Exception as e: - print(f"Error extracting MongoDB schema: {str(e)}") - return {"type": "mongodb", "tables": {}, "tables_list": []} - - def _extract_document_fields(self, doc: Dict[str, Any], fields: Dict[str, str], - prefix: str = "", max_depth: int = 3, current_depth: int = 0) -> None: - """ - Recursively extracts fields and types from a MongoDB document. - - Args: - doc: Document to analyze - fields: Dictionary to store fields and types - prefix: Prefix for nested fields - max_depth: Maximum depth for nested fields - current_depth: Current depth - """ - if current_depth >= max_depth: - return - - for field, value in doc.items(): - # For _id and other special fields - if field == "_id": - field_type = "ObjectId" - elif isinstance(value, dict): - if current_depth < max_depth - 1: - # Recursion for nested fields - self._extract_document_fields(value, fields, - f"{prefix}{field}.", max_depth, current_depth + 1) - field_type = "object" - elif isinstance(value, list): - if value and current_depth < max_depth - 1: - # If we have elements in the list, analyze the first one - if isinstance(value[0], dict): - self._extract_document_fields(value[0], fields, - f"{prefix}{field}[].", max_depth, current_depth + 1) - else: - # For lists of primitive types - field_type = f"array<{type(value[0]).__name__}>" - else: - field_type = "array" - else: - field_type = type(value).__name__ - - # Save the current field type - field_key = f"{prefix}{field}" - if field_key not in fields: - fields[field_key] = field_type - - def _process_document_for_serialization(self, doc: Dict[str, Any]) -> Dict[str, Any]: - """ - Processes a document to be JSON serializable. - - Args: - doc: Document to process - - Returns: - Processed document - """ - processed_doc = {} - for field, value in doc.items(): - # Convert ObjectId to string - if field == "_id": - processed_doc[field] = str(value) - # Handling nested objects - elif isinstance(value, dict): - processed_doc[field] = self._process_document_for_serialization(value) - # Handling arrays - elif isinstance(value, list): - processed_items = [] - for item in value: - if isinstance(item, dict): - processed_items.append(self._process_document_for_serialization(item)) - elif hasattr(item, "__str__"): - processed_items.append(str(item)) - else: - processed_items.append(item) - processed_doc[field] = processed_items - # Convert dates to ISO - elif hasattr(value, 'isoformat'): - processed_doc[field] = value.isoformat() - # Other types of data - else: - processed_doc[field] = value - - return processed_doc - - def execute_query(self, query: str) -> List[Dict[str, Any]]: - """ - Executes a MongoDB query with improved error handling. - - Args: - query: MongoDB query in JSON format or query language - - Returns: - List of resulting documents - """ - if not self.client and not self.connect(): - raise ConnectionError("No se pudo establecer conexión con MongoDB") - - try: - # Determine whether the query is a JSON string or a query in another format - filter_dict, projection, collection_name, limit = self._parse_query(query) - - # Get the collection - if not collection_name: - raise ValueError("No se especificó el nombre de la colección en la consulta") - - collection = self.db[collection_name] - - # Run the query - if projection: - cursor = collection.find(filter_dict, projection).limit(limit or 100) - else: - cursor = collection.find(filter_dict).limit(limit or 100) - - # Convert results to serializable format - results = [] - for doc in cursor: - processed_doc = self._process_document_for_serialization(doc) - results.append(processed_doc) - - return results - - except Exception as e: - # Try to reconnect and try again once - try: - self.close() - if self.connect(): - print("Reconectando y reintentando consulta...") - - # Retry the query - filter_dict, projection, collection_name, limit = self._parse_query(query) - collection = self.db[collection_name] - - if projection: - cursor = collection.find(filter_dict, projection).limit(limit or 100) - else: - cursor = collection.find(filter_dict).limit(limit or 100) - - results = [] - for doc in cursor: - processed_doc = self._process_document_for_serialization(doc) - results.append(processed_doc) - - return results - except Exception as retry_error: - # If the retry fails, propagate the original error - raise Exception(f"Error al ejecutar consulta MongoDB: {str(e)}") - - # If we get here, there was an error in the retry. - raise Exception(f"Error al ejecutar consulta MongoDB (después de reconexión): {str(e)}") - - def _parse_query(self, query: str) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]], str, Optional[int]]: - """ - Analyzes a query and extracts the necessary components. - - Args: - query: Query in string format - - Returns: - Tuple with (filter, projection, collection name, limit) - """ - # Trying to parse as JSON - try: - query_dict = json.loads(query) - - # Extract components from the query - filter_dict = query_dict.get("filter", {}) - projection = query_dict.get("projection") - collection_name = query_dict.get("collection") - limit = query_dict.get("limit") - - return filter_dict, projection, collection_name, limit - - except json.JSONDecodeError: - # If not valid JSON, attempt to parse the alternative query format - collection_match = re.search(r'from\s+([a-zA-Z0-9_]+)', query, re.IGNORECASE) - collection_name = collection_match.group(1) if collection_match else None - - # Try to extract filters - filter_match = re.search(r'where\s+(.+?)(?:limit|$)', query, re.IGNORECASE | re.DOTALL) - filter_str = filter_match.group(1).strip() if filter_match else "{}" - - # Try to parse the filters as JSON - try: - filter_dict = json.loads(filter_str) - except json.JSONDecodeError: - # If parsing is not possible, use empty filter - filter_dict = {} - - # Extract limit if it exists - limit_match = re.search(r'limit\s+(\d+)', query, re.IGNORECASE) - limit = int(limit_match.group(1)) if limit_match else None - - return filter_dict, None, collection_name, limit - - def count_documents(self, collection_name: str, filter_dict: Optional[Dict[str, Any]] = None) -> int: - """ - Counts documents in a collection. - - Args: - collection_name: Name of the collection - filter_dict: Optional filter - - Returns: - Number of documents - """ - if not self.client and not self.connect(): - raise ConnectionError("No se pudo establecer conexión con MongoDB") - - try: - collection = self.db[collection_name] - return collection.count_documents(filter_dict or {}) - except Exception as e: - print(f"Error al contar documentos: {str(e)}") - return 0 - - def list_collections(self) -> List[str]: - """ - Returns a list of collections in the database. - - Returns: - List of collection names - """ - if not self.client and not self.connect(): - raise ConnectionError("No se pudo establecer conexión con MongoDB") - - try: - return self.db.list_collection_names() - except Exception as e: - print(f"Error al listar colecciones: {str(e)}") - return [] - - def close(self) -> None: - """Closes the MongoDB connection.""" - if self.client: - try: - self.client.close() - except: - pass - finally: - self.client = None - self.db = None - - def __del__(self): - """Destructor to ensure the connection is closed.""" - self.close() \ No newline at end of file diff --git a/corebrain/corebrain/db/connectors/nosql.py b/corebrain/corebrain/db/connectors/nosql.py deleted file mode 100644 index 0a153f4..0000000 --- a/corebrain/corebrain/db/connectors/nosql.py +++ /dev/null @@ -1,366 +0,0 @@ -''' -NoSQL Database Connector -This module provides a basic structure for connecting to a NoSQL database. -It includes methods for connecting, disconnecting, and executing queries. -''' - -import time -import json -import re - -from typing import Dict, Any, List, Optional, Callable, Tuple - -# Try'ies for imports DB's (for now only mongoDB) -try: - import pymongo - from pymongo.errors import ConnectionFailure, ServerSelectionTimeoutError - PYMONGO_IMPORTED = True -except ImportError: - PYMONGO_IMPORTED = False -# When adding new DB type write a try to it from user - -from corebrain.db.connector import DatabaseConnector -class NoSQLConnector(DatabaseConnector): - ''' - NoSQL Database Connector - This class provides a basic structure for connecting to a NoSQL database. - It includes methods for connecting, disconnecting, and executing queries. - ''' - def __init__(self, config: Dict[str, Any]): - ''' - Initialize the NoSQL database connector. - Args: - engine (str): Name of the database. - config (dict): Configuration dictionary containing connection parameters. - ''' - super().__init__(config) - self.engine = config.get("engine", "").lower() - self.client = None - self.db = None - self.config = config - self.connection_timeout = 30 # seconds - - match self.engine: - case "mongodb": - if not PYMONGO_IMPORTED: - raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") - case _: - pass - - - def connect(self) -> bool: - ''' - Connection with NoSQL DB's - Args: - self.engine (str): Name of the database. - ''' - - match self.engine: - case "mongodb": - if not PYMONGO_IMPORTED: - raise ImportError("Pymongo is not installed. Please install it to use MongoDB connector.") - try: - start_time = time.time() - # Check if connection string is provided - if "connection_string" in self.config: - connection_string = self.config["connection_string"] - if "connectTimeoutMS=" not in connection_string: - if "?" in connection_string: - connection_string += "&connectTimeoutMS=10000" - else: - connection_string += "?connectTimeoutMS=10000" - self.client = pymongo.MongoClient(connection_string) - else: - - # Setup for MongoDB connection parameters - - mongo_params = { - "host": self.config.get("host", "localhost"), - "port": int(self.config.get("port", 27017)), - "connection_timeoutMS": 10000, - "serverSelectionTimeoutMS": 10000, - } - - # Required parameters - - if self.config.get("user"): - mongo_params["username"] = self.config["user"] - if self.config.get("password"): - mongo_params["password"] = self.config["password"] - - #Optional parameters - - if self.config.get("authSource"): - mongo_params["authSource"] = self.config["authSource"] - if self.config.get("authMechanism"): - mongo_params["authMechanism"] = self.config["authMechanism"] - - # Insert parameters for MongoDB - self.client = pymongo.MongoClient(**mongo_params) - # Ping test for DB connection - self.client.admin.command('ping') - - db_name = self.config.get("database", "") - - if not db_name: - db_names = self.client.list_database_names() - if not db_names: - raise ValueError("No database names found in the MongoDB server.") - system_dbs = ["admin", "local", "config"] - for name in db_names: - if name not in system_dbs: - db_name = name - break - if not db_name: - db_name = db_names[0] - print(f"Not specified database name. Using the first available database: {db_name}") - self.db = self.client[db_name] - return True - except (ConnectionFailure, ServerSelectionTimeoutError) as e: - if time.time() - start_time > self.connection_timeout: - print(f"Connection to MongoDB timed out after {self.connection_timeout} seconds.") - time.sleep(2) - self.close() - return False - # Add case when is needed new DB type - case _ : - raise ValueError(f"Unsupported NoSQL database: {self.engine}") - pass - - def extract_schema(self, sample_limit: int = 5, collection_limit: Optional[int] = None, - progress_callback: Optional[Callable] = None) -> Dict[str, Any]: - ''' - Extract schema from the NoSQL database. - Args: - sample_limit (int): Number of samples to extract for schema inference. - collection_limit (int): Maximum number of collections to process. - progress_callback (Callable): Optional callback function for progress updates. - Returns: - Dict[str, Any]: Extracted schema information. - ''' - - match self.engine: - case "mongodb": - if not PYMONGO_IMPORTED: - raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") - if not self.client and not self.connect(): - return { - "type": "mongodb", - "tables": {}, - "tables_list": [] - } - schema = { - "type": "mongodb", - "database": self.db.name, - "tables": {}, # In MongoDB, tables are collections - } - try: - collections = self.db.list_collection_names() - if collection_limit is not None and collection_limit > 0: - collections = collections[:collection_limit] - total_collections = len(collections) - for i, collection_name in enumerate(collections): - if progress_callback: - progress_callback(i, total_collections, f"Processing collection: {collection_name}") - collection = self.db[collection_name] - - try: - doc_count = collection.count_documents({}) - if doc_count <= 0: - schema["tables"][collection_name] = { - "fields": [], - "sample_data": [], - "count": 0, - "empty": True - } - else: - sample_docs = list(collection.find().limit(sample_limit)) - fields = {} - sample_data = [] - - for doc in sample_docs: - self._extract_document_fields(doc, fields) - processed_doc = self._process_document_for_serialization(doc) - sample_data.append(processed_doc) - - formatted_fields = [{"name": field, "type": type_name} for field, type_name in fields.items()] - - schema["tables"][collection_name] = { - "fields": formatted_fields, - "sample_data": sample_data, - "count": doc_count, - } - except Exception as e: - print(f"Error processing collection {collection_name}: {e}") - schema["tables"][collection_name] = { - "fields": [], - "error": str(e) - } - # Convert the schema to a list of tables - table_list = [] - for collection_name, collection_info in schema["tables"].items(): - table_data = {"name": collection_name} - table_data.update(collection_info) - table_list.append(table_data) - schema["tables_list"] = table_list - return schema - except Exception as e: - print(f"Error extracting schema: {e}") - return { - "type": "mongodb", - "tables": {}, - "tabbles_list": [] - } - # Add case when is needed new DB type - case _ : - raise ValueError(f"Unsupported NoSQL database: {self.engine}") - def _extract_document_fields(self, doc: Dict[str, Any], fields: Dict[str, str], - prefix: str = "", max_depth: int = 3, current_depth: int = 0) -> None: - ''' - - Recursively extract fields from a document and determine their types. - Args: - doc (Dict[str, Any]): The document to extract fields from. - fields (Dict[str, str]): Dictionary to store field names and types. - prefix (str): Prefix for nested fields. - max_depth (int): Maximum depth for nested fields. - current_depth (int): Current depth in the recursion. - ''' - match self.engine: - case "mongodb": - if not PYMONGO_IMPORTED: - raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") - if current_depth >= max_depth: - return - for field, value in doc.items(): - if field == "_id": - field_type = "ObjectId" - elif isinstance(value, dict): - if value and current_depth < max_depth - 1: - self._extract_document_fields(value, fields, f"{prefix}{field}.", max_depth, current_depth + 1) - else: - field_type = f"array<{type(value[0]).__name__}>" - else: - field_type = "array" - else: - field_type = type(value).__name__ - - field_key = f"{prefix}{field}" - if field_key not in fields: - fields[field_key] = field_type - # Add case when is needed new DB type - case _ : - raise ValueError(f"Unsupported NoSQL database: {self.engine}") - - def _process_document_for_serialization(self, doc: Dict[str, Any]) -> Dict[str, Any]: - ''' - Proccesig a document for serialization of a JSON. - Args: - doc (Dict[str, Any]): The document to process. - Returns: - Procesed document - ''' - match self.engine: - case "mongodb": - if not PYMONGO_IMPORTED: - raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") - processed_doc = {} - for field, value in doc.items(): - if field == "_id": - processed_doc[field] = self._process_document_for_serialization(value) - elif isinstance(value, list): - processed_items = [] - for item in value: - if isinstance(item, dict): - processed_items.append(self._process_document_for_serialization(item)) - elif hasattr(item, "__str__"): - processed_items.append(str(item)) - else: - processed_items.append(item) - processed_doc[field] = processed_items - # Convert fetch to ISO - elif hasattr(value, 'isoformat'): - processed_doc[field] = value.isoformat() - # Convert data - else: - processed_doc[field] = value - return processed_doc - # Add case when is needed new DB type - case _ : - raise ValueError(f"Unsupported NoSQL database: {self.engine}") - - def execute_query(self, query: str) -> List[Dict[str, Any]]: - """ - Runs a NoSQL (or other) query with improved error handling - - Args: - query: A NoSQL (or other) query in JSON format or query language - - Returns: - List of resulting documents. - """ - - match self.engine: - case "nosql": - if not PYMONGO_IMPORTED: - raise ImportError("Pymongo is not installed. Please install it to use NoSQL connector.") - - if not self.client and not self.connect(): - raise ConnectionError("Couldn't estabilish a connection with NoSQL") - - try: - # Determine whether the query is a JSON string or a query in another format - filter_dict, projection, collection_name, limit = self._parse_query(query) - - # Get the collection - if not collection_name: - raise ValueError("Name of the colletion not specified in the query") - - collection = self.db[collection_name] - - # Execute the query - if projection: - cursor = collection.find(filter_dict, projection).limit(limit or 100) - else: - cursor = collection.find(filter_dict).limit(limit or 100) - - # Convert the results to a serializable format - results = [] - for doc in cursor: - processed_doc = self._process_document_for_serialization(doc) - results.append(processed_doc) - - return results - - except Exception as e: - # Reconnect and retry the query - try: - self.close() - if self.connect(): - print("Reconnecting and retrying the query...") - - # Retry the query - filter_dict, projection, collection_name, limit = self._parse_query(query) - collection = self.db[collection_name] - - if projection: - cursor = collection.find(filter_dict, projection).limit(limit or 100) - else: - cursor = collection.find(filter_dict).limit(limit or 100) - - results = [] - for doc in cursor: - processed_doc = self._process_document_for_serialization(doc) - results.append(processed_doc) - - return results - except Exception as retry_error: - # If retrying fails, show the original error - raise Exception(f"Failed to execute the NoSQL query: {str(e)}") - - # This code is will be executed if the retry fails - raise Exception(f"Failed to execute the NoSQL query (after the reconnection): {str(e)}") - - # Add case when is needed new DB type - case _ : - raise ValueError(f"Unsupported NoSQL database: {self.self.engine}") \ No newline at end of file diff --git a/corebrain/corebrain/db/connectors/sql.py b/corebrain/corebrain/db/connectors/sql.py deleted file mode 100644 index d9190b9..0000000 --- a/corebrain/corebrain/db/connectors/sql.py +++ /dev/null @@ -1,598 +0,0 @@ -""" -Conector para bases de datos SQL. -""" -import sqlite3 -import time -from typing import Dict, Any, List, Optional, Callable - -try: - import mysql.connector -except ImportError: - pass - -try: - import psycopg2 - import psycopg2.extras -except ImportError: - pass - -from corebrain.db.connector import DatabaseConnector - -class SQLConnector(DatabaseConnector): - """Optimized connector for SQL databases.""" - - def __init__(self, config: Dict[str, Any]): - """ - Initializes the SQL connector with the provided configuration. - - Args: - config: Dictionary with the connection configuration - """ - super().__init__(config) - self.conn = None - self.cursor = None - self.engine = config.get("engine", "").lower() - self.config = config - self.connection_timeout = 30 # seconds - - def connect(self) -> bool: - """ - Establishes a connection with optimized timeout. - - Returns: - True if the connection was successful, False otherwise - """ - try: - start_time = time.time() - - # Try the connection with a time limit - while time.time() - start_time < self.connection_timeout: - try: - if self.engine == "sqlite": - if "connection_string" in self.config: - self.conn = sqlite3.connect(self.config["connection_string"], timeout=10.0) - else: - self.conn = sqlite3.connect(self.config.get("database", ""), timeout=10.0) - - # Configure to return rows as dictionaries - self.conn.row_factory = sqlite3.Row - - elif self.engine == "mysql": - if "connection_string" in self.config: - self.conn = mysql.connector.connect( - connection_string=self.config["connection_string"], - connection_timeout=10 - ) - else: - self.conn = mysql.connector.connect( - host=self.config.get("host", "localhost"), - user=self.config.get("user", ""), - password=self.config.get("password", ""), - database=self.config.get("database", ""), - port=self.config.get("port", 3306), - connection_timeout=10 - ) - - elif self.engine == "postgresql": - # Determine whether to use connection string or parameters - if "connection_string" in self.config: - # Add timeout to connection string if not present - conn_str = self.config["connection_string"] - if "connect_timeout" not in conn_str: - if "?" in conn_str: - conn_str += "&connect_timeout=10" - else: - conn_str += "?connect_timeout=10" - - self.conn = psycopg2.connect(conn_str) - else: - self.conn = psycopg2.connect( - host=self.config.get("host", "localhost"), - user=self.config.get("user", ""), - password=self.config.get("password", ""), - dbname=self.config.get("database", ""), - port=self.config.get("port", 5432), - connect_timeout=10 - ) - - # If we get here, the connection was successful - if self.conn: - # Verify connection with a simple query - cursor = self.conn.cursor() - cursor.execute("SELECT 1") - cursor.close() - return True - - except (sqlite3.Error, mysql.connector.Error, psycopg2.Error) as e: - # If the error is not a timeout, propagate the exception - if "timeout" not in str(e).lower() and "tiempo de espera" not in str(e).lower(): - raise - - # If it's a timeout error, wait a bit and retry - time.sleep(1.0) - - # If we get here, the timeout was exceeded - raise TimeoutError(f"Could not connect to database in {self.connection_timeout} seconds") - - except Exception as e: - if self.conn: - try: - self.conn.close() - except: - pass - self.conn = None - - print(f"Error connecting to database: {str(e)}") - return False - - def extract_schema(self, sample_limit: int = 5, table_limit: Optional[int] = None, - progress_callback: Optional[Callable] = None) -> Dict[str, Any]: - """ - Extracts the schema with limits and progress. - - Args: - sample_limit: Data sample limit per table - table_limit: Limit of tables to process (None for all) - progress_callback: Optional function to report progress - - Returns: - Dictionary with the database schema - """ - # Ensure we are connected - if not self.conn and not self.connect(): - return {"type": "sql", "tables": {}, "tables_list": []} - - # Initialize schema - schema = { - "type": "sql", - "engine": self.engine, - "database": self.config.get("database", ""), - "tables": {} - } - - # Select the extractor function according to the engine - if self.engine == "sqlite": - return self._extract_sqlite_schema(sample_limit, table_limit, progress_callback) - elif self.engine == "mysql": - return self._extract_mysql_schema(sample_limit, table_limit, progress_callback) - elif self.engine == "postgresql": - return self._extract_postgresql_schema(sample_limit, table_limit, progress_callback) - else: - return schema # Empty schema if engine is not recognized - - def execute_query(self, query: str) -> List[Dict[str, Any]]: - """ - Executes an SQL query with improved error handling. - - Args: - query: SQL query to execute - - Returns: - List of resulting rows as dictionaries - """ - if not self.conn and not self.connect(): - raise ConnectionError("Could not establish connection to database") - - try: - # Execute query according to engine - if self.engine == "sqlite": - return self._execute_sqlite_query(query) - elif self.engine == "mysql": - return self._execute_mysql_query(query) - elif self.engine == "postgresql": - return self._execute_postgresql_query(query) - else: - raise ValueError(f"Unsupported database engine: {self.engine}") - - except Exception as e: - # Try to reconnect and retry once - try: - self.close() - if self.connect(): - print("Reconnecting and retrying query...") - - if self.engine == "sqlite": - return self._execute_sqlite_query(query) - elif self.engine == "mysql": - return self._execute_mysql_query(query) - elif self.engine == "postgresql": - return self._execute_postgresql_query(query) - - except Exception as retry_error: - # If retry fails, propagate the original error - raise Exception(f"Error executing query: {str(e)}") - - # If we get here without returning, there was an error in the retry - raise Exception(f"Error executing query (after reconnection): {str(e)}") - - def _execute_sqlite_query(self, query: str) -> List[Dict[str, Any]]: - """Executes a query in SQLite.""" - cursor = self.conn.cursor() - cursor.execute(query) - - # Convertir filas a diccionarios - columns = [desc[0] for desc in cursor.description] if cursor.description else [] - rows = cursor.fetchall() - result = [] - - for row in rows: - row_dict = {} - for i, column in enumerate(columns): - row_dict[column] = row[i] - result.append(row_dict) - - cursor.close() - return result - - def _execute_mysql_query(self, query: str) -> List[Dict[str, Any]]: - """Executes a query in MySQL.""" - cursor = self.conn.cursor(dictionary=True) - cursor.execute(query) - result = cursor.fetchall() - cursor.close() - return result - - def _execute_postgresql_query(self, query: str) -> List[Dict[str, Any]]: - """Executes a query in PostgreSQL.""" - cursor = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) - cursor.execute(query) - results = [dict(row) for row in cursor.fetchall()] - cursor.close() - return results - - def _extract_sqlite_schema(self, sample_limit: int, table_limit: Optional[int], progress_callback: Optional[Callable]) -> Dict[str, Any]: - """ - Extracts specific schema for SQLite. - - Args: - sample_limit: Maximum number of sample rows per table - table_limit: Maximum number of tables to extract - progress_callback: Function to report progress - - Returns: - Dictionary with the database schema - """ - schema = { - "type": "sql", - "engine": "sqlite", - "database": self.config.get("database", ""), - "tables": {} - } - - try: - cursor = self.conn.cursor() - - # Obtener la lista de tablas - cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;") - tables = [row[0] for row in cursor.fetchall()] - - # Limitar tablas si es necesario - if table_limit is not None and table_limit > 0: - tables = tables[:table_limit] - - # Procesar cada tabla - total_tables = len(tables) - for i, table_name in enumerate(tables): - # Reportar progreso si hay callback - if progress_callback: - progress_callback(i, total_tables, f"Procesando tabla {table_name}") - - # Extraer información de columnas - cursor.execute(f"PRAGMA table_info({table_name});") - columns = [{"name": col[1], "type": col[2]} for col in cursor.fetchall()] - - # Guardar información básica de la tabla - schema["tables"][table_name] = { - "columns": columns, - "sample_data": [] - } - - # Obtener muestra de datos - try: - cursor.execute(f"SELECT * FROM {table_name} LIMIT {sample_limit};") - - # Obtener nombres de columnas - col_names = [desc[0] for desc in cursor.description] - - # Procesar las filas - sample_data = [] - for row in cursor.fetchall(): - row_dict = {} - for j, value in enumerate(row): - # Convertir a string los valores que no son serializable directamente - if isinstance(value, (bytes, bytearray)): - row_dict[col_names[j]] = f"" - else: - row_dict[col_names[j]] = value - sample_data.append(row_dict) - - schema["tables"][table_name]["sample_data"] = sample_data - - except Exception as e: - print(f"Error al obtener muestra de datos para tabla {table_name}: {str(e)}") - - cursor.close() - - except Exception as e: - print(f"Error al extraer esquema SQLite: {str(e)}") - - # Crear la lista de tablas para compatibilidad - table_list = [] - for table_name, table_info in schema["tables"].items(): - table_data = {"name": table_name} - table_data.update(table_info) - table_list.append(table_data) - - schema["tables_list"] = table_list - return schema - - def _extract_mysql_schema(self, sample_limit: int, table_limit: Optional[int], progress_callback: Optional[Callable]) -> Dict[str, Any]: - """ - Extracts specific schema for MySQL. - - Args: - sample_limit: Maximum number of sample rows per table - table_limit: Maximum number of tables to extract - progress_callback: Function to report progress - - Returns: - Dictionary with the database schema - """ - schema = { - "type": "sql", - "engine": "mysql", - "database": self.config.get("database", ""), - "tables": {} - } - - try: - cursor = self.conn.cursor(dictionary=True) - - # Obtener la lista de tablas - cursor.execute("SHOW TABLES;") - tables_result = cursor.fetchall() - tables = [] - - # Extraer nombres de tablas (el formato puede variar según versión) - for row in tables_result: - if len(row) == 1: # Si es una lista simple - tables.extend(row.values()) - else: # Si tiene estructura compleja - for value in row.values(): - if isinstance(value, str): - tables.append(value) - break - - # Limitar tablas si es necesario - if table_limit is not None and table_limit > 0: - tables = tables[:table_limit] - - # Procesar cada tabla - total_tables = len(tables) - for i, table_name in enumerate(tables): - # Reportar progreso si hay callback - if progress_callback: - progress_callback(i, total_tables, f"Procesando tabla {table_name}") - - # Extraer información de columnas - cursor.execute(f"DESCRIBE `{table_name}`;") - columns = [{"name": col.get("Field"), "type": col.get("Type")} for col in cursor.fetchall()] - - # Guardar información básica de la tabla - schema["tables"][table_name] = { - "columns": columns, - "sample_data": [] - } - - # Obtener muestra de datos - try: - cursor.execute(f"SELECT * FROM `{table_name}` LIMIT {sample_limit};") - sample_data = cursor.fetchall() - - # Procesar valores que no son JSON serializable - processed_samples = [] - for row in sample_data: - processed_row = {} - for key, value in row.items(): - if isinstance(value, (bytes, bytearray)): - processed_row[key] = f"" - elif hasattr(value, 'isoformat'): # Para fechas y horas - processed_row[key] = value.isoformat() - else: - processed_row[key] = value - processed_samples.append(processed_row) - - schema["tables"][table_name]["sample_data"] = processed_samples - - except Exception as e: - print(f"Error al obtener muestra de datos para tabla {table_name}: {str(e)}") - - cursor.close() - - except Exception as e: - print(f"Error al extraer esquema MySQL: {str(e)}") - - # Crear la lista de tablas para compatibilidad - table_list = [] - for table_name, table_info in schema["tables"].items(): - table_data = {"name": table_name} - table_data.update(table_info) - table_list.append(table_data) - - schema["tables_list"] = table_list - return schema - - def _extract_postgresql_schema(self, sample_limit: int, table_limit: Optional[int], progress_callback: Optional[Callable]) -> Dict[str, Any]: - """ - Extracts specific schema for PostgreSQL with optimizations. - - Args: - sample_limit: Maximum number of sample rows per table - table_limit: Maximum number of tables to extract - progress_callback: Function to report progress - - Returns: - Dictionary with the database schema - """ - schema = { - "type": "sql", - "engine": "postgresql", - "database": self.config.get("database", ""), - "tables": {} - } - - try: - cursor = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) - - # Estrategia 1: Buscar en todos los esquemas accesibles - cursor.execute(""" - SELECT table_schema, table_name - FROM information_schema.tables - WHERE table_schema NOT IN ('pg_catalog', 'information_schema') - AND table_type = 'BASE TABLE' - ORDER BY table_schema, table_name; - """) - tables = cursor.fetchall() - - # Si no se encontraron tablas, intentar estrategia alternativa - if not tables: - cursor.execute(""" - SELECT schemaname AS table_schema, tablename AS table_name - FROM pg_tables - WHERE schemaname NOT IN ('pg_catalog', 'information_schema') - ORDER BY schemaname, tablename; - """) - tables = cursor.fetchall() - - # Si aún no hay tablas, intentar buscar en esquemas específicos - if not tables: - cursor.execute(""" - SELECT DISTINCT table_schema - FROM information_schema.tables - ORDER BY table_schema; - """) - schemas = cursor.fetchall() - - # Intentar con esquemas que no sean del sistema - user_schemas = [s[0] for s in schemas if s[0] not in ('pg_catalog', 'information_schema')] - for schema_name in user_schemas: - cursor.execute(f""" - SELECT '{schema_name}' AS table_schema, table_name - FROM information_schema.tables - WHERE table_schema = '{schema_name}' - AND table_type = 'BASE TABLE'; - """) - schema_tables = cursor.fetchall() - if schema_tables: - tables.extend(schema_tables) - - # Limitar tablas si es necesario - if table_limit is not None and table_limit > 0: - tables = tables[:table_limit] - - # Procesar cada tabla - total_tables = len(tables) - for i, (schema_name, table_name) in enumerate(tables): - # Reportar progreso si hay callback - if progress_callback: - progress_callback(i, total_tables, f"Procesando tabla {schema_name}.{table_name}") - - # Determinar el nombre completo de la tabla - full_name = f"{schema_name}.{table_name}" if schema_name != 'public' else table_name - - # Extraer información de columnas - cursor.execute(f""" - SELECT column_name, data_type - FROM information_schema.columns - WHERE table_schema = '{schema_name}' AND table_name = '{table_name}' - ORDER BY ordinal_position; - """) - - columns_data = cursor.fetchall() - if columns_data: - columns = [{"name": col[0], "type": col[1]} for col in columns_data] - schema["tables"][full_name] = {"columns": columns, "sample_data": []} - - # Obtener muestra de datos - try: - cursor.execute(f""" - SELECT * FROM "{schema_name}"."{table_name}" LIMIT {sample_limit}; - """) - rows = cursor.fetchall() - - # Obtener nombres de columnas - col_names = [desc[0] for desc in cursor.description] - - # Convertir filas a diccionarios - sample_data = [] - for row in rows: - row_dict = {} - for j, value in enumerate(row): - # Convertir a formato serializable - if hasattr(value, 'isoformat'): # Para fechas y horas - row_dict[col_names[j]] = value.isoformat() - elif isinstance(value, (bytes, bytearray)): - row_dict[col_names[j]] = f"" - else: - row_dict[col_names[j]] = str(value) if value is not None else None - sample_data.append(row_dict) - - schema["tables"][full_name]["sample_data"] = sample_data - - except Exception as e: - print(f"Error al obtener muestra de datos para tabla {full_name}: {str(e)}") - else: - # Registrar la tabla aunque no tenga columnas - schema["tables"][full_name] = {"columns": [], "sample_data": []} - - cursor.close() - - except Exception as e: - print(f"Error al extraer esquema PostgreSQL: {str(e)}") - - # Intento de recuperación para diagnosticar problemas - try: - if self.conn and self.conn.closed == 0: # 0 = conexión abierta - recovery_cursor = self.conn.cursor() - - # Verificar versión - recovery_cursor.execute("SELECT version();") - version = recovery_cursor.fetchone() - print(f"Versión PostgreSQL: {version[0] if version else 'Desconocida'}") - - # Verificar permisos - recovery_cursor.execute(""" - SELECT has_schema_privilege(current_user, 'public', 'USAGE') AS has_usage, - has_schema_privilege(current_user, 'public', 'CREATE') AS has_create; - """) - perms = recovery_cursor.fetchone() - if perms: - print(f"Permisos en esquema public: USAGE={perms[0]}, CREATE={perms[1]}") - - recovery_cursor.close() - except Exception as diag_err: - print(f"Error durante el diagnóstico: {str(diag_err)}") - - # Crear la lista de tablas para compatibilidad - table_list = [] - for table_name, table_info in schema["tables"].items(): - table_data = {"name": table_name} - table_data.update(table_info) - table_list.append(table_data) - - schema["tables_list"] = table_list - return schema - - def close(self) -> None: - """Closes the database connection.""" - if self.conn: - try: - self.conn.close() - except: - pass - finally: - self.conn = None - - def __del__(self): - """Destructor to ensure the connection is closed.""" - self.close() \ No newline at end of file diff --git a/corebrain/corebrain/db/engines.py b/corebrain/corebrain/db/engines.py deleted file mode 100644 index 51b51d6..0000000 --- a/corebrain/corebrain/db/engines.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -Information about supported database engines. -""" -from typing import Dict, List - -def get_available_engines() -> Dict[str, List[str]]: - """ - Returns the available database engines by type. - - Returns: - Dict with DB types and a list of engines per type - """ - return { - "sql": ["sqlite", "mysql", "postgresql"], - "nosql": ["mongodb"] - } \ No newline at end of file diff --git a/corebrain/corebrain/db/factory.py b/corebrain/corebrain/db/factory.py deleted file mode 100644 index 092f827..0000000 --- a/corebrain/corebrain/db/factory.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -Database connector factory. -""" -from typing import Dict, Any - -from corebrain.db.connector import DatabaseConnector -from corebrain.db.connectors.sql import SQLConnector -from corebrain.db.connectors.nosql import NoSQLConnector - -def get_connector(db_config: Dict[str, Any], timeout: int = 10) -> DatabaseConnector: - """ - Database connector factory based on configuration. - - Args: - db_config: Database configuration - timeout: Timeout for DB operations - - Returns: - Instance of the appropriate connector - """ - db_type = db_config.get("type", "").lower() - engine = db_config.get("engine", "").lower() - - if db_type == "sql": - return SQLConnector(db_config, timeout) - elif db_type == "nosql": - return NoSQLConnector(db_config, timeout) - else: - raise ValueError(f"Tipo de base de datos no soportado: {db_type}") \ No newline at end of file diff --git a/corebrain/corebrain/db/interface.py b/corebrain/corebrain/db/interface.py deleted file mode 100644 index 25fd433..0000000 --- a/corebrain/corebrain/db/interface.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Abstract interfaces for database connections. -""" -from typing import Dict, Any, List, Optional, Protocol -from abc import ABC, abstractmethod - -from corebrain.core.common import ConfigDict, SchemaDict - -class DatabaseConnector(ABC): - """Abstract interface for database connectors.""" - - @abstractmethod - def connect(self, config: ConfigDict) -> Any: - """Establishes a connection with the database.""" - pass - - @abstractmethod - def extract_schema(self, connection: Any) -> SchemaDict: - """Extracts the database schema.""" - pass - - @abstractmethod - def execute_query(self, connection: Any, query: str) -> List[Dict[str, Any]]: - """Executes a query and returns results.""" - pass - - @abstractmethod - def close(self, connection: Any) -> None: - """Closes the connection.""" - pass - -# Posteriormente se podrían implementar conectores específicos: -# - SQLiteConnector -# - MySQLConnector -# - PostgresConnector -# - NoSQLConnector \ No newline at end of file diff --git a/corebrain/corebrain/db/schema/__init__.py b/corebrain/corebrain/db/schema/__init__.py deleted file mode 100644 index 388ee63..0000000 --- a/corebrain/corebrain/db/schema/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -Components for extracting and optimizing database schemas. -""" -from .extractor import extract_schema -from .optimizer import SchemaOptimizer - -# Alias para compatibilidad con código existente -extract_db_schema = extract_schema -schemaOptimizer = SchemaOptimizer - -__all__ = ['extract_schema', 'extract_db_schema', 'schemaOptimizer'] \ No newline at end of file diff --git a/corebrain/corebrain/db/schema/extractor.py b/corebrain/corebrain/db/schema/extractor.py deleted file mode 100644 index 4aeabcf..0000000 --- a/corebrain/corebrain/db/schema/extractor.py +++ /dev/null @@ -1,123 +0,0 @@ -# db/schema/extractor.py (replaces the circular import in db/schema.py) - -""" -Independent database schema extractor. -""" - -from typing import Dict, Any, Optional, Callable - -from corebrain.utils.logging import get_logger - -logger = get_logger(__name__) - -def extract_db_schema(db_config: Dict[str, Any], client_factory: Optional[Callable] = None) -> Dict[str, Any]: - """ - Extracts the database schema with dependency injection. - - Args: - db_config: Database configuration - client_factory: Optional function to create a client (avoids circular imports) - - Returns: - Dictionary with the database structure - """ - db_type = db_config.get("type", "").lower() - schema = { - "type": db_type, - "database": db_config.get("database", ""), - "tables": {}, - "tables_list": [] - } - - try: - # If we have a specialized client, use it - if client_factory: - # The factory creates a client and extracts the schema - client = client_factory(db_config) - return client.extract_schema() - - # Direct extraction without using Corebrain client - if db_type == "sql": - # Code for SQL databases (without circular dependencies) - engine = db_config.get("engine", "").lower() - if engine == "sqlite": - # Extract SQLite schema - import sqlite3 - # (implementation...) - elif engine == "mysql": - # Extract MySQL schema - import mysql.connector - # (implementation...) - elif engine == "postgresql": - # Extract PostgreSQL schema - import psycopg2 - # (implementation...) - - elif db_type in ["nosql", "mongodb"]: - # Extract MongoDB schema - import pymongo - # (implementation...) - - # Convert dictionary to list for compatibility - table_list = [] - for table_name, table_info in schema["tables"].items(): - table_data = {"name": table_name} - table_data.update(table_info) - table_list.append(table_data) - - schema["tables_list"] = table_list - return schema - - except Exception as e: - logger.error(f"Error extracting schema: {str(e)}") - return {"type": db_type, "tables": {}, "tables_list": []} - - -def create_schema_from_corebrain() -> Callable: - """ - Creates an extraction function that uses Corebrain internally. - Loads dynamically to avoid circular imports. - - Returns: - Function that extracts schema using Corebrain - """ - def extract_with_corebrain(db_config: Dict[str, Any]) -> Dict[str, Any]: - # Import dynamically to avoid circular dependencies - from corebrain.core.client import Corebrain - - # Create temporary client only to extract the schema - try: - client = Corebrain( - api_token="temp_token", - db_config=db_config, - skip_verification=True - ) - schema = client.db_schema - client.close() - return schema - except Exception as e: - logger.error(f"Error extracting schema with Corebrain: {str(e)}") - return {"type": db_config.get("type", ""), "tables": {}, "tables_list": []} - - return extract_with_corebrain - - -# Public function exposed -def extract_schema(db_config: Dict[str, Any], use_corebrain: bool = False) -> Dict[str, Any]: - """ - Public function that decides how to extract the schema. - - Args: - db_config: Database configuration - use_corebrain: If True, uses the Corebrain class for extraction - - Returns: - Database schema - """ - if use_corebrain: - # Try to use Corebrain if requested - factory = create_schema_from_corebrain() - return extract_db_schema(db_config, client_factory=factory) - else: - # Use direct extraction without circular dependencies - return extract_db_schema(db_config) \ No newline at end of file diff --git a/corebrain/corebrain/db/schema/optimizer.py b/corebrain/corebrain/db/schema/optimizer.py deleted file mode 100644 index c7840b0..0000000 --- a/corebrain/corebrain/db/schema/optimizer.py +++ /dev/null @@ -1,157 +0,0 @@ -""" -Components for database schema optimization. -""" -import re -from typing import Dict, Any, Optional - -from corebrain.utils.logging import get_logger - -logger = get_logger(__name__) - -class SchemaOptimizer: - """Optimizes the database schema to reduce context size.""" - - def __init__(self, max_tables: int = 10, max_columns_per_table: int = 15, max_samples: int = 2): - """ - Initializes the schema optimizer. - - Args: - max_tables: Maximum number of tables to include - max_columns_per_table: Maximum number of columns per table - max_samples: Maximum number of sample rows per table - """ - self.max_tables = max_tables - self.max_columns_per_table = max_columns_per_table - self.max_samples = max_samples - - # Tablas importantes que siempre deben incluirse si existen - self.priority_tables = set([ - "users", "customers", "products", "orders", "transactions", - "invoices", "accounts", "clients", "employees", "services" - ]) - - # Tablas típicamente menos importantes - self.low_priority_tables = set([ - "logs", "sessions", "tokens", "temp", "cache", "metrics", - "statistics", "audit", "history", "archives", "settings" - ]) - - def optimize_schema(self, db_schema: Dict[str, Any], query: str = None) -> Dict[str, Any]: - """ - Optimizes the schema to reduce its size. - - Args: - db_schema: Original database schema - query: User query (to prioritize relevant tables) - - Returns: - Optimized schema - """ - # Crear copia para no modificar el original - optimized_schema = { - "type": db_schema.get("type", ""), - "database": db_schema.get("database", ""), - "engine": db_schema.get("engine", ""), - "tables": {}, - "tables_list": [] - } - - # Determinar tablas relevantes para la consulta - query_relevant_tables = set() - if query: - # Extraer potenciales nombres de tablas de la consulta - normalized_query = query.lower() - - # Obtener nombres de todas las tablas - all_table_names = [ - name.lower() for name in db_schema.get("tables", {}).keys() - ] - - # Buscar menciones a tablas en la consulta - for table_name in all_table_names: - # Buscar el nombre exacto (como palabra completa) - if re.search(r'\b' + re.escape(table_name) + r'\b', normalized_query): - query_relevant_tables.add(table_name) - - # También buscar formas singulares/plurales simples - if table_name.endswith('s') and re.search(r'\b' + re.escape(table_name[:-1]) + r'\b', normalized_query): - query_relevant_tables.add(table_name) - elif not table_name.endswith('s') and re.search(r'\b' + re.escape(table_name + 's') + r'\b', normalized_query): - query_relevant_tables.add(table_name) - - # Priorizar tablas a incluir - table_scores = {} - for table_name in db_schema.get("tables", {}): - score = 0 - - # Tablas mencionadas en la consulta tienen máxima prioridad - if table_name.lower() in query_relevant_tables: - score += 100 - - # Tablas importantes - if table_name.lower() in self.priority_tables: - score += 50 - - # Tablas poco importantes - if table_name.lower() in self.low_priority_tables: - score -= 30 - - # Tablas con más columnas pueden ser más relevantes - table_info = db_schema["tables"].get(table_name, {}) - column_count = len(table_info.get("columns", [])) - score += min(column_count, 20) # Limitar a 20 puntos máximo - - # Guardar puntuación - table_scores[table_name] = score - - # Ordenar tablas por puntuación - sorted_tables = sorted(table_scores.items(), key=lambda x: x[1], reverse=True) - - # Limitar número de tablas - selected_tables = [name for name, _ in sorted_tables[:self.max_tables]] - - # Copiar tablas seleccionadas con optimizaciones - for table_name in selected_tables: - table_info = db_schema["tables"].get(table_name, {}) - - # Optimizar columnas - columns = table_info.get("columns", []) - if len(columns) > self.max_columns_per_table: - # Mantener las columnas más importantes (id, nombre, clave primaria, etc) - important_columns = [] - other_columns = [] - - for col in columns: - col_name = col.get("name", "").lower() - if col_name in ["id", "uuid", "name", "key", "code"] or "id" in col_name: - important_columns.append(col) - else: - other_columns.append(col) - - # Tomar las columnas importantes y completar con otras hasta el límite - optimized_columns = important_columns - remaining_slots = self.max_columns_per_table - len(optimized_columns) - if remaining_slots > 0: - optimized_columns.extend(other_columns[:remaining_slots]) - else: - optimized_columns = columns - - # Optimizar datos de muestra - sample_data = table_info.get("sample_data", []) - optimized_samples = sample_data[:self.max_samples] if sample_data else [] - - # Guardar tabla optimizada - optimized_schema["tables"][table_name] = { - "columns": optimized_columns, - "sample_data": optimized_samples - } - - # Añadir a la lista de tablas - optimized_schema["tables_list"].append({ - "name": table_name, - "columns": optimized_columns, - "sample_data": optimized_samples - }) - - return optimized_schema - diff --git a/corebrain/corebrain/db/schema_file.py b/corebrain/corebrain/db/schema_file.py deleted file mode 100644 index 0f75ebc..0000000 --- a/corebrain/corebrain/db/schema_file.py +++ /dev/null @@ -1,566 +0,0 @@ -""" -Components for extracting and optimizing database schemas. -""" -import json - -from typing import Dict, Any, Optional - -def _print_colored(message: str, color: str) -> None: - """Simplified version of _print_colored that doesn't depend on cli.utils.""" - colors = { - "red": "\033[91m", - "green": "\033[92m", - "yellow": "\033[93m", - "blue": "\033[94m", - "default": "\033[0m" - } - color_code = colors.get(color, colors["default"]) - print(f"{color_code}{message}{colors['default']}") - -def extract_db_schema(db_config: Dict[str, Any]) -> Dict[str, Any]: - """ - Extracts the database schema directly without using the SDK. - - Args: - db_config: Database configuration - - Returns: - Dictionary with the database structure organized by tables/collections - """ - db_type = db_config["type"].lower() - schema = { - "type": db_type, - "database": db_config.get("database", ""), - "tables": {} # Changed to dictionary to facilitate direct access to tables by name - } - - try: - if db_type == "sql": - # Code for SQL databases... - # [Keeps the same] - pass - - # Handle both "nosql" and valid types - elif db_type == "nosql": - match db_config.get("engine", "").lower(): - case "mongodb": - try: - import pymongo - PYMONGO_IMPORTED = True - except ImportError: - PYMONGO_IMPORTED = False - if not PYMONGO_IMPORTED: - raise ImportError("pymongo is not installed. Please install it to use MongoDB connector.") - # Defining the engine - engine = db_config.get("engine", "").lower() - - if engine == "mongodb": - if "connection_string" in db_config: - client = pymongo.MongoClient(db_config["connection_string"]) - else: - # Dictionary of parameters for MongoClient - mongo_params = { - "host": db_config.get("host", "localhost"), - "port": db_config.get("port", 27017) - } - - # Add credentials only if they are present - if db_config.get("user"): - mongo_params["username"] = db_config["user"] - if db_config.get("password"): - mongo_params["password"] = db_config["password"] - - client = pymongo.MongoClient(**mongo_params) - - db_name = db_config.get("database","") - - if not db_name: - _print_colored("⚠️ Database is not specified", "yellow") - return schema - - try: - db = client[db_name] - collection_names = db.list_collection_names() - - # Process collection - for collection_name in collection_names: - collection = db[collection_name] - - try: - sample_docs = list(collection.find().limit(5)) - - field_types = {} - # Process sample documents to infer field types - for doc in sample_docs: - for field, value in doc.items(): - if field not in field_types: - field_types[field] = type(value).__name__ - - # Add collection to schema - schema["tables"][collection_name] = { - "fields": [{"name": field, "type": field_type} for field, field_type in field_types.items()], - "sample_data": sample_docs - } - - except Exception as e: - _print_colored(f"Error processing collection {collection_name}: {str(e)}", "red") - - except Exception as e: - _print_colored(f"Error accessing database: {str(e)}", "red") - - except Exception as e: - _print_colored(f"Error extracting schema: {str(e)}", "red") - - return schema - -def extract_db_schema_direct(db_config: Dict[str, Any]) -> Dict[str, Any]: - """ - Extracts the schema directly without using the Corebrain client. - This is a reduced version that doesn't require importing core. - """ - db_type = db_config["type"].lower() - schema = { - "type": db_type, - "database": db_config.get("database", ""), - "tables": {}, - "tables_list": [] # Initially empty list - } - - try: - # [Existing implementation to extract schema without using Corebrain] - # ... - - return schema - except Exception as e: - _print_colored(f"Error extracting schema directly: {str(e)}", "red") - return {"type": db_type, "tables": {}, "tables_list": []} - -def extract_schema_with_lazy_init(api_key: str, db_config: Dict[str, Any], api_url: Optional[str] = None) -> Dict[str, Any]: - """ - Extracts the schema using late import of the client. - - This function avoids the circular import issue by dynamically loading - the Corebrain client only when necessary. - """ - try: - # The import is moved here to avoid the circular import problem - # It is only executed when we really need to create the client - import importlib - core_module = importlib.import_module('core') - init_func = getattr(core_module, 'init') - - # Create client with the configuration - api_url_to_use = api_url or "https://api.corebrain.com" - cb = init_func( - api_token=api_key, - db_config=db_config, - api_url=api_url_to_use, - skip_verification=True # We don't need to verify token to extract schema - ) - - # Get the schema and close the client - schema = cb.db_schema - cb.close() - - return schema - - except Exception as e: - _print_colored(f"Error extracting schema with client: {str(e)}", "red") - # As an alternative, use direct extraction without client - return extract_db_schema_direct(db_config) - -from typing import Dict, Any - -def test_connection(db_config: Dict[str, Any]) -> bool: - try: - if db_config["type"].lower() == "sql": - # Code to test SQL connection... - pass - elif db_config["type"].lower() in ["nosql", "mongodb"]: - import pymongo - - # Create MongoDB client - client = pymongo.MongoClient(db_config["connection_string"]) - client.admin.command('ping') # Test connection - - return True - else: - _print_colored("Unsupported database type.", "red") - return False - except Exception as e: - _print_colored(f"Failed to connect to the database: {str(e)}", "red") - return False - -def extract_schema_to_file(api_key: str, config_id: Optional[str] = None, output_file: Optional[str] = None, api_url: Optional[str] = None) -> bool: - """ - Extracts the database schema and saves it to a file. - - Args: - api_key: API Key to identify the configuration - config_id: Specific configuration ID (optional) - output_file: Path to the file where the schema will be saved - api_url: Optional API URL - - Returns: - True if extraction is successful, False otherwise - """ - try: - # Explicit import with try-except to handle errors - try: - from corebrain.config.manager import ConfigManager - except ImportError as e: - _print_colored(f"Error importing ConfigManager: {e}", "red") - return False - - # Get the available configurations - config_manager = ConfigManager() - configs = config_manager.list_configs(api_key) - - if not configs: - _print_colored("No configurations saved for this API Key.", "yellow") - return False - - selected_config_id = config_id - - # If no config_id is specified, show list to select - if not selected_config_id: - _print_colored("\n=== Available configurations ===", "blue") - for i, conf_id in enumerate(configs, 1): - print(f"{i}. {conf_id}") - - try: - choice = int(input(f"\nSelect a configuration (1-{len(configs)}): ").strip()) - if 1 <= choice <= len(configs): - selected_config_id = configs[choice - 1] - else: - _print_colored("Invalid option.", "red") - return False - except ValueError: - _print_colored("Please enter a valid number.", "red") - return False - - # Verify that the config_id exists - if selected_config_id not in configs: - _print_colored(f"Configuration with ID not found: {selected_config_id}", "red") - return False - - # Get the selected configuration - db_config = config_manager.get_config(api_key, selected_config_id) - - if not db_config: - _print_colored(f"Error getting configuration with ID: {selected_config_id}", "red") - return False - - _print_colored(f"\nExtracting schema for configuration: {selected_config_id}", "blue") - print(f"Type: {db_config['type'].upper()}, Engine: {db_config.get('engine', 'Not specified').upper()}") - print(f"Database: {db_config.get('database', 'Not specified')}") - - # Extract the database schema - _print_colored("\nExtracting database schema...", "blue") - schema = extract_schema_with_lazy_init(api_key, db_config, api_url) - - # Verify if a valid schema was obtained - if not schema or not schema.get("tables"): - _print_colored("No tables/collections found in the database.", "yellow") - return False - - # Save the schema to a file - output_path = output_file or "db_schema.json" - try: - with open(output_path, 'w', encoding='utf-8') as f: - json.dump(schema, f, indent=2, default=str) - _print_colored(f"✅ Schema extracted and saved in: {output_path}", "green") - except Exception as e: - _print_colored(f"❌ Error saving the file: {str(e)}", "red") - return False - - # Show a summary of the tables/collections found - tables = schema.get("tables", {}) - _print_colored(f"\nExtracted schema summary: {len(tables)} tables/collections", "green") - - for table_name in tables: - print(f"- {table_name}") - - return True - - except Exception as e: - _print_colored(f"❌ Error extracting schema: {str(e)}", "red") - return False - -def show_db_schema(api_token: str, config_id: Optional[str] = None, api_url: Optional[str] = None) -> None: - """ - Displays the schema of the configured database. - - Args: - api_token: API token - config_id: Specific configuration ID (optional) - api_url: Optional API URL - """ - try: - # Explicit import with try-except to handle errors - try: - from corebrain.config.manager import ConfigManager - except ImportError as e: - _print_colored(f"Error importing ConfigManager: {e}", "red") - return False - - # Get the available configurations - config_manager = ConfigManager() - configs = config_manager.list_configs(api_token) - - if not configs: - _print_colored("No configurations saved for this token.", "yellow") - return - - selected_config_id = config_id - - # If no config_id is specified, show list to select - if not selected_config_id: - _print_colored("\n=== Available configurations ===", "blue") - for i, conf_id in enumerate(configs, 1): - print(f"{i}. {conf_id}") - - try: - choice = int(input(f"\nSelect a configuration (1-{len(configs)}): ").strip()) - if 1 <= choice <= len(configs): - selected_config_id = configs[choice - 1] - else: - _print_colored("Invalid option.", "red") - return - except ValueError: - _print_colored("Please enter a valid number.", "red") - return - - # Verify that the config_id exists - if selected_config_id not in configs: - _print_colored(f"Configuration with ID not found: {selected_config_id}", "red") - return - - if config_id and config_id in configs: - db_config = config_manager.get_config(api_token, config_id) - else: - # Get the selected configuration - db_config = config_manager.get_config(api_token, selected_config_id) - - if not db_config: - _print_colored(f"Error getting configuration with ID: {selected_config_id}", "red") - return - - _print_colored(f"\nGetting schema for configuration: {selected_config_id}", "blue") - _print_colored("Database type:", "blue") - print(f" {db_config['type'].upper()}") - - if db_config.get('engine'): - _print_colored("Engine:", "blue") - print(f" {db_config['engine'].upper()}") - - _print_colored("Database:", "blue") - print(f" {db_config.get('database', 'Not specified')}") - - # Extract and show the schema - _print_colored("\nExtracting database schema...", "blue") - - # Try to connect to the database and extract the schema - try: - - # Create a Corebrain instance with the selected configuration - """ - cb = init( - api_token=api_token, - config_id=selected_config_id, - api_url=api_url, - skip_verification=True # Skip verification for simplicity - ) - """ - - import importlib - core_module = importlib.import_module('core.client') - init_func = getattr(core_module, 'init') - - # Create a Corebrain instance with the selected configuration - cb = init_func( - api_token=api_token, - config_id=config_id, - api_url=api_url, - skip_verification=True # Skip verification for simplicity - ) - - # The schema is automatically extracted when initializing - schema = get_schema_with_dynamic_import( - api_token=api_token, - config_id=selected_config_id, - db_config=db_config, - api_url=api_url - ) - - # If there's no schema, we try to extract it explicitly - if not schema or not schema.get("tables"): - _print_colored("Trying to extract schema explicitly...", "yellow") - schema = cb._extract_db_schema() - - # Close the connection - cb.close() - - except Exception as conn_error: - _print_colored(f"Connection error: {str(conn_error)}", "red") - print("Trying alternative method...") - - # Alternative method: use extract_db_schema function directly - schema = extract_db_schema(db_config) - - # Verify if a valid schema was obtained - if not schema or not schema.get("tables"): - _print_colored("No tables/collections found in the database.", "yellow") - - # Additional information to help diagnose the problem - print("\nDebug information:") - print(f" Database type: {db_config.get('type', 'Not specified')}") - print(f" Engine: {db_config.get('engine', 'Not specified')}") - print(f" Host: {db_config.get('host', 'Not specified')}") - print(f" Port: {db_config.get('port', 'Not specified')}") - print(f" Database: {db_config.get('database', 'Not specified')}") - - # For PostgreSQL, suggest checking the schema - if db_config.get('engine') == 'postgresql': - print("\nFor PostgreSQL, verify that tables exist in the 'public' schema or") - print("that you have access to the schemas where the tables are located.") - print("You can check available schemas with: SELECT DISTINCT table_schema FROM information_schema.tables;") - - return - - # Show schema information - tables = schema.get("tables", {}) - - # Separate SQL tables and NoSQL collections to display them appropriately - sql_tables = {} - nosql_collections = {} - - for name, info in tables.items(): - if "columns" in info: - sql_tables[name] = info - elif "fields" in info: - nosql_collections[name] = info - - # Show SQL tables - if sql_tables: - _print_colored(f"\n{len(sql_tables)} SQL tables found:", "green") - for table_name, table_info in sql_tables.items(): - _print_colored(f"\n=== Table: {table_name} ===", "bold") - - # Show columns - columns = table_info.get("columns", []) - if columns: - _print_colored("Columns:", "blue") - for column in columns: - print(f" - {column['name']} ({column['type']})") - else: - _print_colored("No columns found.", "yellow") - - # Show sample data if available - sample_data = table_info.get("sample_data", []) - if sample_data: - _print_colored("\nSample data:", "blue") - for i, row in enumerate(sample_data[:2], 1): # Limit to 2 rows for simplicity - print(f" Record {i}: {row}") - - if len(sample_data) > 2: - print(f" ... ({len(sample_data) - 2} more records)") - - # Show NoSQL collections - if nosql_collections: - _print_colored(f"\n{len(nosql_collections)} NoSQL collections found:", "green") - for coll_name, coll_info in nosql_collections.items(): - _print_colored(f"\n=== Collection: {coll_name} ===", "bold") - - # Show fields - fields = coll_info.get("fields", []) - if fields: - _print_colored("Fields:", "blue") - for field in fields: - print(f" - {field['name']} ({field['type']})") - else: - _print_colored("No fields found.", "yellow") - - # Show sample data if available - sample_data = coll_info.get("sample_data", []) - if sample_data: - _print_colored("\nSample data:", "blue") - for i, doc in enumerate(sample_data[:2], 1): # Limit to 2 documents - # Simplify visualization for large documents - if isinstance(doc, dict) and len(doc) > 5: - simplified = {k: doc[k] for k in list(doc.keys())[:5]} - print(f" Document {i}: {simplified} ... (and {len(doc) - 5} more fields)") - else: - print(f" Document {i}: {doc}") - - if len(sample_data) > 2: - print(f" ... ({len(sample_data) - 2} more documents)") - - _print_colored("\n✅ Schema extracted successfully!", "green") - - # Ask if they want to save the schema to a file - save_option = input("\nDo you want to save the schema to a file? (y/n): ").strip().lower() - if save_option == "y": - filename = input("File name (default: db_schema.json): ").strip() or "db_schema.json" - try: - with open(filename, 'w') as f: - json.dump(schema, f, indent=2, default=str) - _print_colored(f"\n✅ Schema saved in: {filename}", "green") - except Exception as e: - _print_colored(f"❌ Error saving the file: {str(e)}", "red") - - except Exception as e: - _print_colored(f"❌ Error showing schema: {str(e)}", "red") - import traceback - traceback.print_exc() - - -def get_schema_with_dynamic_import(api_token: str, config_id: str, db_config: Dict[str, Any], api_url: Optional[str] = None) -> Dict[str, Any]: - """ - Retrieves the database schema using dynamic import. - - Args: - api_token: API token - config_id: Configuration ID - db_config: Database configuration - api_url: Optional API URL - - Returns: - Database schema - """ - try: - # Dynamic import of the core module - import importlib - core_module = importlib.import_module('core.client') - init_func = getattr(core_module, 'init') - - # Create a Corebrain instance with the selected configuration - cb = init_func( - api_token=api_token, - config_id=config_id, - api_url=api_url, - skip_verification=True # Skip verification for simplicity - ) - - # The schema is automatically extracted when initializing - schema = cb.db_schema - - # If there's no schema, we try to extract it explicitly - if not schema or not schema.get("tables"): - _print_colored("Trying to extract schema explicitly...", "yellow") - schema = cb._extract_db_schema() - - # Close the connection - cb.close() - - return schema - - except ImportError: - # If dynamic import fails, we try an alternative approach - _print_colored("Could not import client. Using alternative method.", "yellow") - return extract_db_schema(db_config) - - except Exception as e: - _print_colored(f"Error extracting schema with client: {str(e)}", "red") - # Fallback to direct extraction - return extract_db_schema(db_config) diff --git a/corebrain/corebrain/lib/sso/__init__.py b/corebrain/corebrain/lib/sso/__init__.py deleted file mode 100644 index 033a277..0000000 --- a/corebrain/corebrain/lib/sso/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from corebrain.lib.sso.auth import GlobodainSSOAuth -from corebrain.lib.sso.client import GlobodainSSOClient - -__all__ = ['GlobodainSSOAuth', 'GlobodainSSOClient'] \ No newline at end of file diff --git a/corebrain/corebrain/lib/sso/auth.py b/corebrain/corebrain/lib/sso/auth.py deleted file mode 100644 index a83736e..0000000 --- a/corebrain/corebrain/lib/sso/auth.py +++ /dev/null @@ -1,171 +0,0 @@ -import requests -import logging -from urllib.parse import urlencode - -class GlobodainSSOAuth: - def __init__(self, config=None): - self.config = config or {} - self.logger = logging.getLogger(__name__) - - # Default configuration - self.sso_url = self.config.get('GLOBODAIN_SSO_URL', 'http://localhost:3000/login') # SSO URL - self.client_id = self.config.get('GLOBODAIN_CLIENT_ID', '') - self.client_secret = self.config.get('GLOBODAIN_CLIENT_SECRET', '') - self.redirect_uri = self.config.get('GLOBODAIN_REDIRECT_URI', '') - self.success_redirect = self.config.get('GLOBODAIN_SUCCESS_REDIRECT', 'https://sso.globodain.com/cli/success') - - def requires_auth(self, session_handler): - """ - Generic decorator that checks if the user is authenticated - - Args: - session_handler: Function that retrieves the current session object - - Returns: - A decorator function that can be applied to routes/views - """ - def decorator(func): - def wrapper(*args, **kwargs): - # Get the current session using the provided handler - session = session_handler() - - if 'user' not in session: - # Here we return information for the framework to redirect - return { - 'authenticated': False, - 'redirect_url': self.get_login_url() - } - return func(*args, **kwargs) - return wrapper - return decorator - - def get_login_url(self, state=None): - """ - Generates the URL to initiate SSO authentication - - Args: - state: Optional parameter to maintain state between requests - - Returns: - Full URL for SSO login initiation - """ - params = { - 'client_id': self.client_id, - 'redirect_uri': self.redirect_uri, - 'response_type': 'code', - } - - if state: - params['state'] = state - - return f"{self.sso_url}/api/auth/authorize?{urlencode(params)}" - - def verify_token(self, token): - """ - Verifies the token with the SSO server - - Args: - token: Access token to verify - - Returns: - Token data if valid, None otherwise - """ - try: - response = requests.post( - f"{self.sso_url}/api/auth/service-auth", - headers={'Authorization': f'Bearer {token}'}, - json={'service_id': self.client_id} - ) - if response.status_code == 200: - return response.json() - return None - except Exception as e: - self.logger.error(f"Error verifying token: {str(e)}") - return None - - def get_user_info(self, token): - """ - Retrieves user information using the token - - Args: - token: User access token - - Returns: - User profile information if the token is valid, None otherwise - """ - try: - response = requests.get( - f"{self.sso_url}/api/users/me/profile", - headers={'Authorization': f'Bearer {token}'} - ) - if response.status_code == 200: - return response.json() - return None - except Exception as e: - self.logger.error(f"Error getting user info: {str(e)}") - return None - - def exchange_code_for_token(self, code): - """ - Exchanges the authorization code for an access token - - Args: - code: Authorization code received from the SSO server - - Returns: - Access token data if the exchange is successful, None otherwise - """ - try: - response = requests.post( - f"{self.sso_url}/api/auth/token", - json={ - 'client_id': self.client_id, - 'client_secret': self.client_secret, - 'code': code, - 'grant_type': 'authorization_code', - 'redirect_uri': self.redirect_uri - } - ) - if response.status_code == 200: - return response.json() - return None - except Exception as e: - self.logger.error(f"Error exchanging code: {str(e)}") - return None - - def handle_callback(self, code, session_handler, store_user_func=None): - """ - Handles the SSO callback by processing the received code - - Args: - code: Authorization code received - session_handler: Function that retrieves the current session object - store_user_func: Optional function to store user data elsewhere - - Returns: - Redirect URL after processing the code - """ - # Exchange code for token - token_data = self.exchange_code_for_token(code) - if not token_data: - # Error getting the token - return self.get_login_url() - - # Get user information - user_info = self.get_user_info(token_data.get('access_token')) - if not user_info: - # Error getting user information - return self.get_login_url() - - # Save information in the session - session = session_handler() - session['user'] = user_info - session['token'] = token_data - - # If there is a function to store the user, execute it - if store_user_func and callable(store_user_func): - store_user_func(user_info, token_data) - - # Redirect to the success URL or the previously saved URL - next_url = session.pop('next_url', self.success_redirect) - return next_url \ No newline at end of file diff --git a/corebrain/corebrain/lib/sso/client.py b/corebrain/corebrain/lib/sso/client.py deleted file mode 100644 index d47e1ac..0000000 --- a/corebrain/corebrain/lib/sso/client.py +++ /dev/null @@ -1,194 +0,0 @@ -# /auth/sso_client.py -import requests - -from typing import Dict, Any -from datetime import datetime, timedelta - -class GlobodainSSOClient: - """ - SDK client for Globodain services that connect to the central SSO - """ - - def __init__( - self, - sso_url: str, - client_id: str, - client_secret: str, - service_id: int, - redirect_uri: str - ): - """ - Initialize the SSO client - - Args: - sso_url: Base URL of the SSO service (e.g., https://sso.globodain.com) - client_id: Client ID of the service - client_secret: Client secret of the service - service_id: Numeric ID of the service on the SSO platform - redirect_uri: Redirect URI for OAuth - """ - self.sso_url = sso_url.rstrip('/') - self.client_id = client_id - self.client_secret = client_secret - self.service_id = service_id - self.redirect_uri = redirect_uri - self._token_cache = {} # Verified tokens cache - - - def get_login_url(self, provider: str = None) -> str: - """ - Get URL to initiate SSO login - - Args: - provider: OAuth provider (google, microsoft, github) or None for normal login - - Returns: - URL to redirect the user - """ - if provider: - return f"{self.sso_url}/api/auth/oauth/{provider}?service_id={self.service_id}" - else: - return f"{self.sso_url}/login?service_id={self.service_id}&redirect_uri={self.redirect_uri}" - - def verify_token(self, token: str) -> Dict[str, Any]: - """ - Verify an access token and retrieve user information - - Args: - token: JWT token to verify - - Returns: - User information if the token is valid - - Raises: - Exception: If the token is not valid - """ - # Check if we have cached and valid token information - now = datetime.now() - if token in self._token_cache: - cache_data = self._token_cache[token] - if cache_data['expires_at'] > now: - return cache_data['user_info'] - else: - # Delete expired token from cache - del self._token_cache[token] - - # Verify token with the SSO service - headers = { - "Authorization": f"Bearer {token}", - "Content-Type": "application/json" - } - - response = requests.post( - f"{self.sso_url}/api/auth/service-auth", - headers=headers, - json={"service_id": self.service_id} - ) - - if response.status_code != 200: - raise Exception(f"Invalid token: {response.text}") - - # Get user information - user_response = requests.get( - f"{self.sso_url}/api/users/me", - headers=headers - ) - - if user_response.status_code != 200: - raise Exception(f"Error getting user information: {user_response.text}") - - user_info = user_response.json() - - # Save in cache (15 minutes) - self._token_cache[token] = { - 'user_info': user_info, - 'expires_at': now + timedelta(minutes=15) - } - - return user_info - - def authenticate_service(self, token: str) -> Dict[str, Any]: - """ - Authenticate a token for use with this specific service - - Args: - token: JWT token obtained from the SSO - - Returns: - New service-specific token - - Raises: - Exception: If there is an authentication error - """ - headers = { - "Authorization": f"Bearer {token}", - "Content-Type": "application/json" - } - - response = requests.post( - f"{self.sso_url}/api/auth/service-auth", - headers=headers, - json={"service_id": self.service_id} - ) - - if response.status_code != 200: - raise Exception(f"Authentication error: {response.text}") - - return response.json() - - def refresh_token(self, refresh_token: str) -> Dict[str, Any]: - """ - Renew an access token using a refresh token - - Args: - refresh_token: Refresh token - - Returns: - New access token - - Raises: - Exception: If there is an error renewing the token - """ - response = requests.post( - f"{self.sso_url}/api/auth/refresh", - json={"refresh_token": refresh_token} - ) - - if response.status_code != 200: - raise Exception(f"Error renewing token: {response.text}") - - return response.json() - - def logout(self, refresh_token: str, access_token: str) -> bool: - """ - Log out (revoke refresh token) - - Args: - refresh_token: Refresh token to revoke - access_token: Valid access token - - Returns: - True if the logout was successful - - Raises: - Exception: If there is an error logging out - """ - headers = { - "Authorization": f"Bearer {access_token}", - "Content-Type": "application/json" - } - - response = requests.post( - f"{self.sso_url}/api/auth/logout", - headers=headers, - json={"refresh_token": refresh_token} - ) - - if response.status_code != 200: - raise Exception(f"Error logging out: {response.text}") - - # Clean any cached token - if access_token in self._token_cache: - del self._token_cache[access_token] - - return True \ No newline at end of file diff --git a/corebrain/corebrain/network/__init__.py b/corebrain/corebrain/network/__init__.py deleted file mode 100644 index 20328ee..0000000 --- a/corebrain/corebrain/network/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -""" -Network components for Corebrain SDK. - -This package provides utilities and clients for communication -with the Corebrain API and other web services. -""" -from corebrain.network.client import ( - APIClient, - APIError, - APITimeoutError, - APIConnectionError, - APIAuthError -) - -__all__ = [ - 'APIClient', - 'APIError', - 'APITimeoutError', - 'APIConnectionError', - 'APIAuthError' -] \ No newline at end of file diff --git a/corebrain/corebrain/network/client.py b/corebrain/corebrain/network/client.py deleted file mode 100644 index 8875f2c..0000000 --- a/corebrain/corebrain/network/client.py +++ /dev/null @@ -1,502 +0,0 @@ -""" -HTTP client for communication with the Corebrain API. -""" -import time -import logging -import httpx - -from typing import Dict, Any, Optional, List -from urllib.parse import urljoin -from httpx import Response, ConnectError, ReadTimeout, WriteTimeout, PoolTimeout - -logger = logging.getLogger(__name__) -http_session = httpx.Client(timeout=10.0, verify=True) - -def __init__(self, verbose=False): - self.verbose = verbose - -class APIError(Exception): - """Generic error in the API.""" - def __init__(self, message: str, status_code: Optional[int] = None, - detail: Optional[str] = None, response: Optional[Response] = None): - self.message = message - self.status_code = status_code - self.detail = detail - self.response = response - super().__init__(message) - -class APITimeoutError(APIError): - """Timeout error in the API.""" - pass - -class APIConnectionError(APIError): - """Connection error to the API.""" - pass - -class APIAuthError(APIError): - """Authentication error in the API.""" - pass - -class APIClient: - """Optimized HTTP client for communication with the Corebrain API.""" - - # Constants for retry handling and errors - MAX_RETRIES = 3 - RETRY_DELAY = 0.5 # seconds - RETRY_STATUS_CODES = [408, 429, 500, 502, 503, 504] - - def __init__(self, base_url: str, default_timeout: int = 10, - verify_ssl: bool = True, user_agent: Optional[str] = None): - """ - Initializes the API client with optimized configuration. - - Args: - base_url: Base URL for all requests - default_timeout: Default timeout in seconds - verify_ssl: Whether to verify the SSL certificate - user_agent: Custom user agent - """ - # Normalize base URL to ensure it ends with '/' - self.base_url = base_url if base_url.endswith('/') else base_url + '/' - self.default_timeout = default_timeout - self.verify_ssl = verify_ssl - - # Default headers - self.default_headers = { - 'User-Agent': user_agent or 'CorebrainSDK/1.0', - 'Accept': 'application/json', - 'Content-Type': 'application/json' - } - - # Create HTTP session with optimized limits and timeouts - self.session = httpx.Client( - timeout=httpx.Timeout(timeout=default_timeout), - verify=verify_ssl, - http2=True, # Use HTTP/2 if available - limits=httpx.Limits(max_connections=100, max_keepalive_connections=20) - ) - - # Statistics and metrics - self.request_count = 0 - self.error_count = 0 - self.total_request_time = 0 - - logger.debug(f"Cliente API inicializado con base_url={base_url}, timeout={default_timeout}s") - - def __del__(self): - """Ensure the session is closed when the client is deleted.""" - self.close() - - def close(self): - """Closes the HTTP session.""" - if hasattr(self, 'session') and self.session: - try: - self.session.close() - logger.debug("Sesión HTTP cerrada correctamente") - except Exception as e: - logger.warning(f"Error al cerrar sesión HTTP: {e}") - - def get_full_url(self, endpoint: str) -> str: - """ - Builds the full URL for an endpoint. - - Args: - endpoint: Relative path of the endpoint - - Returns: - Full URL - """ - # Remove '/' if it exists to avoid duplicate paths - endpoint = endpoint.lstrip('/') - return urljoin(self.base_url, endpoint) - - def prepare_headers(self, headers: Optional[Dict[str, str]] = None, - auth_token: Optional[str] = None) -> Dict[str, str]: - """ - Prepares the headers for a request. - - Args: - headers: Additional headers - auth_token: Authentication token - - Returns: - Combined headers - """ - # Start with default headers - final_headers = self.default_headers.copy() - - # Add custom headers - if headers: - final_headers.update(headers) - - # Add authentication token if provided - if auth_token: - final_headers['Authorization'] = f'Bearer {auth_token}' - - return final_headers - - def handle_response(self, response: Response) -> Response: - """ - Processes the response to handle common errors. - - Args: - response: HTTP response - - Returns: - The same response if there are no errors - - Raises: - APIError: If there are errors in the response - """ - status_code = response.status_code - - # Process errors according to status code - if 400 <= status_code < 500: - error_detail = None - - # Try to extract error details from JSON body - try: - json_data = response.json() - if isinstance(json_data, dict): - error_detail = ( - json_data.get('detail') or - json_data.get('message') or - json_data.get('error') - ) - except Exception: - # If we can't parse JSON, use the full text - error_detail = response.text[:200] + ('...' if len(response.text) > 200 else '') - - # Specific errors according to status code - if status_code == 401: - msg = "Authentication error: invalid or expired token" - logger.error(f"{msg} - {error_detail or ''}") - raise APIAuthError(msg, status_code, error_detail, response) - - elif status_code == 403: - msg = "Access denied: you don't have enough permissions" - logger.error(f"{msg} - {error_detail or ''}") - raise APIAuthError(msg, status_code, error_detail, response) - - elif status_code == 404: - msg = f"Resource not found: {response.url}" - logger.error(msg) - raise APIError(msg, status_code, error_detail, response) - - elif status_code == 429: - msg = "Too many requests: rate limit exceeded" - logger.warning(msg) - raise APIError(msg, status_code, error_detail, response) - - else: - msg = f"Client error ({status_code}): {error_detail or 'no details'}" - logger.error(msg) - raise APIError(msg, status_code, error_detail, response) - - elif 500 <= status_code < 600: - msg = f"Server error ({status_code}): the API server found an error" - logger.error(msg) - raise APIError(msg, status_code, response.text[:200], response) - - return response - - def request(self, method: str, endpoint: str, *, - headers: Optional[Dict[str, str]] = None, - json: Optional[Any] = None, - data: Optional[Any] = None, - params: Optional[Dict[str, Any]] = None, - timeout: Optional[int] = None, - auth_token: Optional[str] = None, - retry: bool = True) -> Response: - """ - Makes an HTTP request with error handling and retries. - - Args: - method: HTTP method (GET, POST, etc.) - endpoint: Relative path of the endpoint - headers: Additional headers - json: Data to send as JSON - data: Data to send as form or bytes - params: Query string parameters - timeout: Timeout in seconds (overrides the default) - auth_token: Authentication token - retry: Whether to retry failed requests - - Returns: - Processed HTTP response - - Raises: - APIError: If there are errors in the request or response - APITimeoutError: If the request exceeds the timeout - APIConnectionError: If there are connection errors - """ - url = self.get_full_url(endpoint) - final_headers = self.prepare_headers(headers, auth_token) - - # Configure timeout - request_timeout = timeout or self.default_timeout - - # Counter for retries - retries = 0 - last_error = None - - # Register start of request - start_time = time.time() - self.request_count += 1 - - while retries <= (self.MAX_RETRIES if retry else 0): - try: - if retries > 0: - # Wait before retrying with exponential backoff - wait_time = self.RETRY_DELAY * (2 ** (retries - 1)) - logger.info(f"Retrying request ({retries}/{self.MAX_RETRIES}) to {url} after {wait_time:.2f}s") - time.sleep(wait_time) - - # Make the request - logger.debug(f"Sending request {method} to {url}") - response = self.session.request( - method=method, - url=url, - headers=final_headers, - json=json, - data=data, - params=params, - timeout=request_timeout - ) - - # Check if we should retry by status code - if response.status_code in self.RETRY_STATUS_CODES and retry and retries < self.MAX_RETRIES: - logger.warning(f"Status code {response.status_code} received, retrying") - retries += 1 - continue - - # Process the response - processed_response = self.handle_response(response) - - # Register total time - elapsed = time.time() - start_time - self.total_request_time += elapsed - logger.debug(f"Request completed in {elapsed:.3f}s with status {response.status_code}") - - return processed_response - - except (ConnectError, httpx.HTTPError) as e: - last_error = e - - # Decide if we should retry depending on the type of error - if isinstance(e, (ReadTimeout, WriteTimeout, PoolTimeout, ConnectError)) and retry and retries < self.MAX_RETRIES: - logger.warning(f"Connection error: {str(e)}, retrying {retries+1}/{self.MAX_RETRIES}") - retries += 1 - continue - - # No more retries or unrecoverable error - self.error_count += 1 - elapsed = time.time() - start_time - - if isinstance(e, (ReadTimeout, WriteTimeout, PoolTimeout)): - logger.error(f"Timeout in request to {url} after {elapsed:.3f}s: {str(e)}") - raise APITimeoutError(f"The request to {endpoint} exceeded the maximum time of {request_timeout}s", - response=getattr(e, 'response', None)) - else: - logger.error(f"Connection error to {url} after {elapsed:.3f}s: {str(e)}") - raise APIConnectionError(f"Connection error to {endpoint}: {str(e)}", - response=getattr(e, 'response', None)) - - except Exception as e: - # Unexpected error - self.error_count += 1 - elapsed = time.time() - start_time - logger.error(f"Unexpected error in request to {url} after {elapsed:.3f}s: {str(e)}") - raise APIError(f"Unexpected error in request to {endpoint}: {str(e)}") - - # If we get here, we have exhausted the retries - if last_error: - self.error_count += 1 - raise APIError(f"Request to {endpoint} failed after {retries} retries: {str(last_error)}") - - # This point should never be reached - raise APIError(f"Unexpected error in request to {endpoint}") - - def get(self, endpoint: str, **kwargs) -> Response: - """Makes a GET request.""" - return self.request("GET", endpoint, **kwargs) - - def post(self, endpoint: str, **kwargs) -> Response: - """Makes a POST request.""" - return self.request("POST", endpoint, **kwargs) - - def put(self, endpoint: str, **kwargs) -> Response: - """Makes a PUT request.""" - return self.request("PUT", endpoint, **kwargs) - - def delete(self, endpoint: str, **kwargs) -> Response: - """Makes a DELETE request.""" - return self.request("DELETE", endpoint, **kwargs) - - def patch(self, endpoint: str, **kwargs) -> Response: - """Makes a PATCH request.""" - return self.request("PATCH", endpoint, **kwargs) - - def get_json(self, endpoint: str, **kwargs) -> Any: - """ - Makes a GET request and returns the JSON data. - - Args: - endpoint: Endpoint to query - **kwargs: Additional arguments for request() - - Returns: - Parsed JSON data - """ - response = self.get(endpoint, **kwargs) - try: - return response.json() - except Exception as e: - raise APIError(f"Error parsing JSON response: {str(e)}", response=response) - - def post_json(self, endpoint: str, **kwargs) -> Any: - """ - Makes a POST request and returns the JSON data. - - Args: - endpoint: Endpoint to query - **kwargs: Additional arguments for request() - - Returns: - Parsed JSON data - """ - response = self.post(endpoint, **kwargs) - try: - return response.json() - except Exception as e: - raise APIError(f"Error parsing JSON response: {str(e)}", response=response) - - # High-level methods for common operations in the Corebrain API - - def check_health(self, timeout: int = 5) -> bool: - """ - Checks if the API is available. - - Args: - timeout: Maximum wait time - - Returns: - True if the API is available - """ - try: - response = self.get("health", timeout=timeout, retry=False) - return response.status_code == 200 - except Exception: - return False - - def verify_token(self, token: str, timeout: int = 5) -> Dict[str, Any]: - """ - Verifies if a token is valid. - - Args: - token: Token to verify - timeout: Maximum wait time - - Returns: - User information if the token is valid - - Raises: - APIAuthError: If the token is invalid - """ - try: - response = self.get("api/auth/me", auth_token=token, timeout=timeout) - return response.json() - except APIAuthError: - raise - except Exception as e: - raise APIAuthError(f"Error verifying token: {str(e)}") - - def get_api_keys(self, token: str) -> List[Dict[str, Any]]: - """ - Retrieves the available API keys for a user. - - Args: - token: Authentication token - - Returns: - List of API keys - """ - return self.get_json("api/auth/api-keys", auth_token=token) - - def update_api_key_metadata(self, token: str, api_key: str, metadata: Dict[str, Any]) -> Dict[str, Any]: - """ - Updates the metadata of an API key. - - Args: - token: Authentication token - api_key: API key ID - metadata: Metadata to update - - Returns: - Updated API key data - """ - data = {"metadata": metadata} - return self.put_json(f"api/auth/api-keys/{api_key}", auth_token=token, json=data) - - def query_database(self, token: str, question: str, db_schema: Dict[str, Any], - config_id: str, timeout: int = 30) -> Dict[str, Any]: - """ - Makes a natural language query. - - Args: - token: Authentication token - question: Natural language question - db_schema: Database schema - config_id: Configuration ID - timeout: Maximum wait time - - Returns: - Query result - """ - data = { - "question": question, - "db_schema": db_schema, - "config_id": config_id - } - return self.post_json("api/database/sdk/query", auth_token=token, json=data, timeout=timeout) - - def exchange_sso_token(self, sso_token: str, user_data: Dict[str, Any]) -> Dict[str, Any]: - """ - Exchanges an SSO token for an API token. - - Args: - sso_token: SSO token - user_data: User data - - Returns: - API token data - """ - headers = {"Authorization": f"Bearer {sso_token}"} - data = {"user_data": user_data} - return self.post_json("api/auth/sso/token", headers=headers, json=data) - - # Methods for statistics and diagnostics - - def get_stats(self) -> Dict[str, Any]: - """ - Retrieves client usage statistics. - - Returns: - Request statistics - """ - avg_time = self.total_request_time / max(1, self.request_count) - error_rate = (self.error_count / max(1, self.request_count)) * 100 - - return { - "request_count": self.request_count, - "error_count": self.error_count, - "error_rate": f"{error_rate:.2f}%", - "total_request_time": f"{self.total_request_time:.3f}s", - "average_request_time": f"{avg_time:.3f}s", - } - - def reset_stats(self) -> None: - """Resets the usage statistics.""" - self.request_count = 0 - self.error_count = 0 - self.total_request_time = 0 \ No newline at end of file diff --git a/corebrain/corebrain/sdk.py b/corebrain/corebrain/sdk.py deleted file mode 100644 index b08226c..0000000 --- a/corebrain/corebrain/sdk.py +++ /dev/null @@ -1,7 +0,0 @@ -""" -Corebrain SDK for compatibility. -""" -from corebrain.config.manager import ConfigManager - -list_configurations = ConfigManager().list_configs -remove_configuration = ConfigManager().remove_config \ No newline at end of file diff --git a/corebrain/corebrain/services/schema.py b/corebrain/corebrain/services/schema.py deleted file mode 100644 index d18beea..0000000 --- a/corebrain/corebrain/services/schema.py +++ /dev/null @@ -1,30 +0,0 @@ - -# Nuevo directorio: services/ -# Nuevo archivo: services/schema_service.py -""" -Services for managing database schemas. -""" -from typing import Dict, Any, Optional - -from corebrain.config.manager import ConfigManager -from corebrain.db.schema import extract_db_schema, SchemaOptimizer - -class SchemaService: - """Service for database schema operations.""" - - def __init__(self): - self.config_manager = ConfigManager() - self.schema_optimizer = SchemaOptimizer() - - def get_schema(self, api_token: str, config_id: str) -> Optional[Dict[str, Any]]: - """Retrieves the schema for a specific configuration.""" - config = self.config_manager.get_config(api_token, config_id) - if not config: - return None - - return extract_db_schema(config) - - def optimize_schema(self, schema: Dict[str, Any], query: str = None) -> Dict[str, Any]: - """Optimizes an existing schema.""" - return self.schema_optimizer.optimize_schema(schema, query) - \ No newline at end of file diff --git a/corebrain/corebrain/utils/__init__.py b/corebrain/corebrain/utils/__init__.py deleted file mode 100644 index e264aff..0000000 --- a/corebrain/corebrain/utils/__init__.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -General utilities for Corebrain SDK. - -This package provides utilities shared by different -SDK components, such as serialization, encryption, and logging. -""" -import logging - -from corebrain.utils.serializer import serialize_to_json, JSONEncoder -from corebrain.utils.encrypter import ( - create_cipher, - generate_key, - derive_key_from_password, - ConfigEncrypter -) - -logger = logging.getLogger('corebrain') - -def setup_logger(level=logging.INFO, - file_path=None, - format_string=None): - """ - Configures the main Corebrain logger. - - Args: - level: Logging level - file_path: Path to log file (optional) - format_string: Custom log format - """ - # Formato predeterminado - fmt = format_string or '%(asctime)s - %(name)s - %(levelname)s - %(message)s' - formatter = logging.Formatter(fmt) - - # Handler de consola - console_handler = logging.StreamHandler() - console_handler.setFormatter(formatter) - - # Configurar logger principal - logger.setLevel(level) - logger.addHandler(console_handler) - - # File handler if path is provided - if file_path: - file_handler = logging.FileHandler(file_path) - file_handler.setFormatter(formatter) - logger.addHandler(file_handler) - - logger.debug(f"Logger configurado con nivel {logging.getLevelName(level)}") - if file_path: - logger.debug(f"Logs escritos a {file_path}") - - return logger - - # Exportación explícita de componentes públicos -__all__ = [ - 'serialize_to_json', - 'JSONEncoder', - 'create_cipher', - 'generate_key', - 'derive_key_from_password', - 'ConfigEncrypter', - 'setup_logger', - 'logger' -] \ No newline at end of file diff --git a/corebrain/corebrain/utils/encrypter.py b/corebrain/corebrain/utils/encrypter.py deleted file mode 100644 index 1d18abd..0000000 --- a/corebrain/corebrain/utils/encrypter.py +++ /dev/null @@ -1,264 +0,0 @@ -""" -Encryption utilities for Corebrain SDK. -""" -import os -import base64 -import logging - -from pathlib import Path -from typing import Optional, Union -from cryptography.fernet import Fernet, InvalidToken -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC - -logger = logging.getLogger(__name__) - -def derive_key_from_password(password: Union[str, bytes], salt: Optional[bytes] = None) -> bytes: - """ - Derives a secure encryption key from a password and salt. - - Args: - password: Password or passphrase - salt: Cryptographic salt (generated if not provided) - - Returns: - Derived key in bytes - """ - if isinstance(password, str): - password = password.encode() - - # Generate salt if not provided - if salt is None: - salt = os.urandom(16) - - # Derive key using PBKDF2 - kdf = PBKDF2HMAC( - algorithm=hashes.SHA256(), - length=32, - salt=salt, - iterations=100000 # Higher number of iterations = higher security - ) - - key = kdf.derive(password) - return base64.urlsafe_b64encode(key) - -def generate_key() -> str: - """ - Generates a new random encryption key. - - Returns: - Encryption key in base64 format - """ - key = Fernet.generate_key() - return key.decode() - -def create_cipher(key: Optional[Union[str, bytes]] = None) -> Fernet: - """ - Creates a Fernet encryption object with the given key or generates a new one. - - Args: - key: Encryption key in base64 format or None to generate a new one - - Returns: - Fernet object for encryption/decryption - """ - if key is None: - key = Fernet.generate_key() - elif isinstance(key, str): - key = key.encode() - - return Fernet(key) - -class ConfigEncrypter: - """ - Encryption manager for configurations with key management. - """ - - def __init__(self, key_path: Optional[Union[str, Path]] = None): - """ - Initializes the encryptor with an optional key path. - - Args: - key_path: Path to the key file (will be created if it doesn't exist) - """ - self.key_path = Path(key_path) if key_path else None - self.cipher = None - self._init_cipher() - - def _init_cipher(self) -> None: - """Initializes the encryption object, creating or loading the key as needed.""" - key = None - - # If there is a key path, try to load or create - if self.key_path: - try: - if self.key_path.exists(): - with open(self.key_path, 'rb') as f: - key = f.read().strip() - logger.debug(f"Clave cargada desde {self.key_path}") - else: - # Create parent directory if it doesn't exist - self.key_path.parent.mkdir(parents=True, exist_ok=True) - - # Generate new key - key = Fernet.generate_key() - - # Save key - with open(self.key_path, 'wb') as f: - f.write(key) - - # Ensure restrictive permissions (only the owner can read) - try: - os.chmod(self.key_path, 0o600) - except Exception as e: - logger.warning(f"No se pudieron establecer permisos en archivo de clave: {e}") - - logger.debug(f"Nueva clave generada y guardada en {self.key_path}") - except Exception as e: - logger.error(f"Error al gestionar clave en {self.key_path}: {e}") - # If there is an error, generate a temporary key - key = None - - # If we don't have a key, generate a temporary key - if not key: - key = Fernet.generate_key() - logger.debug("Usando clave efímera generada") - - self.cipher = Fernet(key) - - def encrypt(self, data: Union[str, bytes]) -> bytes: - """ - Encrypts data. - - Args: - data: Data to encrypt - - Returns: - Encrypted data in bytes - """ - if isinstance(data, str): - data = data.encode() - - try: - return self.cipher.encrypt(data) - except Exception as e: - logger.error(f"Error al cifrar datos: {e}") - raise - - def decrypt(self, encrypted_data: Union[str, bytes]) -> bytes: - """ - Decrypts data. - - Args: - encrypted_data: Encrypted data - - Returns: - Decrypted data in bytes - """ - if isinstance(encrypted_data, str): - encrypted_data = encrypted_data.encode() - - try: - return self.cipher.decrypt(encrypted_data) - except InvalidToken: - logger.error("Token inválido o datos corruptos") - raise ValueError("Los datos no pueden ser descifrados: token inválido o datos corruptos") - except Exception as e: - logger.error(f"Error al descifrar datos: {e}") - raise - - def encrypt_file(self, input_path: Union[str, Path], output_path: Optional[Union[str, Path]] = None) -> Path: - """ - Encrypts a complete file. - - Args: - input_path: Path to the file to encrypt - output_path: Path to save the encrypted file (if None, .enc is added) - - Returns: - Path of the encrypted file - """ - input_path = Path(input_path) - - if not output_path: - output_path = input_path.with_suffix(input_path.suffix + '.enc') - else: - output_path = Path(output_path) - - try: - with open(input_path, 'rb') as f: - data = f.read() - - encrypted_data = self.encrypt(data) - - with open(output_path, 'wb') as f: - f.write(encrypted_data) - - return output_path - except Exception as e: - logger.error(f"Error al cifrar archivo {input_path}: {e}") - raise - - def decrypt_file(self, input_path: Union[str, Path], output_path: Optional[Union[str, Path]] = None) -> Path: - """ - Decrypts a complete file. - - Args: - input_path: Path to the encrypted file - output_path: Path to save the decrypted file - - Returns: - Path of the decrypted file - """ - input_path = Path(input_path) - - if not output_path: - # If it ends in .enc, remove that extension - if input_path.suffix == '.enc': - output_path = input_path.with_suffix('') - else: - output_path = input_path.with_suffix(input_path.suffix + '.dec') - else: - output_path = Path(output_path) - - try: - with open(input_path, 'rb') as f: - encrypted_data = f.read() - - decrypted_data = self.decrypt(encrypted_data) - - with open(output_path, 'wb') as f: - f.write(decrypted_data) - - return output_path - except Exception as e: - logger.error(f"Error al descifrar archivo {input_path}: {e}") - raise - - @staticmethod - def generate_key_file(key_path: Union[str, Path]) -> None: - """ - Generates and saves a new key to a file. - - Args: - key_path: Path to save the key - """ - key_path = Path(key_path) - - # Create parent directory if it doesn't exist - key_path.parent.mkdir(parents=True, exist_ok=True) - - # Generate key - key = Fernet.generate_key() - - # Save key - with open(key_path, 'wb') as f: - f.write(key) - - # Set restrictive permissions - try: - os.chmod(key_path, 0o600) - except Exception as e: - logger.warning(f"No se pudieron establecer permisos en archivo de clave: {e}") - - logger.info(f"Nueva clave generada y guardada en {key_path}") \ No newline at end of file diff --git a/corebrain/corebrain/utils/logging.py b/corebrain/corebrain/utils/logging.py deleted file mode 100644 index 6b7c5e7..0000000 --- a/corebrain/corebrain/utils/logging.py +++ /dev/null @@ -1,243 +0,0 @@ -""" -Logging utilities for Corebrain SDK. - -This module provides functions and classes to manage logging -within the SDK consistently. -""" -import logging -import sys -from datetime import datetime -from pathlib import Path -from typing import Optional, Any, Union - -# Custom logging levels -VERBOSE = 15 # Between DEBUG and INFO - -# Default configuration -DEFAULT_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' -DEFAULT_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' -DEFAULT_LEVEL = logging.INFO -DEFAULT_LOG_DIR = Path.home() / ".corebrain" / "logs" - -# Colores de logging en terminal -LOG_COLORS = { - "DEBUG": "\033[94m", # Azul - "VERBOSE": "\033[96m", # Cian - "INFO": "\033[92m", # Verde - "WARNING": "\033[93m", # Amarillo - "ERROR": "\033[91m", # Rojo - "CRITICAL": "\033[95m", # Magenta - "RESET": "\033[0m" # Reset -} - -class VerboseLogger(logging.Logger): - """Custom logger with VERBOSE level.""" - - def verbose(self, msg: str, *args: Any, **kwargs: Any) -> None: - """ - Logs a message with VERBOSE level. - - Args: - msg: Message to log - *args: Arguments to format the message - **kwargs: Additional arguments for the logger - """ - return self.log(VERBOSE, msg, *args, **kwargs) - -class ColoredFormatter(logging.Formatter): - """Formatter that adds colors to log messages in the terminal.""" - - def __init__(self, fmt: str = DEFAULT_FORMAT, datefmt: str = DEFAULT_DATE_FORMAT, - use_colors: bool = True): - """ - Initializes the formatter. - - Args: - fmt: Message format - datefmt: Date format - use_colors: If True, uses colors in the terminal - """ - super().__init__(fmt, datefmt) - self.use_colors = use_colors and sys.stdout.isatty() - - def format(self, record: logging.LogRecord) -> str: - """ - Formats a log record with colors. - - Args: - record: Record to format - - Returns: - Formatted message - """ - levelname = record.levelname - message = super().format(record) - - if self.use_colors and levelname in LOG_COLORS: - return f"{LOG_COLORS[levelname]}{message}{LOG_COLORS['RESET']}" - return message - -def setup_logger(name: str = "corebrain", - level: int = DEFAULT_LEVEL, - file_path: Optional[Union[str, Path]] = None, - format_string: Optional[str] = None, - use_colors: bool = True, - propagate: bool = False) -> logging.Logger: - """ - Configures a logger with custom options. - - Args: - name: Logger name - level: Logging level - file_path: Path to the log file (optional) - format_string: Custom message format - use_colors: If True, uses colors in the terminal - propagate: If True, propagates messages to parent loggers - - Returns: - Configured logger - """ - # Register custom VERBOSE level - if not hasattr(logging, 'VERBOSE'): - logging.addLevelName(VERBOSE, 'VERBOSE') - - # Register custom logger class - logging.setLoggerClass(VerboseLogger) - - # Get or create logger - logger = logging.getLogger(name) - - # Clean existing handlers - for handler in logger.handlers[:]: - logger.removeHandler(handler) - - # Configure logging level - logger.setLevel(level) - logger.propagate = propagate - - # Default format - fmt = format_string or DEFAULT_FORMAT - formatter = ColoredFormatter(fmt, use_colors=use_colors) - - # Console handler - console_handler = logging.StreamHandler() - console_handler.setFormatter(formatter) - logger.addHandler(console_handler) - - # File handler if path is provided - if file_path: - # Ensure directory exists - file_path = Path(file_path) - file_path.parent.mkdir(parents=True, exist_ok=True) - - file_handler = logging.FileHandler(file_path) - # For files, use formatter without colors - file_formatter = logging.Formatter(fmt) - file_handler.setFormatter(file_formatter) - logger.addHandler(file_handler) - - # Diagnostic messages - logger.debug(f"Logger '{name}' configurado con nivel {logging.getLevelName(level)}") - if file_path: - logger.debug(f"Logs escritos a {file_path}") - - return logger - -def get_logger(name: str, level: Optional[int] = None) -> logging.Logger: - """ - Retrieves an existing logger or creates a new one. - - Args: - name: Logger name - level: Optional logging level - - Returns: - Configured logger - """ - logger = logging.getLogger(name) - - # If logger has no handlers, configure it - if not logger.handlers: - # Determine if it is a secondary logger - if '.' in name: - # It is a sublogger, configure to propagate to parent logger - logger.propagate = True - if level is not None: - logger.setLevel(level) - else: - # It is a main logger, configure completely - logger = setup_logger(name, level or DEFAULT_LEVEL) - elif level is not None: - # Update level only if specified - logger.setLevel(level) - - return logger - -def enable_file_logging(logger_name: str = "corebrain", - log_dir: Optional[Union[str, Path]] = None, - filename: Optional[str] = None) -> str: - """ - Enables file logging for an existing logger. - - Args: - logger_name: Logger name - log_dir: Directory for the logs (optional) - filename: Custom file name (optional) - - Returns: - Path to the log file - """ - logger = logging.getLogger(logger_name) - - # Determine the log file path - log_dir = Path(log_dir) if log_dir else DEFAULT_LOG_DIR - log_dir.mkdir(parents=True, exist_ok=True) - - # Generate file name if not provided - if not filename: - timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') - filename = f"{logger_name}_{timestamp}.log" - - file_path = log_dir / filename - - # Check if there is already a FileHandler - for handler in logger.handlers: - if isinstance(handler, logging.FileHandler): - logger.removeHandler(handler) - - # Add new FileHandler - file_handler = logging.FileHandler(file_path) - formatter = logging.Formatter(DEFAULT_FORMAT) - file_handler.setFormatter(formatter) - logger.addHandler(file_handler) - - logger.info(f"Logging a archivo activado: {file_path}") - return str(file_path) - -def set_log_level(level: Union[int, str], - logger_name: Optional[str] = None) -> None: - """ - Sets the logging level for one or all loggers. - - Args: - level: Logging level (name or integer value) - logger_name: Specific logger name (if None, affects all) - """ - # Convert level name to value if necessary - if isinstance(level, str): - level = getattr(logging, level.upper(), logging.INFO) - - if logger_name: - # Affect only the specified logger - logger = logging.getLogger(logger_name) - logger.setLevel(level) - logger.info(f"Nivel de log cambiado a {logging.getLevelName(level)}") - else: - # Affect the root logger and all existing loggers - root = logging.getLogger() - root.setLevel(level) - - # Also affect specific SDK loggers - for name in logging.root.manager.loggerDict: - if name.startswith("corebrain"): - logging.getLogger(name).setLevel(level) \ No newline at end of file diff --git a/corebrain/corebrain/utils/serializer.py b/corebrain/corebrain/utils/serializer.py deleted file mode 100644 index 6652e02..0000000 --- a/corebrain/corebrain/utils/serializer.py +++ /dev/null @@ -1,33 +0,0 @@ -""" -Serialization utilities for Corebrain SDK. -""" -import json - -from datetime import datetime, date, time -from bson import ObjectId -from decimal import Decimal - -class JSONEncoder(json.JSONEncoder): - """Custom JSON serializer for special types.""" - def default(self, obj): - # Objects datetime - if isinstance(obj, (datetime, date, time)): - return obj.isoformat() - # Objects timedelta - elif hasattr(obj, 'total_seconds'): # For objects timedelta - return obj.total_seconds() - # ObjectId from MongoDB - elif isinstance(obj, ObjectId): - return str(obj) - # Bytes o bytearray - elif isinstance(obj, (bytes, bytearray)): - return obj.hex() - # Decimal - elif isinstance(obj, Decimal): - return float(obj) - # Other types - return super().default(obj) - -def serialize_to_json(obj): - """Serializes any object to JSON using the custom encoder""" - return json.dumps(obj, cls=JSONEncoder) \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp/.editorconfig b/corebrain/corebrain/wrappers/csharp/.editorconfig deleted file mode 100644 index e4eb58c..0000000 --- a/corebrain/corebrain/wrappers/csharp/.editorconfig +++ /dev/null @@ -1,432 +0,0 @@ -# This file is the top-most EditorConfig file -root = true - -#All Files -[*] -charset = utf-8 -indent_style = space -indent_size = 2 -insert_final_newline = true -trim_trailing_whitespace = true -end_of_line = lf - -########################################## -# File Extension Settings -########################################## - -# Visual Studio Solution Files -[*.sln] -indent_style = tab - -# Visual Studio XML Project Files -[*.{csproj,vbproj,vcxproj.filters,proj,projitems,shproj}] -indent_size = 2 - -# XML Configuration Files -[*.{xml,config,props,targets,nuspec,resx,ruleset,vsixmanifest,vsct}] -indent_size = 2 - -# JSON Files -[*.{json,json5,webmanifest}] -indent_size = 2 - -# YAML Files -[*.{yml,yaml}] -indent_size = 2 - -# Markdown Files -[*.{md,mdx}] -trim_trailing_whitespace = false - -# Web Files -[*.{htm,html,js,jsm,ts,tsx,cjs,cts,ctsx,mjs,mts,mtsx,css,sass,scss,less,pcss,svg,vue}] -indent_size = 2 - -# Batch Files -[*.{cmd,bat}] -end_of_line = crlf - -# Bash Files -[*.sh] -end_of_line = lf - -# Makefiles -[Makefile] -indent_style = tab - -[{*_Generated.cs, *.g.cs, *.generated.cs}] -# Ignore a lack of documentation for generated code. Doesn't apply to builds, -# just to viewing generation output. -dotnet_diagnostic.CS1591.severity = none - -########################################## -# Default .NET Code Style Severities -########################################## - -[*.{cs,csx,cake,vb,vbx}] -# Default Severity for all .NET Code Style rules below -dotnet_analyzer_diagnostic.severity = warning - -########################################## -# Language Rules -########################################## - -# .NET Style Rules -[*.{cs,csx,cake,vb,vbx}] - -# "this." and "Me." qualifiers -dotnet_style_qualification_for_field = false -dotnet_style_qualification_for_property = false -dotnet_style_qualification_for_method = false -dotnet_style_qualification_for_event = false - -# Language keywords instead of framework type names for type references -dotnet_style_predefined_type_for_locals_parameters_members = true:warning -dotnet_style_predefined_type_for_member_access = true:warning - -# Modifier preferences -dotnet_style_require_accessibility_modifiers = always:warning -csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:warning -visual_basic_preferred_modifier_order = Partial,Default,Private,Protected,Public,Friend,NotOverridable,Overridable,MustOverride,Overloads,Overrides,MustInherit,NotInheritable,Static,Shared,Shadows,ReadOnly,WriteOnly,Dim,Const,WithEvents,Widening,Narrowing,Custom,Async:warning -dotnet_style_readonly_field = true:warning -dotnet_diagnostic.IDE0036.severity = warning - - -# Parentheses preferences -dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:warning -dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:warning -dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:warning -dotnet_style_parentheses_in_other_operators = never_if_unnecessary:warning - -# Expression-level preferences -dotnet_style_object_initializer = true:warning -dotnet_style_collection_initializer = true:warning -dotnet_style_explicit_tuple_names = true:warning -dotnet_style_prefer_inferred_tuple_names = true:warning -dotnet_style_prefer_inferred_anonymous_type_member_names = true:warning -dotnet_style_prefer_auto_properties = true:warning -dotnet_style_prefer_conditional_expression_over_assignment = true:suggestion -dotnet_diagnostic.IDE0045.severity = suggestion -dotnet_style_prefer_conditional_expression_over_return = true:suggestion -dotnet_diagnostic.IDE0046.severity = suggestion -dotnet_style_prefer_compound_assignment = true:warning -dotnet_style_prefer_simplified_interpolation = true:warning -dotnet_style_prefer_simplified_boolean_expressions = true:warning - -# Null-checking preferences -dotnet_style_coalesce_expression = true:warning -dotnet_style_null_propagation = true:warning -dotnet_style_prefer_is_null_check_over_reference_equality_method = true:warning - -# File header preferences -# Keep operators at end of line when wrapping. -dotnet_style_operator_placement_when_wrapping = end_of_line:warning -csharp_style_prefer_null_check_over_type_check = true:warning - -# Code block preferences -csharp_prefer_braces = true:warning -csharp_prefer_simple_using_statement = true:suggestion -dotnet_diagnostic.IDE0063.severity = suggestion - -# C# Style Rules -[*.{cs,csx,cake}] -# 'var' preferences -csharp_style_var_for_built_in_types = true:warning -csharp_style_var_when_type_is_apparent = true:warning -csharp_style_var_elsewhere = true:warning -# Expression-bodied members -csharp_style_expression_bodied_methods = true:warning -csharp_style_expression_bodied_constructors = false:warning -csharp_style_expression_bodied_operators = true:warning -csharp_style_expression_bodied_properties = true:warning -csharp_style_expression_bodied_indexers = true:warning -csharp_style_expression_bodied_accessors = true:warning -csharp_style_expression_bodied_lambdas = true:warning -csharp_style_expression_bodied_local_functions = true:warning -# Pattern matching preferences -csharp_style_pattern_matching_over_is_with_cast_check = true:warning -csharp_style_pattern_matching_over_as_with_null_check = true:warning -csharp_style_prefer_switch_expression = true:warning -csharp_style_prefer_pattern_matching = true:warning -csharp_style_prefer_not_pattern = true:warning -# Expression-level preferences -csharp_style_inlined_variable_declaration = true:warning -csharp_prefer_simple_default_expression = true:warning -csharp_style_pattern_local_over_anonymous_function = true:warning -csharp_style_deconstructed_variable_declaration = true:warning -csharp_style_prefer_index_operator = true:warning -csharp_style_prefer_range_operator = true:warning -csharp_style_implicit_object_creation_when_type_is_apparent = true:warning -# "Null" checking preferences -csharp_style_throw_expression = true:warning -csharp_style_conditional_delegate_call = true:warning -# Code block preferences -csharp_prefer_braces = true:warning -csharp_prefer_simple_using_statement = true:suggestion -dotnet_diagnostic.IDE0063.severity = suggestion -# 'using' directive preferences -csharp_using_directive_placement = inside_namespace:warning -# Modifier preferences -# Don't suggest making public methods static. Very annoying. -csharp_prefer_static_local_function = false -# Only suggest making private methods static (if they don't use instance data). -dotnet_code_quality.CA1822.api_surface = private - -########################################## -# Unnecessary Code Rules -########################################## - -# .NET Unnecessary code rules -[*.{cs,csx,cake,vb,vbx}] - -dotnet_code_quality_unused_parameters = non_public:suggestion -dotnet_remove_unnecessary_suppression_exclusions = none -dotnet_diagnostic.IDE0079.severity = warning - -# C# Unnecessary code rules -[*.{cs,csx,cake}] - - -# Don't remove method parameters that are unused. -dotnet_diagnostic.IDE0060.severity = none -dotnet_diagnostic.RCS1163.severity = none - -# Don't remove methods that are unused. -dotnet_diagnostic.IDE0051.severity = none -dotnet_diagnostic.RCS1213.severity = none - -# Use discard variable for unused expression values. -csharp_style_unused_value_expression_statement_preference = discard_variable - -# .NET formatting rules -[*.{cs,csx,cake,vb,vbx}] - -# Organize using directives -dotnet_sort_system_directives_first = true -dotnet_separate_import_directive_groups = false - -dotnet_sort_accessibility = true - -# Dotnet namespace options -# -# We don't care about namespaces matching folder structure. Games and apps -# are complicated and you are free to organize them however you like. Change -# this if you want to enforce it. -dotnet_style_namespace_match_folder = false -dotnet_diagnostic.IDE0130.severity = none - -# C# formatting rules -[*.{cs,csx,cake}] - -# Newline options -csharp_new_line_before_open_brace = none -csharp_new_line_before_else = true -csharp_new_line_before_catch = true -csharp_new_line_before_finally = true -csharp_new_line_before_members_in_object_initializers = true -csharp_new_line_before_members_in_anonymous_types = true -csharp_new_line_between_query_expression_clauses = true - -# Indentation options -csharp_indent_switch_labels = true -csharp_indent_case_contents = true -csharp_indent_case_contents_when_block = true -csharp_indent_labels = no_change -csharp_indent_block_contents = true -csharp_indent_braces = false - -# Spacing options -csharp_space_after_cast = false -csharp_space_after_keywords_in_control_flow_statements = true -csharp_space_between_parentheses = false -csharp_space_before_colon_in_inheritance_clause = true -csharp_space_after_colon_in_inheritance_clause = true -csharp_space_around_binary_operators = before_and_after -csharp_space_between_method_declaration_parameter_list_parentheses = false -csharp_space_between_method_declaration_empty_parameter_list_parentheses = false -csharp_space_between_method_declaration_name_and_open_parenthesis = false -csharp_space_between_method_call_parameter_list_parentheses = false -csharp_space_between_method_call_empty_parameter_list_parentheses = false -csharp_space_between_method_call_name_and_opening_parenthesis = false -csharp_space_after_comma = true -csharp_space_before_comma = false -csharp_space_after_dot = false -csharp_space_before_dot = false -csharp_space_after_semicolon_in_for_statement = true -csharp_space_before_semicolon_in_for_statement = false -csharp_space_around_declaration_statements = false -csharp_space_before_open_square_brackets = false -csharp_space_between_empty_square_brackets = false -csharp_space_between_square_brackets = false - -# Wrap options -csharp_preserve_single_line_statements = false -csharp_preserve_single_line_blocks = true - -# Namespace options -csharp_style_namespace_declarations = file_scoped:warning - -########################################## -# .NET Naming Rules -########################################## -[*.{cs,csx,cake,vb,vbx}] - -# Allow underscores in names. -dotnet_diagnostic.CA1707.severity = none - -# Styles -dotnet_naming_style.pascal_case_style.capitalization = pascal_case - -dotnet_naming_style.upper_case_style.capitalization = all_upper -dotnet_naming_style.upper_case_style.word_separator = _ - -dotnet_naming_style.camel_case_style.capitalization = camel_case - -dotnet_naming_style.camel_case_underscore_style.required_prefix = _ -dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case - -# Use uppercase for all constant fields. -dotnet_naming_rule.constants_uppercase.severity = suggestion -dotnet_naming_rule.constants_uppercase.symbols = constant_fields -dotnet_naming_rule.constants_uppercase.style = upper_case_style -dotnet_naming_symbols.constant_fields.applicable_kinds = field -dotnet_naming_symbols.constant_fields.applicable_accessibilities = * -dotnet_naming_symbols.constant_fields.required_modifiers = const - -# Non-public fields should be _camelCase -dotnet_naming_rule.non_public_fields_under_camel.severity = suggestion -dotnet_naming_rule.non_public_fields_under_camel.symbols = non_public_fields -dotnet_naming_rule.non_public_fields_under_camel.style = camel_case_underscore_style -dotnet_naming_symbols.non_public_fields.applicable_kinds = field -dotnet_naming_symbols.non_public_fields.required_modifiers = -dotnet_naming_symbols.non_public_fields.applicable_accessibilities = private,private_protected,internal,protected,protected_internal - -# Public fields should be PascalCase -dotnet_naming_rule.public_fields_pascal.severity = suggestion -dotnet_naming_rule.public_fields_pascal.symbols = public_fields -dotnet_naming_rule.public_fields_pascal.style = pascal_case_style -dotnet_naming_symbols.public_fields.applicable_kinds = field -dotnet_naming_symbols.public_fields.required_modifiers = -dotnet_naming_symbols.public_fields.applicable_accessibilities = public - -# Async methods should have "Async" suffix. -# Disabled because it makes tests too verbose. -# dotnet_naming_style.end_in_async.required_suffix = Async -# dotnet_naming_style.end_in_async.capitalization = pascal_case -# dotnet_naming_rule.methods_end_in_async.symbols = methods_async -# dotnet_naming_rule.methods_end_in_async.style = end_in_async -# dotnet_naming_rule.methods_end_in_async.severity = warning -# dotnet_naming_symbols.methods_async.applicable_kinds = method -# dotnet_naming_symbols.methods_async.required_modifiers = async -# dotnet_naming_symbols.methods_async.applicable_accessibilities = * - -########################################## -# Other Naming Rules -########################################## - -# All of the following must be PascalCase: -dotnet_naming_symbols.element_group.applicable_kinds = namespace, class, enum, struct, delegate, event, method, property -dotnet_naming_rule.element_rule.symbols = element_group -dotnet_naming_rule.element_rule.style = pascal_case_style -dotnet_naming_rule.element_rule.severity = warning - -# Interfaces use PascalCase and are prefixed with uppercase 'I' -# https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-classes-structs-and-interfaces -dotnet_naming_style.prefix_interface_with_i_style.capitalization = pascal_case -dotnet_naming_style.prefix_interface_with_i_style.required_prefix = I -dotnet_naming_symbols.interface_group.applicable_kinds = interface -dotnet_naming_rule.interface_rule.symbols = interface_group -dotnet_naming_rule.interface_rule.style = prefix_interface_with_i_style -dotnet_naming_rule.interface_rule.severity = warning - -# Generics Type Parameters use PascalCase and are prefixed with uppercase 'T' -# https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-classes-structs-and-interfaces -dotnet_naming_style.prefix_type_parameters_with_t_style.capitalization = pascal_case -dotnet_naming_style.prefix_type_parameters_with_t_style.required_prefix = T -dotnet_naming_symbols.type_parameter_group.applicable_kinds = type_parameter -dotnet_naming_rule.type_parameter_rule.symbols = type_parameter_group -dotnet_naming_rule.type_parameter_rule.style = prefix_type_parameters_with_t_style -dotnet_naming_rule.type_parameter_rule.severity = warning - -# Function parameters use camelCase -# https://docs.microsoft.com/dotnet/standard/design-guidelines/naming-parameters -dotnet_naming_symbols.parameters_group.applicable_kinds = parameter -dotnet_naming_rule.parameters_rule.symbols = parameters_group -dotnet_naming_rule.parameters_rule.style = camel_case_style -dotnet_naming_rule.parameters_rule.severity = warning - -# Anything not specified uses camel case. -dotnet_naming_rule.unspecified_naming.severity = warning -dotnet_naming_rule.unspecified_naming.symbols = unspecified -dotnet_naming_rule.unspecified_naming.style = camel_case_style -dotnet_naming_symbols.unspecified.applicable_kinds = * -dotnet_naming_symbols.unspecified.applicable_accessibilities = * - -########################################## -# Rule Overrides -########################################## - -roslyn_correctness.assembly_reference_validation = relaxed - -# Allow using keywords as names -# dotnet_diagnostic.CA1716.severity = none -# Don't require culture info for ToString() -dotnet_diagnostic.CA1304.severity = none -# Don't require a string comparison for comparing strings. -dotnet_diagnostic.CA1310.severity = none -# Don't require a string format specifier. -dotnet_diagnostic.CA1305.severity = none -# Allow protected fields. -dotnet_diagnostic.CA1051.severity = none -# Don't warn about checking values that are supposedly never null. Sometimes -# they are actually null. -dotnet_diagnostic.CS8073.severity = none -# Don't remove seemingly "unnecessary" assignments, as they often have -# intended side-effects. -dotnet_diagnostic.IDE0059.severity = none -# Switch/case should always have a default clause. Tell that to Roslynator. -dotnet_diagnostic.RCS1070.severity = none -# Tell roslynator not to eat unused parameters. -dotnet_diagnostic.RCS1163.severity = none -# Tell dotnet not to remove unused parameters. -dotnet_diagnostic.IDE0060.severity = none -# Tell roslynator not to remove `partial` modifiers. -dotnet_diagnostic.RCS1043.severity = none -# Tell roslynator not to make classes static so aggressively. -dotnet_diagnostic.RCS1102.severity = none -# Roslynator wants to make properties readonly all the time, so stop it. -# The developer knows best when it comes to contract definitions with Godot. -dotnet_diagnostic.RCS1170.severity = none -# Allow expression values to go unused, even without discard variable. -# Otherwise, using Moq would be way too verbose. -dotnet_diagnostic.IDE0058.severity = none -# Don't let roslynator turn every local variable into a const. -# If we did, we'd have to specify the types of local variables far more often, -# and this style prefers type inference. -dotnet_diagnostic.RCS1118.severity = none -# Enums don't need to declare explicit values. Everyone knows they start at 0. -dotnet_diagnostic.RCS1161.severity = none -# Allow unconstrained type parameter to be checked for null. -dotnet_diagnostic.RCS1165.severity = none -# Allow keyword-based names so that parameter names like `@event` can be used. -dotnet_diagnostic.CA1716.severity = none -# Allow me to use the word Collection if I want. -dotnet_diagnostic.CA1711.severity = none -# Not disposing of objects in a test is normal within Godot because of scene tree stuff. -dotnet_diagnostic.CA1001.severity = none -# No primary constructors — not supported well by tooling. -dotnet_diagnostic.IDE0290.severity = none -# Let me comment where I like -dotnet_diagnostic.RCS1181.severity = none -# Let me write dumb if checks, keeps it readable -dotnet_diagnostic.IDE0046.severity = none -# Don't make me use expression bodies for methods -dotnet_diagnostic.IDE0022.severity = none -# Don't use collection shorhand. -dotnet_diagnostic.IDE0300.severity = none -dotnet_diagnostic.IDE0028.severity = none -dotnet_diagnostic.IDE0305.severity = none -# Don't make me populate a switch expression redundantly -dotnet_diagnostic.IDE0072.severity = none -# Leave me alone about primary constructors -dotnet_diagnostic.IDE0290.severity = none diff --git a/corebrain/corebrain/wrappers/csharp/.gitignore b/corebrain/corebrain/wrappers/csharp/.gitignore deleted file mode 100644 index d37d4fc..0000000 --- a/corebrain/corebrain/wrappers/csharp/.gitignore +++ /dev/null @@ -1,417 +0,0 @@ -# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,csharp -# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,csharp - -### Csharp ### -## Ignore Visual Studio temporary files, build results, and -## files generated by popular Visual Studio add-ons. -## -## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore - -# User-specific files -*.rsuser -*.suo -*.user -*.userosscache -*.sln.docstates - -# User-specific files (MonoDevelop/Xamarin Studio) -*.userprefs - -# Mono auto generated files -mono_crash.* - -# Build results -[Dd]ebug/ -[Dd]ebugPublic/ -[Rr]elease/ -[Rr]eleases/ -x64/ -x86/ -[Ww][Ii][Nn]32/ -[Aa][Rr][Mm]/ -[Aa][Rr][Mm]64/ -bld/ -[Bb]in/ -[Oo]bj/ -[Ll]og/ -[Ll]ogs/ - -# Visual Studio 2015/2017 cache/options directory -.vs/ -# Uncomment if you have tasks that create the project's static files in wwwroot -#wwwroot/ - -# Visual Studio 2017 auto generated files -Generated\ Files/ - -# MSTest test Results -[Tt]est[Rr]esult*/ -[Bb]uild[Ll]og.* - -# NUnit -*.VisualState.xml -TestResult.xml -nunit-*.xml - -# Build Results of an ATL Project -[Dd]ebugPS/ -[Rr]eleasePS/ -dlldata.c - -# Benchmark Results -BenchmarkDotNet.Artifacts/ - -# .NET Core -project.lock.json -project.fragment.lock.json -artifacts/ - -# ASP.NET Scaffolding -ScaffoldingReadMe.txt - -# StyleCop -StyleCopReport.xml - -# Files built by Visual Studio -*_i.c -*_p.c -*_h.h -*.ilk -*.meta -*.obj -*.iobj -*.pch -*.pdb -*.ipdb -*.pgc -*.pgd -*.rsp -*.sbr -*.tlb -*.tli -*.tlh -*.tmp -*.tmp_proj -*_wpftmp.csproj -*.log -*.tlog -*.vspscc -*.vssscc -.builds -*.pidb -*.svclog -*.scc - -# Chutzpah Test files -_Chutzpah* - -# Visual C++ cache files -ipch/ -*.aps -*.ncb -*.opendb -*.opensdf -*.sdf -*.cachefile -*.VC.db -*.VC.VC.opendb - -# Visual Studio profiler -*.psess -*.vsp -*.vspx -*.sap - -# Visual Studio Trace Files -*.e2e - -# TFS 2012 Local Workspace -$tf/ - -# Guidance Automation Toolkit -*.gpState - -# ReSharper is a .NET coding add-in -_ReSharper*/ -*.[Rr]e[Ss]harper -*.DotSettings.user - -# TeamCity is a build add-in -_TeamCity* - -# DotCover is a Code Coverage Tool -*.dotCover - -# AxoCover is a Code Coverage Tool -.axoCover/* -!.axoCover/settings.json - -# Coverlet is a free, cross platform Code Coverage Tool -coverage*.json -coverage*.xml -coverage*.info - -# Visual Studio code coverage results -*.coverage -*.coveragexml - -# NCrunch -_NCrunch_* -.*crunch*.local.xml -nCrunchTemp_* - -# MightyMoose -*.mm.* -AutoTest.Net/ - -# Web workbench (sass) -.sass-cache/ - -# Installshield output folder -[Ee]xpress/ - -# DocProject is a documentation generator add-in -DocProject/buildhelp/ -DocProject/Help/*.HxT -DocProject/Help/*.HxC -DocProject/Help/*.hhc -DocProject/Help/*.hhk -DocProject/Help/*.hhp -DocProject/Help/Html2 -DocProject/Help/html - -# Click-Once directory -publish/ - -# Publish Web Output -*.[Pp]ublish.xml -*.azurePubxml -# Note: Comment the next line if you want to checkin your web deploy settings, -# but database connection strings (with potential passwords) will be unencrypted -*.pubxml -*.publishproj - -# Microsoft Azure Web App publish settings. Comment the next line if you want to -# checkin your Azure Web App publish settings, but sensitive information contained -# in these scripts will be unencrypted -PublishScripts/ - -# NuGet Packages -*.nupkg -# NuGet Symbol Packages -*.snupkg -# The packages folder can be ignored because of Package Restore -**/[Pp]ackages/* -# except build/, which is used as an MSBuild target. -!**/[Pp]ackages/build/ -# Uncomment if necessary however generally it will be regenerated when needed -#!**/[Pp]ackages/repositories.config -# NuGet v3's project.json files produces more ignorable files -*.nuget.props -*.nuget.targets - -# Microsoft Azure Build Output -csx/ -*.build.csdef - -# Microsoft Azure Emulator -ecf/ -rcf/ - -# Windows Store app package directories and files -AppPackages/ -BundleArtifacts/ -Package.StoreAssociation.xml -_pkginfo.txt -*.appx -*.appxbundle -*.appxupload - -# Visual Studio cache files -# files ending in .cache can be ignored -*.[Cc]ache -# but keep track of directories ending in .cache -!?*.[Cc]ache/ - -# Others -ClientBin/ -~$* -*~ -*.dbmdl -*.dbproj.schemaview -*.jfm -*.pfx -*.publishsettings -orleans.codegen.cs - -# Including strong name files can present a security risk -# (https://github.com/github/gitignore/pull/2483#issue-259490424) -#*.snk - -# Since there are multiple workflows, uncomment next line to ignore bower_components -# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) -#bower_components/ - -# RIA/Silverlight projects -Generated_Code/ - -# Backup & report files from converting an old project file -# to a newer Visual Studio version. Backup files are not needed, -# because we have git ;-) -_UpgradeReport_Files/ -Backup*/ -UpgradeLog*.XML -UpgradeLog*.htm -ServiceFabricBackup/ -*.rptproj.bak - -# SQL Server files -*.mdf -*.ldf -*.ndf - -# Business Intelligence projects -*.rdl.data -*.bim.layout -*.bim_*.settings -*.rptproj.rsuser -*- [Bb]ackup.rdl -*- [Bb]ackup ([0-9]).rdl -*- [Bb]ackup ([0-9][0-9]).rdl - -# Microsoft Fakes -FakesAssemblies/ - -# GhostDoc plugin setting file -*.GhostDoc.xml - -# Node.js Tools for Visual Studio -.ntvs_analysis.dat -node_modules/ - -# Visual Studio 6 build log -*.plg - -# Visual Studio 6 workspace options file -*.opt - -# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) -*.vbw - -# Visual Studio 6 auto-generated project file (contains which files were open etc.) -*.vbp - -# Visual Studio 6 workspace and project file (working project files containing files to include in project) -*.dsw -*.dsp - -# Visual Studio 6 technical files - -# Visual Studio LightSwitch build output -**/*.HTMLClient/GeneratedArtifacts -**/*.DesktopClient/GeneratedArtifacts -**/*.DesktopClient/ModelManifest.xml -**/*.Server/GeneratedArtifacts -**/*.Server/ModelManifest.xml -_Pvt_Extensions - -# Paket dependency manager -.paket/paket.exe -paket-files/ - -# FAKE - F# Make -.fake/ - -# CodeRush personal settings -.cr/personal - -# Python Tools for Visual Studio (PTVS) -__pycache__/ -*.pyc - -# Cake - Uncomment if you are using it -# tools/** -# !tools/packages.config - -# Tabs Studio -*.tss - -# Telerik's JustMock configuration file -*.jmconfig - -# BizTalk build output -*.btp.cs -*.btm.cs -*.odx.cs -*.xsd.cs - -# OpenCover UI analysis results -OpenCover/ - -# Azure Stream Analytics local run output -ASALocalRun/ - -# MSBuild Binary and Structured Log -*.binlog - -# NVidia Nsight GPU debugger configuration file -*.nvuser - -# MFractors (Xamarin productivity tool) working folder -.mfractor/ - -# Local History for Visual Studio -.localhistory/ - -# Visual Studio History (VSHistory) files -.vshistory/ - -# BeatPulse healthcheck temp database -healthchecksdb - -# Backup folder for Package Reference Convert tool in Visual Studio 2017 -MigrationBackup/ - -# Ionide (cross platform F# VS Code tools) working folder -.ionide/ - -# Fody - auto-generated XML schema -FodyWeavers.xsd - -# VS Code files for those working on multiple tools -.vscode/* -!.vscode/settings.json -!.vscode/tasks.json -!.vscode/launch.json -!.vscode/extensions.json -*.code-workspace - -.venv - -# Local History for Visual Studio Code -.history/ - -# Windows Installer files from build outputs -*.cab -*.msi -*.msix -*.msm -*.msp - -# JetBrains Rider -*.sln.iml - -### VisualStudioCode ### -!.vscode/*.code-snippets - -# Local History for Visual Studio Code - -# Built Visual Studio Code Extensions -*.vsix - -### VisualStudioCode Patch ### -# Ignore all local history of files -.history -.ionide - -# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,csharp \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp/.vscode/settings.json b/corebrain/corebrain/wrappers/csharp/.vscode/settings.json deleted file mode 100644 index 23c1fea..0000000 --- a/corebrain/corebrain/wrappers/csharp/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "dotnet.defaultSolution": "CorebrainCS.sln" -} \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp/.vscode/tasks.json b/corebrain/corebrain/wrappers/csharp/.vscode/tasks.json deleted file mode 100644 index f058896..0000000 --- a/corebrain/corebrain/wrappers/csharp/.vscode/tasks.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "version": "2.0.0", - "tasks": [ - { - "label": "Build Project", - "command": "dotnet", - "type": "process", - "args": [ - "build", - "${workspaceFolder}/CorebrainCS.Tests/CorebrainCS.Tests.csproj", - "--configuration", - "Release" - ], - "problemMatcher": "$msCompile", - "group": { - "kind": "build", - "isDefault": true - } - }, - { - "label": "Run Project", - "command": "dotnet", - "type": "process", - "args": [ - "run", - "--project", - "${workspaceFolder}/CorebrainCS.Tests/CorebrainCS.Tests.csproj" - ], - "dependsOn": ["Build Project"] - }, - ] -} \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/CorebrainCS.Tests.csproj b/corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/CorebrainCS.Tests.csproj deleted file mode 100644 index 6e20930..0000000 --- a/corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/CorebrainCS.Tests.csproj +++ /dev/null @@ -1,14 +0,0 @@ - - - - Exe - net9.0 - enable - enable - - - - - - - diff --git a/corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs b/corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs deleted file mode 100644 index d58d0fa..0000000 --- a/corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/Program.cs +++ /dev/null @@ -1,10 +0,0 @@ -using CorebrainCS; - -Console.WriteLine("Hello, World!"); - -// For now it only works on windows -var corebrain = new CorebrainCS.CorebrainCS("../../../../venv/Scripts/python.exe", "../../../cli", false); - -Console.WriteLine(corebrain.Version()); - - diff --git a/corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/README.md b/corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/README.md deleted file mode 100644 index 9aa8159..0000000 --- a/corebrain/corebrain/wrappers/csharp/CorebrainCS.Tests/README.md +++ /dev/null @@ -1,4 +0,0 @@ -### Quick start - -* Create venv in the root directory and install all the dependencies. The instalation guide is in corebrain README.md -* Go to the CorebrainCS.Tests directory to see how the program runs and run `dotnet run` \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp/CorebrainCS.sln b/corebrain/corebrain/wrappers/csharp/CorebrainCS.sln deleted file mode 100644 index e9542bb..0000000 --- a/corebrain/corebrain/wrappers/csharp/CorebrainCS.sln +++ /dev/null @@ -1,48 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.0.31903.59 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CorebrainCS", "CorebrainCS\CorebrainCS.csproj", "{152890AC-4B76-42F7-813B-CB7F3F902B9F}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CorebrainCS.Tests", "CorebrainCS.Tests\CorebrainCS.Tests.csproj", "{664BB3EB-0364-4989-879A-D8CCDBCF6B89}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 - Release|Any CPU = Release|Any CPU - Release|x64 = Release|x64 - Release|x86 = Release|x86 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Debug|x64.ActiveCfg = Debug|Any CPU - {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Debug|x64.Build.0 = Debug|Any CPU - {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Debug|x86.ActiveCfg = Debug|Any CPU - {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Debug|x86.Build.0 = Debug|Any CPU - {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Release|Any CPU.Build.0 = Release|Any CPU - {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Release|x64.ActiveCfg = Release|Any CPU - {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Release|x64.Build.0 = Release|Any CPU - {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Release|x86.ActiveCfg = Release|Any CPU - {152890AC-4B76-42F7-813B-CB7F3F902B9F}.Release|x86.Build.0 = Release|Any CPU - {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Debug|Any CPU.Build.0 = Debug|Any CPU - {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Debug|x64.ActiveCfg = Debug|Any CPU - {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Debug|x64.Build.0 = Debug|Any CPU - {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Debug|x86.ActiveCfg = Debug|Any CPU - {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Debug|x86.Build.0 = Debug|Any CPU - {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Release|Any CPU.ActiveCfg = Release|Any CPU - {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Release|Any CPU.Build.0 = Release|Any CPU - {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Release|x64.ActiveCfg = Release|Any CPU - {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Release|x64.Build.0 = Release|Any CPU - {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Release|x86.ActiveCfg = Release|Any CPU - {664BB3EB-0364-4989-879A-D8CCDBCF6B89}.Release|x86.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal diff --git a/corebrain/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs b/corebrain/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs deleted file mode 100644 index c685d3f..0000000 --- a/corebrain/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs +++ /dev/null @@ -1,175 +0,0 @@ -namespace CorebrainCS; - -using System; -using System.Diagnostics; - -/// -/// Creates the main corebrain interface. -/// -/// Path to the python which works with the corebrain cli, for example if you create the ./.venv you pass the path to the ./.venv python executable -/// Path to the corebrain cli script, if you installed it globally you just pass the `corebrain` path -/// -public class CorebrainCS(string pythonPath = "python", string scriptPath = "corebrain", bool verbose = false) { - private readonly string _pythonPath = Path.GetFullPath(pythonPath); - private readonly string _scriptPath = Path.GetFullPath(scriptPath); - private readonly bool _verbose = verbose; - - - public string Help() { - return ExecuteCommand("--help"); - } - - public string Version() { - return ExecuteCommand("--version"); - } - - public string Configure() { - return ExecuteCommand("--configure"); - } - - public string ListConfigs() { - return ExecuteCommand("--list-configs"); - } - - public string RemoveConfig() { - return ExecuteCommand("--remove-config"); - } - - public string ShowSchema() { - return ExecuteCommand("--show-schema"); - } - - public string ExtractSchema() { - return ExecuteCommand("--extract-schema"); - } - - public string ExtractSchemaToDefaultFile() { - return ExecuteCommand("--extract-schema --output-file test"); - } - - public string ConfigID() { - return ExecuteCommand("--extract-schema --config-id config"); - } - - public string SetToken(string token) { - return ExecuteCommand($"--token {token}"); - } - - public string ApiKey(string apikey) { - return ExecuteCommand($"--api-key {apikey}"); - } - - public string ApiUrl(string apiurl) { - if (string.IsNullOrWhiteSpace(apiurl)) { - throw new ArgumentException("API URL cannot be empty or whitespace", nameof(apiurl)); - } - - if (!Uri.TryCreate(apiurl, UriKind.Absolute, out var uriResult) || - (uriResult.Scheme != Uri.UriSchemeHttp && uriResult.Scheme != Uri.UriSchemeHttps)) { - throw new ArgumentException("Invalid API URL format. Must be a valid HTTP/HTTPS URL", nameof(apiurl)); - } - - var escapedUrl = apiurl.Replace("\"", "\\\""); - return ExecuteCommand($"--api-url \"{escapedUrl}\""); - } - public string SsoUrl(string ssoUrl) { - if (string.IsNullOrWhiteSpace(ssoUrl)) { - throw new ArgumentException("SSO URL cannot be empty or whitespace", nameof(ssoUrl)); - } - - if (!Uri.TryCreate(ssoUrl, UriKind.Absolute, out var uriResult) || - (uriResult.Scheme != Uri.UriSchemeHttp && uriResult.Scheme != Uri.UriSchemeHttps)) { - throw new ArgumentException("Invalid SSO URL format. Must be a valid HTTP/HTTPS URL", nameof(ssoUrl)); - } - - var escapedUrl = ssoUrl.Replace("\"", "\\\""); - return ExecuteCommand($"--sso-url \"{escapedUrl}\""); - } - public string Login(string username, string password){ - if (string.IsNullOrWhiteSpace(username)){ - throw new ArgumentException("Username cannot be empty or whitespace", nameof(username)); - } - - if (string.IsNullOrWhiteSpace(password)){ - throw new ArgumentException("Password cannot be empty or whitespace", nameof(password)); - } - - var escapedUsername = username.Replace("\"", "\\\""); - var escapedPassword = password.Replace("\"", "\\\""); - - return ExecuteCommand($"--login --username \"{escapedUsername}\" --password \"{escapedPassword}\""); - } - - public string LoginWithToken(string token) { - if (string.IsNullOrWhiteSpace(token)) { - throw new ArgumentException("Token cannot be empty or whitespace", nameof(token)); - } - - var escapedToken = token.Replace("\"", "\\\""); - return ExecuteCommand($"--login --token \"{escapedToken}\""); - } - - //When youre logged in use this function - public string TestAuth() { - return ExecuteCommand("--test-auth"); - } - - //Without beeing logged - public string TestAuth(string? apiUrl = null, string? token = null) { - var args = new List { "--test-auth" }; - - if (!string.IsNullOrEmpty(apiUrl)) { - if (!Uri.IsWellFormedUriString(apiUrl, UriKind.Absolute)) - throw new ArgumentException("Invalid API URL format", nameof(apiUrl)); - - args.Add($"--api-url \"{apiUrl}\""); - } - - if (!string.IsNullOrEmpty(token)) - args.Add($"--token \"{token}\""); - - return ExecuteCommand(string.Join(" ", args)); - } - public string ExecuteCommand(string arguments) - { - if (_verbose) - { - Console.WriteLine($"Executing: {_pythonPath} {_scriptPath} {arguments}"); - } - - var process = new Process - { - StartInfo = new ProcessStartInfo - { - FileName = _pythonPath, - Arguments = $"\"{_scriptPath}\" {arguments}", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - } - }; - - process.Start(); - var output = process.StandardOutput.ReadToEnd(); - var error = process.StandardError.ReadToEnd(); - process.WaitForExit(); - - if (_verbose) - { - Console.WriteLine("Command output:"); - Console.WriteLine(output); - if (!string.IsNullOrEmpty(error)) - { - Console.WriteLine("Error output:\n" + error); - } - } - - if (!string.IsNullOrEmpty(error)) - { - throw new InvalidOperationException($"Python CLI error: {error}"); - } - - return output.Trim(); - } -} \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.csproj b/corebrain/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.csproj deleted file mode 100644 index bf9d3be..0000000 --- a/corebrain/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.csproj +++ /dev/null @@ -1,7 +0,0 @@ - - - net9.0 - enable - enable - - diff --git a/corebrain/corebrain/wrappers/csharp/LICENSE b/corebrain/corebrain/wrappers/csharp/LICENSE deleted file mode 100644 index 8e423f6..0000000 --- a/corebrain/corebrain/wrappers/csharp/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2025 Oliwier Adamczyk - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/corebrain/corebrain/wrappers/csharp/README.md b/corebrain/corebrain/wrappers/csharp/README.md deleted file mode 100644 index 3d113c5..0000000 --- a/corebrain/corebrain/wrappers/csharp/README.md +++ /dev/null @@ -1,77 +0,0 @@ -# CoreBrain-CS - -[![NuGet Version](https://img.shields.io/nuget/v/CorebrainCS.svg)](https://www.nuget.org/packages/CorebrainCS/) -[![Python Requirement](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/) - -A C# wrapper for the CoreBrain Python CLI tool, providing seamless integration between .NET applications and CoreBrain's cognitive computing capabilities. - -## Features - -- 🚀 Native C# interface for CoreBrain functions -- 🛠️ Supports both development and production workflows - -## Installation - -### Prerequisites - -- [.NET 8.0 SDK](https://dotnet.microsoft.com/download) -- [Python 3.8+](https://www.python.org/downloads/) - -## Corebrain installation - -See the main corebrain package installation on https://github.com/ceoweggo/Corebrain/blob/main/README.md#installation - -## Basic Usage - -```csharp -using CorebrainCS; - -// Initialize wrapper (auto-detects Python environment) -var corebrain = new CorebrainCS(); - -// Get version -Console.WriteLine($"CoreBrain version: {corebrain.Version()}"); -``` - -## Advanced Configuration - -```csharp -// Custom configuration -var corebrain = new CorebrainCS( - pythonPath: "path/to/python", // Custom python path - scriptPath: "path/to/cli", // Custom CoreBrain CLI path - verbose: true // Enable debug logging -); -``` - -## Common Commands - -| Command | C# Method | Description | -|---------|-----------|-------------| -| `--version` | `.Version()` | Get CoreBrain version | - - - -### File Structure - -``` -Corebrain-CS/ -├── CorebrainCS/ # C# wrapper library -├── CorebrainCLI/ # Example consumer app -├── corebrain/ # Embedded Python package -``` - -## License - -MIT License - See [LICENSE](LICENSE) for details. diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/.gitignore b/corebrain/corebrain/wrappers/csharp_cli_api/.gitignore deleted file mode 100644 index 5a8763b..0000000 --- a/corebrain/corebrain/wrappers/csharp_cli_api/.gitignore +++ /dev/null @@ -1,548 +0,0 @@ -# Created by https://www.toptal.com/developers/gitignore/api/csharp,dotnetcore,aspnetcore,visualstudiocode -# Edit at https://www.toptal.com/developers/gitignore?templates=csharp,dotnetcore,aspnetcore,visualstudiocode - -### ASPNETCore ### -## Ignore Visual Studio temporary files, build results, and -## files generated by popular Visual Studio add-ons. - -# User-specific files -*.suo -*.user -*.userosscache -*.sln.docstates - -# User-specific files (MonoDevelop/Xamarin Studio) -*.userprefs - -# Build results -[Dd]ebug/ -[Dd]ebugPublic/ -[Rr]elease/ -[Rr]eleases/ -x64/ -x86/ -bld/ -[Bb]in/ -[Oo]bj/ -[Ll]og/ - -# Visual Studio 2015 cache/options directory -.vs/ -# Uncomment if you have tasks that create the project's static files in wwwroot -#wwwroot/ - -# MSTest test Results -[Tt]est[Rr]esult*/ -[Bb]uild[Ll]og.* - -# NUNIT -*.VisualState.xml -TestResult.xml - -# Build Results of an ATL Project -[Dd]ebugPS/ -[Rr]eleasePS/ -dlldata.c - -# DNX -project.lock.json -project.fragment.lock.json -artifacts/ - -*_i.c -*_p.c -*_i.h -*.ilk -*.meta -*.obj -*.pch -*.pdb -*.pgc -*.pgd -*.rsp -*.sbr -*.tlb -*.tli -*.tlh -*.tmp -*.tmp_proj -*.log -*.vspscc -*.vssscc -.builds -*.pidb -*.svclog -*.scc - -# Chutzpah Test files -_Chutzpah* - -# Visual C++ cache files -ipch/ -*.aps -*.ncb -*.opendb -*.opensdf -*.sdf -*.cachefile -*.VC.db -*.VC.VC.opendb - -# Visual Studio profiler -*.psess -*.vsp -*.vspx -*.sap - -# TFS 2012 Local Workspace -$tf/ - -# Guidance Automation Toolkit -*.gpState - -# ReSharper is a .NET coding add-in -_ReSharper*/ -*.[Rr]e[Ss]harper -*.DotSettings.user - -# JustCode is a .NET coding add-in -.JustCode - -# TeamCity is a build add-in -_TeamCity* - -# DotCover is a Code Coverage Tool -*.dotCover - -# Visual Studio code coverage results -*.coverage -*.coveragexml - -# NCrunch -_NCrunch_* -.*crunch*.local.xml -nCrunchTemp_* - -# MightyMoose -*.mm.* -AutoTest.Net/ - -# Web workbench (sass) -.sass-cache/ - -# Installshield output folder -[Ee]xpress/ - -# DocProject is a documentation generator add-in -DocProject/buildhelp/ -DocProject/Help/*.HxT -DocProject/Help/*.HxC -DocProject/Help/*.hhc -DocProject/Help/*.hhk -DocProject/Help/*.hhp -DocProject/Help/Html2 -DocProject/Help/html - -# Click-Once directory -publish/ - -# Publish Web Output -*.[Pp]ublish.xml -*.azurePubxml -# TODO: Comment the next line if you want to checkin your web deploy settings -# but database connection strings (with potential passwords) will be unencrypted -*.pubxml -*.publishproj - -# Microsoft Azure Web App publish settings. Comment the next line if you want to -# checkin your Azure Web App publish settings, but sensitive information contained -# in these scripts will be unencrypted -PublishScripts/ - -# NuGet Packages -*.nupkg -# The packages folder can be ignored because of Package Restore -**/packages/* -# except build/, which is used as an MSBuild target. -!**/packages/build/ -# Uncomment if necessary however generally it will be regenerated when needed -#!**/packages/repositories.config -# NuGet v3's project.json files produces more ignoreable files -*.nuget.props -*.nuget.targets - -# Microsoft Azure Build Output -csx/ -*.build.csdef - -# Microsoft Azure Emulator -ecf/ -rcf/ - -# Windows Store app package directories and files -AppPackages/ -BundleArtifacts/ -Package.StoreAssociation.xml -_pkginfo.txt - -# Visual Studio cache files -# files ending in .cache can be ignored -*.[Cc]ache -# but keep track of directories ending in .cache -!*.[Cc]ache/ - -# Others -ClientBin/ -~$* -*~ -*.dbmdl -*.dbproj.schemaview -*.jfm -*.pfx -*.publishsettings -node_modules/ -orleans.codegen.cs - -# Since there are multiple workflows, uncomment next line to ignore bower_components -# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) -#bower_components/ - -# RIA/Silverlight projects -Generated_Code/ - -# Backup & report files from converting an old project file -# to a newer Visual Studio version. Backup files are not needed, -# because we have git ;-) -_UpgradeReport_Files/ -Backup*/ -UpgradeLog*.XML -UpgradeLog*.htm - -# SQL Server files -*.mdf -*.ldf - -# Business Intelligence projects -*.rdl.data -*.bim.layout -*.bim_*.settings - -# Microsoft Fakes -FakesAssemblies/ - -# GhostDoc plugin setting file -*.GhostDoc.xml - -# Node.js Tools for Visual Studio -.ntvs_analysis.dat - -# Visual Studio 6 build log -*.plg - -# Visual Studio 6 workspace options file -*.opt - -# Visual Studio LightSwitch build output -**/*.HTMLClient/GeneratedArtifacts -**/*.DesktopClient/GeneratedArtifacts -**/*.DesktopClient/ModelManifest.xml -**/*.Server/GeneratedArtifacts -**/*.Server/ModelManifest.xml -_Pvt_Extensions - -# Paket dependency manager -.paket/paket.exe -paket-files/ - -# FAKE - F# Make -.fake/ - -# JetBrains Rider -.idea/ -*.sln.iml - -# CodeRush -.cr/ - -# Python Tools for Visual Studio (PTVS) -__pycache__/ -*.pyc - -# Cake - Uncomment if you are using it -# tools/ - -### Csharp ### -## -## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore - -# User-specific files -*.rsuser - -# User-specific files (MonoDevelop/Xamarin Studio) - -# Mono auto generated files -mono_crash.* - -# Build results -[Ww][Ii][Nn]32/ -[Aa][Rr][Mm]/ -[Aa][Rr][Mm]64/ -[Ll]ogs/ - -# Visual Studio 2015/2017 cache/options directory -# Uncomment if you have tasks that create the project's static files in wwwroot - -# Visual Studio 2017 auto generated files -Generated\ Files/ - -# MSTest test Results - -# NUnit -nunit-*.xml - -# Build Results of an ATL Project - -# Benchmark Results -BenchmarkDotNet.Artifacts/ - -# .NET Core - -# ASP.NET Scaffolding -ScaffoldingReadMe.txt - -# StyleCop -StyleCopReport.xml - -# Files built by Visual Studio -*_h.h -*.iobj -*.ipdb -*_wpftmp.csproj -*.tlog - -# Chutzpah Test files - -# Visual C++ cache files - -# Visual Studio profiler - -# Visual Studio Trace Files -*.e2e - -# TFS 2012 Local Workspace - -# Guidance Automation Toolkit - -# ReSharper is a .NET coding add-in - -# TeamCity is a build add-in - -# DotCover is a Code Coverage Tool - -# AxoCover is a Code Coverage Tool -.axoCover/* -!.axoCover/settings.json - -# Coverlet is a free, cross platform Code Coverage Tool -coverage*.json -coverage*.xml -coverage*.info - -# Visual Studio code coverage results - -# NCrunch - -# MightyMoose - -# Web workbench (sass) - -# Installshield output folder - -# DocProject is a documentation generator add-in - -# Click-Once directory - -# Publish Web Output -# Note: Comment the next line if you want to checkin your web deploy settings, -# but database connection strings (with potential passwords) will be unencrypted - -# Microsoft Azure Web App publish settings. Comment the next line if you want to -# checkin your Azure Web App publish settings, but sensitive information contained -# in these scripts will be unencrypted - -# NuGet Packages -# NuGet Symbol Packages -*.snupkg -# The packages folder can be ignored because of Package Restore -**/[Pp]ackages/* -# except build/, which is used as an MSBuild target. -!**/[Pp]ackages/build/ -# Uncomment if necessary however generally it will be regenerated when needed -#!**/[Pp]ackages/repositories.config -# NuGet v3's project.json files produces more ignorable files - -# Microsoft Azure Build Output - -# Microsoft Azure Emulator - -# Windows Store app package directories and files -*.appx -*.appxbundle -*.appxupload - -# Visual Studio cache files -# files ending in .cache can be ignored -# but keep track of directories ending in .cache -!?*.[Cc]ache/ - -# Others - -# Including strong name files can present a security risk -# (https://github.com/github/gitignore/pull/2483#issue-259490424) -#*.snk - -# Since there are multiple workflows, uncomment next line to ignore bower_components -# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) - -# RIA/Silverlight projects - -# Backup & report files from converting an old project file -# to a newer Visual Studio version. Backup files are not needed, -# because we have git ;-) -ServiceFabricBackup/ -*.rptproj.bak - -# SQL Server files -*.ndf - -# Business Intelligence projects -*.rptproj.rsuser -*- [Bb]ackup.rdl -*- [Bb]ackup ([0-9]).rdl -*- [Bb]ackup ([0-9][0-9]).rdl - -# Microsoft Fakes - -# GhostDoc plugin setting file - -# Node.js Tools for Visual Studio - -# Visual Studio 6 build log - -# Visual Studio 6 workspace options file - -# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) -*.vbw - -# Visual Studio 6 auto-generated project file (contains which files were open etc.) -*.vbp - -# Visual Studio 6 workspace and project file (working project files containing files to include in project) -*.dsw -*.dsp - -# Visual Studio 6 technical files - -# Visual Studio LightSwitch build output - -# Paket dependency manager - -# FAKE - F# Make - -# CodeRush personal settings -.cr/personal - -# Python Tools for Visual Studio (PTVS) - -# Cake - Uncomment if you are using it -# tools/** -# !tools/packages.config - -# Tabs Studio -*.tss - -# Telerik's JustMock configuration file -*.jmconfig - -# BizTalk build output -*.btp.cs -*.btm.cs -*.odx.cs -*.xsd.cs - -# OpenCover UI analysis results -OpenCover/ - -# Azure Stream Analytics local run output -ASALocalRun/ - -# MSBuild Binary and Structured Log -*.binlog - -# NVidia Nsight GPU debugger configuration file -*.nvuser - -# MFractors (Xamarin productivity tool) working folder -.mfractor/ - -# Local History for Visual Studio -.localhistory/ - -# Visual Studio History (VSHistory) files -.vshistory/ - -# BeatPulse healthcheck temp database -healthchecksdb - -# Backup folder for Package Reference Convert tool in Visual Studio 2017 -MigrationBackup/ - -# Ionide (cross platform F# VS Code tools) working folder -.ionide/ - -# Fody - auto-generated XML schema -FodyWeavers.xsd - -# VS Code files for those working on multiple tools -.vscode/* -!.vscode/settings.json -!.vscode/tasks.json -!.vscode/launch.json -!.vscode/extensions.json -*.code-workspace - -# Local History for Visual Studio Code -.history/ - -# Windows Installer files from build outputs -*.cab -*.msi -*.msix -*.msm -*.msp - -# JetBrains Rider - -### DotnetCore ### -# .NET Core build folders -bin/ -obj/ - -# Common node modules locations -/node_modules -/wwwroot/node_modules - -### VisualStudioCode ### -!.vscode/*.code-snippets - -# Local History for Visual Studio Code - -# Built Visual Studio Code Extensions -*.vsix - -### VisualStudioCode Patch ### -# Ignore all local history of files -.history -.ionide - -# End of https://www.toptal.com/developers/gitignore/api/csharp,dotnetcore,aspnetcore,visualstudiocode \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/.vscode/settings.json b/corebrain/corebrain/wrappers/csharp_cli_api/.vscode/settings.json deleted file mode 100644 index 8de99dd..0000000 --- a/corebrain/corebrain/wrappers/csharp_cli_api/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "dotnet.defaultSolution": "CorebrainCLIAPI.sln" -} \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/CorebrainCLIAPI.sln b/corebrain/corebrain/wrappers/csharp_cli_api/CorebrainCLIAPI.sln deleted file mode 100644 index 81bb4a4..0000000 --- a/corebrain/corebrain/wrappers/csharp_cli_api/CorebrainCLIAPI.sln +++ /dev/null @@ -1,50 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.0.31903.59 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CorebrainCLIAPI", "src\CorebrainCLIAPI\CorebrainCLIAPI.csproj", "{3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 - Release|Any CPU = Release|Any CPU - Release|x64 = Release|x64 - Release|x86 = Release|x86 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Debug|x64.ActiveCfg = Debug|Any CPU - {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Debug|x64.Build.0 = Debug|Any CPU - {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Debug|x86.ActiveCfg = Debug|Any CPU - {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Debug|x86.Build.0 = Debug|Any CPU - {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Release|Any CPU.ActiveCfg = Release|Any CPU - {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Release|Any CPU.Build.0 = Release|Any CPU - {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Release|x64.ActiveCfg = Release|Any CPU - {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Release|x64.Build.0 = Release|Any CPU - {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Release|x86.ActiveCfg = Release|Any CPU - {3CA0D8CC-EC03-4FE6-93CB-1BCB5D34BB07}.Release|x86.Build.0 = Release|Any CPU - {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Debug|x64.ActiveCfg = Debug|Any CPU - {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Debug|x64.Build.0 = Debug|Any CPU - {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Debug|x86.ActiveCfg = Debug|Any CPU - {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Debug|x86.Build.0 = Debug|Any CPU - {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Release|Any CPU.ActiveCfg = Release|Any CPU - {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Release|Any CPU.Build.0 = Release|Any CPU - {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Release|x64.ActiveCfg = Release|Any CPU - {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Release|x64.Build.0 = Release|Any CPU - {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Release|x86.ActiveCfg = Release|Any CPU - {1B7A4995-2D77-4398-BE28-B3B52C1E351B}.Release|x86.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(NestedProjects) = preSolution - {C5CF7B2F-DA16-24C6-929A-8AB8C4831AB0} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} - {1B7A4995-2D77-4398-BE28-B3B52C1E351B} = {C5CF7B2F-DA16-24C6-929A-8AB8C4831AB0} - EndGlobalSection -EndGlobal diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/README.md b/corebrain/corebrain/wrappers/csharp_cli_api/README.md deleted file mode 100644 index 41b69c8..0000000 --- a/corebrain/corebrain/wrappers/csharp_cli_api/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Corebrain CLI API - -## Quick Start - -### Prerequisites - -- Python 3.8+ -- .NET 6.0+ -- Node.js 14+ -- Git - -### Installation - -1. Create **venv** in corebrain directory -2. Continue with installation provided here https://github.com/ceoweggo/Corebrain/blob/pre-release-v0.2.0/README.md#development-installation -3. If you changed the installation directory of venv or corebrain, change the paths in `CorebrainCLIAPI/appsettings.json` -4. Go to `src/CorebrainCLIAPI` -5. run `dotnet run` \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/src/.editorconfig b/corebrain/corebrain/wrappers/csharp_cli_api/src/.editorconfig deleted file mode 100644 index 0bcaf64..0000000 --- a/corebrain/corebrain/wrappers/csharp_cli_api/src/.editorconfig +++ /dev/null @@ -1,432 +0,0 @@ -# This file is the top-most EditorConfig file -root = true - -#All Files -[*] -charset = utf-8 -indent_style = space -indent_size = 2 -insert_final_newline = true -trim_trailing_whitespace = true -end_of_line = lf - -########################################## -# File Extension Settings -########################################## - -# Visual Studio Solution Files -[*.sln] -indent_style = tab - -# Visual Studio XML Project Files -[*.{csproj,vbproj,vcxproj.filters,proj,projitems,shproj}] -indent_size = 2 - -# XML Configuration Files -[*.{xml,config,props,targets,nuspec,resx,ruleset,vsixmanifest,vsct}] -indent_size = 2 - -# JSON Files -[*.{json,json5,webmanifest}] -indent_size = 2 - -# YAML Files -[*.{yml,yaml}] -indent_size = 2 - -# Markdown Files -[*.{md,mdx}] -trim_trailing_whitespace = false - -# Web Files -[*.{htm,html,js,jsm,ts,tsx,cjs,cts,ctsx,mjs,mts,mtsx,css,sass,scss,less,pcss,svg,vue}] -indent_size = 2 - -# Batch Files -[*.{cmd,bat}] -end_of_line = crlf - -# Bash Files -[*.sh] -end_of_line = lf - -# Makefiles -[Makefile] -indent_style = tab - -[{*_Generated.cs, *.g.cs, *.generated.cs}] -# Ignore a lack of documentation for generated code. Doesn't apply to builds, -# just to viewing generation output. -dotnet_diagnostic.CS1591.severity = none - -########################################## -# Default .NET Code Style Severities -########################################## - -[*.{cs,csx,cake,vb,vbx}] -# Default Severity for all .NET Code Style rules below -dotnet_analyzer_diagnostic.severity = warning - -########################################## -# Language Rules -########################################## - -# .NET Style Rules -[*.{cs,csx,cake,vb,vbx}] - -# "this." and "Me." qualifiers -dotnet_style_qualification_for_field = false -dotnet_style_qualification_for_property = false -dotnet_style_qualification_for_method = false -dotnet_style_qualification_for_event = false - -# Language keywords instead of framework type names for type references -dotnet_style_predefined_type_for_locals_parameters_members = true:warning -dotnet_style_predefined_type_for_member_access = true:warning - -# Modifier preferences -dotnet_style_require_accessibility_modifiers = always:warning -csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:warning -visual_basic_preferred_modifier_order = Partial,Default,Private,Protected,Public,Friend,NotOverridable,Overridable,MustOverride,Overloads,Overrides,MustInherit,NotInheritable,Static,Shared,Shadows,ReadOnly,WriteOnly,Dim,Const,WithEvents,Widening,Narrowing,Custom,Async:warning -dotnet_style_readonly_field = true:warning -dotnet_diagnostic.IDE0036.severity = warning - - -# Parentheses preferences -dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:warning -dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:warning -dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:warning -dotnet_style_parentheses_in_other_operators = never_if_unnecessary:warning - -# Expression-level preferences -dotnet_style_object_initializer = true:warning -dotnet_style_collection_initializer = true:warning -dotnet_style_explicit_tuple_names = true:warning -dotnet_style_prefer_inferred_tuple_names = true:warning -dotnet_style_prefer_inferred_anonymous_type_member_names = true:warning -dotnet_style_prefer_auto_properties = true:warning -dotnet_style_prefer_conditional_expression_over_assignment = true:suggestion -dotnet_diagnostic.IDE0045.severity = suggestion -dotnet_style_prefer_conditional_expression_over_return = true:suggestion -dotnet_diagnostic.IDE0046.severity = suggestion -dotnet_style_prefer_compound_assignment = true:warning -dotnet_style_prefer_simplified_interpolation = true:warning -dotnet_style_prefer_simplified_boolean_expressions = true:warning - -# Null-checking preferences -dotnet_style_coalesce_expression = true:warning -dotnet_style_null_propagation = true:warning -dotnet_style_prefer_is_null_check_over_reference_equality_method = true:warning - -# File header preferences -# Keep operators at end of line when wrapping. -dotnet_style_operator_placement_when_wrapping = end_of_line:warning -csharp_style_prefer_null_check_over_type_check = true:warning - -# Code block preferences -csharp_prefer_braces = true:warning -csharp_prefer_simple_using_statement = true:suggestion -dotnet_diagnostic.IDE0063.severity = suggestion - -# C# Style Rules -[*.{cs,csx,cake}] -# 'var' preferences -csharp_style_var_for_built_in_types = true:warning -csharp_style_var_when_type_is_apparent = true:warning -csharp_style_var_elsewhere = true:warning -# Expression-bodied members -csharp_style_expression_bodied_methods = true:warning -csharp_style_expression_bodied_constructors = false:warning -csharp_style_expression_bodied_operators = true:warning -csharp_style_expression_bodied_properties = true:warning -csharp_style_expression_bodied_indexers = true:warning -csharp_style_expression_bodied_accessors = true:warning -csharp_style_expression_bodied_lambdas = true:warning -csharp_style_expression_bodied_local_functions = true:warning -# Pattern matching preferences -csharp_style_pattern_matching_over_is_with_cast_check = true:warning -csharp_style_pattern_matching_over_as_with_null_check = true:warning -csharp_style_prefer_switch_expression = true:warning -csharp_style_prefer_pattern_matching = true:warning -csharp_style_prefer_not_pattern = true:warning -# Expression-level preferences -csharp_style_inlined_variable_declaration = true:warning -csharp_prefer_simple_default_expression = true:warning -csharp_style_pattern_local_over_anonymous_function = true:warning -csharp_style_deconstructed_variable_declaration = true:warning -csharp_style_prefer_index_operator = true:warning -csharp_style_prefer_range_operator = true:warning -csharp_style_implicit_object_creation_when_type_is_apparent = true:warning -# "Null" checking preferences -csharp_style_throw_expression = true:warning -csharp_style_conditional_delegate_call = true:warning -# Code block preferences -csharp_prefer_braces = true:warning -csharp_prefer_simple_using_statement = true:suggestion -dotnet_diagnostic.IDE0063.severity = suggestion -# 'using' directive preferences -csharp_using_directive_placement = inside_namespace:warning -# Modifier preferences -# Don't suggest making public methods static. Very annoying. -csharp_prefer_static_local_function = false -# Only suggest making private methods static (if they don't use instance data). -dotnet_code_quality.CA1822.api_surface = private - -########################################## -# Unnecessary Code Rules -########################################## - -# .NET Unnecessary code rules -[*.{cs,csx,cake,vb,vbx}] - -dotnet_code_quality_unused_parameters = non_public:suggestion -dotnet_remove_unnecessary_suppression_exclusions = none -dotnet_diagnostic.IDE0079.severity = warning - -# C# Unnecessary code rules -[*.{cs,csx,cake}] - - -# Don't remove method parameters that are unused. -dotnet_diagnostic.IDE0060.severity = none -dotnet_diagnostic.RCS1163.severity = none - -# Don't remove methods that are unused. -dotnet_diagnostic.IDE0051.severity = none -dotnet_diagnostic.RCS1213.severity = none - -# Use discard variable for unused expression values. -csharp_style_unused_value_expression_statement_preference = discard_variable - -# .NET formatting rules -[*.{cs,csx,cake,vb,vbx}] - -# Organize using directives -dotnet_sort_system_directives_first = true -dotnet_separate_import_directive_groups = false - -dotnet_sort_accessibility = true - -# Dotnet namespace options -# -# We don't care about namespaces matching folder structure. Games and apps -# are complicated and you are free to organize them however you like. Change -# this if you want to enforce it. -dotnet_style_namespace_match_folder = false -dotnet_diagnostic.IDE0130.severity = none - -# C# formatting rules -[*.{cs,csx,cake}] - -# Newline options -csharp_new_line_before_open_brace = none -csharp_new_line_before_else = true -csharp_new_line_before_catch = true -csharp_new_line_before_finally = true -csharp_new_line_before_members_in_object_initializers = true -csharp_new_line_before_members_in_anonymous_types = true -csharp_new_line_between_query_expression_clauses = true - -# Indentation options -csharp_indent_switch_labels = true -csharp_indent_case_contents = true -csharp_indent_case_contents_when_block = true -csharp_indent_labels = no_change -csharp_indent_block_contents = true -csharp_indent_braces = false - -# Spacing options -csharp_space_after_cast = false -csharp_space_after_keywords_in_control_flow_statements = true -csharp_space_between_parentheses = false -csharp_space_before_colon_in_inheritance_clause = true -csharp_space_after_colon_in_inheritance_clause = true -csharp_space_around_binary_operators = before_and_after -csharp_space_between_method_declaration_parameter_list_parentheses = false -csharp_space_between_method_declaration_empty_parameter_list_parentheses = false -csharp_space_between_method_declaration_name_and_open_parenthesis = false -csharp_space_between_method_call_parameter_list_parentheses = false -csharp_space_between_method_call_empty_parameter_list_parentheses = false -csharp_space_between_method_call_name_and_opening_parenthesis = false -csharp_space_after_comma = true -csharp_space_before_comma = false -csharp_space_after_dot = false -csharp_space_before_dot = false -csharp_space_after_semicolon_in_for_statement = true -csharp_space_before_semicolon_in_for_statement = false -csharp_space_around_declaration_statements = false -csharp_space_before_open_square_brackets = false -csharp_space_between_empty_square_brackets = false -csharp_space_between_square_brackets = false - -# Wrap options -csharp_preserve_single_line_statements = false -csharp_preserve_single_line_blocks = true - -# Namespace options -csharp_style_namespace_declarations = file_scoped:warning - -########################################## -# .NET Naming Rules -########################################## -[*.{cs,csx,cake,vb,vbx}] - -# Allow underscores in names. -dotnet_diagnostic.CA1707.severity = none - -# Styles -dotnet_naming_style.pascal_case_style.capitalization = pascal_case - -dotnet_naming_style.upper_case_style.capitalization = all_upper -dotnet_naming_style.upper_case_style.word_separator = _ - -dotnet_naming_style.camel_case_style.capitalization = camel_case - -dotnet_naming_style.camel_case_underscore_style.required_prefix = _ -dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case - -# Use uppercase for all constant fields. -dotnet_naming_rule.constants_uppercase.severity = suggestion -dotnet_naming_rule.constants_uppercase.symbols = constant_fields -dotnet_naming_rule.constants_uppercase.style = upper_case_style -dotnet_naming_symbols.constant_fields.applicable_kinds = field -dotnet_naming_symbols.constant_fields.applicable_accessibilities = * -dotnet_naming_symbols.constant_fields.required_modifiers = const - -# Non-public fields should be _camelCase -dotnet_naming_rule.non_public_fields_under_camel.severity = suggestion -dotnet_naming_rule.non_public_fields_under_camel.symbols = non_public_fields -dotnet_naming_rule.non_public_fields_under_camel.style = camel_case_underscore_style -dotnet_naming_symbols.non_public_fields.applicable_kinds = field -dotnet_naming_symbols.non_public_fields.required_modifiers = -dotnet_naming_symbols.non_public_fields.applicable_accessibilities = private,private_protected,internal,protected,protected_internal - -# Public fields should be PascalCase -dotnet_naming_rule.public_fields_pascal.severity = suggestion -dotnet_naming_rule.public_fields_pascal.symbols = public_fields -dotnet_naming_rule.public_fields_pascal.style = pascal_case_style -dotnet_naming_symbols.public_fields.applicable_kinds = field -dotnet_naming_symbols.public_fields.required_modifiers = -dotnet_naming_symbols.public_fields.applicable_accessibilities = public - -# Async methods should have "Async" suffix. -# Disabled because it makes tests too verbose. -# dotnet_naming_style.end_in_async.required_suffix = Async -# dotnet_naming_style.end_in_async.capitalization = pascal_case -# dotnet_naming_rule.methods_end_in_async.symbols = methods_async -# dotnet_naming_rule.methods_end_in_async.style = end_in_async -# dotnet_naming_rule.methods_end_in_async.severity = warning -# dotnet_naming_symbols.methods_async.applicable_kinds = method -# dotnet_naming_symbols.methods_async.required_modifiers = async -# dotnet_naming_symbols.methods_async.applicable_accessibilities = * - -########################################## -# Other Naming Rules -########################################## - -# All of the following must be PascalCase: -dotnet_naming_symbols.element_group.applicable_kinds = namespace, class, enum, struct, delegate, event, method, property -dotnet_naming_rule.element_rule.symbols = element_group -dotnet_naming_rule.element_rule.style = pascal_case_style -dotnet_naming_rule.element_rule.severity = warning - -# Interfaces use PascalCase and are prefixed with uppercase 'I' -# https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-classes-structs-and-interfaces -dotnet_naming_style.prefix_interface_with_i_style.capitalization = pascal_case -dotnet_naming_style.prefix_interface_with_i_style.required_prefix = I -dotnet_naming_symbols.interface_group.applicable_kinds = interface -dotnet_naming_rule.interface_rule.symbols = interface_group -dotnet_naming_rule.interface_rule.style = prefix_interface_with_i_style -dotnet_naming_rule.interface_rule.severity = warning - -# Generics Type Parameters use PascalCase and are prefixed with uppercase 'T' -# https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-classes-structs-and-interfaces -dotnet_naming_style.prefix_type_parameters_with_t_style.capitalization = pascal_case -dotnet_naming_style.prefix_type_parameters_with_t_style.required_prefix = T -dotnet_naming_symbols.type_parameter_group.applicable_kinds = type_parameter -dotnet_naming_rule.type_parameter_rule.symbols = type_parameter_group -dotnet_naming_rule.type_parameter_rule.style = prefix_type_parameters_with_t_style -dotnet_naming_rule.type_parameter_rule.severity = warning - -# Function parameters use camelCase -# https://docs.microsoft.com/dotnet/standard/design-guidelines/naming-parameters -dotnet_naming_symbols.parameters_group.applicable_kinds = parameter -dotnet_naming_rule.parameters_rule.symbols = parameters_group -dotnet_naming_rule.parameters_rule.style = camel_case_style -dotnet_naming_rule.parameters_rule.severity = warning - -# Anything not specified uses camel case. -dotnet_naming_rule.unspecified_naming.severity = warning -dotnet_naming_rule.unspecified_naming.symbols = unspecified -dotnet_naming_rule.unspecified_naming.style = camel_case_style -dotnet_naming_symbols.unspecified.applicable_kinds = * -dotnet_naming_symbols.unspecified.applicable_accessibilities = * - -########################################## -# Rule Overrides -########################################## - -roslyn_correctness.assembly_reference_validation = relaxed - -# Allow using keywords as names -# dotnet_diagnostic.CA1716.severity = none -# Don't require culture info for ToString() -dotnet_diagnostic.CA1304.severity = none -# Don't require a string comparison for comparing strings. -dotnet_diagnostic.CA1310.severity = none -# Don't require a string format specifier. -dotnet_diagnostic.CA1305.severity = none -# Allow protected fields. -dotnet_diagnostic.CA1051.severity = none -# Don't warn about checking values that are supposedly never null. Sometimes -# they are actually null. -dotnet_diagnostic.CS8073.severity = none -# Don't remove seemingly "unnecessary" assignments, as they often have -# intended side-effects. -dotnet_diagnostic.IDE0059.severity = none -# Switch/case should always have a default clause. Tell that to Roslynator. -dotnet_diagnostic.RCS1070.severity = none -# Tell roslynator not to eat unused parameters. -dotnet_diagnostic.RCS1163.severity = none -# Tell dotnet not to remove unused parameters. -dotnet_diagnostic.IDE0060.severity = none -# Tell roslynator not to remove `partial` modifiers. -dotnet_diagnostic.RCS1043.severity = none -# Tell roslynator not to make classes static so aggressively. -dotnet_diagnostic.RCS1102.severity = none -# Roslynator wants to make properties readonly all the time, so stop it. -# The developer knows best when it comes to contract definitions with Godot. -dotnet_diagnostic.RCS1170.severity = none -# Allow expression values to go unused, even without discard variable. -# Otherwise, using Moq would be way too verbose. -dotnet_diagnostic.IDE0058.severity = none -# Don't let roslynator turn every local variable into a const. -# If we did, we'd have to specify the types of local variables far more often, -# and this style prefers type inference. -dotnet_diagnostic.RCS1118.severity = none -# Enums don't need to declare explicit values. Everyone knows they start at 0. -dotnet_diagnostic.RCS1161.severity = none -# Allow unconstrained type parameter to be checked for null. -dotnet_diagnostic.RCS1165.severity = none -# Allow keyword-based names so that parameter names like `@event` can be used. -dotnet_diagnostic.CA1716.severity = none -# Allow me to use the word Collection if I want. -dotnet_diagnostic.CA1711.severity = none -# Not disposing of objects in a test is normal within Godot because of scene tree stuff. -dotnet_diagnostic.CA1001.severity = none -# No primary constructors — not supported well by tooling. -dotnet_diagnostic.IDE0290.severity = none -# Let me comment where I like -dotnet_diagnostic.RCS1181.severity = none -# Let me write dumb if checks, keeps it readable -dotnet_diagnostic.IDE0046.severity = none -# Don't make me use expression bodies for methods -dotnet_diagnostic.IDE0022.severity = none -# Don't use collection shorhand. -dotnet_diagnostic.IDE0300.severity = none -dotnet_diagnostic.IDE0028.severity = none -dotnet_diagnostic.IDE0305.severity = none -# Don't make me populate a switch expression redundantly -dotnet_diagnostic.IDE0072.severity = none -# Leave me alone about primary constructors -dotnet_diagnostic.IDE0290.severity = none \ No newline at end of file diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CommandController.cs b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CommandController.cs deleted file mode 100644 index e0236d2..0000000 --- a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CommandController.cs +++ /dev/null @@ -1,70 +0,0 @@ -namespace CorebrainCLIAPI; - -using CorebrainCS; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Options; - -/// -/// Controller for executing Corebrain CLI commands -/// -[ApiController] -[Route("api/[controller]")] -[Produces("application/json")] -public class CommandController : ControllerBase { - private readonly CorebrainCS _corebrain; - - public CommandController(IOptions settings) { - var config = settings.Value; - _corebrain = new CorebrainCS( - config.PythonPath, - config.ScriptPath, - config.Verbose - ); - } - - /// - /// Executes a Corebrain CLI command - /// - /// - /// Sample request: - /// - /// POST /api/command - /// { - /// "arguments": "--help" - /// } - /// - /// - /// Command request containing the arguments - /// The output of the executed command - /// Returns the command output - /// If the arguments are empty - /// If there was an error executing the command - [HttpPost] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesResponseType(StatusCodes.Status400BadRequest)] - [ProducesResponseType(StatusCodes.Status500InternalServerError)] - public IActionResult ExecuteCommand([FromBody] CommandRequest request) { - if (string.IsNullOrWhiteSpace(request.Arguments)) { - return BadRequest("Command arguments are required"); - } - - try { - var result = _corebrain.ExecuteCommand(request.Arguments); - return Ok(result); - } - catch (Exception ex) { - return StatusCode(500, $"Error executing command: {ex.Message}"); - } - } - - /// - /// Command request model - /// - public class CommandRequest { - /// - /// The arguments to pass to the Corebrain CLI - /// - /// --help - public required string Arguments { get; set; } - } -} diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainCLIAPI.csproj b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainCLIAPI.csproj deleted file mode 100644 index 279f7d0..0000000 --- a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainCLIAPI.csproj +++ /dev/null @@ -1,20 +0,0 @@ - - - - net9.0 - enable - enable - true - $(NoWarn);1591 - - - - - - - - - - - - diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainSettings.cs b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainSettings.cs deleted file mode 100644 index 82143b9..0000000 --- a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/CorebrainSettings.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace CorebrainCLIAPI; - -public class CorebrainSettings -{ - public string PythonPath { get; set; } - public string ScriptPath { get; set; } - public bool Verbose { get; set; } = false; -} diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Program.cs b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Program.cs deleted file mode 100644 index 3ddd6a1..0000000 --- a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Program.cs +++ /dev/null @@ -1,49 +0,0 @@ -using System.Reflection; -using CorebrainCLIAPI; -using Microsoft.OpenApi.Models; - -var builder = WebApplication.CreateBuilder(args); - -// CORS policy to allow requests from the frontend -builder.Services.AddCors(options => options.AddPolicy("AllowFrontend", policy => - policy.WithOrigins("http://localhost:5173") - .AllowAnyMethod() - .AllowAnyHeader() -)); - -// Configure controllers and settings -builder.Services.AddControllers(); -builder.Services.Configure( - builder.Configuration.GetSection("CorebrainSettings")); - -// Swagger / OpenAPI -builder.Services.AddEndpointsApiExplorer(); -builder.Services.AddSwaggerGen(c => { - c.SwaggerDoc("v1", new OpenApiInfo { - Title = "Corebrain CLI API", - Version = "v1", - Description = "ASP.NET Core Web API for interfacing with Corebrain CLI commands" - }); - - var xmlFile = $"{Assembly.GetExecutingAssembly().GetName().Name}.xml"; - var xmlPath = Path.Combine(AppContext.BaseDirectory, xmlFile); - if (File.Exists(xmlPath)) { - c.IncludeXmlComments(xmlPath); - } -}); - -var app = builder.Build(); - -// Middleware pipeline -app.UseCors("AllowFrontend"); - -if (app.Environment.IsDevelopment()) { - app.UseSwagger(); - app.UseSwaggerUI(c => - c.SwaggerEndpoint("/swagger/v1/swagger.json", "Corebrain CLI API v1")); -} - -app.UseHttpsRedirection(); -app.UseAuthorization(); -app.MapControllers(); -app.Run(); diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Properties/launchSettings.json b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Properties/launchSettings.json deleted file mode 100644 index f212316..0000000 --- a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/Properties/launchSettings.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "$schema": "https://json.schemastore.org/launchsettings.json", - "profiles": { - "http": { - "commandName": "Project", - "dotnetRunMessages": true, - "launchBrowser": false, - "applicationUrl": "http://localhost:5140", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "https": { - "commandName": "Project", - "dotnetRunMessages": true, - "launchBrowser": false, - "applicationUrl": "https://localhost:7261;http://localhost:5140", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - } - } -} diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.Development.json b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.Development.json deleted file mode 100644 index 0c208ae..0000000 --- a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.Development.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - } -} diff --git a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.json b/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.json deleted file mode 100644 index 0ab3335..0000000 --- a/corebrain/corebrain/wrappers/csharp_cli_api/src/CorebrainCLIAPI/appsettings.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - }, - "AllowedHosts": "*", - "CorebrainSettings": { - "PythonPath": "../../../../../venv/Scripts/python.exe", - "ScriptPath": "../../../../cli", - "Verbose": false - } -} diff --git a/corebrain/docs/Makefile b/corebrain/docs/Makefile deleted file mode 100644 index d0c3cbf..0000000 --- a/corebrain/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/corebrain/docs/README.md b/corebrain/docs/README.md deleted file mode 100644 index cd094fd..0000000 --- a/corebrain/docs/README.md +++ /dev/null @@ -1,13 +0,0 @@ -### Generating docs - -Run in terminal: - -```bash - -.\docs\make.bat html - -``` - - - - diff --git a/corebrain/docs/make.bat b/corebrain/docs/make.bat deleted file mode 100644 index dc1312a..0000000 --- a/corebrain/docs/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=source -set BUILDDIR=build - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.https://www.sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "" goto help - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/corebrain/docs/source/_static/custom.css b/corebrain/docs/source/_static/custom.css deleted file mode 100644 index e69de29..0000000 diff --git a/corebrain/docs/source/conf.py b/corebrain/docs/source/conf.py deleted file mode 100644 index a59ab3a..0000000 --- a/corebrain/docs/source/conf.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import sys - -sys.path.insert(0, os.path.abspath('../..')) - -project = 'Corebrain Documentation' -copyright = '2025, Corebrain' -author = 'Corebrain' -release = '0.1' - -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.napoleon', - 'sphinx.ext.viewcode', - 'sphinx_copybutton', - 'sphinx_design', -] - -templates_path = ['_templates'] -exclude_patterns = [] - -html_theme = 'furo' -html_css_files = ['custom.css'] -html_static_path = ['_static'] - diff --git a/corebrain/docs/source/corebrain.cli.auth.rst b/corebrain/docs/source/corebrain.cli.auth.rst deleted file mode 100644 index 85bb14a..0000000 --- a/corebrain/docs/source/corebrain.cli.auth.rst +++ /dev/null @@ -1,29 +0,0 @@ -corebrain.cli.auth package -========================== - -Submodules ----------- - -corebrain.cli.auth.api\_keys module ------------------------------------ - -.. automodule:: corebrain.cli.auth.api_keys - :members: - :show-inheritance: - :undoc-members: - -corebrain.cli.auth.sso module ------------------------------ - -.. automodule:: corebrain.cli.auth.sso - :members: - :show-inheritance: - :undoc-members: - -Module contents ---------------- - -.. automodule:: corebrain.cli.auth - :members: - :show-inheritance: - :undoc-members: diff --git a/corebrain/docs/source/corebrain.cli.rst b/corebrain/docs/source/corebrain.cli.rst deleted file mode 100644 index 3fdb48b..0000000 --- a/corebrain/docs/source/corebrain.cli.rst +++ /dev/null @@ -1,53 +0,0 @@ -corebrain.cli package -===================== - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - corebrain.cli.auth - -Submodules ----------- - -corebrain.cli.commands module ------------------------------ - -.. automodule:: corebrain.cli.commands - :members: - :show-inheritance: - :undoc-members: - -corebrain.cli.common module ---------------------------- - -.. automodule:: corebrain.cli.common - :members: - :show-inheritance: - :undoc-members: - -corebrain.cli.config module ---------------------------- - -.. automodule:: corebrain.cli.config - :members: - :show-inheritance: - :undoc-members: - -corebrain.cli.utils module --------------------------- - -.. automodule:: corebrain.cli.utils - :members: - :show-inheritance: - :undoc-members: - -Module contents ---------------- - -.. automodule:: corebrain.cli - :members: - :show-inheritance: - :undoc-members: diff --git a/corebrain/docs/source/corebrain.config.rst b/corebrain/docs/source/corebrain.config.rst deleted file mode 100644 index 4168d30..0000000 --- a/corebrain/docs/source/corebrain.config.rst +++ /dev/null @@ -1,21 +0,0 @@ -corebrain.config package -======================== - -Submodules ----------- - -corebrain.config.manager module -------------------------------- - -.. automodule:: corebrain.config.manager - :members: - :show-inheritance: - :undoc-members: - -Module contents ---------------- - -.. automodule:: corebrain.config - :members: - :show-inheritance: - :undoc-members: diff --git a/corebrain/docs/source/corebrain.core.rst b/corebrain/docs/source/corebrain.core.rst deleted file mode 100644 index 0313ae3..0000000 --- a/corebrain/docs/source/corebrain.core.rst +++ /dev/null @@ -1,45 +0,0 @@ -corebrain.core package -====================== - -Submodules ----------- - -corebrain.core.client module ----------------------------- - -.. automodule:: corebrain.core.client - :members: - :show-inheritance: - :undoc-members: - -corebrain.core.common module ----------------------------- - -.. automodule:: corebrain.core.common - :members: - :show-inheritance: - :undoc-members: - -corebrain.core.query module ---------------------------- - -.. automodule:: corebrain.core.query - :members: - :show-inheritance: - :undoc-members: - -corebrain.core.test\_utils module ---------------------------------- - -.. automodule:: corebrain.core.test_utils - :members: - :show-inheritance: - :undoc-members: - -Module contents ---------------- - -.. automodule:: corebrain.core - :members: - :show-inheritance: - :undoc-members: diff --git a/corebrain/docs/source/corebrain.db.connectors.rst b/corebrain/docs/source/corebrain.db.connectors.rst deleted file mode 100644 index d2710b3..0000000 --- a/corebrain/docs/source/corebrain.db.connectors.rst +++ /dev/null @@ -1,29 +0,0 @@ -corebrain.db.connectors package -=============================== - -Submodules ----------- - -corebrain.db.connectors.mongodb module --------------------------------------- - -.. automodule:: corebrain.db.connectors.mongodb - :members: - :show-inheritance: - :undoc-members: - -corebrain.db.connectors.sql module ----------------------------------- - -.. automodule:: corebrain.db.connectors.sql - :members: - :show-inheritance: - :undoc-members: - -Module contents ---------------- - -.. automodule:: corebrain.db.connectors - :members: - :show-inheritance: - :undoc-members: diff --git a/corebrain/docs/source/corebrain.db.rst b/corebrain/docs/source/corebrain.db.rst deleted file mode 100644 index 751b1d4..0000000 --- a/corebrain/docs/source/corebrain.db.rst +++ /dev/null @@ -1,62 +0,0 @@ -corebrain.db package -==================== - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - corebrain.db.connectors - corebrain.db.schema - -Submodules ----------- - -corebrain.db.connector module ------------------------------ - -.. automodule:: corebrain.db.connector - :members: - :show-inheritance: - :undoc-members: - -corebrain.db.engines module ---------------------------- - -.. automodule:: corebrain.db.engines - :members: - :show-inheritance: - :undoc-members: - -corebrain.db.factory module ---------------------------- - -.. automodule:: corebrain.db.factory - :members: - :show-inheritance: - :undoc-members: - -corebrain.db.interface module ------------------------------ - -.. automodule:: corebrain.db.interface - :members: - :show-inheritance: - :undoc-members: - -corebrain.db.schema\_file module --------------------------------- - -.. automodule:: corebrain.db.schema_file - :members: - :show-inheritance: - :undoc-members: - -Module contents ---------------- - -.. automodule:: corebrain.db - :members: - :show-inheritance: - :undoc-members: diff --git a/corebrain/docs/source/corebrain.db.schema.rst b/corebrain/docs/source/corebrain.db.schema.rst deleted file mode 100644 index ccc435b..0000000 --- a/corebrain/docs/source/corebrain.db.schema.rst +++ /dev/null @@ -1,29 +0,0 @@ -corebrain.db.schema package -=========================== - -Submodules ----------- - -corebrain.db.schema.extractor module ------------------------------------- - -.. automodule:: corebrain.db.schema.extractor - :members: - :show-inheritance: - :undoc-members: - -corebrain.db.schema.optimizer module ------------------------------------- - -.. automodule:: corebrain.db.schema.optimizer - :members: - :show-inheritance: - :undoc-members: - -Module contents ---------------- - -.. automodule:: corebrain.db.schema - :members: - :show-inheritance: - :undoc-members: diff --git a/corebrain/docs/source/corebrain.network.rst b/corebrain/docs/source/corebrain.network.rst deleted file mode 100644 index 3d94c4f..0000000 --- a/corebrain/docs/source/corebrain.network.rst +++ /dev/null @@ -1,21 +0,0 @@ -corebrain.network package -========================= - -Submodules ----------- - -corebrain.network.client module -------------------------------- - -.. automodule:: corebrain.network.client - :members: - :show-inheritance: - :undoc-members: - -Module contents ---------------- - -.. automodule:: corebrain.network - :members: - :show-inheritance: - :undoc-members: diff --git a/corebrain/docs/source/corebrain.rst b/corebrain/docs/source/corebrain.rst deleted file mode 100644 index fb44332..0000000 --- a/corebrain/docs/source/corebrain.rst +++ /dev/null @@ -1,42 +0,0 @@ -corebrain package -================= - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - corebrain.cli - corebrain.config - corebrain.core - corebrain.db - corebrain.network - corebrain.utils - -Submodules ----------- - -corebrain.cli module --------------------- - -.. automodule:: corebrain.cli - :members: - :show-inheritance: - :undoc-members: - -corebrain.sdk module --------------------- - -.. automodule:: corebrain.sdk - :members: - :show-inheritance: - :undoc-members: - -Module contents ---------------- - -.. automodule:: corebrain - :members: - :show-inheritance: - :undoc-members: diff --git a/corebrain/docs/source/corebrain.utils.rst b/corebrain/docs/source/corebrain.utils.rst deleted file mode 100644 index 4294529..0000000 --- a/corebrain/docs/source/corebrain.utils.rst +++ /dev/null @@ -1,37 +0,0 @@ -corebrain.utils package -======================= - -Submodules ----------- - -corebrain.utils.encrypter module --------------------------------- - -.. automodule:: corebrain.utils.encrypter - :members: - :show-inheritance: - :undoc-members: - -corebrain.utils.logging module ------------------------------- - -.. automodule:: corebrain.utils.logging - :members: - :show-inheritance: - :undoc-members: - -corebrain.utils.serializer module ---------------------------------- - -.. automodule:: corebrain.utils.serializer - :members: - :show-inheritance: - :undoc-members: - -Module contents ---------------- - -.. automodule:: corebrain.utils - :members: - :show-inheritance: - :undoc-members: diff --git a/corebrain/docs/source/index.rst b/corebrain/docs/source/index.rst deleted file mode 100644 index 03ce071..0000000 --- a/corebrain/docs/source/index.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. Documentation documentation master file, created by - sphinx-quickstart on Fri May 16 16:20:00 2025. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to Corebrain's documentation! -=========================== - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - modules - diff --git a/corebrain/docs/source/modules.rst b/corebrain/docs/source/modules.rst deleted file mode 100644 index 7f3849e..0000000 --- a/corebrain/docs/source/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -corebrain -========= - -.. toctree:: - :maxdepth: 4 - - corebrain diff --git a/corebrain/examples/add_config.py b/corebrain/examples/add_config.py deleted file mode 100644 index 963996a..0000000 --- a/corebrain/examples/add_config.py +++ /dev/null @@ -1,27 +0,0 @@ -from corebrain import ConfigManager - -# Initialize config manager -config_manager = ConfigManager() - -# API key -api_key = "sk_bH8rnkIHCDF1BlRmgS9s6QAK" - -# Database configuration -db_config = { - "type": "sql", # or "mongodb" for MongoDB - "engine": "postgresql", # or "mysql", "sqlite", etc. - "host": "localhost", - "port": 5432, - "database": "your_database", - "user": "your_username", - "password": "your_password" -} - -# Add configuration -config_id = config_manager.add_config(api_key, db_config) -print(f"Configuration added with ID: {config_id}") - -# List all configurations -print("\nAvailable configurations:") -configs = config_manager.list_configs(api_key) -print(configs) \ No newline at end of file diff --git a/corebrain/examples/complex.py b/corebrain/examples/complex.py deleted file mode 100644 index 41ccc4e..0000000 --- a/corebrain/examples/complex.py +++ /dev/null @@ -1,23 +0,0 @@ -from corebrain import init - -api_key = "sk_NPNLbEAjxQm86u6OX97An5ev" -#config_id = "c9913a04-a530-4ae3-a877-8e295be87f78" # MONGODB -config_id = "be43981c-0015-4ba4-9861-f12e82f6805e" # POSTGRES - -# Initialize the SDK with API key and configuration ID -corebrain = init( - api_key=api_key, - config_id=config_id -) - -""" -Corebrain possible arguments (all optionals): - -- execute_query (bool) -- explain_results (bool) -- detail_level (string = "full") -""" - -result = corebrain.ask("Devuélveme 5 datos interesantes sobre mis usuarios", detail_level="full") - -print(result['explanation']) diff --git a/corebrain/examples/list_schema.py b/corebrain/examples/list_schema.py deleted file mode 100644 index daeba01..0000000 --- a/corebrain/examples/list_schema.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -Example script to list database schema and configuration details. -This helps diagnose issues with database connections and schema extraction. -""" -import os -import json -import logging -import psycopg2 -from corebrain import init - -# Configure logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -def verify_postgres_connection(db_config): - """Verify PostgreSQL connection and list tables directly""" - logger.info("\n=== Direct PostgreSQL Connection Test ===") - try: - # Create connection - conn = psycopg2.connect( - host=db_config.get("host", "localhost"), - user=db_config.get("user", ""), - password=db_config.get("password", ""), - dbname=db_config.get("database", ""), - port=db_config.get("port", 5432) - ) - - # Create cursor - cur = conn.cursor() - - # Test connection - cur.execute("SELECT version();") - version = cur.fetchone() - logger.info(f"PostgreSQL Version: {version[0]}") - - # List all schemas - cur.execute(""" - SELECT schema_name - FROM information_schema.schemata - WHERE schema_name NOT IN ('information_schema', 'pg_catalog'); - """) - schemas = cur.fetchall() - logger.info("\nAvailable Schemas:") - for schema in schemas: - logger.info(f" - {schema[0]}") - - # List all tables in public schema - cur.execute(""" - SELECT table_name - FROM information_schema.tables - WHERE table_schema = 'public'; - """) - tables = cur.fetchall() - logger.info("\nTables in public schema:") - for table in tables: - logger.info(f" - {table[0]}") - - # Get column info for each table - cur.execute(f""" - SELECT column_name, data_type - FROM information_schema.columns - WHERE table_schema = 'public' - AND table_name = '{table[0]}'; - """) - columns = cur.fetchall() - logger.info(" Columns:") - for col in columns: - logger.info(f" - {col[0]}: {col[1]}") - - cur.close() - conn.close() - - except Exception as e: - logger.error(f"Error in direct PostgreSQL connection: {str(e)}", exc_info=True) - -def main(): - # Get API key from environment variable - api_key = "sk_bH8rnkIHCDF1BlRmgS9s6QAK" - if not api_key: - raise ValueError("Please set COREBRAIN_API_KEY environment variable") - - # Get config ID from environment variable - config_id = "8bdba894-34a7-4453-b665-e640d11fd463" - if not config_id: - raise ValueError("Please set COREBRAIN_CONFIG_ID environment variable") - - logger.info("Initializing Corebrain SDK...") - try: - corebrain = init( - api_key=api_key, - config_id=config_id, - skip_verification=True # Skip API key verification due to the error - ) - except Exception as e: - logger.error(f"Error initializing SDK: {str(e)}") - return - - # Print configuration details - logger.info("\n=== Configuration Details ===") - logger.info(f"Database Type: {corebrain.db_config.get('type')}") - logger.info(f"Database Engine: {corebrain.db_config.get('engine')}") - logger.info(f"Database Name: {corebrain.db_config.get('database')}") - logger.info(f"Config ID: {corebrain.config_id}") - - # Print full database configuration - logger.info("\n=== Full Database Configuration ===") - logger.info(json.dumps(corebrain.db_config, indent=2)) - - # If PostgreSQL, verify connection directly - if corebrain.db_config.get("type", "").lower() == "sql" and \ - corebrain.db_config.get("engine", "").lower() == "postgresql": - verify_postgres_connection(corebrain.db_config) - - # Extract and print schema - logger.info("\n=== Database Schema ===") - try: - schema = corebrain._extract_db_schema(detail_level="full") - - # Print schema summary - logger.info(f"Schema Type: {schema.get('type')}") - logger.info(f"Total Collections: {schema.get('total_collections', 0)}") - logger.info(f"Included Collections: {schema.get('included_collections', 0)}") - - # Print tables/collections - if schema.get("tables"): - logger.info("\n=== Tables/Collections ===") - for table_name, table_info in schema["tables"].items(): - logger.info(f"\nTable/Collection: {table_name}") - - # Print columns/fields - if "columns" in table_info: - logger.info("Columns:") - for col in table_info["columns"]: - logger.info(f" - {col['name']}: {col['type']}") - elif "fields" in table_info: - logger.info("Fields:") - for field in table_info["fields"]: - logger.info(f" - {field['name']}: {field['type']}") - - # Print document count if available - if "doc_count" in table_info: - logger.info(f"Document Count: {table_info['doc_count']}") - - # Print sample data if available - if "sample_data" in table_info and table_info["sample_data"]: - logger.info("Sample Data:") - for doc in table_info["sample_data"][:2]: # Show only first 2 documents - logger.info(f" {json.dumps(doc, indent=2)}") - else: - logger.warning("No tables/collections found in schema!") - - # Print raw schema for debugging - logger.info("\n=== Raw Schema ===") - logger.info(json.dumps(schema, indent=2)) - except Exception as e: - logger.error(f"Error extracting schema: {str(e)}", exc_info=True) - -if __name__ == "__main__": - try: - main() - except Exception as e: - logger.error(f"Error: {str(e)}", exc_info=True) \ No newline at end of file diff --git a/corebrain/examples/simple.py b/corebrain/examples/simple.py deleted file mode 100644 index 483c546..0000000 --- a/corebrain/examples/simple.py +++ /dev/null @@ -1,15 +0,0 @@ -from corebrain import init - -api_key = "sk_bH8rnkIHCDF1BlRmgS9s6QAK" -#config_id = "c9913a04-a530-4ae3-a877-8e295be87f78" MONGODB -config_id = "8bdba894-34a7-4453-b665-e640d11fd463" # POSTGRES - -# Initialize the SDK with API key and configuration ID -corebrain = init( - api_key=api_key, - config_id=config_id -) - -result = corebrain.ask("Analiza los usuarios y los servicios asociados a estos usuarios.") -print(result["explanation"]) - diff --git a/corebrain/health.py b/corebrain/health.py deleted file mode 100644 index 55ffd9d..0000000 --- a/corebrain/health.py +++ /dev/null @@ -1,47 +0,0 @@ -# check_imports.py -import os -import importlib -import sys - -def check_imports(package_name, directory): - """ - Recursively checks imports into a directory. - """ - - for item in os.listdir(directory): - path = os.path.join(directory, item) - - # Ignore hidden folders or __pycache__ - if item.startswith('.') or item == '__pycache__': - continue - - if os.path.isdir(path): - - if os.path.exists(os.path.join(path, '__init__.py')): - subpackage = f"{package_name}.{item}" - try: - print(f"Verificating subpackage: {subpackage}") - importlib.import_module(subpackage) - check_imports(subpackage, path) - except Exception as e: - print(f"ERROR in {subpackage}: {e}") - - elif item.endswith('.py') and item != '__init__.py': - module_name = f"{package_name}.{item[:-3]}" # quitar .py - try: - print(f"Verificating module: {module_name}") - importlib.import_module(module_name) - except Exception as e: - print(f"ERROR in {module_name}: {e}") - -sys.path.insert(0, '.') - -# Verify all main modules -for pkg in ['corebrain']: - if os.path.exists(pkg): - try: - print(f"\Verificating pkg: {pkg}") - importlib.import_module(pkg) - check_imports(pkg, pkg) - except Exception as e: - print(f"ERROR in pkg {pkg}: {e}") \ No newline at end of file diff --git a/corebrain/pyproject.toml b/corebrain/pyproject.toml deleted file mode 100644 index 76f919d..0000000 --- a/corebrain/pyproject.toml +++ /dev/null @@ -1,85 +0,0 @@ -[project] -name = "corebrain" -version = "0.1.0" -description = "SDK de Corebrain para consultas en lenguaje natural a bases de datos" -readme = "README.md" -authors = [ - {name = "Rubén Ayuso", email = "ruben@globodain.com"} -] -license = {text = "MIT"} -classifiers = [ - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", -] -requires-python = ">=3.8" -dependencies = [ - "httpx>=0.24.0", - "sqlalchemy>=2.0.0", - "pydantic>=2.0.0", - "cryptography>=40.0.0", - "python-dotenv>=1.0.0", - "typing-extensions>=4.4.0", - "requests>=2.28.0", - "asyncio>=3.4.3", - "psycopg2-binary>=2.9.0", # En lugar de psycopg2 para evitar problemas de compilación - "mysql-connector-python>=8.0.23", - "pymongo>=4.4.0", -] - -[project.optional-dependencies] -postgres = ["psycopg2-binary>=2.9.0"] -mongodb = ["pymongo>=4.4.0"] -mysql = ["mysql-connector-python>=8.0.23"] -all_db = [ - "psycopg2-binary>=2.9.0", - "pymongo>=4.4.0", - "mysql-connector-python>=8.0.23", -] -dev = [ - "pytest>=7.0.0", - "pytest-cov>=4.0.0", - "black>=23.0.0", - "isort>=5.12.0", - "mypy>=1.3.0", - "flake8>=6.0.0", - "sphinx>=8.2.3", - "furo>=2024.8.6", -] - - -[tool.setuptools] -packages = ["corebrain"] - -[project.urls] -"Homepage" = "https://github.com/ceoweggo/Corebrain" -"Bug Tracker" = "https://github.com/ceoweggo/Corebrain/issues" - -[project.scripts] -corebrain = "corebrain.cli.__main__:main" - -[tool.black] -line-length = 100 -target-version = ["py38"] -include = '\.pyi?$' - -[tool.isort] -profile = "black" -line_length = 100 - -[tool.mypy] -python_version = "3.8" -warn_return_any = true -warn_unused_configs = true -disallow_untyped_defs = false -disallow_incomplete_defs = false - -[tool.pytest.ini_options] -minversion = "7.0" -testpaths = ["tests"] -pythonpath = ["."] \ No newline at end of file diff --git a/corebrain/setup.ps1 b/corebrain/setup.ps1 deleted file mode 100644 index 3d031a4..0000000 --- a/corebrain/setup.ps1 +++ /dev/null @@ -1,5 +0,0 @@ -python -m venv venv - -.\venv\Scripts\Activate.ps1 - -pip install -e ".[dev,all_db]" \ No newline at end of file diff --git a/corebrain/setup.py b/corebrain/setup.py deleted file mode 100644 index b14bc71..0000000 --- a/corebrain/setup.py +++ /dev/null @@ -1,38 +0,0 @@ -""" -Installer configuration for Corebrain package. -""" - -from setuptools import setup, find_packages - -setup( - name="corebrain", - version="1.0.0", - description="SDK for natural language ask to DB", - author="Rubén Ayuso", - author_email="ruben@globodain.com", - packages=find_packages(), - install_requires=[ - "httpx>=0.23.0", - "pymongo>=4.3.0", - "psycopg2-binary>=2.9.5", - "mysql-connector-python>=8.0.31", - "sqlalchemy>=2.0.0", - "cryptography>=39.0.0", - "pydantic>=1.10.0", - ], - python_requires=">=3.8", - entry_points={ - "console_scripts": [ - "corebrain=corebrain.__main__:main", - ], - }, - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - ], -) \ No newline at end of file diff --git a/corebrain/setup.sh b/corebrain/setup.sh deleted file mode 100644 index d32b7c8..0000000 --- a/corebrain/setup.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# Utwórz i aktywuj środowisko wirtualne -python3 -m venv venv -source venv/bin/activate - -pip install -e ".[dev,all_db]" \ No newline at end of file diff --git a/examples/complex.py b/examples/complex.py index e66c21b..fea6643 100644 --- a/examples/complex.py +++ b/examples/complex.py @@ -1,8 +1,8 @@ from corebrain import init -api_key = "sk_bH8rnkIHCDF1BlRmgS9s6QAK" -#config_id = "c9913a04-a530-4ae3-a877-8e295be87f78" # MONGODB -config_id = "8bdba894-34a7-4453-b665-e640d11fd463" # POSTGRES +api_key = "sk_HljTkVLkT2TGMwrpbezpqBmR" +config_id = "59c3e839-fe0a-4675-925d-762064da350b" # MONGODB +#config_id = "8bdba894-34a7-4453-b665-e640d11fd463" # POSTGRES # Initialize the SDK with API key and configuration ID corebrain = init( From a6a4c1a9928daee9d3408b62b7e4b59a35686d8e Mon Sep 17 00:00:00 2001 From: bunny70pl Date: Tue, 27 May 2025 12:50:01 +0200 Subject: [PATCH 67/81] Adding list configurations... --- corebrain/cli/common.py | 2 +- corebrain/config/manager.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/corebrain/cli/common.py b/corebrain/cli/common.py index 8599bee..022e221 100644 --- a/corebrain/cli/common.py +++ b/corebrain/cli/common.py @@ -2,7 +2,7 @@ Default values for SSO and API connection """ -DEFAULT_API_URL = "http://localhost:1000" # Use 5000 in Windows / 1000 in MacOS by default +DEFAULT_API_URL = "http://localhost:5000" # Use 5000 in Windows / 1000 in MacOS by default #DEFAULT_SSO_URL = "http://localhost:3000" # localhost DEFAULT_SSO_URL = "https://sso.globodain.com" # remote DEFAULT_PORT = 8765 diff --git a/corebrain/config/manager.py b/corebrain/config/manager.py index 8b645ef..c26d058 100644 --- a/corebrain/config/manager.py +++ b/corebrain/config/manager.py @@ -194,6 +194,12 @@ def list_configs(self, api_key_selected: str) -> List[str]: Returns: List of configuration IDs """ + + + + + + return list(self.configs.get(api_key_selected, {}).keys()) def remove_config(self, api_key_selected: str, config_id: str) -> bool: From 8397e7df91ead376138d4e348a3edda8a4cde3ec Mon Sep 17 00:00:00 2001 From: bunny70pl Date: Tue, 27 May 2025 13:06:45 +0200 Subject: [PATCH 68/81] Any changes --- corebrain/cli/commands.py | 2 +- corebrain/config/manager.py | 118 +++++++++++++++++++++++++++++++++++- 2 files changed, 117 insertions(+), 3 deletions(-) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index f3314dc..50babff 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -687,7 +687,7 @@ def check_library(library_name, min_version): Note: This command provides a safe environment for configuration management with confirmation prompts for destructive operations. """ - ConfigManager.list_configs(api_key, api_url) + ConfigManager.list_configs(api_key_selected,api_url) elif args.show_schema: """ diff --git a/corebrain/config/manager.py b/corebrain/config/manager.py index c26d058..e2fa740 100644 --- a/corebrain/config/manager.py +++ b/corebrain/config/manager.py @@ -61,6 +61,7 @@ class ConfigManager: CONFIG_DIR = Path.home() / ".corebrain" CONFIG_FILE = CONFIG_DIR / "config.json" SECRET_KEY_FILE = CONFIG_DIR / "secret.key" + ACTIVE_CONFIG_FILE = CONFIG_DIR / "active_config.json" def __init__(self): self.configs = {} @@ -184,8 +185,9 @@ def get_config(self, api_key_selected: str, config_id: str) -> Optional[Dict[str """ return self.configs.get(api_key_selected, {}).get(config_id) + """ --> Default version def list_configs(self, api_key_selected: str) -> List[str]: - """ + Lists the available configuration IDs for an API Key. Args: @@ -193,15 +195,127 @@ def list_configs(self, api_key_selected: str) -> List[str]: Returns: List of configuration IDs + + return list(self.configs.get(api_key_selected, {}).keys()) + """ + + def set_active_config(self, config_id_to_activate: str) -> bool: """ + Sets a given config as active, regardless of which API key it's under. + Args: + config_id_to_activate: The config ID to set as active globally. + Returns: + True if the config was found and activated, False otherwise. + """ + found = False + for api_key, configs in self.configs.items(): + for config_id, config_data in configs.items(): + if config_id == config_id_to_activate: + config_data["active"] = True + found = True + else: + config_data.pop("active", None) + if found: + self._save_configs() + _print_colored(f"Activated configuration {config_id_to_activate}", "green") + return True + else: + _print_colored(f"Invalid Config ID: {config_id_to_activate}", "red") + return False + def get_active_config_id(self, api_key: str) -> Optional[str]: + """ + Retrieve the currently active configuration ID for a given API key. - return list(self.configs.get(api_key_selected, {}).keys()) + Returns None if not set. + """ + try: + if self.ACTIVE_CONFIG_FILE.exists(): + with open(self.ACTIVE_CONFIG_FILE, "r") as f: + data = json.load(f) + if data.get("api_key") == api_key: + return data.get("config_id") + except Exception as e: + _print_colored(f"Could not load active configuration: {e}", "yellow") + return None + def list_configs(self, api_key_selected: str) -> List[str]: + """ + Interactively select an API key, then display and manage its configurations. + + Returns: + ID of the selected or activated configuration (or None if nothing selected). + """ + configs = self.configs + if configs: + print("No saved configurations found.") + return None + + api_keys = list(self.configs.keys()) + print("\nAvailable API Keys:") + for idx, key in enumerate(api_keys, 1): + print(f" {idx}. {key}") + + try: + selected_index = int(input("Select an API Key by number: ").strip()) + selected_api_key = api_keys[selected_index - 1] + except (ValueError, IndexError): + _print_colored("Invalid selection.", "red") + return None + + configs = self.configs[selected_api_key] + if not configs: + _print_colored("No configurations found for the selected API Key.", "yellow") + return None + + print(f"\nConfigurations for API Key {selected_api_key}.") + config_ids = list(configs.keys()) + for idx, config_id in enumerate(config_ids, 1): + status = " [ACTIVE]" if configs[config_id].get("active") else "" + if status == " [ACTIVE]": + _print_colored(f" {idx}. {config_id}{status}","blue") + else: + print(f" {idx}. {config_id}{status}") + + for k, v in configs[config_id].items(): + print(f" {k}: {v}") + + action_prompt = input("\nWould you like to perform an action? (y/n): ").strip().lower() + if action_prompt == 'y': + print("\nAvailable actions:") + print(" 1. Activate configuration") + print(" 2. Delete configuration") + print(" 3. Exit") + + choice = input("Enter your choice (1/2/3): ").strip() + if choice == '1': + selected_idx = input("Enter the number of the configuration to activate: ").strip() + try: + config_id = config_ids[int(selected_idx) - 1] + self.set_active_config(config_id) + return config_id + except (ValueError, IndexError): + _print_colored("Invalid configuration number.", "red") + elif choice == '2': + selected_idx = input("Enter the number of the configuration to delete: ").strip() + try: + config_id = config_ids[int(selected_idx) - 1] + self.remove_config(selected_api_key, config_id) + except (ValueError, IndexError): + _print_colored("Invalid configuration number.", "red") + elif choice == '3': + print("Exit selected.") + else: + print("Invalid action.") + elif action_prompt != 'n': + print("Invalid input. Please enter 'y' or 'n'.") + + return None + def remove_config(self, api_key_selected: str, config_id: str) -> bool: """ Deletes a configuration. From 9d70621259fead928f069a07cfbb8231b59c1e35 Mon Sep 17 00:00:00 2001 From: bunny70pl Date: Tue, 27 May 2025 13:10:58 +0200 Subject: [PATCH 69/81] Added remove, active and list functions about configuration --- corebrain/cli/commands.py | 3 ++- corebrain/config/manager.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index 50babff..a10363a 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -687,7 +687,8 @@ def check_library(library_name, min_version): Note: This command provides a safe environment for configuration management with confirmation prompts for destructive operations. """ - ConfigManager.list_configs(api_key_selected,api_url) + manager = ConfigManager() + manager.list_configs(api_key_selected) elif args.show_schema: """ diff --git a/corebrain/config/manager.py b/corebrain/config/manager.py index e2fa740..e41f278 100644 --- a/corebrain/config/manager.py +++ b/corebrain/config/manager.py @@ -250,8 +250,7 @@ def list_configs(self, api_key_selected: str) -> List[str]: Returns: ID of the selected or activated configuration (or None if nothing selected). """ - configs = self.configs - if configs: + if not self.configs: print("No saved configurations found.") return None From f5ff10c874033e4896b36b92fe9200c25f96dcee Mon Sep 17 00:00:00 2001 From: KarixD2137 Date: Tue, 27 May 2025 13:11:57 +0200 Subject: [PATCH 70/81] API key creation - fixed compatibility with merged code --- corebrain/cli/commands.py | 16 ++++++++++++---- corebrain/cli/common.py | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index cfcb687..5ca902c 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -12,7 +12,7 @@ from typing import Optional, List from corebrain.cli.common import DEFAULT_API_URL, DEFAULT_SSO_URL, DEFAULT_PORT, SSO_CLIENT_ID, SSO_CLIENT_SECRET -from corebrain.cli.auth.sso import authenticate_with_sso, authenticate_with_sso_and_api_key_request, save_api_token, load_api_token +from corebrain.cli.auth.sso import authenticate_with_sso, authenticate_with_sso_and_api_key_request, load_api_token, save_api_token from corebrain.cli.config import configure_sdk, get_api_credential from corebrain.cli.utils import print_colored from corebrain.config.manager import ConfigManager @@ -48,6 +48,12 @@ def authentication(): print_colored(f"{sso_token}", "blue") print_colored("✅ Returning User data.", "green") print_colored(f"{sso_user}", "blue") + + # Saving api token + api_key, user_data, api_token = get_api_credential(sso_token, DEFAULT_SSO_URL) + save_api_token(api_key) + print_colored("✅ API token saved.", "green") + return sso_token, sso_user except Exception as e: @@ -450,6 +456,8 @@ def check_library(library_name, min_version): if sso_token and sso_user: print_colored("✅ Enter to create an user and API Key.", "green") + save_sso_token(sso_token) + print_colored("✅ SSO token saved.", "green") # Get API URL from environment or use default api_url = os.environ.get("COREBRAIN_API_URL", DEFAULT_API_URL) @@ -1068,12 +1076,12 @@ def run_in_background_silent(cmd, cwd): # Handles the CLI command to create a new API key using stored credentials (token from SSO) if args.create_api_key: - sso_token, sso_user = authentication() # Authentica use with SSO + sso_token = load_api_token() key_name = args.key_name or "default-key" key_level = args.key_level or "read" - api_url = args.api_url or os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + api_url = os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL # Sending request to Corebrain-API payload = { @@ -1082,7 +1090,7 @@ def run_in_background_silent(cmd, cwd): } headers = { - "Authorization": f"Bearer {api_token}", + "Authorization": f"Bearer {sso_token}", "Content-Type": "application/json" } diff --git a/corebrain/cli/common.py b/corebrain/cli/common.py index 8599bee..022e221 100644 --- a/corebrain/cli/common.py +++ b/corebrain/cli/common.py @@ -2,7 +2,7 @@ Default values for SSO and API connection """ -DEFAULT_API_URL = "http://localhost:1000" # Use 5000 in Windows / 1000 in MacOS by default +DEFAULT_API_URL = "http://localhost:5000" # Use 5000 in Windows / 1000 in MacOS by default #DEFAULT_SSO_URL = "http://localhost:3000" # localhost DEFAULT_SSO_URL = "https://sso.globodain.com" # remote DEFAULT_PORT = 8765 From 00ceb178711023a0a39380948e3ff519c983ccac Mon Sep 17 00:00:00 2001 From: Piotr_Piskorz Date: Tue, 27 May 2025 15:35:14 +0200 Subject: [PATCH 71/81] fixing comments comments now work properly --- .../csharp/CorebrainCS/CorebrainCS.cs | 399 ++++++++++-------- 1 file changed, 213 insertions(+), 186 deletions(-) diff --git a/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs b/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs index 1b7522f..d421f28 100644 --- a/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs +++ b/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs @@ -1,186 +1,213 @@ -namespace CorebrainCS; - -using System; -using System.Diagnostics; -using System.Collections.Generic; - -/// -/// Creates the main corebrain interface. -/// -/// Path to the python which works with the corebrain cli, for example if you create the ./.venv you pass the path to the ./.venv python executable -/// Path to the corebrain cli script, if you installed it globally you just pass the `corebrain` path -/// -public class CorebrainCS(string pythonPath = "python", string scriptPath = "corebrain", bool verbose = false) -{ - private readonly string _pythonPath = Path.GetFullPath(pythonPath); - private readonly string _scriptPath = Path.GetFullPath(scriptPath); - private readonly bool _verbose = verbose; - - /// Shows help message with all available commands - - public string Help() - { - return ExecuteCommand("--help"); - } - - - /// Shows the current version of the Corebrain SDK - - public string Version() - { - return ExecuteCommand("--version"); - } - - /// Checks system status including: - /// - API Server status - /// - Redis status - /// - SSO Server status - /// - MongoDB status - /// - Required libraries installation - - public string CheckStatus() - { - return ExecuteCommand("--check-status"); - } - - - /// Checks system status with optional API URL and token parameters - - public string CheckStatus(string? apiUrl = null, string? token = null) - { - var args = new List { "--check-status" }; - - if (!string.IsNullOrEmpty(apiUrl)) - { - if (!Uri.IsWellFormedUriString(apiUrl, UriKind.Absolute)) - throw new ArgumentException("Invalid API URL format", nameof(apiUrl)); - - args.Add($"--api-url \"{apiUrl}\""); - } - - if (!string.IsNullOrEmpty(token)) - args.Add($"--token \"{token}\""); - - return ExecuteCommand(string.Join(" ", args)); - } - - /// Authenticates with SSO using username and password - - public string Authentication(string username, string password) - { - if (string.IsNullOrWhiteSpace(username)) - { - throw new ArgumentException("Username cannot be empty or whitespace", nameof(username)); - } - - if (string.IsNullOrWhiteSpace(password)) - { - throw new ArgumentException("Password cannot be empty or whitespace", nameof(password)); - } - - var escapedUsername = username.Replace("\"", "\\\""); - var escapedPassword = password.Replace("\"", "\\\""); - - - return ExecuteCommand($"--authentication --username \"{escapedUsername}\" --password \"{escapedPassword}\""); - } - - - /// Authenticates with SSO using a token - - public string AuthenticationWithToken(string token) - { - if (string.IsNullOrWhiteSpace(token)) - { - throw new ArgumentException("Token cannot be empty or whitespace", nameof(token)); - } - - var escapedToken = token.Replace("\"", "\\\""); - return ExecuteCommand($"--authentication --token \"{escapedToken}\""); - } - - - /// Creates a new user account and generates an associated API Key - - public string CreateUser() - { - return ExecuteCommand("--create-user"); - } - - /// Launches the configuration wizard for setting up database connections - - public string Configure() - { - return ExecuteCommand("--configure"); - } - - /// Lists all available database configurations - - public string ListConfigs() - { - return ExecuteCommand("--list-configs"); - } - - /// Displays the database schema for a configured database - public string ShowSchema() - { - return ExecuteCommand("--show-schema"); - } - - /// Displays information about the currently authenticated user - public string WhoAmI() - { - return ExecuteCommand("--woami"); - } - - /// Launches the web-based graphical user interface - public string Gui() - { - return ExecuteCommand("--gui"); - } - - - private string ExecuteCommand(string arguments) - - { - if (_verbose) - { - Console.WriteLine($"Executing: {_pythonPath} {_scriptPath} {arguments}"); - } - - var process = new Process - { - StartInfo = new ProcessStartInfo - { - FileName = _pythonPath, - Arguments = $"\"{_scriptPath}\" {arguments}", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - } - }; - - process.Start(); - var output = process.StandardOutput.ReadToEnd(); - var error = process.StandardError.ReadToEnd(); - process.WaitForExit(); - - if (_verbose) - { - Console.WriteLine("Command output:"); - Console.WriteLine(output); - if (!string.IsNullOrEmpty(error)) - { - Console.WriteLine("Error output:\n" + error); - } - } - - if (!string.IsNullOrEmpty(error)) - { - throw new InvalidOperationException($"Python CLI error: {error}"); - } - - return output.Trim(); - } -} \ No newline at end of file +namespace CorebrainCS; + +using System; +using System.Diagnostics; +using System.Collections.Generic; + +/// +/// Creates the main corebrain interface. +/// +///Path to the python which works with the corebrain cli, for example if you create the ./.venv you pass the path to the ./.venv python executable +/// Path to the corebrain cli script, if you installed it globally you just pass the `corebrain` path +/// +public class CorebrainCS(string pythonPath = "python", string scriptPath = "corebrain", bool verbose = false) +{ + private readonly string _pythonPath = Path.GetFullPath(pythonPath); + private readonly string _scriptPath = Path.GetFullPath(scriptPath); + private readonly bool _verbose = verbose; + + /// + /// Shows help message with all available commands + /// + /// + public string Help() + { + return ExecuteCommand("--help"); + } + + /// + /// Shows the current version of the Corebrain SDK + /// + /// + public string Version() + { + return ExecuteCommand("--version"); + } + + /// + /// Checks system status including: + /// + /// + /// - API Server status + /// - Redis status + /// - SSO Server status + /// - MongoDB status + /// - Required libraries installation + public string CheckStatus() + { + return ExecuteCommand("--check-status"); + } + /// + /// Checks system status with optional API URL and token parameters + /// + /// + /// + /// + /// + public string CheckStatus(string? apiUrl = null, string? token = null) + { + var args = new List { "--check-status" }; + + if (!string.IsNullOrEmpty(apiUrl)) + { + if (!Uri.IsWellFormedUriString(apiUrl, UriKind.Absolute)) + throw new ArgumentException("Invalid API URL format", nameof(apiUrl)); + + args.Add($"--api-url \"{apiUrl}\""); + } + + if (!string.IsNullOrEmpty(token)) + args.Add($"--token \"{token}\""); + + return ExecuteCommand(string.Join(" ", args)); + } + + /// + /// Authenticates with SSO using username and password + /// + /// + /// + /// + /// + public string Authentication(string username, string password) + { + if (string.IsNullOrWhiteSpace(username)) + { + throw new ArgumentException("Username cannot be empty or whitespace", nameof(username)); + } + + if (string.IsNullOrWhiteSpace(password)) + { + throw new ArgumentException("Password cannot be empty or whitespace", nameof(password)); + } + + var escapedUsername = username.Replace("\"", "\\\""); + var escapedPassword = password.Replace("\"", "\\\""); + + + return ExecuteCommand($"--authentication --username \"{escapedUsername}\" --password \"{escapedPassword}\""); + } + + /// + /// Authenticates with SSO using a token + /// + /// + /// + /// + public string AuthenticationWithToken(string token) + { + if (string.IsNullOrWhiteSpace(token)) + { + throw new ArgumentException("Token cannot be empty or whitespace", nameof(token)); + } + + var escapedToken = token.Replace("\"", "\\\""); + return ExecuteCommand($"--authentication --token \"{escapedToken}\""); + } + + /// + /// Creates a new user account and generates an associated API Key + /// + /// + public string CreateUser() + { + return ExecuteCommand("--create-user"); + } + + /// + /// Launches the configuration wizard for setting up database connections + /// + /// + public string Configure() + { + return ExecuteCommand("--configure"); + } + + /// + /// Lists all available database configurations + /// + /// + public string ListConfigs() + { + return ExecuteCommand("--list-configs"); + } + + /// + /// Displays the database schema for a configured database + /// + /// + public string ShowSchema() + { + return ExecuteCommand("--show-schema"); + } + + /// + /// Displays information about the currently authenticated user + /// + /// + public string WhoAmI() + { + return ExecuteCommand("--woami"); + } + + /// + /// Launches the web-based graphical user interface + /// + /// + public string Gui() + { + return ExecuteCommand("--gui"); + } + private string ExecuteCommand(string arguments) + { + if (_verbose) + { + Console.WriteLine($"Executing: {_pythonPath} {_scriptPath} {arguments}"); + } + + var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = _pythonPath, + Arguments = $"\"{_scriptPath}\" {arguments}", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + process.Start(); + var output = process.StandardOutput.ReadToEnd(); + var error = process.StandardError.ReadToEnd(); + process.WaitForExit(); + + if (_verbose) + { + Console.WriteLine("Command output:"); + Console.WriteLine(output); + if (!string.IsNullOrEmpty(error)) + { + Console.WriteLine("Error output:\n" + error); + } + } + + if (!string.IsNullOrEmpty(error)) + { + throw new InvalidOperationException($"Python CLI error: {error}"); + } + + return output.Trim(); + } +} From 74247cf86d77c98c67b221862a8bf6eab1f50acb Mon Sep 17 00:00:00 2001 From: Albert Stankowski Date: Tue, 27 May 2025 16:39:22 +0100 Subject: [PATCH 72/81] Add translations to commands --- corebrain/__init__.py | 8 +- corebrain/cli/__init__.py | 6 +- corebrain/cli/auth/__init__.py | 2 +- corebrain/config/__init__.py | 2 +- corebrain/config/manager.py | 6 +- corebrain/core/client.py | 124 +++++++++---------- corebrain/core/query.py | 212 ++++++++++++++++----------------- corebrain/core/test_utils.py | 30 ++--- corebrain/sdk.py | 2 +- setup.sh | 1 - 10 files changed, 196 insertions(+), 197 deletions(-) diff --git a/corebrain/__init__.py b/corebrain/__init__.py index d1107b1..6d2bc2e 100644 --- a/corebrain/__init__.py +++ b/corebrain/__init__.py @@ -7,16 +7,16 @@ import logging from typing import Dict, Any, List, Optional -# Configuración básica de logging +# Basic logging configuration logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) -# Importaciones seguras (sin dependencias circulares) +# Safe imports (no circular dependencies) from corebrain.db.engines import get_available_engines from corebrain.core.client import Corebrain from corebrain.config.manager import ConfigManager -# Exportación explícita de componentes públicos +# Explicit export of public components __all__ = [ 'init', 'extract_db_schema', @@ -40,7 +40,7 @@ def init(api_key: str, config_id: str, skip_verification: bool = False) -> Coreb """ return Corebrain(api_key=api_key, config_id=config_id, skip_verification=skip_verification) -# Funciones de conveniencia a nivel de paquete +# Package-level convenience features def list_configurations(api_key: str) -> List[str]: """ Lists the available configurations for an API key. diff --git a/corebrain/cli/__init__.py b/corebrain/cli/__init__.py index 53672d1..108326c 100644 --- a/corebrain/cli/__init__.py +++ b/corebrain/cli/__init__.py @@ -7,7 +7,7 @@ import sys from typing import Optional, List -# Importar componentes principales para CLI +# Import core components for CLI from corebrain.cli.commands import main_cli from corebrain.cli.utils import print_colored, ProgressTracker, get_free_port from corebrain.cli.config import ( @@ -26,7 +26,7 @@ ) -# Exportación explícita de componentes públicos +# Explicit export of public components __all__ = [ 'main_cli', 'run_cli', @@ -40,7 +40,7 @@ 'verify_api_token' ] -# Función de conveniencia para ejecutar CLI +# Convenience function for running CLI def run_cli(argv: Optional[List[str]] = None) -> int: """ Run the CLI with the provided arguments. diff --git a/corebrain/cli/auth/__init__.py b/corebrain/cli/auth/__init__.py index 873572d..a601c7b 100644 --- a/corebrain/cli/auth/__init__.py +++ b/corebrain/cli/auth/__init__.py @@ -11,7 +11,7 @@ verify_api_token, get_api_key_id_from_token ) -# Exportación explícita de componentes públicos +# Explicit export of public components __all__ = [ 'authenticate_with_sso', 'TokenHandler', diff --git a/corebrain/config/__init__.py b/corebrain/config/__init__.py index 7149948..3363886 100644 --- a/corebrain/config/__init__.py +++ b/corebrain/config/__init__.py @@ -6,5 +6,5 @@ """ from .manager import ConfigManager -# Exportación explícita de componentes públicos +# Explicit export of public components __all__ = ['ConfigManager'] \ No newline at end of file diff --git a/corebrain/config/manager.py b/corebrain/config/manager.py index e41f278..bbd7c96 100644 --- a/corebrain/config/manager.py +++ b/corebrain/config/manager.py @@ -42,7 +42,7 @@ def export_config(filepath="config.json"): json.dump(config, f, indent=4) print(f"Configuration exported to {filepath}") -# Función para imprimir mensajes coloreados +# Function to print colored messages def _print_colored(message: str, color: str) -> None: """Simplified version of _print_colored that does not depend on cli.utils.""" colors = { @@ -114,11 +114,11 @@ def _load_configs(self) -> Dict[str, Dict[str, Any]]: return {} try: - # Intentar descifrar los datos + # Trying to decipher the data decrypted_data = self.cipher.decrypt(encrypted_data.encode()).decode() configs = json.loads(decrypted_data) except Exception as e: - # Si falla el descifrado, intentar cargar como JSON plano + # If decryption fails, attempt to load as plain JSON logger.warning(f"Error decrypting configuration: {e}") configs = json.loads(encrypted_data) diff --git a/corebrain/core/client.py b/corebrain/core/client.py index 9b65225..7a83c57 100644 --- a/corebrain/core/client.py +++ b/corebrain/core/client.py @@ -480,8 +480,8 @@ def _extract_db_schema(self, detail_level: str = "full", specific_collections: L "type": db_type, "database": self.db_config.get("database", ""), "tables": {}, - "total_collections": 0, # Añadir contador total - "included_collections": 0 # Contador de incluidas + "total_collections": 0, # Add total counter + "included_collections": 0 # Included counter } excluded_tables = set(self.db_config.get("excluded_tables", [])) logger.info(f"Tablas excluidas: {excluded_tables}") @@ -496,7 +496,7 @@ def _extract_db_schema(self, detail_level: str = "full", specific_collections: L if engine == "sqlite": logger.info("Obteniendo tablas de SQLite") - # Obtener listado de tablas + # Get list of tables cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") tables = cursor.fetchall() logger.info(f"Tablas encontradas en SQLite: {tables}") @@ -516,18 +516,18 @@ def _extract_db_schema(self, detail_level: str = "full", specific_collections: L tables = cursor.fetchall() logger.info(f"Tablas encontradas en PostgreSQL: {tables}") - # Procesar las tablas encontradas + # Process the found tables for table in tables: table_name = table[0] logger.info(f"Procesando tabla: {table_name}") - # Saltar tablas excluidas + # Skip excluded tables if table_name in excluded_tables: logger.info(f"Saltando tabla excluida: {table_name}") continue try: - # Obtener información de columnas según el motor + # Get column information based on the engine if engine == "sqlite": cursor.execute(f"PRAGMA table_info({table_name});") elif engine == "mysql": @@ -542,7 +542,7 @@ def _extract_db_schema(self, detail_level: str = "full", specific_collections: L columns = cursor.fetchall() logger.info(f"Columnas encontradas para {table_name}: {columns}") - # Estructura de columnas según el motor + # Column structure according to the engine if engine == "sqlite": column_info = [{"name": col[1], "type": col[2]} for col in columns] elif engine == "mysql": @@ -550,17 +550,17 @@ def _extract_db_schema(self, detail_level: str = "full", specific_collections: L elif engine == "postgresql": column_info = [{"name": col[0], "type": col[1]} for col in columns] - # Guardar información de la tabla + # Save table information schema["tables"][table_name] = { "columns": column_info, - "sample_data": [] # No obtenemos datos de muestra por defecto + "sample_data": [] # We don't get sample data by default } except Exception as e: logger.error(f"Error procesando tabla {table_name}: {str(e)}") else: - # Usando SQLAlchemy + # Using SQLAlchemy logger.info("Usando SQLAlchemy para obtener el esquema") inspector = inspect(self.db_connection) table_names = inspector.get_table_names() @@ -598,12 +598,12 @@ def _extract_db_schema(self, detail_level: str = "full", specific_collections: L logger.error(f"Error al obtener colecciones MongoDB: {str(e)}") return schema - # Si solo queremos los nombres + # If we only want the names if detail_level == "names_only": schema["collection_names"] = collection_names return schema - # Procesar cada colección + # Process each collection for collection_name in collection_names: if collection_name in excluded_tables: logger.info(f"Saltando colección excluida: {collection_name}") @@ -611,7 +611,7 @@ def _extract_db_schema(self, detail_level: str = "full", specific_collections: L try: collection = self.db_connection[collection_name] - # Obtener un documento para inferir estructura + # Convert table dictionary to list first_doc = collection.find_one() if first_doc: @@ -638,7 +638,7 @@ def _extract_db_schema(self, detail_level: str = "full", specific_collections: L except Exception as e: logger.error(f"Error general procesando MongoDB: {str(e)}") - # Convertir el diccionario de tablas en una lista + # Convert table dictionary to list table_list = [] for table_name, table_info in schema["tables"].items(): table_data = {"name": table_name} @@ -684,27 +684,27 @@ def ask(self, question: str, **kwargs) -> Dict: Dictionary with the query results and explanation """ try: - # Verificar opciones de comportamiento + # Check behavior options execute_query = kwargs.get("execute_query", True) explain_results = kwargs.get("explain_results", True) - # Obtener esquema con el nivel de detalle apropiado + # Obtain an outline with the appropriate level of detail detail_level = kwargs.get("detail_level", "full") schema = self._extract_db_schema(detail_level=detail_level) - # Validar que el esquema tiene tablas/colecciones + # Validate that the schema has tables/collections if not schema.get("tables"): print("Error: No se encontraron tablas/colecciones en la base de datos") return {"error": True, "explanation": "No se encontraron tablas/colecciones en la base de datos"} - # Obtener nombres de tablas disponibles para validación + # Get table names available for validation available_tables = set() if isinstance(schema.get("tables"), dict): available_tables.update(schema["tables"].keys()) elif isinstance(schema.get("tables_list"), list): available_tables.update(table["name"] for table in schema["tables_list"]) - # Preparar datos de la solicitud con información de esquema mejorada + # Prepare application data with enhanced schema information request_data = { "question": question, "db_schema": schema, @@ -718,30 +718,30 @@ def ask(self, question: str, **kwargs) -> Dict: } } - # Añadir configuración de la base de datos al request - # Esto permite a la API ejecutar directamente las consultas si es necesario + # Add database configuration to the request + # This allows the API to directly execute queries if needed. if execute_query: request_data["db_config"] = self.db_config - # Añadir datos de usuario si están disponibles + # Add user data if available if self.user_data: request_data["user_data"] = self.user_data - # Preparar headers para la solicitud + # Prepare headers for the request headers = { "X-API-Key": self.api_key, "Content-Type": "application/json" } - # Determinar el endpoint adecuado según el modo de ejecución + # Determine the appropriate endpoint based on the execution mode if execute_query: - # Usar el endpoint de ejecución completa + # Use the full execution endpoint endpoint = f"{self.api_url}/api/database/sdk/query" else: - # Usar el endpoint de solo generación de consulta + # Use the query-only generation endpoint endpoint = f"{self.api_url}/api/database/generate" - # Realizar solicitud a la API + # Make a request to the API response = httpx.post( endpoint, headers=headers, @@ -749,7 +749,7 @@ def ask(self, question: str, **kwargs) -> Dict: timeout=60.0 ) - # Verificar respuesta + # Check answer if response.status_code != 200: error_msg = f"Error {response.status_code} al realizar la consulta" try: @@ -760,29 +760,29 @@ def ask(self, question: str, **kwargs) -> Dict: error_msg += f": {response.text}" return {"error": True, "explanation": error_msg} - # Procesar respuesta de la API + # Process API response api_response = response.json() - # Verificar si la API reportó un error + # Check if the API reported an error if api_response.get("error", False): return api_response - # Verificar si se generó una consulta válida + # Check if a valid query was generated if "query" not in api_response: return { "error": True, "explanation": "La API no generó una consulta válida." } - # Si se debe ejecutar la consulta pero la API no lo hizo - # (esto ocurriría solo en caso de cambios de configuración o fallbacks) + # If the query should be executed but the API did not + # (this would only occur in the case of configuration changes or fallbacks) if execute_query and "result" not in api_response: try: - # Preparar la consulta para ejecución local + # Prepare the query for local execution query_type = self.db_config.get("engine", "").lower() if self.db_config["type"].lower() == "sql" else self.db_config["type"].lower() query_value = api_response["query"] - # Para SQL, asegurarse de que la consulta es un string + # For SQL, make sure the query is a string if query_type in ["sqlite", "mysql", "postgresql"]: if isinstance(query_value, dict): sql_candidate = query_value.get("sql") or query_value.get("query") @@ -791,15 +791,15 @@ def ask(self, question: str, **kwargs) -> Dict: else: raise CorebrainError(f"La consulta SQL generada no es un string: {query_value}") - # Preparar la consulta con el formato adecuado + # Prepare the consultation with the appropriate format query_to_execute = { "type": query_type, "query": query_value } - # Para MongoDB, añadir información específica + # For MongoDB, add specific information if query_type in ["nosql", "mongodb"]: - # Obtener nombre de colección + # Get collection name collection_name = None if isinstance(api_response["query"], dict): collection_name = api_response["query"].get("collection") @@ -810,29 +810,29 @@ def ask(self, question: str, **kwargs) -> Dict: if not collection_name and available_tables: collection_name = list(available_tables)[0] - # Validar nombre de colección + # Validate collection name if not collection_name: raise CorebrainError("No se especificó colección y no se encontraron colecciones en el esquema") if not isinstance(collection_name, str) or not collection_name.strip(): raise CorebrainError("Nombre de colección inválido: debe ser un string no vacío") - # Añadir colección a la consulta + # Add collection to query query_to_execute["collection"] = collection_name - # Añadir tipo de operación + # Add operation type if isinstance(api_response["query"], dict): query_to_execute["operation"] = api_response["query"].get("operation", "find") - # Añadir límite si se especifica + # Add limit if specified if "limit" in kwargs: query_to_execute["limit"] = kwargs["limit"] - # Ejecutar la consulta + # Run the query start_time = datetime.now() query_result = self._execute_query(query_to_execute) query_time_ms = int((datetime.now() - start_time).total_seconds() * 1000) - # Actualizar la respuesta con los resultados + # Update the response with the results api_response["result"] = { "data": query_result, "count": len(query_result) if isinstance(query_result, list) else 1, @@ -840,13 +840,13 @@ def ask(self, question: str, **kwargs) -> Dict: "has_more": False } - # Si se debe generar explicación pero la API no lo hizo + # If explanation should be generated but API didn't do it if explain_results and ( "explanation" not in api_response or not isinstance(api_response.get("explanation"), str) or - len(str(api_response.get("explanation", ""))) < 15 # Detectar explicaciones numéricas o muy cortas + len(str(api_response.get("explanation", ""))) < 15 # Detect numerical or very short explanations ): - # Preparar datos para obtener explicación + # Prepare data for explanation explanation_data = { "question": question, "query": api_response["query"], @@ -861,7 +861,7 @@ def ask(self, question: str, **kwargs) -> Dict: } try: - # Obtener explicación de la API + # Get API explanation explanation_response = httpx.post( f"{self.api_url}/api/database/sdk/query/explain", headers=headers, @@ -890,9 +890,9 @@ def ask(self, question: str, **kwargs) -> Dict: } } - # Verificar si la explicación es un número (probablemente el tiempo de ejecución) y corregirlo + # Check if the explanation is a number (probably the runtime) and correct it if "explanation" in api_response and not isinstance(api_response["explanation"], str): - # Si la explicación es un número, reemplazarla con una explicación generada + # If the explanation is a number, replace it with a generated explanation try: is_sql = False if "query" in api_response: @@ -908,7 +908,7 @@ def ask(self, question: str, **kwargs) -> Dict: sql_query = api_response["query"].get("sql", "") api_response["explanation"] = self._generate_sql_explanation(sql_query, result_data) else: - # Para MongoDB o genérico + # For MongoDB or generic api_response["explanation"] = self._generate_generic_explanation(api_response["query"], result_data) else: api_response["explanation"] = "La consulta se ha ejecutado correctamente." @@ -916,7 +916,7 @@ def ask(self, question: str, **kwargs) -> Dict: logger.error(f"Error al corregir explicación: {str(exp_fix_error)}") api_response["explanation"] = "La consulta se ha ejecutado correctamente." - # Preparar la respuesta final + # Prepare the final response result = { "question": question, "query": api_response["query"], @@ -926,7 +926,7 @@ def ask(self, question: str, **kwargs) -> Dict: } } - # Añadir resultados si están disponibles + # Add results if available if "result" in api_response: if isinstance(api_response["result"], dict) and "data" in api_response["result"]: result["result"] = api_response["result"] @@ -938,7 +938,7 @@ def ask(self, question: str, **kwargs) -> Dict: "has_more": False } - # Añadir explicación si está disponible + # Add explanation if available if "explanation" in api_response: result["explanation"] = api_response["explanation"] @@ -967,7 +967,7 @@ def _generate_fallback_explanation(self, query, results): Returns: Generated explanation """ - # Determinar si es SQL o MongoDB + # Determine if it is SQL or MongoDB if isinstance(query, dict): query_type = query.get("type", "").lower() @@ -976,7 +976,7 @@ def _generate_fallback_explanation(self, query, results): elif query_type in ["nosql", "mongodb"]: return self._generate_mongodb_explanation(query, results) - # Fallback genérico + # Generic Fallback result_count = len(results) if isinstance(results, list) else (1 if results else 0) return f"La consulta devolvió {result_count} resultados." @@ -994,7 +994,7 @@ def _generate_sql_explanation(self, sql_query, results): sql_lower = sql_query.lower() if isinstance(sql_query, str) else "" result_count = len(results) if isinstance(results, list) else (1 if results else 0) - # Extraer nombres de tablas si es posible + # Extract table names if possible tables = [] from_match = re.search(r'from\s+([a-zA-Z0-9_]+)', sql_lower) if from_match: @@ -1004,7 +1004,7 @@ def _generate_sql_explanation(self, sql_query, results): if join_matches: tables.extend(join_matches) - # Detectar tipo de consulta + # Detect query type if "select" in sql_lower: if "join" in sql_lower: if len(tables) > 1: @@ -1021,7 +1021,7 @@ def _generate_sql_explanation(self, sql_query, results): else: return f"La consulta devolvió {result_count} registros de la base de datos." - # Para otros tipos de consultas (INSERT, UPDATE, DELETE) + # For other types of queries (INSERT, UPDATE, DELETE) if "insert" in sql_lower: return "Se insertaron correctamente los datos en la base de datos." elif "update" in sql_lower: @@ -1029,7 +1029,7 @@ def _generate_sql_explanation(self, sql_query, results): elif "delete" in sql_lower: return "Se eliminaron correctamente los datos de la base de datos." - # Fallback genérico + # Generic Fallback return f"La consulta SQL se ejecutó correctamente y devolvió {result_count} resultados." @@ -1048,7 +1048,7 @@ def _generate_mongodb_explanation(self, query, results): operation = query.get("operation", "find") result_count = len(results) if isinstance(results, list) else (1 if results else 0) - # Generar explicación según la operación + # Generate explanation according to the operation if operation == "find": return f"Se encontraron {result_count} documentos en la colección {collection} que coinciden con los criterios de búsqueda." elif operation == "findOne": @@ -1065,7 +1065,7 @@ def _generate_mongodb_explanation(self, query, results): elif operation == "deleteOne": return f"Se ha eliminado correctamente un documento de la colección {collection}." - # Fallback genérico + # Generic Fallback return f"La operación {operation} se ejecutó correctamente en la colección {collection} y devolvió {result_count} resultados." diff --git a/corebrain/core/query.py b/corebrain/core/query.py index b23eddb..4d9f87b 100644 --- a/corebrain/core/query.py +++ b/corebrain/core/query.py @@ -27,23 +27,23 @@ def __init__(self, cache_dir: str = None, ttl: int = 86400, memory_limit: int = ttl: Time-to-live of the cache in seconds (default: 24 hours) memory_limit: Memory cache entry limit """ - # Caché en memoria (más rápido, pero volátil) + # In-memory cache (faster, but volatile) self.memory_cache = {} self.memory_timestamps = {} self.memory_limit = memory_limit - self.memory_lru = [] # Lista para seguimiento de menos usados recientemente + self.memory_lru = [] # Least recently used tracking list - # Caché persistente (más lento, pero permanente) + # Persistent cache (slower, but permanent) self.ttl = ttl if cache_dir: self.cache_dir = Path(cache_dir) else: self.cache_dir = Path.home() / ".corebrain_cache" - # Crear directorio de caché si no existe + # Create cache directory if it does not exist self.cache_dir.mkdir(parents=True, exist_ok=True) - # Inicializar base de datos SQLite para metadatos + # Initialize SQLite database for metadata self.db_path = self.cache_dir / "cache_metadata.db" self._init_db() @@ -54,7 +54,7 @@ def _init_db(self): conn = sqlite3.connect(str(self.db_path)) cursor = conn.cursor() - # Crear tabla de metadatos si no existe + # Create metadata table if it does not exist cursor.execute(''' CREATE TABLE IF NOT EXISTS cache_metadata ( query_hash TEXT PRIMARY KEY, @@ -71,21 +71,21 @@ def _init_db(self): def _get_hash(self, query: str, config_id: str, collection_name: Optional[str] = None) -> str: """Generates a unique hash for the query.""" - # Normalizar la consulta (eliminar espacios extra, convertir a minúsculas) + # Normalize the query (remove extra spaces, convert to lowercase) normalized_query = re.sub(r'\s+', ' ', query.lower().strip()) - # Crear string compuesto para el hash + # Create composite string for the hash hash_input = f"{normalized_query}|{config_id}" if collection_name: hash_input += f"|{collection_name}" - # Generar el hash + # Generate the hash return hashlib.md5(hash_input.encode()).hexdigest() def _get_cache_path(self, query_hash: str) -> Path: """Gets the cache file path for a given hash.""" - # Usar los primeros caracteres del hash para crear subdirectorios - # Esto evita tener demasiados archivos en un solo directorio + # Use the first characters of the hash to create subdirectories + # This prevents having too many files in a single directory subdir = query_hash[:2] cache_subdir = self.cache_dir / subdir cache_subdir.mkdir(exist_ok=True) @@ -99,12 +99,12 @@ def _update_metadata(self, query_hash: str, query: str, config_id: str): now = datetime.now().isoformat() - # Verificar si el hash ya existe + # Check if the hash already exists cursor.execute("SELECT hit_count FROM cache_metadata WHERE query_hash = ?", (query_hash,)) result = cursor.fetchone() if result: - # Actualizar entrada existente + # Update existing entry hit_count = result[0] + 1 cursor.execute(''' UPDATE cache_metadata @@ -112,7 +112,7 @@ def _update_metadata(self, query_hash: str, query: str, config_id: str): WHERE query_hash = ? ''', (now, hit_count, query_hash)) else: - # Insertar nueva entrada + # Insert new entry cursor.execute(''' INSERT INTO cache_metadata (query_hash, query, config_id, created_at, last_accessed, hit_count) VALUES (?, ?, ?, ?, ?, 1) @@ -124,12 +124,12 @@ def _update_metadata(self, query_hash: str, query: str, config_id: str): def _update_memory_lru(self, query_hash: str): """Updates the LRU (Least Recently Used) list for the in-memory cache.""" if query_hash in self.memory_lru: - # Mover al final (más recientemente usado) + # Move to end (most recently used) self.memory_lru.remove(query_hash) self.memory_lru.append(query_hash) - # Si excedemos el límite, eliminar el elemento menos usado recientemente + # If we exceed the limit, delete the least recently used item if len(self.memory_lru) > self.memory_limit: oldest_hash = self.memory_lru.pop(0) if oldest_hash in self.memory_cache: @@ -150,7 +150,7 @@ def get(self, query: str, config_id: str, collection_name: Optional[str] = None) """ query_hash = self._get_hash(query, config_id, collection_name) - # 1. Verificar caché en memoria (más rápido) + # 1. Check in-memory cache (faster) if query_hash in self.memory_cache: timestamp = self.memory_timestamps[query_hash] if (time.time() - timestamp) < self.ttl: @@ -159,23 +159,23 @@ def get(self, query: str, config_id: str, collection_name: Optional[str] = None) print_colored(f"Cache hit (memory): {query[:30]}...", "green") return self.memory_cache[query_hash] else: - # Expirado en memoria + # Expired in memory del self.memory_cache[query_hash] del self.memory_timestamps[query_hash] if query_hash in self.memory_lru: self.memory_lru.remove(query_hash) - # 2. Verificar caché en disco + # 2. Check disk cache cache_path = self._get_cache_path(query_hash) if cache_path.exists(): - # Verificar edad del archivo + # Check file age file_age = time.time() - cache_path.stat().st_mtime if file_age < self.ttl: try: with open(cache_path, 'rb') as f: result = pickle.load(f) - # Guardar también en caché de memoria + # Also save in memory cache self.memory_cache[query_hash] = result self.memory_timestamps[query_hash] = time.time() self._update_memory_lru(query_hash) @@ -185,10 +185,10 @@ def get(self, query: str, config_id: str, collection_name: Optional[str] = None) return result except Exception as e: print_colored(f"Error al cargar caché: {str(e)}", "red") - # Si hay error al cargar, eliminar el archivo corrupto + # If there is an error when uploading, delete the corrupted file cache_path.unlink(missing_ok=True) else: - # Archivo expirado, eliminarlo + # Expired file, delete it cache_path.unlink(missing_ok=True) return None @@ -205,18 +205,18 @@ def set(self, query: str, config_id: str, result: Dict[str, Any], collection_nam """ query_hash = self._get_hash(query, config_id, collection_name) - # 1. Guardar en caché de memoria + # 1. Save to memory cache self.memory_cache[query_hash] = result self.memory_timestamps[query_hash] = time.time() self._update_memory_lru(query_hash) - # 2. Guardar en caché persistente + # 2. Save to persistent cache try: cache_path = self._get_cache_path(query_hash) with open(cache_path, 'wb') as f: pickle.dump(result, f) - # 3. Actualizar metadatos + # 3. Update metadata self._update_metadata(query_hash, query, config_id) print_colored(f"Cached: {query[:30]}...", "green") @@ -230,7 +230,7 @@ def clear(self, older_than: int = None): Args: older_than: Only clear entries older than this number of seconds """ - # Limpiar caché en memoria + # Clear cache in memory if older_than: current_time = time.time() keys_to_remove = [ @@ -250,15 +250,15 @@ def clear(self, older_than: int = None): self.memory_timestamps.clear() self.memory_lru.clear() - # Limpiar caché en disco + # Clear disk cache if older_than: cutoff_time = time.time() - older_than - # Usar la base de datos para encontrar archivos antiguos + # Using the database to find old files conn = sqlite3.connect(str(self.db_path)) cursor = conn.cursor() - # Convertir cutoff_time a formato ISO + # Convert cutoff_time to ISO format cutoff_datetime = datetime.fromtimestamp(cutoff_time).isoformat() cursor.execute( @@ -268,13 +268,13 @@ def clear(self, older_than: int = None): old_hashes = [row[0] for row in cursor.fetchall()] - # Eliminar archivos antiguos + # Delete old files for query_hash in old_hashes: cache_path = self._get_cache_path(query_hash) if cache_path.exists(): cache_path.unlink() - # Eliminar de la base de datos + # Delete from the database cursor.execute( "DELETE FROM cache_metadata WHERE query_hash = ?", (query_hash,) @@ -283,13 +283,13 @@ def clear(self, older_than: int = None): conn.commit() conn.close() else: - # Eliminar todos los archivos de caché + # Delete all cache files for subdir in self.cache_dir.iterdir(): if subdir.is_dir(): for cache_file in subdir.glob("*.cache"): cache_file.unlink() - # Reiniciar la base de datos + # Restart the database conn = sqlite3.connect(str(self.db_path)) cursor = conn.cursor() cursor.execute("DELETE FROM cache_metadata") @@ -298,27 +298,27 @@ def clear(self, older_than: int = None): def get_stats(self) -> Dict[str, Any]: """Gets cache statistics.""" - # Contar archivos en disco + # Count files on disk disk_count = 0 for subdir in self.cache_dir.iterdir(): if subdir.is_dir(): disk_count += len(list(subdir.glob("*.cache"))) - # Obtener estadísticas de la base de datos + # Obtaining database statistics conn = sqlite3.connect(str(self.db_path)) cursor = conn.cursor() - # Total de entradas + # Total entries cursor.execute("SELECT COUNT(*) FROM cache_metadata") total_entries = cursor.fetchone()[0] - # Consultas más frecuentes + # Most frequent queries cursor.execute( "SELECT query, hit_count FROM cache_metadata ORDER BY hit_count DESC LIMIT 5" ) top_queries = cursor.fetchall() - # Edad promedio + # Average age cursor.execute( "SELECT AVG(strftime('%s', 'now') - strftime('%s', created_at)) FROM cache_metadata" ) @@ -361,27 +361,27 @@ def __init__(self, pattern: str, description: str, self.db_type = db_type self.applicable_tables = applicable_tables or [] - # Compilar expresión regular para el patrón + # Compile regular expression for the pattern self.regex = self._compile_pattern(pattern) def _compile_pattern(self, pattern: str) -> re.Pattern: """Compiles the pattern into a regular expression.""" - # Reemplazar marcadores especiales con grupos de captura + # Replace special markers with capture groups regex_pattern = pattern - # {table} se convierte en grupo de captura para el nombre de tabla + # {table} becomes a capturing group for the table name regex_pattern = regex_pattern.replace("{table}", r"(\w+)") - # {field} se convierte en grupo de captura para el nombre de campo + # {field} becomes a capturing group for the field name regex_pattern = regex_pattern.replace("{field}", r"(\w+)") - # {value} se convierte en grupo de captura para un valor + # {value} becomes a capturing group for a value regex_pattern = regex_pattern.replace("{value}", r"([^,.\s]+)") - # {number} se convierte en grupo de captura para un número + # {number} becomes a capture group for a number regex_pattern = regex_pattern.replace("{number}", r"(\d+)") - # Hacer coincidir el patrón completo + # Match the entire pattern regex_pattern = f"^{regex_pattern}$" return re.compile(regex_pattern, re.IGNORECASE) @@ -413,22 +413,22 @@ def generate_query(self, params: List[str], db_schema: Dict[str, Any]) -> Option Generated query or None if it cannot be generated """ if self.generator_func: - # Usar función personalizada + # Use custom function return self.generator_func(params, db_schema) if not self.sql_template: return None - # Intentar aplicar la plantilla SQL con los parámetros + # Try to apply the SQL template with the parameters try: sql_query = self.sql_template - # Reemplazar parámetros en la plantilla + # Replace parameters in the template for i, param in enumerate(params): placeholder = f"${i+1}" sql_query = sql_query.replace(placeholder, param) - # Verificar si hay algún parámetro sin reemplazar + # Check if there are any unreplaced parameters if "$" in sql_query: return None @@ -455,16 +455,16 @@ def __init__(self, query_log_path: str = None, template_path: str = None): Path.home(), ".corebrain_cache", "templates.json" ) - # Inicializar base de datos + # Initialize database self._init_db() - # Plantillas predefinidas para consultas comunes + # Predefined templates for common queries self.templates = self._load_default_templates() - # Cargar plantillas personalizadas + # Upload custom templates self._load_custom_templates() - # Plantillas comunes para identificar patrones + # Common templates for identifying patterns self.common_patterns = [ r"muestra\s+(?:todos\s+)?los\s+(\w+)", r"lista\s+(?:de\s+)?(?:todos\s+)?los\s+(\w+)", @@ -475,13 +475,13 @@ def __init__(self, query_log_path: str = None, template_path: str = None): def _init_db(self): """Initializes the database for query logging.""" - # Asegurar que el directorio existe + # Ensure that the directory exists os.makedirs(os.path.dirname(self.query_log_path), exist_ok=True) conn = sqlite3.connect(self.query_log_path) cursor = conn.cursor() - # Crear tabla de registro si no existe + # Create log table if it does not exist cursor.execute(''' CREATE TABLE IF NOT EXISTS query_log ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -496,7 +496,7 @@ def _init_db(self): ) ''') - # Crear tabla de patrones detectados + # Create table of detected patterns cursor.execute(''' CREATE TABLE IF NOT EXISTS query_patterns ( pattern TEXT PRIMARY KEY, @@ -514,7 +514,7 @@ def _load_default_templates(self) -> List[QueryTemplate]: """Carga las plantillas predefinidas para consultas comunes.""" templates = [] - # Listar todos los registros de una tabla + # List all records in a table templates.append( QueryTemplate( pattern="muestra todos los {table}", @@ -524,7 +524,7 @@ def _load_default_templates(self) -> List[QueryTemplate]: ) ) - # Contar registros + # Count records templates.append( QueryTemplate( pattern="cuántos {table} hay", @@ -534,7 +534,7 @@ def _load_default_templates(self) -> List[QueryTemplate]: ) ) - # Buscar por ID + # Search by ID templates.append( QueryTemplate( pattern="busca el {table} con id {value}", @@ -544,7 +544,7 @@ def _load_default_templates(self) -> List[QueryTemplate]: ) ) - # Listar ordenados + # List sorted templates.append( QueryTemplate( pattern="lista los {table} ordenados por {field}", @@ -554,7 +554,7 @@ def _load_default_templates(self) -> List[QueryTemplate]: ) ) - # Buscar por email + # Search by email templates.append( QueryTemplate( pattern="busca el usuario con email {value}", @@ -564,7 +564,7 @@ def _load_default_templates(self) -> List[QueryTemplate]: ) ) - # Contar por campo + # Count by field templates.append( QueryTemplate( pattern="cuántos {table} hay por {field}", @@ -574,7 +574,7 @@ def _load_default_templates(self) -> List[QueryTemplate]: ) ) - # Contar usuarios activos + # Count active users templates.append( QueryTemplate( pattern="cuántos usuarios activos hay", @@ -585,7 +585,7 @@ def _load_default_templates(self) -> List[QueryTemplate]: ) ) - # Listar usuarios por fecha de registro + # List users by registration date templates.append( QueryTemplate( pattern="usuarios registrados en los últimos {number} días", @@ -601,7 +601,7 @@ def _load_default_templates(self) -> List[QueryTemplate]: ) ) - # Buscar empresas + # Search companies templates.append( QueryTemplate( pattern="usuarios que tienen empresa", @@ -617,7 +617,7 @@ def _load_default_templates(self) -> List[QueryTemplate]: ) ) - # Buscar negocios + # Find businesses templates.append( QueryTemplate( pattern="busca negocios en {value}", @@ -632,7 +632,7 @@ def _load_default_templates(self) -> List[QueryTemplate]: ) ) - # MongoDB: Listar documentos + # MongoDB: List documents templates.append( QueryTemplate( pattern="muestra todos los documentos de {table}", @@ -659,7 +659,7 @@ def _load_custom_templates(self): custom_templates = json.load(f) for template_data in custom_templates: - # Crear plantilla desde datos JSON + # Create template from JSON data template = QueryTemplate( pattern=template_data.get("pattern", ""), description=template_data.get("description", ""), @@ -683,7 +683,7 @@ def save_custom_template(self, template: QueryTemplate) -> bool: Returns: True if saved successfully """ - # Cargar plantillas existentes + # Load existing templates custom_templates = [] if os.path.exists(self.template_path): try: @@ -692,7 +692,7 @@ def save_custom_template(self, template: QueryTemplate) -> bool: except: custom_templates = [] - # Convertir plantilla a diccionario + # Convert template to dictionary template_data = { "pattern": template.pattern, "description": template.description, @@ -701,22 +701,22 @@ def save_custom_template(self, template: QueryTemplate) -> bool: "applicable_tables": template.applicable_tables } - # Verificar si ya existe una plantilla con el mismo patrón + # Check if a template with the same pattern already exists for i, existing in enumerate(custom_templates): if existing.get("pattern") == template.pattern: - # Actualizar existente + # Update existing custom_templates[i] = template_data break else: - # Agregar nueva + # Add new custom_templates.append(template_data) - # Guardar plantillas + # Save templates try: with open(self.template_path, 'w') as f: json.dump(custom_templates, f, indent=2) - # Actualizar lista de plantillas + # Update template list self.templates.append(template) return True @@ -738,7 +738,7 @@ def find_matching_template(self, query: str, db_schema: Dict[str, Any]) -> Optio for template in self.templates: matches, params = template.matches(query) if matches: - # Verificar si la plantilla es aplicable a las tablas existentes + # Check if the template is applicable to existing tables if template.applicable_tables: available_tables = set(db_schema.get("tables", {}).keys()) if not any(table in available_tables for table in template.applicable_tables): @@ -761,10 +761,10 @@ def log_query(self, query: str, config_id: str, collection_name: str = None, cost: Estimated cost of the query result_count: Number of results obtained """ - # Detectar patrón + # Detect pattern pattern = self._detect_pattern(query) - # Registrar en la base de datos + # Register in the database conn = sqlite3.connect(self.query_log_path) cursor = conn.cursor() @@ -776,7 +776,7 @@ def log_query(self, query: str, config_id: str, collection_name: str = None, execution_time, cost, result_count, pattern )) - # Actualizar estadísticas de patrones + # Update pattern statistics if pattern: cursor.execute( "SELECT count, avg_execution_time, avg_cost FROM query_patterns WHERE pattern = ?", @@ -785,7 +785,7 @@ def log_query(self, query: str, config_id: str, collection_name: str = None, result = cursor.fetchone() if result: - # Actualizar patrón existente + # Update existing pattern count, avg_exec_time, avg_cost = result new_count = count + 1 new_avg_exec_time = (avg_exec_time * count + execution_time) / new_count @@ -797,7 +797,7 @@ def log_query(self, query: str, config_id: str, collection_name: str = None, WHERE pattern = ? ''', (new_count, new_avg_exec_time, new_avg_cost, datetime.now().isoformat(), pattern)) else: - # Insertar nuevo patrón + # Insert new pattern cursor.execute(''' INSERT INTO query_patterns (pattern, count, avg_execution_time, avg_cost, last_updated) VALUES (?, 1, ?, ?, ?) @@ -818,20 +818,20 @@ def _detect_pattern(self, query: str) -> Optional[str]: """ normalized_query = query.lower() - # Comprobar patrones predefinidos + # Check predefined patterns for pattern in self.common_patterns: match = re.search(pattern, normalized_query) if match: - # Devolver el patrón con comodines + # Return the pattern with wildcards entity = match.group(1) return pattern.replace(r'(\w+)', f"{entity}") - # Si no se detecta ningún patrón predefinido, intentar generalizar + # If no predefined pattern is detected, try to generalize words = normalized_query.split() if len(words) < 3: return None - # Intentar generalizar consultas simples + # Try to generalize simple queries if "mostrar" in words or "muestra" in words or "listar" in words or "lista" in words: for i, word in enumerate(words): if word in ["de", "los", "las", "todos", "todas"]: @@ -885,36 +885,36 @@ def suggest_new_template(self, query: str, sql_query: str) -> Optional[QueryTemp Returns: Suggested template or None """ - # Detectar patrón + # Detect pattern pattern = self._detect_pattern(query) if not pattern: return None - # Generalizar la consulta SQL + # Generalize the SQL query generalized_sql = sql_query - # Reemplazar valores específicos con marcadores - # Esto es una simplificación, idealmente se usaría un parser SQL + # Replace specific values ​​with markers + # This is a simplification; ideally, you would use an SQL parser tokens = query.lower().split() - # Identificar posibles valores a parametrizar + # Identify possible values ​​to parameterize for i, token in enumerate(tokens): if token.isdigit(): - # Reemplazar números + # Replace numbers generalized_sql = re.sub(r'\b' + re.escape(token) + r'\b', '$1', generalized_sql) pattern = pattern.replace(token, "{number}") elif '@' in token and '.' in token: - # Reemplazar emails + # Replace emails generalized_sql = re.sub(r'\b' + re.escape(token) + r'\b', '$1', generalized_sql) pattern = pattern.replace(token, "{value}") elif token.startswith('"') or token.startswith("'"): # Reemplazar strings value = token.strip('"\'') - if len(value) > 2: # Evitar reemplazar strings muy cortos + if len(value) > 2: # Avoid replacing very short strings generalized_sql = re.sub(r'[\'"]' + re.escape(value) + r'[\'"]', "'$1'", generalized_sql) pattern = pattern.replace(token, "{value}") - # Crear plantilla + # Create template return QueryTemplate( pattern=pattern, description=f"Plantilla generada automáticamente para: {pattern}", @@ -931,11 +931,11 @@ def get_optimization_suggestions(self) -> List[Dict[str, Any]]: """ suggestions = [] - # Calcular estadísticas generales + # Calculate general statistics conn = sqlite3.connect(self.query_log_path) cursor = conn.cursor() - # Total de consultas y costo en los últimos 30 días + # Total consultations and cost in the last 30 days cursor.execute(''' SELECT COUNT(*) as query_count, SUM(cost) as total_cost FROM query_log @@ -947,7 +947,7 @@ def get_optimization_suggestions(self) -> List[Dict[str, Any]]: query_count, total_cost = row if query_count and query_count > 100: - # Si hay muchas consultas en total, sugerir plan de volumen + # If there are many queries in total, suggest volume plan suggestions.append({ "type": "volume_plan", "query_count": query_count, @@ -955,7 +955,7 @@ def get_optimization_suggestions(self) -> List[Dict[str, Any]]: "suggestion": f"Considerar negociar un plan por volumen. Actualmente ~{query_count} consultas/mes." }) - # Sugerir ajustar el TTL del caché según frecuencia + # Suggest adjusting cache TTL based on frequency avg_queries_per_day = query_count / 30 suggested_ttl = max(3600, min(86400 * 3, 86400 * (100 / avg_queries_per_day))) @@ -965,21 +965,21 @@ def get_optimization_suggestions(self) -> List[Dict[str, Any]]: "suggestion": f"Ajustar TTL del caché a {suggested_ttl/3600:.1f} horas basado en su patrón de uso" }) - # Obtener patrones comunes + # Get common patterns common_patterns = self.get_common_patterns(10) for pattern in common_patterns: if pattern["count"] >= 5: - # Si un patrón se repite mucho, sugerir precompilación + # If a pattern repeats a lot, suggest precompilation suggestions.append({ "type": "precompile", "pattern": pattern["pattern"], "count": pattern["count"], - "estimated_savings": round(pattern["avg_cost"] * pattern["count"] * 0.9, 2), # 90% de ahorro + "estimated_savings": round(pattern["avg_cost"] * pattern["count"] * 0.9, 2), # 90% savings "suggestion": f"Crear una plantilla SQL para consultas del tipo '{pattern['pattern']}'" }) - # Si un patrón es costoso pero poco frecuente + # If a pattern is expensive but rare if pattern["avg_cost"] > 0.1 and pattern["count"] < 5: suggestions.append({ "type": "analyze", @@ -988,7 +988,7 @@ def get_optimization_suggestions(self) -> List[Dict[str, Any]]: "suggestion": f"Revisar manualmente consultas del tipo '{pattern['pattern']}' para optimizar" }) - # Buscar períodos con alta carga para ajustar parámetros + # Find periods with high load to adjust parameters cursor.execute(''' SELECT strftime('%Y-%m-%d %H', timestamp) as hour, COUNT(*) as count, SUM(cost) as total_cost FROM query_log @@ -1000,7 +1000,7 @@ def get_optimization_suggestions(self) -> List[Dict[str, Any]]: for row in cursor.fetchall(): hour, count, total_cost = row - if count > 20: # Si hay más de 20 consultas en una hora + if count > 20: # If there are more than 20 queries in an hour suggestions.append({ "type": "load_balancing", "hour": hour, @@ -1009,7 +1009,7 @@ def get_optimization_suggestions(self) -> List[Dict[str, Any]]: "suggestion": f"Alta carga de consultas detectada el {hour} ({count} consultas). Considerar técnicas de agrupación." }) - # Buscar consultas redundantes (misma consulta en corto tiempo) + # Find redundant queries (same query in a short time) cursor.execute(''' SELECT query, COUNT(*) as count FROM query_log diff --git a/corebrain/core/test_utils.py b/corebrain/core/test_utils.py index 5e334d9..7097e39 100644 --- a/corebrain/core/test_utils.py +++ b/corebrain/core/test_utils.py @@ -27,18 +27,18 @@ def generate_test_question_from_schema(schema: Dict[str, Any]) -> str: if not tables: return "¿Cuáles son las tablas disponibles?" - # Seleccionar una tabla aleatoria + # Select a random table table = random.choice(tables) table_name = table["name"] - # Determinar el tipo de pregunta + # Determine the type of question question_types = [ f"¿Cuántos registros hay en la tabla {table_name}?", f"Muestra los primeros 5 registros de {table_name}", f"¿Cuáles son los campos de la tabla {table_name}?", ] - # Obtener columnas según la estructura (SQL vs NoSQL) + # Getting columns by structure (SQL vs NoSQL) columns = [] if "columns" in table and table["columns"]: columns = table["columns"] @@ -46,10 +46,10 @@ def generate_test_question_from_schema(schema: Dict[str, Any]) -> str: columns = table["fields"] if columns: - # Si tenemos información de columnas/campos + # If we have information from columns/fields column_name = columns[0]["name"] if columns else "id" - # Añadir preguntas específicas con columnas + # Add specific questions with columns question_types.extend([ f"¿Cuál es el valor máximo de {column_name} en {table_name}?", f"¿Cuáles son los valores únicos de {column_name} en {table_name}?", @@ -73,16 +73,16 @@ def test_natural_language_query(api_token: str, db_config: Dict[str, Any], api_u try: print_colored("\nRealizando prueba de consulta en lenguaje natural...", "blue") - # Importación dinámica para evitar circular imports + # Dynamic import to avoid circular imports from db.schema_file import extract_db_schema - # Generar una pregunta de prueba basada en el esquema extraído directamente + # Generate a test question based on the extracted schema directly schema = extract_db_schema(db_config) print("REcoge esquema: ", schema) question = generate_test_question_from_schema(schema) print(f"Pregunta de prueba: {question}") - # Preparar los datos para la petición + # Prepare the data for the request api_url = api_url or DEFAULT_API_URL if not api_url.startswith(("http://", "https://")): api_url = "https://" + api_url @@ -90,23 +90,23 @@ def test_natural_language_query(api_token: str, db_config: Dict[str, Any], api_u if api_url.endswith('/'): api_url = api_url[:-1] - # Construir endpoint para la consulta + # Build endpoint for the query endpoint = f"{api_url}/api/database/sdk/query" - # Datos para la consulta + # Data for consultation request_data = { "question": question, "db_schema": schema, "config_id": db_config["config_id"] } - # Realizar la petición al API + # Make the request to the API headers = { "Authorization": f"Bearer {api_token}", "Content-Type": "application/json" } - timeout = 15.0 # Tiempo máximo de espera reducido + timeout = 15.0 # Reduced maximum waiting time try: print_colored("Enviando consulta al API...", "blue") @@ -117,11 +117,11 @@ def test_natural_language_query(api_token: str, db_config: Dict[str, Any], api_u timeout=timeout ) - # Verificar la respuesta + # Check the answer if response.status_code == 200: result = response.json() - # Verificar si hay explicación en el resultado + # Check if there is an explanation in the result if "explanation" in result: print_colored("\nRespuesta:", "green") print(result["explanation"]) @@ -129,7 +129,7 @@ def test_natural_language_query(api_token: str, db_config: Dict[str, Any], api_u print_colored("\n✅ Prueba de consulta exitosa!", "green") return True else: - # Si no hay explicación pero la API responde, puede ser un formato diferente + # If there is no explanation but the API responds, it may be a different format print_colored("\nRespuesta recibida del API (formato diferente al esperado):", "yellow") print(json.dumps(result, indent=2)) print_colored("\n⚠️ La API respondió, pero con un formato diferente al esperado.", "yellow") diff --git a/corebrain/sdk.py b/corebrain/sdk.py index 7de1491..a843e24 100644 --- a/corebrain/sdk.py +++ b/corebrain/sdk.py @@ -3,6 +3,6 @@ """ from corebrain.config.manager import ConfigManager -# Re-exportar elementos principales +# Re-export main elements list_configurations = ConfigManager().list_configs remove_configuration = ConfigManager().remove_config \ No newline at end of file diff --git a/setup.sh b/setup.sh index d32b7c8..fde0e3c 100644 --- a/setup.sh +++ b/setup.sh @@ -1,5 +1,4 @@ #!/bin/bash -# Utwórz i aktywuj środowisko wirtualne python3 -m venv venv source venv/bin/activate From 1db210c67dfa936251d8c70819dc0ebb3d89f273 Mon Sep 17 00:00:00 2001 From: Ruben Ayuso Date: Tue, 27 May 2025 18:03:39 +0200 Subject: [PATCH 73/81] Pyproject error fixed. --- corebrain/core/client.py | 164 +++++++++++++++++------------------ corebrain/core/test_utils.py | 72 +++++++-------- package-lock.json | 6 ++ pyproject.toml | 2 +- 4 files changed, 125 insertions(+), 119 deletions(-) create mode 100644 package-lock.json diff --git a/corebrain/core/client.py b/corebrain/core/client.py index 9b65225..16e554e 100644 --- a/corebrain/core/client.py +++ b/corebrain/core/client.py @@ -473,61 +473,61 @@ def _extract_db_schema(self, detail_level: str = "full", specific_collections: L Returns: Dictionary with the database structure organized by tables/collections """ - logger.info(f"Extrayendo esquema de base de datos. Tipo: {self.db_config['type']}, Motor: {self.db_config.get('engine')}") + logger.info(f"Extracting database schema. Type: {self.db_config['type']}, Engine: {self.db_config.get('engine')}") db_type = self.db_config["type"].lower() schema = { "type": db_type, "database": self.db_config.get("database", ""), "tables": {}, - "total_collections": 0, # Añadir contador total - "included_collections": 0 # Contador de incluidas + "total_collections": 0, # Add total counter + "included_collections": 0 # Counter for included ones } excluded_tables = set(self.db_config.get("excluded_tables", [])) - logger.info(f"Tablas excluidas: {excluded_tables}") + logger.info(f"Excluded tables: {excluded_tables}") try: if db_type == "sql": engine = self.db_config.get("engine", "").lower() - logger.info(f"Procesando base de datos SQL con motor: {engine}") + logger.info(f"Processing SQL database with engine: {engine}") if engine in ["sqlite", "mysql", "postgresql"]: cursor = self.db_connection.cursor() if engine == "sqlite": - logger.info("Obteniendo tablas de SQLite") - # Obtener listado de tablas + logger.info("Getting SQLite tables") + # Get table listing cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") tables = cursor.fetchall() - logger.info(f"Tablas encontradas en SQLite: {tables}") + logger.info(f"Tables found in SQLite: {tables}") elif engine == "mysql": - logger.info("Obteniendo tablas de MySQL") + logger.info("Getting MySQL tables") cursor.execute("SHOW TABLES;") tables = cursor.fetchall() - logger.info(f"Tablas encontradas en MySQL: {tables}") + logger.info(f"Tables found in MySQL: {tables}") elif engine == "postgresql": - logger.info("Obteniendo tablas de PostgreSQL") + logger.info("Getting PostgreSQL tables") cursor.execute(""" SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'; """) tables = cursor.fetchall() - logger.info(f"Tablas encontradas en PostgreSQL: {tables}") + logger.info(f"Tables found in PostgreSQL: {tables}") - # Procesar las tablas encontradas + # Process the found tables for table in tables: table_name = table[0] - logger.info(f"Procesando tabla: {table_name}") + logger.info(f"Processing table: {table_name}") - # Saltar tablas excluidas + # Skip excluded tables if table_name in excluded_tables: - logger.info(f"Saltando tabla excluida: {table_name}") + logger.info(f"Skipping excluded table: {table_name}") continue try: - # Obtener información de columnas según el motor + # Get column information according to engine if engine == "sqlite": cursor.execute(f"PRAGMA table_info({table_name});") elif engine == "mysql": @@ -540,9 +540,9 @@ def _extract_db_schema(self, detail_level: str = "full", specific_collections: L """) columns = cursor.fetchall() - logger.info(f"Columnas encontradas para {table_name}: {columns}") + logger.info(f"Columns found for {table_name}: {columns}") - # Estructura de columnas según el motor + # Column structure according to engine if engine == "sqlite": column_info = [{"name": col[1], "type": col[2]} for col in columns] elif engine == "mysql": @@ -550,25 +550,25 @@ def _extract_db_schema(self, detail_level: str = "full", specific_collections: L elif engine == "postgresql": column_info = [{"name": col[0], "type": col[1]} for col in columns] - # Guardar información de la tabla + # Save table information schema["tables"][table_name] = { "columns": column_info, - "sample_data": [] # No obtenemos datos de muestra por defecto + "sample_data": [] # We don't get sample data by default } except Exception as e: - logger.error(f"Error procesando tabla {table_name}: {str(e)}") + logger.error(f"Error processing table {table_name}: {str(e)}") else: - # Usando SQLAlchemy - logger.info("Usando SQLAlchemy para obtener el esquema") + # Using SQLAlchemy + logger.info("Using SQLAlchemy to get schema") inspector = inspect(self.db_connection) table_names = inspector.get_table_names() - logger.info(f"Tablas encontradas con SQLAlchemy: {table_names}") + logger.info(f"Tables found with SQLAlchemy: {table_names}") for table_name in table_names: if table_name in excluded_tables: - logger.info(f"Saltando tabla excluida: {table_name}") + logger.info(f"Skipping excluded table: {table_name}") continue try: @@ -580,12 +580,12 @@ def _extract_db_schema(self, detail_level: str = "full", specific_collections: L "sample_data": [] } except Exception as e: - logger.error(f"Error procesando tabla {table_name} con SQLAlchemy: {str(e)}") + logger.error(f"Error processing table {table_name} with SQLAlchemy: {str(e)}") elif db_type in ["nosql", "mongodb"]: - logger.info("Procesando base de datos MongoDB") + logger.info("Processing MongoDB database") if not hasattr(self, 'db_connection') or self.db_connection is None: - logger.error("La conexión a MongoDB no está disponible") + logger.error("MongoDB connection is not available") return schema try: @@ -593,25 +593,25 @@ def _extract_db_schema(self, detail_level: str = "full", specific_collections: L try: collection_names = self.db_connection.list_collection_names() schema["total_collections"] = len(collection_names) - logger.info(f"Colecciones encontradas en MongoDB: {collection_names}") + logger.info(f"Collections found in MongoDB: {collection_names}") except Exception as e: - logger.error(f"Error al obtener colecciones MongoDB: {str(e)}") + logger.error(f"Error getting MongoDB collections: {str(e)}") return schema - # Si solo queremos los nombres + # If we only want the names if detail_level == "names_only": schema["collection_names"] = collection_names return schema - # Procesar cada colección + # Process each collection for collection_name in collection_names: if collection_name in excluded_tables: - logger.info(f"Saltando colección excluida: {collection_name}") + logger.info(f"Skipping excluded collection: {collection_name}") continue try: collection = self.db_connection[collection_name] - # Obtener un documento para inferir estructura + # Get a document to infer structure first_doc = collection.find_one() if first_doc: @@ -625,20 +625,20 @@ def _extract_db_schema(self, detail_level: str = "full", specific_collections: L "fields": fields, "doc_count": collection.estimated_document_count() } - logger.info(f"Procesada colección {collection_name} con {len(fields)} campos") + logger.info(f"Processed collection {collection_name} with {len(fields)} fields") else: - logger.info(f"Colección {collection_name} está vacía") + logger.info(f"Collection {collection_name} is empty") schema["tables"][collection_name] = { "fields": [], "doc_count": 0 } except Exception as e: - logger.error(f"Error procesando colección {collection_name}: {str(e)}") + logger.error(f"Error processing collection {collection_name}: {str(e)}") except Exception as e: - logger.error(f"Error general procesando MongoDB: {str(e)}") + logger.error(f"General error processing MongoDB: {str(e)}") - # Convertir el diccionario de tablas en una lista + # Convert the table dictionary to a list table_list = [] for table_name, table_info in schema["tables"].items(): table_data = {"name": table_name} @@ -646,13 +646,13 @@ def _extract_db_schema(self, detail_level: str = "full", specific_collections: L table_list.append(table_data) schema["tables_list"] = table_list - logger.info(f"Esquema final - Tablas encontradas: {len(schema['tables'])}") - logger.info(f"Nombres de tablas: {list(schema['tables'].keys())}") + logger.info(f"Final schema - Tables found: {len(schema['tables'])}") + logger.info(f"Table names: {list(schema['tables'].keys())}") return schema except Exception as e: - logger.error(f"Error al extraer el esquema de la base de datos: {str(e)}") + logger.error(f"Error extracting database schema: {str(e)}") return {"type": db_type, "tables": {}, "tables_list": []} def list_collections_name(self) -> List[str]: @@ -694,8 +694,8 @@ def ask(self, question: str, **kwargs) -> Dict: # Validar que el esquema tiene tablas/colecciones if not schema.get("tables"): - print("Error: No se encontraron tablas/colecciones en la base de datos") - return {"error": True, "explanation": "No se encontraron tablas/colecciones en la base de datos"} + print("Error: No tables/collections found in the database") + return {"error": True, "explanation": "No tables/collections found in the database"} # Obtener nombres de tablas disponibles para validación available_tables = set() @@ -751,7 +751,7 @@ def ask(self, question: str, **kwargs) -> Dict: # Verificar respuesta if response.status_code != 200: - error_msg = f"Error {response.status_code} al realizar la consulta" + error_msg = f"Error {response.status_code} while performing query" try: error_data = response.json() if isinstance(error_data, dict): @@ -771,7 +771,7 @@ def ask(self, question: str, **kwargs) -> Dict: if "query" not in api_response: return { "error": True, - "explanation": "La API no generó una consulta válida." + "explanation": "The API did not generate a valid query." } # Si se debe ejecutar la consulta pero la API no lo hizo @@ -789,7 +789,7 @@ def ask(self, question: str, **kwargs) -> Dict: if isinstance(sql_candidate, str): query_value = sql_candidate else: - raise CorebrainError(f"La consulta SQL generada no es un string: {query_value}") + raise CorebrainError(f"The generated SQL query is not a string: {query_value}") # Preparar la consulta con el formato adecuado query_to_execute = { @@ -812,9 +812,9 @@ def ask(self, question: str, **kwargs) -> Dict: # Validar nombre de colección if not collection_name: - raise CorebrainError("No se especificó colección y no se encontraron colecciones en el esquema") + raise CorebrainError("No collection specified and no collections found in schema") if not isinstance(collection_name, str) or not collection_name.strip(): - raise CorebrainError("Nombre de colección inválido: debe ser un string no vacío") + raise CorebrainError("Invalid collection name: must be a non-empty string") # Añadir colección a la consulta query_to_execute["collection"] = collection_name @@ -871,15 +871,15 @@ def ask(self, question: str, **kwargs) -> Dict: if explanation_response.status_code == 200: explanation_result = explanation_response.json() - api_response["explanation"] = explanation_result.get("explanation", "No se pudo generar una explicación.") + api_response["explanation"] = explanation_result.get("explanation", "Could not generate an explanation.") else: api_response["explanation"] = self._generate_fallback_explanation(query_to_execute, query_result) except Exception as explain_error: - logger.error(f"Error al obtener explicación: {str(explain_error)}") + logger.error(f"Error getting explanation: {str(explain_error)}") api_response["explanation"] = self._generate_fallback_explanation(query_to_execute, query_result) except Exception as e: - error_msg = f"Error al ejecutar la consulta: {str(e)}" + error_msg = f"Error executing query: {str(e)}" logger.error(error_msg) return { "error": True, @@ -911,10 +911,10 @@ def ask(self, question: str, **kwargs) -> Dict: # Para MongoDB o genérico api_response["explanation"] = self._generate_generic_explanation(api_response["query"], result_data) else: - api_response["explanation"] = "La consulta se ha ejecutado correctamente." + api_response["explanation"] = "The query executed successfully." except Exception as exp_fix_error: - logger.error(f"Error al corregir explicación: {str(exp_fix_error)}") - api_response["explanation"] = "La consulta se ha ejecutado correctamente." + logger.error(f"Error correcting explanation: {str(exp_fix_error)}") + api_response["explanation"] = "The query executed successfully." # Preparar la respuesta final result = { @@ -945,16 +945,16 @@ def ask(self, question: str, **kwargs) -> Dict: return result except httpx.TimeoutException: - return {"error": True, "explanation": "Tiempo de espera agotado al conectar con el servidor."} + return {"error": True, "explanation": "Timeout waiting to connect to server."} except httpx.RequestError as e: - return {"error": True, "explanation": f"Error de conexión con el servidor: {str(e)}"} + return {"error": True, "explanation": f"Connection error with server: {str(e)}"} except Exception as e: import traceback error_details = traceback.format_exc() - logger.error(f"Error inesperado en ask(): {error_details}") - return {"error": True, "explanation": f"Error inesperado: {str(e)}"} + logger.error(f"Unexpected error in ask(): {error_details}") + return {"error": True, "explanation": f"Unexpected error: {str(e)}"} def _generate_fallback_explanation(self, query, results): """ @@ -978,7 +978,7 @@ def _generate_fallback_explanation(self, query, results): # Fallback genérico result_count = len(results) if isinstance(results, list) else (1 if results else 0) - return f"La consulta devolvió {result_count} resultados." + return f"The query returned {result_count} results." def _generate_sql_explanation(self, sql_query, results): """ @@ -1009,28 +1009,28 @@ def _generate_sql_explanation(self, sql_query, results): if "join" in sql_lower: if len(tables) > 1: if "where" in sql_lower: - return f"Se encontraron {result_count} registros que cumplen con los criterios especificados, relacionando información de las tablas {', '.join(tables)}." + return f"Found {result_count} records that meet the specified criteria, relating information from tables {', '.join(tables)}." else: - return f"Se obtuvieron {result_count} registros relacionando información de las tablas {', '.join(tables)}." + return f"Found {result_count} records relating information from tables {', '.join(tables)}." else: - return f"Se obtuvieron {result_count} registros relacionando datos entre tablas." + return f"Found {result_count} records relating data between tables." elif "where" in sql_lower: - return f"Se encontraron {result_count} registros que cumplen con los criterios de búsqueda." + return f"Found {result_count} records that meet the search criteria." else: - return f"La consulta devolvió {result_count} registros de la base de datos." + return f"The query returned {result_count} records from the database." # Para otros tipos de consultas (INSERT, UPDATE, DELETE) if "insert" in sql_lower: - return "Se insertaron correctamente los datos en la base de datos." + return "Data inserted successfully into the database." elif "update" in sql_lower: - return "Se actualizaron correctamente los datos en la base de datos." + return "Data updated successfully in the database." elif "delete" in sql_lower: - return "Se eliminaron correctamente los datos de la base de datos." + return "Data deleted successfully from the database." # Fallback genérico - return f"La consulta SQL se ejecutó correctamente y devolvió {result_count} resultados." + return f"The SQL query executed successfully and returned {result_count} results." def _generate_mongodb_explanation(self, query, results): @@ -1044,29 +1044,29 @@ def _generate_mongodb_explanation(self, query, results): Returns: Generated explanation """ - collection = query.get("collection", "la colección") + collection = query.get("collection", "the collection") operation = query.get("operation", "find") result_count = len(results) if isinstance(results, list) else (1 if results else 0) # Generar explicación según la operación if operation == "find": - return f"Se encontraron {result_count} documentos en la colección {collection} que coinciden con los criterios de búsqueda." + return f"Found {result_count} documents in the {collection} that meet the search criteria." elif operation == "findOne": if result_count > 0: - return f"Se encontró el documento solicitado en la colección {collection}." + return f"Found the requested document in the {collection}." else: - return f"No se encontró ningún documento en la colección {collection} que coincida con los criterios." + return f"No documents found in the {collection} that meet the criteria." elif operation == "aggregate": - return f"La agregación en la colección {collection} devolvió {result_count} resultados." + return f"The aggregation in the {collection} returned {result_count} results." elif operation == "insertOne": - return f"Se ha insertado correctamente un nuevo documento en la colección {collection}." + return f"A new document inserted successfully into the {collection}." elif operation == "updateOne": - return f"Se ha actualizado correctamente un documento en la colección {collection}." + return f"A document updated successfully in the {collection}." elif operation == "deleteOne": - return f"Se ha eliminado correctamente un documento de la colección {collection}." + return f"A document deleted successfully from the {collection}." # Fallback genérico - return f"La operación {operation} se ejecutó correctamente en la colección {collection} y devolvió {result_count} resultados." + return f"The {operation} operation executed successfully in the {collection} and returned {result_count} results." def _generate_generic_explanation(self, query, results): @@ -1083,11 +1083,11 @@ def _generate_generic_explanation(self, query, results): result_count = len(results) if isinstance(results, list) else (1 if results else 0) if result_count == 0: - return "La consulta no devolvió ningún resultado." + return "The query returned no results." elif result_count == 1: - return "La consulta devolvió 1 resultado." + return "The query returned 1 result." else: - return f"La consulta devolvió {result_count} resultados." + return f"The query returned {result_count} results." def close(self) -> None: diff --git a/corebrain/core/test_utils.py b/corebrain/core/test_utils.py index 5e334d9..8fead19 100644 --- a/corebrain/core/test_utils.py +++ b/corebrain/core/test_utils.py @@ -20,25 +20,25 @@ def generate_test_question_from_schema(schema: Dict[str, Any]) -> str: Generated test question """ if not schema or not schema.get("tables"): - return "¿Cuáles son las tablas disponibles?" + return "What are the available tables?" tables = schema["tables"] if not tables: - return "¿Cuáles son las tablas disponibles?" + return "What are the available tables?" - # Seleccionar una tabla aleatoria + # Select a random table table = random.choice(tables) table_name = table["name"] - # Determinar el tipo de pregunta + # Determine the type of question question_types = [ - f"¿Cuántos registros hay en la tabla {table_name}?", - f"Muestra los primeros 5 registros de {table_name}", - f"¿Cuáles son los campos de la tabla {table_name}?", + f"How many records are in the {table_name} table?", + f"Show the first 5 records from {table_name}", + f"What are the fields in the {table_name} table?", ] - # Obtener columnas según la estructura (SQL vs NoSQL) + # Get columns according to structure (SQL vs NoSQL) columns = [] if "columns" in table and table["columns"]: columns = table["columns"] @@ -46,13 +46,13 @@ def generate_test_question_from_schema(schema: Dict[str, Any]) -> str: columns = table["fields"] if columns: - # Si tenemos información de columnas/campos + # If we have column/field information column_name = columns[0]["name"] if columns else "id" - # Añadir preguntas específicas con columnas + # Add specific questions with columns question_types.extend([ - f"¿Cuál es el valor máximo de {column_name} en {table_name}?", - f"¿Cuáles son los valores únicos de {column_name} en {table_name}?", + f"What is the maximum value of {column_name} in {table_name}?", + f"What are the unique values of {column_name} in {table_name}?", ]) return random.choice(question_types) @@ -71,18 +71,18 @@ def test_natural_language_query(api_token: str, db_config: Dict[str, Any], api_u True if the test is successful, False otherwise """ try: - print_colored("\nRealizando prueba de consulta en lenguaje natural...", "blue") + print_colored("\nPerforming natural language query test...", "blue") - # Importación dinámica para evitar circular imports + # Dynamic import to avoid circular imports from db.schema_file import extract_db_schema - # Generar una pregunta de prueba basada en el esquema extraído directamente + # Generate a test question based on the directly extracted schema schema = extract_db_schema(db_config) - print("REcoge esquema: ", schema) + print("Retrieved schema: ", schema) question = generate_test_question_from_schema(schema) - print(f"Pregunta de prueba: {question}") + print(f"Test question: {question}") - # Preparar los datos para la petición + # Prepare the data for the request api_url = api_url or DEFAULT_API_URL if not api_url.startswith(("http://", "https://")): api_url = "https://" + api_url @@ -90,26 +90,26 @@ def test_natural_language_query(api_token: str, db_config: Dict[str, Any], api_u if api_url.endswith('/'): api_url = api_url[:-1] - # Construir endpoint para la consulta + # Build endpoint for the query endpoint = f"{api_url}/api/database/sdk/query" - # Datos para la consulta + # Data for the query request_data = { "question": question, "db_schema": schema, "config_id": db_config["config_id"] } - # Realizar la petición al API + # Make the API request headers = { "Authorization": f"Bearer {api_token}", "Content-Type": "application/json" } - timeout = 15.0 # Tiempo máximo de espera reducido + timeout = 15.0 # Reduced maximum wait time try: - print_colored("Enviando consulta al API...", "blue") + print_colored("Sending query to API...", "blue") response = http_session.post( endpoint, headers=headers, @@ -117,25 +117,25 @@ def test_natural_language_query(api_token: str, db_config: Dict[str, Any], api_u timeout=timeout ) - # Verificar la respuesta + # Verify the response if response.status_code == 200: result = response.json() - # Verificar si hay explicación en el resultado + # Check if there's an explanation in the result if "explanation" in result: - print_colored("\nRespuesta:", "green") + print_colored("\nResponse:", "green") print(result["explanation"]) - print_colored("\n✅ Prueba de consulta exitosa!", "green") + print_colored("\n✅ Query test successful!", "green") return True else: - # Si no hay explicación pero la API responde, puede ser un formato diferente - print_colored("\nRespuesta recibida del API (formato diferente al esperado):", "yellow") + # If there's no explanation but the API responds, it may be a different format + print_colored("\nResponse received from API (different format than expected):", "yellow") print(json.dumps(result, indent=2)) - print_colored("\n⚠️ La API respondió, pero con un formato diferente al esperado.", "yellow") + print_colored("\n⚠️ The API responded, but with a different format than expected.", "yellow") return True else: - print_colored(f"❌ Error en la respuesta: Código {response.status_code}", "red") + print_colored(f"❌ Error in response: Code {response.status_code}", "red") try: error_data = response.json() print(json.dumps(error_data, indent=2)) @@ -144,14 +144,14 @@ def test_natural_language_query(api_token: str, db_config: Dict[str, Any], api_u return False except http_session.TimeoutException: - print_colored("⚠️ Timeout al realizar la consulta. El API puede estar ocupado o no disponible.", "yellow") - print_colored("Esto no afecta a la configuración guardada.", "yellow") + print_colored("⚠️ Timeout while performing query. The API may be busy or unavailable.", "yellow") + print_colored("This does not affect the saved configuration.", "yellow") return False except http_session.RequestError as e: - print_colored(f"⚠️ Error de conexión: {str(e)}", "yellow") - print_colored("Verifica la URL de la API y tu conexión a internet.", "yellow") + print_colored(f"⚠️ Connection error: {str(e)}", "yellow") + print_colored("Check the API URL and your internet connection.", "yellow") return False except Exception as e: - print_colored(f"❌ Error al realizar la consulta: {str(e)}", "red") + print_colored(f"❌ Error performing query: {str(e)}", "red") return False \ No newline at end of file diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..e70049e --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "[Tecnología] SDK", + "lockfileVersion": 3, + "requires": true, + "packages": {} +} diff --git a/pyproject.toml b/pyproject.toml index 03bb8ad..436d24b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ [project.optional-dependencies] postgres = ["psycopg2-binary>=2.9.0"] - = ["pymongo>=4.4.0"] +mongodb = ["pymongo>=4.4.0"] mysql = ["mysql-connector-python>=8.0.23"] all_db = [ "psycopg2-binary>=2.9.0", From 9c779d18d6eb62c112c88d6a8e67e1af8331dc7f Mon Sep 17 00:00:00 2001 From: Vapniak <113126917+Vapniak@users.noreply.github.com> Date: Tue, 27 May 2025 18:04:09 +0200 Subject: [PATCH 74/81] Add installation instruction to include submodules --- README.md | 3 +- .../csharp/CorebrainCS/CorebrainCS.cs | 347 ++++++++---------- 2 files changed, 163 insertions(+), 187 deletions(-) diff --git a/README.md b/README.md index 750d1be..4b340fc 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,8 @@ pip install corebrain ```bash -git clone https://github.com/ceoweggo/Corebrain.git +git clone https://github.com/ceoweggo/Corebrain.git +git submodule update --init --recursive pip install -e . ``` diff --git a/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs b/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs index 1b7522f..4a29a99 100644 --- a/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs +++ b/corebrain/wrappers/csharp/CorebrainCS/CorebrainCS.cs @@ -1,186 +1,161 @@ -namespace CorebrainCS; - -using System; -using System.Diagnostics; -using System.Collections.Generic; - -/// -/// Creates the main corebrain interface. -/// -/// Path to the python which works with the corebrain cli, for example if you create the ./.venv you pass the path to the ./.venv python executable -/// Path to the corebrain cli script, if you installed it globally you just pass the `corebrain` path -/// -public class CorebrainCS(string pythonPath = "python", string scriptPath = "corebrain", bool verbose = false) -{ - private readonly string _pythonPath = Path.GetFullPath(pythonPath); - private readonly string _scriptPath = Path.GetFullPath(scriptPath); - private readonly bool _verbose = verbose; - - /// Shows help message with all available commands - - public string Help() - { - return ExecuteCommand("--help"); - } - - - /// Shows the current version of the Corebrain SDK - - public string Version() - { - return ExecuteCommand("--version"); - } - - /// Checks system status including: - /// - API Server status - /// - Redis status - /// - SSO Server status - /// - MongoDB status - /// - Required libraries installation - - public string CheckStatus() - { - return ExecuteCommand("--check-status"); - } - - - /// Checks system status with optional API URL and token parameters - - public string CheckStatus(string? apiUrl = null, string? token = null) - { - var args = new List { "--check-status" }; - - if (!string.IsNullOrEmpty(apiUrl)) - { - if (!Uri.IsWellFormedUriString(apiUrl, UriKind.Absolute)) - throw new ArgumentException("Invalid API URL format", nameof(apiUrl)); - - args.Add($"--api-url \"{apiUrl}\""); - } - - if (!string.IsNullOrEmpty(token)) - args.Add($"--token \"{token}\""); - - return ExecuteCommand(string.Join(" ", args)); - } - - /// Authenticates with SSO using username and password - - public string Authentication(string username, string password) - { - if (string.IsNullOrWhiteSpace(username)) - { - throw new ArgumentException("Username cannot be empty or whitespace", nameof(username)); - } - - if (string.IsNullOrWhiteSpace(password)) - { - throw new ArgumentException("Password cannot be empty or whitespace", nameof(password)); - } - - var escapedUsername = username.Replace("\"", "\\\""); - var escapedPassword = password.Replace("\"", "\\\""); - - - return ExecuteCommand($"--authentication --username \"{escapedUsername}\" --password \"{escapedPassword}\""); - } - - - /// Authenticates with SSO using a token - - public string AuthenticationWithToken(string token) - { - if (string.IsNullOrWhiteSpace(token)) - { - throw new ArgumentException("Token cannot be empty or whitespace", nameof(token)); - } - - var escapedToken = token.Replace("\"", "\\\""); - return ExecuteCommand($"--authentication --token \"{escapedToken}\""); - } - - - /// Creates a new user account and generates an associated API Key - - public string CreateUser() - { - return ExecuteCommand("--create-user"); - } - - /// Launches the configuration wizard for setting up database connections - - public string Configure() - { - return ExecuteCommand("--configure"); - } - - /// Lists all available database configurations - - public string ListConfigs() - { - return ExecuteCommand("--list-configs"); - } - - /// Displays the database schema for a configured database - public string ShowSchema() - { - return ExecuteCommand("--show-schema"); - } - - /// Displays information about the currently authenticated user - public string WhoAmI() - { - return ExecuteCommand("--woami"); - } - - /// Launches the web-based graphical user interface - public string Gui() - { - return ExecuteCommand("--gui"); - } - - - private string ExecuteCommand(string arguments) - - { - if (_verbose) - { - Console.WriteLine($"Executing: {_pythonPath} {_scriptPath} {arguments}"); - } - - var process = new Process - { - StartInfo = new ProcessStartInfo - { - FileName = _pythonPath, - Arguments = $"\"{_scriptPath}\" {arguments}", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - } - }; - - process.Start(); - var output = process.StandardOutput.ReadToEnd(); - var error = process.StandardError.ReadToEnd(); - process.WaitForExit(); - - if (_verbose) - { - Console.WriteLine("Command output:"); - Console.WriteLine(output); - if (!string.IsNullOrEmpty(error)) - { - Console.WriteLine("Error output:\n" + error); - } - } - - if (!string.IsNullOrEmpty(error)) - { - throw new InvalidOperationException($"Python CLI error: {error}"); - } - - return output.Trim(); - } -} \ No newline at end of file +namespace CorebrainCS; + +using System; +using System.Diagnostics; +using System.Collections.Generic; + +/// +/// Creates the main corebrain interface. +/// +/// Path to the python which works with the corebrain cli, for example if you create the ./.venv you pass the path to the ./.venv python executable +/// Path to the corebrain cli script, if you installed it globally you just pass the `corebrain` path +/// +public class CorebrainCS(string pythonPath = "python", string scriptPath = "corebrain", bool verbose = false) { + private readonly string _pythonPath = Path.GetFullPath(pythonPath); + private readonly string _scriptPath = Path.GetFullPath(scriptPath); + private readonly bool _verbose = verbose; + + /// Shows help message with all available commands + + public string Help() { + return ExecuteCommand("--help"); + } + + + /// Shows the current version of the Corebrain SDK + + public string Version() { + return ExecuteCommand("--version"); + } + + /// Checks system status including: + /// - API Server status + /// - Redis status + /// - SSO Server status + /// - MongoDB status + /// - Required libraries installation + + public string CheckStatus() { + return ExecuteCommand("--check-status"); + } + + + /// Checks system status with optional API URL and token parameters + + public string CheckStatus(string? apiUrl = null, string? token = null) { + var args = new List { "--check-status" }; + + if (!string.IsNullOrEmpty(apiUrl)) { + if (!Uri.IsWellFormedUriString(apiUrl, UriKind.Absolute)) + throw new ArgumentException("Invalid API URL format", nameof(apiUrl)); + + args.Add($"--api-url \"{apiUrl}\""); + } + + if (!string.IsNullOrEmpty(token)) + args.Add($"--token \"{token}\""); + + return ExecuteCommand(string.Join(" ", args)); + } + + /// Authenticates with SSO using username and password + + public string Authentication(string username, string password) { + if (string.IsNullOrWhiteSpace(username)) { + throw new ArgumentException("Username cannot be empty or whitespace", nameof(username)); + } + + if (string.IsNullOrWhiteSpace(password)) { + throw new ArgumentException("Password cannot be empty or whitespace", nameof(password)); + } + + var escapedUsername = username.Replace("\"", "\\\""); + var escapedPassword = password.Replace("\"", "\\\""); + + + return ExecuteCommand($"--authentication --username \"{escapedUsername}\" --password \"{escapedPassword}\""); + } + + + /// Authenticates with SSO using a token + + public string AuthenticationWithToken(string token) { + if (string.IsNullOrWhiteSpace(token)) { + throw new ArgumentException("Token cannot be empty or whitespace", nameof(token)); + } + + var escapedToken = token.Replace("\"", "\\\""); + return ExecuteCommand($"--authentication --token \"{escapedToken}\""); + } + + + /// Creates a new user account and generates an associated API Key + + public string CreateUser() { + return ExecuteCommand("--create-user"); + } + + /// Launches the configuration wizard for setting up database connections + + public string Configure() { + return ExecuteCommand("--configure"); + } + + /// Lists all available database configurations + + public string ListConfigs() { + return ExecuteCommand("--list-configs"); + } + + /// Displays the database schema for a configured database + public string ShowSchema() { + return ExecuteCommand("--show-schema"); + } + + /// Displays information about the currently authenticated user + public string WhoAmI() { + return ExecuteCommand("--woami"); + } + + /// Launches the web-based graphical user interface + public string Gui() { + return ExecuteCommand("--gui"); + } + + + private string ExecuteCommand(string arguments) { + if (_verbose) { + Console.WriteLine($"Executing: {_pythonPath} {_scriptPath} {arguments}"); + } + + var process = new Process { + StartInfo = new ProcessStartInfo { + FileName = _pythonPath, + Arguments = $"\"{_scriptPath}\" {arguments}", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + process.Start(); + var output = process.StandardOutput.ReadToEnd(); + var error = process.StandardError.ReadToEnd(); + process.WaitForExit(); + + if (_verbose) { + Console.WriteLine("Command output:"); + Console.WriteLine(output); + if (!string.IsNullOrEmpty(error)) { + Console.WriteLine("Error output:\n" + error); + } + } + + if (!string.IsNullOrEmpty(error)) { + throw new InvalidOperationException($"Python CLI error: {error}"); + } + + return output.Trim(); + } +} From c01881804d624a23b8719f2870048c11e95a2c20 Mon Sep 17 00:00:00 2001 From: Ruben Ayuso Date: Tue, 27 May 2025 18:05:47 +0200 Subject: [PATCH 75/81] Update submodule reference --- corebrain/CLI-UI | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corebrain/CLI-UI b/corebrain/CLI-UI index 1f4167d..98143ed 160000 --- a/corebrain/CLI-UI +++ b/corebrain/CLI-UI @@ -1 +1 @@ -Subproject commit 1f4167dfc3e9d8b8c654983912b8cb4139d63224 +Subproject commit 98143ed62dd0ec68a5bd181292af4bea88f0218b From 12dd39797959af10c16b030d541fbe7b88d43c33 Mon Sep 17 00:00:00 2001 From: Vapniak <113126917+Vapniak@users.noreply.github.com> Date: Tue, 27 May 2025 18:09:58 +0200 Subject: [PATCH 76/81] Fix the initialization guide --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4b340fc..0dfa42f 100644 --- a/README.md +++ b/README.md @@ -47,10 +47,10 @@ pip install -e . ### Initialization -# > **⚠️ IMPORTANT:** -# > * If you don't have an existing configuration, first run `corebrain --configure` -# > * If you need to generate a new API key, use `corebrain --create` -# > * Never share your API key in public repositories. Use environment variables instead. +> **⚠️ IMPORTANT:** +> * If you don't have an existing configuration, first run `corebrain --configure` +> * If you need to generate a new API key, use `corebrain --create` +> * Never share your API key in public repositories. Use environment variables instead. ```python From d0978ac9f1e441ed2d0a923bb3b4fea78ac6f943 Mon Sep 17 00:00:00 2001 From: barto3214 Date: Wed, 28 May 2025 10:19:38 +0200 Subject: [PATCH 77/81] Function to test connection added --- corebrain/cli/commands.py | 49 ++++++++++++++++++++++++++++++++++--- corebrain/db/schema_file.py | 37 +++++++++++----------------- 2 files changed, 59 insertions(+), 27 deletions(-) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index a10363a..dbb4201 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -85,13 +85,14 @@ def authentication_with_api_key_return(): parser.add_argument("--check-status",action="store_true",help="Checks status of task") parser.add_argument("--authentication", action="store_true", help="Authenticate with SSO") parser.add_argument("--test-auth", action="store_true", help="Test SSO authentication system") # Is this command really useful? + parser.add_argument("--test-connection",action="store_true",help="Tests the connection to the Corebrain API using the provide credentials") # Arguments to use the SDK parser.add_argument("--create-user", action="store_true", help="Create an user and API Key by default") parser.add_argument("--configure", action="store_true", help="Configure the Corebrain SDK") parser.add_argument("--list-configs", action="store_true", help="List available configurations") parser.add_argument("--show-schema", action="store_true", help="Display database schema for a configuration") - parser.add_argument("--woami",action="store_true",help="Display information about the current user") + parser.add_argument("--whoami",action="store_true",help="Display information about the current user") parser.add_argument("--gui", action="store_true", help="Check setup and launch the web interface") args = parser.parse_args(argv) @@ -412,7 +413,47 @@ def check_library(library_name, min_version): except Exception as e: print_colored(f"❌ Error during test: {str(e)}", "red") return 1 - + if args.test_connection: + """ + Test the connection to the Corebrain API using the provided credentials. + + This command verifies that the Corebrain SDK can successfully connect to the + Corebrain API server using the provided API key or token. It checks if the + API is reachable and responds correctly. + + Usage: corebrain --test-connection [--api-key ] [--api-url ] + """ + # Test connection to the Corebrain API + api_url = os.environ.get("COREBRAIN_API_URL", DEFAULT_API_URL) + sso_url = os.environ.get("COREBRAIN_SSO_URL", DEFAULT_SSO_URL) + + try: + # Retrieve API credentials + api_key, user_data, api_token = get_api_credential(sso_url) + except Exception as e: + # Handle errors while retrieving credentials + print_colored(f"Error while retrieving API credentials: {e}", "red") + return 1 + + if not api_key: + # If no API key is provided, print an error message + # and return an error code + print_colored( + "Error: An API key is required. You can generate one at dashboard.corebrain.com.", + "red" + ) + return 1 + + try: + # Test the connection + # Import the test_connection function from the schema_file module + # and call it with the provided API key and URL + from corebrain.db.schema_file import test_connection + test_connection(api_key, api_url) + print_colored("Successfully connected to Corebrain API.", "green") + except Exception as e: + print_colored(f"Failed to connect to Corebrain API: {e}", "red") + return 1 ## ** SDK ** ## @@ -890,7 +931,7 @@ def check_library(library_name, min_version): # return 1 - if args.woami: + if args.whoami: """ Display information about the currently authenticated user. @@ -906,7 +947,7 @@ def check_library(library_name, min_version): 4. COREBRAIN_API_TOKEN environment variable 5. SSO authentication (if no other credentials found) - Usage: corebrain --woami [--api-key ] [--token ] [--sso-url ] + Usage: corebrain --whoami [--api-key ] [--token ] [--sso-url ] Information displayed: - User ID and email diff --git a/corebrain/db/schema_file.py b/corebrain/db/schema_file.py index c5979c3..0fdb3b9 100644 --- a/corebrain/db/schema_file.py +++ b/corebrain/db/schema_file.py @@ -172,6 +172,20 @@ def extract_db_schema_direct(db_config: Dict[str, Any]) -> Dict[str, Any]: _print_colored(f"Error extracting schema directly: {str(e)}", "red") return {"type": db_type, "tables": {}, "tables_list": []} +from typing import Dict, Any +import requests +# Function to test connection to the API +def test_connection(api_key: str, api_url: str) -> bool: + try: + headers = {"Authorization": f"Bearer {api_key}"} + response = requests.get(api_url, headers=headers, timeout=5) + response.raise_for_status() # if status != 200, raises an exception + return True + except Exception as e: + _print_colored(f"Failed to connect to the API: {str(e)}", "red") + return False + + def extract_schema_with_lazy_init(api_key: str, db_config: Dict[str, Any], api_url: Optional[str] = None) -> Dict[str, Any]: """ Extracts the schema using late import of the client. @@ -205,30 +219,7 @@ def extract_schema_with_lazy_init(api_key: str, db_config: Dict[str, Any], api_u _print_colored(f"Error extracting schema with client: {str(e)}", "red") # As an alternative, use direct extraction without client return extract_db_schema_direct(db_config) -from typing import Dict, Any -def test_connection(db_config: Dict[str, Any]) -> bool: - try: - if db_config["type"].lower() == "sql": - # Code to test SQL connection... - pass - elif db_config["type"].lower() == "nosql": - if db_config["engine"].lower() == "mongodb": - import pymongo - else: - raise ValueError(f"Unsupported NoSQL engine: {db_config['engine']}") - - # Create MongoDB client - client = pymongo.MongoClient(db_config["connection_string"]) - client.admin.command('ping') # Test connection - - return True - else: - _print_colored("Unsupported database type.", "red") - return False - except Exception as e: - _print_colored(f"Failed to connect to the database: {str(e)}", "red") - return False def extract_schema_to_file(api_key: str, config_id: Optional[str] = None, output_file: Optional[str] = None, api_url: Optional[str] = None) -> bool: """ From dc69c1546d4b4f3dcd9732f5da9a1a8be30d0108 Mon Sep 17 00:00:00 2001 From: bunny70pl Date: Wed, 28 May 2025 12:49:56 +0200 Subject: [PATCH 78/81] all commands for list-config --- config.json | 10 ++ corebrain/cli/commands.py | 4 +- corebrain/config/manager.py | 332 +++++++++++++++++++++++++++++------- corebrain/db/schema_file.py | 53 +----- db_schema.json | 47 +++++ 5 files changed, 337 insertions(+), 109 deletions(-) create mode 100644 config.json create mode 100644 db_schema.json diff --git a/config.json b/config.json new file mode 100644 index 0000000..5092964 --- /dev/null +++ b/config.json @@ -0,0 +1,10 @@ +{ + "type": "nosql", + "engine": "mongodb", + "host": "localhost", + "port": 27017, + "database": "baza", + "config_id": "a1e0694f-112d-4ade-aa31-68e6d83abab6", + "excluded_tables": [], + "active": true +} \ No newline at end of file diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index a10363a..45082c4 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -688,7 +688,7 @@ def check_library(library_name, min_version): management with confirmation prompts for destructive operations. """ manager = ConfigManager() - manager.list_configs(api_key_selected) + manager.list_configs(api_key_selected,user_data,api_token) elif args.show_schema: """ @@ -719,7 +719,7 @@ def check_library(library_name, min_version): Note: This command only reads schema information and doesn't modify the database in any way. It's safe to run on production databases. """ - show_db_schema(api_key, args.config_id, api_url) + show_db_schema(api_key_selected, args.config_id, api_url) diff --git a/corebrain/config/manager.py b/corebrain/config/manager.py index bbd7c96..e43afb3 100644 --- a/corebrain/config/manager.py +++ b/corebrain/config/manager.py @@ -10,6 +10,12 @@ from cryptography.fernet import Fernet from corebrain.utils.serializer import serialize_to_json from corebrain.core.common import logger +from corebrain.cli.config import DEFAULT_API_URL, DEFAULT_SSO_URL +from corebrain.db.schema_file import show_db_schema +from corebrain.cli.utils import print_colored +from tkinter import filedialog +import tkinter as tk + # Made by Lukasz # get data from pyproject.toml @@ -23,25 +29,49 @@ def load_project_metadata(): print(f"Warning: Could not load project metadata: {e}") return {} + # Made by Lukasz # get the name, version, etc. def get_config(): - metadata = load_project_metadata() # ^ + metadata = load_project_metadata() # ^ return { "model": metadata.get("name", "unknown"), "version": metadata.get("version", "0.0.0"), "debug": False, - "logging": {"level": "info"} - } + "logging": {"level": "info"}, + } + # Made by Lukasz # export config to file -def export_config(filepath="config.json"): - config = get_config() # ^ - with open(filepath, "w") as f: - json.dump(config, f, indent=4) - print(f"Configuration exported to {filepath}") - +def export_config(config, filepath="config.json", include_credentials=False, shareable=False): + """ + Export configuration to a file with options for credentials and shareable formats. + + Args: + config (dict): The configuration dictionary to export. + filepath (str): Path to the file to export. + include_credentials (bool): Whether to include sensitive fields like passwords or tokens. + shareable (bool): Whether to create a sanitized, shareable version (removes credentials). + """ + config_to_export = config.copy() + + sensitive_keys = {"password", "api_key", "token", "secret", "access_token", "credentials"} + + if shareable or not include_credentials: + config_to_export = { + k: ("***REDACTED***" if k in sensitive_keys else v) + for k, v in config_to_export.items() + } + + try: + with open(filepath, "w") as f: + json.dump(config_to_export, f, indent=4) + _print_colored(f"Configuration exported to {filepath}", "green") + except Exception as e: + _print_colored(f"Failed to export configuration: {e}", "red") + + # Function to print colored messages def _print_colored(message: str, color: str) -> None: """Simplified version of _print_colored that does not depend on cli.utils.""" @@ -50,26 +80,27 @@ def _print_colored(message: str, color: str) -> None: "green": "\033[92m", "yellow": "\033[93m", "blue": "\033[94m", - "default": "\033[0m" + "default": "\033[0m", } color_code = colors.get(color, colors["default"]) print(f"{color_code}{message}{colors['default']}") + class ConfigManager: """SDK configuration manager with improved security and performance.""" - + CONFIG_DIR = Path.home() / ".corebrain" CONFIG_FILE = CONFIG_DIR / "config.json" SECRET_KEY_FILE = CONFIG_DIR / "secret.key" ACTIVE_CONFIG_FILE = CONFIG_DIR / "active_config.json" - + def __init__(self): self.configs = {} self.cipher = None self._ensure_config_dir() self._load_secret_key() self._load_configs() - + def _ensure_config_dir(self) -> None: """Ensures that the configuration directory exists.""" try: @@ -79,40 +110,40 @@ def _ensure_config_dir(self) -> None: except Exception as e: logger.error(f"Error creating configuration directory: {str(e)}") _print_colored(f"Error creating configuration directory: {str(e)}", "red") - + def _load_secret_key(self) -> None: """Loads or generates the secret key to encrypt sensitive data.""" try: if not self.SECRET_KEY_FILE.exists(): key = Fernet.generate_key() - with open(self.SECRET_KEY_FILE, 'wb') as key_file: + with open(self.SECRET_KEY_FILE, "wb") as key_file: key_file.write(key) _print_colored(f"New secret key generated in: {self.SECRET_KEY_FILE}", "green") - - with open(self.SECRET_KEY_FILE, 'rb') as key_file: + + with open(self.SECRET_KEY_FILE, "rb") as key_file: self.secret_key = key_file.read() - + self.cipher = Fernet(self.secret_key) except Exception as e: _print_colored(f"Error loading/generating secret key: {str(e)}", "red") # Fallback a una clave temporal (menos segura pero funcional) self.secret_key = Fernet.generate_key() self.cipher = Fernet(self.secret_key) - + def _load_configs(self) -> Dict[str, Dict[str, Any]]: """Loads the saved configurations.""" if not self.CONFIG_FILE.exists(): _print_colored(f"Configuration file not found: {self.CONFIG_FILE}", "yellow") return {} - + try: - with open(self.CONFIG_FILE, 'r') as f: + with open(self.CONFIG_FILE, "r") as f: encrypted_data = f.read() - + if not encrypted_data: _print_colored("Configuration file is empty", "yellow") return {} - + try: # Trying to decipher the data decrypted_data = self.cipher.decrypt(encrypted_data.encode()).decode() @@ -121,70 +152,72 @@ def _load_configs(self) -> Dict[str, Dict[str, Any]]: # If decryption fails, attempt to load as plain JSON logger.warning(f"Error decrypting configuration: {e}") configs = json.loads(encrypted_data) - + if isinstance(configs, str): configs = json.loads(configs) - + _print_colored(f"Configuration loaded", "green") self.configs = configs return configs except Exception as e: _print_colored(f"Error loading configurations: {str(e)}", "red") return {} - + def _save_configs(self) -> None: """Saves the current configurations.""" try: configs_json = serialize_to_json(self.configs) encrypted_data = self.cipher.encrypt(json.dumps(configs_json).encode()).decode() - - with open(self.CONFIG_FILE, 'w') as f: + + with open(self.CONFIG_FILE, "w") as f: f.write(encrypted_data) - + _print_colored(f"Configurations saved in: {self.CONFIG_FILE}", "green") except Exception as e: _print_colored(f"Error saving configurations: {str(e)}", "red") - - def add_config(self, api_key: str, db_config: Dict[str, Any], config_id: Optional[str] = None) -> str: + + def add_config( + self, api_key: str, db_config: Dict[str, Any], config_id: Optional[str] = None + ) -> str: """ Adds a new configuration. - + Args: api_key: Selected API Key db_config: Database configuration config_id: Optional ID for the configuration (one is generated if not provided) - + Returns: Configuration ID """ if not config_id: config_id = str(uuid.uuid4()) db_config["config_id"] = config_id - + # Create or update the entry for this token if api_key not in self.configs: self.configs[api_key] = {} - + # Add the configuration self.configs[api_key][config_id] = db_config self._save_configs() - + _print_colored(f"Configuration added: {config_id} for API Key: {api_key[:8]}...", "green") return config_id - + def get_config(self, api_key_selected: str, config_id: str) -> Optional[Dict[str, Any]]: """ Retrieves a specific configuration. - + Args: api_key_selected: Selected API Key config_id: Configuration ID - + Returns: Configuration or None if it does not exist """ return self.configs.get(api_key_selected, {}).get(config_id) - + """ --> Default version def list_configs(self, api_key_selected: str) -> List[str]: @@ -230,7 +263,7 @@ def set_active_config(self, config_id_to_activate: str) -> bool: def get_active_config_id(self, api_key: str) -> Optional[str]: """ Retrieve the currently active configuration ID for a given API key. - + Returns None if not set. """ try: @@ -242,8 +275,8 @@ def get_active_config_id(self, api_key: str) -> Optional[str]: except Exception as e: _print_colored(f"Could not load active configuration: {e}", "yellow") return None - - def list_configs(self, api_key_selected: str) -> List[str]: + + def list_configs(self, api_key_selected: str, user_data=None, api_token=None) -> List[str]: """ Interactively select an API key, then display and manage its configurations. @@ -276,22 +309,28 @@ def list_configs(self, api_key_selected: str) -> List[str]: for idx, config_id in enumerate(config_ids, 1): status = " [ACTIVE]" if configs[config_id].get("active") else "" if status == " [ACTIVE]": - _print_colored(f" {idx}. {config_id}{status}","blue") + _print_colored(f" {idx}. {config_id}{status}", "blue") else: print(f" {idx}. {config_id}{status}") - + for k, v in configs[config_id].items(): print(f" {k}: {v}") action_prompt = input("\nWould you like to perform an action? (y/n): ").strip().lower() - if action_prompt == 'y': + if action_prompt == "y": print("\nAvailable actions:") print(" 1. Activate configuration") print(" 2. Delete configuration") - print(" 3. Exit") + print(" 3. Show schema") + print(" 4. Validate configuration") + print(" 5. Edit configuration") + print(" 6. Export configuration") + print(" 7. Improt configuration") + print(" 8. Create configuration") + print(" q. Exit") - choice = input("Enter your choice (1/2/3): ").strip() - if choice == '1': + choice = input("Enter your choice (1/2/3/4/5/6/7/8/q): ").strip() + if choice == "1": selected_idx = input("Enter the number of the configuration to activate: ").strip() try: config_id = config_ids[int(selected_idx) - 1] @@ -299,43 +338,216 @@ def list_configs(self, api_key_selected: str) -> List[str]: return config_id except (ValueError, IndexError): _print_colored("Invalid configuration number.", "red") - elif choice == '2': + elif choice == "2": selected_idx = input("Enter the number of the configuration to delete: ").strip() try: config_id = config_ids[int(selected_idx) - 1] self.remove_config(selected_api_key, config_id) except (ValueError, IndexError): _print_colored("Invalid configuration number.", "red") - elif choice == '3': + elif choice == "3": + api_url = os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + selected_idx = input("Enter the number of the configuration to show: ").strip() + try: + config_id = config_ids[int(selected_idx) - 1] + config = self.get_config(api_key_selected, config_id) + show_db_schema(config, selected_api_key, config_id, api_url) + except (ValueError, IndexError): + _print_colored("Invalid configuration number.", "red") + elif choice == "4": + api_url = os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + selected_idx = input("Enter the number of the configuration to validate: ").strip() + try: + config_id = config_ids[int(selected_idx) - 1] + self.validate_config(selected_api_key, config_id) + except (ValueError, IndexError): + _print_colored("Invalid configuration number.", "red") + elif choice == "5": + selected_idx = input("Enter the number of the configuration to modify: ").strip() + try: + config_id = config_ids[int(selected_idx) - 1] + self.modify_config(selected_api_key, config_id) + except (ValueError, IndexError): + _print_colored("Invalid configuration number.", "red") + elif choice == "6": + selected_idx = input("Enter the number of the configuration to export: ").strip() + try: + config_id = config_ids[int(selected_idx) - 1] + config = self.get_config(api_key_selected, config_id) + + # Prompt for credentials handling + include_credentials = input("Include credentials in export? (y/n): ").strip().lower() == "y" + shareable = input("Export as shareable version? (y/n): ").strip().lower() == "y" + + export_config(config, include_credentials=include_credentials, shareable=shareable) + + except (ValueError, IndexError): + _print_colored("Invalid configuration number.", "red") + elif choice == "7": + try: + self.validate_config(selected_api_key, config_id) + self.import_config(selected_api_key) + except (ValueError, IndexError): + _print_colored("Invalid configuration number.", "red") + elif choice == "8": + api_url = os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + sso_url = os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + from corebrain.cli.config import configure_sdk + + configure_sdk(api_token, api_key_selected, api_url, sso_url, user_data) + + elif choice == "q": print("Exit selected.") else: print("Invalid action.") - elif action_prompt != 'n': + elif action_prompt != "n": print("Invalid input. Please enter 'y' or 'n'.") return None + def modify_config(self, api_key_selected: str, config_id: str) -> None: + """ + Allows the user to interactively modify multiple parameters of an existing configuration. + """ + config = self.get_config(api_key_selected, config_id) + if not config: + _print_colored(f"Configuration with ID '{config_id}' not found", "red") + return + + print_colored(f"\nEditing configuration: {config_id}", "blue") + + while True: + keys = [key for key in config.keys() if key != "config_id"] + print("\nCurrent parameters:") + for idx, key in enumerate(keys, 1): + print(f" {idx}. {key}: {config[key]}") + + print(" 0. Exit edit mode") + + try: + key_idx = int(input("\nSelect parameter number to edit (or 0 to exit): ").strip()) + if key_idx == 0: + _print_colored("Exiting edit mode.", "yellow") + break + + key_to_edit = keys[key_idx - 1] + new_value = input(f"Enter new value for '{key_to_edit}': ").strip() + + # Try to interpret value types + if new_value.lower() in ["true", "false"]: + new_value = new_value.lower() == "true" + elif new_value.isdigit(): + new_value = int(new_value) + elif new_value.lower() == "null": + new_value = None + + config[key_to_edit] = new_value + self.configs[api_key_selected][config_id] = config + self._save_configs() + _print_colored(f"Updated '{key_to_edit}'", "green") + + except (ValueError, IndexError): + _print_colored("Invalid selection.", "red") + + validate = ( + input("Would you like to validate the modified config now? (y/n): ").strip().lower() + ) + if validate == "y": + self.validate_config(api_key_selected, config_id) + + def import_config(self, api_key: str) -> str | None: + """ + Opens a file dialog to select a JSON config file and imports it. + + Args: + api_key: The API key under which to store the configuration. + + Returns: + The ID of the imported configuration, or None if import failed or cancelled. + """ + # Open file dialog for selecting config JSON file + root = tk.Tk() + filepath = filedialog.askopenfilename( + title="Select configuration JSON file", + filetypes=[("JSON files", "*.json"), ("All files", "*.*")], + ) + root.destroy() + + if not filepath: + _print_colored("Import cancelled or no file selected.", "yellow") + return None + + try: + with open(filepath, "r") as f: + config = json.load(f) + + config_id = config.get("id") or str(uuid.uuid4()) + config["id"] = config_id + + self.add_config(api_key, config, config_id) + _print_colored(f"Configuration imported as {config_id}", "green") + return config_id + + except Exception as e: + _print_colored(f"Failed to import configuration: {e}", "red") + return None + + def validate_config(self, api_key, config_id): + api_url = os.environ.get("COREBRAIN_API_URL") or DEFAULT_API_URL + sso_url = os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + if not api_key: + print_colored( + "Error: An API Key is required. Use --api-key or login via --login", "red" + ) + return 1 + try: + config = self.get_config(api_key, config_id) + if not config: + print_colored(f"Configuration with ID '{config_id}' not found", "red") + return 1 + print_colored( + f"✅ Validating configuration: {config_id}", "blue" + ) # Create a temporary Corebrain instance to validate + from corebrain.core.client import Corebrain + + try: + temp_client = Corebrain(api_key=api_key, db_config=config, skip_verification=True) + print_colored("✅ Configuration validation passed!", "green") + print_colored(f"Database type: {config.get('type', 'Unknown')}", "blue") + print_colored(f"Engine: {config.get('engine', 'Unknown')}", "blue") + return 0 + except Exception as validation_error: + print_colored(f"❌ Configuration validation failed: {str(validation_error)}", "red") + return 1 + except Exception as e: + print_colored(f"❌ Error during validation: {str(e)}", "red") + return 1 + def remove_config(self, api_key_selected: str, config_id: str) -> bool: """ Deletes a configuration. - + Args: api_key_selected: Selected API Key config_id: Configuration ID - + Returns: True if deleted successfully, False otherwise """ if api_key_selected in self.configs and config_id in self.configs[api_key_selected]: del self.configs[api_key_selected][config_id] - + # If there are no configurations for this token, delete the entry if not self.configs[api_key_selected]: del self.configs[api_key_selected] - + self._save_configs() - _print_colored(f"Configuration {config_id} removed for API Key: {api_key_selected[:8]}...", "green") + _print_colored( + f"Configuration {config_id} removed for API Key: {api_key_selected[:8]}...", "green" + ) return True - - _print_colored(f"Configuration {config_id} not found for API Key: {api_key_selected[:8]}...", "yellow") - return False \ No newline at end of file + + _print_colored( + f"Configuration {config_id} not found for API Key: {api_key_selected[:8]}...", "yellow" + ) + return False diff --git a/corebrain/db/schema_file.py b/corebrain/db/schema_file.py index c5979c3..d010816 100644 --- a/corebrain/db/schema_file.py +++ b/corebrain/db/schema_file.py @@ -230,7 +230,7 @@ def test_connection(db_config: Dict[str, Any]) -> bool: _print_colored(f"Failed to connect to the database: {str(e)}", "red") return False -def extract_schema_to_file(api_key: str, config_id: Optional[str] = None, output_file: Optional[str] = None, api_url: Optional[str] = None) -> bool: +def extract_schema_to_file(config ,api_key: str, config_id: Optional[str] = None, output_file: Optional[str] = None, api_url: Optional[str] = None) -> bool: """ Extracts the database schema and saves it to a file. @@ -326,7 +326,7 @@ def extract_schema_to_file(api_key: str, config_id: Optional[str] = None, output _print_colored(f"❌ Error extracting schema: {str(e)}", "red") return False -def show_db_schema(api_token: str, config_id: Optional[str] = None, api_url: Optional[str] = None) -> None: +def show_db_schema(config ,api_token: str, config_id: Optional[str] = None, api_url: Optional[str] = None) -> None: """ Displays the schema of the configured database. @@ -335,51 +335,10 @@ def show_db_schema(api_token: str, config_id: Optional[str] = None, api_url: Opt config_id: Specific configuration ID (optional) api_url: Optional API URL """ - try: - # Explicit import with try-except to handle errors - try: - from corebrain.config.manager import ConfigManager - except ImportError as e: - _print_colored(f"Error importing ConfigManager: {e}", "red") - return False - - # Get the available configurations - config_manager = ConfigManager() - configs = config_manager.list_configs(api_token) - - if not configs: - _print_colored("No configurations saved for this token.", "yellow") - return - + try: + configs = config selected_config_id = config_id - - # If no config_id is specified, show list to select - if not selected_config_id: - _print_colored("\n=== Available configurations ===", "blue") - for i, conf_id in enumerate(configs, 1): - print(f"{i}. {conf_id}") - - try: - choice = int(input(f"\nSelect a configuration (1-{len(configs)}): ").strip()) - if 1 <= choice <= len(configs): - selected_config_id = configs[choice - 1] - else: - _print_colored("Invalid option.", "red") - return - except ValueError: - _print_colored("Please enter a valid number.", "red") - return - - # Verify that the config_id exists - if selected_config_id not in configs: - _print_colored(f"No configuration found with ID: {selected_config_id}", "red") - return - - if config_id and config_id in configs: - db_config = config_manager.get_config(api_token, config_id) - else: - # Get the selected configuration - db_config = config_manager.get_config(api_token, selected_config_id) + db_config = config if not db_config: _print_colored(f"Error getting configuration with ID: {selected_config_id}", "red") @@ -397,7 +356,7 @@ def show_db_schema(api_token: str, config_id: Optional[str] = None, api_url: Opt print(f" {db_config.get('database', 'No specified')}") # Extract and show the schema - _print_colored("\nExtracting schema from the database...", "blue") + _print_colored("\nExtracting schema from the database...", "bglue") # Try to connect to the database and extract the schema try: diff --git a/db_schema.json b/db_schema.json new file mode 100644 index 0000000..d0cb4db --- /dev/null +++ b/db_schema.json @@ -0,0 +1,47 @@ +{ + "type": "nosql", + "database": "baza2", + "tables": { + "test": { + "fields": [ + { + "name": "test", + "type": "str" + }, + { + "name": "test1", + "type": "str" + } + ], + "sample_data": [ + { + "_id": "6836e4095b51d864a9263a34", + "test": "tak", + "test1": "shimon" + } + ] + } + }, + "tables_list": [ + { + "name": "test", + "fields": [ + { + "name": "test", + "type": "str" + }, + { + "name": "test1", + "type": "str" + } + ], + "sample_data": [ + { + "_id": "6836e4095b51d864a9263a34", + "test": "tak", + "test1": "shimon" + } + ] + } + ] +} \ No newline at end of file From b9452eb13138e261c41ede2f130a9e6af755ac8b Mon Sep 17 00:00:00 2001 From: bunny70pl Date: Wed, 28 May 2025 12:54:52 +0200 Subject: [PATCH 79/81] added commands --- corebrain/config/manager.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/corebrain/config/manager.py b/corebrain/config/manager.py index e43afb3..d2899e4 100644 --- a/corebrain/config/manager.py +++ b/corebrain/config/manager.py @@ -106,7 +106,7 @@ def _ensure_config_dir(self) -> None: try: self.CONFIG_DIR.mkdir(parents=True, exist_ok=True) logger.debug(f"Configuration directory ensured: {self.CONFIG_DIR}") - _print_colored(f"Configuration directory ensured: {self.CONFIG_DIR}", "blue") + _print_colored(f"Configuration dire:ctory ensured: {self.CONFIG_DIR}", "blue") except Exception as e: logger.error(f"Error creating configuration directory: {str(e)}") _print_colored(f"Error creating configuration directory: {str(e)}", "red") @@ -374,11 +374,8 @@ def list_configs(self, api_key_selected: str, user_data=None, api_token=None) -> try: config_id = config_ids[int(selected_idx) - 1] config = self.get_config(api_key_selected, config_id) - - # Prompt for credentials handling include_credentials = input("Include credentials in export? (y/n): ").strip().lower() == "y" shareable = input("Export as shareable version? (y/n): ").strip().lower() == "y" - export_config(config, include_credentials=include_credentials, shareable=shareable) except (ValueError, IndexError): From 32ea038d242a6933a45e065694a64139cfd8565f Mon Sep 17 00:00:00 2001 From: KarixD2137 Date: Thu, 29 May 2025 11:30:21 +0200 Subject: [PATCH 80/81] API key creation - Finally working again --- 1 | 0 corebrain/cli/commands.py | 52 +++++++++++++++++++++++++-------------- 2 files changed, 33 insertions(+), 19 deletions(-) create mode 100644 1 diff --git a/1 b/1 new file mode 100644 index 0000000..e69de29 diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index 621c6e5..d8d94a5 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -41,28 +41,42 @@ def main_cli(argv: Optional[List[str]] = None) -> int: # Functions def authentication(): sso_url = os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL - sso_token, sso_user = authenticate_with_sso(sso_url) - if sso_token: - try: - print_colored("✅ Returning SSO Token.", "green") - print_colored(f"{sso_token}", "blue") - print_colored("✅ Returning User data.", "green") - print_colored(f"{sso_user}", "blue") + api_key_selected, user_data, api_token = authenticate_with_sso_and_api_key_request(sso_url) + + if api_token: + save_api_token(api_token) + print_colored("✅ API token saved.", "green") + print_colored("✅ Returning User data.", "green") + print_colored(f"{user_data}", "blue") + return api_token, user_data + else: + print_colored("❌ Could not authenticate with SSO.", "red") + return None, None + + # Previous authentication method that wasn't working (commented out for now) + # + # sso_url = os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + # sso_token, sso_user = authenticate_with_sso(sso_url) + # if sso_token: + # try: + # print_colored("✅ Returning SSO Token.", "green") + # print_colored(f"{sso_token}", "blue") + # print_colored("✅ Returning User data.", "green") + # print_colored(f"{sso_user}", "blue") - # Saving api token - api_key, user_data, api_token = get_api_credential(sso_token, DEFAULT_SSO_URL) - save_api_token(api_key) - print_colored("✅ API token saved.", "green") + # # Saving api token + # save_api_token(sso_token) + # print_colored("✅ API token saved.", "green") - return sso_token, sso_user + # return sso_token, sso_user - except Exception as e: - print_colored("❌ Could not return SSO Token or SSO User data.", "red") - return sso_token, sso_user + # except Exception as e: + # print_colored("❌ Could not return SSO Token or SSO User data.", "red") + # return sso_token, sso_user - else: - print_colored("❌ Could not authenticate with SSO.", "red") - return None, None + # else: + # print_colored("❌ Could not authenticate with SSO.", "red") + # return None, None def authentication_with_api_key_return(): sso_url = os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL @@ -497,7 +511,7 @@ def check_library(library_name, min_version): if sso_token and sso_user: print_colored("✅ Enter to create an user and API Key.", "green") - save_sso_token(sso_token) + save_api_token(sso_token) print_colored("✅ SSO token saved.", "green") # Get API URL from environment or use default From 495b130e8e95af170d5a0b94a6172b299b56c82c Mon Sep 17 00:00:00 2001 From: KarixD2137 Date: Thu, 29 May 2025 12:15:13 +0200 Subject: [PATCH 81/81] API key creation - Finally working again --- corebrain/cli/commands.py | 52 ++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/corebrain/cli/commands.py b/corebrain/cli/commands.py index d8d94a5..ba12d91 100644 --- a/corebrain/cli/commands.py +++ b/corebrain/cli/commands.py @@ -39,7 +39,7 @@ def main_cli(argv: Optional[List[str]] = None) -> int: argv = sys.argv[1:] # Functions - def authentication(): + def authentication_api_token(): sso_url = os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL api_key_selected, user_data, api_token = authenticate_with_sso_and_api_key_request(sso_url) @@ -52,31 +52,30 @@ def authentication(): else: print_colored("❌ Could not authenticate with SSO.", "red") return None, None - - # Previous authentication method that wasn't working (commented out for now) - # - # sso_url = os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL - # sso_token, sso_user = authenticate_with_sso(sso_url) - # if sso_token: - # try: - # print_colored("✅ Returning SSO Token.", "green") - # print_colored(f"{sso_token}", "blue") - # print_colored("✅ Returning User data.", "green") - # print_colored(f"{sso_user}", "blue") + + def authentication(): + sso_url = os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL + sso_token, sso_user = authenticate_with_sso(sso_url) + if sso_token: + try: + print_colored("✅ Returning SSO Token.", "green") + print_colored(f"{sso_token}", "blue") + print_colored("✅ Returning User data.", "green") + print_colored(f"{sso_user}", "blue") - # # Saving api token - # save_api_token(sso_token) - # print_colored("✅ API token saved.", "green") + # Saving api token + save_api_token(sso_token) + print_colored("✅ API token saved.", "green") - # return sso_token, sso_user + return sso_token, sso_user - # except Exception as e: - # print_colored("❌ Could not return SSO Token or SSO User data.", "red") - # return sso_token, sso_user + except Exception as e: + print_colored("❌ Could not return SSO Token or SSO User data.", "red") + return sso_token, sso_user - # else: - # print_colored("❌ Could not authenticate with SSO.", "red") - # return None, None + else: + print_colored("❌ Could not authenticate with SSO.", "red") + return None, None def authentication_with_api_key_return(): sso_url = os.environ.get("COREBRAIN_SSO_URL") or DEFAULT_SSO_URL @@ -374,7 +373,7 @@ def check_library(library_name, min_version): Note: This command only authenticates but doesn't save credentials for future use. """ - authentication() + authentication_api_token() if args.test_auth: """ @@ -1128,9 +1127,12 @@ def run_in_background_silent(cmd, cwd): webbrowser.open(url) - - # Handles the CLI command to create a new API key using stored credentials (token from SSO) + # + # Usage example: + # corebrain --create-api-key --key-name "Name of key" --key-level read | write | admin + # + if args.create_api_key: sso_token = load_api_token()