diff --git a/NEWS.md b/NEWS.md new file mode 100644 index 0000000..7df2f57 --- /dev/null +++ b/NEWS.md @@ -0,0 +1,248 @@ +# FZ Release Notes + +## Version 0.9.1 (2026-01-25) + +### New Features + +#### `fzl` Command - List and Validate Models/Calculators +- **New CLI command**: `fzl` (or `fz list`) for listing and validating installed models and calculators +- Supports glob patterns for filtering: `fzl --models "perfect*" --calculators "ssh*"` +- Validation mode with `--check` flag to verify model/calculator integrity +- Multiple output formats: JSON, Markdown (default), Table +- Shows supported calculators for each model +- Example usage: + ```bash + fzl --models "*" --calculators "*" --check --format markdown + ``` + +#### Enhanced Calculator Support + +**SLURM Workload Manager Integration** +- New calculator type: `slurm://[user@host[:port]]:partition/script` +- Supports both local and remote SLURM execution +- Local: `slurm://:compute/bash script.sh` +- Remote: `slurm://user@cluster.edu:gpu/bash script.sh` +- Automatic partition scheduling and job management +- Interrupt handling (Ctrl+C terminates SLURM jobs) + +**Funz Server Protocol Support** +- New calculator type: `funz://[host]:/` +- Compatible with legacy Java Funz calculator servers +- TCP socket-based communication with reservation/unreservation +- Automatic file upload/download +- UDP discovery support for automatic server detection +- Example: `funz://:5555/R` or `funz://server.example.com:5555/Python` +- See `FUNZ_UDP_DISCOVERY.md` for detailed protocol documentation + +#### Shell Path Configuration (FZ_SHELL_PATH) +- **New environment variable**: `FZ_SHELL_PATH` for custom binary resolution +- Overrides system PATH for shell commands in models and calculators +- Essential for Windows users with MSYS2, Git Bash, or custom tool locations +- Format: Semicolon-separated on Windows, colon-separated on Unix/Linux +- Example: `SET FZ_SHELL_PATH=C:\msys64\usr\bin;C:\msys64\mingw64\bin` +- Automatic `.exe` extension handling on Windows +- Binary path caching for performance +- See `SHELL_PATH_IMPLEMENTATION.md` for implementation details + +#### R Interpreter Support +- **Formula evaluation with R**: Set interpreter to "R" for statistical computing +- Configure via `model["interpreter"] = "R"` or `set_interpreter("R")` +- Supports R statistical functions: `mean()`, `sd()`, `median()`, `rnorm()`, etc. +- Multi-line R function definitions in formula context +- Requires `rpy2` package and R system installation +- Example: + ```text + #@ samples <- rnorm($n, mean=$mu, sd=$sigma) + Mean: @{mean(samples)} + ``` +- See `examples/r_interpreter_example.md` for installation guide + +#### Variable Default Values +- **New syntax**: `${var~default}` for specifying default values +- Falls back to default when variable not provided in `input_variables` +- Useful for configuration templates with sensible defaults +- Example: `${host~localhost}`, `${port~8080}` +- Warning issued when default value is used +- See `examples/variable_substitution.md` for comprehensive documentation + +#### Old Funz Syntax Compatibility +- Support for legacy Java Funz variable syntax: `?var` (equivalent to `$var`) +- Backward compatibility for existing Funz users migrating to Python +- Automatic detection and replacement +- Example: `Temperature: ?T_celsius` → `Temperature: 25` + +#### Progress Callbacks +- **New callback system** for monitoring execution progress +- Callback functions receive events: `case_start`, `case_complete`, `case_failed` +- Real-time progress tracking for long-running calculations +- Custom progress bars, logging, or UI updates +- Example: + ```python + def progress_callback(event_type, case_info): + if event_type == "case_complete": + print(f"✓ Case {case_info['case_name']} done") + + fzr(..., callbacks=[progress_callback]) + ``` + +### Improvements + +#### Enhanced Argument Parsing +- CLI arguments now support three formats: + 1. **Inline JSON**: `--model '{"varprefix": "$"}'` + 2. **JSON file**: `--model model.json` + 3. **Alias**: `--model perfectgas` (loads from `.fz/models/perfectgas.json`) +- Automatic detection with fallback and helpful error messages +- Better type validation with detailed warnings +- Consistent behavior across all CLI commands + +#### Calculator-Model Compatibility +- Automatic validation that calculators support specified models +- Prevents incompatible calculator/model combinations +- Clear error messages when model not supported by calculator +- Alias resolution for both models and calculators + +#### Timeout Configuration +- Flexible timeout settings at multiple levels: + - Environment variable: `FZ_EXECUTION_TIMEOUT` + - Model configuration: `model["timeout"]` + - Calculator URI: `sh://script.sh?timeout=300` +- Per-calculator timeout overrides +- Default timeout handling for long-running calculations + +#### Better Error Handling +- Comprehensive error messages with context +- Automatic help display on TypeError (missing/wrong arguments) +- Detailed warnings for argument parsing failures +- Stack trace preservation for debugging + +#### Code Quality & Sanitization +- Extensive code cleanup and refactoring +- Improved type hints and docstrings +- Better separation of concerns +- Enhanced test coverage (100+ new tests) +- Fixed unsafe bash command replacement vulnerabilities + +### Testing & CI + +#### New Test Suites +- `test_funz_protocol.py` - Comprehensive Funz protocol tests (TCP/UDP) +- `test_funz_integration.py` - End-to-end Funz server integration +- `test_slurm_runner.py` - SLURM workload manager tests +- `test_shell_path.py` - Shell path resolution tests +- `test_interpreter_r.py` - R interpreter tests +- `test_interpreter_default_values.py` - Default value substitution tests +- `test_calculator_discovery.py` - Calculator/model discovery tests +- `test_callbacks.py` - Progress callback tests +- `test_no_*.py` - Edge case tests (no models, no calculators, no formulas, etc.) + +#### CI/CD Enhancements +- GitHub Actions workflow for SLURM testing +- Funz calculator integration CI workflow +- Windows CI improvements with better error handling +- More robust test fixtures and cleanup + +### Documentation + +#### New Documentation Files +- `FUNZ_UDP_DISCOVERY.md` - Funz protocol and UDP discovery guide +- `SHELL_PATH_IMPLEMENTATION.md` - Shell path configuration details +- `CLAUDE.md` - LLM-friendly codebase documentation +- `context/` directory - Modular documentation for different aspects: + - `INDEX.md` - Documentation overview + - `core-functions.md` - API reference for fzi, fzc, fzo, fzr + - `calculators.md` - Calculator types and configuration + - `model-definition.md` - Model structure and aliases + - `formulas-and-interpreters.md` - Formula evaluation guide + - `parallel-and-caching.md` - Performance optimization + - `quick-examples.md` - Common usage patterns + - `syntax-guide.md` - Input template syntax reference + +#### Updated Documentation +- README.md - Added sections for: + - FZ_SHELL_PATH configuration + - R interpreter support + - Variable default values + - SLURM calculator usage + - Funz server integration + - Progress callbacks + - Old Funz syntax compatibility +- Examples: + - `examples/r_interpreter_example.md` - R setup and usage + - `examples/variable_substitution.md` - Default values guide + - `examples/shell_path_example.md` - Shell path configuration + - `examples/java_funz_syntax_example.py` - Legacy syntax examples + - `examples/fzi_formulas_example.py` - Formula evaluation examples + - `examples/fzi_static_objects_example.py` - Static object examples + +### Bug Fixes + +- Fixed Windows path separator handling in file operations +- Fixed unsafe bash command string replacement (security issue) +- Fixed PATH environment variable not respecting FZ_SHELL_PATH priority +- Fixed SLURM URI parsing for local execution (`:partition` prefix required) +- Fixed scalar value result extraction in various contexts +- Fixed calculator XML configuration for Funz integration +- Fixed missing JSON files during model/plugin installation +- Fixed OSError handling in Windows script execution +- Fixed R interpreter initialization and expression evaluation +- Fixed cache matching when outputs are None +- Fixed directory structure creation when no input variables specified + +### Breaking Changes + +- **fzr directory structure**: Now creates subdirectories in `results_dir` as long as any `input_variable` is set up + - No subdirectories only when `input_variables={}` + - More consistent with user expectations + - Better organization for parametric studies + +### Deprecations + +None in this release. + +--- + +## Version 0.9.0 (Initial Release) + +### Core Features + +- Four core functions: `fzi`, `fzc`, `fzo`, `fzr` +- Command-line interface with dedicated commands +- Parametric study automation with Cartesian product +- Parallel execution with thread pool +- Smart caching based on input file hashes +- Retry mechanism with calculator fallback +- Remote execution via SSH +- Interrupt handling (Ctrl+C) with graceful shutdown +- DataFrame output with automatic type casting +- Formula evaluation with Python interpreter +- Model and calculator aliases +- Comprehensive test suite +- BSD 3-Clause License + +### Calculator Types + +- Local shell execution (`sh://`) +- SSH remote execution (`ssh://`) +- Cache calculator (`cache://`) + +### Model Definition + +- Variable substitution with `$var` syntax +- Formula evaluation with `@{expression}` syntax +- Comment-based formula context +- Output command specification +- Model aliasing with JSON files + +### Installation + +- PyPI package: `funz-fz` +- pipx support for CLI tools +- Optional dependencies: `paramiko`, `pandas` + +--- + +For detailed information, see: +- README.md - User guide and examples +- CLAUDE.md - Developer documentation +- context/ - Modular documentation by topic diff --git a/README.md b/README.md index 53fb84c..e8bd33f 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![Python Version](https://img.shields.io/pypi/pyversions/funz.svg)](https://pypi.org/project/funz/) --> [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) -[![Version](https://img.shields.io/badge/version-0.9.0-blue.svg)](https://github.com/Funz/fz/releases) +[![Version](https://img.shields.io/badge/version-0.9.1-blue.svg)](https://github.com/Funz/fz/releases) A powerful Python package for parametric simulations and computational experiments. FZ wraps your simulation codes to automatically run parametric studies, manage input/output files, handle parallel execution, and collect results in structured DataFrames. @@ -214,6 +214,7 @@ Available commands: - `fzc` - Compile input files - `fzo` - Read output files - `fzr` - Run parametric calculations +- `fzl` - List and validate installed models and calculators ### fzi - Parse Input Variables @@ -352,6 +353,60 @@ fzo results/ \ --format json ``` +### fzl - List and Validate Models/Calculators + +List installed models and calculators with optional validation: + +```bash +# List all models and calculators +fzl + +# List with validation checks +fzl --check + +# Filter by pattern +fzl --models "perfect*" --calculators "ssh*" + +# Different output formats +fzl --format json +fzl --format table +fzl --format markdown # default +``` + +**Example output:** + +```bash +$ fzl --check --format table + +=== MODELS === + +Model: perfectgas ✓ + Path: /home/user/project/.fz/models/perfectgas.json + Supported Calculators: 2 + - local + - ssh_cluster + +Model: navier-stokes ✗ + Path: /home/user/.fz/models/navier-stokes.json + Error: Missing required field 'output' + Supported Calculators: 0 + +=== CALCULATORS === + +Calculator: local ✓ + Path: /home/user/project/.fz/calculators/local.json + URI: sh:// + Models: 1 + - perfectgas + +Calculator: ssh_cluster ✓ + Path: /home/user/.fz/calculators/ssh_cluster.json + URI: ssh://user@cluster.edu + Models: 2 + - perfectgas + - navier-stokes +``` + ### fzr - Run Parametric Calculations Execute complete parametric studies from the command line: @@ -1413,8 +1468,61 @@ export FZ_SSH_AUTO_ACCEPT_HOSTKEYS=0 # Default formula interpreter (python or R) export FZ_INTERPRETER=python + +# Custom shell binary search path (overrides system PATH) +# Windows example: SET FZ_SHELL_PATH=C:\msys64\usr\bin;C:\Program Files\Git\usr\bin +# Linux/macOS example: export FZ_SHELL_PATH=/opt/custom/bin:/usr/local/bin +export FZ_SHELL_PATH=/usr/local/bin:/usr/bin + +# Execution timeout in seconds (default per calculator) +export FZ_EXECUTION_TIMEOUT=3600 +``` + +### Shell Path Configuration (FZ_SHELL_PATH) + +The `FZ_SHELL_PATH` environment variable allows you to specify custom locations for shell binaries (grep, awk, sed, etc.) used in model output expressions and calculator commands. This is particularly important on Windows where Unix-like tools may be installed in non-standard locations. + +**Why use FZ_SHELL_PATH?** +- **Windows compatibility**: Locate tools in MSYS2, Git Bash, Cygwin, or WSL +- **Custom installations**: Use specific versions of tools from custom directories +- **Priority control**: Override system PATH to ensure correct tool versions +- **Performance**: Cached binary paths for faster resolution + +**Usage examples:** + +```bash +# Windows with MSYS2 (use semicolon separator) +SET FZ_SHELL_PATH=C:\msys64\usr\bin;C:\msys64\mingw64\bin + +# Windows with Git Bash +SET FZ_SHELL_PATH=C:\Program Files\Git\usr\bin;C:\Program Files\Git\bin + +# Linux/macOS (use colon separator) +export FZ_SHELL_PATH=/opt/homebrew/bin:/usr/local/bin + +# Priority: FZ_SHELL_PATH paths are checked BEFORE system PATH +``` + +**How it works:** +1. Commands in model `output` dictionaries are parsed for binary names (grep, awk, etc.) +2. Binary names are resolved to absolute paths using FZ_SHELL_PATH +3. Commands in `sh://` calculators are similarly resolved +4. Windows: Automatically tries both `command` and `command.exe` +5. Resolved paths are cached for performance + +**Example in model:** +```python +model = { + "output": { + "pressure": "grep 'pressure' output.txt | awk '{print $2}'" + } +} +# With FZ_SHELL_PATH=C:\msys64\usr\bin, executes: +# C:\msys64\usr\bin\grep.exe 'pressure' output.txt | C:\msys64\usr\bin\awk.exe '{print $2}' ``` +See `SHELL_PATH_IMPLEMENTATION.md` and `examples/shell_path_example.md` for detailed documentation. + ### Python Configuration ```python diff --git a/completions/README.md b/completions/README.md index d7923d9..914fc9b 100644 --- a/completions/README.md +++ b/completions/README.md @@ -84,25 +84,28 @@ Once installed, the completion scripts provide intelligent tab completion for al ### Commands with completion support -- `fz` - Main command with subcommands (input, compile, output, run) +- `fz` - Main command with subcommands (input, compile, output, run, list) - `fzi` - Parse input to find variables - `fzc` - Compile input with variable values - `fzo` - Parse output files - `fzr` - Run full parametric calculations +- `fzl` - List available models and calculators ### Examples ```bash # Press TAB after typing these to see available options: -fz # Shows: input, compile, output, run, --help, -h +fz # Shows: input, compile, output, run, list, --help, -h fzi -- # Shows: --input_path, --model, --help fzc - # Shows: -i, -m, -v, -o, -h fzr --input_path # Shows available files in current directory fzc --output_dir # Shows available directories +fzl -- # Shows: --models, --calculators, --check, --format # Subcommand completion: fz input -- # Shows options for input command fz run --results_dir # Shows available directories +fz list --format # Shows: json, markdown, table ``` ### Features diff --git a/completions/fz-completion.bash b/completions/fz-completion.bash index 7dfd5ed..c3a246b 100644 --- a/completions/fz-completion.bash +++ b/completions/fz-completion.bash @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Bash completion script for fz, fzi, fzc, fzo, fzr commands +# Bash completion script for fz, fzi, fzc, fzo, fzr, fzl commands # Helper function to complete file paths _fz_complete_files() { @@ -147,6 +147,36 @@ _fzr() { return 0 } +# Completion for fzl command +_fzl() { + local cur prev opts + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + opts="--models -m --calculators -c --check --format -f --help -h --version" + + case "${prev}" in + --models|-m) + COMPREPLY=($(compgen -W "* *.json" -- ${cur})) + return 0 + ;; + --calculators|-c) + COMPREPLY=($(compgen -W "* sh:// ssh:// slurm:// funz:// cache://" -- ${cur})) + return 0 + ;; + --format|-f) + COMPREPLY=($(compgen -W "json markdown table" -- ${cur})) + return 0 + ;; + *) + ;; + esac + + COMPREPLY=($(compgen -W "${opts}" -- ${cur})) + return 0 +} + # Completion for fz main command _fz() { local cur prev opts subcommand @@ -158,7 +188,7 @@ _fz() { subcommand="" for ((i=1; i < COMP_CWORD; i++)); do case "${COMP_WORDS[i]}" in - input|compile|output|run) + input|compile|output|run|list) subcommand="${COMP_WORDS[i]}" break ;; @@ -167,7 +197,7 @@ _fz() { # If no subcommand yet, offer subcommands and main options if [[ -z "$subcommand" ]]; then - opts="input compile output run --help -h --version" + opts="input compile output run list --help -h --version" COMPREPLY=($(compgen -W "${opts}" -- ${cur})) return 0 fi @@ -274,6 +304,27 @@ _fz() { ;; esac ;; + list) + opts="--models -m --calculators -c --check --format -f --help -h --version" + case "${prev}" in + --models|-m) + COMPREPLY=($(compgen -W "* *.json" -- ${cur})) + return 0 + ;; + --calculators|-c) + COMPREPLY=($(compgen -W "* sh:// ssh:// slurm:// funz:// cache://" -- ${cur})) + return 0 + ;; + --format|-f) + COMPREPLY=($(compgen -W "json markdown table" -- ${cur})) + return 0 + ;; + *) + COMPREPLY=($(compgen -W "${opts}" -- ${cur})) + return 0 + ;; + esac + ;; esac return 0 @@ -285,3 +336,4 @@ complete -F _fzi fzi complete -F _fzc fzc complete -F _fzo fzo complete -F _fzr fzr +complete -F _fzl fzl diff --git a/completions/fz-completion.zsh b/completions/fz-completion.zsh index 3993d81..20736cc 100644 --- a/completions/fz-completion.zsh +++ b/completions/fz-completion.zsh @@ -1,5 +1,5 @@ -#compdef fz fzi fzc fzo fzr -# Zsh completion script for fz, fzi, fzc, fzo, fzr commands +#compdef fz fzi fzc fzo fzr fzl +# Zsh completion script for fz, fzi, fzc, fzo, fzr, fzl commands # Completion for fzi command _fzi() { @@ -45,6 +45,17 @@ _fzr() { '--version[Show version]' } +# Completion for fzl command +_fzl() { + _arguments \ + '(-m --models)'{-m,--models}'[Model pattern to match (supports glob)]:model pattern:' \ + '(-c --calculators)'{-c,--calculators}'[Calculator pattern to match (supports glob/regex)]:calculator pattern:' \ + '--check[Validate each model and calculator]' \ + '(-f --format)'{-f,--format}'[Output format]:format:(json markdown table)' \ + '(-h --help)'{-h,--help}'[Show help message]' \ + '--version[Show version]' +} + # Completion for fz main command with subcommands _fz() { local curcontext="$curcontext" state line @@ -64,6 +75,7 @@ _fz() { 'compile:Compile input with variable values' 'output:Parse output files' 'run:Run full parametric calculations' + 'list:List available models and calculators' ) _describe -t commands 'fz command' subcommands ;; @@ -105,6 +117,15 @@ _fz() { '(-h --help)'{-h,--help}'[Show help message]' \ '--version[Show version]' ;; + list) + _arguments \ + '(-m --models)'{-m,--models}'[Model pattern to match (supports glob)]:model pattern:' \ + '(-c --calculators)'{-c,--calculators}'[Calculator pattern to match (supports glob/regex)]:calculator pattern:' \ + '--check[Validate each model and calculator]' \ + '(-f --format)'{-f,--format}'[Output format]:format:(json markdown table)' \ + '(-h --help)'{-h,--help}'[Show help message]' \ + '--version[Show version]' + ;; esac ;; esac @@ -116,3 +137,4 @@ compdef _fzi fzi compdef _fzc fzc compdef _fzo fzo compdef _fzr fzr +compdef _fzl fzl diff --git a/context/calculators.md b/context/calculators.md index f73669c..ade506e 100644 --- a/context/calculators.md +++ b/context/calculators.md @@ -236,6 +236,208 @@ mpirun -np 16 ./simulation input.txt # Results written to output.txt ``` +## SLURM Workload Manager (`slurm://`) + +Execute calculations on SLURM clusters (local or remote). + +### Basic Syntax + +```python +# Local SLURM +calculators = "slurm://:partition/command" + +# Remote SLURM via SSH +calculators = "slurm://user@host:partition/command" +calculators = "slurm://user@host:port:partition/command" # with custom SSH port +``` + +**Note**: For local execution, the partition must be prefixed with a colon (`:partition`). + +### Examples + +**Example 1: Local SLURM execution** + +```python +calculators = "slurm://:compute/bash script.sh" +``` + +**Example 2: Remote SLURM on HPC cluster** + +```python +calculators = "slurm://user@cluster.edu:gpu/bash simulation.sh" +``` + +**Example 3: Remote SLURM with custom SSH port** + +```python +calculators = "slurm://user@hpc.university.edu:2222:compute/bash run.sh" +``` + +**Example 4: Multiple SLURM partitions for parallel execution** + +```python +calculators = [ + "slurm://user@cluster:compute/bash calc.sh", + "slurm://user@cluster:gpu/bash calc.sh", + "slurm://user@cluster:highmem/bash calc.sh" +] +``` + +### How it Works + +**Local execution**: +1. Uses `srun --partition= ` directly +2. Automatically handles SLURM partition scheduling +3. Supports interrupt handling (Ctrl+C terminates SLURM jobs) + +**Remote execution**: +1. Connects via SSH to remote cluster +2. Transfers input files via SFTP +3. Executes `srun` on remote cluster +4. Retrieves results via SFTP + +### Features + +- **Partition specification**: Control which SLURM partition to use +- **Automatic file transfer**: For remote execution +- **Timeout handling**: Configurable execution timeouts +- **Interrupt support**: Graceful job termination with Ctrl+C +- **Compatible with all SLURM schedulers** + +### Requirements + +- **Local**: SLURM installed (`srun` command available) +- **Remote**: SSH access to SLURM cluster + `paramiko` library + +### Configuration + +**Environment variables**: +```bash +export FZ_EXECUTION_TIMEOUT=3600 # Timeout in seconds +export FZ_SSH_KEEPALIVE=300 # For remote SLURM +``` + +**Python**: +```python +import os +os.environ['FZ_EXECUTION_TIMEOUT'] = '7200' # 2 hours +``` + +## Funz Server Calculator (`funz://`) + +Execute calculations using legacy Java Funz calculator servers via TCP socket protocol. + +### Basic Syntax + +```python +# Local Funz server +calculators = "funz://:port/code" + +# Remote Funz server +calculators = "funz://host:port/code" +``` + +**URI Format**: `funz://[host]:/` +- `host`: Server hostname (default: localhost) +- `port`: Server port (required) +- `code`: Calculator code/model name (e.g., "R", "Python", "Modelica", "bash") + +### Examples + +**Example 1: Connect to local Funz server** + +```python +calculators = "funz://:5555/R" +``` + +**Example 2: Connect to remote Funz server** + +```python +calculators = "funz://server.example.com:5555/Python" +``` + +**Example 3: Multiple Funz servers for parallel execution** + +```python +calculators = [ + "funz://:5555/R", + "funz://:5556/R", + "funz://:5557/R" +] +``` + +**Example 4: Complete parametric study** + +```python +import fz + +model = { + "output": { + "pressure": "grep 'pressure = ' output.txt | awk '{print $3}'" + } +} + +results = fz.fzr( + "input.txt", + {"temp": [100, 200, 300]}, + model, + calculators="funz://:5555/bash" +) +``` + +### How it Works + +1. **Calculator reservation**: Connects to Funz server and reserves calculator +2. **File upload**: Transfers input files to server +3. **Remote execution**: Executes calculation via Funz protocol +4. **Result download**: Retrieves output files +5. **Unreservation**: Releases calculator and cleans up + +### Funz Protocol + +The Funz calculator uses a text-based TCP socket communication protocol: + +- **RESERVE**: Request calculator reservation with authentication +- **EXECUTE**: Submit calculation job +- **STATUS**: Check job status +- **DOWNLOAD**: Retrieve result files +- **UNRESERVE**: Release calculator + +### UDP Discovery + +Funz calculators broadcast their availability via UDP: + +``` +Port 5555 (UDP): Broadcasts availability every ~5 seconds + Message format: + Line 1: Protocol version (e.g., "FUNZ1.0") + Line 2: TCP port number + Line 3+: Available codes (bash, R, Python, etc.) + +Port (dynamic): Actual calculator communication +``` + +See `FUNZ_UDP_DISCOVERY.md` for detailed protocol documentation. + +### Features + +- **Compatible with legacy Java Funz servers** +- **Automatic file upload/download** +- **TCP socket communication** +- **Calculator reservation system** +- **Interrupt handling support** +- **Authentication support** + +### Requirements + +- Funz calculator server running (Java-based) +- Network access to server port +- No Python dependencies beyond standard library + +### Starting a Funz Calculator + +See `tools/start_funz_calculator.sh` and `tools/setup_funz_calculator.sh` for helper scripts. + ## Cache Calculator (`cache://`) Reuse results from previous calculations based on input file hashes. diff --git a/context/core-functions.md b/context/core-functions.md index 3f5ac0c..2ab5b1d 100644 --- a/context/core-functions.md +++ b/context/core-functions.md @@ -1,13 +1,98 @@ # FZ Core Functions -## The Four Core Functions +## The Core Functions -FZ provides four main functions for parametric computing: +FZ provides five main functions for parametric computing: 1. **`fzi`** - Parse **I**nput files to identify variables 2. **`fzc`** - **C**ompile input files by substituting variables 3. **`fzo`** - Parse **O**utput files from calculations 4. **`fzr`** - **R**un complete parametric calculations end-to-end +5. **`fzl`** - **L**ist and validate installed models and calculators + +## fzl - List and Validate Models/Calculators + +**Purpose**: List installed models and calculators, optionally validating their configuration. + +### Function Signature + +```python +import fz + +result = fz.fzl(models="*", calculators="*", check=False) +``` + +**Parameters**: +- `models` (str): Model pattern to match (default: "*" for all). Supports glob patterns. +- `calculators` (str): Calculator pattern to match (default: "*" for all). Supports glob/regex. +- `check` (bool): Validate each model and calculator (default: False) + +**Returns**: Dictionary with two keys: "models" and "calculators" + +### Examples + +**Example 1: List all models and calculators** + +```python +import fz + +result = fz.fzl() +print(result) +# Output: +# { +# "models": { +# "perfectgas": { +# "path": "/home/user/.fz/models/perfectgas.json", +# "supported_calculators": ["local", "ssh_cluster"] +# } +# }, +# "calculators": { +# "local": { +# "path": "/home/user/.fz/calculators/local.json", +# "uri": "sh://", +# "models": ["perfectgas"] +# } +# } +# } +``` + +**Example 2: Filter by pattern** + +```python +result = fz.fzl(models="perfect*", calculators="ssh*") +# Only lists models matching "perfect*" and calculators matching "ssh*" +``` + +**Example 3: Validate configurations** + +```python +result = fz.fzl(check=True) +# Each model/calculator includes check_status: "passed" or "failed" +# Failed items include check_error with details +``` + +**Example 4: CLI usage** + +```bash +# List all +fzl + +# List with validation +fzl --check + +# Filter and format +fzl --models "navier*" --format table + +# JSON output +fzl --format json > config.json +``` + +### Use Cases + +- **Discover available models**: See what models are installed +- **Verify configurations**: Check for errors before running calculations +- **Document setup**: Export configuration inventory +- **Troubleshoot**: Identify missing or broken configurations ## fzi - Parse Input Variables diff --git a/context/overview.md b/context/overview.md index c4a08fe..38ab861 100644 --- a/context/overview.md +++ b/context/overview.md @@ -10,6 +10,19 @@ FZ is a parametric scientific computing framework that automates running computa - **Remote execution**: Run calculations on remote servers via SSH - **Result management**: Organize and parse results into structured DataFrames +## What's New in 0.9.1 + +- **`fzl` command**: List and validate installed models and calculators +- **SLURM support**: New `slurm://` calculator for workload managers +- **Funz server support**: New `funz://` calculator for legacy Java Funz servers +- **R interpreter**: Formula evaluation with R statistical computing +- **FZ_SHELL_PATH**: Custom binary path configuration (essential for Windows) +- **Variable defaults**: `${var~default}` syntax for default values +- **Progress callbacks**: Real-time monitoring of calculation progress +- **Old Funz syntax**: Backward compatibility with `?var` syntax + +See `NEWS.md` for complete release notes. + ## When to Use FZ Use FZ when you need to: diff --git a/context/syntax-guide.md b/context/syntax-guide.md index 0fc6fcf..cd6f67e 100644 --- a/context/syntax-guide.md +++ b/context/syntax-guide.md @@ -30,6 +30,29 @@ model = { - Case-sensitive: `$Temp` ≠ `$temp` - Cannot start with number: `$1var` is invalid +### Legacy Funz Syntax Compatibility + +FZ supports the legacy Java Funz variable syntax for backward compatibility: + +```text +# Old Funz syntax (question mark prefix) +Temperature: ?T_celsius +Pressure: ?pressure + +# Equivalent to modern FZ syntax +Temperature: $T_celsius +Pressure: $pressure +``` + +**Automatic detection**: `?var` is automatically converted to `$var` internally. + +**Use cases**: +- Migrating from Java Funz to Python FZ +- Reusing existing Funz input templates +- Backward compatibility with legacy projects + +See `examples/java_funz_syntax_example.py` for complete examples. + ### Default Values Variables can specify default values using `~` syntax: diff --git a/fz/__init__.py b/fz/__init__.py index d685fa5..8359e44 100644 --- a/fz/__init__.py +++ b/fz/__init__.py @@ -88,7 +88,7 @@ def list_models(global_list=False): return list_installed_models(global_list=global_list) -__version__ = "0.9.0" +__version__ = "0.9.1" __all__ = [ "fzi", "fzc", "fzo", "fzr", "fzl", "install", "uninstall", "list_models", diff --git a/fz/_version.py b/fz/_version.py index 5b30a31..e22bea5 100644 --- a/fz/_version.py +++ b/fz/_version.py @@ -5,6 +5,6 @@ DO NOT EDIT MANUALLY - changes will be overwritten. """ -__version__ = "0.9.0" -__commit_date__ = "2025-10-17 18:15:40 +0200" -__commit_hash__ = "f846dd0" +__version__ = "0.9.1" +__commit_date__ = "2026-01-25 18:25:30 +0100" +__commit_hash__ = "5b42273" diff --git a/fz/cli.py b/fz/cli.py index 2ab5f66..a7c1233 100644 --- a/fz/cli.py +++ b/fz/cli.py @@ -254,7 +254,6 @@ def fzl_main(): result = fzl_func(models=args.models, calculators=args.calculators, check=args.check) if args.format == "json": - import json print(json.dumps(result, indent=2)) elif args.format == "table": # Table format @@ -598,7 +597,6 @@ def main(): result = fzl_func(models=args.models, calculators=args.calculators, check=args.check) if args.format == "json": - import json print(json.dumps(result, indent=2)) elif args.format == "table": # Table format diff --git a/fz/config.py b/fz/config.py index 2df989d..6fb5277 100755 --- a/fz/config.py +++ b/fz/config.py @@ -5,7 +5,7 @@ """ import os -from typing import Optional, Union +from typing import Optional from enum import Enum diff --git a/fz/core.py b/fz/core.py index 75e7422..c50061c 100644 --- a/fz/core.py +++ b/fz/core.py @@ -4,8 +4,6 @@ import os import re -import subprocess -import tempfile import json import ast import logging @@ -13,12 +11,9 @@ import uuid import signal import sys -import io import platform from pathlib import Path -from typing import Dict, List, Union, Any, Optional, Tuple, TYPE_CHECKING -from concurrent.futures import ThreadPoolExecutor, as_completed -from contextlib import contextmanager +from typing import Dict, List, Union, Any, Optional, TYPE_CHECKING # Configure UTF-8 encoding for Windows to handle emoji output if platform.system() == "Windows": @@ -72,30 +67,23 @@ def utf8_open( import threading from collections import defaultdict -import shutil -from .logging import log_error, log_warning, log_info, log_debug, log_progress -from .config import get_config, get_interpreter +from .logging import log_error, log_warning, log_debug +from .config import get_interpreter from .helpers import ( fz_temporary_directory, - _get_result_directory, - _get_case_directories, _cleanup_fzr_resources, _resolve_model, - get_calculator_manager, - try_calculators_with_retry, - run_single_case, + _resolve_calculators_arg, + _calculator_supports_model, run_cases_parallel, compile_to_result_directories, prepare_temp_directories, - prepare_case_directories, ) from .shell import run_command, replace_commands_in_string from .io import ( ensure_unique_directory, - create_hash_file, resolve_cache_paths, - find_cache_match, load_aliases, ) from .interpreter import ( @@ -105,7 +93,7 @@ def utf8_open( _get_var_prefix, _get_formula_prefix, ) -from .runners import resolve_calculators, run_calculation +from .runners import resolve_calculators def _print_function_help(func_name: str, func_doc: str): @@ -155,12 +143,12 @@ def wrapper(*args, **kwargs): # Check if it's an argument name error if "got an unexpected keyword argument" in error_msg or \ "missing" in error_msg and "required positional argument" in error_msg: - print(f"\n⚠️ This error suggests improper argument names were used.", file=sys.stderr) - print(f"Please check the function signature below:\n", file=sys.stderr) + print("\n⚠️ This error suggests improper argument names were used.", file=sys.stderr) + print("Please check the function signature below:\n", file=sys.stderr) _print_function_help(func.__name__, func.__doc__) else: - print(f"\n⚠️ This error suggests an argument has an invalid type.", file=sys.stderr) - print(f"Please check the expected types in the function signature:\n", file=sys.stderr) + print("\n⚠️ This error suggests an argument has an invalid type.", file=sys.stderr) + print("Please check the expected types in the function signature:\n", file=sys.stderr) _print_function_help(func.__name__, func.__doc__) # Re-raise the exception @@ -169,8 +157,8 @@ def wrapper(*args, **kwargs): except ValueError as e: error_msg = str(e) print(f"\n❌ ValueError in {func.__name__}(): {error_msg}", file=sys.stderr) - print(f"\n⚠️ This error suggests an argument has an improper value.", file=sys.stderr) - print(f"Please check the constraints in the function documentation:\n", file=sys.stderr) + print("\n⚠️ This error suggests an argument has an improper value.", file=sys.stderr) + print("Please check the constraints in the function documentation:\n", file=sys.stderr) _print_function_help(func.__name__, func.__doc__) # Re-raise the exception @@ -179,7 +167,7 @@ def wrapper(*args, **kwargs): except FileNotFoundError as e: error_msg = str(e) print(f"\n❌ FileNotFoundError in {func.__name__}(): {error_msg}", file=sys.stderr) - print(f"\n⚠️ Please check that the file or directory path is correct.\n", file=sys.stderr) + print("\n⚠️ Please check that the file or directory path is correct.\n", file=sys.stderr) # Re-raise the exception raise @@ -265,14 +253,7 @@ def _parse_argument(arg, alias_type=None): return arg -# Import calculator-related functions from helpers -from .helpers import ( - _calculator_supports_model, - _extract_calculator_uri, - _find_all_calculators, - _filter_calculators_by_model, - _resolve_calculators_arg -) +# Calculator-related functions imported later where needed def check_bash_availability_on_windows(): @@ -543,7 +524,7 @@ def cleanup_all_calculators(self): f"🧹 Cleanup: Force-releasing calculator {calc_id} from thread {thread_id}" ) calc_lock.release() - except Exception as e: + except Exception: # Lock might not be held, which is fine pass @@ -609,8 +590,6 @@ def _validate_calculator(calc_spec, calc_display): - error_message: Error description if failed, None if passed """ import tempfile - import os - from pathlib import Path try: # Extract URI from calculator spec @@ -719,9 +698,6 @@ def fzl(models: str = "*", calculators: str = "*", check: bool = False) -> Dict[ >>> result = fzl(models="*", calculators="*", check=True) >>> print(result["calculators"]["sh://"]["check_status"]) """ - from pathlib import Path - from .helpers import find_items_by_pattern, _calculator_supports_model, _resolve_calculators_arg, _resolve_model - # Find all matching models models_list = [] search_dirs = [Path.cwd() / ".fz" / "models", Path.home() / ".fz" / "models"] @@ -1411,10 +1387,6 @@ def fzr( if not input_path_obj.exists(): raise FileNotFoundError(f"Input path '{input_path}' not found") - # Get the global formula interpreter - from .config import get_interpreter - interpreter = get_interpreter() - # Get model ID for calculator filtering model_id = model.get("id") if isinstance(model, dict) else None diff --git a/fz/helpers.py b/fz/helpers.py index d6a7683..28d47cf 100644 --- a/fz/helpers.py +++ b/fz/helpers.py @@ -6,6 +6,7 @@ import shutil import threading import time +import uuid import itertools from pathlib import Path from typing import Dict, List, Tuple, Union, Any, Optional @@ -17,6 +18,24 @@ from .spinner import CaseSpinner, CaseStatus +def format_time(seconds): + """ + Format seconds into human-readable time string + + Args: + seconds: Number of seconds + + Returns: + Formatted string (e.g., "5.3s", "2.5m", "1.2h") + """ + if seconds < 60: + return f"{seconds:.1f}s" + elif seconds < 3600: + return f"{seconds/60:.1f}m" + else: + return f"{seconds/3600:.1f}h" + + @contextmanager def fz_temporary_directory(session_cwd=None): """ @@ -28,9 +47,6 @@ def fz_temporary_directory(session_cwd=None): Yields: Path to temporary directory """ - import time - import uuid - # Use centralized temp base directory if session_cwd is None: session_cwd = os.getcwd() @@ -265,7 +281,6 @@ def _resolve_model(model: Union[str, Dict]) -> Dict: if isinstance(model, str): import json import sys - from pathlib import Path from .io import load_aliases json_error = None @@ -1099,15 +1114,6 @@ def run_cases_parallel(var_combinations: List[Dict], temp_path: Path, resultsdir remaining_cases = len(var_combinations) - completed_count estimated_remaining = avg_time_per_case * remaining_cases - # Format time estimates - def format_time(seconds): - if seconds < 60: - return f"{seconds:.1f}s" - elif seconds < 3600: - return f"{seconds/60:.1f}m" - else: - return f"{seconds/3600:.1f}h" - log_progress(f"📊 Progress: {completed_count}/{len(var_combinations)} cases completed " f"({completed_count/len(var_combinations)*100:.1f}%), " f"ETA: {format_time(estimated_remaining)}") @@ -1141,15 +1147,6 @@ def format_time(seconds): case_results = [None] * len(var_combinations) completed_count = 0 - # Helper function to format time - def format_time(seconds): - if seconds < 60: - return f"{seconds:.1f}s" - elif seconds < 3600: - return f"{seconds/60:.1f}m" - else: - return f"{seconds/3600:.1f}h" - for future in as_completed(future_to_index): # Check for interrupt if is_interrupted(): diff --git a/fz/runners.py b/fz/runners.py index 24ad217..3cbff1c 100644 --- a/fz/runners.py +++ b/fz/runners.py @@ -695,7 +695,6 @@ def _resolve_paths_in_segment(segment: str, original_cwd: str) -> tuple[str, boo """ import shlex import re - import os # Parse command parts using shlex for proper quote handling try: @@ -1134,8 +1133,6 @@ def run_ssh_calculation( if not username: # Try to get username from environment or use current user - import getpass - username = os.getenv("SSH_USER") or getpass.getuser() # Validate connection security @@ -1421,8 +1418,6 @@ def _run_local_slurm_calculation( with open(out_file_path, "w") as out_file, open(err_file_path, "w") as err_file: # Start process with Popen to allow interrupt handling - from .shell import run_command - process = run_command( full_command, shell=True, @@ -2162,7 +2157,6 @@ def read_response(): log_debug(f"✅ Phase 1 complete") # Phase 2: Send project code and tagged values - import getpass tagged_values = { "USERNAME": getpass.getuser() }