From 7eecef1d1760868176eb06e9c7c775eb1e3cb64a Mon Sep 17 00:00:00 2001 From: Fitsum <138158520+fitsblb@users.noreply.github.com> Date: Thu, 16 Oct 2025 10:25:48 -0400 Subject: [PATCH 1/2] feat: Add SHAP explainability dashboard with feature contribution visualization --- .dockerignore | 5 +- .env.example | 54 +- .github/workflows/ci.yml | 15 +- .github/workflows/data-contract.yml | 23 +- .gitignore | 3 + CHANGELOG.md | 66 +++ Readme.md | 215 ++++++-- .../dev/{ => archive}/thresholds_7feat.json | 0 configs/dev/config.yaml | 62 ++- configs/dev/thresholds.json | 51 +- configs/dev/whitelist.json | 38 ++ docker-compose.yml | 40 ++ docker/compose.yml | 22 +- docker/feature.Dockerfile | 8 - docker/gateway.Dockerfile | 1 + docker/model.Dockerfile | 6 +- docs/FEATURE_EXTRACTION.md | 88 +++- gx/expectations/.ge_store_backend_id | 2 +- .../phiusiil_8feature_production.json | 263 +++++++++ gx/expectations/phiusiil_minimal.json | 331 ------------ gx/great_expectations.yml | 62 ++- gx/uncommitted/config_variables.yml | 2 +- .../validations/.ge_store_backend_id | 2 +- init_ge.py | 0 model_logs.txt | Bin 0 -> 31068 bytes models/dev/{ => archive}/model_7feat.pkl | Bin .../dev/{ => archive}/model_7feat_meta.json | 0 notebooks/03_prod_valid.ipynb | 498 ++++++++++++++++++ outputs/benchmark_results.txt | 9 + pyproject.toml | 18 + requirements-docker.txt | 3 + .../{ => archive}/materialize_url_features.py | 0 scripts/benchmark.py | 193 +++++++ scripts/ge_build_phiusiil_suite.py | 426 ++++++++++++--- scripts/ge_check.py | 126 +++-- scripts/smoke_judge_selector.py | 40 +- src/__pycache__/__init__.cpython-311.pyc | Bin 149 -> 0 bytes .../__pycache__/thresholds.cpython-311.pyc | Bin 1760 -> 2502 bytes src/common/thresholds.py | 26 +- src/feature_svc/__init__.py | 1 - src/feature_svc/main.py | 24 - .../__pycache__/judge_wire.cpython-311.pyc | Bin 6836 -> 10182 bytes src/gateway/__pycache__/main.cpython-311.pyc | Bin 12086 -> 16904 bytes src/gateway/judge_wire.py | 109 +++- src/gateway/main.py | 166 +++++- src/gateway/static/explain.html | 481 +++++++++++++++++ .../__pycache__/adapter.cpython-311.pyc | Bin 4852 -> 5392 bytes .../__pycache__/contracts.cpython-311.pyc | Bin 2693 -> 3444 bytes .../__pycache__/stub.cpython-311.pyc | Bin 3315 -> 5278 bytes src/judge_svc/adapter.py | 27 +- src/judge_svc/contracts.py | 40 +- src/judge_svc/stub.py | 110 ++-- .../__pycache__/main.cpython-311.pyc | Bin 24708 -> 26428 bytes src/model_svc/main.py | 208 +++++++- test_shap_locally.py | 101 ++++ ...t_gateway_e2e.cpython-311-pytest-8.4.1.pyc | Bin 11363 -> 11401 bytes ...l_integration.cpython-311-pytest-8.4.1.pyc | Bin 22599 -> 22579 bytes ...e_llm_adapter.cpython-311-pytest-8.4.1.pyc | Bin 7930 -> 8155 bytes ...est_model_svc.cpython-311-pytest-8.4.1.pyc | Bin 19676 -> 18543 bytes tests/test_enhanced_routing.py | 199 +++++++ tests/test_gateway_e2e.py | 9 +- tests/test_gateway_model_integration.py | 30 +- tests/test_judge_llm_adapter.py | 10 + tests/test_model_svc.py | 14 +- 64 files changed, 3440 insertions(+), 787 deletions(-) rename configs/dev/{ => archive}/thresholds_7feat.json (100%) create mode 100644 configs/dev/whitelist.json create mode 100644 docker-compose.yml delete mode 100644 docker/feature.Dockerfile create mode 100644 gx/expectations/phiusiil_8feature_production.json delete mode 100644 gx/expectations/phiusiil_minimal.json delete mode 100644 init_ge.py create mode 100644 model_logs.txt rename models/dev/{ => archive}/model_7feat.pkl (100%) rename models/dev/{ => archive}/model_7feat_meta.json (100%) create mode 100644 notebooks/03_prod_valid.ipynb create mode 100644 outputs/benchmark_results.txt rename scripts/{ => archive}/materialize_url_features.py (100%) create mode 100644 scripts/benchmark.py delete mode 100644 src/__pycache__/__init__.cpython-311.pyc delete mode 100644 src/feature_svc/__init__.py delete mode 100644 src/feature_svc/main.py create mode 100644 src/gateway/static/explain.html create mode 100644 test_shap_locally.py create mode 100644 tests/test_enhanced_routing.py diff --git a/.dockerignore b/.dockerignore index 53742ac..ee0a507 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,7 +2,7 @@ .git .venv venv -data +# data # REMOVED: Need data/tld_probs.json for 8-feature model mlruns mlartifacts outputs @@ -30,8 +30,9 @@ mlflow.db .bandit .flake8 pytest.ini -requirements*.txt +requirements.txt requirements.in +# requirements-docker.txt is needed for Docker builds! # Environment & Secrets .env diff --git a/.env.example b/.env.example index 6a96c90..8e5de33 100644 --- a/.env.example +++ b/.env.example @@ -1,35 +1,47 @@ -# Judge backend: stub | llm +# ============================================================ +# JUDGE CONFIGURATION +# ============================================================ JUDGE_BACKEND=stub -# LLM Judge (used when JUDGE_BACKEND=llm) +# LLM Judge (only used when JUDGE_BACKEND=llm) OLLAMA_HOST=http://localhost:11434 JUDGE_MODEL=llama3.2:1b JUDGE_TIMEOUT_SECS=12 -# Optional: store models off C: to save space -# OLLAMA_MODELS=D:\ollama\models - -# MongoDB Audit Logging (Optional) -MONGO_URI= -MONGO_DB=phishguard - -# Thresholds (use the tuned URL-only thresholds by default) -THRESHOLDS_JSON=configs/dev/thresholds.json # ============================================================ # MODEL SERVICE CONFIGURATION # ============================================================ -# Configuration file path (can be overridden for different environments) +# Configuration file path CONFIG_PATH=configs/dev/config.yaml -# Primary model (production model used for decisions) -MODEL_PATH=models/dev/model_7feat.pkl -MODEL_META_PATH=models/dev/model_7feat_meta.json +# Primary model (8-feature production model) +PRIMARY_MODEL_PATH=models/dev/model_8feat.pkl +PRIMARY_META_PATH=models/dev/model_8feat_meta.json + +# Shadow testing (DISABLED for production) +SHADOW_ENABLED=false +SHADOW_MODEL_PATH=models/dev/model_7feat.pkl +SHADOW_META_PATH=models/dev/model_7feat_meta.json + +# Service URLs +MODEL_SVC_URL=http://localhost:9000 +GATEWAY_PORT=8000 +MODEL_SVC_PORT=9000 + +# Thresholds (gray-zone policy bands) +THRESHOLDS_JSON=configs/dev/thresholds.json + +# ============================================================ +# DATA & LOGGING +# ============================================================ + +# MongoDB Audit Logging (Optional - disabled by default) +MONGO_URI= +MONGO_DB=phishguard -# Shadow testing (A/B testing with 8-feature model) -SHADOW_ENABLED=true -SHADOW_MODEL_PATH=models/dev/model_8feat.pkl -SHADOW_META_PATH=models/dev/model_8feat_meta.json +# Logging +LOG_LEVEL=INFO -# Model service URL (used by gateway to call model service) -MODEL_SVC_URL=http://localhost:9000 \ No newline at end of file +# Optional: Ollama models storage (uncomment if needed) +# OLLAMA_MODELS=D:\ollama\models \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c893a93..2739c66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,9 +1,9 @@ -# .github/workflows/ci.yml β€” Option A (paused CI for feature/dev) +# .github/workflows/ci.yml β€” CI/CD for 8-feature PhishGuard project name: Tests on: pull_request: - branches: ["main"] # only runs on PRs into main - workflow_dispatch: # allow manual runs from Actions tab + branches: ["main"] # only runs on PRs into main + workflow_dispatch: # allow manual runs from Actions tab jobs: Tests: @@ -13,7 +13,8 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.11" - - run: | + - name: Install dependencies + run: | python -m pip install --upgrade pip # Skip pywin32 on Linux runners, install everything else grep -v "pywin32" requirements.txt > requirements-linux.txt || cp requirements.txt requirements-linux.txt @@ -21,9 +22,11 @@ jobs: pip install pytest pytest-cov black isort flake8 mypy - name: Set PYTHONPATH run: echo "PYTHONPATH=$PYTHONPATH:$(pwd)/src:$(pwd)" >> $GITHUB_ENV - - run: | + - name: Code quality checks + run: | black --check . isort --check-only . flake8 . mypy src - - run: python -m pytest tests/ -q + - name: Run tests + run: python -m pytest tests/ -v --tb=short diff --git a/.github/workflows/data-contract.yml b/.github/workflows/data-contract.yml index 2a64bfc..5e43af2 100644 --- a/.github/workflows/data-contract.yml +++ b/.github/workflows/data-contract.yml @@ -12,13 +12,24 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.11" - - run: pip install -U pip pandas numpy - - name: Run data contract check (if file exists) + - run: pip install -U pip pandas numpy great-expectations + - name: Set PYTHONPATH + run: echo "PYTHONPATH=$PYTHONPATH:$(pwd)/src:$(pwd)" >> $GITHUB_ENV + - name: Run data contract check (8-feature model) shell: bash run: | - CSV="data/processed/phiusiil_clean_urlfeats.csv" - if [ -f "$CSV" ]; then - python scripts/ge_check.py --csv "$CSV" + # Check for 8-feature model data (current) + CSV_V2="data/processed/phiusiil_features_v2.csv" + # Legacy fallback + CSV_LEGACY="data/processed/phiusiil_clean_urlfeats.csv" + + if [ -f "$CSV_V2" ]; then + echo "Found 8-feature model data: $CSV_V2" + python scripts/ge_check.py --csv "$CSV_V2" + elif [ -f "$CSV_LEGACY" ]; then + echo "Found legacy data: $CSV_LEGACY" + python scripts/ge_check.py --csv "$CSV_LEGACY" else - echo "No processed CSV found ($CSV); skipping." + echo "No processed CSV found. Checked: $CSV_V2, $CSV_LEGACY" + echo "Skipping data contract validation." fi diff --git a/.gitignore b/.gitignore index 7a1c05d..b62baa2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ venv/ # Python cache __pycache__/ +**/__pycache__/ *.py[cod] *$py.class *.so @@ -43,4 +44,6 @@ outputs/*.csv mlflow.db +docs/*.md + diff --git a/CHANGELOG.md b/CHANGELOG.md index ff9f802..6df8eec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,72 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.2.0] - 2025-10-15 + +### Added +- **8-Feature Model**: Complete upgrade to production-ready 8-feature phishing detection + - `IsHTTPS`: Binary HTTPS indicator for security baseline + - `TLDLegitimateProb`: Bayesian TLD legitimacy probability with 1401+ TLD dataset + - `CharContinuationRate`: Character repetition pattern detection + - `SpacialCharRatioInURL`: Special character density analysis + - `URLCharProb`: URL character sequence probability scoring + - `LetterRatioInURL`: Alphabetic character ratio for readability assessment + - `NoOfOtherSpecialCharsInURL`: Special character count for complexity analysis + - `DomainLength`: RFC-compliant domain length validation +- **Enhanced Judge System**: Modernized LLM integration with 8-feature model + - Updated judge contracts to use production features + - Enhanced stub logic with sophisticated heuristics + - Improved LLM prompts with detailed feature descriptions + - Graceful fallback from modern to legacy features +- **Comprehensive Test Suite**: 52 tests with 100% pass rate + - Updated all tests for 8-feature model compatibility + - Enhanced integration tests for microservice communication + - Modernized judge system tests with production features + - Fixed whitelist behavior validation + +### Changed +- **Great Expectations**: Updated data validation for 8-feature model + - Migrated from 3-feature legacy validation to 8-feature production validation + - Updated thresholds and expectations for new feature ranges + - Enhanced data contract validation with feature-specific checks +- **Judge System Architecture**: Complete alignment with production features + - FeatureDigest contract updated with 8 required + 3 optional legacy fields + - Enhanced decision logic using modern feature signals + - Improved context and audit trail with comprehensive feature logging +- **GitHub Workflows**: Updated CI/CD for modern project structure + - Enhanced data contract workflow with 8-feature model support + - Updated CI workflow with better error reporting + - Added fallback logic for legacy data file compatibility + +### Removed +- **Feature Service**: Eliminated redundant microservice + - Removed `src/feature_svc/` directory and related code + - Updated Docker compose to remove feature service dependency + - Cleaned up unused `docker/feature.Dockerfile` + - Streamlined architecture to gateway + model services only +- **Legacy Scripts**: Deprecated obsolete feature extraction + - Identified `scripts/materialize_url_features.py` as obsolete + - Removed references to deprecated 3-feature model components + +### Fixed +- **Docker Configuration**: Enhanced for 8-feature model deployment + - Added `data/` directory to Docker images for TLD probability data + - Updated environment variables for proper service communication + - Fixed `.dockerignore` to include necessary data files +- **Test Infrastructure**: Resolved all compatibility issues + - Fixed whitelist behavior in integration tests + - Updated API contract expectations for current implementation + - Resolved version mismatches and dependency issues + - Enhanced test reliability with non-whitelisted test domains + +### Technical Details +- **Feature Engineering**: Advanced URL-only features with statistical and linguistic analysis +- **Data Validation**: 31 comprehensive Great Expectations rules for production data quality +- **Performance**: Maintained 204ms API response time with enhanced feature extraction +- **Compatibility**: Backward compatibility maintained through optional legacy feature support + +--- + ## [0.1.0] - 2025-09-17 ### Added diff --git a/Readme.md b/Readme.md index 1b7020f..120958c 100644 --- a/Readme.md +++ b/Readme.md @@ -44,41 +44,75 @@ This design mirrors real incident response workflows and scales from local demos ## πŸ—‚οΈ Repository Structure (what goes where) ``` -β”œβ”€ src/ -β”‚ β”œβ”€ gateway/ # FastAPI gateway (policy bands, judge wiring, /stats) -β”‚ β”‚ β”œβ”€ main.py -β”‚ β”‚ └─ judge_wire.py -β”‚ β”œβ”€ model_svc/ # FastAPI model service (serves calibrated URL-only model) -β”‚ β”‚ └─ main.py -β”‚ β”œβ”€ judge_svc/ -β”‚ β”‚ β”œβ”€ contracts.py # JudgeRequest/JudgeResponse schema -β”‚ β”‚ β”œβ”€ stub.py # Deterministic, explainable heuristic -β”‚ β”‚ └─ adapter.py # LLM adapter (Ollama) with safe stub fallback -β”‚ └─ common/ -β”‚ β”œβ”€ thresholds.py # load/decide helpers for policy bands -β”‚ β”œβ”€ stats.py # in-process counters + /stats snapshot -β”‚ └─ audit.py # optional Mongo audit writer (fail-open, dev-only) -β”œβ”€ configs/ -β”‚ └─ dev/ -β”‚ └─ thresholds.json # policy band config (default β‰ˆ14% gray-zone) -β”œβ”€ models/ -β”‚ └─ dev/ -β”‚ β”œβ”€ model.pkl # calibrated URL-only classifier (frozen) -β”‚ └─ model_meta.json # feature order, class mapping, proba column index -β”œβ”€ notebooks/ -β”‚ β”œβ”€ 00_eda.ipynb # dataset-first exploration (EDA) -β”‚ β”œβ”€ 01_baseline_and_calibration.ipynb -β”‚ └─ 03_ablation_url_only.ipynb # source of truth for URL-only pipeline + thresholds -β”œβ”€ scripts/ -β”‚ β”œβ”€ materialize_url_features.py # reproducible feature build (URL morphology, etc.) -β”‚ └─ ge_check.py # lightweight data contract checks (columns/dtypes/ranges) -β”œβ”€ docker/ -β”‚ └─ gateway.Dockerfile # slim multi-stage build (runtime only) -β”œβ”€ .github/workflows/ -β”‚ β”œβ”€ ci.yml # tests + docker build -β”‚ └─ data-contract.yml # runs scripts/ge_check.py on PRs -β”œβ”€ README.md # (this file) -└─ .env.example # environment toggles (judge backend, thresholds, etc.) +β”œβ”€ src/ # Core application source code +β”‚ β”œβ”€ gateway/ # FastAPI gateway service - handles policy bands, judge integration, and API endpoints +β”‚ β”‚ β”œβ”€ main.py # Gateway FastAPI application with /predict, /health, /stats endpoints +β”‚ β”‚ └─ judge_wire.py # Judge service integration and wiring logic +β”‚ β”œβ”€ model_svc/ # FastAPI model service - serves calibrated ML models for phishing prediction +β”‚ β”‚ └─ main.py # Model service FastAPI app with /predict endpoint for p_malicious scoring +β”‚ β”œβ”€ judge_svc/ # Judge service components - provides second opinion for gray-zone cases +β”‚ β”‚ β”œβ”€ contracts.py # Pydantic schemas for JudgeRequest/JudgeResponse data contracts +β”‚ β”‚ β”œβ”€ stub.py # Deterministic rule-based judge implementation (default, fast, explainable) +β”‚ β”‚ └─ adapter.py # LLM judge adapter for Ollama integration with automatic fallback to stub +β”‚ └─ common/ # Shared utilities and cross-service components +β”‚ β”œβ”€ thresholds.py # Threshold loading and decision logic helpers for policy bands +β”‚ β”œβ”€ stats.py # In-process metrics collection and /stats endpoint implementation +β”‚ └─ audit.py # Optional MongoDB audit logging (fail-open, development-only) +β”œβ”€ configs/ # Configuration files for different environments +β”‚ └─ dev/ # Development environment configurations +β”‚ └─ thresholds.json # Policy band thresholds (low/high bounds, gray-zone rate ~14%) +β”œβ”€ models/ # Trained ML model artifacts and metadata +β”‚ └─ dev/ # Development model versions +β”‚ β”œβ”€ model.pkl # Serialized calibrated classifier pipeline (production-ready) +β”‚ └─ model_meta.json # Model metadata (feature order, class mapping, probability column index) +β”œβ”€ notebooks/ # Jupyter notebooks for data exploration, model development, and analysis +β”‚ β”œβ”€ 00_eda.ipynb # Exploratory Data Analysis (EDA) - dataset profiling and feature discovery +β”‚ β”œβ”€ 01_baseline_and_calibration.ipynb # Baseline model training and probability calibration experiments +β”‚ └─ 03_ablation_url_only.ipynb # URL-only model development, ablation studies, and threshold optimization +β”œβ”€ scripts/ # Utility scripts for data processing and validation +β”‚ β”œβ”€ materialize_url_features.py # Feature engineering pipeline for URL morphological analysis +β”‚ └─ ge_check.py # Data contract validation using Great Expectations (columns, types, ranges) +β”œβ”€ docker/ # Docker-related files and configurations +β”‚ └─ gateway.Dockerfile # Multi-stage Docker build for slim production gateway image +β”œβ”€ .github/workflows/ # GitHub Actions CI/CD pipelines +β”‚ β”œβ”€ ci.yml # Main CI pipeline (tests, linting, Docker build) +β”‚ └─ data-contract.yml # Data validation pipeline (runs ge_check.py on pull requests) +β”œβ”€ README.md # Project documentation (this file) +└─ .env.example # Environment variable template (judge backend, thresholds, etc.) + +# Additional Project Directories & Files + +β”œβ”€ data/ # Dataset storage and processing artifacts +β”‚ β”œβ”€ raw/ # Raw, unmodified datasets (PhiUSIIL Phishing URL Dataset) +β”‚ β”œβ”€ processed/ # Cleaned and engineered datasets ready for model training +β”‚ └─ tld_probs.json # Pre-computed legitimate probability scores for top-level domains +β”œβ”€ docs/ # Documentation and analysis artifacts +β”‚ β”œβ”€ EDA_INVESTIGATION.md # Exploratory data analysis findings and insights +β”‚ β”œβ”€ model_docs.md # Model architecture and performance documentation +β”‚ └─ MODELING.md # Modeling methodology, experiments, and results +β”œβ”€ outputs/ # Analysis outputs and generated artifacts +β”‚ β”œβ”€ eda/ # EDA visualizations and statistical summaries +β”‚ β”œβ”€ model/ # Model training outputs and evaluation metrics +β”‚ └─ feature_comparison_v1_vs_v2.csv # Feature engineering comparison results +β”œβ”€ tests/ # Test suite for quality assurance +β”‚ β”œβ”€ test_gateway_*.py # Gateway service integration and unit tests +β”‚ β”œβ”€ test_judge_*.py # Judge service functionality tests +β”‚ β”œβ”€ test_model_svc.py # Model service API and prediction tests +β”‚ └─ test_threshold_*.py # Threshold logic and policy band tests +β”œβ”€ gx/ # Great Expectations data validation suite +β”‚ β”œβ”€ great_expectations.yml # GX configuration and data source definitions +β”‚ β”œβ”€ expectations/ # Data quality expectation suites +β”‚ β”œβ”€ checkpoints/ # Validation checkpoints and test definitions +β”‚ └─ validations/ # Validation run results and reports +β”œβ”€ mlartifacts/ # MLflow experiment tracking artifacts +β”‚ └─ [experiment_id]/ # Individual experiment runs and metadata +β”œβ”€ mlruns/ # MLflow run tracking database and logs +β”œβ”€ requirements*.txt # Python dependency specifications for different environments +β”œβ”€ pyproject.toml # Python project configuration (dependencies, tools, metadata) +β”œβ”€ pytest.ini # Pytest testing framework configuration +β”œβ”€ .pre-commit-config.yaml # Pre-commit hooks configuration (linting, formatting) +β”œβ”€ .flake8, .bandit # Code quality and security linting configurations +└─ docker-compose.yml # Multi-service Docker composition for local development ``` --- @@ -86,6 +120,7 @@ This design mirrors real incident response workflows and scales from local demos ## Quick Start ### **Local (stub judge, no Docker)** + ```bash pip install -U pip && pip install -e ".[dev]" uvicorn model_svc.main:app --reload --port 9000 # terminal A (serves model) @@ -96,6 +131,7 @@ uvicorn gateway.main:app --reload ``` **Test:** + ```bash curl -X POST localhost:8000/predict -H "Content-Type: application/json" \ -d '{"url":"http://ex.com/login?acct=12345","p_malicious":0.45}' @@ -104,11 +140,13 @@ curl -X POST localhost:8000/predict -H "Content-Type: application/json" \ ### **Docker (mount your thresholds; stub or LLM)** **Build:** + ```bash docker build -f docker/gateway.Dockerfile -t phishguard-gateway:local . ``` **Run (stub judge; thresholds mounted):** + ```bash docker run --rm -p 8000:8000 \ -e THRESHOLDS_JSON=/app/configs/dev/thresholds.json \ @@ -117,6 +155,7 @@ docker run --rm -p 8000:8000 \ ``` **Run (LLM judge via Ollama on host):** + ```bash docker run --rm -p 8000:8000 \ -e THRESHOLDS_JSON=/app/configs/dev/thresholds.json \ @@ -129,7 +168,7 @@ docker run --rm -p 8000:8000 \ ### **Endpoints** - `/health` – service liveness -- `/config` – active thresholds & source +- `/config` – active thresholds & source - `/predict` – decision API (POST JSON: `{"url": "...", "p_malicious": 0.45}` or omit `p_malicious` to let the gateway call the model service) - `/stats`, `/stats/reset` – simple counters for demos @@ -214,8 +253,10 @@ MIT License. See [LICENSE](LICENSE) file for details. This project runs fully **locally** with a URL-only model and a judge that’s either a **deterministic stub** (default) or an **LLM via Ollama** (optional). Follow these steps in order. ### 0) Prereqs + - Python 3.11 in a virtual env (conda or venv) - Editable install: + ```bash pip install -U pip pip install -e ".[dev]" @@ -226,11 +267,13 @@ This project runs fully **locally** with a URL-only model and a judge that’s e This image runs the **gateway** with either the deterministic **stub** judge (default) or an **LLM** judge via **Ollama**. It's a slim multi-stage image; no dev deps included. ### Build (local image) + ```bash docker build -f docker/gateway.Dockerfile -t phishguard-gateway:local . ``` ### Run with stub judge (no Ollama needed) + ```bash docker run --rm -p 8000:8000 \ -e THRESHOLDS_JSON=/app/configs/dev/thresholds.json \ @@ -242,6 +285,7 @@ docker run --rm -p 8000:8000 \ On the host, start Ollama and pull a small model (e.g., llama3.2:1b). Start the container and point it at the host: + ```bash docker run --rm -p 8000:8000 \ -e THRESHOLDS_JSON=/app/configs/dev/thresholds.json \ @@ -253,6 +297,7 @@ docker run --rm -p 8000:8000 \ ``` ### Smoke checks + ```bash curl http://127.0.0.1:8000/health curl http://127.0.0.1:8000/config @@ -263,17 +308,22 @@ curl http://127.0.0.1:8000/stats ``` --- + ### Latest aditions will organize + --- + ## Model Performance **Validation Metrics (PhiUSIIL Dataset):** + - PR-AUC (phishing detection): **99.92%** - F1-Macro: **99.70%** - Brier Score: **0.0026** - False Positive Rate: **0.09%** (23/26,970 legitimate URLs) **Feature Set (8 features):** + - IsHTTPS, TLDLegitimateProb, CharContinuationRate - SpacialCharRatioInURL, URLCharProb, LetterRatioInURL - NoOfOtherSpecialCharsInURL, DomainLength @@ -286,4 +336,95 @@ curl http://127.0.0.1:8000/stats **Known Limitations:** - Model trained on PhiUSIIL dataset (2019-2020 URLs) - Major tech companies (google.com, github.com) are out-of-distribution -- Whitelist override implemented for known legitimate short domains \ No newline at end of file +- Whitelist override implemented for known legitimate short domains + +## 🎯 Model Performance + +**Validation Metrics (PhiUSIIL Dataset, 47,074 samples):** +- **PR-AUC:** 99.92% +- **F1-Macro:** 99.70% +- **Brier Score:** 0.0026 +- **False Positive Rate:** 0.09% (23/26,970 legitimate URLs) + +**Feature Set (8 URL-only features):** +1. IsHTTPS - Protocol security +2. TLDLegitimateProb - TLD legitimacy (695 TLDs, Bayesian priors) +3. CharContinuationRate - Character repetition ratio +4. SpacialCharRatioInURL - Special character density +5. URLCharProb - Character probability score +6. LetterRatioInURL - Alphabetic ratio +7. NoOfOtherSpecialCharsInURL - Special character count +8. DomainLength - Domain length + +**Decision Framework:** +- **Whitelist:** 15 major domains (google.com, github.com, etc.) β†’ Fast-path ALLOW +- **Policy Bands:** 89% automated (p<0.004 β†’ ALLOW, p>0.999 β†’ BLOCK) +- **Short Domain Routing:** len≀10, p<0.5 β†’ Judge review (handles npm.org, bit.ly edge cases) +- **Gray Zone:** 11% escalated to judge for explainable decisions + +**Performance (Single Instance):** +- Whitelist path: <10ms (p95) +- Model path: 20-30ms (p95) +- Judge path: 50-100ms (p95) +- Throughput: ~150 req/sec + +See [JUDGE_LOGIC.md](docs/JUDGE_LOGIC.md) for complete decision flow. + + +Add to README.md: + +## Performance Characteristics + +**Latency (p95):** +- Whitelist path: <10ms +- Policy band (no judge): ~20-30ms +- Gray zone (with judge): ~50-100ms + +**Throughput:** +- Single instance: ~150 req/sec +- Scalability: Horizontal scaling via Kubernetes + +**Tested on:** Local development machine +═══════════════════════════════════════════════════════════════ PHASE 4: FINAL POLISH (15 minutes) ═══════════════════════════════════════════════════════════════ + +4.1: Update README.md (10 min) +Add these sections: + +## Model Performance + +**Validation Metrics (PhiUSIIL Dataset, 47,074 samples):** +- **PR-AUC:** 99.92% +- **F1-Macro:** 99.70% +- **Brier Score:** 0.0026 +- **False Positive Rate:** 0.09% (23/26,970 legitimate URLs) + +**Feature Set (8 URL-only features):** +1. IsHTTPS - Protocol security +2. TLDLegitimateProb - TLD legitimacy (Bayesian priors) +3. CharContinuationRate - Character repetition +4. SpacialCharRatioInURL - Special character density +5. URLCharProb - Character probability +6. LetterRatioInURL - Alphabetic ratio +7. NoOfOtherSpecialCharsInURL - Special char count +8. DomainLength - Domain length + +**Enhanced Routing:** +- Whitelist: 14 major tech domains (OOD handling) +- Policy Bands: 89% automated (ALLOW/BLOCK) +- Short Domain Routing: len≀10, p<0.5 β†’ Judge +- Gray Zone: 11% escalated for review + +### Enhanced Routing Logic + +PhishGuardAI uses intelligent routing for edge cases: + +- **Whitelist:** Known legitimate domains (Google, GitHub, etc.) β†’ Fast-path ALLOW +- **Policy Bands:** High confidence cases (p<0.004 or p>0.999) β†’ Automated ALLOW/BLOCK +- **Short Domain Routing:** Short domains (≀10 chars) with moderate confidence (p<0.5) β†’ Judge review +- **Standard Gray Zone:** Normal domains in gray zone β†’ Judge review + +This handles distribution shift for short legitimate domains (npm.org, bit.ly) that aren't in the training data. + +See [JUDGE_LOGIC.md](docs/JUDGE_LOGIC.md) for full decision flow. + +See [JUDGE_LOGIC.md](docs/JUDGE_LOGIC.md) for decision flow details. \ No newline at end of file diff --git a/configs/dev/thresholds_7feat.json b/configs/dev/archive/thresholds_7feat.json similarity index 100% rename from configs/dev/thresholds_7feat.json rename to configs/dev/archive/thresholds_7feat.json diff --git a/configs/dev/config.yaml b/configs/dev/config.yaml index 8dfe6f8..d9d407d 100644 --- a/configs/dev/config.yaml +++ b/configs/dev/config.yaml @@ -1,42 +1,54 @@ +# ============================================================ # PhishGuard Configuration - Development Environment -# This file specifies which models to load and how to configure services +# ============================================================ +# Model Service Configuration model_service: - # Primary model - used for actual predictions/decisions + # Primary model (production) primary: - path: "models/dev/model_7feat.pkl" - meta_path: "models/dev/model_7feat_meta.json" - name: "7-feature-production" - description: "URL-only model without IsHTTPS, robust to 2025 HTTPS phishing" - - # Shadow model - runs in parallel for A/B testing (optional) - shadow: - enabled: true # Set to false to disable shadow testing path: "models/dev/model_8feat.pkl" meta_path: "models/dev/model_8feat_meta.json" - name: "8-feature-research" - description: "URL-only model with IsHTTPS for comparison" + name: "8-feature-production-v1" + description: "8-feature URL model with IsHTTPS (PR-AUC: 99.92%)" + + # Shadow model (DISABLED for production) + shadow: + enabled: false + path: "models/dev/model_7feat.pkl" + meta_path: "models/dev/model_7feat_meta.json" + name: "7-feature-baseline" + description: "7-feature URL model without IsHTTPS (research only)" + log_path: "outputs/shadow_predictions.jsonl" +# Gateway Configuration gateway: - # Thresholds must match the primary model - thresholds_path: "configs/dev/thresholds_7feat.json" + # Thresholds file (tuned on 8-feature model) + thresholds_path: "configs/dev/thresholds.json" - # Policy band configuration - # These values can be overridden by the thresholds file + # Default thresholds (fallback if file load fails) default_thresholds: - low: 0.20 # Auto-ALLOW below this - high: 1.00 # Auto-BLOCK at/above this - t_star: 0.60 # F1-optimal threshold + low: 0.004 # Auto-ALLOW below this (99.6% safe URLs) + high: 0.999 # Auto-BLOCK at/above this (99.5% phishing) + t_star: 0.350 # F1-optimal threshold + gray_zone_rate: 0.109 # ~11% escalated to judge -# Feature extraction configuration +# Feature Extraction Configuration features: - # TLD probability lookup table + # TLD legitimacy lookup table (695 TLDs) tld_probs_path: "data/tld_probs.json" - # Fail-secure behavior when feature extraction fails - fail_secure: true # If true, return suspicious features on error + # Bayesian priors for TLD scoring + tld_alpha: 1 + tld_beta: 2 + + # Minimum samples for TLD reliability + min_samples: 10 + + # Fail-secure: return suspicious features on extraction errors + fail_secure: true -# Logging configuration +# Logging Configuration logging: level: "INFO" - shadow_logging: true # Log shadow model predictions for analysis \ No newline at end of file + shadow_logging: false + audit_enabled: false \ No newline at end of file diff --git a/configs/dev/thresholds.json b/configs/dev/thresholds.json index a2c6dbc..77a26cd 100644 --- a/configs/dev/thresholds.json +++ b/configs/dev/thresholds.json @@ -1,12 +1,45 @@ { - "optimal_threshold": 0.35, - "gray_zone_low": 0.004, - "gray_zone_high": 0.9990000000000006, - "gray_zone_rate": 0.10936468383276894, - "f1_score_at_optimal": 0.002766509680600472, + "model": "xgb_8feat", + "version": "v1.0", + "optimization": { + "method": "f1_macro", + "validation_metric": 0.997, + "description": "Optimized for F1-macro on validation set with enhanced routing" + }, + "thresholds": { + "t_star": 0.35, + "low": 0.004, + "high": 0.999, + "gray_zone_rate": 0.109 + }, + "routing": { + "short_domain_length": 10, + "short_domain_confidence": 0.5, + "description": "Route short domains (≀10 chars) with moderate confidence (p<0.5) to judge for edge case handling" + }, + + "performance_metrics": { + "pr_auc": 0.9992, + "f1_macro": 0.9970, + "brier_score": 0.002637, + "false_positive_rate": 0.0009 + }, + + "class_mapping": { + "phish": 0, + "legit": 1 + }, + "decision_distribution": { - "allow_rate": 0.48099162992780015, - "review_rate": 0.10936468383276894, - "block_rate": 0.4096436862394309 - } + "allow_rate": 0.886, + "review_rate": 0.109, + "block_rate": 0.005 + }, + + "notes": [ + "Thresholds tuned for 8-feature model with IsHTTPS", + "Gray zone rate ~11% for judge escalation", + "Low threshold (0.004) catches 99.6% of legitimate URLs", + "High threshold (0.999) blocks 99.5% of phishing URLs" + ] } \ No newline at end of file diff --git a/configs/dev/whitelist.json b/configs/dev/whitelist.json new file mode 100644 index 0000000..f5d7437 --- /dev/null +++ b/configs/dev/whitelist.json @@ -0,0 +1,38 @@ +{ + "well_known_domains": [ + "google.com", + "www.google.com", + "github.com", + "example.com", + "www.example.com", + "openai.com", + "www.openai.com", + "www.github.com", + "microsoft.com", + "www.microsoft.com", + "amazon.com", + "www.amazon.com", + "apple.com", + "www.apple.com", + "facebook.com", + "www.facebook.com", + "twitter.com", + "www.twitter.com", + "linkedin.com", + "www.linkedin.com", + "youtube.com", + "www.youtube.com", + "wikipedia.org", + "www.wikipedia.org", + "stackoverflow.com", + "www.stackoverflow.com", + "netflix.com", + "www.netflix.com", + "paypal.com", + "www.paypal.com", + "ebay.com", + "www.ebay.com" + ], + "short_domain_threshold": 10, + "short_domain_confidence_threshold": 0.5 + } \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8d49c61 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,40 @@ +services: + model-svc: + build: + context: . + dockerfile: docker/model.Dockerfile + ports: ["8002:8002"] + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://localhost:8002/health')", + ] + interval: 10s + timeout: 3s + retries: 10 + + gateway: + build: + context: . + dockerfile: docker/gateway.Dockerfile + ports: ["8080:8000"] + depends_on: + model-svc: { condition: service_healthy } + environment: + - MODEL_SVC_URL=http://model-svc:8002 + - THRESHOLDS_JSON=configs/dev/thresholds.json + - JUDGE_BACKEND=stub + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')", + ] + interval: 10s + timeout: 3s + retries: 10 diff --git a/docker/compose.yml b/docker/compose.yml index 5a23384..e10ca31 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -1,15 +1,8 @@ services: - feature-svc: - build: { context: .., dockerfile: docker/feature.Dockerfile } - ports: ["8001:8001"] - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8001/health"] - interval: 10s - timeout: 3s - retries: 10 - model-svc: - build: { context: .., dockerfile: docker/model.Dockerfile } + build: + context: . + dockerfile: docker/model.Dockerfile ports: ["8002:8002"] healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8002/health"] @@ -18,11 +11,16 @@ services: retries: 10 gateway: - build: { context: .., dockerfile: docker/gateway.Dockerfile } + build: + context: . + dockerfile: docker/gateway.Dockerfile ports: ["8080:8000"] depends_on: - feature-svc: { condition: service_healthy } model-svc: { condition: service_healthy } + environment: + - MODEL_SVC_URL=http://model-svc:8002 + - THRESHOLDS_JSON=configs/dev/thresholds.json + - JUDGE_BACKEND=stub healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8080/health"] interval: 10s diff --git a/docker/feature.Dockerfile b/docker/feature.Dockerfile deleted file mode 100644 index d0b8291..0000000 --- a/docker/feature.Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM python:3.11-slim -WORKDIR /app -ENV PYTHONUNBUFFERED=1 PYTHONPATH=/app -COPY requirements-docker.txt . -RUN pip install --no-cache-dir -r requirements-docker.txt -COPY src ./src -EXPOSE 8001 -CMD ["uvicorn", "src.feature_svc.main:app", "--host", "0.0.0.0", "--port", "8001"] diff --git a/docker/gateway.Dockerfile b/docker/gateway.Dockerfile index 3b93adc..f25685e 100644 --- a/docker/gateway.Dockerfile +++ b/docker/gateway.Dockerfile @@ -30,6 +30,7 @@ WORKDIR /app COPY --from=builder /opt/venv /opt/venv COPY src ./src COPY configs ./configs +COPY data/tld_probs.json ./data/tld_probs.json EXPOSE 8000 CMD ["uvicorn", "gateway.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/docker/model.Dockerfile b/docker/model.Dockerfile index 375c53c..0d046e5 100644 --- a/docker/model.Dockerfile +++ b/docker/model.Dockerfile @@ -2,7 +2,11 @@ FROM python:3.11-slim WORKDIR /app ENV PYTHONUNBUFFERED=1 PYTHONPATH=/app COPY requirements-docker.txt . -RUN pip install --no-cache-dir -r requirements-docker.txt +COPY pyproject.toml Readme.md ./ COPY src ./src +RUN pip install --no-cache-dir -r requirements-docker.txt +COPY data/tld_probs.json ./data/tld_probs.json +COPY configs ./configs +COPY models ./models EXPOSE 8002 CMD ["uvicorn", "src.model_svc.main:app", "--host", "0.0.0.0", "--port", "8002"] diff --git a/docs/FEATURE_EXTRACTION.md b/docs/FEATURE_EXTRACTION.md index 4a4285c..0ecd0f3 100644 --- a/docs/FEATURE_EXTRACTION.md +++ b/docs/FEATURE_EXTRACTION.md @@ -5,40 +5,78 @@ All features extracted using `src/common/feature_extraction.py` for training/ser ## Feature Definitions -### 1. IsHTTPS +### IsHTTPS - **Type:** Binary (0/1) -- **Definition:** URL uses HTTPS protocol -- **Range:** [0, 1] +- **Definition:** Whether URL uses HTTPS protocol +- **Legitimate URLs:** 95% use HTTPS +- **Phishing URLs:** 60% use HTTPS (mixed) -### 2. TLDLegitimateProb -- **Type:** Float -- **Definition:** Bayesian legitimacy probability for TLD -- **Range:** [0, 1] -- **Source:** `common/tld_probs.json` (695 TLDs) -- **Priors:** Ξ±=1, Ξ²=2 (conservative) +### TLDLegitimateProb +- **Type:** Float [0, 1] +- **Definition:** Bayesian legitimacy probability for top-level domain +- **Source:** `common/tld_probs.json` (695 TLDs with frequency counts) +- **Priors:** Ξ±=1, Ξ²=2 (conservative, assumes unknown TLDs are risky) +- **Examples:** + - .com: 0.611 + - .org: 0.709 + - .tk (Tokelau): 0.019 (high phishing) ### 3. CharContinuationRate -- **Type:** Float +- **Type:** Float [0, 1] - **Definition:** Ratio of consecutive identical characters -- **Range:** [0, 1] -- **Example:** "google.com" β†’ 0.176 +- **Formula:** (count of repeated chars) / (total chars - 1) +- **Examples:** + - "abc" β†’ 0.0 (no repetition) + - "aaa" β†’ 1.0 (all repeated) + - "google.com" β†’ 0.176 (some repetition) -[... continue for all 8 features ...] +### 4. SpacialCharRatioInURL +- **Type:** Float [0, 1] +- **Definition:** Density of special characters in URL +- **Special chars:** ! @ # $ % ^ & * ( ) _ + - = [ ] { } | ; : , . < > ? / +- **Formula:** (count of special chars) / (total chars) +- **Examples:** + - "http://example.com" β†’ 0.16 + - "http://ex.com/login?id=123&token=abc" β†’ 0.23 + +### 5. URLCharProb +- **Type:** Float [0, 1] +- **Definition:** Proportion of common URL characters (alphanumeric + :/.?=&-_) +- **Formula:** (count of common chars) / (total chars) +- **Purpose:** Measures how "URL-like" the character distribution is +- **Examples:** + - "http://example.com" β†’ 0.95 (all common chars) + - "http://ex.com/@@##$$" β†’ 0.70 (unusual chars) + +### 6. LetterRatioInURL +- **Type:** Float [0, 1] +- **Definition:** Density of letter characters (A-Za-z) in URL +- **Formula:** (count of letters) / (total chars) +- **Examples:** + - "http://example.com" β†’ 0.63 + - "http://ex.com/123" β†’ 0.47 + +### 7. NoOfOtherSpecialCharsInURL +- **Type:** Integer [0, ∞) +- **Definition:** Total count of special characters in URL +- **Same character set as SpacialCharRatioInURL but returns count** +- **Examples:** + - "http://example.com" β†’ 3 + - "http://ex.com/login?id=123&token=abc" β†’ 8 + +### 8. DomainLength +- **Type:** Integer [0, ∞) +- **Definition:** Length of the domain component (netloc) +- **Examples:** + - "http://example.com" β†’ 11 + - "https://www.very-long-suspicious-domain.com" β†’ 32 ## Training/Serving Consistency -- βœ… Same extraction logic for training and production -- βœ… No data leakage (trained on raw PhiUSIIL URLs) -- βœ… Validated: Batch vs live extraction matches +- βœ… Same extraction code for training and production +- βœ… No data leakage (trained on raw PhiUSIIL URLs only) +- βœ… Validated: Batch extraction matches live extraction +- βœ… Deterministic (same URL always gives same features) ### **Step 4: Clean Up Notebooks (30 min)** -``` -notebooks/ - β”œβ”€β”€ 00_eda.ipynb - β”œβ”€β”€ feature_engineering.ipynb - β”œβ”€β”€ 03_ablation_url_only.ipynb - β”œβ”€β”€ 03_ablation_url_only_copy.ipynb - └── archive/ - └── old_experiments/ -``` \ No newline at end of file diff --git a/gx/expectations/.ge_store_backend_id b/gx/expectations/.ge_store_backend_id index 2685e94..965d980 100644 --- a/gx/expectations/.ge_store_backend_id +++ b/gx/expectations/.ge_store_backend_id @@ -1 +1 @@ -store_backend_id = bc10f424-b14f-4fc5-9966-37b37fb49ef4 +store_backend_id = 21207da8-5d01-43dc-b9d7-55f06d678fac diff --git a/gx/expectations/phiusiil_8feature_production.json b/gx/expectations/phiusiil_8feature_production.json new file mode 100644 index 0000000..5e9f7a3 --- /dev/null +++ b/gx/expectations/phiusiil_8feature_production.json @@ -0,0 +1,263 @@ +{ + "data_asset_type": null, + "expectation_suite_name": "phiusiil_8feature_production", + "expectations": [ + { + "expectation_type": "expect_column_values_to_not_be_null", + "kwargs": { + "column": "label" + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_be_in_set", + "kwargs": { + "column": "label", + "value_set": [ + 0, + 1 + ] + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_not_be_null", + "kwargs": { + "column": "URL" + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_be_unique", + "kwargs": { + "column": "URL" + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_not_be_null", + "kwargs": { + "column": "IsHTTPS" + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_be_in_set", + "kwargs": { + "column": "IsHTTPS", + "value_set": [ + 0, + 1 + ] + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_be_of_type", + "kwargs": { + "column": "IsHTTPS", + "type_": "float64" + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_not_be_null", + "kwargs": { + "column": "TLDLegitimateProb" + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "TLDLegitimateProb", + "max_value": 1.0, + "min_value": 0.0 + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_be_of_type", + "kwargs": { + "column": "TLDLegitimateProb", + "type_": "float64" + }, + "meta": {} + }, + { + "expectation_type": "expect_column_unique_value_count_to_be_between", + "kwargs": { + "column": "TLDLegitimateProb", + "max_value": 1000, + "min_value": 10 + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_not_be_null", + "kwargs": { + "column": "CharContinuationRate" + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "CharContinuationRate", + "max_value": 1.0, + "min_value": 0.0 + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_be_of_type", + "kwargs": { + "column": "CharContinuationRate", + "type_": "float64" + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_not_be_null", + "kwargs": { + "column": "SpacialCharRatioInURL" + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "SpacialCharRatioInURL", + "max_value": 1.0, + "min_value": 0.0 + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_be_of_type", + "kwargs": { + "column": "SpacialCharRatioInURL", + "type_": "float64" + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_not_be_null", + "kwargs": { + "column": "URLCharProb" + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "URLCharProb", + "max_value": 1.0, + "min_value": 0.0 + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_be_of_type", + "kwargs": { + "column": "URLCharProb", + "type_": "float64" + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_not_be_null", + "kwargs": { + "column": "LetterRatioInURL" + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "LetterRatioInURL", + "max_value": 1.0, + "min_value": 0.0 + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_be_of_type", + "kwargs": { + "column": "LetterRatioInURL", + "type_": "float64" + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_not_be_null", + "kwargs": { + "column": "NoOfOtherSpecialCharsInURL" + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "NoOfOtherSpecialCharsInURL", + "max_value": 1000, + "min_value": 0 + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_be_of_type", + "kwargs": { + "column": "NoOfOtherSpecialCharsInURL", + "type_": "int64" + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_not_be_null", + "kwargs": { + "column": "DomainLength" + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "DomainLength", + "max_value": 253, + "min_value": 1 + }, + "meta": {} + }, + { + "expectation_type": "expect_column_values_to_be_of_type", + "kwargs": { + "column": "DomainLength", + "type_": "int64" + }, + "meta": {} + }, + { + "expectation_type": "expect_column_mean_to_be_between", + "kwargs": { + "column": "IsHTTPS", + "max_value": 0.98, + "min_value": 0.3 + }, + "meta": {} + }, + { + "expectation_type": "expect_column_mean_to_be_between", + "kwargs": { + "column": "TLDLegitimateProb", + "max_value": 0.9, + "min_value": 0.2 + }, + "meta": {} + } + ], + "ge_cloud_id": null, + "meta": { + "great_expectations_version": "0.18.22" + } +} \ No newline at end of file diff --git a/gx/expectations/phiusiil_minimal.json b/gx/expectations/phiusiil_minimal.json deleted file mode 100644 index a1af39e..0000000 --- a/gx/expectations/phiusiil_minimal.json +++ /dev/null @@ -1,331 +0,0 @@ -{ - "expectations": [ - { - "id": "b25ecf96-25f5-485e-9713-6b9f5c214f86", - "kwargs": { - "column": "label" - }, - "meta": {}, - "type": "expect_column_values_to_not_be_null" - }, - { - "id": "554e03e0-e5c1-45fd-9ad7-44ff908d1eec", - "kwargs": { - "column": "label", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - }, - { - "id": "f5526aee-2ed7-4a0e-a780-00b96147812f", - "kwargs": { - "column": "URL" - }, - "meta": {}, - "type": "expect_column_values_to_not_be_null" - }, - { - "id": "7a368679-8c8b-4e3b-b0b5-ea48e07f7c86", - "kwargs": { - "column": "URL" - }, - "meta": {}, - "type": "expect_column_values_to_be_unique" - }, - { - "id": "f992073f-456c-4617-97a5-2d5d599c53f4", - "kwargs": { - "column": "CharContinuationRate", - "max_value": 1.0, - "min_value": 0.0 - }, - "meta": {}, - "type": "expect_column_values_to_be_between" - }, - { - "id": "a23e659d-c543-4ea4-b71b-3760444ac581", - "kwargs": { - "column": "URLCharProb", - "max_value": 1.0, - "min_value": 0.0 - }, - "meta": {}, - "type": "expect_column_values_to_be_between" - }, - { - "id": "98460811-3b05-4668-b7ef-ee9a8058d998", - "kwargs": { - "column": "TLDLegitimateProb", - "max_value": 1.0, - "min_value": 0.0 - }, - "meta": {}, - "type": "expect_column_values_to_be_between" - }, - { - "id": "6fd92605-c61b-4a68-b40e-ebe86e997d99", - "kwargs": { - "column": "Domain", - "type_": "object" - }, - "meta": {}, - "type": "expect_column_values_to_be_of_type" - }, - { - "id": "dcaca9a3-438d-4ef3-b5b2-ab6cf8a8aa0f", - "kwargs": { - "column": "TLD", - "type_": "object" - }, - "meta": {}, - "type": "expect_column_values_to_be_of_type" - }, - { - "id": "acb0ab75-64b3-4eb6-b59c-486fb043213f", - "kwargs": { - "column": "Title", - "type_": "object" - }, - "meta": {}, - "type": "expect_column_values_to_be_of_type" - }, - { - "id": "0ab1b810-5aa7-4bc0-887f-c2217c52486b", - "kwargs": { - "column": "IsDomainIP", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - }, - { - "id": "201dbd07-ec98-462c-9f3d-dc8517fa9f69", - "kwargs": { - "column": "HasObfuscation", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - }, - { - "id": "0c1f9c9e-cb79-432b-8549-b29fd0deff9e", - "kwargs": { - "column": "IsHTTPS", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - }, - { - "id": "eb2fb96f-51e4-4d4a-bc63-ed1c5052edb0", - "kwargs": { - "column": "HasTitle", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - }, - { - "id": "7b9d3fb2-d65f-4ccd-bd85-d80811a263c0", - "kwargs": { - "column": "HasFavicon", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - }, - { - "id": "8505aa31-6b83-4f24-af8b-632479f91fbc", - "kwargs": { - "column": "Robots", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - }, - { - "id": "12229542-3f65-43e3-8ee6-6d264dbfd620", - "kwargs": { - "column": "IsResponsive", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - }, - { - "id": "fff2dc2a-732f-459d-9dc4-8599d83779de", - "kwargs": { - "column": "NoOfURLRedirect", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - }, - { - "id": "788360c3-e093-466d-a126-a44da2eccf47", - "kwargs": { - "column": "NoOfSelfRedirect", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - }, - { - "id": "c386ea87-92f4-48af-8cda-37c41c58e72c", - "kwargs": { - "column": "HasDescription", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - }, - { - "id": "07854ae4-f943-4265-b090-8b2cdb2ec941", - "kwargs": { - "column": "HasExternalFormSubmit", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - }, - { - "id": "659367d5-248c-4d5c-b3a2-21782eecb223", - "kwargs": { - "column": "HasSocialNet", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - }, - { - "id": "5af0dc82-3ffe-4ff5-8a03-a6211fd0b4db", - "kwargs": { - "column": "HasSubmitButton", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - }, - { - "id": "21458498-0be9-47fd-9d4d-241fa2c2ce56", - "kwargs": { - "column": "HasHiddenFields", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - }, - { - "id": "14c6c1fe-125b-417c-a19f-cb571e2acc6e", - "kwargs": { - "column": "HasPasswordField", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - }, - { - "id": "bf2e79b9-7146-4c8b-a888-4a3cef68f28f", - "kwargs": { - "column": "Bank", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - }, - { - "id": "32d6edd0-f1a6-4d05-b540-3b5109788dd4", - "kwargs": { - "column": "Pay", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - }, - { - "id": "8af1c25f-574f-4671-9558-0999b0d9af41", - "kwargs": { - "column": "Crypto", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - }, - { - "id": "1cc80804-85de-4225-9828-9d8af49cc66e", - "kwargs": { - "column": "HasCopyrightInfo", - "value_set": [ - 0, - 1 - ] - }, - "meta": {}, - "type": "expect_column_values_to_be_in_set" - } - ], - "id": "ce807dda-2a20-4de3-b7f2-d5a8d31e35bb", - "meta": { - "great_expectations_version": "1.4.0" - }, - "name": "phiusiil_minimal", - "notes": null -} \ No newline at end of file diff --git a/gx/great_expectations.yml b/gx/great_expectations.yml index 0c408fa..d0f3672 100644 --- a/gx/great_expectations.yml +++ b/gx/great_expectations.yml @@ -8,7 +8,23 @@ # config_version refers to the syntactic version of this config file, and is used in maintaining backwards compatibility # It is auto-generated and usually does not need to be changed. -config_version: 4.0 +config_version: 3.0 + +# Datasources tell Great Expectations where your data lives and how to get it. +# Read more at https://docs.greatexpectations.io/docs/guides/connecting_to_your_data/connect_to_data_overview +datasources: + phishguard_features: + class_name: Datasource + module_name: great_expectations.datasource + execution_engine: + class_name: PandasExecutionEngine + module_name: great_expectations.execution_engine + data_connectors: + default_runtime_data_connector: + class_name: RuntimeDataConnector + module_name: great_expectations.datasource.data_connector + batch_identifiers: + - default_identifier_name # This config file supports variable substitution which enables: 1) keeping # secrets out of source control & 2) environment-based configuration changes @@ -38,18 +54,24 @@ stores: # Stores are configurable places to store things like Expectations, Validations # Data Docs, and more. These are for advanced users only - most users can simply # leave this section alone. +# +# Three stores are required: expectations, validations, and +# evaluation_parameters, and must exist with a valid store entry. Additional +# stores can be configured for uses such as data_docs, etc. expectations_store: class_name: ExpectationsStore store_backend: class_name: TupleFilesystemStoreBackend base_directory: expectations/ - validation_results_store: - class_name: ValidationResultsStore + validations_store: + class_name: ValidationsStore store_backend: class_name: TupleFilesystemStoreBackend base_directory: uncommitted/validations/ + evaluation_parameter_store: + class_name: EvaluationParameterStore checkpoint_store: class_name: CheckpointStore store_backend: @@ -57,14 +79,16 @@ stores: suppress_store_backend_id: true base_directory: checkpoints/ - validation_definition_store: - class_name: ValidationDefinitionStore + profiler_store: + class_name: ProfilerStore store_backend: class_name: TupleFilesystemStoreBackend - base_directory: validation_definitions/ + suppress_store_backend_id: true + base_directory: profilers/ expectations_store_name: expectations_store -validation_results_store_name: validation_results_store +validations_store_name: validations_store +evaluation_parameter_store_name: evaluation_parameter_store checkpoint_store_name: checkpoint_store data_docs_sites: @@ -80,18 +104,12 @@ data_docs_sites: base_directory: uncommitted/data_docs/local_site/ site_index_builder: class_name: DefaultSiteIndexBuilder -fluent_datasources: - phiusiil_src: - type: pandas - id: e089c4c4-945f-465c-b0cc-5ef79bcdc6af - assets: - phiusiil_df: - type: dataframe - id: e6ea6284-df37-40a3-82e6-201badb3e382 - batch_metadata: {} - batch_definitions: - phiusiil_batch: - id: 8d2dc0cf-fa7c-4081-b524-a17aeafb068e - partitioner: -analytics_enabled: -data_context_id: bc10f424-b14f-4fc5-9966-37b37fb49ef4 + +anonymous_usage_statistics: + data_context_id: 21207da8-5d01-43dc-b9d7-55f06d678fac + enabled: true +notebooks: +include_rendered_content: + globally: false + expectation_suite: false + expectation_validation_result: false diff --git a/gx/uncommitted/config_variables.yml b/gx/uncommitted/config_variables.yml index 9902691..61408bc 100644 --- a/gx/uncommitted/config_variables.yml +++ b/gx/uncommitted/config_variables.yml @@ -16,4 +16,4 @@ # # https://docs.greatexpectations.io/docs/guides/setup/configuring_data_contexts/how_to_configure_credentials -instance_id: 690e19ab-2446-40b6-8e01-027c073ee386 +instance_id: d726507c-8280-48d7-884c-1671c37977a4 diff --git a/gx/uncommitted/validations/.ge_store_backend_id b/gx/uncommitted/validations/.ge_store_backend_id index af6cfc4..df85df8 100644 --- a/gx/uncommitted/validations/.ge_store_backend_id +++ b/gx/uncommitted/validations/.ge_store_backend_id @@ -1 +1 @@ -store_backend_id = 2eda7416-6aa7-4e8f-a7f6-f73d0ea6100a +store_backend_id = a83db70f-087e-4a1e-ac90-67c26c3e3e5a diff --git a/init_ge.py b/init_ge.py deleted file mode 100644 index e69de29..0000000 diff --git a/model_logs.txt b/model_logs.txt new file mode 100644 index 0000000000000000000000000000000000000000..8678da602c69b6a80c855817c00c425a6f78777c GIT binary patch literal 31068 zcmeI5+iu&)8OI0My8?X&A-#x$O=LT=dQx#b!@NWT^~}(Ie3fq zCH4{efW7HWZ`%HU^JzlLrew+-4HY;bWQxP#%y&M`Q2yV4U%DUNzT0tU?w#xC+bexO z>Z+r*U7ZizsXKAsyM}wLZ#!;J_xE%)>*m~y{x;mU?yTtiR?nQe*Xr|HPaWyQ)BCRL ze(QdtXHGTV&${lY?*o1A>-qbx?e1xm8UMH4Yki=WMfI6=Eq7P*EU8yRJsKLTqfwgf zSfc^Sk>KsR_r10q&3Dr4x$d62HMh~b`t_@Sx_|3?KMuPZ2jnw4}?)e zaP4b^54!G%YF%CJdKjVVxoDA5;9idc&=D%#Qx7O{q$_AVqx0|e$)0$ndp1u~ec?7e z8=@;mfb1V`QVt+$^bTmYEd0(ykpoYibGPofc>1YN=*t;Nd#ZkCZc7ph6%Refkz(#m zT_J}?6J%{LzaMC9GzRHAaCbE7OKHqqouBDB=nk~2qV%rLZ*_I3dl^G8VmJ#>$xDXU(qr+Rw5Cnz#|kUew9LmspZNWuY4kalnfN@TbrDw+i_YXLpFI-uCB zXa;OWV|}NQ%$`7Dv=i(bYJIB%+U)9kroG)ybHCGllj*YdrY%V3G(m<4Z+-X01!5+sZ$4>F*TBD@9J0FQe_9rV7;jqm6E8vsz8{J{)8f{-?D`uo zoh@1QSsk=uSK#Bm+UP^HwV)3%WJ5N^JP$skBU+oy;m%8MC-c!Jaqf5Wf6o*%;D^5T zBS4kuTrBohBVoy=J*>MrqcM1E;tRYGUT#{@H1&5+z43(Mn!b2vbZlDvf_HwUwywNe zQy4xF?DzfL>yp-na5@!b@OQ-M#FZ9{2m86BHpB8zeW2Gb>WA0OBp)lw9b#~cKJt`n zc)B#)W{$=d!K}z$hr7>wbc64OCV4wS_K^3$2bzT#)>;jg*BTofh|!@J{sAv;F+o=l zKojnP^PffiNBVv$%+7o?g{LQ10n>&)AH{88A$~ra$w3L2=ir#{fcGK{XUguX}|a%T~eS)FP1e5VhzH2J}H z8cmF9dAcB9_gtH+FA-co9A;HjLPjmdJ1Ih!7#emSqG9GY-e);+P z`D$_PWr;UfE!qq8LRQXt*b<3O<-S{daUGk3tTNg?alyE{BOVaTLN~KTW^JZD-+?07 zra48ILASvknP)sWVzTf;XC%t>;j-P@*4=a6wOAOwnLfW$OTSJ75%!2&!1&vcUF+%z zuCRr>K1Vt5ydV+}I+#iRMeUpwCT)Gl7(aLkgIn0`08!Bs3;qeDdRxzcEj%UjMw-wT zlUX!5;Pcu;wIh0lk)al{g^j>+(EeVyq1{+JB&sV|jx-7qAFMwT$Sh_@d6LWrxRH1` zM&GsGsF@ryKGIEfBX4_{$0XR`H66XEmt9d8dyA~x>phPLYWjH&E@@$AdtPb2AOk@z z(34E}ekZ6e*95ErVwz`BMX1D3Gk{9aockeHvg!<$<%3!?{|NO$%apf1H%(8dGC(n; z5PD%rm(_y*LDsSR@Mzh+5cTTQ7c8CW%olxuFO#A#Wpbi9#Wl04Z?+^sRq74J_07zl zA7>u=#ad{YRig5%5eD0lAUzg*7BfKcj~>#QS|IJG9&tR`=xraH z{Xr4Zw(_toU6X;`?yayq7R(2-Z&^IIuU@w1Vm*uWzwGF#Fz>}t3|1=h8}gkS>QAoH z5!RtfkM|98bPJ$Z(X+(0;K!Uq!;EsfS4SR2ljlU|D)tv_HihlF^qK0xUo>8*Vm{XQ zWnsvA)Vgw7sP|AjUH5GkIi`datJ5!k7EPlQtWAB9#Lrn0;cngzk}<`zeO2`Y=&~4^ z4A53^(BT6wdw-MuzLgde$Dzx}<4n&El7X28wDwu|cW=FmR*Yul$;mlqdY@U1#7gp# z&%RCDZ%G?hrHKt?V(=V`g9T~0V1uU>ol$4Jr8abwwbTvw$B}(TRl5E9)qmW-G4)2&%%ze3QQ36=Kt>xQ19eAGuMfV{J36a6R+Vw5NFPrg5T37avNauU@6K zIPKW?B!ZyY0riSTeBPrqRUoV45POE&KK_i>P>-=HX|M!#roxUsQ^lI{aa34|UY125 z+WA_RDpaHXstAh6tIUezk}M4?$amemtj)Z}s2W*Ju4eV)x=<0c3t${^>K(*Z`Z|@;A6KuC#h+_6PLG)mjre|4GqwmiSkzhOrma(SJa(jYt+YLT}=m8~Kuhqb{e zBA!vXu%zdz#;g}BOxY@Js0f~TzaO%xO?lrf9eCf-;BCuhREKv(*+1)|Yl=hI#myQs z6$t7Re4q0B6Ktmg*Q@}OeOT4(j>lz6+D*lFafCd8ir~-JepqiWgTtUIG(0I^ ziT+ZBE?0%lTlwlT0FTI=$d1@Qn`KP4K5V-{c725_)Q;FSIhwu^UscmLY}Tf<$1=09 z<7CF`P=CZb8IVys4DY!Ni()4OTH9MxHV&N(m|LF00z zuohL>ifib+t#x0A*??*_^zyT|6vw2VwWVB(JTg>_)8GoG)HOooFwLx3b>6p&Gwbm6 z%)Hkdmp7UBl7F3CU@|m!!NWf&6BrFf>Lk@XHuDulnEes*j#lz*-9L7GTpfcV=r6~| zS7GoaI!?JqBX1|FO8_#=4veSDG})7Jakb%vZ-1`1dGu=Uf@*TrG>n+|rEKu`I`_xG z!!rEg_sIXRQ}vC74R3UoiJOOG5$=KwJRDE;Hx|zKJ}Ls z_v(??d5`aPBEv!W+CHOtuvzff42lzrARg8R_ox_;zD7}98lhjE>*35_oR2PuChP)R z^mJ*8J`emV9Qk3Wh22s`YeSL4wkXIusnmyg(~b8*dFPII6?u1rod(7G%NUE%SSzq( zmkTlXqQrv?pHWv<5*Lff-is0!*GbMoGE75t^}4_9g=|i8_7NA;y0VVASWIrV5f{(6 zS;naGUJT8Oomr&Df%(S27bTw50B>;?!WpiH0W)rB$FT!u+%S)42h6yk zuD%0i+z^)E0W(T)NvCLOdVE& zj3c$rVMczjA$0^Zr&-h6VPrUUp2K@fi_)GrD`91Q4mdc2_N-a(E;%n-7 z80xmf)zqGDNt%@$g~YtnPP-{tsM{ BP8I+F literal 0 HcmV?d00001 diff --git a/models/dev/model_7feat.pkl b/models/dev/archive/model_7feat.pkl similarity index 100% rename from models/dev/model_7feat.pkl rename to models/dev/archive/model_7feat.pkl diff --git a/models/dev/model_7feat_meta.json b/models/dev/archive/model_7feat_meta.json similarity index 100% rename from models/dev/model_7feat_meta.json rename to models/dev/archive/model_7feat_meta.json diff --git a/notebooks/03_prod_valid.ipynb b/notebooks/03_prod_valid.ipynb new file mode 100644 index 0000000..b7075a9 --- /dev/null +++ b/notebooks/03_prod_valid.ipynb @@ -0,0 +1,498 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "e5dc52e5", + "metadata": {}, + "source": [ + "## **Verify Model Artifacts**" + ] + }, + { + "cell_type": "markdown", + "id": "8289c920", + "metadata": {}, + "source": [ + "### **Section 0: Imports**" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "6727ac1a", + "metadata": {}, + "outputs": [], + "source": [ + "import joblib\n", + "import json\n", + "import os\n", + "import sys\n", + "import numpy as np\n", + "from pathlib import Path\n" + ] + }, + { + "cell_type": "markdown", + "id": "69d02036", + "metadata": {}, + "source": [ + "- **Set working directory**" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "51094118", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Working directory: d:\\MLops\\NetworkSecurity\n", + "[feature_extraction] Loaded 1401 TLD probabilities\n" + ] + } + ], + "source": [ + "# Set working directory to project root\n", + "if Path.cwd().name == \"notebooks\":\n", + " os.chdir(\"..\")\n", + "\n", + "print(f\"Working directory: {Path.cwd()}\")\n", + "\n", + "# Add src to path so we can import common modules\n", + "sys.path.insert(0, str(Path.cwd() / \"src\"))\n", + "from common.feature_extraction import extract_features\n" + ] + }, + { + "cell_type": "markdown", + "id": "10eb5850", + "metadata": {}, + "source": [ + "### **Section 1: Check 8-Feature Model**" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "831fa2fc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "============================================================\n", + "VERIFYING 8-FEATURE MODEL (PRODUCTION)\n", + "============================================================\n", + "\n", + "1. Model Type: \n", + " Classes: [0 1]\n", + " Calibration: βœ… CalibratedClassifierCV detected\n", + " Base estimator: XGBClassifier\n", + "\n", + "2. Metadata:\n", + " Features (8):\n", + " 1. IsHTTPS\n", + " 2. TLDLegitimateProb\n", + " 3. CharContinuationRate\n", + " 4. SpacialCharRatioInURL\n", + " 5. URLCharProb\n", + " 6. LetterRatioInURL\n", + " 7. NoOfOtherSpecialCharsInURL\n", + " 8. DomainLength\n", + "\n", + "3. Class Mapping:\n", + " Phish (0) at column index: 0\n", + " Class mapping: {'phish': 0, 'legit': 1}\n", + "\n", + "4. Performance Metrics:\n", + " pr_auc: 0.9991584033257773\n", + " f1_macro: 0.9969925280550227\n", + " brier: 0.0026371303574400343\n", + "\n", + "============================================================\n", + "EXPECTED VALUES:\n", + "============================================================\n", + "βœ… Classes: [0 1]\n", + "βœ… Features: 8 (IsHTTPS + 7 URL features)\n", + "βœ… Phish column index: 0\n", + "βœ… PR-AUC: ~0.999\n", + "============================================================\n" + ] + } + ], + "source": [ + "print(\"=\" * 60)\n", + "print(\"VERIFYING 8-FEATURE MODEL (PRODUCTION)\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Load model\n", + "model_path = Path(\"models/dev/model_8feat.pkl\")\n", + "model = joblib.load(model_path)\n", + "\n", + "print(f\"\\n1. Model Type: {type(model)}\")\n", + "print(f\" Classes: {model.classes_}\")\n", + "\n", + "# Check if it's calibrated\n", + "if hasattr(model, \"calibrated_classifiers_\"):\n", + " print(f\" Calibration: βœ… CalibratedClassifierCV detected\")\n", + " base = model.calibrated_classifiers_[0].estimator\n", + " print(f\" Base estimator: {type(base).__name__}\")\n", + "else:\n", + " print(f\" Calibration: ❌ No calibration wrapper found\")\n", + "\n", + "# Load metadata\n", + "meta_path = Path(\"models/dev/model_8feat_meta.json\")\n", + "meta = json.load(open(meta_path))\n", + "\n", + "print(f\"\\n2. Metadata:\")\n", + "print(f\" Features ({len(meta['feature_order'])}):\")\n", + "for i, feat in enumerate(meta[\"feature_order\"], 1):\n", + " print(f\" {i}. {feat}\")\n", + "\n", + "print(f\"\\n3. Class Mapping:\")\n", + "print(f\" Phish (0) at column index: {meta['phish_proba_col_index']}\")\n", + "print(f\" Class mapping: {meta.get('class_mapping', 'NOT FOUND')}\")\n", + "\n", + "print(f\"\\n4. Performance Metrics:\")\n", + "if \"metrics\" in meta:\n", + " for key, val in meta[\"metrics\"].items():\n", + " print(f\" {key}: {val}\")\n", + "else:\n", + " print(\" ⚠️ No metrics found in metadata\")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"EXPECTED VALUES:\")\n", + "print(\"=\" * 60)\n", + "print(\"βœ… Classes: [0 1]\")\n", + "print(\"βœ… Features: 8 (IsHTTPS + 7 URL features)\")\n", + "print(\"βœ… Phish column index: 0\")\n", + "print(\"βœ… PR-AUC: ~0.999\")\n", + "print(\"=\" * 60)\n" + ] + }, + { + "cell_type": "markdown", + "id": "2d10f6c8", + "metadata": {}, + "source": [ + "### **Section 2: Quick Prediction Test**" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "fa664259", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "============================================================\n", + "PREDICTION TEST WITH WHITELIST INTEGRATION\n", + "============================================================\n", + "βœ“ Imported whitelist function\n", + "βœ“ Known legitimate domains: 26 entries\n", + " Sample domains: ['microsoft.com', 'www.apple.com', 'www.youtube.com', 'facebook.com', 'www.wikipedia.org']\n", + "\n", + "------------------------------------------------------------\n", + "WHITELIST TESTS\n", + "------------------------------------------------------------\n", + "βœ… WHITELISTED https://google.com\n", + "βœ… WHITELISTED https://www.github.com\n", + "βœ… WHITELISTED https://microsoft.com/login\n", + "❌ NOT WHITELISTED https://example.com/login?id=123\n", + "❌ NOT WHITELISTED http://suspicious-phishing-site.top/verify-account\n", + "\n", + "------------------------------------------------------------\n", + "MODEL PREDICTION WITH WHITELIST INTEGRATION\n", + "------------------------------------------------------------\n", + "\n", + "Testing URL: https://google.com\n", + "βœ… WHITELIST HIT: https://google.com\n", + " β†’ Bypassing model prediction\n", + " β†’ p_malicious = 0.01 (whitelist override)\n", + " β†’ source = 'whitelist'\n", + "\n", + "============================================================\n", + "CONCLUSION:\n", + "============================================================\n", + "βœ… Whitelist function imported successfully\n", + "βœ… Major tech domains bypass model prediction\n", + "βœ… Fast-path optimization working as designed\n", + "βœ… Production model service logic validated\n", + "============================================================\n" + ] + } + ], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"PREDICTION TEST WITH WHITELIST INTEGRATION\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Import whitelist function from model service\n", + "from model_svc.main import _check_whitelist, KNOWN_LEGITIMATE_DOMAINS\n", + "\n", + "print(f\"βœ“ Imported whitelist function\")\n", + "print(f\"βœ“ Known legitimate domains: {len(KNOWN_LEGITIMATE_DOMAINS)} entries\")\n", + "print(f\" Sample domains: {list(KNOWN_LEGITIMATE_DOMAINS)[:5]}\")\n", + "\n", + "# Test URLs including whitelisted ones\n", + "test_urls_with_whitelist = [\n", + " \"https://google.com\",\n", + " \"https://www.github.com\",\n", + " \"https://microsoft.com/login\",\n", + " \"https://example.com/login?id=123\", # Not whitelisted\n", + " \"http://suspicious-phishing-site.top/verify-account\", # Not whitelisted\n", + "]\n", + "\n", + "print(f\"\\n\" + \"-\" * 60)\n", + "print(\"WHITELIST TESTS\")\n", + "print(\"-\" * 60)\n", + "\n", + "for test_url in test_urls_with_whitelist:\n", + " is_whitelisted = _check_whitelist(test_url)\n", + " status = \"βœ… WHITELISTED\" if is_whitelisted else \"❌ NOT WHITELISTED\"\n", + " print(f\"{status:20s} {test_url}\")\n", + "\n", + "print(f\"\\n\" + \"-\" * 60)\n", + "print(\"MODEL PREDICTION WITH WHITELIST INTEGRATION\")\n", + "print(\"-\" * 60)\n", + "\n", + "# Test feature array (simulating google.com features)\n", + "test_features = {\n", + " \"IsHTTPS\": 1.0,\n", + " \"TLDLegitimateProb\": 0.6111,\n", + " \"CharContinuationRate\": 0.1765,\n", + " \"SpacialCharRatioInURL\": 0.2222,\n", + " \"URLCharProb\": 0.06,\n", + " \"LetterRatioInURL\": 0.7778,\n", + " \"NoOfOtherSpecialCharsInURL\": 4.0,\n", + " \"DomainLength\": 10.0,\n", + "}\n", + "\n", + "# Simulate full prediction pipeline (like model service does)\n", + "google_url = \"https://google.com\"\n", + "\n", + "print(f\"\\nTesting URL: {google_url}\")\n", + "\n", + "# Step 1: Check whitelist first (fast path)\n", + "if _check_whitelist(google_url):\n", + " print(f\"βœ… WHITELIST HIT: {google_url}\")\n", + " print(f\" β†’ Bypassing model prediction\")\n", + " print(f\" β†’ p_malicious = 0.01 (whitelist override)\")\n", + " print(f\" β†’ source = 'whitelist'\")\n", + "else:\n", + " print(f\"❌ NOT WHITELISTED: Proceeding with model prediction...\")\n", + "\n", + " # Step 2: Model prediction (only if not whitelisted)\n", + " feature_array = np.array([[test_features[f] for f in meta[\"feature_order\"]]])\n", + "\n", + " print(f\"\\nTest input (google.com-like features):\")\n", + " print(f\" Shape: {feature_array.shape}\")\n", + " print(f\" Values: {feature_array[0]}\")\n", + "\n", + " # Predict\n", + " probas = model.predict_proba(feature_array)\n", + " print(f\"\\nModel output:\")\n", + " print(f\" Raw probabilities: {probas[0]}\")\n", + " print(\n", + " f\" P(phishing) [col {meta['phish_proba_col_index']}]: {probas[0, meta['phish_proba_col_index']]:.6f}\"\n", + " )\n", + " print(\n", + " f\" P(legitimate) [col {1 - meta['phish_proba_col_index']}]: {probas[0, 1 - meta['phish_proba_col_index']]:.6f}\"\n", + " )\n", + "\n", + " if probas[0, 0] > 0.5:\n", + " print(f\"\\n⚠️ WARNING: Model predicts PHISHING for google.com-like features!\")\n", + " print(f\" This demonstrates why whitelist is essential for OOD domains.\")\n", + " else:\n", + " print(f\"\\nβœ… Model predicts LEGITIMATE for google.com-like features\")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"CONCLUSION:\")\n", + "print(\"=\" * 60)\n", + "print(\"βœ… Whitelist function imported successfully\")\n", + "print(\"βœ… Major tech domains bypass model prediction\")\n", + "print(\"βœ… Fast-path optimization working as designed\")\n", + "print(\"βœ… Production model service logic validated\")\n", + "print(\"=\" * 60)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4ae79b7f", + "metadata": {}, + "outputs": [], + "source": [ + "# # Test whitelist functionality with various URLs\n", + "# test_urls = [\n", + "# \"https://google.com\",\n", + "# \"https://www.google.com\",\n", + "# \"https://github.com/user/repo\",\n", + "# \"https://suspicious-phishing-site.top/verify-account\",\n", + "# \"http://example.com/login?acct=12345\",\n", + "# \"https://paypal.com/signin\",\n", + "# \"https://evil-paypal-clone.tk/login\",\n", + "# ]\n", + "\n", + "# print(\"=\" * 60)\n", + "# print(\"WHITELIST TESTING\")\n", + "# print(\"=\" * 60)\n", + "\n", + "# for url in test_urls:\n", + "# is_whitelisted = _check_whitelist(url)\n", + "# status = \"βœ… WHITELISTED\" if is_whitelisted else \"❌ NOT WHITELISTED\"\n", + "# print(f\"{status:20s} | {url}\")\n", + "\n", + "# print(\"=\" * 60)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e6f9aad", + "metadata": {}, + "outputs": [], + "source": [ + "# # Enhanced prediction test with whitelist integration\n", + "# def predict_with_whitelist(url: str, model, meta):\n", + "# \"\"\"\n", + "# Simulate the model service prediction logic with whitelist.\n", + "# Returns prediction result with source information.\n", + "# \"\"\"\n", + "# # Fast path: Check whitelist FIRST\n", + "# if _check_whitelist(url):\n", + "# return {\n", + "# \"url\": url,\n", + "# \"p_malicious\": 0.01,\n", + "# \"source\": \"whitelist\",\n", + "# \"decision\": \"ALLOW\",\n", + "# \"reason\": \"Known legitimate domain\",\n", + "# }\n", + "\n", + "# # Extract features for model prediction\n", + "# features = extract_features(url, include_https=True)\n", + "# feature_array = np.array([[features[f] for f in meta[\"feature_order\"]]])\n", + "\n", + "# # Model prediction\n", + "# probas = model.predict_proba(feature_array)\n", + "# p_malicious = probas[0, meta[\"phish_proba_col_index\"]]\n", + "\n", + "# return {\n", + "# \"url\": url,\n", + "# \"p_malicious\": float(p_malicious),\n", + "# \"source\": \"model\",\n", + "# \"decision\": \"BLOCK\" if p_malicious > 0.5 else \"ALLOW\",\n", + "# \"reason\": f\"Model prediction (p={p_malicious:.4f})\",\n", + "# }\n", + "\n", + "\n", + "# print(\"βœ“ Enhanced prediction function with whitelist ready\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "987edc10", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "============================================================\n", + "PREDICTION TEST\n", + "============================================================\n", + "\n", + "Test input (google.com-like features):\n", + " Shape: (1, 8)\n", + " Values: [ 1. 0.6111 0.1765 0.2222 0.06 0.7778 4. 10. ]\n", + "\n", + "Model output:\n", + " Raw probabilities: [1. 0.]\n", + " P(phishing) [col 0]: 1.000000\n", + " P(legitimate) [col 1]: 0.000000\n", + "\n", + "⚠️ WARNING: Model predicts PHISHING for google.com-like features!\n", + " This is expected due to OOD - whitelist will handle it.\n", + "============================================================\n" + ] + } + ], + "source": [ + "# print(\"\\n\" + \"=\" * 60)\n", + "# print(\"PREDICTION TEST\")\n", + "# print(\"=\" * 60)\n", + "\n", + "\n", + "# # Test feature array (simulating google.com features)\n", + "# test_features = {\n", + "# \"IsHTTPS\": 1.0,\n", + "# \"TLDLegitimateProb\": 0.6111,\n", + "# \"CharContinuationRate\": 0.1765,\n", + "# \"SpacialCharRatioInURL\": 0.2222,\n", + "# \"URLCharProb\": 0.06,\n", + "# \"LetterRatioInURL\": 0.7778,\n", + "# \"NoOfOtherSpecialCharsInURL\": 4.0,\n", + "# \"DomainLength\": 10.0,\n", + "# }\n", + "\n", + "# # Create feature array in correct order\n", + "\n", + "# feature_array = np.array([[test_features[f] for f in meta[\"feature_order\"]]])\n", + "\n", + "# print(f\"\\nTest input (google.com-like features):\")\n", + "# print(f\" Shape: {feature_array.shape}\")\n", + "# print(f\" Values: {feature_array[0]}\")\n", + "\n", + "# # Predict\n", + "# probas = model.predict_proba(feature_array)\n", + "# print(f\"\\nModel output:\")\n", + "# print(f\" Raw probabilities: {probas[0]}\")\n", + "# print(\n", + "# f\" P(phishing) [col {meta['phish_proba_col_index']}]: {probas[0, meta['phish_proba_col_index']]:.6f}\"\n", + "# )\n", + "# print(\n", + "# f\" P(legitimate) [col {1 - meta['phish_proba_col_index']}]: {probas[0, 1 - meta['phish_proba_col_index']]:.6f}\"\n", + "# )\n", + "\n", + "# if probas[0, 0] > 0.5:\n", + "# print(f\"\\n⚠️ WARNING: Model predicts PHISHING for google.com-like features!\")\n", + "# print(f\" This is expected due to OOD - whitelist will handle it.\")\n", + "# else:\n", + "# print(f\"\\nβœ… Model predicts LEGITIMATE for google.com-like features\")\n", + "\n", + "# print(\"=\" * 60)\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/outputs/benchmark_results.txt b/outputs/benchmark_results.txt new file mode 100644 index 0000000..a7b8fbc --- /dev/null +++ b/outputs/benchmark_results.txt @@ -0,0 +1,9 @@ + +====================================================================== +PhishGuardAI - Performance Benchmark +====================================================================== + +1. LATENCY TESTS (100 requests per path) +---------------------------------------------------------------------- + +Testing: whitelist diff --git a/pyproject.toml b/pyproject.toml index 61ead86..0a2c531 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,22 @@ # pyproject.toml +[project] +name = "phishguard" +version = "0.2.0" +description = "PhishGuard AI - URL phishing detection system" +requires-python = ">=3.11" +dependencies = [ + "fastapi", + "uvicorn[standard]", + "pydantic>=2", + "scikit-learn", + "xgboost", + "numpy", + "pandas", + "joblib", + "pyyaml", + "tldextract", +] + [tool.black] line-length = 88 diff --git a/requirements-docker.txt b/requirements-docker.txt index 00ae679..be885a9 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -36,3 +36,6 @@ isort # Sorts imports automatically flake8 # Python style guide enforcement mypy # Static type checker for Python bandit # Security linter for Python code + +# --- Local Package Installation --- +-e . # Install current package in editable mode diff --git a/scripts/materialize_url_features.py b/scripts/archive/materialize_url_features.py similarity index 100% rename from scripts/materialize_url_features.py rename to scripts/archive/materialize_url_features.py diff --git a/scripts/benchmark.py b/scripts/benchmark.py new file mode 100644 index 0000000..5c79ca2 --- /dev/null +++ b/scripts/benchmark.py @@ -0,0 +1,193 @@ +""" +PhishGuardAI Performance Benchmark + +Measures latency and throughput for different request paths. +""" + +import statistics +import time +from concurrent.futures import ThreadPoolExecutor + +import requests + +GATEWAY_URL = "http://localhost:8000/predict" + +test_cases = [ + ("https://google.com", "whitelist"), + ("https://phishing.top", "high_confidence_block"), + ("https://example.com", "low_confidence_allow"), + ("https://npm.org", "short_domain_judge"), +] + + +def test_latency(url, label, n=100): + """Measure latency for a single URL""" + latencies = [] + errors = 0 + + for _ in range(n): + try: + start = time.time() + response = requests.post(GATEWAY_URL, json={"url": url}, timeout=5) + response.raise_for_status() + latencies.append((time.time() - start) * 1000) # Convert to ms + except Exception: + errors += 1 + + if not latencies: + return {"label": label, "url": url, "error": "All requests failed"} + + return { + "label": label, + "url": url, + "n": n, + "errors": errors, + "p50": statistics.median(latencies), + "p95": ( + statistics.quantiles(latencies, n=20)[18] + if len(latencies) > 20 + else max(latencies) + ), + "p99": ( + statistics.quantiles(latencies, n=100)[98] + if len(latencies) > 100 + else max(latencies) + ), + "mean": statistics.mean(latencies), + "min": min(latencies), + "max": max(latencies), + } + + +def test_throughput(n_requests=1000, n_workers=10): + """Measure throughput with concurrent requests""" + + def make_request(_): + try: + response = requests.post( + GATEWAY_URL, json={"url": "https://example.com"}, timeout=5 + ) + return response.status_code == 200 + except Exception: + return False + + start = time.time() + with ThreadPoolExecutor(max_workers=n_workers) as executor: + results = list(executor.map(make_request, range(n_requests))) + elapsed = time.time() - start + + success_rate = sum(results) / len(results) + throughput = n_requests / elapsed + + return { + "n_requests": n_requests, + "n_workers": n_workers, + "elapsed_seconds": elapsed, + "throughput_req_per_sec": throughput, + "success_rate": success_rate, + } + + +if __name__ == "__main__": + print("\n" + "=" * 70) + print("PhishGuardAI - Performance Benchmark") + print("=" * 70) + + print("\n1. LATENCY TESTS (100 requests per path)") + print("-" * 70) + + latency_results = [] + for url, label in test_cases: + print(f"\nTesting: {label}") + result = test_latency(url, label, n=100) + latency_results.append(result) + + if "error" in result: + print(f" ❌ ERROR: {result['error']}") + else: + print(" βœ… SUCCESS") + print(f" p50: {result['p50']:6.2f}ms") + print(f" p95: {result['p95']:6.2f}ms") + print(f" p99: {result['p99']:6.2f}ms") + print(f" mean: {result['mean']:6.2f}ms") + if result["errors"] > 0: + print(f" errors: {result['errors']}/{result['n']}") + + print("\n\n2. THROUGHPUT TEST (1000 requests, 10 concurrent workers)") + print("-" * 70) + + throughput_result = test_throughput(n_requests=1000, n_workers=10) + print(f" Throughput: {throughput_result['throughput_req_per_sec']:7.2f} req/sec") + print(f" Success Rate: {throughput_result['success_rate'] * 100:5.1f}%") + print(f" Total Time: {throughput_result['elapsed_seconds']:7.2f}s") + + print("\n\n3. SUMMARY - PRODUCTION READINESS") + print("=" * 70) + + # Check against targets + targets = { + "whitelist_p95": 10, # ms + "model_p95": 50, # ms + "judge_p95": 100, # ms + "throughput": 100, # req/sec + "success_rate": 0.99, # 99% + } + + whitelist_result = next( + (r for r in latency_results if r["label"] == "whitelist"), None + ) + model_result = next( + (r for r in latency_results if r["label"] == "low_confidence_allow"), None + ) + judge_result = next( + (r for r in latency_results if r["label"] == "short_domain_judge"), None + ) + + print("\nLatency Targets:") + if whitelist_result and "p95" in whitelist_result: + status = "βœ…" if whitelist_result["p95"] < targets["whitelist_p95"] else "⚠️" + target_ms = targets["whitelist_p95"] + print( + f" {status} Whitelist p95: {whitelist_result['p95']:.2f}ms " + f"(target: <{target_ms}ms)" + ) + + if model_result and "p95" in model_result: + status = "βœ…" if model_result["p95"] < targets["model_p95"] else "⚠️" + target_ms = targets["model_p95"] + print( + f" {status} Model p95: {model_result['p95']:.2f}ms " + f"(target: <{target_ms}ms)" + ) + + if judge_result and "p95" in judge_result: + status = "βœ…" if judge_result["p95"] < targets["judge_p95"] else "⚠️" + target_ms = targets["judge_p95"] + print( + f" {status} Judge p95: {judge_result['p95']:.2f}ms " + f"(target: <{target_ms}ms)" + ) + + print("\nThroughput Targets:") + status = ( + "βœ…" + if throughput_result["throughput_req_per_sec"] > targets["throughput"] + else "⚠️" + ) + throughput_val = throughput_result["throughput_req_per_sec"] + target_throughput = targets["throughput"] + print( + f" {status} Throughput: {throughput_val:.2f} req/sec " + f"(target: >{target_throughput} req/sec)" + ) + + status = ( + "βœ…" if throughput_result["success_rate"] > targets["success_rate"] else "⚠️" + ) + success_pct = throughput_result["success_rate"] * 100 + target_pct = targets["success_rate"] * 100 + print(f" {status} Success Rate: {success_pct:.1f}% (target: >{target_pct:.0f}%)") + + print("\n" + "=" * 70) + print("Benchmark Complete!") + print("=" * 70) diff --git a/scripts/ge_build_phiusiil_suite.py b/scripts/ge_build_phiusiil_suite.py index 1d3c194..97e550c 100644 --- a/scripts/ge_build_phiusiil_suite.py +++ b/scripts/ge_build_phiusiil_suite.py @@ -1,119 +1,391 @@ -# scripts/ge_build_phiusiil_suite.py +""" +Build Great Expectations suite for PhishGuard 8-Feature Model. +Creates comprehensive data validation expectations for the production-ready feature set. + +This script: +1. Loads processed features (phiusiil_features_v2.csv) +2. Creates GE expectations for all 8 required features +3. Validates data quality for ML pipeline + +Features validated match docs/FEATURE_EXTRACTION.md +""" + from pathlib import Path import great_expectations as gx import pandas as pd +from great_expectations.core.batch import RuntimeBatchRequest + +# Updated paths for 8-feature model +PROCESSED_CSV = Path("data/processed/phiusiil_features_v2.csv") +SUITE_NAME = "phiusiil_8feature_production" + +# 8-Feature Model Definition (matches ge_check.py and FEATURE_EXTRACTION.md) +REQUIRED_FEATURES = { + # Binary features + "IsHTTPS": ("binary", 0, 1), + # Probability features [0, 1] + "TLDLegitimateProb": ("float", 0.0, 1.0), + "CharContinuationRate": ("float", 0.0, 1.0), + "SpacialCharRatioInURL": ("float", 0.0, 1.0), + "URLCharProb": ("float", 0.0, 1.0), + "LetterRatioInURL": ("float", 0.0, 1.0), + # Count features + "NoOfOtherSpecialCharsInURL": ("int_like", 0, 1000), + "DomainLength": ("int_like", 1, 253), # RFC 1035 limit +} + +# Legacy features to warn about +DEPRECATED_FEATURES = {"url_len", "url_digit_ratio", "url_subdomains"} + +print(f"πŸ” Loading processed features: {PROCESSED_CSV}") +if not PROCESSED_CSV.exists(): + raise FileNotFoundError(f"Processed features not found: {PROCESSED_CSV}") -CSV = Path("data/raw/PhiUSIIL_Phishing_URL_Dataset.csv") -OUT_CSV = Path("data/processed/phiusiil_clean.csv") -SUITE_NAME = "phiusiil_minimal" +# Load the processed features dataset +df = pd.read_csv(PROCESSED_CSV) +print(f"βœ… Loaded dataset: {df.shape[0]:,} rows Γ— {df.shape[1]} columns") -# 1) Load & deduplicate by exact URL (prevents train/test contamination) -df = pd.read_csv(CSV, encoding_errors="ignore") -dup_total = df.duplicated(subset=["URL"]).sum() if "URL" in df.columns else 0 -df = df.drop_duplicates(subset=["URL"]).reset_index(drop=True) -OUT_CSV.parent.mkdir(parents=True, exist_ok=True) -df.to_csv(OUT_CSV, index=False) +# Check for deprecated features +deprecated_present = [col for col in DEPRECATED_FEATURES if col in df.columns] +if deprecated_present: + print(f"⚠️ Found deprecated features: {deprecated_present}") -# 2) GE context - initialize project if needed +# Verify all required features are present +missing_features = [feat for feat in REQUIRED_FEATURES if feat not in df.columns] +if missing_features: + raise ValueError(f"Missing required features: {missing_features}") + +print(f"βœ… All 8 required features present: {list(REQUIRED_FEATURES.keys())}") + +# Initialize Great Expectations context (handle corrupted config) +print("πŸ”§ Setting up Great Expectations...") try: - # Try to get existing context (if project already initialized) ctx = gx.get_context() if not hasattr(ctx, "root_directory") or ctx.root_directory is None: raise ValueError("No Great Expectations project found") -except (ValueError, gx.exceptions.DataContextError): - # Initialize a new GE project in the current directory - print("[GE] No Great Expectations project found. Initializing...") - ctx = gx.get_context(mode="file") # Creates file-based context - print(f"[GE] Initialized Great Expectations project at: {ctx.root_directory}") + print(f"βœ… Using existing GE project: {ctx.root_directory}") +except ( + ValueError, + gx.exceptions.DataContextError, + gx.exceptions.InvalidDataContextConfigError, +): + print("πŸ”¨ GE config corrupted or missing - initializing fresh project...") -# Create pandas datasource -try: - datasource = ctx.data_sources.get("phiusiil_src") -except (ValueError, KeyError): - datasource = ctx.data_sources.add_pandas("phiusiil_src") + # Remove corrupted GE directory if it exists + import shutil -# Add dataframe asset -try: - asset = datasource.get_asset("phiusiil_df") -except (ValueError, KeyError, AttributeError, LookupError): - asset = datasource.add_dataframe_asset("phiusiil_df") + gx_dir = Path("gx") + if gx_dir.exists(): + print("πŸ—‘οΈ Removing corrupted GE directory...") + shutil.rmtree(gx_dir) + # Initialize fresh GE project + try: + ctx = gx.get_context(mode="file") + print(f"βœ… Initialized fresh GE project: {ctx.root_directory}") + except Exception as e: + print(f"❌ Failed to initialize GE: {e}") + print("πŸ’‘ Continuing with basic validation instead...") -# Create batch definition for the whole dataframe -try: - batch_definition = asset.get_batch_definition("phiusiil_batch") -except (ValueError, KeyError, AttributeError, LookupError): - batch_definition = asset.add_batch_definition_whole_dataframe("phiusiil_batch") + # Simple validation without GE + print("πŸ” Running basic feature validation...") + + # Check all required features are present and valid + validation_errors = [] -# Create batch parameters with the dataframe -batch_parameters = {"dataframe": df} + for feature_name, (dtype, min_val, max_val) in REQUIRED_FEATURES.items(): + if feature_name not in df.columns: + validation_errors.append(f"Missing feature: {feature_name}") + continue -# Get the batch -batch = batch_definition.get_batch(batch_parameters=batch_parameters) + series = df[feature_name] + # Check for nulls + null_count = series.isnull().sum() + if null_count > 0: + validation_errors.append(f"{feature_name}: {null_count} null values") -# Create or get expectation suite + # Check data type and range + if dtype == "binary": + if not series.isin([0, 1]).all(): + validation_errors.append(f"{feature_name}: not binary (0/1)") + elif dtype == "float": + if not pd.api.types.is_numeric_dtype(series): + validation_errors.append(f"{feature_name}: not numeric") + elif (series < min_val).any() or (series > max_val).any(): + validation_errors.append( + f"{feature_name}: values outside [{min_val}, {max_val}]" + ) + elif dtype == "int_like": + if not pd.api.types.is_integer_dtype(series) and not ( + pd.api.types.is_float_dtype(series) and (series % 1 == 0).all() + ): + validation_errors.append(f"{feature_name}: not integer-like") + elif (series < min_val).any() or (series > max_val).any(): + validation_errors.append( + f"{feature_name}: values outside [{min_val}, {max_val}]" + ) + + if validation_errors: + print("❌ Validation errors found:") + for error in validation_errors[:10]: # Show first 10 errors + print(f" πŸ’₯ {error}") + if len(validation_errors) > 10: + print(f" ... and {len(validation_errors) - 10} more errors") + else: + print("βœ… All basic validations PASSED!") + print(f"πŸ“Š Dataset: {df.shape[0]:,} rows Γ— {df.shape[1]} columns") + print( + f"🎯 Features: {len(REQUIRED_FEATURES)} production features validated" + ) + + print("πŸš€ Data ready for ML pipeline (basic validation)") + exit(0) + +# Create or get pandas datasource for PhishGuard features +datasource_name = "phishguard_features" try: - ctx.suites.delete(SUITE_NAME) -except Exception: # nosec B110 - # Suite doesn't exist, which is fine - pass -suite = ctx.suites.add(gx.ExpectationSuite(name=SUITE_NAME)) + datasource = ctx.datasources[datasource_name] + print(f"βœ… Using existing datasource: {datasource_name}") +except (ValueError, KeyError): + # Create new pandas datasource using modern GE API + datasource_config = { + "name": datasource_name, + "class_name": "Datasource", + "execution_engine": {"class_name": "PandasExecutionEngine"}, + "data_connectors": { + "default_runtime_data_connector": { + "class_name": "RuntimeDataConnector", + "batch_identifiers": ["default_identifier_name"], + } + }, + } + datasource = ctx.add_datasource(**datasource_config) + print(f"βœ… Created new datasource: {datasource_name}") -# Get validator using the batch -validator = ctx.get_validator(batch=batch, expectation_suite=suite) +# Create batch request for our DataFrame using proper GE API +batch_request = RuntimeBatchRequest( + datasource_name=datasource_name, + data_connector_name="default_runtime_data_connector", + data_asset_name="processed_8features", + runtime_parameters={"batch_data": df}, + batch_identifiers={"default_identifier_name": "production_features"}, +) # Remove existing suite if it exists (fresh start) +try: + existing_suites = ctx.list_expectation_suite_names() + if SUITE_NAME in existing_suites: + ctx.delete_expectation_suite(SUITE_NAME) + print(f"πŸ—‘οΈ Removed existing suite: {SUITE_NAME}") +except Exception as e: + # Ignore deletion errors - suite might not exist + print(f"Note: Could not delete existing suite: {e}") -# 3) Expectations grounded in your EDA -# --- GE hardening derived from URL-only policy --- +# Create new expectation suite using add_expectation_suite +try: + suite = ctx.add_expectation_suite(expectation_suite_name=SUITE_NAME) + print(f"βœ… Created expectation suite: {SUITE_NAME}") +except Exception: + # Suite might already exist, get it instead + suite = ctx.get_expectation_suite(expectation_suite_name=SUITE_NAME) + print(f"βœ… Using existing expectation suite: {SUITE_NAME}") +# Get validator using batch request +validator = ctx.get_validator( + batch_request=batch_request, expectation_suite_name=SUITE_NAME +) -def has(col: str) -> bool: +print("🎯 Building expectations for 8-feature production model...") + + +def has_column(col: str) -> bool: + """Check if column exists in dataframe""" return col in df.columns -# 1) Core invariants +# === CORE DATA INTEGRITY === +print(" πŸ“‹ Core data integrity checks...") + +# Label column validation (phish=0, legit=1) label_col = next( - (c for c in df.columns if c.lower() in {"label", "result", "y", "target"}), "label" + (c for c in df.columns if c.lower() in {"label", "result", "y", "target", "class"}), + "label", ) -validator.expect_column_values_to_not_be_null(label_col) -validator.expect_column_values_to_be_in_set(label_col, [0, 1]) +if has_column(label_col): + validator.expect_column_values_to_not_be_null(label_col) + validator.expect_column_values_to_be_in_set(label_col, [0, 1]) + print(f" βœ… Label column '{label_col}' validated") -if has("URL"): +# URL uniqueness (prevent data leakage) +if has_column("URL"): validator.expect_column_values_to_not_be_null("URL") validator.expect_column_values_to_be_unique("URL") + print(" βœ… URL uniqueness validated") -# 2) URL-only engineered features (ranges/dtypes) -if has("url_len"): - validator.expect_column_values_to_be_between("url_len", min_value=0) - validator.expect_column_values_to_be_of_type("url_len", "int64") +# === 8-FEATURE MODEL VALIDATION === +print(" 🧠 8-Feature model validation...") -if has("url_subdomains"): - validator.expect_column_values_to_be_between("url_subdomains", min_value=0) - validator.expect_column_values_to_be_of_type("url_subdomains", "int64") +# 1. IsHTTPS - Binary feature (0=HTTP, 1=HTTPS) +if has_column("IsHTTPS"): + validator.expect_column_values_to_not_be_null("IsHTTPS") + validator.expect_column_values_to_be_in_set("IsHTTPS", [0, 1]) + # Note: Accept both int64 and float64 for binary features (common in pandas) + if df["IsHTTPS"].dtype == "int64": + validator.expect_column_values_to_be_of_type("IsHTTPS", "int64") + elif df["IsHTTPS"].dtype == "float64": + validator.expect_column_values_to_be_of_type("IsHTTPS", "float64") + print(" βœ… IsHTTPS (binary) validated") -if has("url_digit_ratio"): +# 2. TLDLegitimateProb - Bayesian TLD probability [0,1] +if has_column("TLDLegitimateProb"): + validator.expect_column_values_to_not_be_null("TLDLegitimateProb") validator.expect_column_values_to_be_between( - "url_digit_ratio", min_value=0.0, max_value=1.0 + "TLDLegitimateProb", min_value=0.0, max_value=1.0 ) - validator.expect_column_values_to_be_of_type("url_digit_ratio", "float64") + validator.expect_column_values_to_be_of_type("TLDLegitimateProb", "float64") + # Reasonable distribution check - TLD probs should vary + validator.expect_column_unique_value_count_to_be_between( + "TLDLegitimateProb", min_value=10, max_value=1000 + ) + print(" βœ… TLDLegitimateProb (Bayesian) validated") -# 3) Probability-like URL priors (must be in [0,1]) -for c in ["CharContinuationRate", "URLCharProb", "TLDLegitimateProb"]: - if has(c): - validator.expect_column_values_to_be_between(c, min_value=0.0, max_value=1.0) +# 3. CharContinuationRate - Character repetition [0,1] +if has_column("CharContinuationRate"): + validator.expect_column_values_to_not_be_null("CharContinuationRate") + validator.expect_column_values_to_be_between( + "CharContinuationRate", min_value=0.0, max_value=1.0 + ) + validator.expect_column_values_to_be_of_type("CharContinuationRate", "float64") + print(" βœ… CharContinuationRate (repetition) validated") -# 4) Keep page-source strings as strings (so they never sneak in numerically) -for c in ["Domain", "TLD", "Title"]: - if has(c): - validator.expect_column_values_to_be_of_type(c, "object") +# 4. SpacialCharRatioInURL - Special character density [0,1] +if has_column("SpacialCharRatioInURL"): + validator.expect_column_values_to_not_be_null("SpacialCharRatioInURL") + validator.expect_column_values_to_be_between( + "SpacialCharRatioInURL", min_value=0.0, max_value=1.0 + ) + validator.expect_column_values_to_be_of_type("SpacialCharRatioInURL", "float64") + print(" βœ… SpacialCharRatioInURL (density) validated") + +# 5. URLCharProb - Common URL character proportion [0,1] +if has_column("URLCharProb"): + validator.expect_column_values_to_not_be_null("URLCharProb") + validator.expect_column_values_to_be_between( + "URLCharProb", min_value=0.0, max_value=1.0 + ) + validator.expect_column_values_to_be_of_type("URLCharProb", "float64") + print(" βœ… URLCharProb (URL-likeness) validated") + +# 6. LetterRatioInURL - Letter density [0,1] +if has_column("LetterRatioInURL"): + validator.expect_column_values_to_not_be_null("LetterRatioInURL") + validator.expect_column_values_to_be_between( + "LetterRatioInURL", min_value=0.0, max_value=1.0 + ) + validator.expect_column_values_to_be_of_type("LetterRatioInURL", "float64") + print(" βœ… LetterRatioInURL (letter density) validated") + +# 7. NoOfOtherSpecialCharsInURL - Special character count [0,∞) +if has_column("NoOfOtherSpecialCharsInURL"): + validator.expect_column_values_to_not_be_null("NoOfOtherSpecialCharsInURL") + validator.expect_column_values_to_be_between( + "NoOfOtherSpecialCharsInURL", min_value=0, max_value=1000 + ) + validator.expect_column_values_to_be_of_type("NoOfOtherSpecialCharsInURL", "int64") + print(" βœ… NoOfOtherSpecialCharsInURL (count) validated") + +# 8. DomainLength - Domain component length [1,253] +if has_column("DomainLength"): + validator.expect_column_values_to_not_be_null("DomainLength") + validator.expect_column_values_to_be_between( + "DomainLength", min_value=1, max_value=253 + ) # RFC 1035 + validator.expect_column_values_to_be_of_type("DomainLength", "int64") + print(" βœ… DomainLength (RFC compliant) validated") -# 5) Optional: boolean flags constrained to {0,1} (skip label itself) -for c in df.select_dtypes(include=["int64", "bool"]).columns: - if c != label_col and df[c].dropna().isin([0, 1]).all(): - validator.expect_column_values_to_be_in_set(c, [0, 1]) +# === DATA QUALITY CHECKS === +print(" πŸ“Š Data quality and distribution checks...") -# Save suite -ctx.suites.add_or_update(validator.expectation_suite) +# Check reasonable HTTPS adoption (should be 60-95% for mixed phish/legit) +if has_column("IsHTTPS"): + https_rate = df["IsHTTPS"].mean() + if 0.3 <= https_rate <= 0.98: + validator.expect_column_mean_to_be_between( + "IsHTTPS", min_value=0.3, max_value=0.98 + ) + print(f" βœ… HTTPS rate reasonable: {https_rate:.1%}") + else: + print(f" ⚠️ Unusual HTTPS rate: {https_rate:.1%}") + +# Check TLD legitimacy distribution +if has_column("TLDLegitimateProb"): + tld_mean = df["TLDLegitimateProb"].mean() + if 0.2 <= tld_mean <= 0.9: + validator.expect_column_mean_to_be_between( + "TLDLegitimateProb", min_value=0.2, max_value=0.9 + ) + print(f" βœ… TLD legitimacy reasonable: {tld_mean:.3f}") + else: + print(f" ⚠️ Unusual TLD legitimacy: {tld_mean:.3f}") + +# No duplicate rows by URL (critical for train/test split) +if has_column("URL"): + duplicate_count = df.duplicated(subset=["URL"]).sum() + if duplicate_count == 0: + print(" βœ… No duplicate URLs found") + else: + print(f" ⚠️ Found {duplicate_count} duplicate URLs") + +# === DEPRECATED FEATURE WARNINGS === +if deprecated_present: + print(f" ⚠️ Deprecated features detected: {deprecated_present}") + print(" These features are no longer used in the 8-feature model") + +# Save the expectation suite +ctx.save_expectation_suite(validator.expectation_suite) expectations_count = len(validator.expectation_suite.expectations) -print(f"[GE] Hardened suite saved with {expectations_count} expectations.") + +print("\nπŸŽ‰ PhishGuard 8-Feature Expectation Suite Complete!") +print(f"πŸ“‹ Suite: {SUITE_NAME}") +print(f"πŸ” Expectations: {expectations_count}") +print(f"πŸ“Š Dataset: {df.shape[0]:,} rows validated") +print(f"🎯 Features: {len(REQUIRED_FEATURES)} production features") + +# Quick validation run +print("\nπŸ§ͺ Running validation checkpoint...") +try: + results = validator.validate() + if results.success: + print("βœ… All expectations PASSED - Data ready for ML pipeline!") + else: + failed_expectations = len([exp for exp in results.results if not exp.success]) + print(f"❌ {failed_expectations} expectations FAILED - Review data quality") + + # Show detailed failure information + print("\nπŸ” Failed Expectations Details:") + for i, result in enumerate(results.results): + if not result.success: + exp_type = result.expectation_config.expectation_type + column = result.expectation_config.kwargs.get("column", "N/A") + + print(f" πŸ’₯ {exp_type}") + print(f" Column: {column}") + print(f" Config: {result.expectation_config.kwargs}") + + # Show result details if available + if hasattr(result, "result") and result.result: + obs_value = result.result.get("observed_value", "N/A") + exp_range = result.result.get("element_count", "N/A") + print(f" Observed: {obs_value}") + print(f" Details: {result.result}") + print() + + passed = len([r for r in results.results if r.success]) + total = len(results.results) + print(f"πŸ“Š Success Rate: {passed}/{total} passed") + +except Exception as e: + print(f"⚠️ Validation error: {e}") + +print("\nπŸ“ Expectation suite saved to: gx/expectations/") +print("πŸš€ Ready for production ML pipeline!") diff --git a/scripts/ge_check.py b/scripts/ge_check.py index 43cc802..7b42c0f 100644 --- a/scripts/ge_check.py +++ b/scripts/ge_check.py @@ -1,10 +1,11 @@ """ -Lightweight data contract check for URL-only features. +Lightweight data contract check for PhishGuard 8-feature model. +Validates the 8 features documented in docs/FEATURE_EXTRACTION.md Fails (exit 1) if required columns are missing or out-of-range. Run: python scripts/ge_check.py - python scripts/ge_check.py --csv data/processed/phiusiil_clean_urlfeats.csv + python scripts/ge_check.py --csv data/processed/phiusiil_final_features.csv """ from __future__ import annotations @@ -17,21 +18,27 @@ import numpy as np import pandas as pd -DEF_CSV = "data/processed/phiusiil_clean_urlfeats.csv" -META_PATH = Path("models/dev/model_meta.json") # for feature_order consistency check +DEF_CSV = "data/processed/phiusiil_features_v2.csv" +META_PATH = Path("models/dev/model_8feat_meta.json") # Updated to 8-feature model -REQUIRED_NUMERIC = { - "url_len": ("int_like", 0, 8192), # Increased from 4096 to handle outliers - "url_digit_ratio": ("float", 0.0, 1.0), - "url_subdomains": ("int_like", 0, 10), -} -OPTIONAL_BOUNDED = { +# 8-Feature Model: All features are required for production model +REQUIRED_FEATURES = { + # Binary features + "IsHTTPS": ("binary", 0, 1), + # Probability features [0, 1] "TLDLegitimateProb": ("float", 0.0, 1.0), - "SpacialCharRatioInURL": ("float", 0.0, 1.0), "CharContinuationRate": ("float", 0.0, 1.0), + "SpacialCharRatioInURL": ("float", 0.0, 1.0), "URLCharProb": ("float", 0.0, 1.0), + "LetterRatioInURL": ("float", 0.0, 1.0), + # Count features + "NoOfOtherSpecialCharsInURL": ("int_like", 0, 1000), # Reasonable upper bound + "DomainLength": ("int_like", 1, 253), # RFC 1035 domain length limit } +# Legacy features no longer used (for backward compatibility warnings) +DEPRECATED_FEATURES = {"url_len", "url_digit_ratio", "url_subdomains"} + def fail(msg: str) -> None: print(f"❌ {msg}") @@ -46,6 +53,15 @@ def ok(msg: str) -> None: print(f"βœ… {msg}") +def is_binary(s: pd.Series) -> bool: + """Check if series contains only 0s and 1s""" + if pd.api.types.is_integer_dtype(s): + return s.isin([0, 1]).all() + if pd.api.types.is_float_dtype(s): + return s.isin([0.0, 1.0]).all() + return False + + def is_int_like(s: pd.Series) -> bool: if pd.api.types.is_integer_dtype(s): return True @@ -81,31 +97,40 @@ def main(): df = pd.read_csv(csv_path) ok(f"Loaded {csv_path} β†’ shape={df.shape}") - # 1) Required columns present - missing = [c for c in REQUIRED_NUMERIC if c not in df.columns] + # 1) Required columns present (all 8 features must be present) + missing = [c for c in REQUIRED_FEATURES if c not in df.columns] if missing: - fail(f"Missing required columns: {missing}") - ok("Required columns present") + fail(f"Missing required features: {missing}") + ok("All 8 required features present") - # 2) Dtype & range checks + # 2) Check for deprecated features (warn only) + deprecated_present = [c for c in DEPRECATED_FEATURES if c in df.columns] + if deprecated_present: + warn(f"Found deprecated features (no longer used): {deprecated_present}") + + # 3) Dtype & range checks for each feature errors: list[str] = [] - for col, (kind, lo, hi) in REQUIRED_NUMERIC.items(): + for col, (kind, lo, hi) in REQUIRED_FEATURES.items(): s = df[col] - if kind == "int_like" and not is_int_like(s): + + # Type validation + if kind == "binary" and not is_binary(s): + errors.append( + f"{col}: expected binary (0/1) values, got: {s.unique()[:10]}" + ) + elif kind == "int_like" and not is_int_like(s): errors.append(f"{col}: expected integer-like dtype") - if kind == "float" and not pd.api.types.is_numeric_dtype(s): + elif kind == "float" and not pd.api.types.is_numeric_dtype(s): errors.append(f"{col}: expected numeric dtype") + + # Range validation errors.extend(check_range(col, pd.to_numeric(s, errors="coerce"), lo, hi)) + + # Null check if s.isna().any(): - errors.append(f"{col}: {s.isna().sum()} nulls") - ok("Basic dtype/range checks computed") + errors.append(f"{col}: {s.isna().sum()} null values (not allowed)") - # 3) Optional bounded features (if present) - for col, (_, lo, hi) in OPTIONAL_BOUNDED.items(): - if col in df.columns: - s = pd.to_numeric(df[col], errors="coerce") - errors.extend(check_range(col, s, lo, hi)) - ok("Optional bounded columns validated (if present)") + ok("Feature type and range checks completed") # 4) No duplicate rows by URL-like keys if URL column exists for key in ("URL", "url"): @@ -114,7 +139,7 @@ def main(): if dups: warn(f"Found {dups} duplicate URLs based on column '{key}'") - # 5) Feature order compatibility with model metadata (if present) + # 5) Feature order compatibility with 8-feature model metadata if META_PATH.exists(): meta = json.loads(META_PATH.read_text(encoding="utf-8")) feat_order = meta.get("feature_order") or [] @@ -122,18 +147,49 @@ def main(): missing_for_model = [c for c in feat_order if c not in df.columns] if missing_for_model: errors.append( - f"Model feature_order missing in CSV: {missing_for_model}" + f"8-feature model requires missing columns: {missing_for_model}" ) else: - ok("CSV covers model feature_order") - - # 6) Summarize & exit + ok("CSV matches 8-feature model requirements") + + # Check feature order matches exactly + required_feat = list(REQUIRED_FEATURES.keys()) + if feat_order != required_feat: + warn( + f"Feature order mismatch - Model: {feat_order}, " + f"Script: {required_feat}" + ) + else: + warn(f"Model metadata not found: {META_PATH}") + + # 6) Data quality checks + total_rows = len(df) + if total_rows == 0: + errors.append("Dataset is empty") + else: + ok(f"Dataset contains {total_rows:,} rows") + + # Check for reasonable feature distributions + if "IsHTTPS" in df.columns: + https_rate = df["IsHTTPS"].mean() + if https_rate < 0.3 or https_rate > 0.98: + warn(f"Unusual HTTPS rate: {https_rate:.3f} (expected ~0.6-0.95)") + + if "TLDLegitimateProb" in df.columns: + tld_mean = df["TLDLegitimateProb"].mean() + if tld_mean < 0.2 or tld_mean > 0.9: + warn(f"Unusual TLD legitimacy mean: {tld_mean:.3f} (expected ~0.4-0.8)") + + # 7) Summarize & exit if errors: - print("\n---- Violations ----") + print("\n---- VIOLATIONS ----") for e in errors: - print(f" - {e}") + print(f" ❌ {e}") fail(f"{len(errors)} violation(s) found") - ok("Data contract PASSED") + + print("\nβœ… PhishGuard 8-Feature Data Contract PASSED") + print(f"βœ… All {len(REQUIRED_FEATURES)} features validated") + print(f"βœ… {total_rows:,} rows ready for model training/inference") if __name__ == "__main__": diff --git a/scripts/smoke_judge_selector.py b/scripts/smoke_judge_selector.py index 12d540e..f0d40f6 100644 --- a/scripts/smoke_judge_selector.py +++ b/scripts/smoke_judge_selector.py @@ -10,17 +10,43 @@ ) TH = load_thresholds(os.getenv("THRESHOLDS_JSON", "configs/dev/thresholds.json")) -out = decide_with_judge("http://ex.com/login?acct=12345", p_malicious=0.45, th=TH) + +# Test with a realistic gray-zone URL that would trigger judge evaluation +test_url = "https://secure-banking-update.net/login?session=abc123" +test_p_malicious = 0.45 # Gray zone score that should trigger judge + +out = decide_with_judge(test_url, p_malicious=test_p_malicious, th=TH) + +print("πŸ§ͺ PhishGuard Judge System Smoke Test") +print( + f"πŸ“Š Thresholds: low={TH['low']:.3f}, high={TH['high']:.3f}, " + f"t_star={TH['t_star']:.3f}" +) +print(f"🌐 Test URL: {test_url}") +print(f"⚠️ Test p_malicious: {test_p_malicious} (gray zone)") +print() print( json.dumps( { - "final_decision": out.final_decision, - "reason": out.policy_reason, - "judge_backend": ( - None if out.judge is None else out.judge.context.get("backend") - ), - "judge_verdict": (None if out.judge is None else out.judge.verdict), + "test_input": { + "url": test_url, + "p_malicious": test_p_malicious, + "thresholds": { + "low": TH["low"], + "high": TH["high"], + "t_star": TH["t_star"], + }, + }, + "result": { + "final_decision": out.final_decision, + "reason": out.policy_reason, + "judge_backend": ( + None if out.judge is None else out.judge.context.get("backend") + ), + "judge_verdict": (None if out.judge is None else out.judge.verdict), + "judge_score": (None if out.judge is None else out.judge.judge_score), + }, }, indent=2, ) diff --git a/src/__pycache__/__init__.cpython-311.pyc b/src/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index 7c415a212a2deebb4354635a3e614adf700bfea5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 149 zcmZ3^%ge<81Z%|3Wbgy&#~=<2FhUuh*?^4c3@Hr344RC7D;bKIfc(!O$zK{SRx!Rl z`31!>eyJtp`9;~msmY~9nI)Am#YM?6@$s2?nI-Y@dIgogIBatBQ%ZAE?TT1|8bL-E TgDm{O%*e?2fdNJoF$2W_B-@M@fHb7aexTLXvuTUUNB}@vvN?s+@m*5%Ypz;j3Me#;LF|7W{B|>2o zc8em2$JlFDt2SuY9^SYQZ&J+v#as5_t;(T)iw^7H%dJk%i^tHC0y?{q^YEyR#Qpf+gGDVJsevsrvAveYeFAZ-0Ibf=yZ` zS+GNQF(o=0NrR8{acdVwaM)9mBxo`8u2opVB&?@3V;f$(T;uvt#I@g{K8bCa@mwey z(wxIZ>N&f^7O%e%pl=rWW}!TXfhr(%&QUV(#?XmAor zAtZ$$KZW?IvLI!;a-OV5>l`ec*dBg5T=cwf!=dmR8VW=EG_p@aVFn2^Wrs6Au63V6 zE+2IGk;7jxGlwk|q87}SA5BDyoSDdlvtiBcE4rW@L~;;{V@Mp+wFtS^Y-@g?a1u&> zB>5pfg7^`g-IeoZy_$2Va30zN$R2>gC=y1??U-*u>+vJk2y_IHBXEEZWDI>|CwOI0 zPCN0`ssmQtdgjtYBMv>3>e2H~98&t9?dVW>yp>obaw)DvR4*>N!4eZBWw6EcH|tMO vGELyaR6Lnb8*r(5!Dr?Oc?=lY6^}q#GS0l=UT=lK=|g*UI~e-GCVS>LZ!-7U delta 353 zcmX>m{D7BlIWI340}!kcJCni3ypd0ziJKkBWd`EUN{o{|nDQp?W0GS@VUc8*Jb_t- zC508nV1qH(!3;5u6plG8!3>(5lWkZPChunzoxG1#jFENnRaT|Re^?E;<$y-=0&#IY z!(?+dt?(P1yd61LIHen$o(hT2V47huhv%Y@>J=f?yZj;(f@ZKziM+@!e}!NEu7K!- zq#0sUvMvfJUJ+2dD4LKH6$O)vQl?j=Oh0h4^00kifDo)~lk3>E8D%H0Voy<& z2ijHy@?DW4h)@6#fQ>g%p@VC&VT diff --git a/src/common/thresholds.py b/src/common/thresholds.py index 0986be9..cf1737f 100644 --- a/src/common/thresholds.py +++ b/src/common/thresholds.py @@ -12,13 +12,25 @@ class Thresholds(TypedDict): def load_thresholds(path: str | Path) -> Thresholds: data = json.loads(Path(path).read_text(encoding="utf-8")) - th = data["thresholds"] - return { - "t_star": float(th["t_star"]), - "low": float(th["low"]), - "high": float(th["high"]), - "gray_zone_rate": float(th["gray_zone_rate"]), - } + + # Handle both nested and flat threshold file formats + if "thresholds" in data: + # Nested format (thresholds_8feat.json, thresholds_7feat.json) + th = data["thresholds"] + return { + "t_star": float(th.get("optimal_threshold", th.get("t_star", 0.35))), + "low": float(th.get("gray_zone_low", th.get("low", 0.004))), + "high": float(th.get("gray_zone_high", th.get("high", 0.999))), + "gray_zone_rate": float(th["gray_zone_rate"]), + } + else: + # Flat format (legacy thresholds.json) + return { + "t_star": float(data.get("optimal_threshold", 0.35)), + "low": float(data.get("gray_zone_low", 0.004)), + "high": float(data.get("gray_zone_high", 0.999)), + "gray_zone_rate": float(data["gray_zone_rate"]), + } Decision = Literal["ALLOW", "REVIEW", "BLOCK"] diff --git a/src/feature_svc/__init__.py b/src/feature_svc/__init__.py deleted file mode 100644 index cc56cdf..0000000 --- a/src/feature_svc/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Feature service package diff --git a/src/feature_svc/main.py b/src/feature_svc/main.py deleted file mode 100644 index 9023161..0000000 --- a/src/feature_svc/main.py +++ /dev/null @@ -1,24 +0,0 @@ -import re - -from fastapi import FastAPI -from pydantic import BaseModel - -app = FastAPI(title="Feature Service") - - -class TextIn(BaseModel): - text: str - - -@app.get("/health") -def health(): - return {"status": "ok", "service": "feature-svc", "version": "0.0.1"} - - -@app.post("/featurize") -def featurize(payload: TextIn): - # ultra-simplified placeholder features to prove the path - text = payload.text or "" - num_links = len(re.findall(r"https?://", text)) - length = len(text) - return {"length": length, "num_links": num_links} diff --git a/src/gateway/__pycache__/judge_wire.cpython-311.pyc b/src/gateway/__pycache__/judge_wire.cpython-311.pyc index 8f025211eee0fa9b75cf73f27d8c85951c51493f..87b276b6485fd3823c0404d61abbe872e1023029 100644 GIT binary patch literal 10182 zcmbVRYit`=cAg=J@3-EfUZaO?QI;qvwxy-E z+Q4n;g%_($Q& zZJamdi~3Ujs6Q2m25_F8t4q~K>r)NU2K?^e8dFWtCY*M1&8e1XOR6>6nhHjPIM2nk zrP`w%B*EGlcWZg zTRk-;y^z;e(}$l482k8;gx27%U(rLrf4HVj9aH~1>l~@!Z>VXrzoxvAX)?AK4Kb$Z z0Q`@_e~|6xc?a6RW^O1Ij)qgi(P5H+Hip@Lrsb2?nz{q*zz;6418k^jWk=W%CRp`; z6yDpa-Ur#E#bf0;S3j9{HoRh5C5gw6k}Q>LZ^O&dG*W|h^PC^wtol8N<@Npmi(0rOghV+760u!$stTJvDm7_=YbxD+(yIj1)}om^!F zQFC8HO)aq3a_G%ddK0oR3sl`vsSZ^w6SK4P-_fiKGp}5n`Hp6poSmP#40T?fK0gzi zoS3>i6Pea1LCh^Ds&^h8I#43y*T*4omk_JU81hYgBzc=yIhq8Kwu;mN6D+bRqM7y4 zdt-*)ct7xI-pU|jsPm7AuTkxX&rF|sZEiM`6<&+5;*AXd<|3QO@kw$0HGxmOwi*}N z8}aqm^tv~aJR8ogYo3_Eax5%9&gMZY5{>W|nt*H(d+h`F*xmL~seSZE&ljHmWv4`) z=aH7t>K8A}FD%8T=jSFaMq;xwk@HIzG}EyWt=Uj8H6J;5ae5{)HM3zJ37^m$bMuk& z^Rbr~E^2l?IX$U4vujCVZ8aC?8Q50G#NrgJ>&3`<%{7-vuV$vWB%2nK|7O8r*CMd| zb(UYmqXuzWN~YLMPMli=0R$NkkT`07eUdzu&N#;knlr(J7_dw%E^4HpnFW#8tUL?y zme$P4w3w*qEtnnJ47`C6NWcbGmF0=*jWeDh-Y3Es%U7N-p8Pb)kZ5Zko|t7<Y8Q0FMG&x{ zW_}bH{Z>aVdv=%JckviJW!E7EsQr*) zKeRQWy4*h!wvXMue&_n_QxesP2WD7!EA)U*(0@bX|067UqFALMp^?(Xm>9G1)G^e7 zHGV=J%OiEjgF2fAA!CqeQ03mLpN!Qgo{V_|(4=uJAdWUHj`*~0m`@E4{}!Fe-~KOL z56R(&WH@B0+CPhs<&vUiPQ`C(mP96(2Eo!dl3L3MqG2!nF(J3iWK!{DI)+(!w2d?H z7dRk~+0TH!p8?h9FIW(sg*(b+{rIDn%;AH3)i;$?hd>?u(SA+;dh3M6LK90!C}R9SfUQ=I>HP#_MsQq_*X%j4$@%q zUw^z16k&XK2}g}0WynuVwaeFQP_F=bMo5F+sMP_#OFxJX=(cp#)?>`?nG>cfaIVV` zDnxCJ&gejd9v><}5jvG&SdK0O6ReW4X`vIDv-&8~p&R<*IFjpOBw)sGLo!e>(S z%L}uE^jbX4aLM#4&GLMP7wA<0qU=hJkY-J@BIs|;!ewp%-xB2UF9WIG z6<=jn^gj4kfgy^Rrh%X>FRpkzN3=;sJ=C%r4DIe zov6M*fiLj;X2?M6?1qNkI@C|xbB>vRL5`Ud@Lsu)rGtz78sUJu1WQCohKQP8hNWQ5 zjOjGwGG<6oIAwv96{l>Vvg{>LJf9O2nG~zpr%OOe^I`}dD+4Rt=y-Gl9s@unWYQbW zWmx~}B^_DoSo(F%f=@~Wya9VGmX4>`SWI)qVsNB#97?-mv1_?FS1R$vVpR@BEXH@j zkg=k{c!qx+y&|# zGLsWDYB`hP64f>bEh(1U>yW6mVMeLlH04btkW<upGLyH+&S4>QfMK8d> zi*!leS@3x1L|kCQPTbT&85f_T7cn2l0;Wl(03esD2y}lsn+j+6)j@hWDTcZAL7Ej4 z;SimQuha2tmW}hcDIq6h^%X?Dife31WtN7)B*8Q!mvacg!kCgPVwhe)BR&`~z+BwY zS}qk&4}nlI@nsGH;w%?W)i{%fY0>xaV`ALXaZB7>(~isXic}_K~*c zSAc-)if5$RlR^ydn$`pe z>w?W!_0*j1 zr|#YJ)STX@p1V&B9ITmGe~_+Sb)!5yHII`WRI2)p5nX)Fb%98A-u zlWE>`D+XQ-R0I{eX9C0j-v+PJ0B@U zYO{3le=j2aS$_Ll@o+f}+VpQAR%vCrxQ0~2rW^DE0k-w%L3yn;&Qp13-ja70n+|vm zQXfL{Tzh~R@RQFx1+WQI{k@q|@`l=B=p^jl~V4YgZ?LoRC9gq%I@}G9p-S63I z$CLI0Vn*nxbOB;EBgA|VOEzdpXWo@}JgOx_J@MORTr!ge5S;)MSVoC7msw3F=o?9} z&juKR>=a<_5(p`SO!^$aYTddY3)2(XELdP2VbaTh&H;%)`7kog2bR8M*q;g07gvA_ z;%Al4!Bp$v+ChK-2r;2Bnqe3f3yw|nD72Bjo_Uik;V0-4Fiw^Rv(A8Pi*O1C!Id6# zOi-EiEShW?oGezMEOJ_lm8<`;y^UpMd8BI(u<2YHlYQUW~U(#S%Q5p@|opTV1n!Z z3tGbzfG5CAuv&9XIT;WtBQk> z%>2qcphJE!3%-S3UC^t#bwoFd0UiGmbiywnvIr#P;;*4J@*H^q5fn`4bBG{R&fh@f zCL%QzO&DQm_OgqtIcH}kBC!`QTwJ`sqqDoA$B2hG7EmV+KyP$l1U-a>Wk|#iNz64zMZPo_9#l+7WO$ZeM{E1Xh~R4N zJhB)(f;l|0M7mps049z>@Bv6^=2ceI5iiERNZTAlGW;g^H1Ht62f{ zLcAZG#PT!*&7}uWk>Cq_9&u6VSZY@ARnwUp=#vmkNsMLEEdK-4(p!kf28RuWXRxmx z@*b=mpF%uo#N$DFqtX+eqvkA&OKco)x&Fb(%(K@t(^X!x=-;E@s{!(R^}*PfOaiqb zAQX(Td4*uNRSTf0KxhbQ!_qgFcYqbGsL%$u6X19s(RHZ!>Zd(&=Lx0r#May%0>PfTzT%V|=vM;$TbF+03EuYy?|R?%?yTSImi>c@ ze^3qfy&rxzEQL;gmXL#!N^nx`IP(4*@4g`oPJTWmcU(|9E~uUT@2B5QOGDG2C*;mc zO6MiDq4RF!?MU(YPkZHtF{NQlZS8%(^W9FV|I}wwa_d>8^{m=E^z$=6JtI96kzRaB z?p;uN7u3eCyKlVxMlte>sbBiOFw5h!%J{5&d`>w&CpSiv#z4HKy*xgrjL*r< z5v4h@-{$pNzaoHqgUHsoZ+i$=^UmnKCdoM>J4Y1f$kx=Jqi%cjcITbWt;u~mVRzoz z*nVx-)-Bn()nIqQq6T^kR@K!7{*vQ_+IYNh?t$uX-3e~zcOAWwqgVAeKbGmfzXj#@ zR4N}=8%GQ0U^^_{!t$M~GS#Y3trFF`M|ldYOf@J}gG4nzx!t=@1TBHT7{HI8wc*FU zsnros-66?%gQT>ysa*qmheG#y{`$Kgd{^xqQG1T4{ioH=!|!L_&B&d@O6Tyt&)y8J z0x6vTwvO;Lf9dYtb$1t!$nF8fJs`OU_S}%yv+M4;-`1&ib-%ZvcK3eJuJ#`K;EYQ5 zeBe|E&#FBGKOg?-u-r4M^o&A_0SH+E`O56{c)uZl>{rNxZ<`2zD_Sz-XO@WS_7|oh zvgHUsWDDl$ZYkzIJNHG8$b6bTf3Y82$c z!6<$NrH%o4H1}COch1Ngd2H@5%C+ix0;REQ&Z8Fe9*}M1pF>Q}$1-8Hf^-1ZEqta@ z5e1q;&!M>2N)CP8hQJo`HZ@ZQbX(<%Ip{XzA8S@|JqrklN1i@!MFhE!yd9OfA#}_o zm&5q8xgMBog>W6H%y3C-l|zu|YatFRyG!Npxlj~zeT@_?R^M8nRCr)Tuq%%hycQ_g zfa{4c#AC8x&Gj$O^#}P##iGyl0#^E1t@CKd%3r4isEfW%^9`s=xp$@X{l!4@-CO&Dc>&PfzlHf>{3)kv~IRw(L4K9(+H_3ZX1> zj9J#)`gh-0GR?3z_0VmNJ$eOIz=jI#6_0HhQqeU5$cOgRdgSs9k9-3_zl3lFwi1Mx zBvqnEGT!$I#}@vnM7LzTRbp8BefOia?h|8@@m7hG(vxqM7?B=#tHe>sc&o%E>B+Zh z>=edu-&H5o55dQxEjy^3y0-1QcUpH|xOYwtj4FZAF9Roc11IIcb4uX3E$_aSBrlL~ z?hmM)@B|BUx99K7Z#kfTb8tJc6WZx{FC_Jy{Oq_KJgWrH%1skW(}YCStHG|FLq$i? z^npVfnEL!>nZB&hm*wEB5}cKYW~lFNE-c)+vi&{T)2n!Tx16fGu`qe(;`Y_zpzI!2 z+{0TA)#2H)s-6H8dh51a_syPt!b`U8n>>~_)&7k1xch#feYKD{U%IwSSH zAiK{h?z58ntcnzC->$o_nE&*Obod47&{^3%p|~d`_r!w-D*8BMwt$W=6E)BXy*9QM zoN80B;6lHB8x(@h-1nNuVX*TC;(CfB~n zO!mT%9wA?$yl)Ws$_#wBT6Io$JsOgc94Gn*_Z?SBG6JITv=6NQ|M|#7*Nlz$mCZHN LX#Q0L3H1K}^){F# delta 3213 zcma(T{cBsvai6}wKP=g@<&*qu`Ad!*<&yf2JLf(e+p(QEc4{Y=i>^e-+80}i^q$!F zbooHW>5(5?DAyc&1ycfb3(ch@(3bRvw!ak0rB`U7wsDDh5E|$oP#@QWe*DszCo8dB z0$teI*_qjyo!yz){r&7Ohx$MBdR+wC!dz)n9urhuRm=E>@2+4FD z(B``*03Uj-WyAw~KMgcm%XCsxrVDxkdbcDp=0tCU#hy}crnl6W=@aG&DCv_#+Vn7V zU~!k!_3cTiOX}R`V87H)!~5n5m^bg6cT0)&B>#mo6eF}n>b-1Q6dE@TN7h}F$h}lT z4km~+&`p>V4tt*i7l~}SMy{JLl4}C(r|q{bnL{*5V=$&*9*1#IvY5yOp&hpg6}itG z3mpcR2k0T%ciWmtQ`0<2rua$o=%b&S+XcbLA6qV2JK#J+yx-bki2?TLBdbL?J;U(M zl4jjiPz$oE<>b7?27siSi;|oaSQ2m*??Pm8Bs;PdR?GsA9f9dlEi4wa zEX!#Hg$)DeHR$Rz05leGo)Th@K5)7PVTNCI#cVar{9{+!8@OlPvZSb5gT_Estt?Qb zlq<;Dyi$=hHV%Ssq1NJ_)@QCO#^LZk`&40okNU5+VI(u$z2QHbKn%ec|HA)Hr@_Ff zZiaKw-Q{d4S1jZUN=4O8+7k2d3!$TK4+xkE5x#oU!aoh&HEC}Cr8{cUytToPgQIFL z3k9<7z_4l~)Y=$O`?>rrSMZ>?{pI4I%<*#W3Zc$M?nBo-2P7c$T3fbiP-N<@N~+d} z?#3KiwrS0EJ|R_GqrPQ()qcfY=TpyBqf9U>^)5T8@A$G)4mSw+p9=&)AP9U56##2m zc4{wjkl-4AE7G%az&U|v(Yi6XtDkB0S_DLPU&AQgYwSmmG$9%;21m~q@Gd(^)!nEEcDQdBHL2Ip`o3cV zJY2T}`19ihyyZC^_JS7BdRxog1II29IglXQVeV>6jytN(s{Ntm9z@{-=(Pq#B69P4 zE)m*Dxm>&~m=M#tyQjZDoTt)|S-!O+2*?wZj-G&++s zvne35QwUB2(1Yiu$EGDH1cg#glg=`evcPY&$A+8d%ej0ZR~%i+u{kUeQ}X$_X+4bd zQAO4Yas{a}@Fd+0EM%Nz%0kk?mplV8_8ywJfZ%-u7ZGFtKsh?g|K1*5!6UyJxKg2u z(qK`_$%Eykg1V%;495*08Ky;pND+5>(_dRu!CI<8wW-xuL3dEt3^*bsZR{-C`z8QA zSUZ3$R0~O6Qi@d7{XpV>eZ}iF4x=BWa0U0rdS)--I)l>6yoh6B!Fvb5C{2FEurwPb z)2MK|c~R0>0BE{1d+Pky#CZ1P45KLH;&)>`xg4@Aw2;?yTSd#u$~AZnWED1_Rb+`t zD0vxW?0Gm@>AI#n$FJw5GM;bJ^THK2hm!Ls>0QXFQkF`2cvcjd55_x1!-}0Mc_gb? z5;5@Vjm3F69tmCtGTysdh9Yr`d>!U%@tAqX9_GJ^zrNyYe;R0A3w`4K)cdrxdwqVh zHM!lId>ZMyZ{Ccgwj-&X_Rh8A>vK<{sjX;gWANS0=*V_-WT$&zefs{wlf=+gVrb*F zxy{7A|Z$0l}~ z+wXj9t&2Sg_ilxI*OxcLL)+n@-JsKF`;q|o2ZCECVY>k_F4%ZljP$J9cAW0-M(JN9+r?T7%yo*d;Lhq}>>HO%eV_F%^yX{!saW zvKdcr$J4uBd&IX(0IW`Y9pK^4p~0?!&(c3mZ+0Eq?m7nSU=y%|O<$UwULS183(|G) zPuSUw&f%5Ehfeq>hRGMhu1Saa4|V|{Jj(c98jmtNgjfo||59H?);Eyls@*vL5{w9D z2Ftj6yj3Fr*g3=ubr_b(Bf2Y_z06m--aV67N+m@;41WzxtpzTAW{o)3gL}*uxxfsS z80mV@NUN;+Vg7KA=E|DHn2*1cXj?I29^;z@QO@^5PSq+44C^(+&nlZlwhgjDl0Vd~ zMdcc(z_)_o*N814*OfgpJ2NqxJwG=E*;2#DPOt(BTtRRZrR?P!;KQQA@U{4WbtBE3 zr3<6lY(b{dbymg&jmcQcQVes}__O%8<9JD-m7?@E`wnpN%ccI>#OJ%;7oPEr z?#KKuJ^wl`2FV diff --git a/src/gateway/__pycache__/main.cpython-311.pyc b/src/gateway/__pycache__/main.cpython-311.pyc index c9ee53cff214c3ed1bad384d423fbf18e7d3693b..68962cc6281aba00521c364357ca3cdf887792e5 100644 GIT binary patch literal 16904 zcmcJ032+-%mRL8?02((z@Bpa`6h%sucu1tI%NAvdq$o-hDT}l`qOCz7bd!XHgYIsM zB23Oe8f6T*G-cYWDRDP;Eq62H)nt@JmC7bgRd!=f#+%t>lIqs>M5s=f(q_xGan)7| zjy#D zFd2w9Og4~rXRI;4X>t=y%}|`{AJUVXxl^3)CNsHZn&PT|pPt;xo#yJ`eVd8mO_zg( z?s4^8;3E@7{S^N7XR-+>wpUO!aE(T)k7&pP|M@)4+@_mUMV`&v7Gs{(9R>RGUv8_h ztO4pHNd9S&WU>Xyb#U9b;Qc0Jd6TUsYKY>t-=w%^-uICia>2j;Ozz}cS9cZO%76Kt z#`VSX!p|Qsee3P-f$u@2mzng1&35ZDhc0>F2 z@SckH?ICIHkk+v-t(~NGLR#0lv<_~B>xOyk++q0b zhF*YJ{A2ZG%cb4nB$yChu z!ROX~yrTC0b+sRQZtVjVc?Z|!J^b9fCo1xu<%ZU^`p9!zeX=6=@Vc5`HNL1p5a7<&zW-&b+1A5zuuqgGN9fqi< zf9%}&nJCA_`1!EFx0y8i@vz9BNpgHlvz&@7YcGFcxfsYy{4x~P&o{d{<4sqljbhz^|_|P5&lv#d4(joV$sAEo{Q@ByXNQT zJBy8U$D=Q&c`xgZ{$A=5c`xf0c`xgl zmo%lUdjwJ5MS>{rCP9>UwF=8HbeCWZUG|AmIC3RxJ#mC4wmQ0JLRjgy(!=_ouZ8j{3e2T$=)u(n7z_=Jo#`7M9oOuG*CIUO5^aot zq99+8-W1KP|ArFD5L7JUxCP;P);@6LjWZ+3S@DfgUYbt|SH}6soDh{3-VlYz8`IF4 z`S8LU*qhGT1+6L+!QO|89TWPYpu_MlW&otA9OJaqX4y8`7#{l?mIEuz?`^rWMfJ5j z^6h!(+oSk8R9{EN`Z%!lcRj!3S-pBsPy*d*pgZG!?5h4%U8n5o%u%%aaMru|SHVNF z_fXDawmY8wnx$N|kYFhTS#M2dE;E-i1NQ0Dr(!iU=0=~lubTR_+R+y@f4YqZ*v4wU ziIIVk;yj%dl9#l`(d5|Fm^8x+R z!Q8H1q(L^BM%PvC$BkSABzLJOwMbpGKm{~M{g&yrb)5PMJ^B;KrI|YSYV;Km*LpB0 z9EYE0j+aPJlJVKzR12&=As&uJQ+!D8bmvR3$ip$Qx3fg{EyyM|0!UM`vqujrbfzyO z6A;ad?g44n#z)Tf3>9>YE+y(sK z8-uI9589TQC?h_C{6QDoIE!;esXMNU_68}H?=8}OiV?m#V(G&+EtRM zotMRALUW(#`&Q`O;47iy7bXVBHRjO1!`J??MPwozU31;UQpl zsiql8nmy9hMOM#DQk2^Bh<)7$JNLrhzGrlMs`~6qRGb-_3kzIuNatiJ3#4`KMSVw# zN-pS%<+EO?%pgryF4AMCkFGtu76S|jd%J|T_6Y95Kkqf$w`ir{i9 zJPbs>hx7H{Aaa_bVFNAFmkX)XYPrNLVmaC<89{!Tg=356)uMzfl0w8d$HE>dZ^a_b zqS;uI#Flbi!M0I?UvidkoQX3>X^y#XF-p*Jl-Q=ligtYuY~r`f=c&8q?=!$RtfMIp zEN&c@4&Z&$(u`*1cqtr>rRvV{Z_e?e6ugw=7J^bT8H|O6X+HWP^dFX(6I2XoP7Fmr zb#1e1wi!Om!Nh3hY0yb+QIRlE%^Xcg!t2;(HvrI;8Ig}op>!l=Yj)rzu}}hdi)M<% zib77`_AD0_XTwrtCe{4>sp~8d%|Zp@bpZbZ{(vL6Yi@@gxq}bgLAhz4;_g=6-5FEX z2|R(ezeX=_zq|YW-S4&EY1iL>+_NhD-oYOo{Kpp`HXo6jkF5E(tp?;h{c>wLC$>+*$_d8KxzTDw#Bt_$o=vJcQOU_+rqIL?Pcnj;j7C%L&8#$BP% zn{(lqo+1c1IJj*|F4A#f3SSW<;a_Y9kS_k^OpIk$j#>}&{FrN^&8#tJ$Hpf#^U&ag zW*a*@F+2te2fZ8Gs!M`H(xv>#54{v{yElaJ(%{RAZFG_RRr{4;OC z;CmhP^{{kN7y#0|5-Hq-c*nAj~1aeNIZ!yrEE; z*%=CLl)rKvr7<^xzYqW79|I_@BiJEro%d?*Ngta(G0SZOO51?iHXt*#G6N8U!VF~h zzVN{EfcyAOd2gSxw@=;MCo@}R1|SB7>C1L>-+T4J{*U`V>6bfBDjg@)j*~LeATt0l zD9p)h`@VaF4|aUq^GT1~KBTk{sqI5D6Ob8z7!+nG+q`S_g?lv*nm=xqn|qb!UbVS5 zeOhLE^`I0Y`Vz+uZM^zsAZ4S~ci&vGt{9Gib!9281uJLc*!%X16>8-iKtxrwWLqTOp6|rLPVc5tl2PWIAZY7 zPbLADPZ~!9Ga$MxXNDLwAJ-GAS;Q8S}5<5OJ)_;2EMU6%02`V0FK!-u| zOH~cP}#W^g|fv4!=Ma@EdFR9(dYC$&eF|2lm2UAXzq7i&4 z+H6p}RZPwa5x&h*IsatV@4y&VPWZcsL-0fR7kL2X6P{&U>67V`*H31dI++B8smn6^ zWD*o+UzP!VIDPv1=`0hJNl=(zwsm)UP-b@PL1A`hnFBHjN+F_8H>o^?Erpn$Iw>XfxtNE zC%q8K>$%j$z&Eu_K!>Dh>UIMM$e{DQTEt+^tXpj6MD2uORZbStEMz4D~t8rCw<`nqujBSBcXw$c6$_$3!7q|((_p$goZDpUY7E+rhIy8Khka-j4>)nIz99({3rAcvaR-Kzy&Z*AeBWLSFXX~nXuTODys?N@PZ>r9H z>47z;C-dg=4m2HB#^39@)3w^CRKo&jRh_Le)4G1HZO{TT*PlRl-?pF^2*wndf%qg&(mgq!1;F&VKC69Lk-kQf-fZnF#rM!H_YSz^OwkgMMP*y$Q zZDsg;SqUfL?PYb?^L6|NhP=cqGMr<>vOrd^FALUmS&Iv!Q)wx`W0CBow2Wnn=3F;f z&i$Ktq=4g6Xmh#cOiM7fg@^NitqkI)bkb^kLtR4WqF<~+9@L=Vd3@{(4t}Pm$n&3% zeQ_4we&@IG`O{<5J>>bbV|mQKI`#$b9|5I&pA;Ax)aI?l_E z^DF(Cp5>OzOAxIHI=i)?w;Rkz%?du5=+uH{otx#*v1%33Ck?)ua75pdsOz@bh>Rty zmh4dhl{C#9PAmw>@nO416Kz(aB5Mq&L02?-L^E-MW)eB!p8w`vn2g0ZGtxVana1CUTeCN;G%NP4s(q_o z%qutM)AJQs-Za#lb$Q<%erNdB>D#B%gKIX2VJezmg;&LKUIl17uiDO|lmdT#zIVv5 z#2@yrm{%s=8@)5CIGa^xv&=M;`FrLuDyGB^iIizzR8ds@HDEzh!RQvA%S9&k9KllI zcv>=*t0^3?xqm=?Xflco8XQ)l&p=n<41=o_;I@K;Da#@irI*YGS}T^x*%r-EGE-S{ zIperoWLzW%`=JRL+?Y*$k!n z7i}B$0geutCEIswq%R~kPZtjoYf_Q}Sx-NB%z|+E6eLDl5DqM^Mk72pNQ2<{43-)3 z!CBos048H6$$5^PyNcwz2#+HRS?3brtKe}BUyAX;q=3hhydWe6JrkTSj13HqgvQVJ zhv2v>RsZ_H;PF?7E(Q(gBf-?>Lc(a$m$buc*<6vByV9;(__pdq&?ilZvdMIyW?0zt7h zBLNxYBf&PeW=e{hjZa*S3dw|U1uEApvjRBjG&b+27ByxT^iCm)X;nfvD)OPJq!7}( zEBqYWOSD(Cb_KKwg<}A~TLVH}_s3{vrK(G<>Pioc0wmMrF^)?3p$HraMQW_@;w;_;~skqU;}6{Nt*BJj0UQFW=jx?(UU6FGEn+ zK9%j0*}g1W^@weH$Tlf#v&uHhY;)FI|L)Q|OSisz`@0#2WbC}h$evCJ3frZ!T{7F1 zt!unH^Zv}rl2X^9)^%i@;JI?u%g(JozWiX*ABmq0ejHO;Pphq`0k!7x$<@am>;V8l zalNd%UY1=iXZ`id`?FR4<+tRjHV6+1T#e|zI*w!=c<`27Jpe&*4XUm|*)>SAevU(S z^nnuq1cmKY*}0{pDjG5C`Cvu)Lb2hE=yv_L#%>jL+D zOqz*6j^cxwCg_#)lkE^e%Y!p*EWkPQ2WHT2nNipXS2SDx3IQA&$RPMTG3hLDA{C-5(#h)GkA>l#-V{%HJPzb-q6F)+hK|A1!e2djo=qbf|A z9fXKc=P36Mar6gxDa$kpmf>#!rRFX9-a>F{Gz0kWAl|8;=7!GpO`KfUUcG+dzXIV? z_!seXBTaqYxcLt^$)qa-V%Lzig84iOq^@UoamN(2I}Hy zR@!Q4Z(g@7GWmRU236;|;||nrP$FxeQ-`U192e>Po{wNGep(z4P|?c90QypDooUf> zxoBxZ+*PQdyu`n&nS}a^c5T|k*%N_s%r}|0Mf3WV(B@5DAZpARyeL)DZGwD)n@xhW z)HR5r)4JWLAhid>2`*SPM1xbHfn0(cZn~B;{Z`FC8T#|o&10H{xZqP&`emDrVsYTW zP0oY+CpryB)DlWwLGw%t;e`-fQsnWb4iE0X$1u*qpCkD12>t@VsAew~0wyRC$Unfu z_Yk~~0QrY-2LbxZH4E9=n!VUq@NAAiLYs$Np3;jT9&ThXBnb|BkR61$x%jMp21Aqp z6wt&b6{@jd&r}p{gocU-3GUe>9QZ7R^z#JL0${$#_V- z8qsW^0m1zZzGM&QwFFy!T7>@zW&B_G7w2G2f@a{N*uaYI1E*r!q1xakjK|_w^EPCw z>hCzSfz8?4%~^lT=k*OMEq4~y0*xyNK5(oxZeID;2kmRycgQU-sM`;Tm+ap4i79>J zvCX5}8f9DS=T)`K(`sFtQng#H+MTgvkMw2C8S}5LR9)M>`um$6OewWT)!L&ON7h*@ zJDajKTQh^%>V`~zrXPrEcdQQm=(JL^SFPEL_$>tM4X)aLH;f)JF&H`cwa!> z+6^79fleWCQGsoDmgM?&6w(iN{_~DU-KQRQpHjL<)b0`R2*XIYtS}NTD~yE83L{~$ z>JrY{g(6gpz*H0;kPEtm*a{J&|JTSpOMZ1^nU%5&?)X5-LJpD+!6b{wVnB2`{kM+X zE;^o5F1-O=AhvjE<7TveO*6pe2Gt>or{Qpn@Ml0h0UN>tzm-iQcS1y1&QO=|zkuK< zHVnBdG>oxdAG$e|9?H6^mUnxU-rGL#85xPG(l^q5?BEecZTMe#9C zGWtBei9di?wPk*T%SA`%WpK6Y4_LY#6+RK{p~GM;S9ny=4qiV35fj+L&L#7AKrz&R z%Pe76YMclHxHfqWZw46|eXw#qO9BE+rr`L=zO%vnOouN;!Og#*ivZzM=tT7Yz%*+X zVw$CbXOO2B4LZEN3-|7Wy3PjfMeZji-6FFA3a>B;PoN0Gp7TcQj+cdh3#e4Bv824^ zH`?YB{seRUI|OL=ky}c_pJMFaBOt=!KVa8)CoioVc);R zJpT!S44~p(lP%wU;Tfb2;1;m}c#i!>3^g;d>rL6frqyY=>lL}>T+VDl-V5L>1bDU4 zkz-vJ$LDB{WxWB}wObF_>iU)X4>rlZmTXNRXJcw08-TA6WZ0a8viqKIhRN(!Fl79y zbF1p?!poFUQvsA3H1?VJi+!J7{rOe7YYaon*$taV+PDEAHjf#KMVkkC;=kx)`;Jkc z9;@l6%s=yQKmMZmXD`qgf6)Rk#TFR_&=w!uYJ^c#6TKT*R*YH#{dE<_cKstUC7e<7 zybPu251hS#4u>9ijv+BqIa2@^r0JXJQX4s%4CR2^a@AU>v;5b{D89BF8=TIS>7%8R z{)XkSP|n}7J>RS+f)8fL9<#;SB{TW10Pgx5=K#*Z{iXJE4$jHBzWHbaJ98Udm>{}5 zse^=Rjf1}qymjEeGcutd7ZztOCGqCz$%!)~;0c@N^Na}v_NZ+kA9awF9 zw7vV`_HJeSes%kP*>M1Z!W>kYgEDiFkQ`cVeAKl6Vbgx4>44e<>4zXF%wd%|EHj4* z$cw1F(tIW23PS`pq;PTedkL`~RoqBlalyYc9Jv0Ic3^il$5e4#k0AC>h z4s3T2)d_?R5m5VaLS8^{003aoSP{+?v3Su2u;e;Ui8j6=7R{FRWddYnS(E(1W`ccM`LFfxG-E=jR>d#k-L&w1FS_M#=|FO@TFAYGrLamT>&b{ zn!l7@TG4Jyae+k@2Uk5VW>HsfQSH`zd3?#eMDD-}=$O!|^UktPa-$1AtP^-*>IuD& zs#}Y%VeA4DTj78Jyv)LXK@1MRelr@`v*s$lEiXL6#3{sk!W)&Qjaub>T5(Pm{ka)g^yjcuaZIm5m|iYT67RMihWd1#%~8Hw_;&8NqgY1CSMfOx5NZt#lQ9QDFcz zlsAyMbUP{s+T=Ya70HQt#tnd;0^W*wPj zsLk|kokz+0N zF>F#L$Sh!sj$w=PNmUjazUx^JmhDK{ij?{EDjR(;N393T4PPSmt5Q-OO@o?i1S~_> z!F_uJ@G$gF#FYb2fCk5k5ggb~!!e5yjDX5xdAt5beWpR7{VMIxnP~bHjn$r_zcf>( zni7uPfV3AWd&>!XbA&E%Baqp>I++eA^j?+Tn=>7x!4**em|t$$d#29NW9^xs=XP z=tMhr+BC{t$EMOLEs%93t-4WI_n~%eRrR0#k*ukdJ*m?+390{VO1D-*Ow)GGdvSuD z?#l1H`_B2@bI-l^oO6#e7x>$cyMJC(O=3il$VTT2BJ7%-}0LsT<{I*U#7h3QraabNkdlu7bg4Qm?j8{pkZ_LEKU?jrqUE4;g5 z9UE|?nz!W|NVZ?#X27@`%gf-n!Hp}}Z{%Tjtif&q>}J&4u~TiE-IcrP0T^v;sOTWyeDk%Ua)EDW?-g(4 z8`+mYx3gx?_dxNlJZ-fh08-RWn~|U*x-Cn#v44BoWmB;2X2-hc_yfJr1v@ ziVi3BXd)hoos7oyQ?v}_UL@s6Du5_Xx)=NVknBfNh@_P9rQVLO!RK4^2%?cZ%tECf zS=KzlJY~22h6vkXh$$L7L%Q*3v|sU~Gk!r#0}p<|tm3F=0qIe#yEr{X(p*fqqM8ixZxnTGkpatOl_hOG&M zjK+XD#?Dl>TB{g_p+5FjqQQ!${snaAe5sAjb!--T}r<0%r6=^XRuPSFp{Dao2!Z&aD|H{LXeo(uQ zKgm9+?JdHEnR~5>w4T)lDrM8blWbR@nZL+Lpc}&Ag+ON)dX8c@_sKNUoW!uj5@%KL zut@wIeF_kUI1!C2G7Yn(fVb@#v_S>}VH*tYHe<&q4v(oR3UL@64ksrfF?tCAPu=Ve zHt^lJdH39VAt#&!$88VpiJv<~PXKcmcRJf_(1B#Y|LYwlZ|oZt(t;*nektJ6oNmve zI4;fSmCj5g|7rTZzRk8x*t#mThJ&i#N~z$L zhLi;7G(0pC)zt7vf`&CcqNg-^nd87R)0KO1*f0(&*!SeHYUW1pE6XzWep5@y5C}Cq zMsylzwDD52@1d)}cmX8>Y|MG?D<`+E^heIi?n~~UOwD_`7Cc?&4=xs#Epfc9Y_YgB zqc2It(5tHXRN3+N)_J9GLFt<-3BfO8XD>Ae#6^dP4Q;6sG7@vOw6nu44JQ^$D=u!n zQ(SiOV+YnK98SbXqNDT;0N#d2^8h)|-Kp~5s-7$C z!JpJ)8SrC}z}@MFIg*Ju9Nk2Ms4+SXY&P9mzJ45{?=l1z9-!5z@BXi8J{ug5#`!2W z$ro^hCqhh^5GUcd;wAN*?#K=`UBL#f%uNc9ad9c@rl4>o8lSGvi&pU{fts_TR3zf0 z6z@+9v`gU90`Tq3)$R*GFg^{>q$Tgj2|vWZ*b0$IFt+^bWK*V0FU^^8C6A=V`-3*k z&)Q}rsL0^4+N7-GCBEFyrj@>!md9NyU-J0Mn(o&~WajKBJab=SAa-AN5SN2={2zY7 zDxWy7NHD8AtKhsqxOnCLoYlh$rlr}!88G+8ipfx|B)1M(3 zLW0@BN(gv0P)wiT?y%wXH2bi#z63SH%+ts>%aW_?tIozYT!fi0Fl9jBurH@L`YwF^ z4jygX%pFS(_Ig)fYDt!C^?z|}o?G+F27NuoyG0gj*Uvestsh2yJw*PJ%sGk|>{WC2 zrq4Xyi=zwW%Dks_!PAijmN{?Bs}-}=Z;s4+dltMs87KR1 zS4rqjS^aeXwFl?Rx);j2Gy4`xYBHfr=#D2aZNFAH?`dD~w1YCYFC*S@o4i+#-|Bn& zi8=TFdH4PW_x|(!pRvAfZJma0c1>>MZn$=czu|X??6=)}dhl|Gs-+lS`Pl*Lv4uz%9?fCwVImt_#+4c>`yRL)m7^pyh%2!cnst7~UP<8qN0RGOg-*2dy zYKAUUK<3H#aa;`&%#y&x8pf*5OfKFAYq{tEz%}oL$8ms(HMvk)tY|Q2U@P(yI7*{L@gwVJag3r|hGhLH int: + """Legacy helper - use extract_features() for production""" return len(s) if isinstance(s, str) else 0 def _digit_ratio(s: str) -> float: + """Legacy helper - use extract_features() for production""" if not isinstance(s, str) or not s: return 0.0 d = sum(ch.isdigit() for ch in s) @@ -59,12 +67,34 @@ def _digit_ratio(s: str) -> float: def _subdomain_count(s: str) -> int: + """Legacy helper - use extract_features() for production""" if not isinstance(s, str) or not s: return 0 host = s.split("://", 1)[-1].split("/", 1)[0] return max(0, host.count(".") - 1) +def _extract_8features(url: str) -> Dict[str, Any]: + """Extract 8-feature model features for judge context.""" + try: + return extract_features(url, include_https=True) + except Exception: + # Fallback to legacy features if extraction fails + return { + "url_len": _url_len(url), + "url_digit_ratio": _digit_ratio(url), + "url_subdomains": _subdomain_count(url), + } + + +def _extract_domain(url: str) -> str: + """Extract domain from URL, handling errors gracefully.""" + try: + return urlparse(url).netloc.lower() + except Exception: + return "" + + @dataclass class JudgeOutcome: final_decision: Decision @@ -72,6 +102,28 @@ class JudgeOutcome: judge: Optional[JudgeResponse] # None if not invoked +def _should_route_to_judge_for_short_domain(url: str, p_malicious: float) -> bool: + """ + Check if URL should be routed to judge due to short domain edge case. + + Rationale: Short legitimate domains (npm.org, bit.ly, etc.) may appear + suspicious to the model due to distribution shift. Route to judge for + human-readable explanation when: + - Domain length ≀ threshold (default 10 chars) + - Confidence is moderate (p < 0.5) - not highly suspicious + + This catches edge cases not covered by the whitelist. + """ + domain = _extract_domain(url) + if not domain: + return False + + is_short = len(domain) <= SHORT_DOMAIN_LENGTH + is_moderate_confidence = p_malicious < SHORT_DOMAIN_CONFIDENCE + + return is_short and is_moderate_confidence + + def decide_with_judge( url: str, p_malicious: float, @@ -79,42 +131,71 @@ def decide_with_judge( extras: Optional[Dict[str, Any]] = None, ) -> JudgeOutcome: """ - Apply policy band first; if REVIEW, invoke judge and map verdict: - LEAN_PHISH -> BLOCK, LEAN_LEGIT -> ALLOW, UNCERTAIN -> REVIEW + Enhanced decision logic with short domain routing. + + Decision Flow: + 1. Apply policy bands (low/high thresholds) + 2. If base decision is REVIEW, check for short domain edge case + 3. Invoke judge and map verdict to final decision + + Enhanced Logic: + - Short domains with moderate confidence routed to judge + - Judge provides human-readable rationale for edge cases """ base_decision: Decision = decide(p_malicious, th) # uses low/high inc_policy(base_decision) + + # Fast path: High confidence ALLOW/BLOCK if base_decision != "REVIEW": inc_final(base_decision) # final == policy when not REVIEW return JudgeOutcome( final_decision=base_decision, policy_reason="policy-band", judge=None ) - # Build the compact digest (URL-only; optional extras may include - # TLDLegitimateProb, etc.) + # === GRAY ZONE ROUTING LOGIC === + # Check if this is a short domain edge case that needs judge review + is_short_domain_case = _should_route_to_judge_for_short_domain(url, p_malicious) + + # Build the feature digest using 8-feature model + features_8 = _extract_8features(url) + digest = FeatureDigest( - url_len=_url_len(url), - url_digit_ratio=_digit_ratio(url), - url_subdomains=_subdomain_count(url), - TLDLegitimateProb=(extras or {}).get("TLDLegitimateProb"), - NoOfOtherSpecialCharsInURL=(extras or {}).get("NoOfOtherSpecialCharsInURL"), - SpacialCharRatioInURL=(extras or {}).get("SpacialCharRatioInURL"), - CharContinuationRate=(extras or {}).get("CharContinuationRate"), - URLCharProb=(extras or {}).get("URLCharProb"), + # 8-feature model (required fields) + IsHTTPS=features_8.get("IsHTTPS", 0), + TLDLegitimateProb=features_8.get("TLDLegitimateProb", 0.5), # neutral default + CharContinuationRate=features_8.get("CharContinuationRate", 0.0), + SpacialCharRatioInURL=features_8.get("SpacialCharRatioInURL", 0.0), + URLCharProb=features_8.get("URLCharProb", 0.5), # neutral default + LetterRatioInURL=features_8.get("LetterRatioInURL", 0.5), # neutral default + NoOfOtherSpecialCharsInURL=features_8.get("NoOfOtherSpecialCharsInURL", 0), + DomainLength=features_8.get("DomainLength", len(_extract_domain(url))), + # Legacy features (optional for backward compatibility) + url_len=features_8.get("url_len", _url_len(url)), + url_digit_ratio=features_8.get("url_digit_ratio", _digit_ratio(url)), + url_subdomains=features_8.get("url_subdomains", _subdomain_count(url)), ) + + # Add routing context to judge request req = JudgeRequest(url=url, features=digest) jr = _JUDGE_FN(req) # uses selected judge backend (stub or llm) + # === VERDICT MAPPING WITH SHORT DOMAIN CONTEXT === # Map judge verdict to final decision if jr.verdict == "LEAN_PHISH": final: Decision = "BLOCK" reason = "judge-lean-phish" + if is_short_domain_case: + reason = "judge-short-domain-lean-phish" elif jr.verdict == "LEAN_LEGIT": final = "ALLOW" reason = "judge-lean-legit" + if is_short_domain_case: + reason = "judge-short-domain-lean-legit" else: final = "REVIEW" reason = "judge-uncertain" + if is_short_domain_case: + reason = "judge-short-domain-uncertain" # Track judge verdict and final decision inc_judge(jr.verdict) @@ -131,6 +212,7 @@ def decide_with_judge( "policy_thresholds": dict(th), "policy_decision": base_decision, "final_decision": final, + "is_short_domain_case": is_short_domain_case, "created_at": datetime.utcnow(), } _decisions.insert_one(doc_dec) @@ -141,6 +223,7 @@ def decide_with_judge( "rationale": jr.rationale, "judge_score": jr.judge_score, "features": jr.context, + "is_short_domain_case": is_short_domain_case, "created_at": datetime.utcnow(), } ) diff --git a/src/gateway/main.py b/src/gateway/main.py index 81d1baf..5d59426 100644 --- a/src/gateway/main.py +++ b/src/gateway/main.py @@ -14,6 +14,61 @@ from common.thresholds import Thresholds, load_thresholds from gateway.judge_wire import decide_with_judge +# =================================================================== +# WHITELIST: Known legitimate domains (handles OOD major tech sites) +# =================================================================== +KNOWN_LEGITIMATE_DOMAINS = { + "google.com", + "www.google.com", + "github.com", + "example.com", + "www.example.com", + "openai.com", + "www.openai.com", + "www.github.com", + "microsoft.com", + "www.microsoft.com", + "amazon.com", + "www.amazon.com", + "apple.com", + "www.apple.com", + "facebook.com", + "www.facebook.com", + "twitter.com", + "www.twitter.com", + "linkedin.com", + "www.linkedin.com", + "youtube.com", + "www.youtube.com", + "wikipedia.org", + "www.wikipedia.org", + "stackoverflow.com", + "www.stackoverflow.com", + "netflix.com", + "www.netflix.com", + "paypal.com", + "www.paypal.com", + "ebay.com", + "www.ebay.com", +} + + +def _check_whitelist(url: str) -> bool: + """Check if URL is on known legitimate domain whitelist.""" + try: + from urllib.parse import urlparse + + domain = urlparse(url).netloc.lower() + # Strip www. prefix for comparison + domain_no_www = domain.replace("www.", "") + return ( + domain in KNOWN_LEGITIMATE_DOMAINS + or domain_no_www in KNOWN_LEGITIMATE_DOMAINS + ) + except Exception: + return False + + # List of expected extras keys for normalization _EXPECTED_EXTRAS_KEYS = [ "TLDLegitimateProb", @@ -97,7 +152,7 @@ class PredictOut(BaseModel): reason: str thresholds: Dict[str, float] judge: Optional[Dict[str, Any]] = None - source: Literal["model", "heuristic"] + source: Literal["model", "heuristic", "whitelist"] # --------- tiny deterministic URL helpers (fallback heuristic) --------- @@ -152,24 +207,34 @@ def _call_model_service(url: str, extras: Dict[str, Any]) -> Optional[float]: Returns None if service unavailable or on error. """ model_url = os.environ.get("MODEL_SVC_URL") + print(f"[DEBUG] MODEL_SVC_URL: {model_url}") # Debug if not model_url: + print("[DEBUG] No MODEL_SVC_URL set") # Debug return None try: - payload = {"url": url, "extras": _normalize_extras(extras)} + # Use model service API schema: {"url": "..."} + payload = {"url": url} + print(f"[DEBUG] Calling {model_url}/predict with payload: {payload}") # Debug response = requests.post(f"{model_url}/predict", json=payload, timeout=3.0) + print(f"[DEBUG] Response status: {response.status_code}") # Debug response.raise_for_status() data = response.json() + print(f"[DEBUG] Response data: {data}") # Debug p_malicious = data.get("p_malicious") # Validate probability is in valid range [0.0, 1.0] if p_malicious is None or not isinstance(p_malicious, (int, float)): + print(f"[DEBUG] Invalid p_malicious: {p_malicious}") # Debug return None if not (0.0 <= p_malicious <= 1.0): + print(f"[DEBUG] p_malicious out of range: {p_malicious}") # Debug return None + print(f"[DEBUG] Model service success: {p_malicious}") # Debug return float(p_malicious) - except Exception: + except Exception as e: + print(f"[DEBUG] Model service error: {e}") # Debug return None @@ -186,33 +251,47 @@ def config(): @app.post("/predict", response_model=PredictOut) def predict(payload: PredictIn): - # choose p_malicious (client-provided/model service/heuristic - # handled upstream in our existing wiring) + """ + Main prediction endpoint with whitelist, model service, and heuristic fallback. + """ + # PHASE 1: Fast-path whitelist check + if _check_whitelist(payload.url): + return PredictOut( + url=payload.url, + p_malicious=0.01, # Very low risk for whitelisted domains + decision="ALLOW", + reason="domain-whitelist", + thresholds={ + "low": TH["low"], + "high": TH["high"], + "t_star": TH["t_star"], + "gray_zone_rate": TH["gray_zone_rate"], + }, + judge=None, + source="whitelist", + ) + + # PHASE 2: Determine p_malicious source extras = payload.extras.model_dump() if payload.extras else {} - # prefer client/model; fallback heuristic - # (gateway-call-model branch already added model call) - try: - # if present from earlier branch - from gateway.main import _call_model_service - except Exception: - _call_model_service = None if payload.p_malicious is not None: + # Client provided probability p_mal = float(payload.p_malicious) - src: Literal["model", "heuristic"] = "model" - elif _call_model_service: + src: Literal["model", "heuristic", "whitelist"] = "model" + else: + # Try model service first p_from_svc = _call_model_service(payload.url, extras) if p_from_svc is not None: p_mal = p_from_svc src = "model" else: + # Fallback to heuristic p_mal = _heuristic_pmal(payload.url) src = "heuristic" - else: - p_mal = _heuristic_pmal(payload.url) - src = "heuristic" + # PHASE 3: Apply business logic and judge outcome = decide_with_judge(payload.url, p_mal, TH, extras=extras) + return PredictOut( url=payload.url, p_malicious=p_mal, @@ -238,3 +317,56 @@ def stats(): def stats_reset(): reset() return {"ok": True} + + +# --------- Explainability endpoints --------- +@app.post("/predict/explain") +def explain(payload: PredictIn): + """ + Proxy to model service /predict/explain endpoint for SHAP explainability. + """ + model_url = os.environ.get("MODEL_SVC_URL") + if not model_url: + return JSONResponse( + status_code=503, content={"error": "Model service URL not configured"} + ) + + try: + # Forward request to model service + response = requests.post( + f"{model_url}/predict/explain", + json={"url": payload.url}, + timeout=10.0, # SHAP computation can take longer + ) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + return JSONResponse( + status_code=503, content={"error": f"Model service error: {str(e)}"} + ) + + +@app.get("/explain") +def explain_dashboard(): + """ + Serve the explainability dashboard HTML page. + """ + import pathlib + + static_dir = pathlib.Path(__file__).parent / "static" + html_file = static_dir / "explain.html" + + print(f"[DEBUG] Looking for dashboard at: {html_file.absolute()}") + print(f"[DEBUG] File exists: {html_file.exists()}") + print(f"[DEBUG] Static dir: {static_dir.absolute()}") + print(f"[DEBUG] Static dir exists: {static_dir.exists()}") + + if html_file.exists(): + from fastapi.responses import FileResponse + + return FileResponse(html_file) + else: + return JSONResponse( + status_code=404, + content={"error": f"Dashboard not found at {html_file.absolute()}"}, + ) diff --git a/src/gateway/static/explain.html b/src/gateway/static/explain.html new file mode 100644 index 0000000..392bd76 --- /dev/null +++ b/src/gateway/static/explain.html @@ -0,0 +1,481 @@ + + + + + + PhishGuard - URL Explainability Dashboard + + + +
+

πŸ›‘οΈ PhishGuard Explainability Dashboard

+

Enter a URL to analyze and see why it was classified as safe or malicious

+ +
+ + +
+ +
+ +
+ +

Analyzing URL and computing feature contributions...

+
+ +
+
+
Malicious Probability
+
--
+
--
+
+ +
🎯 Feature Contributions (SHAP Values)
+

+ Features in red increase phishing risk, + while features in green decrease it. +

+
+ +
+
πŸ“Š Extracted Feature Values
+
+
+
+
+ + + + diff --git a/src/judge_svc/__pycache__/adapter.cpython-311.pyc b/src/judge_svc/__pycache__/adapter.cpython-311.pyc index 34daa5f6b4371f79d7f2acdd4ccea67e994dd506..ec4e758ce1603eebcf9c5ac089bcdbba978fb082 100644 GIT binary patch delta 1030 zcmZ8g&rcIU7;P;etN|hcH5~NwU}Ck9l4vv=NUZHPG;S!_E=n;bPPfzUgmz}#nT?i9 z{S!<|VvHAWUNxLNnfMoI;)#F49`wS=Z?=>Id)OcIeczim@4fwd>)oB1&(qUJ0g=Gb!s$})Qdu+!o+he>5j6>iv>Ff78 zRn${V@MXwJNu?Z>LMC@1lDbRz6t)~2;p*@Zq|oDvc|_5A7aEkPD5TOfj3-tVidN3u za;yrtWyo#h?COqX8cR@;>#n<5F(G~+E@@J@on6zP8CZ#*l{7ZGcEP4irdS){H$zb~ z;W6n@$q3gqXdM^54g^^3k{WC2qdw1OR$kzI-X~#RaK(5;^uZj2G2>O@DH=jZ1F8^S zpbA9c3YRCdm4IlOGz5+@QQ~-N6DkAsz{B+cNS$&y*t9#?75ntmfUlNn)GO($h;om{ zbkDQ3sitsxvs^3iNewA_r!Ydz=7i>kC^yRJNr=R{L&EZyY+giMfoKdpAE8K8ry$yd zahvj{@=b7x`6W-ZgBFUbpdQyyU|lo}^l-DhQGjP9cO9(lT;8>-;I2EC1;vtO7b>Rl z)N*={##Xv`ff%mX33!eCC}h_pX;5u&8{_9O5lO?zxur5D!!n^3GTH$8CTL+& zV^12w8)Ods*`*zbp&761w24p}M(oJmC1aQ~6VlxaN6E}l(o8204M_tuNw@C3G>oD~ z1HsS4E^{tNnUO>%k;ueZC%hBqUep?qL;^ZA_`mp^p`~&;J$^W;)8BhA{@$FMTJB#@ zT%O;**uSCQ6L|Xvi7SiyJDDMj` z0dWin1g?ji$R;3)yrz&*tPkxsIP*ce7`7{AML1Aret_z-Q_nK$HKBo#3^^8aMg@z2 zsg`3RT`25WD|UHT&2o&PL9pitU{t9Dh$D%?BNU5hEV)#qpx*i1cw|D2}+eP6GT{Zn2!h*CfMaw2smNf{J9O$rlLha>V$;?Bs;=XbbS3jAS!jY z^1Z#&aY`G^UUtXnPOd5V%xEKH*E22q_V77pz2vOdoE7)~u|F-b-q)V!xz`g@?e{*n XE3w>OA3W2unM5+y{=<0mmR7&-72YM6|D-6Hl10iE$%IlQrm2g@q2J(xHbQa^S&0*`9cE6Idv6%(2HDmjDmKo)n}&;ZqgZr=0TMkYXsVTXZ#inw|H3 z`)1y}d2jw6i3B-#p8WMc#|b~jeTy$wk7v+%{Ws|Rg;O|%uW~iM!q;3CSIu2<*My2t z^He-M>ReTC%~$c&{1tyKPzmrH5fpdg>RVhT=;9u5itr~+@etn&H^;q%U;n9un9d72 zpF=ib*cgyQL1$nRr_Rl+bdHpXQJw~_+uW%LD9_QkSTj3R#;#Pz+&kF%o55N9X5r~I) z74JvzI>7l9zY=iZe8dm9;219Czy%mCJcf%na6yKf7{f&!xDdllj^U;pxG=-r7{R%e z*xB^hUPst!XRe`cI#3e~H9JJb!FLG>0nv=}tu|y>q!aptRnyG3H2~R+}E}1ZO@whYFZwC0??{%&y?yCDv$+8Ro~I}cJn6*5F^;piG@IgXf~x-o}g_st?J;E zG$`%K{@}>x-r3X}nwi!Q)4F*KaASNnM%G2Az)ds;!k#PW#ls>Z2h&D-Mt^3vcJFWg zRMu7JJRJx57yr90-10OPi2KsyMeRpszuzB9KmC0ey&~ z!Z!5RVMkHR21K^7li4ej(RezrDC3Ik2}BMDjcm;Ji=rmgh$z}YQLO1oql*2oC_ZaQ z)xL%XaHZ3@Qa5BKJ1UA&Kh?o*8lp&NQRO+T>XJ$0h{XV>b0}`1xQ${SMG}P%MG8e4 z#24Iaklgoox7H3G<@LI8P$1@~I{ib5$PEgibzo5W;P(yXh=|4~@&SetMwM}=-lS1r zG6B_tGcpkUrz3wCv3P5x9qUYWo>{*rSg~R^R_w)!rw{&IvVoP}&>Wysk8 zI}6BJK+Xbi673ZyXD-#A8FIFOGl!fxug_IOhQebAfMVy?a z4?2v)TPd8_5;B&Mu>_1{$L(Y!l5J00?nu_kt`#qJIx-rQZm!t z>wNh6y7lWlYkt2wzu%kR2iHGo$)Ek!6852Wh5d7|4ek%OV(pmq{ynh1jMkUY`m*=5 z*m}|?&qe117T@n|BW=n`uL5ZiNsCBY^qv-458I{ZkH$&EU8KubQSOX!*V1q9hkOtb7cbX`m40;Xo-gHqsON!klZ+GmG9#^~K8&l-@U%XAVjU%2c^ zsPt-G{PY+u2nJ*|Vh3xghNm1;gBwn&?LX{iccYN?(|3SI-$ju@f%g~N4d0-6hoXxp z*l}3G9$R7-dq24n_-1T_7(F8Afg7#6Ry+&ss}H|kx8j9vywHooVJY+icti%LWa5@3 zVCymooxTP9_}iWM573M|QQ?_;;JyLw^x=KvO!yz}6#7mCtY7geP*2YSW7RxBf+RGu z0mATp;wRy=&W1rxGC(#7m;rwoFjiO+~?tirA6CMj3TTY{d3`LMW6?v-}F3 zO+Nr(2Uv!MYaA)i863R{;;I_lKtyPet7M&$EN@2@aws*brg#WROQ%iShZ&6g(acVc zZRsFR_WMTrJIW0I9PO}OW|{bx_{x&#duUI z5J^A+g4Puh=g~Nv#N^MQ{d3ydEm_HOH(Bl_%T}V?i|$zhwyv;u0XcBx^XV@$UuCRJ zuA9mAGC9adkoumdEFlN2E95R92aqqftmJApx!Oyv!tpP+GA*OE{Mqj;VHH}JN$B+B z38z@dPS{?vS%>Q|3-bPg*wXNZ%T{jt>rDmfsVY;<{QU~WJhX>RMbXA8gXjPLy) z_3K=Y<^YWGVSEGP(#`YyIk#s0f9-<%mG$4-Mf9dMo9jjIorW*HNgm3FQ?z!z!_hZC nsy28Re;e|aL$p>Qzwx&rzg?+8v)GD?DH8DbRBnKmxxaz?W!X`Vh7i1|N ztwM50BnW{*38-wKI{2VNaTIEba`UC&KcI)%Lp7ghkw^p&9nNrxCTj&HfD+Q?X}H)sH2nk`AbjW^y~3+>WLO%XzfpQhBf! zOC$T4IhNM@Us}&G1^Sp?Jf!y>&>y#gAA4(`EaCr5NZ?3OfzLz?9Eli!2+gy^MF#Df z|0-6ED#00j1y_to0vjITZD9r5{Z&%DS;VEc`+KlwyqkR?kU{3Og(9{GK&9y`x^a`J zhF-P<;HXq@Y2`IlFIDpP(Dw_O1b~nbq}+9%07<%BGW1dvyd^e=Z7+*}2}m0mG#`sme#rboRK7Y11&Viy zWz`_3m`#QO{Qx*5q5w_<&X_`9z7}JRSU+&@$it1Prra8$a@dl?T{&FO{lC|O@@6evovVWDJC8Ec+zx!OEcqLvcvD$#nrk)|lhhJ~WN z$YeX%Hnv>bE*i;Lky@r3$xsw*+;54`Y7{X$W1-AWe5xaMw5=a$JY~gG-FS+k(U#Pf zfAdqs?39I4y@_wyh0dj|^mdv~&3oq&vnMP659zH z%UZE)HnqcTp&WOah!j++bY-IA?v1SvGRi)8G^V2=Garaqt14 zF)9rP=mxXyeY#t7+?0K(WD-=)XFA{I1RYNFk2JGZ)~ZE(g?!7ztgE=nj`_3iM%2U4 XzT=LfG5;RCAO4?GOxb?}!A|`PY)V2p diff --git a/src/judge_svc/__pycache__/stub.cpython-311.pyc b/src/judge_svc/__pycache__/stub.cpython-311.pyc index b4a2938ec35f09bfbf2eab66cece35cf734dccdb..4fd1a1366b641826113f1527673fb04784ae5ebe 100644 GIT binary patch literal 5278 zcmb7ITW{OQ6(&VVBrRIL#9k+sY}2u2N506pXXChWF3!b`SDVJpq6ktLib%_hWlAQg zBo^dA4FXg#0>les5p9tNp(xT6=?~bx73fpnNRBe)6Tpdhs-mK!#F&SE#o)vWycN2 zGI4{zG0wp|If8dBI63D-=M6XK;&CWDpzMY+$&s^IklMrnAV`?ZFH>BMzs9emcv&$U z?UI~Kh%#?Ce#J|i5LHYfCB;p*#49ODjH2$_36Ub;_x3i3-#MOPF`)Gad=1ZF1*oY_ zFQDpGxq989px$=MITmov@sP^kQ`m037hv;M*)9x%2nR{i6;H$j(RAP8C1GLJbf=PB zMBz=h%tuoaZ<3K{G?5Y&)18d0CL@QSL5PXM?g1e4N8tCC1@R4j1;fF&8SFO{3;}=S zxPyI#J;QPAG0kCM77za!dCn0!Xc9_d$#j8V=HoH~VcG1v(A41yd=m=^8m_=a%du+h(_Y_ z=}=jL0u%Beh@XDMe#Ekv>J7nDAae2spUad=mGUj9VnGM;(Oq6L2|2YKBqTRfn(mSj zsSOG?^)pgCYGEo#NIs}S^DCbQ@lzIi(>SYIPxhk0G(DMKpMG@Z@s;d_*B)=B<-#2( zY2J_y(i74>A=Md@IuJY|@hiKc2Qjds=Wiqm=?sRVsFc=l4qtP`ur(rs-!9KQ=U8%} zcBwPt;D~f1hOId=M8T@FJC7$B9SjdLwonvxHY+;x!1Y9ukR3(IX|eGDtA z8ViuNmo~=HOIF@u)li3(ry9v(wS&djScsLY4pw6|7O-+{yOL}81XkW^B#YG!7E!^f z-V8QXV*x7^!dBw8y^dzExz6}ns*x;KJ6Nq1Gg!}8d#bU36~wR2s;vU4-uT+zgM=2k zmAKC;yk2j79o0zI_;#@NR(QSM`1Vy}!T9JhtIi5m^&-?)jbyRf!Sa6stFCG+1#MY& z%HUl03)lr-m3IPZ1Xz2v+kcDJYq`YrELrhr?IQKO!Pc?c`yRW@F1zIttfP;&yT6Xz zzW3OL{jij*@s>+m|0mcD)Y$FXM#az$-LzfWt+(dPIJp7a&d03C!QK6FOk@G!K*j!E zZ)Sqk2uklZHw;c8N?%23DC5}euiYj1ia0pNW>0X#YcA1)c9kXT%Oero2-G;Z(KT0w zklMLW0S;%iDMRdLB;wrInp;6%Z98RL8F$73u~`11LB9M=f_!=RhJ1O?gnW6Yf_!ZXoA`b*-NncGJ8(ke-m91P zTM%-Rj|!1^rAh=~(rmf&2Q+L_{{TCP@%3vn6&Oh=aVv65h{IXX=KN+dy?;5u@e&aF z8Re3r{slfnpmAQY28S4JqPb6ZEJwsuwuX|tiJ~X%`!H#F^DxqAw%snqST3;~5yZ`I zRPBj_YjbC(!)eDUHVpyECT`sk5-EAd+#(Mrc3BakGOI{JEXGSb z7xb9qv}}d0*#hA?V@IdLUxj%!+s;GSoll60Af}KD#B1mp(mtDvph!ZEP$49yMdU9J z2vlYDY@V?uw4%8)ocMC#OJ$LlW=nyHbmdZ4X3CnO8D5MjixS%2qyQ2FAc9Vlgo&|k zYqaHdF%suZkA3NgCeVc_3LN=+O0eCe7WfEU=6G2efzGBol1%aC}))4Oxmh*_od!lD#U~ zTdR9eqYmlRA=P>ex9vbDN3v(ZgSEg%HR_m79aE`erLo=5JBnndN_N(& z?r?yhKx6C9?=RARmG)QbZmelkK&JvK75E=SSyOIhBdC#no%C;EglnYG^2dE@?{UM! zJU*ExiXNZp@fl3>#?+&k+?2uWc@ka^=O;C0SZ9V+X7~rvn_E#^`!up&C;Jh?*vl3@ zIIadS0)+1IL;QI9RZo94hUy#Dn2^qdR3>DD?AORXo$Ny(p_jOR;B)oBlm+6;uM|D3 z>S1d^Ml@zrXGT?K6b94q*+%qfOryJXx;uN$AnDw%9^B2|t@eQ0@wrBw)TxuI^%%4# zx3b=m>!{TWY1Eibjj7ZapfgQ5`J3~3B0u+a_-R=4^y!|y>=c?ACZ{~eWHSakk|T4Z zK{sx+t^0F+qqQ5A-?Vn+NKoq;P^s%V=M#FJ-Wb%VPMzvh;TgWZMPM#3GO*8R?l4+? zM%!Lc-@EVWs?ob&?>%bt2lW1NBQT%`CJdkd`7aE%S7#3!L&N&eMZ@3qJp4mPcYdW1 zd^x9e9M(GyzwQ|*tf)h$zMIo}&gnhpUUvry5jA-7yQ5n78NK_=me1SLu!SK(?_u2k z!9e^Mi7jl8%WeJB5e`x&jW%0!7?CA(@UnJ634N5|zZeDnkP)C2{ljAIW>O2vp+8Bi z&F&pF0Y`&WoIfpHf;x0qlvhD)5jc(;*ctW1&z9SXk8MP^FnHxlFDqXmylEVFZ%k}q O=w0}t^!|BiQTsm=9nOgW delta 1408 zcmaJ=OKclO7~Zvay|e3GKjK#&#@;w>T^DSEny80Xn1=;Ogs3HOfkWhUH;(I9G+r0H za%3ZMNTZ4xO%Dbklnd>l6%r@lgep!*9M;wfRz9E#PQJM+T;RgY#tBg(@oV(5lmqvJ4xc z$sP%XEgj&qEr;SlPnbCB;hJ@)#DfW9Gie_6T_NvHj11%Vwdfnnfg~0cODw86 zQ%Gk>JFnz9M)8s+v=8~&0Q!XCqaVGAskR5(kGEv2z&Y#(*nZw)!)V9|5||*wsVy=; zbq}B8!O>Z0N|O#PP*jV$!682#20KiPQ33~!7J?@!RT-n9l7kT8k#stIvQG2=C#}$0 z;MgjH-scz6(~~ngm6;v3>L%A&vaLa9@j-51cOY!Lmx2iRwJTv`PE z%D>cBVqY*1y_I~mDc2ht zO0AMFR_i13ZA}_|pVb4EMp-Lvm(^n>`c%30f&9)&PA;psTd|VA-qa(Nnxf*q>Iq;( zu3nELr<4N|M}MH#q_r?J#1%`eszB19))T^+zfi8Q}XT%}c zRP$N`s&#k{%MM-n?QONHz$9M4^CYFvcb>$V6bGXOr#FWTEY?dy4yyTjty&*Cpaw6H z!ePmz(ou$Vga$s+!A*g z3wZB%_B`EoA0$nXH0)vV!tI&v#$J3keyULKgM3ZBpZ=*A7o6BF+gU*j`Us_a8ZMco~}LL2PqSz413tZGremAK4$Q-)5XvA zK|5`Nv;oo+Wj;!xNL~af4L|j+`WFc(C1?&AM3ngas2Kz)SdeF;^WBl-KG_n z&G@1fi<_~GHJ>o&m&QEn7seDxydb)sP&oZb(wGv^qrg&}{C<%9bzy|05Ke!CEBI4v fl+_pEWfTmqU3!D2UEM2VioEyVI(Ywow#EJhpJ06p diff --git a/src/judge_svc/adapter.py b/src/judge_svc/adapter.py index 47ff516..43fd0ee 100644 --- a/src/judge_svc/adapter.py +++ b/src/judge_svc/adapter.py @@ -20,19 +20,28 @@ def _prompt(req: JudgeRequest) -> str: - # Compact, deterministic prompt; instruct to emit explicit fields we can parse. + # Enhanced prompt for 8-feature model with detailed feature descriptions feat = req.features.model_dump() return ( - "You are a security analyst. Assess phishing risk from the URL and " - "compact URL-only features.\n" - "Respond with EXACTLY three fields on separate lines:\n" + "You are a cybersecurity analyst specializing in phishing detection. " + "Assess phishing risk using the URL and 8 sophisticated features:\n\n" + "KEY FEATURES TO ANALYZE:\n" + "- IsHTTPS: HTTPS usage (0=HTTP, 1=HTTPS)\n" + "- TLDLegitimateProb: Bayesian TLD legitimacy probability [0,1]\n" + "- CharContinuationRate: Character repetition patterns [0,1]\n" + "- SpacialCharRatioInURL: Special character density [0,1]\n" + "- URLCharProb: URL character sequence probability [0,1]\n" + "- LetterRatioInURL: Alphabetic character ratio [0,1]\n" + "- NoOfOtherSpecialCharsInURL: Count of special characters\n" + "- DomainLength: RFC-compliant domain length\n\n" + "RESPOND WITH EXACTLY THREE FIELDS:\n" "VERDICT: LEAN_PHISH | LEAN_LEGIT | UNCERTAIN\n" - "SCORE: number in [0,1]\n" - "RATIONALE: brief human explanation\n\n" + "SCORE: risk score in [0,1] where 0=safe, 1=malicious\n" + "RATIONALE: brief explanation focusing on key risk indicators\n\n" f"URL: {req.url}\n" - f"FEATURES_JSON: {json.dumps(feat, separators=(',', ':'))}\n" - "Consider length, digit ratio, subdomains, TLD prior, and any " - "suspicious tokens in the URL." + f"FEATURES: {json.dumps(feat, separators=(',', ':'))}\n\n" + "Focus on: HTTPS usage, TLD legitimacy, character patterns, " + "and any URL obfuscation techniques." ) diff --git a/src/judge_svc/contracts.py b/src/judge_svc/contracts.py index fc4ed60..a4a16bf 100644 --- a/src/judge_svc/contracts.py +++ b/src/judge_svc/contracts.py @@ -6,16 +6,36 @@ class FeatureDigest(BaseModel): - # compact, URL-only signals we pass to the judge - url_len: int = Field(..., ge=0) - url_digit_ratio: float = Field(..., ge=0.0, le=1.0) - url_subdomains: int = Field(..., ge=0) - TLDLegitimateProb: Optional[float] = Field(None, ge=0.0, le=1.0) - # optional extras (keep small and explicit) - NoOfOtherSpecialCharsInURL: Optional[int] = Field(None, ge=0) - SpacialCharRatioInURL: Optional[float] = Field(None, ge=0.0, le=1.0) - CharContinuationRate: Optional[float] = Field(None, ge=0.0, le=1.0) - URLCharProb: Optional[float] = Field(None, ge=0.0, le=1.0) + # 8-feature model (production features - required) + IsHTTPS: int = Field(..., ge=0, le=1, description="Binary HTTPS indicator") + TLDLegitimateProb: float = Field( + ..., ge=0.0, le=1.0, description="Bayesian TLD legitimacy probability" + ) + CharContinuationRate: float = Field( + ..., ge=0.0, le=1.0, description="Character continuation pattern rate" + ) + SpacialCharRatioInURL: float = Field( + ..., ge=0.0, le=1.0, description="Special character ratio" + ) + URLCharProb: float = Field( + ..., ge=0.0, le=1.0, description="URL character probability" + ) + LetterRatioInURL: float = Field( + ..., ge=0.0, le=1.0, description="Letter ratio in URL" + ) + NoOfOtherSpecialCharsInURL: int = Field( + ..., ge=0, description="Count of other special characters" + ) + DomainLength: int = Field(..., ge=0, description="RFC-compliant domain length") + + # Legacy features (optional for backward compatibility) + url_len: Optional[int] = Field(None, ge=0, description="Legacy: total URL length") + url_digit_ratio: Optional[float] = Field( + None, ge=0.0, le=1.0, description="Legacy: digit ratio" + ) + url_subdomains: Optional[int] = Field( + None, ge=0, description="Legacy: subdomain count" + ) class JudgeRequest(BaseModel): diff --git a/src/judge_svc/stub.py b/src/judge_svc/stub.py index 057ba86..7eef318 100644 --- a/src/judge_svc/stub.py +++ b/src/judge_svc/stub.py @@ -10,42 +10,83 @@ def _risk_tokens(url: str) -> int: def judge_url(req: JudgeRequest) -> JudgeResponse: f = req.features - # Simple, explainable rules: + # Enhanced heuristics using 8-feature model: risk = 0.0 reasons = [] - # long URL - if f.url_len >= 120: - risk += 0.35 - reasons.append("very long URL") - elif f.url_len >= 80: - risk += 0.20 - reasons.append("long URL") + # HTTPS check (security baseline) + if f.IsHTTPS == 0: + risk += 0.15 + reasons.append("HTTP (not HTTPS)") + + # TLD legitimacy (Bayesian prior) + if f.TLDLegitimateProb < 0.10: + risk += 0.30 + reasons.append("very low TLD legitimacy") + elif f.TLDLegitimateProb < 0.30: + risk += 0.15 + reasons.append("low TLD legitimacy") + + # Character patterns (obfuscation indicators) + if f.CharContinuationRate > 0.80: + risk += 0.25 + reasons.append("high character repetition") + elif f.CharContinuationRate > 0.60: + risk += 0.10 + reasons.append("elevated character repetition") + + # Special character ratio (obfuscation) + if f.SpacialCharRatioInURL > 0.25: + risk += 0.25 + reasons.append("high special character ratio") + elif f.SpacialCharRatioInURL > 0.15: + risk += 0.15 + reasons.append("elevated special character ratio") - # many digits - if f.url_digit_ratio >= 0.25: - risk += 0.35 - reasons.append("high digit ratio") - elif f.url_digit_ratio >= 0.15: + # URL character probability (language model signal) + if f.URLCharProb < 0.30: risk += 0.20 - reasons.append("elevated digit ratio") + reasons.append("low URL character probability") + elif f.URLCharProb < 0.50: + risk += 0.10 + reasons.append("moderate URL character probability") - # many subdomains - if f.url_subdomains >= 4: + # Letter ratio (readability) + if f.LetterRatioInURL < 0.40: + risk += 0.15 + reasons.append("low letter ratio") + + # Special characters count (complexity) + if f.NoOfOtherSpecialCharsInURL > 8: risk += 0.20 - reasons.append("many subdomains") - elif f.url_subdomains >= 3: + reasons.append("many special characters") + elif f.NoOfOtherSpecialCharsInURL > 5: + risk += 0.10 + reasons.append("elevated special characters") + + # Domain length (suspiciously long domains) + if f.DomainLength > 50: + risk += 0.25 + reasons.append("very long domain") + elif f.DomainLength > 30: risk += 0.10 - reasons.append("multiple subdomains") + reasons.append("long domain") + + # Legacy features fallback (if available) + if hasattr(f, "url_len") and f.url_len is not None: + if f.url_len >= 120: + risk += 0.10 # Lower weight since we have better features + reasons.append("very long URL") + + if hasattr(f, "url_digit_ratio") and f.url_digit_ratio is not None: + if f.url_digit_ratio >= 0.25: + risk += 0.10 # Lower weight since we have better features + reasons.append("high digit ratio") - # low TLD legitimacy prior (if provided) - if f.TLDLegitimateProb is not None: - if f.TLDLegitimateProb < 0.10: - risk += 0.25 - reasons.append("low TLD legitimacy") - elif f.TLDLegitimateProb < 0.25: - risk += 0.10 - reasons.append("moderate TLD legitimacy") + if hasattr(f, "url_subdomains") and f.url_subdomains is not None: + if f.url_subdomains >= 4: + risk += 0.10 # Lower weight since we have better features + reasons.append("many subdomains") # suspicious tokens rt = _risk_tokens(req.url) @@ -75,9 +116,18 @@ def judge_url(req: JudgeRequest) -> JudgeResponse: rationale=rationale, judge_score=risk, context={ - "url_len": f.url_len, - "url_digit_ratio": f.url_digit_ratio, - "url_subdomains": f.url_subdomains, + # 8-feature model context + "IsHTTPS": f.IsHTTPS, "TLDLegitimateProb": f.TLDLegitimateProb, + "CharContinuationRate": f.CharContinuationRate, + "SpacialCharRatioInURL": f.SpacialCharRatioInURL, + "URLCharProb": f.URLCharProb, + "LetterRatioInURL": f.LetterRatioInURL, + "NoOfOtherSpecialCharsInURL": f.NoOfOtherSpecialCharsInURL, + "DomainLength": f.DomainLength, + # Legacy context (if available) + "url_len": getattr(f, "url_len", None), + "url_digit_ratio": getattr(f, "url_digit_ratio", None), + "url_subdomains": getattr(f, "url_subdomains", None), }, ) diff --git a/src/model_svc/__pycache__/main.cpython-311.pyc b/src/model_svc/__pycache__/main.cpython-311.pyc index 9ecf6f19bd7af04800e7b0f9d8d777344c41953a..a8bc228124c6cbe3c62c0a4bf1835ff907f8be4a 100644 GIT binary patch delta 6902 zcmai2Yj9h~b-pjWE&!6?6MTbjf)q*dO^K95iZ4-oN~BB~jtxV+mn0+*fESmd#DE3m zva#aY8N(4xOSMP8IPTDY-iG@GcBMZBlgyAB4 zJn0J=j*Nv&MXzQ#z|vK^;!ImMNaG7{&oKW6vpss<-9E22sbr_g*Bfz9RcWPEdXA35(Iht^AonatnOwhRO?k41JV*0Gge#~p zwWcyDD>_<0wpFTpyXy;9kK|e z8d92LTnFTXP^u**iFI|#T~OLbOPuS_9vyX5M~HD9rb_BJl^mf;8a9=5QziRpsZrKR zMerKs9@!v^vs}`oS*_Fr8{nv{k(y=8EGM@8%6qUKj8V?coj54S5(wkejp(>`gg)EMW&&dIT&T+T6Ow zVe58xK*g|uktU?0vf-gBdgU&ukM8y3vnHv3P2(7m`{JHZHL!EVq`@_2*U(2nc8u(m z>0|QXt)ZQ+CmomjVZw&@OxUTe;ye?M37Eu*vnLXRk^7`kY3$ZGHtBMO z6U(oQ`0-*Z?$P~9WL4dOE1>jD4B=vZR)|y6RN%_z=U*`+KZIiOn3yF6~mrN~=V zhx0)31B1)&_sI1#zJ&*(Yr(bb^Tz2pw=%zYW|Q{$<`Ob~*G0d}L+g!RS(){?FT@MV zl3P(^1clJT1LH;CqOy1a8(lO=b&AM=8H2KfTMS%osqOYU=S zKX6_3J~a2hytK4bpENLq7L!KO@-{=`WnXB^T3WHq;#8twyVbNJY5BqfTCv^uG}2_p z0BFsQL0AHcYv!Eqyo_c&zNPvQnM@05w{vi`YUcp6h}6#3&{*Xckw?`c8J5mFvvp{| zatZK~qu_sj3UGx1FD9PCEU+sq3x`IK4Kg=a+}&*A&T{zO=6%8n7v%7&IVZ$R!cI?I z;e(t)Pef1!&w0kL5Ar)}CYdsZW#ClMOm3b7cI|R&@4P&7&gP!AojN&Ua|dibukD=I zx8${XWLOOM0yvpX@-4XBUfa^VTai8PfD)=lB-Xyks(Kik--QD5vOor}G6Qz4YJy6S z`%FD$pjwbrwXjYe-;An(DV33`L$cokexzE5$Hu3}oFn~%LsLVeJyZS8zVXqXp|MH3 zfU2P3X6v{CsyY638ofRzbQd5Kh{I{Em1)iyG7RUIg{T!+B=Q9mfK?_Y7&Te=U{|3l zYN@zmsl9EfjacfUmb%N@yVjx~rhGr;`uR6d#M%_KHeEK~HKqSHyFP5Hk1?$Iz(#7p zZ*3jn)Q*^jH;DIRdRCLQk(zmV@$%yR`}YGT=-}y|)Sh(am+4$j4!^#j&tS1x6!Ei$ zdfX{m@ZTF!wVGL%Cm`d`4Eyk^LBu&?IeazveS>&pIcsv>CHaRca6ooBTt3EhYP zHm%TxT7XtFJU*A?B&|G77gF4?6BKk7IDfWYY{FFo78YBLFJz_9|$U6o&|m5f~>hL4d?U zJB!Z2CqVYarj>e+dsYtkU0#Hs@)v~r-ujh-N>2!I$KE*|*F5q;&$)Es{ATxt5B!FY_ zk8Fpvlyd=_%5q;Wd4~<+pOh?4tq_?%A`l`#)qe*lRRe*-rJzfs_89`J1TF)p0z^s= zdYY7zovo1~0WFKaUHYf-&eqmVWjU<&PYCZj*j|2;eFg{1YtD2-h24z)Q0M5oqyg>v zc|xrcphoBj)lGo*x66j-2!lWhi}e*B8>w})@{#pVD*i>QQCSf!tiQYO6_!1OC+Zgq zk`7^nW~v5a!9oC;h@M&eS-rzZ_bYXwm+(M?gKfdDH7xM2UX$%FK-YQr_Xnzqs16N!2Y9MB z;GXlky{gVNGvix?-S9WK=iqSdLshGqrxs;&@nOm@;V%yEPl+p51<<9S4{%9mf9Mc& zGByVQ&)p;T6jTQUjhd*QPz?l#$>?JMm3MhBB8gC%fZyX*RDJ=B;}={PkW7T=B0GVy z>$k^W8p$PSGvh-tauZ21`RHf?L83XJ2e6T+ySsZc+ThPRGPEcRjK|iiyIfrEDI!gb zRZaeRcVON*q8NQ&m;;S+6u>=nG;cX5A9L}-l)tzn^c!<>G{Y`bl?UC)o0>!Rkm)q$@B z@tNVLhr?-&5uqt6G=+tx^>6i`V!^wAGVoXIBjj`%Qw6VUK}KYD>>5=|g0w85e}$r| zPuPjJ24ELdo$Pg;@qo-HAVUSwr=&55j*yCO{MNCY(0S5cOM`vDF4FL@W4-jfa6xr z;J%?pi@J`7GpAk|y^~pSJG0{Y>>KV#W=Ax$BMiTZsju++LkD25{_oJ?>;fIhfE02H zY(7EeEQ_DAXKPWLB0vOqx4GzB5@bYYw`Ab1%U$DlX7DVd7!n%bCIxwtq>;3@bT>f7 zO?czaB(q)53Y+Lako1xPZgyGxW{-GZc1D8cBqLrmc);Tc!LxbEZqoie4%R>ZHuq1pL(T47Lh~MoV%FNxH!>D|*GA z)DqMwHDrqI7O80GOa%3}ijy~9+ESZTZ%e9A_=PBy1ohwjU%ou3As)%n^)OZ>R}6nKR-5QFTQcnKAYo*-RQZQkoK%JHjaavwNRX<7 z{C4+0DB1V4-L}1gT#ye6!G~-wdkr3+7ziz9z;HhQ0k%ya8=C4L8Je830RgFPp{>q# z2CklgfZIDq(^!vfGsr@6Huvq!w+263UOS>1wi5c~w0Q1cw^cI+R|FEV(engIRHxa_ zGAD}5U1j9KHzpeK+Y?%RXQEsnhs(<+vI{%vHbbf%Qcv1yjx;6F79GLOCz{!R#&ahs z*wgs=6Q|qTx1^UdHq-H>od<2`{Y1BrV^CwL#>-fDacST#LAnpR5Xz z1UEq3ft>pEiJ$wr%^51+!fUg-5Fve&MVk`y! zY_e9Yi3qh(p*Aek;{Tc)WjpYJsSC!A7{eLjfZv;{E^UjL+M}lS@SY#srmm=|iw~Q+@DqWGZ@4qOyybAnBM7#!xxbAdu9=S}!ZxM}zS|#u-fmS?yx-?^j zkYqck+BsQ~z2{K}e(tm_L~o|l^W8*(xV)i)ne51DE=LAuYcEMyUf?{h#Xp0JMVfwpkEr1?2a`HiRLog7*rijoS6`I3BbIi0ZKfcUW`&m}| z;uA53e7EC0p5fY}Y-^08oLQ`U@@QvvN^b@8VY#K343q)%Pv4+B30ZkL__h&Uox(Jl^*uu^sPA96N5~E}k@=O#GhX3A6RixJfDp z2S_PUC@B|8Q3)7+1Za&!F-24=P?7=#iE1Vqk@QJrlc@BeN}q#3K=cWH=e#%LogZA! zJ3i;0<(+%(x#ynu=DXarpK;c2SuAD(<1NF73qKfm!CJ&W(7#X_9S!aG?&pZaNrqrv zDBqhOvUzPPZ44HK3cZD)B5zU1?zMA-Co+Gb7;WGH{D6iLBAHgrFRa@dOEMF(<0ePQ ztMF*UTMEhsc!UcFGjmdwl+B`oqt9tV(`B+#vPjm|eK!Trghv~##Ong?z6<4<8QHbN zb_M$)S1**V9Af@ln~)7X5r4o6jT-o_Dk@m}O=Hv?xR?HBI z_qtEJ!$$4@MeZWGB+QVDoLsTuL5;$^0J+J7Kk#V7TP0VedHK6|)fry3Z5OX*j|ddV z)tW7#P%2_MVzs??4=MYGRIzkGm8t`&t&uDNx?ZkH+j2;y5Z4lR$!_IW8ekdutUkx; zk`8`EU4^%Ck9RmXv+-ac4uuGAayeM@8J!T@(ganI(dCzVWz48&M4@{#%w$y)3=RGd>YnkC5*nV=P!E0nk z9L7QR1ZMv7kUX?Hygm1&5qSWjH@Yi&6S6}Z)3S@&p_K5!N$(^?bn?Q<%wptz>9{n$ zdSZM0pwRHA)WB3W*?AnxvYOd*mO9pH-N)Xse>d#v+!}OgtP)dH-bX| z6d@3f)5A!*5V#RkBJd!n0^s2&W>ySqvXc%1RE)vEyc~=A!*l}3Pp^H|SlV{aZ>oSs1eFM`tuR3qmAQ*`seh(Aux;OrcNClE-;63;~< zK{|`{=9|tUi6BYXGl!qk-gFk;WM}p6f9w6oKF69XAJ(I_rz_Yu4izptD+BY+^yYdf zo<5I-{wJ5Hj5bt<0*I*&sjeB2L~Yf9ot`Zq1HpUn#NGsOHT}oRJg?`dOHgGVN5O`* zKYHHN>mG1sI=c4LhA(kk0ehrrv1BtlvO!GI1>$ljMy;sQ%RXrGW@*|;huCflM+1%(X$q6iE!rTq_HJsY)Kef*axjIn(LCr`joLgVXSAL zZ2JT^xn^$f6|+5@VuT%z`e{sNCy$P}Fzj%?___u~ycnU9qL(5ee;^zafZ=prrGxDC zqitL}d%wGw>AT$AB&+LMuKEl*rs>k>09VAh1wW;q19VdpWu9lRbysu8n9$u|z|eU3 z2kc1qq-UoA`dO65zGE-_0eF31sj9D|p9djD9}Aoh2f~WMKQ|Xygd@UOSI>C$o@N!@ zlZ!IFyqDn<>~}p4R?V;?f~`3H8Y?+Av|Io$nLG~w#vhJ(tV%Y7Ef_dgr-qI`3)6}| zEXRY9IYk$YT$HKmJpCpz1b_H4?LkZziv|O6MF@ebLdbuK9)nq`Zhj|V9wSxvSu>}a zmOy7NGC*ZIfT9@DP0KMOkWL)Ey}jB|&jY`F3-Tfj#NvTDALxSWYan)b&DQJZ?e%D& z4}g*##UkvRiv)dvOLT7S^Zfx){}E)5vVRZ1%1yIZM%uVfu|JL+UfxJgMYkA@!s0OT z6ibxKQUI!Olt#`$3Zbrgih2=?0)PeKyR=oQ)$sdB;;~Y)GPT<$%`1AnOC+h@b=;xmb*osJ4K#vAO4;vS=5y(>Pi%K{lR8`pRG=G!U_IhqNgBVPso?a z61e~dv~h)t^X!H3GFDzy#Hm=Gu)VF|h$bP3OB_2}Wz(dVd1$?LH~5{hlKu)W8LkNM zUyQ@sGUH>fnMmA}p$OT^TjJv8EgYzq+1g%MveHUlzgJT`LPum6Q1XjBP^T17qNd4)m_u%N+ZbbT5+|yp*$=yWO%H z5s+ZdI&xXgLkGD+*6@%+2df6npY1PWp((@io}tmlH$J#vbN>}D*?jfK{<G zEnt5-Rjs8S3k>9^Q?+!bwX|mkwJrZoo(aw+VKbH4tHXtraXfhGvBWP4OX8A#wd|${ zxd)FnP7#lb4L>scg-7Q*FrJf&X`==|otHlK&dT=MbS?2flZHi5tBz^~DPlJsZRUQ! zl8+wZn%F-cecaWtAx&RKS!_W(f?6UqWr=I*?`V z)5j0|*C#`_7e8q>DctWoQ0PHiJS%a zyajG`#RxFF-ve%fc(_ddGE3QY=l0r3tb0{mivhmK0qn(UlNgtl>ER&AL11OSoOR{RAd0D{=;7N;4nIY6+5gQtmpgD?O|=2UdJ$myDcRbm zQ2M%4(^Ac^<0vtX;0Xk;AW(0~GQ?a6$`Po&dj(=x$!Ht_Rxx@4fttgWge-5(WLfOT!q|A1T8S+DkA z8%tPAR|NQ*xLeVBr=sU#d>H$CAe5DdX|`Isugd+!u8DpbQ{Ea@Gsh z9MaQ#UDrcDebiV(n)jHrHw@PyRJB3@d*@$ bool: + """Check if URL is on known legitimate domain whitelist.""" + try: + from urllib.parse import urlparse + + domain = urlparse(url).netloc.lower() + # Strip www. for comparison + domain_no_www = domain.replace("www.", "") + return ( + domain in KNOWN_LEGITIMATE_DOMAINS + or domain_no_www in KNOWN_LEGITIMATE_DOMAINS + ) + except Exception: + return False + + # Configure logging with more detail logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" @@ -49,32 +99,40 @@ PRIMARY_CONFIG = MODEL_CONFIG.get("primary", {}) SHADOW_CONFIG = MODEL_CONFIG.get("shadow", {}) -# Environment variable overrides +# Environment variable overrides - PRIMARY NOW USES 8-FEATURE MODEL PRIMARY_MODEL_PATH = Path( - os.getenv("MODEL_PATH", PRIMARY_CONFIG.get("path", "models/dev/model_7feat.pkl")) + os.getenv("MODEL_PATH", PRIMARY_CONFIG.get("path", "models/dev/model_8feat.pkl")) ) PRIMARY_META_PATH = Path( os.getenv( "MODEL_META_PATH", - PRIMARY_CONFIG.get("meta_path", "models/dev/model_7feat_meta.json"), + PRIMARY_CONFIG.get("meta_path", "models/dev/model_8feat_meta.json"), ) ) -SHADOW_ENABLED = ( - os.getenv("SHADOW_ENABLED", str(SHADOW_CONFIG.get("enabled", True))).lower() - == "true" -) -SHADOW_MODEL_PATH = Path( - os.getenv( - "SHADOW_MODEL_PATH", SHADOW_CONFIG.get("path", "models/dev/model_8feat.pkl") +# Shadow mode disabled for production (Option A: 8-feature primary only) +SHADOW_ENABLED = os.getenv("SHADOW_ENABLED", "false").lower() == "true" + +SHADOW_MODEL_PATH: Optional[Path] +SHADOW_META_PATH: Optional[Path] + +if SHADOW_ENABLED: + SHADOW_MODEL_PATH = Path( + os.getenv( + "SHADOW_MODEL_PATH", SHADOW_CONFIG.get("path", "models/dev/model_7feat.pkl") + ) ) -) -SHADOW_META_PATH = Path( - os.getenv( - "SHADOW_META_PATH", - SHADOW_CONFIG.get("meta_path", "models/dev/model_8feat_meta.json"), + SHADOW_META_PATH = Path( + os.getenv( + "SHADOW_META_PATH", + SHADOW_CONFIG.get("meta_path", "models/dev/model_7feat_meta.json"), + ) ) -) + print(f"Shadow mode ENABLED: {SHADOW_MODEL_PATH}") +else: + SHADOW_MODEL_PATH = None + SHADOW_META_PATH = None + print("Shadow mode DISABLED (production mode)") # ============================================================ # GLOBAL MODEL STORAGE @@ -164,6 +222,8 @@ async def lifespan(app: FastAPI): # Load shadow model if enabled if SHADOW_ENABLED: + assert SHADOW_MODEL_PATH is not None + assert SHADOW_META_PATH is not None _shadow_model, _shadow_meta = load_model_artifact( SHADOW_MODEL_PATH, SHADOW_META_PATH ) @@ -224,6 +284,20 @@ class PredictResponse(BaseModel): ) +class ExplainRequest(BaseModel): + url: str = Field(..., min_length=1, max_length=2048, description="URL to explain") + + +class ExplainResponse(BaseModel): + p_malicious: float = Field( + ..., description="Probability that URL is malicious (0.0-1.0)" + ) + feature_contributions: dict = Field(..., description="SHAP values for each feature") + feature_values: dict = Field(..., description="Feature values for the input URL") + source: str = Field(..., description="Prediction source: 'model' or 'heuristic'") + model_name: Optional[str] = Field(None, description="Primary model identifier") + + # ============================================================ # FEATURE ENGINEERING # ============================================================ @@ -392,6 +466,89 @@ def predict_with_model( # ============================================================ # API ENDPOINTS # ============================================================ +@app.post("/predict/explain", response_model=ExplainResponse) +def explain(request: ExplainRequest): + """ + Return SHAP feature contributions for a given URL using the primary model. + """ + url = request.url + if _primary_model is None: + return JSONResponse( + status_code=503, content={"error": "Primary model not loaded"} + ) + + # Fast path: Check whitelist BEFORE calling model + if _check_whitelist(url): + return ExplainResponse( + p_malicious=0.01, + feature_contributions={}, + feature_values={}, + source="whitelist", + model_name="domain-whitelist", + ) + + # Extract features + try: + features_df = engineer_features_for_model(url, _primary_feature_order) + # Convert numpy values to Python floats for JSON serialization + feature_values = {k: float(v) for k, v in dict(features_df.iloc[0]).items()} + except Exception as e: + return JSONResponse( + status_code=400, content={"error": f"Feature extraction failed: {e}"} + ) + + # Compute prediction + try: + p_malicious = float( + _primary_model.predict_proba(features_df)[0][_primary_phish_col_ix] + ) + except Exception as e: + return JSONResponse( + status_code=500, content={"error": f"Model prediction failed: {e}"} + ) + + # SHAP explainability + try: + # For CalibratedClassifierCV, we need to access the base estimator + # Try TreeExplainer first (for XGBoost), fallback to KernelExplainer + try: + # Access the base estimator from CalibratedClassifierCV + base_estimator = _primary_model.calibrated_classifiers_[0].estimator + explainer = shap.TreeExplainer(base_estimator) + shap_values = explainer.shap_values(features_df) + # For binary classification, shap_values might be a list [neg, pos] + if isinstance(shap_values, list): + shap_values = shap_values[_primary_phish_col_ix] + # Convert numpy values to Python floats for JSON serialization + contributions = { + k: float(v) for k, v in zip(features_df.columns, shap_values[0]) + } + except Exception as tree_err: + logger.warning(f"TreeExplainer failed: {tree_err}, trying KernelExplainer") + + # Fallback to KernelExplainer (slower but more general) + def model_predict(X): + return _primary_model.predict_proba(X)[:, _primary_phish_col_ix] + + explainer = shap.KernelExplainer(model_predict, features_df) + shap_values = explainer.shap_values(features_df) + # Convert numpy values to Python floats for JSON serialization + contributions = { + k: float(v) for k, v in zip(features_df.columns, shap_values[0]) + } + except Exception as e: + logger.error(f"SHAP explainability failed: {e}", exc_info=True) + return JSONResponse( + status_code=500, content={"error": f"SHAP explainability failed: {str(e)}"} + ) + + return ExplainResponse( + p_malicious=p_malicious, + feature_contributions=contributions, + feature_values=feature_values, + source="model", + model_name=PRIMARY_CONFIG.get("name", "primary"), + ) @app.get("/health") @@ -426,6 +583,16 @@ def predict(request: PredictRequest): """ Predict phishing probability with extensive debug logging. """ + # Fast path: Check whitelist BEFORE calling model + if _check_whitelist(request.url): + logger.info(f"βœ“ WHITELIST HIT: {request.url} - bypassing model prediction") + return PredictResponse( + p_malicious=0.01, + source="whitelist", + model_name="domain-whitelist", + shadow=None, + ) + url = request.url logger.info(f"\n\n{'#' * 60}") @@ -448,7 +615,7 @@ def predict(request: PredictRequest): url, _primary_feature_order, _primary_phish_col_ix, - model_name="PRIMARY (7-feature)", + model_name="PRIMARY (8-feature)", ) source = "model" model_name_primary = PRIMARY_CONFIG.get("name", "primary") @@ -475,6 +642,7 @@ def predict(request: PredictRequest): shadow_result = None + # Shadow model (only if enabled) if SHADOW_ENABLED and _shadow_model is not None and source == "model": try: p_malicious_shadow = predict_with_model( @@ -482,9 +650,12 @@ def predict(request: PredictRequest): url, _shadow_feature_order, _shadow_phish_col_ix, - model_name="SHADOW (8-feature)", + model_name="SHADOW (7-feature)", ) + # Log shadow prediction details + logger.info(f"Shadow prediction: {p_malicious_shadow:.6f}") + agreement = abs(p_malicious_primary - p_malicious_shadow) < 0.1 shadow_result = ShadowPrediction( @@ -502,6 +673,7 @@ def predict(request: PredictRequest): ) except Exception as e: + logger.warning(f"Shadow prediction failed: {e}") logger.error(f"\nβœ— SHADOW MODEL FAILED: {e}", exc_info=True) # ======================================== diff --git a/test_shap_locally.py b/test_shap_locally.py new file mode 100644 index 0000000..b4f57ec --- /dev/null +++ b/test_shap_locally.py @@ -0,0 +1,101 @@ +""" +Quick test to verify SHAP works with our CalibratedClassifierCV model +""" + +import joblib +import pandas as pd +import shap + +# Load the model +model = joblib.load("models/dev/model_8feat.pkl") +print(f"Model type: {type(model)}") + +# Create sample features (phishing URL features) +features_df = pd.DataFrame( + [ + { + "IsHTTPS": 0.0, + "TLDLegitimateProb": 0.017663043478260868, + "CharContinuationRate": 0.05714285714285714, + "SpacialCharRatioInURL": 0.19444444444444445, + "URLCharProb": 1.0, + "LetterRatioInURL": 0.6666666666666666, + "NoOfOtherSpecialCharsInURL": 7.0, + "DomainLength": 23.0, + } + ] +) + +print(f"\nFeatures shape: {features_df.shape}") +print(f"Features:\n{features_df}") + +# Test prediction +pred = model.predict_proba(features_df) +print(f"\nPrediction: {pred}") +print(f"Phishing probability: {pred[0][0]}") + +# Test SHAP TreeExplainer +print("\n" + "=" * 60) +print("Testing SHAP TreeExplainer...") +print("=" * 60) +try: + # Access the base estimator from CalibratedClassifierCV + base_estimator = model.calibrated_classifiers_[0].estimator + print(f"Base estimator type: {type(base_estimator)}") + + explainer = shap.TreeExplainer(base_estimator) + shap_values = explainer.shap_values(features_df) + + print(f"SHAP values type: {type(shap_values)}") + print(f"SHAP values: {shap_values}") + + # For binary classification, shap_values might be a list [neg, pos] + if isinstance(shap_values, list): + print(f"SHAP values is a list with {len(shap_values)} elements") + shap_values_phish = shap_values[0] # Index 0 for phishing class + else: + shap_values_phish = shap_values + + print(f"SHAP values for phishing class: {shap_values_phish}") + + contributions = dict(zip(features_df.columns, shap_values_phish[0])) + print("\nFeature contributions:") + for feat, contrib in sorted( + contributions.items(), key=lambda x: abs(x[1]), reverse=True + ): + print(f" {feat:35s}: {contrib:+.6f}") + + print("\nβœ“ TreeExplainer SUCCESS!") + +except Exception as e: + print(f"\nβœ— TreeExplainer FAILED: {e}") + import traceback + + traceback.print_exc() + + # Try KernelExplainer fallback + print("\n" + "=" * 60) + print("Testing SHAP KernelExplainer (fallback)...") + print("=" * 60) + try: + + def model_predict(X): + return model.predict_proba(X)[:, 0] # Phishing class + + explainer = shap.KernelExplainer(model_predict, features_df) + shap_values = explainer.shap_values(features_df, nsamples=100) + + contributions = dict(zip(features_df.columns, shap_values[0])) + print("\nFeature contributions:") + for feat, contrib in sorted( + contributions.items(), key=lambda x: abs(x[1]), reverse=True + ): + print(f" {feat:35s}: {contrib:+.6f}") + + print("\nβœ“ KernelExplainer SUCCESS!") + + except Exception as ke: + print(f"\nβœ— KernelExplainer FAILED: {ke}") + import traceback + + traceback.print_exc() diff --git a/tests/__pycache__/test_gateway_e2e.cpython-311-pytest-8.4.1.pyc b/tests/__pycache__/test_gateway_e2e.cpython-311-pytest-8.4.1.pyc index f47458b8264c764eee44e6419653ae4fffb50f0c..c8e4d344fc5cff2ea7045278b66a74764f710251 100644 GIT binary patch delta 535 zcmaDH(HY6NoR^o20SJE1exH%VzL9Sk596)PTX~-Gu%z&$Gfn<0qdGZC@hnRXLkj<7 zbwy)#fg0vC#uUNHiHgSrN*F;pAs|I4MR@X3X?=N-T6QLe8s-}IWkA{0a22&2H5|po zDWa1dR1ak)F&fugEAfSy$ob zW;q28Mn-fuoo1AstgF;0R3(s-pPQJOr(2$pS(2KQSzI!?URiqbI;D$~XDO@dsb-Xv z6jvpN6On66(`SB*;W65 zo3|sQ$L0cy-3;Fg+O`*@?SRY<8xYM6VnWbW7P~7jHbexXgvIV6SOy~BVS~&A8Uf)# zGyvIf;}9bHEOtP>aN{8wkhP)dnaruRbn{#_aVAd79cC9fovv^?ZQiTV#VjMtDF1;0 KlPK~8MkWAPOsE(D delta 323 zcmeB-d>p~IoR^o20SM$3&t#ln+sL!WJfvM$p$hpjKY(9WpbHC z7BfzsFR98dTEm>im?AbgQSlf{4I9vo&$7ns;$T4ukRYmEk||P?ndKE3rGZZRxmiwu zgOL&LDz+4cUfkhNfdbkg8~3$ds;F8 diff --git a/tests/__pycache__/test_gateway_model_integration.cpython-311-pytest-8.4.1.pyc b/tests/__pycache__/test_gateway_model_integration.cpython-311-pytest-8.4.1.pyc index 7775c5d60dac54e53888741b09c57b44e2147d87..ef694db244b52e32418a7af0df9acdda35dcf9d5 100644 GIT binary patch delta 2775 zcmbuBUu;uV7{Kqj?Y(W+b)|pCR`#!B19tbXD`Pllw<;pr1UI^|A&QK$+jSSZt(@CA z7#md+lR+RxP7EqD`l7N31acpI@P+syBx;J8ajA)k55y23)&=4N@xk-m%f_gp1#fPD z_d9>SbH4MP?|i4`{u33IzZ3bG>(cfJ=%P{AeI3f#Pmy)lW zM`d{;FCo!-Y1L=6!xzv=eGntEJ&a%@ZtfPB(v9ILd5`Zx#ma5|GC~~+F|I{b^Y((I zt1fA)pLZ6d4UDOJkR3xunpyOV7wMk0jLD51FQ#Z08~*3JgxybpELvu z>@rVfxBB3XR7?rPYb+$YEH<**(uM@(fW@mfHG{%d3UV+vpIo+;b@-sh4`>0vC^%aK z44MJ8prcyhkimhP6>iOp4XPP4b}JdQzkoc-C3_LaYi5w|3MvnjfwCJ=4EP`B;A%R6 zY@~Q#0JVU070NuUS6(F7><;DU!ee~%M9EM*9^2~i$YXL$8jwcE7a%C~#lX)ifSgYOOIw!?Macg{DTyL#T$e8Z8ny+q$M$2iLt8J&|x55=^R zglg|AXV&gSXp=Huv5n_cLRceJ%C<- z4bVq1Yu7eO4;2}kv={1L20ZGV#J|U{w&4N{Zc$y7vJQ_99O)kjMWg-0qhdI!+3JCj z_W@o3cmb~h_5)r6)B_Fx`YB`!1&x8+r8sayB=poo&>rQhnhMil%|*1+MDc@2fPJqU zq!S+s4IHTt9}5k~qG1gF%f@}Gm#ok^(}%>n!+ zFjQ*+0#eqCC3l({(MD3TIl>>Nin5nnuW8k{CCl3twzdnAKc4%P+-&|mzcabJ`wN7HtuBNNz)(_9pp`a$l{z~Cul;L$`SL$)nviH z&rmRBoHU4qq^re+TtsTAGp(64P3i|(v50)!a>`OXqf<3_u|zB-O|4;6MlQ5ABsRmc zmfeo+P|P|ym))B(jSorja8#1x0!p#ch=3;S>#AbMy@KqPM%*B766DMvmY4*cPK|N! z58QB3bTB8hz6z;cEp^dw(?g=9XlwM(bkq-BMe$_UV%=OLPdWs3}l+dAOA| zR8`ep2hEzoO0^6f<-#^SXKtfr-+@b1V98z1lZ$hxjLm=@;7hu3<{3?yHEa6C zP(1Wh2li~cKo9Jf?LFwUV(vI=NQ9^*kt=P#PB^bD1k@w)Xi|z{X%tKG<4Z{?qTNn6 z0~oG~?A@Y#>Z~xKV8^HECuKY|CPm}Yh>R=9w|$Mg`X9khxV@c~sxMXJPJ@NZ2H!XO={+|V d4Y%TRWPIR_0<9Kj``2ySj;Bs@}H0 z`}Dj%=Q;oToU6@}k7kMERfl7hz^^yYe>(Kr8;-9pho+Z@Mv&&Oq~&vCSH=1LZ>1$sIu=Y`$>hiReGI}6WUel zMsnb4{+6q9bziML8(}uw>;pQvy&Z; zkZoUe6G+_&&1-*JJh+AXPpg1xpa!S~>VSG+D~Bx6?XdHJCZHME3AAvi5{E~>sE;|a zqE<9l4-l607i%wPxy({4+U{ZRWH-j|f!PMM1L#XLb;N2ovIBiq^v23-4R1Wu$L-|i zNJxzY27)6afzdD3YG*uh_Zg*uc3(lvto@+Y{1|SJs?TfXn5dwS_4XG#PC(u@c@Qb0En4} zfTO@5pzD1I)-Z>Pl)?8kE4{ZKzPs##caQ|MMQ@qC%LoygV|OXSlu4BIV7SnDX{gwl zZ5zlVY-U^C&=J%Z0*LIQGpMQ|8u4tZ96#MGt4R~T(NSI_y9^P8(bP}^tc=&m?%v+O zr4rpfC`EZkwVD0yt<-+@Q^KaI%-YGub0l}q2mN!lz4NxcSJGd4g4NgywEE_8arLA* zW{a7R^Q4B%MKR zp7w?RIl^L%(Gz)day(0NUn3Ge3yZEl5Qg(1jfR~VHZ;97@r`WvTn}nvj+Am#B(~%{4 zcNbRK0o(_40>^-HU=qM1Kpz93sTqOphK&Q4+S5QrQyW=-k8eY%g};GtESLmIp0`xu z-a+Oq?I|;=S_ylvr`lz=$oy3%jkRpCr?Jpw*+)_Wh(5LRmYS5Pmu+PC_I4HP(_%(B zho0rPq^&R diff --git a/tests/__pycache__/test_judge_llm_adapter.cpython-311-pytest-8.4.1.pyc b/tests/__pycache__/test_judge_llm_adapter.cpython-311-pytest-8.4.1.pyc index 9d755c8b6e75bcb2047119b608ec69ee6c9807f5..479359423db2a4b3738b8c94e15d6c9b67467bd3 100644 GIT binary patch delta 978 zcmZ9K%WD%s9LIMy$!@ZnO%uDdXSP*LMvKLpQ#i@t!ZbQth7m-tk{-R zMgIVWDT1X6UR1D`l8fL;Jt=~MmvsfHD3p587WGmOI+Lh~Klpz5z4`61ThX_1*F8xR z7-FM~Uor`C#kF7&+$6LReRbny%asU#9OR)LI-meW=!6n-sJimfughMnq*-RAL65+ypQlF-zSBZtmSQPSPysL<95{& zh7I_!eHt|4Hb*<(RFuhG%~*4E{++8AB2vqLR}4vx889!;+aQxu81bzTR1#2GMF?yBZ;1b zn$~n}I;E?FC^v4(-I)~X&SiBidp@OWx$H2hm^I0KYT|^Jnxa*b(Q>`nW5WqkB%rDC zs5_zRy83U~*Pk1h9MCf=O6Jvx-{XbKsIwn(1)-4BH|ED3{xcW4_}}|aiUW+LIftOXH1Yc*|Qf+cH}TxpoKFODT;B5 z2?~g1f6R7=xcf_^{uY*;a52~(D)08v2708)=TmxSN*hP?zd%Rv%i!+a^c@i$MAZ}? wiYkIFd)IUho}W@<=spPnQQ>amMAyMnTgUzElBG3m)I+&%$v_Y9iG{>_VM+m0Po$ud~HlR zLaxwZ@<)7k+U|f&y+GQb*?2+DmbQ&1LXxE)o*28FKQflM_j-i5h(tGGFt@{#6?Nq87U#-95r&anAY$2(smDdWzdjxfs!Qu1M4#_$E>?4OC8?c|@#u#E8aUC&%xB-=LkMxSk zw$k~?X`b}M?N};Sy)uXmIIyGS?&PQEX71YflSQ8$>mwGOW$-SNyg-aYn#fDg5M@%02r@)AEkXyXLqN+cZaR zUtV`_)iQ()*QXh}ak?mbtm*`8(ku6Q5>sMk%4qWK0ZUA)Aok7wa0o$T`-N0zJ z&qimJTdt^s2&{B8%wANuTWAmtB#?!&hQ{pG4nYp3G9e_p1evy@%0zuIsU;v~|WDh{-m5=s-NDZ>Dct2|mSS{}Y8&LWKehcYUPF0;VTl+ym2iOmxdU6zx zW}{J*3^T&F@Iy$xjqnZv9$1Teqf`r&jpwTU^zG5Fw)=>-`%IViJ&MuyP;&<1)lAX(=w1*uLNUZ2S5CK;l7ADj+Wa#i4Qw_Ptd?f8 zxvQbc(-W<+uL`2Y=>kKFQzb4EcZEywhPPp)uc}L#D zI?Q2~h(-bj@0k|hMAzKnAu*VDr-;URSg5>?{Sxgisaj~qGcbPe&i?W)(_xHS>#z}| z-b6T#@D{=-!WaV9S18%*J=JmfVk#%4Q*v&v4fi+2Sv2yhj+`Xtu#_?#KwtQl(IW`# zj!@mE_r(e1Pa^Dfh!3T%t+ zbpxZ>>ox^9hF{PQ0fql&_`GFbwOUA&4UAS-Xr=I~7Mje=NQN#CYF(~cli9445w%B3 zElEomc%KH?{m}qi!pdfqq-Y0bdEaIHucwNYjx}25VDYrlKlaoV>;+P&P;H5XI5(Y0 zsP?J(R63W+$h3|Pj&~8BT^jE^QG?#`|Ape?M)8@^t}wlTR1yKJN73;HN71Qv3;(u9 zXXnKEv@}GofDA4PCqH3@@hPvJ^cJ`x_*rry%9}~Jz!kycC73${*0Y}`{H_A$A;*Ze F$3MNo*Kq&< delta 1911 zcmZ`)Urbw781E_lNB>Yo`e$nky|4n8G5(Zly9yNO9OxX*za!DCDco9{bZtEs<{TT~ zgGsY2ik@g#vWNL3O?+^<4<>u@!Dym!2`Ed*jgJ~bd>DHnhQ!40yT{e?XE&E$f9Lm| z?>pc9zH<%_mdS57iQ`L$!>WU)MfmxhPwqJWHn4x{c1%+Jt2(hsAto1566UMKIsjAb z@F(vm-&-d2i<>s*a{Z0Onjy531t=$0vcK3u!1#OL5ZZXexq>r*t!oBuE+=Y7D_K6` z(}8nRMXrD+I(KVk?ojJion1?mA8dp5q@4+32kUY3}HH6Y!B0J$1p4S_jfDlUyGtxS2`MiNnq{wag$8Bl1H-3uNgG5GdQ(@7MPEGV4l8$o&o;YEZv!dZZdT}zuz zqOdpX~n*Sua04~#Yb`R?11+CnxLYlA z^fr>D@@}u4@LG|}pAu2A*(VJq$qq-(#nRzUN40k9f7MH9t<$5?~+1zS+Ms7L%|j)jsePvgZ)S5hf5YCHexw2*P1oW8W6iYT#7sXEX9F z_Fkq@^xS{WP{Veyis*Xc9FAT_=tJmd{keih^%7Ja?#dJLt1*gWR|A1E7sh54~mF*Ow0S7Qs@SWIv4s+D3qiPSr@$ zg>2^W$HJ|e(xpHk$ODsyVtCw|Y`klA}GS^dj)`13UE6W4Dr> z#sXL~%8$t{y)Ob(Y*NjslvJEarBuuGTrNMGE66m+?wuYW&5VqP&*DW)@gJCWNm9J8 zwEKWwK$b?p-ck%)(NPSl9gJPe;Kiaem(L8-_dvsP$iK6t@oA@-v~TOG(8L~&4>ntf Ve_K}tcLRayR-u(mP6P}X`ClVv$>{(9 diff --git a/tests/test_enhanced_routing.py b/tests/test_enhanced_routing.py new file mode 100644 index 0000000..210294e --- /dev/null +++ b/tests/test_enhanced_routing.py @@ -0,0 +1,199 @@ +""" +Tests for enhanced short domain routing logic in judge_wire.py + +Test Scenarios: +1. High confidence cases (no judge) +2. Gray zone cases (standard judge routing) +3. Short domain edge cases (enhanced judge routing) +4. Whitelist fast path (handled in main.py) +""" + +from common.thresholds import Thresholds +from gateway.judge_wire import ( + _extract_domain, + _should_route_to_judge_for_short_domain, + decide_with_judge, +) + +# Constants for short domain routing (matching judge_wire.py) +SHORT_DOMAIN_LENGTH = 10 +SHORT_DOMAIN_CONFIDENCE = 0.5 + + +# Mock thresholds for testing +MOCK_THRESHOLDS = Thresholds( + low=0.004, + high=0.999, + t_star=0.35, + gray_zone_rate=0.109, +) + + +class TestDomainExtraction: + """Test domain extraction helper.""" + + def test_extract_valid_domain(self): + assert _extract_domain("https://example.com/path") == "example.com" + assert _extract_domain("http://sub.example.com") == "sub.example.com" + + def test_extract_short_domain(self): + assert _extract_domain("https://npm.org") == "npm.org" + assert _extract_domain("https://bit.ly/abc") == "bit.ly" + + def test_extract_malformed_url(self): + assert _extract_domain("not-a-url") == "" + assert _extract_domain("") == "" + + +class TestShortDomainRouting: + """Test short domain routing logic.""" + + def test_short_legitimate_domain_moderate_confidence(self): + """Short domain with p < 0.5 should route to judge.""" + url = "https://npm.org/package" + p_malicious = 0.35 + assert _should_route_to_judge_for_short_domain(url, p_malicious) is True + + def test_short_domain_high_confidence(self): + """Short domain with p >= 0.5 should NOT route (high suspicion).""" + url = "https://evil.io/phish" + p_malicious = 0.75 + assert _should_route_to_judge_for_short_domain(url, p_malicious) is False + + def test_long_domain_moderate_confidence(self): + """Long domain should NOT route via short domain path.""" + url = "https://verylongdomainname.com/path" + p_malicious = 0.35 + assert _should_route_to_judge_for_short_domain(url, p_malicious) is False + + def test_boundary_cases(self): + """Test boundary conditions.""" + # Exactly 10 chars (should trigger) + url_10 = "https://tenchar.co" + assert len("tenchar.co") == 10 + assert _should_route_to_judge_for_short_domain(url_10, 0.4) is True + + # 11 chars (should NOT trigger) + url_11 = "https://elevenchar.co" + assert len("elevenchar.co") == 13 # Actually longer + assert _should_route_to_judge_for_short_domain(url_11, 0.4) is False + + # Exactly p = 0.5 (boundary) + url_short = "https://bit.ly" + assert _should_route_to_judge_for_short_domain(url_short, 0.5) is False + assert _should_route_to_judge_for_short_domain(url_short, 0.499) is True + + +class TestEnhancedDecisionLogic: + """Integration tests for enhanced decision logic.""" + + def test_high_confidence_allow(self): + """p < low threshold should ALLOW without judge.""" + url = "https://example.com" + p_malicious = 0.001 + outcome = decide_with_judge(url, p_malicious, MOCK_THRESHOLDS) + + assert outcome.final_decision == "ALLOW" + assert outcome.policy_reason == "policy-band" + assert outcome.judge is None + + def test_high_confidence_block(self): + """p > high threshold should BLOCK without judge.""" + url = "https://phishing-site.evil" + p_malicious = 0.999 + outcome = decide_with_judge(url, p_malicious, MOCK_THRESHOLDS) + + assert outcome.final_decision == "BLOCK" + assert outcome.policy_reason == "policy-band" + assert outcome.judge is None + + def test_gray_zone_standard(self): + """Normal domain in gray zone should invoke judge.""" + url = "https://suspicious-but-long-domain.com" + p_malicious = 0.35 + outcome = decide_with_judge(url, p_malicious, MOCK_THRESHOLDS) + + # Judge should be invoked + assert outcome.judge is not None + # Decision depends on judge verdict + assert outcome.final_decision in ["ALLOW", "REVIEW", "BLOCK"] + # Reason should indicate judge was used + assert "judge" in outcome.policy_reason + + def test_short_domain_gray_zone(self): + """Short domain in gray zone should have enhanced routing.""" + url = "https://npm.org" + p_malicious = 0.35 + outcome = decide_with_judge(url, p_malicious, MOCK_THRESHOLDS) + + # Judge should be invoked + assert outcome.judge is not None + # Reason should indicate short domain handling + assert "short-domain" in outcome.policy_reason + + +class TestEdgeCases: + """Test edge cases and error handling.""" + + def test_malformed_url(self): + """Malformed URL should still process.""" + url = "not-a-valid-url" + p_malicious = 0.35 + # Should not crash + outcome = decide_with_judge(url, p_malicious, MOCK_THRESHOLDS) + assert outcome.final_decision in ["ALLOW", "REVIEW", "BLOCK"] + + def test_empty_extras(self): + """Empty extras should not crash.""" + url = "https://example.com" + p_malicious = 0.35 + outcome = decide_with_judge(url, p_malicious, MOCK_THRESHOLDS, extras={}) + assert outcome is not None + + def test_none_extras(self): + """None extras should not crash.""" + url = "https://example.com" + p_malicious = 0.35 + outcome = decide_with_judge(url, p_malicious, MOCK_THRESHOLDS, extras=None) + assert outcome is not None + + +# Example test cases for manual validation +MANUAL_TEST_CASES = [ + # (url, p_malicious, expected_behavior) + ("https://github.com", 0.001, "Whitelist fast path (main.py)"), + ("https://npm.org", 0.35, "Short domain β†’ judge"), + ("https://bit.ly/abc", 0.45, "Short domain β†’ judge"), + ("https://t.co/xyz", 0.40, "Short domain β†’ judge"), + ( + "https://evil.io/phish", + 0.75, + "Short domain but high confidence β†’ standard gray zone", + ), + ("https://legitimate-company.com", 0.35, "Normal domain β†’ standard gray zone"), + ("https://phishing-site.evil", 0.999, "High confidence β†’ BLOCK (no judge)"), + ("https://safe-site.com", 0.001, "High confidence β†’ ALLOW (no judge)"), +] + +if __name__ == "__main__": + print("\n" + "=" * 60) + print("MANUAL TEST CASES - Expected Behavior") + print("=" * 60) + + for url, p_mal, expected in MANUAL_TEST_CASES: + print("\nURL: {url}") + print(f" p_malicious: {p_mal:.3f}") + print(f" Expected: {expected}") + + # Check if short domain routing applies + is_short = _should_route_to_judge_for_short_domain(url, p_mal) + if is_short: + print("βœ“ Short domain routing triggered") + + # Show threshold classification + if p_mal < MOCK_THRESHOLDS["low"]: + print(f" β†’ Policy: ALLOW (p < {MOCK_THRESHOLDS['low']})") + elif p_mal > MOCK_THRESHOLDS["high"]: + print(f" β†’ Policy: BLOCK (p > {MOCK_THRESHOLDS['high']})") + else: + print("β†’ Policy: REVIEW (gray zone)") diff --git a/tests/test_gateway_e2e.py b/tests/test_gateway_e2e.py index ddee62e..bdbf95b 100644 --- a/tests/test_gateway_e2e.py +++ b/tests/test_gateway_e2e.py @@ -33,10 +33,11 @@ def _predict(url: str, p: float): def test_allow_review_block_paths(): - # Below low => ALLOW (no judge) + # Below low => ALLOW (whitelist or policy band) j1 = _predict("http://example.com/", 0.05) assert j1["decision"] == "ALLOW" - assert j1["reason"] == "policy-band" + # example.com is whitelisted, so expect whitelist reason + assert j1["reason"] in ["policy-band", "domain-whitelist"] assert j1["judge"] is None # Inside band => REVIEW path (judge runs; reason starts with 'judge-') @@ -46,8 +47,8 @@ def test_allow_review_block_paths(): # mapping depends on stub rules assert j2["judge"] is not None # judge invoked - # At/above high => BLOCK (no judge) - j3 = _predict("http://example.com/?id=999", 0.95) + # At/above high => BLOCK (no judge) - use value above high threshold (0.999) + j3 = _predict("http://suspicious-domain.test/?id=999", 0.9995) assert j3["decision"] == "BLOCK" assert j3["reason"] == "policy-band" assert j3["judge"] is None diff --git a/tests/test_gateway_model_integration.py b/tests/test_gateway_model_integration.py index 88d9830..e8fa460 100644 --- a/tests/test_gateway_model_integration.py +++ b/tests/test_gateway_model_integration.py @@ -34,19 +34,7 @@ def test_call_model_service_success(self, mock_post): assert result == 0.75 mock_post.assert_called_once_with( "http://localhost:9000/predict", - json={ - "url": "http://example.com", - "extras": { - "TLDLegitimateProb": None, - "NoOfOtherSpecialCharsInURL": None, - "SpacialCharRatioInURL": None, - "CharContinuationRate": None, - "URLCharProb": None, - "url_len": None, - "url_digit_ratio": None, - "url_subdomains": None, - }, - }, + json={"url": "http://example.com"}, timeout=3.0, ) @@ -84,7 +72,8 @@ def test_call_model_service_invalid_response(self): def test_predict_with_p_malicious_provided(self): """Test /predict when p_malicious is provided by caller.""" response = client.post( - "/predict", json={"url": "http://example.com", "p_malicious": 0.8} + "/predict", + json={"url": "http://suspicious-domain.test", "p_malicious": 0.8}, ) assert response.status_code == 200 @@ -121,16 +110,19 @@ def test_predict_fallback_to_heuristic(self, mock_call_model): mock_call_model.return_value = None with patch.dict(os.environ, {"MODEL_SVC_URL": "http://localhost:9000"}): - response = client.post("/predict", json={"url": "http://example.com"}) + response = client.post( + "/predict", json={"url": "http://test-fallback.example"} + ) assert response.status_code == 200 data = response.json() + # Should fall back to heuristic when model service fails assert data["source"] == "heuristic" # Valid probability from heuristic assert 0.0 <= data["p_malicious"] <= 1.0 # Verify model service was attempted - mock_call_model.assert_called_once_with("http://example.com", {}) + mock_call_model.assert_called_once_with("http://test-fallback.example", {}) def test_predict_no_model_service_url(self): """Test /predict when MODEL_SVC_URL is not set.""" @@ -139,8 +131,8 @@ def test_predict_no_model_service_url(self): assert response.status_code == 200 data = response.json() - # Should fall back to heuristic - assert data["source"] == "heuristic" + # Should fall back to heuristic or whitelist + assert data["source"] in ["heuristic", "whitelist"] assert 0.0 <= data["p_malicious"] <= 1.0 @patch("gateway.main._call_model_service") @@ -182,7 +174,7 @@ def test_predict_priority_order(self): response = client.post( "/predict", json={ - "url": "http://example.com", + "url": "http://test-priority.example", # Use non-whitelisted domain "p_malicious": 0.2, # Caller's value should win }, ) diff --git a/tests/test_judge_llm_adapter.py b/tests/test_judge_llm_adapter.py index 618d8d2..025fc0a 100644 --- a/tests/test_judge_llm_adapter.py +++ b/tests/test_judge_llm_adapter.py @@ -33,6 +33,16 @@ def test_judge_llm_parsing(monkeypatch): req = JudgeRequest( url="http://ex.com/login", features=FeatureDigest( + # 8-feature model (required fields) + IsHTTPS=0, + TLDLegitimateProb=0.15, # low legitimacy + CharContinuationRate=0.30, + SpacialCharRatioInURL=0.20, + URLCharProb=0.25, # low probability + LetterRatioInURL=0.60, + NoOfOtherSpecialCharsInURL=3, + DomainLength=7, # short domain + # Legacy features (optional) url_len=120, url_digit_ratio=0.22, url_subdomains=3, diff --git a/tests/test_model_svc.py b/tests/test_model_svc.py index 1f01db5..5db9792 100644 --- a/tests/test_model_svc.py +++ b/tests/test_model_svc.py @@ -16,7 +16,8 @@ def test_health_endpoint(): data = response.json() assert data["status"] == "ok" assert data["service"] == "model-svc" - assert data["version"] == "0.1.0" + # Accept current version format + assert data["version"] in ["0.1.0", "0.2.0-debug"] def test_predict_endpoint_basic(): @@ -73,12 +74,8 @@ def test_predict_endpoint_invalid_input(): def test_predict_endpoint_empty_url(): """Test predict endpoint with empty URL.""" response = client.post("/predict", json={"url": ""}) - assert response.status_code == 200 - data = response.json() - - # Should still return valid response - assert "p_malicious" in data - assert "source" in data + # Empty URLs should be rejected with validation error + assert response.status_code == 422 def test_predict_endpoint_various_urls(): @@ -95,7 +92,8 @@ def test_predict_endpoint_various_urls(): assert response.status_code == 200 data = response.json() assert 0.0 <= data["p_malicious"] <= 1.0 - assert data["source"] in ["model", "heuristic"] + # Accept whitelist as valid source (some domains are whitelisted) + assert data["source"] in ["model", "heuristic", "whitelist"] def test_heuristic_scoring_consistency(): From eab22eede2054641792b6c8d767d131a0ca58c3c Mon Sep 17 00:00:00 2001 From: Fitsum <138158520+fitsblb@users.noreply.github.com> Date: Thu, 16 Oct 2025 10:47:58 -0400 Subject: [PATCH 2/2] Updated gitignore --- .gitignore | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b62baa2..ee7ee3a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,12 @@ __pycache__/ *.py[cod] *$py.class *.so +*.pyc +*.pyo + +# Pytest cache +.pytest_cache/ +**/.pytest_cache/ # Package build artifacts *.egg-info/ @@ -41,9 +47,24 @@ mlruns/ outputs/*.png outputs/*.csv - +# MLflow database mlflow.db +# Great Expectations cache +gx/uncommitted/ + +# Temporary test files +test_shap_locally.py +model_logs.txt + +# Jupyter notebook checkpoints +.ipynb_checkpoints/ + +# Coverage reports +.coverage +htmlcov/ +*.cover + docs/*.md