Query reformulation

README.md

@@ -8,15 +8,15 @@ A chatbot that provides semantic search and conversational AI capabilities for C
 ## Key Features
 
 ### Core RAG Capabilities
-- **Semantic RAG Pipeline**: Advanced semantic understanding with intent classification and query expansion
+- **Semantic RAG Pipeline**: Advanced semantic understanding with query reformulation
 - **Dual Pipeline Architecture**: Both standard and semantic-enhanced RAG processing
 - **Intelligent Query Processing**: Handles misleading keywords and provides contextually accurate responses
-- **Multi-Query Retrieval**: Semantic query expansion with domain-specific mappings
+- **Vector Retrieval**: Direct vector search with optional semantic re-ranking
 - **Semantic Re-ranking**: Re-ranks results based on semantic similarity and intent matching
 
 ### Advanced Semantic Features  
 - **Intent Classification**: Automatically detects query intent (team_info, project_info, technical_info, feature_info)
-- **Query Expansion**: Generates semantic variations using domain-specific terminology
+- **Query Reformulation**: LLM-based coreference resolution and context clarification
 - **Multilingual Support**: Optimized for French and English content with synonym handling
 - **Contextual Understanding**: Maintains document hierarchy awareness for numerical and structured queries
 
@@ -75,7 +75,6 @@ A chatbot that provides semantic search and conversational AI capabilities for C
 
    # Optional - Semantic Features
    USE_SEMANTIC_FEATURES=true
-   SEMANTIC_EXPANSION_ENABLED=true
    SEMANTIC_RERANKING_ENABLED=true
 
    # Optional - Azure Integration (for production)
@@ -213,7 +212,7 @@ Isschat/
 │   ├── rag/                   # RAG pipeline implementation
 │   │   ├── pipeline.py        # Standard RAG pipeline
 │   │   ├── semantic_pipeline.py # Semantic-enhanced RAG pipeline
-│   │   ├── query_processor.py # Advanced query processing
+│   │   ├── reformulation_service.py # LLM-based query reformulation
 │   │   └── tools/             # RAG tools (retrieval, generation)
 │   ├── storage/               # Storage abstraction
 │   │   ├── storage_factory.py # Storage factory (local/Azure)
@@ -255,7 +254,7 @@ Isschat/
 
 ### Semantic Intelligence
 - **Intent Classification**: Automatically detects and routes queries based on intent (team_info, project_info, technical_info, feature_info)
-- **Query Expansion**: Semantic expansion using domain-specific mappings and synonym handling
+- **Query Reformulation**: LLM-based coreference resolution using conversation context
 - **Context-Aware Retrieval**: Maintains document hierarchy awareness for complex queries
 - **Multilingual Processing**: Optimized for French and English content with cross-language understanding
 

flux_donnees.mmd → data_flow.mmd

@@ -13,8 +13,10 @@ graph TD
     ChatInterface -->|Query| SemanticPipeline[Semantic RAG Pipeline]
 
     %% Semantic Processing Pipeline
-    SemanticPipeline -->|Analyze query| QueryProcessor[Query Processor]
-    QueryProcessor -->|Intent & expansion| SemanticRetrieval[Semantic Retrieval Tool]
+    SemanticPipeline -->|Reformulate query| ReformulationService[Reformulation Service]
+    ReformulationService -->|LLM call| LLM
+    LLM -->|Reformulated query| ReformulationService
+    ReformulationService -->|Resolved query| SemanticRetrieval[Vector Retrieval Tool]
     SemanticRetrieval -->|Vector search| WeaviateDB[(Weaviate Vector DB<br>Collection: isschat_docs)]
     WeaviateDB -->|Relevant documents| SemanticRetrieval
     SemanticRetrieval -->|Ranked results| GenerationTool[Generation Tool]
@@ -74,7 +76,7 @@ graph TD
 
     subgraph "RAG Processing Engine"
         SemanticPipeline
-        QueryProcessor
+        ReformulationService
         SemanticRetrieval
         GenerationTool
         LLM
@@ -121,7 +123,7 @@ graph TD
 
     class User,WebApp,CLI,ChatInterface,AdminInterface,InteractiveCLI interface
     class AzureAuth,KeyVault auth
-    class SemanticPipeline,QueryProcessor,SemanticRetrieval,GenerationTool,LLM processing
+    class SemanticPipeline,ReformulationService,SemanticRetrieval,GenerationTool,LLM processing
     class WeaviateDB,DataManager,StorageSystem,LocalStorage,AzureStorage storage
     class FeaturesManager,HistoryManager,PerformanceDashboard components
     class IngestionPipeline,ConfluenceConnector,ConfluenceAPI,DocumentProcessor,DocumentChunker,EmbeddingService ingestion

rag_evaluation/core/llm_judge.py

@@ -36,7 +36,7 @@ def __init__(self, config: Any):
             temperature=config.judge_temperature,
             max_tokens=config.judge_max_tokens,
             openai_api_key=api_key,
-            openai_api_base="https://openrouter.ai/api/v1",
+            openai_api_base=config.openrouter_base_url,
         )
 
     def evaluate_conversational(self, question: str, response: str, expected: str, context: str = "") -> Dict[str, Any]:

src/config/settings.py

@@ -32,16 +32,24 @@ class IsschatConfig:
 
     # Semantic understanding configuration
     use_semantic_features: bool = True
-    semantic_expansion_enabled: bool = True
     semantic_reranking_enabled: bool = True
     semantic_similarity_threshold: float = 0.7
-    query_expansion_max_variations: int = 5
     intent_classification_enabled: bool = True
 
+    # OpenRouter API configuration
+    openrouter_base_url: str = "https://openrouter.ai/api/v1"
+    openrouter_timeout: int = 30
+
+    # Query reformulation configuration
+    force_reformulate_all_queries: bool = True
+    reformulation_timeout: int = 15  # Shorter for reformulation
+    reformulation_max_tokens: int = 150
+
     # Source filtering configuration
     source_filtering_enabled: bool = True
-    min_source_score_threshold: float = 0.4
-    min_source_relevance_threshold: float = 0.3
+    min_source_score_threshold: float = 0.3  # Réduit de 0.4 → 0.3
+    min_source_relevance_threshold: float = 0.2  # Réduit de 0.3 → 0.2
+    use_flexible_filtering: bool = True  # Enable flexible multi-criteria filtering
 
     confluence_api_key: str = ""
     confluence_space_key: str = ""
@@ -83,24 +91,23 @@ def from_env(cls, env_file: str = ".env") -> "IsschatConfig":
             search_fetch_k=int(os.getenv("SEARCH_FETCH_K", str(defaults.search_fetch_k))),
             use_semantic_features=os.getenv("USE_SEMANTIC_FEATURES", str(defaults.use_semantic_features)).lower()
             == "true",
-            semantic_expansion_enabled=os.getenv(
-                "SEMANTIC_EXPANSION_ENABLED", str(defaults.semantic_expansion_enabled)
-            ).lower()
-            == "true",
             semantic_reranking_enabled=os.getenv(
                 "SEMANTIC_RERANKING_ENABLED", str(defaults.semantic_reranking_enabled)
             ).lower()
             == "true",
             semantic_similarity_threshold=float(
                 os.getenv("SEMANTIC_SIMILARITY_THRESHOLD", str(defaults.semantic_similarity_threshold))
             ),
-            query_expansion_max_variations=int(
-                os.getenv("QUERY_EXPANSION_MAX_VARIATIONS", str(defaults.query_expansion_max_variations))
-            ),
             intent_classification_enabled=os.getenv(
                 "INTENT_CLASSIFICATION_ENABLED", str(defaults.intent_classification_enabled)
             ).lower()
             == "true",
+            force_reformulate_all_queries=os.getenv(
+                "FORCE_REFORMULATE_ALL_QUERIES", str(defaults.force_reformulate_all_queries)
+            ).lower()
+            == "true",
+            reformulation_timeout=int(os.getenv("REFORMULATION_TIMEOUT", str(defaults.reformulation_timeout))),
+            reformulation_max_tokens=int(os.getenv("REFORMULATION_MAX_TOKENS", str(defaults.reformulation_max_tokens))),
             source_filtering_enabled=os.getenv(
                 "SOURCE_FILTERING_ENABLED", str(defaults.source_filtering_enabled)
             ).lower()
@@ -111,6 +118,8 @@ def from_env(cls, env_file: str = ".env") -> "IsschatConfig":
             min_source_relevance_threshold=float(
                 os.getenv("MIN_SOURCE_RELEVANCE_THRESHOLD", str(defaults.min_source_relevance_threshold))
             ),
+            use_flexible_filtering=os.getenv("USE_FLEXIBLE_FILTERING", str(defaults.use_flexible_filtering)).lower()
+            == "true",
             confluence_api_key=secrets.get_confluence_api_key() or defaults.confluence_api_key,
             confluence_space_key=secrets.get_confluence_space_key() or defaults.confluence_space_key,
             confluence_space_name=secrets.get_confluence_space_name() or defaults.confluence_space_name,
@@ -173,10 +182,8 @@ def get_debug_info() -> dict:
             "data_dir": str(config.data_dir),
             "semantic_features": {
                 "use_semantic_features": config.use_semantic_features,
-                "semantic_expansion_enabled": config.semantic_expansion_enabled,
                 "semantic_reranking_enabled": config.semantic_reranking_enabled,
                 "semantic_similarity_threshold": config.semantic_similarity_threshold,
-                "query_expansion_max_variations": config.query_expansion_max_variations,
                 "intent_classification_enabled": config.intent_classification_enabled,
             },
         }

src/rag/__init__.py

@@ -4,5 +4,11 @@
 """
 
 from .pipeline import RAGPipeline, RAGPipelineFactory
+from .semantic_pipeline import SemanticRAGPipeline, SemanticRAGPipelineFactory
 
-__all__ = ["RAGPipeline", "RAGPipelineFactory"]
+__all__ = [
+    "RAGPipeline",
+    "RAGPipelineFactory",  # Legacy pipeline
+    "SemanticRAGPipeline",
+    "SemanticRAGPipelineFactory",  # Modern pipeline
+]