feat: add rag API

Author: lu-yg

Dockerfile

@@ -16,7 +16,7 @@ COPY --from=build /app/app/target/tiny-engine-app-*.jar /app/tiny-engine-app.jar
 COPY --from=build /app/base/target/tiny-engine-base-*.jar /app/tiny-engine-base.jar
 # 设置环境变量
 
-ENV FOLDER_PATH = "/app/documents"
+ENV FOLDER_PATH="/app/documents"
 # 替换为自己的域名接口路径
 ENV TINY_ENGINE_URL="https://agent.opentiny.design/material-center/api/resource/download"
 ENTRYPOINT ["java", "-jar", "tiny-engine-app.jar", "--spring.profiles.active=alpha"]

base/src/main/java/com/tinyengine/it/controller/AiChatController.java

@@ -12,15 +12,10 @@
 
 package com.tinyengine.it.controller;
 
-import com.tinyengine.it.common.base.Result;
 import com.tinyengine.it.common.log.SystemControllerLog;
 import com.tinyengine.it.model.dto.ChatRequest;
 
-import com.tinyengine.it.rag.service.StorageService;
-import com.tinyengine.it.rag.entity.EmbeddingMatchDto;
 import com.tinyengine.it.service.app.v1.AiChatV1Service;
-import dev.langchain4j.data.segment.TextSegment;
-import dev.langchain4j.store.embedding.EmbeddingMatch;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.media.Content;
@@ -40,9 +35,6 @@
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
 
-import java.util.List;
-import java.util.stream.Collectors;
-
 /**
  * The type Ai chat controller.
  *

base/src/main/java/com/tinyengine/it/controller/VectorStorageController.java

@@ -26,6 +26,7 @@
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.tags.Tag;
+import jakarta.validation.constraints.NotEmpty;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.validation.annotation.Validated;
 import org.springframework.web.bind.annotation.DeleteMapping;
@@ -136,11 +137,11 @@ public Result<List<EmbeddingMatchDto>> searchInCollection(@RequestBody SearchReq
             @ApiResponse(responseCode = "400", description = "请求失败")
     })
     @SystemControllerLog(description = "跨集合搜索")
-    @GetMapping("/vector-storage/all-collections")
+    @PostMapping("/vector-storage/all-collections")
     public Result<Map<String, List<EmbeddingMatchDto>>> searchAllCollections(
         @RequestBody SearchRequest searchDto) {
         Map<String, List<EmbeddingMatchDto>> results =
-                vectorStorageService.searchAcrossCollections(searchDto);
+            vectorStorageService.searchAcrossCollections(searchDto);
         return Result.success(results);
     }
 
@@ -219,7 +220,7 @@ public Result<DeleteResult> deleteByFilePath(@RequestParam String filePath, @Req
     @SystemControllerLog(description = "通过路径和集合名称批量删除知识库文档")
     @DeleteMapping("/vector-storage/batch/{collection}")
     public Result<BatchDeleteResult> deleteMultipleFiles(@PathVariable String collection,
-        @RequestBody List<String> filePaths) {
+        @RequestBody @NotEmpty List<@NotEmpty String> filePaths) {
         BatchDeleteResult result = vectorStorageService.deleteMultipleFiles(filePaths,collection);
         return Result.success(result);
     }

base/src/main/java/com/tinyengine/it/rag/config/RAGConfig.java

@@ -34,7 +34,7 @@ public class RAGConfig {
     private String chromaBaseUrl = "http://localhost:8000";
     private String chromaCollectionName = "tinyengine_documents";
     private String modelPath = "./all-MiniLM-L6-v2/model.onnx";
-    private String tokenizerPath = ".//all-MiniLM-L6-v2/tokenizer.json";
+    private String tokenizerPath = "./all-MiniLM-L6-v2/tokenizer.json";
 
     // 连接配置
     private int timeoutSeconds = 30;

base/src/main/java/com/tinyengine/it/rag/service/StorageService.java

@@ -132,6 +132,9 @@ private boolean isValidCollection(String collectionName) {
     public VectorDocument autoAddFolderToKnowledgeBase() {
         try {
             String folderPath = System.getenv("FOLDER_PATH");
+            if (folderPath == null || folderPath.isBlank()) {
+                throw new ServiceException(ExceptionEnum.CM329.getResultCode(), "FOLDER_PATH does not exist: " + folderPath);
+            }
             // 验证文件夹路径
             Path folder = Paths.get(folderPath);
             if (!Files.exists(folder) || !Files.isDirectory(folder)) {
@@ -585,26 +588,30 @@ public DeleteResult deleteByFilePath(String filePath, String collectionName) {
      */
     private List<EmbeddingMatch<TextSegment>> searchBySource(String sourcePath, String collectionName) {
         try {
+            // 使用更合理的查询文本
+            String queryText = "document content analysis";
+
             EmbeddingSearchRequest searchRequest = EmbeddingSearchRequest.builder()
-                .queryEmbedding(embeddingModel.embed("test").content())
-                .maxResults(10000)
-                .minScore(0.0)
-                .build();
+                    .queryEmbedding(embeddingModel.embed(queryText).content())
+                    .maxResults(1000)
+                    .minScore(0.1)
+                    .build();
 
             List<EmbeddingMatch<TextSegment>> allMatches = embeddingStore.search(searchRequest).matches();
 
-            // 根据源文件路径和集合名称过滤
+            // 在应用层过滤
             return allMatches.stream()
-                .filter(match -> {
-                    String source = match.embedded().metadata().getString("source");
-                    String collection = match.embedded().metadata().getString("collection");
+                    .filter(match -> {
+                        String source = match.embedded().metadata().getString("source");
+                        String collection = match.embedded().metadata().getString("collection");
 
-                    boolean sourceMatch = source != null && source.equals(sourcePath);
-                    boolean collectionMatch = collectionName == null ||
-                        (collection != null && collection.equals(collectionName));
+                        boolean sourceMatch = source != null && source.equals(sourcePath);
+                        boolean collectionMatch = collectionName == null ||
+                                (collection != null && collection.equals(collectionName));
 
-                    return sourceMatch && collectionMatch;
-                }).collect(Collectors.toList());
+                        return sourceMatch && collectionMatch;
+                    })
+                    .collect(Collectors.toList());
 
         } catch (Exception e) {
             log.error("Failed to search vectors by source: {} in collection: {}", sourcePath, collectionName, e);