feat: support semantic search in AI chat and embedding ability

docs/docs.go

@@ -11712,13 +11712,36 @@ const docTemplate = `{
                     "type": "string",
                     "maxLength": 256
                 },
+                "embedding_crontab": {
+                    "type": "string",
+                    "maxLength": 100
+                },
+                "embedding_dimensions": {
+                    "type": "integer"
+                },
+                "embedding_level": {
+                    "type": "string",
+                    "enum": [
+                        "question",
+                        "answer"
+                    ]
+                },
+                "embedding_model": {
+                    "type": "string",
+                    "maxLength": 100
+                },
                 "model": {
                     "type": "string",
                     "maxLength": 100
                 },
                 "provider": {
                     "type": "string",
                     "maxLength": 50
+                },
+                "similarity_threshold": {
+                    "type": "number",
+                    "maximum": 1,
+                    "minimum": 0
                 }
             }
         },

docs/swagger.json

@@ -11685,13 +11685,36 @@
                     "type": "string",
                     "maxLength": 256
                 },
+                "embedding_crontab": {
+                    "type": "string",
+                    "maxLength": 100
+                },
+                "embedding_dimensions": {
+                    "type": "integer"
+                },
+                "embedding_level": {
+                    "type": "string",
+                    "enum": [
+                        "question",
+                        "answer"
+                    ]
+                },
+                "embedding_model": {
+                    "type": "string",
+                    "maxLength": 100
+                },
                 "model": {
                     "type": "string",
                     "maxLength": 100
                 },
                 "provider": {
                     "type": "string",
                     "maxLength": 50
+                },
+                "similarity_threshold": {
+                    "type": "number",
+                    "maximum": 1,
+                    "minimum": 0
                 }
             }
         },

docs/swagger.yaml

@@ -2250,12 +2250,29 @@ definitions:
       api_key:
         maxLength: 256
         type: string
+      embedding_crontab:
+        maxLength: 100
+        type: string
+      embedding_dimensions:
+        type: integer
+      embedding_level:
+        enum:
+        - question
+        - answer
+        type: string
+      embedding_model:
+        maxLength: 100
+        type: string
       model:
         maxLength: 100
         type: string
       provider:
         maxLength: 50
         type: string
+      similarity_threshold:
+        maximum: 1
+        minimum: 0
+        type: number
     type: object
   schema.SiteAIReq:
     properties:

i18n/en_US.yaml

@@ -2355,6 +2355,24 @@ ui:
         label: Model
         msg: Model is required
       add_success: AI settings updated successfully.
+      embedding_settings: Embedding Settings
+      embedding_model:
+        label: Embedding model
+        text: "The model used to generate vector embeddings for semantic search (e.g. text-embedding-3-small)."
+      embedding_dimensions:
+        label: Embedding dimensions
+        text: "The number of dimensions for the embedding vectors (e.g. 1536 for text-embedding-3-small)."
+      embedding_level:
+        label: Embedding level
+        text: "Choose whether to create embeddings at the question level (question + all answers + comments) or answer level (each answer separately)."
+        question: Question level
+        answer: Answer level
+      embedding_crontab:
+        label: Embedding schedule
+        text: "Cron expression for periodic embedding calculation (e.g. '0 */6 * * *' for every 6 hours). Leave empty to disable automatic indexing."
+      similarity_threshold:
+        label: Similarity threshold
+        text: "Minimum cosine similarity score (0-1) for semantic search results. Only results with a score above this threshold will be returned. Default is 0 (no filtering)."
     conversations:
       topic: Topic
       helpful: Helpful

i18n/zh_CN.yaml

@@ -2319,6 +2319,24 @@ ui:
         label: 模型
         msg: 模型是必需的
       add_success: AI 设置更新成功。
+      embedding_settings: Embedding 设置
+      embedding_model:
+        label: Embedding 模型
+        text: "用于生成语义搜索向量 Embedding 的模型（例如 text-embedding-3-small）。"
+      embedding_dimensions:
+        label: Embedding 维度
+        text: "Embedding 向量的维度数（例如 text-embedding-3-small 为 1536）。"
+      embedding_level:
+        label: Embedding 级别
+        text: "选择在问题级别（问题 + 所有回答 + 评论）还是回答级别（每个回答单独）创建 Embedding。"
+        question: 问题级别
+        answer: 回答级别
+      embedding_crontab:
+        label: Embedding 计划
+        text: "定期计算 Embedding 的 Cron 表达式（例如 '0 */6 * * *' 表示每 6 小时）。留空则禁用自动索引。"
+      similarity_threshold:
+        label: 相似度阈值
+        text: "语义搜索结果的最低余弦相似度分数（0-1）。只有分数高于此阈值的结果才会被返回。默认值为 0（不过滤）。"
     conversations:
       topic: 主题
       helpful: 有帮助

internal/base/constant/ai_config.go

@@ -33,6 +33,7 @@ const (
 - get_tags: 搜索标签信息
 - get_tag_detail: 获取特定标签的详细信息
 - get_user: 搜索用户信息
+- semantic_search: 通过语义相似度搜索问题和答案。当用户的问题与现有内容概念相关但可能不匹配确切关键词时使用此工具。当 get_questions 关键词搜索返回较差结果时，请使用 semantic_search。
 
 请根据用户的问题智能地使用这些工具来提供准确的答案。如果需要查询系统信息，请先使用相应的工具获取数据。`
 	DefaultAIPromptConfigEnUS = `You are an intelligent assistant that can help users query information in the system. User question: %s
@@ -44,6 +45,7 @@ You can use the following tools to query system information:
 - get_tags: Search for tag information
 - get_tag_detail: Get detailed information about a specific tag
 - get_user: Search for user information
+- semantic_search: Search questions and answers by semantic meaning. Use this when the user's question relates conceptually to existing content but may not match exact keywords. When get_questions keyword search returns poor results, use semantic_search instead.
 
 Please intelligently use these tools based on the user's question to provide accurate answers. If you need to query system information, please use the appropriate tools to get the data first.`
 )

internal/controller/ai_controller.go

@@ -446,6 +446,7 @@ func (c *AIController) handleAIConversation(ctx *gin.Context, w http.ResponseWri
 		toolCalls, newMessages, finished, aiResponse := c.processAIStream(ctx, w, id, conversationCtx.Model, client, aiReq, messages)
 		messages = newMessages
 
+		log.Debugf("Round %d: toolCalls=%v", round+1, toolCalls)
 		if aiResponse != "" {
 			conversationCtx.Messages = append(conversationCtx.Messages, &ai_conversation.ConversationMessage{
 				Role:    "assistant",
@@ -497,6 +498,10 @@ func (c *AIController) processAIStream(
 			break
 		}
 
+		if len(response.Choices) == 0 {
+			continue
+		}
+
 		choice := response.Choices[0]
 
 		if len(choice.Delta.ToolCalls) > 0 {
@@ -735,6 +740,8 @@ func (c *AIController) callMCPTool(ctx context.Context, toolName string, argumen
 		result, err = c.mcpController.MCPTagDetailsHandler()(ctx, request)
 	case "get_user":
 		result, err = c.mcpController.MCPUserDetailsHandler()(ctx, request)
+	case "semantic_search":
+		result, err = c.mcpController.MCPSemanticSearchHandler()(ctx, request)
 	default:
 		return "", fmt.Errorf("unknown tool: %s", toolName)
 	}

internal/controller/mcp_controller.go

@@ -31,6 +31,7 @@ import (
 	answercommon "github.com/apache/answer/internal/service/answer_common"
 	"github.com/apache/answer/internal/service/comment"
 	"github.com/apache/answer/internal/service/content"
+	"github.com/apache/answer/internal/service/embedding"
 	"github.com/apache/answer/internal/service/feature_toggle"
 	questioncommon "github.com/apache/answer/internal/service/question_common"
 	"github.com/apache/answer/internal/service/siteinfo_common"
@@ -49,6 +50,7 @@ type MCPController struct {
 	userCommon       *usercommon.UserCommon
 	answerRepo       answercommon.AnswerRepo
 	featureToggleSvc *feature_toggle.FeatureToggleService
+	embeddingService *embedding.EmbeddingService
 }
 
 // NewMCPController new site info controller.
@@ -61,6 +63,7 @@ func NewMCPController(
 	userCommon *usercommon.UserCommon,
 	answerRepo answercommon.AnswerRepo,
 	featureToggleSvc *feature_toggle.FeatureToggleService,
+	embeddingService *embedding.EmbeddingService,
 ) *MCPController {
 	return &MCPController{
 		searchService:    searchService,
@@ -71,6 +74,7 @@ func NewMCPController(
 		userCommon:       userCommon,
 		answerRepo:       answerRepo,
 		featureToggleSvc: featureToggleSvc,
+		embeddingService: embeddingService,
 	}
 }
 
@@ -349,3 +353,131 @@ func (c *MCPController) MCPUserDetailsHandler() func(ctx context.Context, reques
 		return mcp.NewToolResultText(string(res)), nil
 	}
 }
+
+func (c *MCPController) MCPSemanticSearchHandler() func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) {
+	return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) {
+		if err := c.ensureMCPEnabled(ctx); err != nil {
+			return nil, err
+		}
+		cond := schema.NewMCPSemanticSearchCond(request)
+		if len(cond.Query) == 0 {
+			return mcp.NewToolResultText("Query is required for semantic search."), nil
+		}
+
+		siteGeneral, err := c.siteInfoService.GetSiteGeneral(ctx)
+		if err != nil {
+			log.Errorf("get site general info failed: %v", err)
+			return nil, err
+		}
+
+		results, err := c.embeddingService.SearchSimilar(ctx, cond.Query, cond.TopK)
+		if err != nil {
+			log.Errorf("semantic search failed: %v", err)
+			return mcp.NewToolResultText("Semantic search is not available. Embedding may not be configured."), nil
+		}
+		if len(results) == 0 {
+			return mcp.NewToolResultText("No semantically similar content found."), nil
+		}
+
+		resp := make([]*schema.MCPSemanticSearchResp, 0, len(results))
+		for _, r := range results {
+			var meta entity.EmbeddingMetadata
+			_ = json.Unmarshal([]byte(r.Metadata), &meta)
+
+			item := &schema.MCPSemanticSearchResp{
+				ObjectID:   r.ObjectID,
+				ObjectType: r.ObjectType,
+				Score:      r.Score,
+			}
+
+			// Compose link from metadata
+			if r.ObjectType == "answer" && meta.AnswerID != "" {
+				item.Link = fmt.Sprintf("%s/questions/%s/%s", siteGeneral.SiteUrl, meta.QuestionID, meta.AnswerID)
+			} else {
+				item.Link = fmt.Sprintf("%s/questions/%s", siteGeneral.SiteUrl, meta.QuestionID)
+			}
+
+			// Query content from DB using IDs stored in metadata
+			if r.ObjectType == "question" {
+				question, qErr := c.questioncommon.Info(ctx, meta.QuestionID, "")
+				if qErr != nil {
+					log.Warnf("get question %s for semantic search failed: %v", meta.QuestionID, qErr)
+				} else {
+					item.Title = question.Title
+					item.Content = question.Content
+				}
+
+				// Fetch answers by ID from metadata
+				for _, a := range meta.Answers {
+					answerEntity, exist, aErr := c.answerRepo.GetAnswer(ctx, a.AnswerID)
+					if aErr != nil || !exist {
+						continue
+					}
+					answerItem := &schema.MCPSemanticSearchAnswer{
+						AnswerID: a.AnswerID,
+						Content:  answerEntity.OriginalText,
+					}
+					// Fetch comments on this answer from DB
+					for _, ac := range a.Comments {
+						cmt, cExist, cErr := c.commentRepo.GetComment(ctx, ac.CommentID)
+						if cErr == nil && cExist {
+							answerItem.Comments = append(answerItem.Comments, &schema.MCPSemanticSearchComment{
+								CommentID: ac.CommentID,
+								Content:   cmt.OriginalText,
+							})
+						}
+					}
+					item.Answers = append(item.Answers, answerItem)
+				}
+
+				// Fetch question comments from DB
+				for _, qc := range meta.Comments {
+					cmt, cExist, cErr := c.commentRepo.GetComment(ctx, qc.CommentID)
+					if cErr == nil && cExist {
+						item.Comments = append(item.Comments, &schema.MCPSemanticSearchComment{
+							CommentID: qc.CommentID,
+							Content:   cmt.OriginalText,
+						})
+					}
+				}
+			} else if r.ObjectType == "answer" {
+				// Fetch question title for context
+				question, qErr := c.questioncommon.Info(ctx, meta.QuestionID, "")
+				if qErr == nil {
+					item.Title = question.Title
+				}
+
+				// Fetch answer content from DB
+				if meta.AnswerID != "" {
+					answerEntity, exist, aErr := c.answerRepo.GetAnswer(ctx, meta.AnswerID)
+					if aErr == nil && exist {
+						item.Content = answerEntity.OriginalText
+					}
+				} else if len(meta.Answers) > 0 {
+					answerEntity, exist, aErr := c.answerRepo.GetAnswer(ctx, meta.Answers[0].AnswerID)
+					if aErr == nil && exist {
+						item.Content = answerEntity.OriginalText
+					}
+				}
+
+				// Fetch answer comments from DB
+				if len(meta.Answers) > 0 {
+					for _, ac := range meta.Answers[0].Comments {
+						cmt, cExist, cErr := c.commentRepo.GetComment(ctx, ac.CommentID)
+						if cErr == nil && cExist {
+							item.Comments = append(item.Comments, &schema.MCPSemanticSearchComment{
+								CommentID: ac.CommentID,
+								Content:   cmt.OriginalText,
+							})
+						}
+					}
+				}
+			}
+
+			resp = append(resp, item)
+		}
+
+		data, _ := json.Marshal(resp)
+		return mcp.NewToolResultText(string(data)), nil
+	}
+}