{ "name": "KA Query", "nodes": [ { "parameters": { "httpMethod": "POST", "path": "ka-eval", "responseMode": "responseNode", "options": {} }, "id": "30ccdd9b-5799-4f47-9e0f-a00e049e63d5", "name": "Eval Door", "type": "n8n-nodes-base.webhook", "typeVersion": 2.1, "position": [ 224, 304 ] }, { "parameters": { "assignments": { "assignments": [ { "id": "ni-question", "name": "question", "value": "={{ $json.body?.question ?? $json.question ?? \"\" }}", "type": "string" }, { "id": "ni-channel", "name": "channel", "value": "={{ $json.channel ?? \"eval\" }}", "type": "string" }, { "id": "ni-replychannel", "name": "replyChannel", "value": "={{ $json.replyChannel ?? \"\" }}", "type": "string" }, { "id": "ni-replyts", "name": "replyTs", "value": "={{ $json.replyTs ?? $json.body?.threadTs ?? \"\" }}", "type": "string" } ] }, "options": {} }, "id": "52567bde-0136-4c82-842c-9746a4069d44", "name": "Normalise Input", "type": "n8n-nodes-base.set", "typeVersion": 3.4, "position": [ 672, 304 ] }, { "parameters": { "trigger": [ "app_mention" ], "watchWorkspace": true, "options": {} }, "id": "f6143489-8f64-47b1-99cd-75797d2b82b5", "name": "Slack Door", "type": "n8n-nodes-base.slackTrigger", "typeVersion": 1, "position": [ 224, 480 ] }, { "parameters": { "assignments": { "assignments": [ { "id": "nsi-question", "name": "question", "value": "={{ ($json.text ?? $json.event?.text ?? \"\").replace(/<@[^>]+>/g, \"\").trim() }}", "type": "string" }, { "id": "nsi-channel", "name": "channel", "value": "slack", "type": "string" }, { "id": "nsi-replychannel", "name": "replyChannel", "value": "={{ $json.channel ?? $json.event?.channel ?? \"\" }}", "type": "string" }, { "id": "nsi-replyts", "name": "replyTs", "value": "={{ $json.thread_ts ?? $json.event?.thread_ts ?? $json.ts ?? $json.event?.ts ?? \"\" }}", "type": "string" } ] }, "options": {} }, "id": "3afc713b-8a08-4fd6-94cf-a36261462229", "name": "Normalise Slack Input", "type": "n8n-nodes-base.set", "typeVersion": 3.4, "position": [ 448, 480 ] }, { "parameters": { "mode": "load", "tableName": "document_vectors", "prompt": "={{ $json.question }}", "topK": 5, "options": {} }, "id": "0ec9bc15-b636-4c93-9917-c16d12dfbb2e", "name": "Retrieve Document Chunks", "type": "@n8n/n8n-nodes-langchain.vectorStorePGVector", "typeVersion": 1.3, "position": [ 880, 304 ] }, { "parameters": { "options": {} }, "id": "09098de8-ee1b-49cf-97d2-53d1d70bc49d", "name": "Query Embeddings", "type": "@n8n/n8n-nodes-langchain.embeddingsOpenAi", "typeVersion": 1.2, "position": [ 624, 560 ] }, { "parameters": { "jsCode": "const SIMILARITY_THRESHOLD = 0.5;\nconst input = $('Final Question').first().json;\nconst docs = $input.all().map(i => i.json);\nconst chunks = docs.map(d => ({\n text: d.document?.pageContent,\n source: d.document?.metadata?.source,\n similarity: 1 - Number(d.score ?? 1)\n})).filter(c => c.text);\nconst topScore = chunks.length ? Math.max(...chunks.map(c => c.similarity)) : 0;\nconst excerpts = chunks.map(c => `--- source: ${c.source} ---\\n${c.text}`).join('\\n\\n');\nreturn [{ json: {\n question: input.question,\n channel: input.channel,\n replyChannel: input.replyChannel,\n replyTs: input.replyTs,\n abstain: topScore < SIMILARITY_THRESHOLD,\n topScore: Math.round(topScore * 1000) / 1000,\n chunks,\n excerpts\n} }];" }, "id": "0f219dbf-60b3-4be5-b829-f7104f71d3e9", "name": "Threshold Gate", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 1104, 304 ] }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 1 }, "conditions": [ { "id": "abstain-true", "leftValue": "={{ $json.abstain }}", "rightValue": true, "operator": { "type": "boolean", "operation": "equals" } } ], "combinator": "and" }, "options": {} }, "id": "e1506ede-7a5c-4875-aea9-74d0ea959167", "name": "Abstained?", "type": "n8n-nodes-base.if", "typeVersion": 2.3, "position": [ 1328, 304 ] }, { "parameters": { "promptType": "define", "text": "=QUESTION: {{ $json.question }}", "messages": { "messageValues": [ { "message": "Decide whether the question asks about a specific customer, company account, contact, or CRM data (plans, MRR, account owner, trials, renewal, contact details). Reply with exactly YES or NO. Nothing else." } ] }, "batching": {} }, "id": "784258c4-5dfe-4b82-94c2-2928d0757a54", "name": "Live-Data Intent Check", "type": "@n8n/n8n-nodes-langchain.chainLlm", "typeVersion": 1.9, "position": [ 1504, 128 ] }, { "parameters": { "model": "qwen/qwen3.7-max", "options": { "temperature": 0 } }, "id": "6860b15d-f7f2-4c70-8d71-9384aff61ab4", "name": "Intent Model", "type": "@n8n/n8n-nodes-langchain.lmChatOpenRouter", "typeVersion": 1, "position": [ 1344, 64 ] }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 1 }, "conditions": [ { "id": "intent-yes", "leftValue": "={{ $json.text ?? \"\" }}", "rightValue": "YES", "operator": { "type": "string", "operation": "contains" } } ], "combinator": "and" }, "options": {} }, "id": "393b5ced-f28f-4ce4-bca1-0550a0860867", "name": "Live-Data Question?", "type": "n8n-nodes-base.if", "typeVersion": 2.3, "position": [ 1728, 128 ] }, { "parameters": { "promptType": "define", "text": "=QUESTION: {{ $(\"Threshold Gate\").first().json.question }}\n\nDOCUMENT EXCERPTS:\n{{ $(\"Threshold Gate\").first().json.excerpts }}", "options": { "systemMessage": "=Today's date is {{ $now.toFormat('yyyy-MM-dd') }} ({{ $now.toFormat('cccc') }}).\n\nYou answer questions using ONLY the document excerpts provided in the user message. Rules:\n1. After every factual claim, cite its source file in square brackets, e.g. [values/_index.md].\n2. If the excerpts answer the question, even indirectly, answer from them. Refuse only when they genuinely don't: if the question is not about a specific customer, contact, or account, reply \"The documents don't cover that.\", plus one short cited sentence on any related facts the excerpts do contain. Never include that phrase when you answer using the live_data_lookup tool.\n3. Use the live_data_lookup tool ONLY when the question asks about a specific customer, contact, or account. Prefix every fact that came from the tool with \"Live data:\". Never present tool data as document content.\n4. Be concise. No preamble.", "maxIterations": 5, "enableStreaming": false } }, "id": "4c176ed8-e6ad-4323-a69b-4b2fee295e9f", "name": "Answer Agent", "type": "@n8n/n8n-nodes-langchain.agent", "typeVersion": 3.1, "position": [ 1568, 432 ] }, { "parameters": { "model": "qwen/qwen3.7-max", "options": { "temperature": 0.2 } }, "id": "53fd753e-0b35-4d94-9c1d-a9026b177f53", "name": "Agent Model", "type": "@n8n/n8n-nodes-langchain.lmChatOpenRouter", "typeVersion": 1, "position": [ 1504, 560 ] }, { "parameters": { "operation": "executeQuery", "query": "SELECT name, company, email, plan, mrr, owner, last_contact, notes FROM crm_contacts WHERE name ILIKE '%' || $1 || '%' OR company ILIKE '%' || $1 || '%' LIMIT 5;", "options": { "queryReplacement": "={{ /*n8n-auto-generated-fromAI-override*/ $fromAI('search_term', 'The customer or company name to search for') }}", "treatQueryParametersInSingleQuotesAsText": false } }, "id": "710d9182-6278-471c-ab73-d85aa6f23a88", "name": "live_data_lookup", "type": "n8n-nodes-base.postgresTool", "typeVersion": 2.6, "position": [ 1664, 560 ] }, { "parameters": { "assignments": { "assignments": [ { "id": "ar-answer", "name": "answer", "value": "I don't have that in the documents. Ask [YOUR FALLBACK CONTACT].", "type": "string" }, { "id": "ar-abstained", "name": "abstained", "value": true, "type": "boolean" }, { "id": "ar-citations", "name": "citations", "value": "={{ [] }}", "type": "array" }, { "id": "ar-topscore", "name": "topScore", "value": "={{ $(\"Threshold Gate\").first().json.topScore }}", "type": "number" }, { "id": "ar-channel", "name": "channel", "value": "={{ $(\"Threshold Gate\").first().json.channel }}", "type": "string" }, { "id": "ar-faithful", "name": "faithful", "value": true, "type": "boolean" }, { "id": "ar-judgenotes", "name": "judgeNotes", "value": "abstained, judge not run", "type": "string" } ] }, "options": {} }, "id": "820ea825-7537-4ba9-8ecc-578b1cfd20a3", "name": "Abstain Reply", "type": "n8n-nodes-base.set", "typeVersion": 3.4, "position": [ 1952, 48 ] }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 1 }, "conditions": [ { "id": "channel-slack", "leftValue": "={{ $json.channel }}", "rightValue": "slack", "operator": { "type": "string", "operation": "equals" } } ], "combinator": "and" }, "options": {} }, "id": "e3ccb22e-9c97-4f1e-a151-7dda3467985e", "name": "Reply via Slack?", "type": "n8n-nodes-base.if", "typeVersion": 2.3, "position": [ 2704, 304 ] }, { "parameters": { "promptType": "define", "text": "=EXCERPTS:\n{{ $(\"Threshold Gate\").first().json.excerpts }}\n\nANSWER:\n{{ $json.output }}", "messages": { "messageValues": [ { "message": "You are grading whether an answer is faithful to its source excerpts. Every claim in the answer must be supported by the excerpts, except lines prefixed \"Live data:\" (those came from a database tool and are exempt) and the exact phrase \"The documents don't cover that.\" Reply on the first line with exactly FAITHFUL or UNFAITHFUL, then one sentence of reasoning." } ] }, "batching": {} }, "id": "b0e697f1-76d9-4294-a5bd-4c416a10e07e", "name": "Faithfulness Judge", "type": "@n8n/n8n-nodes-langchain.chainLlm", "typeVersion": 1.9, "position": [ 1824, 432 ] }, { "parameters": { "model": "qwen/qwen3.7-max", "options": { "temperature": 0 } }, "id": "b4e94042-b6f3-4859-a154-afaf9adb233c", "name": "Judge Model", "type": "@n8n/n8n-nodes-langchain.lmChatOpenRouter", "typeVersion": 1, "position": [ 1824, 560 ] }, { "parameters": { "jsCode": "const verdictRaw = $input.first().json.text ?? $input.first().json.output ?? '';\nconst firstLine = String(verdictRaw).trim().split('\\n')[0].toUpperCase();\nconst prev = $('Threshold Gate').first().json;\nconst answer = String($('Answer Agent').first().json.output ?? '');\nconst citations = [...answer.matchAll(/\\[([^\\]]+)\\]/g)].map(m => m[1]);\nreturn [{ json: {\n question: prev.question,\n channel: prev.channel,\n topScore: prev.topScore,\n answer,\n citations,\n abstained: false,\n faithful: firstLine.startsWith('FAITHFUL'),\n judgeNotes: String(verdictRaw).trim()\n} }];" }, "id": "36553a36-1e4a-4386-915f-94b6d1cef431", "name": "Parse Verdict", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 2048, 432 ] }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 1 }, "conditions": [ { "id": "faithful-true", "leftValue": "={{ $json.faithful }}", "rightValue": true, "operator": { "type": "boolean", "operation": "equals" } } ], "combinator": "and" }, "options": {} }, "id": "2caead05-4a7c-4115-8a8f-dcc5449baf21", "name": "Faithful?", "type": "n8n-nodes-base.if", "typeVersion": 2.3, "position": [ 2272, 432 ] }, { "parameters": { "assignments": { "assignments": [ { "id": "er-answer", "name": "answer", "value": "I drafted an answer but it failed the faithfulness check, so I'm not sending it. [YOUR FALLBACK CONTACT] has been flagged.", "type": "string" }, { "id": "er-citations", "name": "citations", "value": "={{ [] }}", "type": "array" } ] }, "includeOtherFields": true, "options": {} }, "id": "bcee7a09-555f-4b9c-9cce-ef9414fc8a37", "name": "Escalate Reply", "type": "n8n-nodes-base.set", "typeVersion": 3.4, "position": [ 2480, 544 ] }, { "parameters": { "select": "channel", "channelId": { "__rl": true, "mode": "id", "value": "={{ $(\"Threshold Gate\").first().json.replyChannel }}" }, "text": "={{ $json.answer }}", "otherOptions": { "includeLinkToWorkflow": false, "thread_ts": { "replyValues": { "thread_ts": "={{ $(\"Threshold Gate\").first().json.replyTs }}" } } } }, "id": "00fadcc7-0337-4e13-b971-53564d347e0f", "name": "Slack Reply", "type": "n8n-nodes-base.slack", "typeVersion": 2.5, "position": [ 2928, 128 ] }, { "parameters": { "options": {} }, "id": "c731959d-0f65-4564-8742-af5749925446", "name": "Respond to Eval Door", "type": "n8n-nodes-base.respondToWebhook", "typeVersion": 1.5, "position": [ 2928, 432 ] }, { "parameters": { "operation": "executeQuery", "query": "INSERT INTO qa_log (channel, question, answer, abstained, top_score, citations, faithful, judge_notes, thread_ts) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9);", "options": { "queryReplacement": "={{ [ $('Threshold Gate').first().json.channel, $('Threshold Gate').first().json.question, $json.answer ?? '', $json.abstained ?? false, $('Threshold Gate').first().json.topScore, JSON.stringify($json.citations ?? []), $json.faithful ?? null, $json.judgeNotes ?? '', $('Threshold Gate').first().json.replyTs || null ] }}" } }, "id": "b7770eaa-07c4-46ae-8fca-243650c360b7", "name": "Log QA", "type": "n8n-nodes-base.postgres", "typeVersion": 2.6, "position": [ 3152, 304 ] }, { "parameters": { "operation": "executeQuery", "query": "SELECT question, answer FROM qa_log WHERE thread_ts = $1 AND thread_ts <> '' ORDER BY id DESC LIMIT 4;", "options": { "queryReplacement": "={{ [ $json.replyTs ] }}" } }, "id": "33f759fa-906a-4148-b38b-6ffcca7a4fb7", "name": "Fetch Thread History", "type": "n8n-nodes-base.postgres", "typeVersion": 2.6, "position": [ 880, 300 ], "alwaysOutputData": true }, { "parameters": { "mode": "runOnceForAllItems", "language": "javaScript", "jsCode": "const input = $('Normalise Input').first().json;\nconst rows = $input.all().map(i => i.json).filter(r => r && r.question);\nconst history = rows.reverse().map(r => `User: ${r.question}\\nAgent: ${r.answer ?? ''}`).join('\\n\\n');\nreturn [{ json: {\n question: input.question,\n channel: input.channel,\n replyChannel: input.replyChannel,\n replyTs: input.replyTs,\n hasHistory: rows.length > 0,\n history\n} }];" }, "id": "e3199897-352b-4cd1-aa77-2c034f59d4c5", "name": "Prepare Condense", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 1100, 300 ] }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict" }, "conditions": [ { "id": "has-history", "leftValue": "={{ $json.hasHistory }}", "rightValue": true, "operator": { "type": "boolean", "operation": "equals" } } ], "combinator": "and" } }, "id": "0c84f933-f98c-4d64-9850-4d21080a4ec5", "name": "Needs Condense?", "type": "n8n-nodes-base.if", "typeVersion": 2.3, "position": [ 1320, 300 ] }, { "parameters": { "promptType": "define", "text": "=CONVERSATION SO FAR:\n{{ $json.history }}\n\nLATEST MESSAGE: {{ $json.question }}", "messages": { "messageValues": [ { "type": "SystemMessagePromptTemplate", "message": "Rewrite the user's latest message as a single standalone question, using the conversation so far to resolve references like \"they\", \"that\", or \"the second one\". If the message already stands alone, return it unchanged. Return only the question, nothing else." } ] } }, "id": "29d3385b-8482-4568-aaf5-0bccb776566f", "name": "Condense Question", "type": "@n8n/n8n-nodes-langchain.chainLlm", "typeVersion": 1.9, "position": [ 1540, 120 ] }, { "parameters": { "model": "qwen/qwen3.7-max", "options": { "temperature": 0 } }, "id": "b3854b14-7532-4519-93cf-4520e2b8470d", "name": "Condense Model", "type": "@n8n/n8n-nodes-langchain.lmChatOpenRouter", "typeVersion": 1, "position": [ 1380, 60 ] }, { "parameters": { "mode": "manual", "includeOtherFields": false, "assignments": { "assignments": [ { "id": "ac-question", "name": "question", "value": "={{ ($json.text ?? \"\").trim() || $(\"Prepare Condense\").first().json.question }}", "type": "string" }, { "id": "ac-channel", "name": "channel", "value": "={{ $(\"Prepare Condense\").first().json.channel }}", "type": "string" }, { "id": "ac-replychannel", "name": "replyChannel", "value": "={{ $(\"Prepare Condense\").first().json.replyChannel }}", "type": "string" }, { "id": "ac-replyts", "name": "replyTs", "value": "={{ $(\"Prepare Condense\").first().json.replyTs }}", "type": "string" } ] } }, "id": "17325d94-5d65-4bae-af2d-29d26e82f279", "name": "Adopt Condensed Question", "type": "n8n-nodes-base.set", "typeVersion": 3.4, "position": [ 1760, 120 ] }, { "parameters": { "mode": "manual", "includeOtherFields": false, "assignments": { "assignments": [ { "id": "fq-question", "name": "question", "value": "={{ $json.question }}", "type": "string" }, { "id": "fq-channel", "name": "channel", "value": "={{ $json.channel }}", "type": "string" }, { "id": "fq-replychannel", "name": "replyChannel", "value": "={{ $json.replyChannel ?? \"\" }}", "type": "string" }, { "id": "fq-replyts", "name": "replyTs", "value": "={{ $json.replyTs ?? \"\" }}", "type": "string" } ] } }, "id": "ebe241a7-1363-4e67-b44e-75b65625306c", "name": "Final Question", "type": "n8n-nodes-base.set", "typeVersion": 3.4, "position": [ 1980, 300 ] } ], "connections": { "Eval Door": { "main": [ [ { "node": "Normalise Input", "type": "main", "index": 0 } ] ] }, "Normalise Input": { "main": [ [ { "node": "Fetch Thread History", "type": "main", "index": 0 } ] ] }, "Slack Door": { "main": [ [ { "node": "Normalise Slack Input", "type": "main", "index": 0 } ] ] }, "Normalise Slack Input": { "main": [ [ { "node": "Normalise Input", "type": "main", "index": 0 } ] ] }, "Retrieve Document Chunks": { "main": [ [ { "node": "Threshold Gate", "type": "main", "index": 0 } ] ] }, "Query Embeddings": { "ai_embedding": [ [ { "node": "Retrieve Document Chunks", "type": "ai_embedding", "index": 0 } ] ] }, "Threshold Gate": { "main": [ [ { "node": "Abstained?", "type": "main", "index": 0 } ] ] }, "Abstained?": { "main": [ [ { "node": "Live-Data Intent Check", "type": "main", "index": 0 } ], [ { "node": "Answer Agent", "type": "main", "index": 0 } ] ] }, "Live-Data Intent Check": { "main": [ [ { "node": "Live-Data Question?", "type": "main", "index": 0 } ] ] }, "Intent Model": { "ai_languageModel": [ [ { "node": "Live-Data Intent Check", "type": "ai_languageModel", "index": 0 } ] ] }, "Live-Data Question?": { "main": [ [ { "node": "Answer Agent", "type": "main", "index": 0 } ], [ { "node": "Abstain Reply", "type": "main", "index": 0 } ] ] }, "Answer Agent": { "main": [ [ { "node": "Faithfulness Judge", "type": "main", "index": 0 } ] ] }, "Agent Model": { "ai_languageModel": [ [ { "node": "Answer Agent", "type": "ai_languageModel", "index": 0 } ] ] }, "live_data_lookup": { "ai_tool": [ [ { "node": "Answer Agent", "type": "ai_tool", "index": 0 } ] ] }, "Abstain Reply": { "main": [ [ { "node": "Reply via Slack?", "type": "main", "index": 0 } ] ] }, "Reply via Slack?": { "main": [ [ { "node": "Slack Reply", "type": "main", "index": 0 }, { "node": "Log QA", "type": "main", "index": 0 } ], [ { "node": "Respond to Eval Door", "type": "main", "index": 0 } ] ] }, "Faithfulness Judge": { "main": [ [ { "node": "Parse Verdict", "type": "main", "index": 0 } ] ] }, "Judge Model": { "ai_languageModel": [ [ { "node": "Faithfulness Judge", "type": "ai_languageModel", "index": 0 } ] ] }, "Parse Verdict": { "main": [ [ { "node": "Faithful?", "type": "main", "index": 0 } ] ] }, "Faithful?": { "main": [ [ { "node": "Reply via Slack?", "type": "main", "index": 0 } ], [ { "node": "Escalate Reply", "type": "main", "index": 0 } ] ] }, "Escalate Reply": { "main": [ [ { "node": "Reply via Slack?", "type": "main", "index": 0 } ] ] }, "Respond to Eval Door": { "main": [ [ { "node": "Log QA", "type": "main", "index": 0 } ] ] }, "Fetch Thread History": { "main": [ [ { "node": "Prepare Condense", "type": "main", "index": 0 } ] ] }, "Prepare Condense": { "main": [ [ { "node": "Needs Condense?", "type": "main", "index": 0 } ] ] }, "Needs Condense?": { "main": [ [ { "node": "Condense Question", "type": "main", "index": 0 } ], [ { "node": "Final Question", "type": "main", "index": 0 } ] ] }, "Condense Question": { "main": [ [ { "node": "Adopt Condensed Question", "type": "main", "index": 0 } ] ] }, "Condense Model": { "ai_languageModel": [ [ { "node": "Condense Question", "type": "ai_languageModel", "index": 0 } ] ] }, "Adopt Condensed Question": { "main": [ [ { "node": "Final Question", "type": "main", "index": 0 } ] ] }, "Final Question": { "main": [ [ { "node": "Retrieve Document Chunks", "type": "main", "index": 0 } ] ] } }, "settings": { "executionOrder": "v1" }, "pinData": {} }