{
  "meta": {
    "templateCredsSetupCompleted": true
  },
  "nodes": [
    {
      "id": "49b274c7-1ff2-43ba-acb9-349a8665b513",
      "name": "OpenAI Chat Model1",
      "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
      "position": [
        148,
        180
      ],
      "parameters": {
        "model": {
          "__rl": true,
          "mode": "list",
          "value": "gpt-4.1-mini",
          "cachedResultName": "gpt-4.1-mini"
        },
        "options": {}
      },
      "credentials": {
        "openAiApi": {
          "name": "<your credential>"
        }
      },
      "typeVersion": 1.2
    },
    {
      "id": "77f0c130-085f-4fa5-b74e-66fd084949b7",
      "name": "When fetching a dataset row",
      "type": "n8n-nodes-base.evaluationTrigger",
      "position": [
        -380,
        -140
      ],
      "parameters": {
        "sheetName": {
          "__rl": true,
          "mode": "list",
          "value": 2015506025,
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit#gid=2015506025",
          "cachedResultName": "Relevance"
        },
        "documentId": {
          "__rl": true,
          "mode": "list",
          "value": "1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y",
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=drivesdk",
          "cachedResultName": "96. Evaluations Test"
        }
      },
      "credentials": {
        "googleSheetsOAuth2Api": {
          "name": "<your credential>"
        }
      },
      "typeVersion": 4.6
    },
    {
      "id": "63d8ff44-5c4b-4cf6-b5b2-1f22de0de495",
      "name": "Remap Input",
      "type": "n8n-nodes-base.set",
      "position": [
        -160,
        -140
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "00924b90-278f-49f5-80f2-c297df0fcc97",
              "name": "chatInput",
              "type": "string",
              "value": "={{ $json.input }}"
            }
          ]
        }
      },
      "typeVersion": 3.4
    },
    {
      "id": "d8ac1b2b-f4ee-4af1-92f5-d194832c5c76",
      "name": "Evaluation",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        436,
        -40
      ],
      "parameters": {
        "operation": "checkIfEvaluating"
      },
      "typeVersion": 4.6
    },
    {
      "id": "1d8d9c97-0c90-4878-95fa-0dc6d3ad82c6",
      "name": "Set Input Fields",
      "type": "n8n-nodes-base.set",
      "position": [
        656,
        -140
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "d58952c1-d346-4fbf-881e-d5c04b6781a5",
              "name": "question",
              "type": "string",
              "value": "={{ $('When fetching a dataset row').first().json.input }}"
            },
            {
              "id": "0f10a3d0-cf6e-4715-9ded-2cee54aa62ec",
              "name": "answer",
              "type": "string",
              "value": "={{ $json.output }}"
            }
          ]
        }
      },
      "typeVersion": 3.4
    },
    {
      "id": "d4ad7c5e-f1cf-463b-b9b3-a45ff5597199",
      "name": "No Operation, do nothing",
      "type": "n8n-nodes-base.noOp",
      "position": [
        656,
        60
      ],
      "parameters": {},
      "typeVersion": 1
    },
    {
      "id": "1aea189f-3e7d-45d2-9802-d44b1dce4797",
      "name": "AI Agent",
      "type": "@n8n/n8n-nodes-langchain.agent",
      "position": [
        60,
        -40
      ],
      "parameters": {
        "options": {}
      },
      "typeVersion": 2
    },
    {
      "id": "54596db1-4ba7-414c-88cf-250f528d6c21",
      "name": "When chat message received",
      "type": "@n8n/n8n-nodes-langchain.chatTrigger",
      "position": [
        -160,
        60
      ],
      "parameters": {
        "options": {}
      },
      "typeVersion": 1.1
    },
    {
      "id": "4184f202-48f5-4e58-a0e8-0227a23553f3",
      "name": "Update Output",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        2132,
        -140
      ],
      "parameters": {
        "outputs": {
          "values": [
            {
              "outputName": "output",
              "outputValue": "={{ $('Set Input Fields').first().json.answer }}"
            },
            {
              "outputName": "score",
              "outputValue": "={{ $json.score }}"
            }
          ]
        },
        "sheetName": {
          "__rl": true,
          "mode": "list",
          "value": 2015506025,
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit#gid=2015506025",
          "cachedResultName": "Relevance"
        },
        "documentId": {
          "__rl": true,
          "mode": "list",
          "value": "1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y",
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=drivesdk",
          "cachedResultName": "96. Evaluations Test"
        }
      },
      "credentials": {
        "googleSheetsOAuth2Api": {
          "name": "<your credential>"
        }
      },
      "typeVersion": 4.6
    },
    {
      "id": "fe7b41d2-e382-4f59-8b0c-06361062581d",
      "name": "Update Metrics",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        2352,
        -140
      ],
      "parameters": {
        "metrics": {
          "assignments": [
            {
              "id": "1fd7759c-f4ef-4eda-87ad-9d9563b63e99",
              "name": "score",
              "type": "number",
              "value": "={{ $json.score }}"
            }
          ]
        },
        "operation": "setMetrics"
      },
      "typeVersion": 4.6
    },
    {
      "id": "04f714cd-99ce-45a9-bc43-351fb6ea14d4",
      "name": "OpenAI Chat Model",
      "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
      "position": [
        920,
        80
      ],
      "parameters": {
        "model": {
          "__rl": true,
          "mode": "list",
          "value": "gpt-4.1-mini",
          "cachedResultName": "gpt-4.1-mini"
        },
        "options": {}
      },
      "credentials": {
        "openAiApi": {
          "name": "<your credential>"
        }
      },
      "typeVersion": 1.2
    },
    {
      "id": "d821b0ec-93c9-46e1-ab03-280aeb9b87b1",
      "name": "Structured Output Parser",
      "type": "@n8n/n8n-nodes-langchain.outputParserStructured",
      "position": [
        1040,
        80
      ],
      "parameters": {
        "jsonSchemaExample": "{\n  \"question\": \"Where was Albert Einstein born?\",\n  \"noncommittal\": 0\n}"
      },
      "typeVersion": 1.2
    },
    {
      "id": "be975ef4-c50b-4bf9-b871-cc8d479661eb",
      "name": "Answer Relevance",
      "type": "@n8n/n8n-nodes-langchain.chainLlm",
      "position": [
        900,
        -140
      ],
      "parameters": {
        "text": "={{ $json.answer }}",
        "batching": {},
        "messages": {
          "messageValues": [
            {
              "message": "=Generate a question for the given answer and Identify if answer is noncommittal. Give noncommittal as 1 if the answer is noncommittal and 0 if the answer is committal. A noncommittal answer is one that is evasive, vague, or ambiguous. For example, \"I don't know\" or \"I'm not sure\" are noncommittal answers\""
            }
          ]
        },
        "promptType": "define",
        "hasOutputParser": true
      },
      "typeVersion": 1.7
    },
    {
      "id": "8e3df14a-7754-4257-8209-a24cfa30dbbc",
      "name": "Sticky Note1",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -452,
        -400
      ],
      "parameters": {
        "color": 7,
        "width": 840,
        "height": 720,
        "content": "## 1. Setup Your AI Workflow to Use Evaluations\n[Learn more about the Evaluations Trigger](https://docs.n8n.io/integrations/builtin/?utm_source=n8n_app&utm_medium=node_settings_modal-credential_link&utm_campaign=n8n-nodes-base.evaluationTrigger)\n\nThe Evaluations Trigger is a separate execution which does not affect your production workflow in any way. It is manually triggered and automatically pulled datasets from the assigned Google Sheet."
      },
      "typeVersion": 1
    },
    {
      "id": "eeeac9d0-39c6-4d17-8d93-9c60b6c612f8",
      "name": "Sticky Note",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        820,
        -400
      ],
      "parameters": {
        "color": 7,
        "width": 1200,
        "height": 720,
        "content": "## 2. Answer Relevance: How relevant is the agent response to the user question?\n[Learn more about the Evaluations Trigger](https://docs.n8n.io/integrations/builtin/?utm_source=n8n_app&utm_medium=node_settings_modal-credential_link&utm_campaign=n8n-nodes-base.evaluationTrigger)\n\nFor this evaluation, we analyse the agent's answer and ask another AI to generate a question for it - kinda like the game show [Jeopardy!](https://en.wikipedia.org/wiki/Jeopardy!).\nWe do this in order to compare the generated question with the original question for the test. If the questions are similar, then relevancy is high."
      },
      "typeVersion": 1
    },
    {
      "id": "c86f66e2-9997-4ca6-9ec6-cfa185d404f2",
      "name": "Sticky Note3",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -920,
        -340
      ],
      "parameters": {
        "width": 420,
        "height": 720,
        "content": "## Try It Out!\n### This n8n template demonstrates how to calculate the evaluation metric \"Relevance\" which in this scenario, measures the relevance of the agent's response to the user's question.\n\nThe scoring approach is adapted from [https://github.com/explodinggradients/ragas/blob/main/ragas/src/ragas/metrics/_answer_relevance.py](https://github.com/explodinggradients/ragas/blob/main/ragas/src/ragas/metrics/_answer_relevance.py)\n\n### How it works\n* This evaluation works best for Q&A agents.\n* For our scoring, we analyse the agent's response and ask another AI to generate a question from it. This generated question is then compared to the original question using cosine similarity.\n* A high score indicates relevance and the agent's successful ability to answer the question whereas a low score means agent may have added too much irrelevant info, went off script or hallucinated.\n\n### Requirements\n* n8n version 1.94+\n* Check out this Google Sheet for a sample data [https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=sharing](https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=sharing)\n\n\n### Need Help?\nJoin the [Discord](https://discord.com/invite/XPKeKXeB7d) or ask in the [Forum](https://community.n8n.io/)!\n\nHappy Hacking!"
      },
      "typeVersion": 1
    },
    {
      "id": "3bdd3254-7cdf-455a-ae63-9639069993bc",
      "name": "Get Embeddings",
      "type": "n8n-nodes-base.httpRequest",
      "position": [
        1460,
        -140
      ],
      "parameters": {
        "url": "https://api.openai.com/v1/embeddings",
        "method": "POST",
        "options": {},
        "sendBody": true,
        "authentication": "predefinedCredentialType",
        "bodyParameters": {
          "parameters": [
            {
              "name": "input",
              "value": "={{ $json.data }}"
            },
            {
              "name": "model",
              "value": "text-embedding-3-small"
            },
            {
              "name": "encoding_format",
              "value": "float"
            }
          ]
        },
        "nodeCredentialType": "openAiApi"
      },
      "credentials": {
        "openAiApi": {
          "name": "<your credential>"
        }
      },
      "typeVersion": 4.2
    },
    {
      "id": "9e3bcdee-5f24-45e6-a85f-6dcfacb58d5d",
      "name": "Calculate Similarity Score",
      "type": "n8n-nodes-base.code",
      "position": [
        1660,
        -140
      ],
      "parameters": {
        "jsCode": "const [vectorsA, vectorsB] = $input.all().map(item => item.json.data[0].embedding);\nconst score = cosineSimilarity(vectorsA, vectorsB);\n\nreturn { json: { similarityScore: score } };\n\nfunction cosineSimilarity(a, b) {  \n  let dotProduct = normA = normB = 0;\n  for (let i = 0; i < a.length; i++) {\n    dotProduct += a[i] * b[i];\n    normA += a[i] ** 2;\n    normB += b[i] ** 2;\n  }\n  return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));\n}"
      },
      "typeVersion": 2
    },
    {
      "id": "69c30814-9725-426d-9572-9f92bd81dea5",
      "name": "Questions to Items",
      "type": "n8n-nodes-base.code",
      "position": [
        1260,
        -140
      ],
      "parameters": {
        "jsCode": "return [\n  { json: { data: $('Set Input Fields').first().json.question } },\n  { json: { data: $input.first().json.output.question } }\n]"
      },
      "typeVersion": 2
    },
    {
      "id": "83be23ff-6857-4265-b1d0-f4b682c66903",
      "name": "Calculate Relevance Score",
      "type": "n8n-nodes-base.set",
      "position": [
        1860,
        -140
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "c5abdf27-cb9e-4969-b669-25c6fbd135f1",
              "name": "score",
              "type": "number",
              "value": "={{ $json.similarityScore * Math.abs(!$('Answer Relevance').first().json.output.noncommittal) }}"
            }
          ]
        }
      },
      "typeVersion": 3.4
    }
  ],
  "connections": {
    "AI Agent": {
      "main": [
        [
          {
            "node": "Evaluation",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Evaluation": {
      "main": [
        [
          {
            "node": "Set Input Fields",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "No Operation, do nothing",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Remap Input": {
      "main": [
        [
          {
            "node": "AI Agent",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Update Output": {
      "main": [
        [
          {
            "node": "Update Metrics",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Get Embeddings": {
      "main": [
        [
          {
            "node": "Calculate Similarity Score",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Answer Relevance": {
      "main": [
        [
          {
            "node": "Questions to Items",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Set Input Fields": {
      "main": [
        [
          {
            "node": "Answer Relevance",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "OpenAI Chat Model": {
      "ai_languageModel": [
        [
          {
            "node": "Answer Relevance",
            "type": "ai_languageModel",
            "index": 0
          }
        ]
      ]
    },
    "OpenAI Chat Model1": {
      "ai_languageModel": [
        [
          {
            "node": "AI Agent",
            "type": "ai_languageModel",
            "index": 0
          }
        ]
      ]
    },
    "Questions to Items": {
      "main": [
        [
          {
            "node": "Get Embeddings",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Structured Output Parser": {
      "ai_outputParser": [
        [
          {
            "node": "Answer Relevance",
            "type": "ai_outputParser",
            "index": 0
          }
        ]
      ]
    },
    "Calculate Relevance Score": {
      "main": [
        [
          {
            "node": "Update Output",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Calculate Similarity Score": {
      "main": [
        [
          {
            "node": "Calculate Relevance Score",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "When chat message received": {
      "main": [
        [
          {
            "node": "AI Agent",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "When fetching a dataset row": {
      "main": [
        [
          {
            "node": "Remap Input",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}