Evaluations Metric: Answer Similarity

The workflow JSON

Copy or download the full n8n JSON below. Paste it into a new n8n workflow, add your credentials, activate. Full import guide →

{
  "nodes": [
    {
      "id": "c5365531-2d66-4e67-9db2-37059e9e97a3",
      "name": "OpenAI Chat Model1",
      "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
      "position": [
        -1512,
        420
      ],
      "parameters": {
        "model": {
          "__rl": true,
          "mode": "list",
          "value": "gpt-4.1-mini",
          "cachedResultName": "gpt-4.1-mini"
        },
        "options": {}
      },
      "credentials": {
        "openAiApi": {
          "name": "<your credential>"
        }
      },
      "typeVersion": 1.2
    },
    {
      "id": "3881b831-b575-4b2f-8ebb-ac956a714f10",
      "name": "When fetching a dataset row",
      "type": "n8n-nodes-base.evaluationTrigger",
      "position": [
        -2040,
        100
      ],
      "parameters": {
        "sheetName": {
          "__rl": true,
          "mode": "list",
          "value": 1176192270,
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit#gid=1176192270",
          "cachedResultName": "Similarity"
        },
        "documentId": {
          "__rl": true,
          "mode": "list",
          "value": "1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y",
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=drivesdk",
          "cachedResultName": "96. Evaluations Test"
        }
      },
      "credentials": {
        "googleSheetsOAuth2Api": {
          "name": "<your credential>"
        }
      },
      "typeVersion": 4.6
    },
    {
      "id": "dd5a14f7-7321-4dfd-9f08-1f35efba1a07",
      "name": "Remap Input",
      "type": "n8n-nodes-base.set",
      "position": [
        -1820,
        100
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "00924b90-278f-49f5-80f2-c297df0fcc97",
              "name": "chatInput",
              "type": "string",
              "value": "={{ $json.input }}"
            }
          ]
        }
      },
      "typeVersion": 3.4
    },
    {
      "id": "78d87631-7d52-4fbb-9313-f47263bd101f",
      "name": "Evaluation",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        -1180,
        200
      ],
      "parameters": {
        "operation": "checkIfEvaluating"
      },
      "typeVersion": 4.6
    },
    {
      "id": "1d8b12ba-ec63-4f92-a4dc-1885cbcd4d95",
      "name": "Set Input Fields",
      "type": "n8n-nodes-base.set",
      "position": [
        -960,
        100
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "d58952c1-d346-4fbf-881e-d5c04b6781a5",
              "name": "question",
              "type": "string",
              "value": "={{ $('When fetching a dataset row').first().json.input }}"
            },
            {
              "id": "0f10a3d0-cf6e-4715-9ded-2cee54aa62ec",
              "name": "answer",
              "type": "string",
              "value": "={{ $json.output }}"
            },
            {
              "id": "edbe42ed-36a7-438a-989f-900673e61d0f",
              "name": "groundTruth",
              "type": "array",
              "value": "={{ $('When fetching a dataset row').first().json['ground truth'].split('\\n') }}"
            }
          ]
        }
      },
      "typeVersion": 3.4
    },
    {
      "id": "518cecea-9bcc-47dd-83b1-73ef0880e808",
      "name": "No Operation, do nothing",
      "type": "n8n-nodes-base.noOp",
      "position": [
        -960,
        300
      ],
      "parameters": {},
      "typeVersion": 1
    },
    {
      "id": "c1a9cf2f-8e5d-4db9-a29b-b426e1932ab3",
      "name": "AI Agent",
      "type": "@n8n/n8n-nodes-langchain.agent",
      "position": [
        -1600,
        200
      ],
      "parameters": {
        "options": {}
      },
      "typeVersion": 2
    },
    {
      "id": "6c5602d1-59c0-479e-b9aa-fb8ce037d4b1",
      "name": "When chat message received",
      "type": "@n8n/n8n-nodes-langchain.chatTrigger",
      "position": [
        -1820,
        300
      ],
      "parameters": {
        "options": {}
      },
      "typeVersion": 1.1
    },
    {
      "id": "6fb01095-7b93-41dd-8d7c-364cb685ddbb",
      "name": "Update Output",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        520,
        300
      ],
      "parameters": {
        "outputs": {
          "values": [
            {
              "outputName": "output",
              "outputValue": "={{ $('Set Input Fields').item.json.answer }}"
            },
            {
              "outputName": "score",
              "outputValue": "={{ $json.score }}"
            }
          ]
        },
        "sheetName": {
          "__rl": true,
          "mode": "list",
          "value": 1176192270,
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit#gid=1176192270",
          "cachedResultName": "Similarity"
        },
        "documentId": {
          "__rl": true,
          "mode": "list",
          "value": "1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y",
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=drivesdk",
          "cachedResultName": "96. Evaluations Test"
        }
      },
      "credentials": {
        "googleSheetsOAuth2Api": {
          "name": "<your credential>"
        }
      },
      "typeVersion": 4.6
    },
    {
      "id": "18e8dbb6-df98-4d36-9686-9ec481a37754",
      "name": "Update Metrics",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        700,
        300
      ],
      "parameters": {
        "metrics": {
          "assignments": [
            {
              "id": "1fd7759c-f4ef-4eda-87ad-9d9563b63e99",
              "name": "score",
              "type": "number",
              "value": "={{ $json.score }}"
            }
          ]
        },
        "operation": "setMetrics"
      },
      "typeVersion": 4.6
    },
    {
      "id": "a685c9e7-0b21-4873-a502-5826a916e31f",
      "name": "Sticky Note1",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -2120,
        -120
      ],
      "parameters": {
        "color": 7,
        "width": 840,
        "height": 720,
        "content": "## 1. Setup Your AI Workflow to Use Evaluations\n[Learn more about the Evaluations Trigger](https://docs.n8n.io/integrations/builtin/?utm_source=n8n_app&utm_medium=node_settings_modal-credential_link&utm_campaign=n8n-nodes-base.evaluationTrigger)\n\nThe Evaluations Trigger is a separate execution which does not affect your production workflow in any way. It is manually triggered and automatically pulled datasets from the assigned Google Sheet."
      },
      "typeVersion": 1
    },
    {
      "id": "2e57d911-8fef-456d-8e5c-3849350553e1",
      "name": "Sticky Note",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -1240,
        -120
      ],
      "parameters": {
        "color": 7,
        "width": 1700,
        "height": 720,
        "content": "## 2. Answer Similarity: How similar is the AI response to the Ground Truth?\n[Learn more about the Evaluations Trigger](https://docs.n8n.io/integrations/builtin/?utm_source=n8n_app&utm_medium=node_settings_modal-credential_link&utm_campaign=n8n-nodes-base.evaluationTrigger)\n\nFor this evaluation, we want to do a simple check to see how similar the AI's response is to the ground truth.\nThis test is most effective for closed-ended questions where there can be little to no deviation in the answers. "
      },
      "typeVersion": 1
    },
    {
      "id": "9962bbf8-01bc-463c-ad3b-499618c5b1f9",
      "name": "Sticky Note3",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -2580,
        -120
      ],
      "parameters": {
        "width": 420,
        "height": 720,
        "content": "## Try It Out!\n### This n8n template demonstrates how to calculate the evaluation metric \"Similarity\" which in this scenario, measures the consistency of the agent.\n\nThe scoring approach is adapted from [https://github.com/explodinggradients/ragas/blob/main/ragas/src/ragas/metrics/_answer_similarity.py](https://github.com/explodinggradients/ragas/blob/main/ragas/src/ragas/metrics/_answer_similarity.py)\n\n### How it works\n* This evaluation works best where questions are close-ended or about facts where the answer can have little to no deviation.\n* For our scoring, we generate embeddings for both the AI's response and ground truth and calculate the cosine similarity between them.\n* A high score indicates LLM consistency with expected results whereas a low score could signal model hallucination.\n\n### Requirements\n* n8n version 1.94+\n* Check out this Google Sheet for a sample data [https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=sharing](https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=sharing)\n\n\n### Need Help?\nJoin the [Discord](https://discord.com/invite/XPKeKXeB7d) or ask in the [Forum](https://community.n8n.io/)!\n\nHappy Hacking!"
      },
      "typeVersion": 1
    },
    {
      "id": "47699677-545f-4eca-8b46-89c6862a56f1",
      "name": "Get Embeddings",
      "type": "n8n-nodes-base.httpRequest",
      "position": [
        -520,
        300
      ],
      "parameters": {
        "url": "https://api.openai.com/v1/embeddings",
        "method": "POST",
        "options": {},
        "sendBody": true,
        "authentication": "predefinedCredentialType",
        "bodyParameters": {
          "parameters": [
            {
              "name": "input",
              "value": "={{ $json.groundTruth }}"
            },
            {
              "name": "model",
              "value": "text-embedding-3-small"
            },
            {
              "name": "encoding_format",
              "value": "float"
            }
          ]
        },
        "nodeCredentialType": "openAiApi"
      },
      "credentials": {
        "openAiApi": {
          "name": "<your credential>"
        }
      },
      "typeVersion": 4.2
    },
    {
      "id": "1a022046-80f2-4df3-b075-be8e9294b951",
      "name": "GroundTruth to Items",
      "type": "n8n-nodes-base.splitOut",
      "position": [
        -720,
        300
      ],
      "parameters": {
        "options": {},
        "fieldToSplitOut": "groundTruth"
      },
      "typeVersion": 1
    },
    {
      "id": "0cea3a79-23ab-4d0f-9c6c-2522af6a506f",
      "name": "Get Embeddings1",
      "type": "n8n-nodes-base.httpRequest",
      "position": [
        -520,
        100
      ],
      "parameters": {
        "url": "https://api.openai.com/v1/embeddings",
        "method": "POST",
        "options": {},
        "sendBody": true,
        "authentication": "predefinedCredentialType",
        "bodyParameters": {
          "parameters": [
            {
              "name": "input",
              "value": "={{ $json.answer }}"
            },
            {
              "name": "model",
              "value": "text-embedding-3-small"
            },
            {
              "name": "encoding_format",
              "value": "float"
            }
          ]
        },
        "nodeCredentialType": "openAiApi"
      },
      "credentials": {
        "openAiApi": {
          "name": "<your credential>"
        }
      },
      "typeVersion": 4.2
    },
    {
      "id": "1b5740cc-4b79-4690-88ae-fa31c5a1c16c",
      "name": "Aggregate",
      "type": "n8n-nodes-base.aggregate",
      "position": [
        -120,
        300
      ],
      "parameters": {
        "options": {},
        "aggregate": "aggregateAllItemData",
        "destinationFieldName": "groundTruth"
      },
      "typeVersion": 1
    },
    {
      "id": "d32f979b-7d96-4730-8324-de0d34ed659a",
      "name": "Remap Embeddings",
      "type": "n8n-nodes-base.set",
      "position": [
        -320,
        100
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "3db07c98-f926-46e0-85f2-ed1eb137f842",
              "name": "answer",
              "type": "array",
              "value": "={{ $json.data[0].embedding }}"
            }
          ]
        }
      },
      "typeVersion": 3.4
    },
    {
      "id": "188864fb-3d6a-4c01-9d23-73c731de023d",
      "name": "Remap Embeddings1",
      "type": "n8n-nodes-base.set",
      "position": [
        -320,
        300
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "3db07c98-f926-46e0-85f2-ed1eb137f842",
              "name": "data",
              "type": "array",
              "value": "={{ $json.data[0].embedding }}"
            }
          ]
        }
      },
      "typeVersion": 3.4
    },
    {
      "id": "5cc6c1c7-240b-4129-b57a-e7d0ef2e72e6",
      "name": "Create Embeddings Result",
      "type": "n8n-nodes-base.merge",
      "position": [
        80,
        300
      ],
      "parameters": {
        "mode": "combine",
        "options": {},
        "combineBy": "combineByPosition"
      },
      "typeVersion": 3.1
    },
    {
      "id": "7aa037d2-31a6-4efe-81b1-26a95c4c6d38",
      "name": "Calculate Similarity Score",
      "type": "n8n-nodes-base.code",
      "position": [
        280,
        300
      ],
      "parameters": {
        "jsCode": "const { answer, groundTruth = [] } = $input.item.json;\n\nconst scores = await Promise.all(groundTruth.map(truth =>\n  cosineSimilarity(answer, truth.data)\n));\n\nconst averageScore = scores.reduce((acc,i) => acc + i, 0) / scores.length;\n\nreturn { json: { score: averageScore } };\n\nfunction cosineSimilarity(a, b) {  \n  let dotProduct = normA = normB = 0;\n  for (let i = 0; i < a.length; i++) {\n    dotProduct += a[i] * b[i];\n    normA += a[i] ** 2;\n    normB += b[i] ** 2;\n  }\n  return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));\n}"
      },
      "typeVersion": 2
    }
  ],
  "connections": {
    "AI Agent": {
      "main": [
        [
          {
            "node": "Evaluation",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Aggregate": {
      "main": [
        [
          {
            "node": "Create Embeddings Result",
            "type": "main",
            "index": 1
          }
        ]
      ]
    },
    "Evaluation": {
      "main": [
        [
          {
            "node": "Set Input Fields",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "No Operation, do nothing",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Remap Input": {
      "main": [
        [
          {
            "node": "AI Agent",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Update Output": {
      "main": [
        [
          {
            "node": "Update Metrics",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Get Embeddings": {
      "main": [
        [
          {
            "node": "Remap Embeddings1",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Get Embeddings1": {
      "main": [
        [
          {
            "node": "Remap Embeddings",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Remap Embeddings": {
      "main": [
        [
          {
            "node": "Create Embeddings Result",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Set Input Fields": {
      "main": [
        [
          {
            "node": "Get Embeddings1",
            "type": "main",
            "index": 0
          },
          {
            "node": "GroundTruth to Items",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Remap Embeddings1": {
      "main": [
        [
          {
            "node": "Aggregate",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "OpenAI Chat Model1": {
      "ai_languageModel": [
        [
          {
            "node": "AI Agent",
            "type": "ai_languageModel",
            "index": 0
          }
        ]
      ]
    },
    "GroundTruth to Items": {
      "main": [
        [
          {
            "node": "Get Embeddings",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Create Embeddings Result": {
      "main": [
        [
          {
            "node": "Calculate Similarity Score",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Calculate Similarity Score": {
      "main": [
        [
          {
            "node": "Update Output",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "When chat message received": {
      "main": [
        [
          {
            "node": "AI Agent",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "When fetching a dataset row": {
      "main": [
        [
          {
            "node": "Remap Input",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}

Credentials you'll need

Each integration node will prompt for credentials when you import. We strip credential IDs before publishing — you'll add your own.

• googleSheetsOAuth2Api
• openAiApi

About this workflow

The scoring approach is adapted from the open-source evaluations project RAGAS and you can see the source here https://github.com/explodinggradients/ragas/blob/main/ragas/src/ragas/metrics/answersimilarity.py This evaluation works best where questions are close-ended or about…

Source: https://n8n.io/workflows/4423/ — original creator credit. Request a take-down →

More AI & RAG workflows → · Browse all categories →

Related workflows

Workflows that share integrations, category, or trigger type with this one. All free to copy and import.