CRM List Hygiene & Duplicate Merge Planner

The workflow JSON

Copy or download the full n8n JSON below. Paste it into a new n8n workflow, add your credentials, activate. Full import guide →

{
  "name": "CRM List Hygiene & Duplicate Merge Planner",
  "active": false,
  "settings": {
    "executionOrder": "v1",
    "saveDataErrorExecution": "all",
    "saveDataSuccessExecution": "all",
    "saveManualExecutions": true
  },
  "nodes": [
    {
      "name": "Run Audit Now",
      "type": "n8n-nodes-base.manualTrigger",
      "typeVersion": 1,
      "position": [
        -1180,
        260
      ],
      "parameters": {},
      "id": "242dc348-b0d6-4daa-8dd5-fc10d69013bc"
    },
    {
      "name": "Weekly Audit",
      "type": "n8n-nodes-base.scheduleTrigger",
      "typeVersion": 1.2,
      "position": [
        -1180,
        440
      ],
      "parameters": {
        "rule": {
          "interval": [
            {
              "field": "weeks",
              "triggerAtDay": [
                1
              ],
              "triggerAtHour": 6,
              "triggerAtMinute": 0
            }
          ]
        }
      },
      "id": "7a5b4bd8-8cc9-4cd0-8213-bd82f24aa404"
    },
    {
      "name": "Init Audit Run",
      "type": "n8n-nodes-base.code",
      "typeVersion": 2,
      "position": [
        -960,
        350
      ],
      "parameters": {
        "mode": "runOnceForAllItems",
        "jsCode": "/**\n * NODE: \"Init Audit Run\"  (Code node, Run Once for All Items)\n * ----------------------------------------------------------\n * Sets up one audit pass. Everything tunable lives here so the rest of the\n * workflow has no configuration in it.\n *\n * DRY_RUN is on by default and stays on until someone deliberately turns it off.\n * A list-cleaning tool that merges 400 contacts on its first run, in a client's\n * production CRM, is not a tool \u2014 it is an incident. This one produces a plan\n * first and merges only when told to.\n */\n\nconst CONFIG = {\n  // --- safety ------------------------------------------------------------\n  dryRun: true,              // produce the plan; write nothing\n  mockMode: true,            // use the built-in fixture contacts, no credentials\n\n  // --- source ------------------------------------------------------------\n  locationId: 'REPLACE_WITH_LOCATION_ID',\n  pageSize: 10,              // raise to 100 against a real API\n  maxPages: 50,              // hard ceiling \u2014 a paginator with no ceiling is a bug\n  maxContacts: 25000,        // memory guard, see README for the >25k approach\n\n  // --- matching ----------------------------------------------------------\n  // A pair is a duplicate if any rule below fires. Rules are ordered by how much\n  // you should trust them; the weakest one is deliberately conservative.\n  nameSimilarityThreshold: 0.90,   // Jaro-Winkler, 0-1\n  requireSecondSignal: true,       // a fuzzy name match alone is never enough\n\n  // --- deliverability ----------------------------------------------------\n  defaultCountry: 'US',\n  staleAfterDays: 730,             // no engagement in 2 years\n};\n\nfunction processAll(items) {\n  const startedAt = new Date().toISOString();\n\n  return [{\n    json: {\n      config: CONFIG,\n      audit: {\n        runId: 'audit_' + startedAt.slice(0, 10).replace(/-/g, '') + '_'\n          + Math.random().toString(16).slice(2, 8),\n        startedAt,\n        page: 0,\n        cursor: null,\n        hasMore: true,\n        contacts: [],\n        pagesFetched: 0,\n        partial: false,\n        partialReason: null,\n      },\n      // Seeds the shared retry classifier.\n      retry: { attempt: 0, maxAttempts: 5, waitSeconds: 0, history: [] },\n    },\n    pairedItem: items.length ? 0 : undefined,\n  }];\n}\n\n// --- exported for the local test harness; n8n ignores this block -----------\nif (typeof module !== 'undefined' && module.exports) {\n  module.exports = { CONFIG, processAll };\n}\n\n// --- n8n entry point -------------------------------------------------------\nif (typeof $input !== 'undefined') {\n  return processAll($input.all());\n}"
      },
      "id": "08932967-05b5-4e8b-8226-bdfa8900e70d"
    },
    {
      "name": "Build Page Request",
      "type": "n8n-nodes-base.code",
      "typeVersion": 2,
      "position": [
        -740,
        350
      ],
      "parameters": {
        "mode": "runOnceForAllItems",
        "jsCode": "/**\n * NODE: \"Build Page Request\"  (Code node, Run Once for All Items)\n * --------------------------------------------------------------\n * The head of the pagination loop. Three nodes feed into it \u2014 the initialiser,\n * the accumulator (next page), and the backoff wait (retry the same page) \u2014 and\n * only one of them hands over a complete state object.\n *\n * That is the whole reason this node exists. When an HTTP node fails, what comes\n * out of its error output is the error, not the item that went in, so anything\n * downstream of a retry has to be able to rebuild the loop state from somewhere\n * else. This node does that in one place, and the HTTP node reads its cursor from\n * here rather than from whatever happened to be on the item.\n */\n\n/** Pull the freshest loop state from wherever it survived. */\nfunction resolveState(item, fallbacks) {\n  if (item && item.json && item.json.audit && item.json.config) return item.json;\n  for (const fallback of fallbacks) {\n    if (fallback && fallback.audit && fallback.config) return fallback;\n  }\n  return null;\n}\n\nfunction buildRequest(state) {\n  const { config, audit } = state;\n\n  return {\n    config,\n    audit,\n    retry: state.retry || { attempt: 0, maxAttempts: 5, waitSeconds: 0, history: [] },\n    request: {\n      locationId: config.locationId,\n      limit: config.pageSize,\n      // Cursor-based, not offset-based. Offset pagination over a table that is\n      // being written to while you read it silently skips and repeats records;\n      // for a dedupe job that is the one failure mode you cannot tolerate.\n      cursor: audit.cursor || '',\n      page: audit.page + 1,\n    },\n    useLive: config.mockMode !== true,\n  };\n}\n\n// --- exported for the local test harness; n8n ignores this block -----------\nif (typeof module !== 'undefined' && module.exports) {\n  module.exports = { resolveState, buildRequest };\n}\n\n// --- n8n entry point -------------------------------------------------------\nif (typeof $input !== 'undefined') {\n  const fallbacks = [];\n  for (const nodeName of ['Accumulate Page', 'Init Audit Run']) {\n    try {\n      const upstream = $(nodeName).all();\n      if (upstream && upstream.length) fallbacks.push(upstream[upstream.length - 1].json);\n    } catch (error) {\n      // That node has not run on this path yet. Expected on the first pass.\n    }\n  }\n\n  const state = resolveState($input.all()[0], fallbacks);\n  if (!state) {\n    throw new Error('Build Page Request could not resolve the audit state. '\n      + 'Check that \"Init Audit Run\" ran before this node.');\n  }\n  return [{ json: buildRequest(state) }];\n}"
      },
      "id": "2be14fa8-35cc-4fa9-97ae-0848e35e1185"
    },
    {
      "name": "Use Live CRM?",
      "type": "n8n-nodes-base.if",
      "typeVersion": 2.2,
      "position": [
        -520,
        350
      ],
      "parameters": {
        "conditions": {
          "options": {
            "caseSensitive": true,
            "leftValue": "",
            "typeValidation": "loose",
            "version": 2
          },
          "conditions": [
            {
              "id": "use-live",
              "leftValue": "={{ $json.useLive }}",
              "rightValue": "",
              "operator": {
                "type": "boolean",
                "operation": "true",
                "singleValue": true
              }
            }
          ],
          "combinator": "and"
        },
        "looseTypeValidation": true,
        "options": {}
      },
      "id": "b4dccc94-313d-448b-be00-61b1afc194cd"
    },
    {
      "name": "CRM: Fetch Contact Page",
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 4.2,
      "position": [
        -280,
        220
      ],
      "parameters": {
        "method": "GET",
        "url": "https://services.leadconnectorhq.com/contacts/",
        "authentication": "genericCredentialType",
        "genericAuthType": "httpHeaderAuth",
        "sendQuery": true,
        "specifyQuery": "keypair",
        "queryParameters": {
          "parameters": [
            {
              "name": "locationId",
              "value": "={{ $('Build Page Request').item.json.request.locationId }}"
            },
            {
              "name": "limit",
              "value": "={{ $('Build Page Request').item.json.request.limit }}"
            },
            {
              "name": "startAfterId",
              "value": "={{ $('Build Page Request').item.json.request.cursor }}"
            }
          ]
        },
        "sendHeaders": true,
        "specifyHeaders": "keypair",
        "headerParameters": {
          "parameters": [
            {
              "name": "Version",
              "value": "2021-07-28"
            },
            {
              "name": "Accept",
              "value": "application/json"
            }
          ]
        },
        "options": {
          "timeout": 30000,
          "response": {
            "response": {
              "fullResponse": true,
              "neverError": true
            }
          }
        }
      },
      "id": "36baeac7-13c0-4aaa-9d02-e1286cef3ae5"
    },
    {
      "name": "Mock Contact Page",
      "type": "n8n-nodes-base.code",
      "typeVersion": 2,
      "position": [
        -280,
        480
      ],
      "parameters": {
        "mode": "runOnceForAllItems",
        "jsCode": "/**\n * NODE: \"Mock Contact Page\"  (Code node, Run Once for All Items)\n * -------------------------------------------------------------\n * Stands in for the CRM's paginated contact endpoint so the whole audit runs\n * with no credentials and no client account. Same response shape as the live\n * call, same cursor semantics, so nothing downstream can tell the difference.\n *\n * The fixture list is 24 contacts from a CRM that has been running for three\n * years: five real duplicate clusters, two near-misses that must NOT be merged,\n * dead email addresses, un-textable numbers, a DND record, and a contact with no\n * reachable channel at all. If a dedupe tool gets this list right it will get a\n * real one right.\n */\n\nconst MOCK_CONTACTS = [\n  // --- cluster A: same person, three records, three spellings of the email ---\n  { id: 'ct_1001', firstName: 'Robert', lastName: 'Chen', email: 'rob.chen@northsidehvac.example.com',\n    phone: '+14155550143', companyName: 'Northside HVAC', city: 'San Francisco', postalCode: '94110',\n    country: 'US', tags: ['source:website_form'], dnd: false, emailStatus: 'valid',\n    createdAt: '2023-02-11T10:00:00Z', lastActivityAt: '2026-07-30T14:22:00Z' },\n  { id: 'ct_1002', firstName: 'Bob', lastName: 'Chen', email: 'r.chen@northsidehvac.example.com',\n    phone: '415-555-0143', companyName: 'Northside HVAC Inc', city: 'San Francisco', postalCode: '94110',\n    country: 'US', tags: ['trade-show-2024'], dnd: false, emailStatus: 'valid',\n    createdAt: '2024-05-02T09:30:00Z', lastActivityAt: '2025-11-02T09:00:00Z' },\n  { id: 'ct_1003', firstName: 'Robert', lastName: 'Chen', email: '',\n    phone: '(415) 555 0143', companyName: '', city: '', postalCode: '',\n    country: 'US', tags: [], dnd: false, emailStatus: '',\n    createdAt: '2025-01-19T16:45:00Z', lastActivityAt: null },\n\n  // --- cluster B: one Gmail inbox, two \"different\" addresses -----------------\n  { id: 'ct_1004', firstName: 'Sarah', lastName: 'Okonkwo', email: 'sarah.okonkwo@gmail.com',\n    phone: '', companyName: '', city: 'Manchester', postalCode: 'M1 4BT',\n    country: 'GB', tags: ['source:facebook_lead_ads'], dnd: false, emailStatus: 'valid',\n    createdAt: '2024-08-01T11:00:00Z', lastActivityAt: '2026-06-14T08:10:00Z' },\n  { id: 'ct_1005', firstName: 'Sarah', lastName: 'Okonkwo', email: 's.arah.okonkwo+forms@googlemail.com',\n    phone: '+447700900461', companyName: '', city: 'Manchester', postalCode: 'M1 4BT',\n    country: 'GB', tags: ['newsletter'], dnd: false, emailStatus: 'valid',\n    createdAt: '2025-03-22T13:05:00Z', lastActivityAt: '2026-08-01T19:40:00Z' },\n\n  // --- cluster C: same number, written two ways -----------------------------\n  { id: 'ct_1006', firstName: 'Aminath', lastName: 'Rasheed', email: 'aminath.r@example.com',\n    phone: '+9607712345', companyName: '', city: 'Mal\u00e9', postalCode: '20026',\n    country: 'MV', tags: ['existing-client'], dnd: false, emailStatus: 'valid',\n    createdAt: '2023-09-14T07:00:00Z', lastActivityAt: '2026-08-05T06:30:00Z' },\n  { id: 'ct_1007', firstName: 'Aminath', lastName: '', email: '',\n    phone: '00960 771 2345', companyName: 'Rasheed Trading', city: '', postalCode: '',\n    country: '', tags: [], dnd: false, emailStatus: '',\n    createdAt: '2024-01-08T05:20:00Z', lastActivityAt: null },\n\n  // --- cluster D: nickname + full name, one mobile --------------------------\n  { id: 'ct_1008', firstName: 'Mike', lastName: 'Johnson', email: 'mike.johnson@example.com',\n    phone: '+14155550188', companyName: '', city: 'Oakland', postalCode: '94607',\n    country: 'US', tags: [], dnd: false, emailStatus: 'valid',\n    createdAt: '2023-11-30T18:00:00Z', lastActivityAt: '2025-02-11T12:00:00Z' },\n  { id: 'ct_1009', firstName: 'Michael', lastName: 'Johnson', email: 'mjohnson@example.com',\n    phone: '4155550188', companyName: 'Johnson Plumbing', city: 'Oakland', postalCode: '94607',\n    country: 'US', tags: ['quote-requested'], dnd: false, emailStatus: 'valid',\n    createdAt: '2025-06-04T10:15:00Z', lastActivityAt: '2026-07-19T15:45:00Z' },\n\n  // --- cluster E: typo'd surname, identical email ---------------------------\n  { id: 'ct_1010', firstName: 'Jennifer', lastName: 'Nakamura', email: 'jen.nakamura@brightpath.example.com',\n    phone: '+61412345678', companyName: 'Brightpath', city: 'Sydney', postalCode: '2000',\n    country: 'AU', tags: ['vip'], dnd: false, emailStatus: 'valid',\n    createdAt: '2023-04-17T08:00:00Z', lastActivityAt: '2026-08-09T02:00:00Z' },\n  { id: 'ct_1011', firstName: 'Jennifer', lastName: 'Nakamurra', email: 'jen.nakamura@brightpath.example.com',\n    phone: '', companyName: '', city: '', postalCode: '',\n    country: 'AU', tags: [], dnd: false, emailStatus: 'valid',\n    createdAt: '2025-10-01T00:00:00Z', lastActivityAt: null },\n\n  // --- near-misses: similar people who are NOT the same person --------------\n  { id: 'ct_1012', firstName: 'David', lastName: 'Smith', email: 'dsmith@acmedental.example.com',\n    phone: '+14155550201', companyName: 'Acme Dental', city: 'San Jose', postalCode: '95112',\n    country: 'US', tags: [], dnd: false, emailStatus: 'valid',\n    createdAt: '2024-02-20T09:00:00Z', lastActivityAt: '2026-05-30T11:00:00Z' },\n  { id: 'ct_1013', firstName: 'David', lastName: 'Smyth', email: 'dsmyth@acmedental.example.com',\n    phone: '+14155550202', companyName: 'Acme Dental', city: 'San Jose', postalCode: '95112',\n    country: 'US', tags: [], dnd: false, emailStatus: 'valid',\n    createdAt: '2024-02-20T09:05:00Z', lastActivityAt: '2026-05-30T11:05:00Z' },\n  // Shared family address, two real people.\n  { id: 'ct_1014', firstName: 'Anna', lastName: 'Petrova', email: 'petrov.family@example.net',\n    phone: '+14155550210', companyName: '', city: 'Daly City', postalCode: '94014',\n    country: 'US', tags: [], dnd: false, emailStatus: 'valid',\n    createdAt: '2024-07-07T07:07:00Z', lastActivityAt: '2026-03-01T10:00:00Z' },\n  { id: 'ct_1015', firstName: 'Igor', lastName: 'Petrov', email: 'petrov.family@example.net',\n    phone: '+14155550211', companyName: '', city: 'Daly City', postalCode: '94014',\n    country: 'US', tags: [], dnd: false, emailStatus: 'valid',\n    createdAt: '2024-07-07T07:09:00Z', lastActivityAt: '2026-03-01T10:02:00Z' },\n\n  // --- deliverability problems ---------------------------------------------\n  { id: 'ct_1016', firstName: 'Info', lastName: '', email: 'info@westsideclinic.example.com',\n    phone: '+14155550222', companyName: 'Westside Clinic', city: 'San Francisco', postalCode: '94115',\n    country: 'US', tags: [], dnd: false, emailStatus: 'valid',\n    createdAt: '2023-06-01T12:00:00Z', lastActivityAt: '2026-01-15T09:00:00Z' },\n  { id: 'ct_1017', firstName: 'Test', lastName: 'Test', email: 'test@mailinator.com',\n    phone: '5555555555', companyName: '', city: '', postalCode: '',\n    country: 'US', tags: [], dnd: false, emailStatus: 'valid',\n    createdAt: '2025-04-04T04:04:00Z', lastActivityAt: null },\n  { id: 'ct_1018', firstName: 'Karen', lastName: 'Whitfield', email: 'karen.whitfield@gmial.com',\n    phone: '+14155550233', companyName: '', city: 'Sacramento', postalCode: '95814',\n    country: 'US', tags: [], dnd: false, emailStatus: 'bounced',\n    createdAt: '2023-01-05T08:00:00Z', lastActivityAt: '2023-03-01T08:00:00Z' },\n  { id: 'ct_1019', firstName: 'Daniel', lastName: 'Okafor', email: 'daniel.okafor@example.com',\n    phone: '+2348012345678', companyName: '', city: 'Lagos', postalCode: '',\n    country: '', tags: ['unsubscribed'], dnd: true, emailStatus: 'unsubscribed',\n    createdAt: '2024-09-09T09:09:00Z', lastActivityAt: '2024-10-01T09:00:00Z' },\n  { id: 'ct_1020', firstName: '', lastName: '', email: 'not-an-email',\n    phone: '123', companyName: '', city: '', postalCode: '',\n    country: 'US', tags: [], dnd: false, emailStatus: '',\n    createdAt: '2025-12-25T00:00:00Z', lastActivityAt: null },\n\n  // --- clean singles --------------------------------------------------------\n  { id: 'ct_1021', firstName: 'Priya', lastName: 'Venkatesan', email: 'priya.v@brightpathdental.example.com',\n    phone: '+919845012345', companyName: 'Brightpath Dental', city: 'Bengaluru', postalCode: '560001',\n    country: 'IN', tags: ['referral'], dnd: false, emailStatus: 'valid',\n    createdAt: '2025-08-14T16:40:00Z', lastActivityAt: '2026-08-10T12:00:00Z' },\n  { id: 'ct_1022', firstName: 'Lucas', lastName: 'van der Berg', email: 'lucas@vanderberg-bouw.example.com',\n    phone: '', companyName: 'Van der Berg Bouw', city: 'Utrecht', postalCode: '3511',\n    country: '', tags: [], dnd: false, emailStatus: 'valid',\n    createdAt: '2026-08-14T00:00:00Z', lastActivityAt: '2026-08-14T00:00:00Z' },\n  { id: 'ct_1023', firstName: 'Dana', lastName: 'Reyes', email: '',\n    phone: '+14155550177', companyName: '', city: '', postalCode: '',\n    country: 'US', tags: ['referral'], dnd: false, emailStatus: '',\n    createdAt: '2026-08-14T15:15:00Z', lastActivityAt: '2026-08-14T15:15:00Z' },\n  // Nobody has touched this record in three years.\n  { id: 'ct_1024', firstName: 'Harold', lastName: 'Beckett', email: 'h.beckett@example.org',\n    phone: '+14155550244', companyName: '', city: 'Fresno', postalCode: '93701',\n    country: 'US', tags: [], dnd: false, emailStatus: 'valid',\n    createdAt: '2022-05-05T05:05:00Z', lastActivityAt: '2023-01-20T05:05:00Z' },\n];\n\n/** Mimics a cursor-paginated endpoint. The cursor is opaque on purpose \u2014 the\n *  loop must not be able to cheat by doing arithmetic on it. */\nfunction fetchPage(cursor, limit) {\n  const offset = cursor ? Number(Buffer.from(String(cursor), 'base64').toString('utf8')) || 0 : 0;\n  const size = Math.max(1, Number(limit) || 10);\n  const slice = MOCK_CONTACTS.slice(offset, offset + size);\n  const nextOffset = offset + slice.length;\n  const hasMore = nextOffset < MOCK_CONTACTS.length;\n\n  return {\n    contacts: slice,\n    meta: {\n      total: MOCK_CONTACTS.length,\n      returned: slice.length,\n      nextCursor: hasMore ? Buffer.from(String(nextOffset), 'utf8').toString('base64') : null,\n      source: 'MOCK_MODE',\n    },\n  };\n}\n\nfunction processAll(items) {\n  return items.map((item, index) => {\n    const request = (item.json && item.json.request) || {};\n    return {\n      json: { ...item.json, response: fetchPage(request.cursor, request.limit) },\n      pairedItem: index,\n    };\n  });\n}\n\n// --- exported for the local test harness; n8n ignores this block -----------\nif (typeof module !== 'undefined' && module.exports) {\n  module.exports = { MOCK_CONTACTS, fetchPage, processAll };\n}\n\n// --- n8n entry point -------------------------------------------------------\nif (typeof $input !== 'undefined') {\n  return processAll($input.all());\n}"
      },
      "id": "aecf03a6-5adb-45d0-853e-57b708745903"
    },
    {
      "name": "Classify Fetch Error",
      "type": "n8n-nodes-base.code",
      "typeVersion": 2,
      "position": [
        -40,
        20
      ],
      "parameters": {
        "mode": "runOnceForAllItems",
        "jsCode": "/**\n * NODE: \"Classify API Error\"  (Code node, Run Once for All Items)\n * --------------------------------------------------------------\n * Wired to the non-2xx branch of the paginated fetch. Identical to the file in demo 01 apart\n * from this header \u2014 the same three-way decision applies to any API in any workflow, which\n * is exactly why it lives in one reviewable file instead of being retyped.\n * Turns a raw failure into a decision: retry after N seconds, alert a human, or\n * give up and queue the page for manual handling \u2014 never \"fail silently\" and\n * never \"retry forever\".\n *\n * Why this is a Code node and not the HTTP node's built-in retry:\n *\n *   n8n's node-level \"Retry On Fail\" is ignored when \"On Error\" is set to either\n *   Continue option \u2014 the node continues down the error branch on the first\n *   failure instead of retrying (n8n issue #10763). So if you want retries *and*\n *   a graceful error path, you have to own the retry loop yourself. This node\n *   plus a Wait node is that loop.\n *\n * It also does the thing most retry code skips: it reads Retry-After. A 429 that\n * says \"wait 30 seconds\" and gets retried in 2 seconds is not a retry, it is a\n * second offence, and providers extend the block for it.\n */\n\nconst RETRY = {\n  maxAttempts: 5,\n  baseDelaySeconds: 2,     // 2, 4, 8, 16, 32 before jitter\n  maxDelaySeconds: 120,\n  jitterRatio: 0.3,        // \u00b130%, so parallel executions do not retry in lockstep\n};\n\n// Transport-level failures worth retrying \u2014 the request never reached the app.\nconst RETRYABLE_NETWORK_CODES = [\n  'ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'ENOTFOUND', 'EAI_AGAIN',\n  'EPIPE', 'ESOCKETTIMEDOUT', 'ERR_SOCKET_CONNECTION_TIMEOUT',\n];\n\nconst RETRYABLE_STATUSES = [408, 425, 429, 500, 502, 503, 504, 522, 524];\n\n/**\n * n8n wraps HTTP failures differently depending on node version, whether the\n * response was JSON, and whether the failure was transport or application level.\n * Dig the useful bits out of all of them rather than assuming one shape.\n */\nfunction extractError(json) {\n  // Preferred shape: the HTTP node is configured Full Response + Never Error, so\n  // a 429 arrives as an ordinary item with statusCode, headers and body intact.\n  // That is the only configuration in which Retry-After is readable at all \u2014 on\n  // the error output, n8n hands you the error, not the response envelope.\n  const fullResponse = typeof json.statusCode === 'number' && json.headers !== undefined;\n  const container = fullResponse ? json : (json.error || json);\n\n  const status = Number(\n    container.statusCode ?? container.status ?? container.httpCode ??\n    (container.response && (container.response.status ?? container.response.statusCode)) ??\n    (container.context && container.context.statusCode) ??\n    (typeof container.code === 'number' ? container.code : undefined) ?? 0\n  ) || 0;\n\n  const headers =\n    container.headers ||\n    (container.response && container.response.headers) ||\n    (container.context && container.context.headers) || {};\n\n  const body =\n    (fullResponse ? container.body : undefined) ??\n    (container.response && (container.response.body ?? container.response.data)) ??\n    container.body ?? container.data ?? null;\n\n  const networkCode = typeof container.code === 'string' ? container.code\n    : typeof container.errno === 'string' ? container.errno : '';\n\n  const message = String(\n    container.message || container.description ||\n    (body && (body.message || body.error || body.msg)) ||\n    'Unknown error'\n  ).slice(0, 500);\n\n  return { status, headers, body, networkCode, message };\n}\n\n/** Header lookup that does not care about casing, because proxies do not either. */\nfunction header(headers, name) {\n  if (!headers) return undefined;\n  const target = name.toLowerCase();\n  for (const [key, value] of Object.entries(headers)) {\n    if (String(key).toLowerCase() === target) return value;\n  }\n  return undefined;\n}\n\n/**\n * Retry-After is legal in two forms: delay-seconds (\"120\") and an HTTP-date\n * (\"Wed, 21 Oct 2026 07:28:00 GMT\"). Both appear in the wild. Also handles the\n * x-ratelimit-reset variants, which some APIs give as a unix timestamp.\n */\nfunction retryAfterSeconds(headers, nowMs) {\n  const raw = header(headers, 'retry-after');\n  if (raw !== undefined && raw !== null && String(raw).trim() !== '') {\n    const text = String(raw).trim();\n    if (/^\\d+$/.test(text)) return Number(text);\n    const date = new Date(text);\n    if (!isNaN(date.getTime())) {\n      return Math.max(0, Math.ceil((date.getTime() - nowMs) / 1000));\n    }\n  }\n\n  const reset = header(headers, 'x-ratelimit-reset') ?? header(headers, 'ratelimit-reset');\n  if (reset !== undefined && /^\\d+$/.test(String(reset))) {\n    const value = Number(reset);\n    // Ten digits or more is an absolute unix timestamp; anything less is a delta.\n    const seconds = String(value).length >= 10\n      ? Math.ceil((value * 1000 - nowMs) / 1000)\n      : value;\n    if (seconds > 0) return seconds;\n  }\n\n  return null;\n}\n\nfunction backoffSeconds(attempt) {\n  const exponential = RETRY.baseDelaySeconds * Math.pow(2, Math.max(0, attempt - 1));\n  const capped = Math.min(exponential, RETRY.maxDelaySeconds);\n  const jitter = capped * RETRY.jitterRatio * (Math.random() * 2 - 1);\n  return Math.max(1, Math.round(capped + jitter));\n}\n\n/**\n * Returns a decision object. `disposition` is the only field the IF nodes read:\n *   'retry'      -> Wait node, then back to the HTTP node\n *   'alert'      -> notify a human immediately; the whole integration is down\n *   'quarantine' -> this one lead is bad; park it, keep processing the rest\n */\nfunction classify(json, nowMs, runIndex) {\n  const state = json.retry || { attempt: 0, maxAttempts: RETRY.maxAttempts, history: [] };\n\n  // Attempt counting has to survive the item losing its state: when an HTTP node\n  // fails, what arrives on the error output is the error, not necessarily the\n  // item that went in. n8n's $runIndex counts how many times this node has run\n  // inside the current loop, so it is the reliable floor for \"which attempt is\n  // this\". Without it, a retry loop that loses state retries forever.\n  const floor = Number.isFinite(Number(runIndex)) ? Number(runIndex) : 0;\n  const attempt = Math.max(Number(state.attempt || 0), floor) + 1;\n  const maxAttempts = Number(state.maxAttempts || RETRY.maxAttempts);\n\n  const error = extractError(json);\n  const { status, networkCode } = error;\n\n  let category;\n  let disposition;\n  let operatorMessage;\n  let retryable = false;\n\n  if (status === 401 || status === 403) {\n    category = 'auth';\n    disposition = 'alert';\n    operatorMessage = status === 401\n      ? 'CRM rejected the credentials (401). The token has expired or been revoked \u2014 reconnect the credential in n8n. Nothing will sync until this is fixed.'\n      : 'CRM accepted the credentials but refused the action (403). The connected user is missing a scope or the location ID is wrong.';\n  } else if (status === 429) {\n    category = 'rate_limit';\n    retryable = true;\n    operatorMessage = 'Rate limited by the CRM. Backing off and retrying.';\n  } else if (status === 422 || status === 400) {\n    category = 'validation';\n    disposition = 'quarantine';\n    operatorMessage = 'The CRM rejected this payload (' + status + '). This is a data problem with one lead, not an outage \u2014 parked for review. Detail: ' + error.message;\n  } else if (status === 404) {\n    category = 'not_found';\n    disposition = 'quarantine';\n    operatorMessage = 'Endpoint or record not found (404). Check the URL and the location/subaccount ID.';\n  } else if (status === 409) {\n    category = 'conflict';\n    disposition = 'quarantine';\n    operatorMessage = 'Conflict (409) \u2014 the CRM believes this contact already exists. Re-run the search rather than retrying the write.';\n  } else if (RETRYABLE_STATUSES.includes(status)) {\n    category = 'server';\n    retryable = true;\n    operatorMessage = 'CRM returned ' + status + '. Transient server-side failure, retrying.';\n  } else if (RETRYABLE_NETWORK_CODES.includes(networkCode)) {\n    category = 'network';\n    retryable = true;\n    operatorMessage = 'Network failure (' + networkCode + ') before the CRM answered. Retrying.';\n  } else if (status >= 400 && status < 500) {\n    category = 'client';\n    disposition = 'quarantine';\n    operatorMessage = 'CRM returned ' + status + '. Not retryable \u2014 the request itself is wrong.';\n  } else {\n    category = 'unknown';\n    retryable = true;\n    operatorMessage = 'Unclassified failure: ' + error.message + '. Treating as transient.';\n  }\n\n  let waitSeconds = 0;\n  if (retryable) {\n    if (attempt >= maxAttempts) {\n      disposition = 'quarantine';\n      operatorMessage = 'Gave up after ' + maxAttempts + ' attempts. Last failure: ' + operatorMessage;\n    } else {\n      disposition = 'retry';\n      // Honour the server's own instruction when it gives one; it knows better\n      // than our exponential curve does.\n      const serverHint = retryAfterSeconds(error.headers, nowMs);\n      waitSeconds = serverHint !== null\n        ? Math.min(serverHint, RETRY.maxDelaySeconds)\n        : backoffSeconds(attempt);\n      if (serverHint !== null) {\n        operatorMessage += ' Server asked for ' + serverHint + 's, honouring it.';\n      }\n    }\n  }\n\n  const history = Array.isArray(state.history) ? state.history.slice(-9) : [];\n  history.push({\n    attempt,\n    at: new Date(nowMs).toISOString(),\n    status,\n    category,\n    waitSeconds,\n  });\n\n  return {\n    ...json,\n    retry: { attempt, maxAttempts, waitSeconds, history },\n    failure: {\n      category,\n      disposition,\n      status,\n      networkCode,\n      message: error.message,\n      operatorMessage,\n      // Body is capped: a 500 from a CRM sometimes returns an entire HTML error\n      // page, and stuffing that into every execution log is how you fill a disk.\n      responseSnippet: error.body\n        ? String(typeof error.body === 'string' ? error.body : JSON.stringify(error.body)).slice(0, 600)\n        : null,\n    },\n    shouldRetry: disposition === 'retry',\n    shouldAlert: disposition === 'alert',\n    isQuarantined: disposition === 'quarantine',\n  };\n}\n\nfunction processAll(items, runIndex) {\n  const nowMs = Date.now();\n  return items.map((item, index) => ({\n    json: classify(item.json || {}, nowMs, runIndex || 0),\n    pairedItem: index,\n  }));\n}\n\n// --- exported for the local test harness; n8n ignores this block -----------\nif (typeof module !== 'undefined' && module.exports) {\n  module.exports = {\n    RETRY, extractError, header, retryAfterSeconds, backoffSeconds, classify, processAll,\n  };\n}\n\n// --- n8n entry point -------------------------------------------------------\nif (typeof $input !== 'undefined') {\n  return processAll($input.all(), typeof $runIndex !== 'undefined' ? $runIndex : 0);\n}"
      },
      "id": "231412fc-e490-40fd-894f-7a1b9ab9da37"
    },
    {
      "name": "Retry Fetch?",
      "type": "n8n-nodes-base.if",
      "typeVersion": 2.2,
      "position": [
        180,
        20
      ],
      "parameters": {
        "conditions": {
          "options": {
            "caseSensitive": true,
            "leftValue": "",
            "typeValidation": "loose",
            "version": 2
          },
          "conditions": [
            {
              "id": "should-retry",
              "leftValue": "={{ $json.shouldRetry }}",
              "rightValue": "",
              "operator": {
                "type": "boolean",
                "operation": "true",
                "singleValue": true
              }
            }
          ],
          "combinator": "and"
        },
        "looseTypeValidation": true,
        "options": {}
      },
      "id": "b39c773e-8141-4e41-94a0-a33b8d1044b6"
    },
    {
      "name": "Backoff Wait",
      "type": "n8n-nodes-base.wait",
      "typeVersion": 1.1,
      "position": [
        180,
        -180
      ],
      "parameters": {
        "amount": "={{ $json.retry.waitSeconds }}",
        "unit": "seconds"
      },
      "id": "af21aea2-9060-4c62-ba86-7ac793e07d0d"
    },
    {
      "name": "Accumulate Page",
      "type": "n8n-nodes-base.code",
      "typeVersion": 2,
      "position": [
        200,
        350
      ],
      "parameters": {
        "mode": "runOnceForAllItems",
        "jsCode": "/**\n * NODE: \"Accumulate Page\"  (Code node, Run Once for All Items)\n * -----------------------------------------------------------\n * Folds one fetched page into the running audit state and decides whether the\n * loop continues.\n *\n * Four stop conditions, all of them necessary:\n *   1. the API says there is no next cursor\n *   2. the page came back empty\n *   3. maxPages \u2014 a paginator with no ceiling will happily run until the\n *      execution times out, and a cursor bug turns that into an infinite loop\n *   4. maxContacts \u2014 this node holds every contact in memory so the clustering\n *      step can see the whole set at once. That is fine for tens of thousands\n *      and wrong for millions; see the README for the batched variant.\n *\n * It also detects a stalled cursor (the API returning the same cursor twice),\n * which is the specific way pagination loops usually fail in production.\n */\n\nfunction accumulate(state, response) {\n  const config = state.config;\n  const audit = state.audit;\n\n  const contacts = (response && (response.contacts || response.data || response.items)) || [];\n  const meta = (response && response.meta) || {};\n  const nextCursor = meta.nextCursor ?? meta.next_cursor ?? meta.cursor ?? null;\n\n  const merged = audit.contacts.concat(contacts);\n  const page = audit.page + 1;\n\n  let hasMore = Boolean(nextCursor) && contacts.length > 0;\n  let stopReason = null;\n\n  if (contacts.length === 0) {\n    hasMore = false;\n    stopReason = 'empty_page';\n  } else if (!nextCursor) {\n    hasMore = false;\n    stopReason = 'no_next_cursor';\n  } else if (nextCursor === audit.cursor) {\n    // The API handed back the cursor we just used. Continuing means fetching\n    // the same page forever.\n    hasMore = false;\n    stopReason = 'cursor_did_not_advance';\n  } else if (page >= config.maxPages) {\n    hasMore = false;\n    stopReason = 'max_pages_reached';\n  } else if (merged.length >= config.maxContacts) {\n    hasMore = false;\n    stopReason = 'max_contacts_reached';\n  }\n\n  const partial = Boolean(stopReason) &&\n    !['no_next_cursor', 'empty_page'].includes(stopReason);\n\n  return {\n    config,\n    retry: { attempt: 0, maxAttempts: 5, waitSeconds: 0, history: [] },\n    audit: {\n      ...audit,\n      page,\n      cursor: nextCursor,\n      hasMore,\n      contacts: merged,\n      pagesFetched: page,\n      partial: audit.partial || partial,\n      partialReason: audit.partialReason || (partial ? stopReason : null),\n      lastStopReason: stopReason,\n      total: meta.total ?? null,\n    },\n  };\n}\n\n// --- exported for the local test harness; n8n ignores this block -----------\nif (typeof module !== 'undefined' && module.exports) {\n  module.exports = { accumulate };\n}\n\n// --- n8n entry point -------------------------------------------------------\nif (typeof $input !== 'undefined') {\n  const item = $input.all()[0];\n  const json = item.json || {};\n\n  // Mock path: the item still carries config + audit + response.\n  // Live path: the HTTP node replaced the item with the raw API response, so\n  // the loop state comes back from the node that built the request.\n  const state = json.audit && json.config\n    ? json\n    : $('Build Page Request').all()[0].json;\n  // Mock: { response }. Live with Full Response on: { statusCode, headers, body }.\n  const response = json.response !== undefined ? json.response\n    : json.body !== undefined ? json.body : json;\n\n  return [{ json: accumulate(state, response) }];\n}"
      },
      "id": "93bad2b4-4670-4d0b-b002-2e56373ac9d7"
    },
    {
      "name": "More Pages?",
      "type": "n8n-nodes-base.if",
      "typeVersion": 2.2,
      "position": [
        400,
        350
      ],
      "parameters": {
        "conditions": {
          "options": {
            "caseSensitive": true,
            "leftValue": "",
            "typeValidation": "loose",
            "version": 2
          },
          "conditions": [
            {
              "id": "has-more",
              "leftValue": "={{ $json.audit.hasMore }}",
              "rightValue": "",
              "operator": {
                "type": "boolean",
                "operation": "true",
                "singleValue": true
              }
            }
          ],
          "combinator": "and"
        },
        "looseTypeValidation": true,
        "options": {}
      },
      "id": "a866b3e8-56cf-4679-bb67-7fe3e607fc2a"
    },
    {
      "name": "Cluster & Score Contacts",
      "type": "n8n-nodes-base.code",
      "typeVersion": 2,
      "position": [
        640,
        460
      ],
      "parameters": {
        "mode": "runOnceForAllItems",
        "jsCode": "/**\n * NODE: \"Cluster & Score Contacts\"  (Code node, Run Once for All Items)\n * --------------------------------------------------------------------\n * The whole audit happens here. Input is every contact the paginator collected;\n * output is one item containing a merge plan, a suppression list, per-contact\n * deliverability scores and a summary.\n *\n * Three things in here are worth reading properly:\n *\n * 1. BLOCKING. Comparing every contact to every other contact is O(n\u00b2) \u2014 on\n *    40,000 contacts that is 800 million comparisons and the execution dies.\n *    Instead each contact is filed under a few cheap keys (last 9 phone digits,\n *    normalised email, soundex of surname + postcode) and only contacts sharing\n *    a key are ever compared. That turns it into roughly O(n).\n *\n * 2. NEGATIVE EVIDENCE. Most dedupe tools only look for reasons to merge. The\n *    expensive mistakes come from ignoring reasons not to: two people sharing a\n *    family email address, a husband and wife at one company, \"David Smith\" and\n *    \"David Smyth\" who are two different dentists at the same practice. Every\n *    pair here is scored for conflict as well as similarity, and a conflicted\n *    pair is sent for review rather than merged. Merging two real customers is\n *    not recoverable; leaving a duplicate is.\n *\n * 3. NOTHING IS ASSERTED THAT HAS NOT BEEN CHECKED. This node does not claim an\n *    address is deliverable \u2014 it has not sent anything. It reports syntax,\n *    known-bad patterns, and the CRM's own bounce/unsubscribe flags, and marks\n *    everything else \"unverified\". A hygiene report that overstates what it\n *    knows is worse than no report.\n */\n\n// ---------------------------------------------------------------------------\n// Phone / email normalisation (kept local: an n8n Code node cannot import)\n// ---------------------------------------------------------------------------\nconst COUNTRY_RULES = {\n  US: { cc: '1', nsnLengths: [10], trunk: '1' }, CA: { cc: '1', nsnLengths: [10], trunk: '1' },\n  GB: { cc: '44', nsnLengths: [10, 9], trunk: '0' }, IE: { cc: '353', nsnLengths: [9, 8], trunk: '0' },\n  AU: { cc: '61', nsnLengths: [9], trunk: '0' }, NZ: { cc: '64', nsnLengths: [9, 8], trunk: '0' },\n  IN: { cc: '91', nsnLengths: [10], trunk: '0' }, AE: { cc: '971', nsnLengths: [9], trunk: '0' },\n  SG: { cc: '65', nsnLengths: [8], trunk: '' }, ZA: { cc: '27', nsnLengths: [9], trunk: '0' },\n  DE: { cc: '49', nsnLengths: [10, 11], trunk: '0' }, NG: { cc: '234', nsnLengths: [10], trunk: '0' },\n  NL: { cc: '31', nsnLengths: [9], trunk: '0' }, MV: { cc: '960', nsnLengths: [7], trunk: '' },\n};\n\nconst CC_LOOKUP = Object.entries(COUNTRY_RULES)\n  .sort((a, b) => b[1].cc.length - a[1].cc.length)\n  .map(([iso, rule]) => ({ iso, cc: rule.cc }));\n\nconst DISPOSABLE_DOMAINS = ['mailinator.com', 'guerrillamail.com', '10minutemail.com',\n  'tempmail.com', 'yopmail.com', 'trashmail.com', 'sharklasers.com', 'getnada.com'];\n\nconst ROLE_LOCAL_PARTS = ['info', 'admin', 'support', 'sales', 'contact', 'hello',\n  'office', 'billing', 'noreply', 'no-reply', 'help', 'team', 'enquiries'];\n\nconst TYPO_DOMAINS = { 'gmial.com': 'gmail.com', 'gmai.com': 'gmail.com', 'gmail.co': 'gmail.com',\n  'yaho.com': 'yahoo.com', 'hotmial.com': 'hotmail.com', 'outlok.com': 'outlook.com' };\n\nfunction normalizePhone(raw, defaultCountry) {\n  const text = String(raw ?? '').trim();\n  if (!text) return { e164: '', valid: false, reason: 'missing' };\n\n  const hadPlus = text.startsWith('+');\n  let digits = text.replace(/\\D/g, '');\n  if (!digits) return { e164: '', valid: false, reason: 'no_digits' };\n\n  let international = hadPlus;\n  if (!international && digits.startsWith('00')) { digits = digits.slice(2); international = true; }\n\n  let cc = '';\n  let nsn = '';\n  if (international) {\n    const match = CC_LOOKUP.find((entry) => digits.startsWith(entry.cc));\n    if (!match) return { e164: '+' + digits, valid: digits.length >= 8 && digits.length <= 15, reason: 'unknown_country' };\n    cc = match.cc;\n    nsn = digits.slice(cc.length);\n  } else {\n    const rule = COUNTRY_RULES[defaultCountry] || COUNTRY_RULES.US;\n    cc = rule.cc;\n    nsn = digits;\n    if (rule.trunk && nsn.startsWith(rule.trunk) && !rule.nsnLengths.includes(nsn.length)) {\n      const stripped = nsn.slice(rule.trunk.length);\n      if (rule.nsnLengths.includes(stripped.length)) nsn = stripped;\n    }\n  }\n\n  const iso = (CC_LOOKUP.find((entry) => entry.cc === cc) || {}).iso || defaultCountry;\n  const rule = COUNTRY_RULES[iso];\n  if (rule && rule.trunk && nsn.startsWith(rule.trunk) &&\n      !rule.nsnLengths.includes(nsn.length) &&\n      rule.nsnLengths.includes(nsn.length - rule.trunk.length)) {\n    nsn = nsn.slice(rule.trunk.length);\n  }\n\n  const lengthOk = rule ? rule.nsnLengths.includes(nsn.length) : (nsn.length >= 7 && nsn.length <= 12);\n  const placeholder = /^(\\d)\\1+$/.test(nsn) ||\n    ['1234567890', '0123456789', '5555555555'].includes(nsn);\n  const nanpBad = cc === '1' && nsn.length === 10 && (/^[01]/.test(nsn) || /^[01]/.test(nsn.slice(3)));\n\n  const valid = lengthOk && !placeholder && !nanpBad;\n  return {\n    e164: valid ? '+' + cc + nsn : '',\n    national: nsn,\n    country: iso,\n    valid,\n    reason: !lengthOk ? 'bad_length' : placeholder ? 'placeholder' : nanpBad ? 'invalid_nanp' : 'ok',\n  };\n}\n\nfunction normalizeEmail(raw) {\n  const text = String(raw ?? '').trim().toLowerCase().replace(/\\s+/g, '');\n  if (!text) return { value: '', key: '', valid: false, reason: 'missing', domain: '', flags: {} };\n\n  const parts = text.split('@');\n  if (parts.length !== 2 || !parts[0] || !parts[1]) {\n    return { value: text, key: '', valid: false, reason: 'unparseable', domain: '', flags: {} };\n  }\n\n  let [local, domain] = parts;\n  const typo = Boolean(TYPO_DOMAINS[domain]);\n\n  const valid = /^[a-z0-9!#$%&'*+/=?^_`{|}~.-]+@[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+$/\n    .test(text) && !/\\.\\./.test(text);\n\n  const bare = local.split('+')[0];\n  let keyLocal = bare;\n  let keyDomain = domain === 'googlemail.com' ? 'gmail.com' : domain;\n  if (keyDomain === 'gmail.com') keyLocal = keyLocal.replace(/\\./g, '');\n\n  return {\n    value: text,\n    key: valid ? keyLocal + '@' + keyDomain : '',\n    domain: keyDomain,\n    valid,\n    reason: valid ? 'ok' : 'bad_syntax',\n    flags: {\n      role: ROLE_LOCAL_PARTS.includes(bare),\n      disposable: DISPOSABLE_DOMAINS.includes(domain),\n      typoDomain: typo,\n      suggestion: typo ? bare + '@' + TYPO_DOMAINS[domain] : '',\n    },\n  };\n}\n\n// ---------------------------------------------------------------------------\n// Name handling\n// ---------------------------------------------------------------------------\nconst NICKNAMES = {\n  bob: 'robert', rob: 'robert', bobby: 'robert', robbie: 'robert',\n  mike: 'michael', mick: 'michael', micky: 'michael',\n  jen: 'jennifer', jenny: 'jennifer', jenn: 'jennifer',\n  dave: 'david', dan: 'daniel', danny: 'daniel', tom: 'thomas', tommy: 'thomas',\n  bill: 'william', will: 'william', billy: 'william', liz: 'elizabeth', beth: 'elizabeth',\n  kate: 'katherine', katie: 'katherine', cathy: 'catherine', sue: 'susan',\n  chris: 'christopher', steve: 'stephen', jim: 'james', jimmy: 'james',\n  tony: 'anthony', nick: 'nicholas', alex: 'alexander', sam: 'samuel',\n  matt: 'matthew', greg: 'gregory', pat: 'patrick', ed: 'edward', ted: 'edward',\n};\n\nfunction canonicalName(value) {\n  return String(value ?? '').toLowerCase().normalize('NFD')\n    .replace(/[\\u0300-\\u036f]/g, '')       // strip accents: \"Mal\u00e9\" -> \"male\"\n    .replace(/[^a-z\\s'-]/g, ' ')\n    .replace(/\\s+/g, ' ')\n    .trim();\n}\n\nfunction expandNickname(name) {\n  const canonical = canonicalName(name);\n  return NICKNAMES[canonical] || canonical;\n}\n\n/** Soundex, used only as a blocking key \u2014 cheap, phonetic, and good enough to\n *  put \"Nakamura\" and \"Nakamurra\" in the same bucket. */\nfunction soundex(value) {\n  const text = canonicalName(value).replace(/[^a-z]/g, '');\n  if (!text) return '';\n  const codes = { b: 1, f: 1, p: 1, v: 1, c: 2, g: 2, j: 2, k: 2, q: 2, s: 2, x: 2, z: 2,\n    d: 3, t: 3, l: 4, m: 5, n: 5, r: 6 };\n  let result = text[0].toUpperCase();\n  let previous = codes[text[0]] || 0;\n  for (let i = 1; i < text.length && result.length < 4; i++) {\n    const code = codes[text[i]] || 0;\n    if (code && code !== previous) result += code;\n    if (!'hw'.includes(text[i])) previous = code;\n  }\n  return (result + '000').slice(0, 4);\n}\n\n/** Jaro-Winkler similarity, 0-1. Better than Levenshtein for short human names:\n *  it rewards a shared prefix, which is exactly how names get mistyped. */\nfunction jaroWinkler(a, b) {\n  const s1 = canonicalName(a);\n  const s2 = canonicalName(b);\n  if (!s1 || !s2) return 0;\n  if (s1 === s2) return 1;\n\n  const matchWindow = Math.max(0, Math.floor(Math.max(s1.length, s2.length) / 2) - 1);\n  const s1Matched = new Array(s1.length).fill(false);\n  const s2Matched = new Array(s2.length).fill(false);\n  let matches = 0;\n\n  for (let i = 0; i < s1.length; i++) {\n    const start = Math.max(0, i - matchWindow);\n    const end = Math.min(i + matchWindow + 1, s2.length);\n    for (let j = start; j < end; j++) {\n      if (s2Matched[j] || s1[i] !== s2[j]) continue;\n      s1Matched[i] = true;\n      s2Matched[j] = true;\n      matches += 1;\n      break;\n    }\n  }\n  if (matches === 0) return 0;\n\n  let transpositions = 0;\n  let k = 0;\n  for (let i = 0; i < s1.length; i++) {\n    if (!s1Matched[i]) continue;\n    while (!s2Matched[k]) k += 1;\n    if (s1[i] !== s2[k]) transpositions += 1;\n    k += 1;\n  }\n  transpositions /= 2;\n\n  const jaro = (matches / s1.length + matches / s2.length +\n    (matches - transpositions) / matches) / 3;\n\n  let prefix = 0;\n  while (prefix < 4 && prefix < s1.length && prefix < s2.length && s1[prefix] === s2[prefix]) {\n    prefix += 1;\n  }\n  return jaro + prefix * 0.1 * (1 - jaro);\n}\n\n// ---------------------------------------------------------------------------\n// Union-Find, so that A~B and B~C put A, B and C in one cluster\n// ---------------------------------------------------------------------------\nfunction createUnionFind(size) {\n  const parent = Array.from({ length: size }, (unused, index) => index);\n  const rank = new Array(size).fill(0);\n\n  function find(index) {\n    while (parent[index] !== index) {\n      parent[index] = parent[parent[index]];  // path compression\n      index = parent[index];\n    }\n    return index;\n  }\n\n  function union(a, b) {\n    const rootA = find(a);\n    const rootB = find(b);\n    if (rootA === rootB) return;\n    if (rank[rootA] < rank[rootB]) parent[rootA] = rootB;\n    else if (rank[rootA] > rank[rootB]) parent[rootB] = rootA;\n    else { parent[rootB] = rootA; rank[rootA] += 1; }\n  }\n\n  return { find, union };\n}\n\n// ---------------------------------------------------------------------------\n// Per-contact preparation\n// ---------------------------------------------------------------------------\nfunction prepareContact(contact, config, nowMs) {\n  const phone = normalizePhone(contact.phone, contact.country || config.defaultCountry);\n  const email = normalizeEmail(contact.email);\n\n  const first = canonicalName(contact.firstName);\n  const last = canonicalName(contact.lastName);\n  const firstCanonical = expandNickname(first);\n\n  const lastActivityMs = contact.lastActivityAt ? Date.parse(contact.lastActivityAt) : NaN;\n  const ageDays = Number.isNaN(lastActivityMs) ? null\n    : Math.floor((nowMs - lastActivityMs) / 86400000);\n\n  // Completeness: how much of this record is actually filled in. Drives which\n  // record survives a merge.\n  const fields = [contact.firstName, contact.lastName, contact.email, contact.phone,\n    contact.companyName, contact.city, contact.postalCode, contact.country];\n  const completeness = fields.filter((value) => String(value ?? '').trim() !== '').length;\n\n  return {\n    raw: contact,\n    id: String(contact.id),\n    phone,\n    email,\n    first,\n    firstCanonical,\n    last,\n    fullName: [first, last].filter(Boolean).join(' '),\n    postalCode: String(contact.postalCode ?? '').toUpperCase().replace(/\\s+/g, ''),\n    company: canonicalName(contact.companyName),\n    completeness,\n    ageDays,\n    createdMs: Date.parse(contact.createdAt) || 0,\n    lastActivityMs: Number.isNaN(lastActivityMs) ? 0 : lastActivityMs,\n  };\n}\n\n// ---------------------------------------------------------------------------\n// Pair comparison\n// ---------------------------------------------------------------------------\n/**\n * Returns { verdict, confidence, signals[], conflicts[] }.\n * verdict is one of 'merge' | 'review' | 'separate'.\n */\nfunction comparePair(a, b, config) {\n  const signals = [];\n  const conflicts = [];\n  let confidence = 0;\n\n  const bothPhones = a.phone.e164 && b.phone.e164;\n  const bothEmails = a.email.key && b.email.key;\n\n  // --- positive evidence -------------------------------------------------\n  if (bothEmails && a.email.key === b.email.key) {\n    confidence = Math.max(confidence, 95);\n    signals.push('exact_email');\n  }\n  if (bothPhones && a.phone.e164 === b.phone.e164) {\n    confidence = Math.max(confidence, 90);\n    signals.push('exact_phone');\n  }\n  if (bothPhones && a.phone.e164 !== b.phone.e164 &&\n      a.phone.national && a.phone.national === b.phone.national) {\n    confidence = Math.max(confidence, 80);\n    signals.push('same_national_number');\n  }\n\n  const nameSimilarity = jaroWinkler(a.fullName, b.fullName);\n  const firstMatch = a.firstCanonical && a.firstCanonical === b.firstCanonical;\n  const lastSimilarity = jaroWinkler(a.last, b.last);\n\n  const secondSignal =\n    (a.postalCode && a.postalCode === b.postalCode) ? 'same_postcode'\n      : (a.company && a.company === b.company) ? 'same_company'\n        : (a.email.domain && a.email.domain === b.email.domain &&\n           !['gmail.com', 'yahoo.com', 'hotmail.com', 'outlook.com', 'icloud.com'].includes(a.email.domain))\n          ? 'same_email_domain'\n          : null;\n\n  if (nameSimilarity >= config.nameSimilarityThreshold && secondSignal) {\n    confidence = Math.max(confidence, 70);\n    signals.push('similar_name');\n    signals.push(secondSignal);\n  }\n\n  if (firstMatch && lastSimilarity >= 0.92 && secondSignal) {\n    confidence = Math.max(confidence, 65);\n    signals.push('nickname_or_typo_name');\n  }\n\n  if (confidence === 0) return { verdict: 'separate', confidence: 0, signals, conflicts, nameSimilarity };\n\n  // --- negative evidence -------------------------------------------------\n  // Two records that each carry a phone number, and the numbers differ, are\n  // usually two people. This is the single most valuable check in the file.\n  if (bothPhones && a.phone.e164 !== b.phone.e164 && !signals.includes('same_national_number')) {\n    conflicts.push('different_phone_numbers');\n  }\n  if (bothEmails && a.email.key !== b.email.key && bothPhones &&\n      a.phone.e164 !== b.phone.e164) {\n    conflicts.push('different_email_and_phone');\n  }\n  // Shared inbox, different humans: \"petrov.family@example.net\".\n  if (a.first && b.first && !firstMatch && jaroWinkler(a.first, b.first) < 0.70) {\n    conflicts.push('different_first_names');\n  }\n  if (a.last && b.last && lastSimilarity < 0.80) {\n    conflicts.push('different_last_names');\n  }\n\n  let verdict;\n  if (conflicts.length === 0 && confidence >= 90) verdict = 'merge';\n  else if (conflicts.length === 0 && confidence >= 65) verdict = 'review';\n  else if (conflicts.length && confidence >= 90) verdict = 'review';   // strong match, real doubt\n  else verdict = 'separate';\n\n  return { verdict, confidence, signals, conflicts, nameSimilarity };\n}\n\n// ---------------------------------------------------------------------------\n// Blocking: only compare contacts that share a cheap key\n// ---------------------------------------------------------------------------\nfunction buildBlocks(prepared) {\n  const blocks = new Map();\n\n  const add = (key, index) => {\n    if (!key) return;\n    if (!blocks.has(key)) blocks.set(key, []);\n    blocks.get(key).push(index);\n  };\n\n  prepared.forEach((contact, index) => {\n    if (contact.phone.national) add('p:' + contact.phone.national.slice(-9), index);\n    if (contact.email.key) add('e:' + contact.email.key, index);\n    if (contact.last) add('n:' + soundex(contact.last) + ':' + (contact.postalCode || contact.email.domain || ''), index);\n    if (contact.last && contact.firstCanonical) add('f:' + contact.firstCanonical + ':' + soundex(contact.last), index);\n  });\n\n  // A block containing most of the list is not a block \u2014 it is a full scan with\n  // extra steps. Usually caused by thousands of contacts sharing a blank field.\n  const MAX_BLOCK = 200;\n  const usable = [];\n  const oversized = [];\n  for (const [key, members] of blocks) {\n    if (members.length < 2) continue;\n    if (members.length > MAX_BLOCK) { oversized.push({ key, size: members.length }); continue; }\n    usable.push(members);\n  }\n\n  return { blocks: usable, oversized };\n}\n\n// ---------------------------------------------------------------------------\n// Deliverability\n// ---------------------------------------------------------------------------\nfunction scoreDeliverability(contact, config) {\n  const issues = [];\n  const raw = contact.raw;\n  const status = String(raw.emailStatus ?? '').toLowerCase();\n\n  if (!contact.email.value) issues.push('EMAIL_MISSING');\n  else if (!contact.email.valid) issues.push('EMAIL_INVALID_SYNTAX');\n  if (contact.email.flags.role) issues.push('EMAIL_ROLE_ADDRESS');\n  if (contact.email.flags.disposable) issues.push('EMAIL_DISPOSABLE');\n  if (contact.email.flags.typoDomain) issues.push('EMAIL_LIKELY_TYPO_DOMAIN');\n  if (status === 'bounced') issues.push('EMAIL_HARD_BOUNCED');\n  if (status === 'unsubscribed') issues.push('EMAIL_UNSUBSCRIBED');\n\n  if (!raw.phone) issues.push('PHONE_MISSING');\n  else if (!contact.phone.valid) issues.push('PHONE_INVALID_' + contact.phone.reason.toUpperCase());\n  if (raw.dnd === true) issues.push('DND_SET');\n\n  if (contact.ageDays === null) issues.push('NO_RECORDED_ACTIVITY');\n  else if (contact.ageDays > config.staleAfterDays) issues.push('STALE_' + contact.ageDays + 'D');\n\n  const mailable = contact.email.valid && !contact.email.flags.disposable &&\n    status !== 'bounced' && status !== 'unsubscribed' && raw.dnd !== true;\n  const textable = contact.phone.valid && raw.dnd !== true;\n\n  return {\n    id: contact.id,\n    name: [raw.firstName, raw.lastName].filter(Boolean).join(' '),\n    email: contact.email.value,\n    emailSuggestion: contact.email.flags.suggestion || '',\n    phone: contact.phone.e164 || raw.phone || '',\n    mailable,\n    textable,\n    // Deliberately not \"verified\". Nothing here has been sent or looked up over\n    // the network, so nothing here can honestly claim an address exists.\n    emailVerification: 'unverified_syntax_only',\n    reachable: mailable || textable,\n    issues,\n  };\n}\n\n// ---------------------------------------------------------------------------\n// Main\n// ---------------------------------------------------------------------------\nfunction audit(contacts, config, nowMs) {\n  const prepared = contacts.map((contact) => prepareContact(contact, config, nowMs));\n  const { blocks, oversized } = buildBlocks(prepared);\n\n  const unionFind = createUnionFind(prepared.length);\n  const pairs = [];\n  const reviewPairs = [];\n  const comparedKeys = new Set();\n  let comparisons = 0;\n\n  for (const members of blocks) {\n    for (let i = 0; i < members.length; i++) {\n      for (let j = i + 1; j < members.length; j++) {\n        const a = members[i];\n        const b = members[j];\n        const key = a < b ? a + ':' + b : b + ':' + a;\n        if (comparedKeys.has(key)) continue;      // blocks overlap; compare once\n        comparedKeys.add(key);\n        comparisons += 1;\n\n        const result = comparePair(prepared[a], prepared[b], config);\n        if (result.verdict === 'merge') {\n          unionFind.union(a, b);\n          pairs.push({ a: prepared[a].id, b: prepared[b].id, ...result });\n        } else if (result.verdict === 'review') {\n          reviewPairs.push({ a: prepared[a].id, b: prepared[b].id, ...result });\n        }\n      }\n    }\n  }\n\n  // --- group into clusters -----------------------------------------------\n  const clusterMap = new Map();\n  prepared.forEach((contact, index) => {\n    const root = unionFind.find(index);\n    if (!clusterMap.has(root)) clusterMap.set(root, []);\n    clusterMap.get(root).push(contact);\n  });\n\n  const mergePlan = [];\n  for (const members of clusterMap.values()) {\n    if (members.length < 2) continue;\n\n    // Survivor: most complete record, then most recently active, then oldest\n    // (keeping the original id preserves the account's own history), then id\n    // order so the plan is byte-identical on every run.\n    const ordered = members.slice().sort((a, b) =>\n      b.completeness - a.completeness ||\n      b.lastActivityMs - a.lastActivityMs ||\n      a.createdMs - b.createdMs ||\n      a.id.localeCompare(b.id));\n\n    const survivor = ordered[0];\n    const losers = ordered.slice(1);\n\n    // What the survivor actually gains. If the answer is \"nothing\", the merge is\n    // still worth doing \u2014 but the client should see that it is a pure cleanup.\n    const gains = {};\n    for (const field of ['firstName', 'lastName', 'email', 'phone', 'companyName',\n      'city', 'postalCode', 'country']) {\n    

About this workflow

CRM List Hygiene & Duplicate Merge Planner. Uses httpRequest. Event-driven trigger; 24 nodes.

Source: https://github.com/ihthicodes/n8n-agency-demos/blob/main/demos/02-crm-list-hygiene/workflow.json — original creator credit. Request a take-down →

More Web Scraping workflows → · Browse all categories →

Related workflows

Workflows that share integrations, category, or trigger type with this one. All free to copy and import.