{ "name": "CRM List Hygiene & Duplicate Merge Planner", "active": false, "settings": { "executionOrder": "v1", "saveDataErrorExecution": "all", "saveDataSuccessExecution": "all", "saveManualExecutions": true }, "nodes": [ { "name": "Run Audit Now", "type": "n8n-nodes-base.manualTrigger", "typeVersion": 1, "position": [ -1180, 260 ], "parameters": {}, "id": "242dc348-b0d6-4daa-8dd5-fc10d69013bc" }, { "name": "Weekly Audit", "type": "n8n-nodes-base.scheduleTrigger", "typeVersion": 1.2, "position": [ -1180, 440 ], "parameters": { "rule": { "interval": [ { "field": "weeks", "triggerAtDay": [ 1 ], "triggerAtHour": 6, "triggerAtMinute": 0 } ] } }, "id": "7a5b4bd8-8cc9-4cd0-8213-bd82f24aa404" }, { "name": "Init Audit Run", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ -960, 350 ], "parameters": { "mode": "runOnceForAllItems", "jsCode": "/**\n * NODE: \"Init Audit Run\" (Code node, Run Once for All Items)\n * ----------------------------------------------------------\n * Sets up one audit pass. Everything tunable lives here so the rest of the\n * workflow has no configuration in it.\n *\n * DRY_RUN is on by default and stays on until someone deliberately turns it off.\n * A list-cleaning tool that merges 400 contacts on its first run, in a client's\n * production CRM, is not a tool \u2014 it is an incident. This one produces a plan\n * first and merges only when told to.\n */\n\nconst CONFIG = {\n // --- safety ------------------------------------------------------------\n dryRun: true, // produce the plan; write nothing\n mockMode: true, // use the built-in fixture contacts, no credentials\n\n // --- source ------------------------------------------------------------\n locationId: 'REPLACE_WITH_LOCATION_ID',\n pageSize: 10, // raise to 100 against a real API\n maxPages: 50, // hard ceiling \u2014 a paginator with no ceiling is a bug\n maxContacts: 25000, // memory guard, see README for the >25k approach\n\n // --- matching ----------------------------------------------------------\n // A pair is a duplicate if any rule below fires. Rules are ordered by how much\n // you should trust them; the weakest one is deliberately conservative.\n nameSimilarityThreshold: 0.90, // Jaro-Winkler, 0-1\n requireSecondSignal: true, // a fuzzy name match alone is never enough\n\n // --- deliverability ----------------------------------------------------\n defaultCountry: 'US',\n staleAfterDays: 730, // no engagement in 2 years\n};\n\nfunction processAll(items) {\n const startedAt = new Date().toISOString();\n\n return [{\n json: {\n config: CONFIG,\n audit: {\n runId: 'audit_' + startedAt.slice(0, 10).replace(/-/g, '') + '_'\n + Math.random().toString(16).slice(2, 8),\n startedAt,\n page: 0,\n cursor: null,\n hasMore: true,\n contacts: [],\n pagesFetched: 0,\n partial: false,\n partialReason: null,\n },\n // Seeds the shared retry classifier.\n retry: { attempt: 0, maxAttempts: 5, waitSeconds: 0, history: [] },\n },\n pairedItem: items.length ? 0 : undefined,\n }];\n}\n\n// --- exported for the local test harness; n8n ignores this block -----------\nif (typeof module !== 'undefined' && module.exports) {\n module.exports = { CONFIG, processAll };\n}\n\n// --- n8n entry point -------------------------------------------------------\nif (typeof $input !== 'undefined') {\n return processAll($input.all());\n}" }, "id": "08932967-05b5-4e8b-8226-bdfa8900e70d" }, { "name": "Build Page Request", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ -740, 350 ], "parameters": { "mode": "runOnceForAllItems", "jsCode": "/**\n * NODE: \"Build Page Request\" (Code node, Run Once for All Items)\n * --------------------------------------------------------------\n * The head of the pagination loop. Three nodes feed into it \u2014 the initialiser,\n * the accumulator (next page), and the backoff wait (retry the same page) \u2014 and\n * only one of them hands over a complete state object.\n *\n * That is the whole reason this node exists. When an HTTP node fails, what comes\n * out of its error output is the error, not the item that went in, so anything\n * downstream of a retry has to be able to rebuild the loop state from somewhere\n * else. This node does that in one place, and the HTTP node reads its cursor from\n * here rather than from whatever happened to be on the item.\n */\n\n/** Pull the freshest loop state from wherever it survived. */\nfunction resolveState(item, fallbacks) {\n if (item && item.json && item.json.audit && item.json.config) return item.json;\n for (const fallback of fallbacks) {\n if (fallback && fallback.audit && fallback.config) return fallback;\n }\n return null;\n}\n\nfunction buildRequest(state) {\n const { config, audit } = state;\n\n return {\n config,\n audit,\n retry: state.retry || { attempt: 0, maxAttempts: 5, waitSeconds: 0, history: [] },\n request: {\n locationId: config.locationId,\n limit: config.pageSize,\n // Cursor-based, not offset-based. Offset pagination over a table that is\n // being written to while you read it silently skips and repeats records;\n // for a dedupe job that is the one failure mode you cannot tolerate.\n cursor: audit.cursor || '',\n page: audit.page + 1,\n },\n useLive: config.mockMode !== true,\n };\n}\n\n// --- exported for the local test harness; n8n ignores this block -----------\nif (typeof module !== 'undefined' && module.exports) {\n module.exports = { resolveState, buildRequest };\n}\n\n// --- n8n entry point -------------------------------------------------------\nif (typeof $input !== 'undefined') {\n const fallbacks = [];\n for (const nodeName of ['Accumulate Page', 'Init Audit Run']) {\n try {\n const upstream = $(nodeName).all();\n if (upstream && upstream.length) fallbacks.push(upstream[upstream.length - 1].json);\n } catch (error) {\n // That node has not run on this path yet. Expected on the first pass.\n }\n }\n\n const state = resolveState($input.all()[0], fallbacks);\n if (!state) {\n throw new Error('Build Page Request could not resolve the audit state. '\n + 'Check that \"Init Audit Run\" ran before this node.');\n }\n return [{ json: buildRequest(state) }];\n}" }, "id": "2be14fa8-35cc-4fa9-97ae-0848e35e1185" }, { "name": "Use Live CRM?", "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ -520, 350 ], "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "loose", "version": 2 }, "conditions": [ { "id": "use-live", "leftValue": "={{ $json.useLive }}", "rightValue": "", "operator": { "type": "boolean", "operation": "true", "singleValue": true } } ], "combinator": "and" }, "looseTypeValidation": true, "options": {} }, "id": "b4dccc94-313d-448b-be00-61b1afc194cd" }, { "name": "CRM: Fetch Contact Page", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ -280, 220 ], "parameters": { "method": "GET", "url": "https://services.leadconnectorhq.com/contacts/", "authentication": "genericCredentialType", "genericAuthType": "httpHeaderAuth", "sendQuery": true, "specifyQuery": "keypair", "queryParameters": { "parameters": [ { "name": "locationId", "value": "={{ $('Build Page Request').item.json.request.locationId }}" }, { "name": "limit", "value": "={{ $('Build Page Request').item.json.request.limit }}" }, { "name": "startAfterId", "value": "={{ $('Build Page Request').item.json.request.cursor }}" } ] }, "sendHeaders": true, "specifyHeaders": "keypair", "headerParameters": { "parameters": [ { "name": "Version", "value": "2021-07-28" }, { "name": "Accept", "value": "application/json" } ] }, "options": { "timeout": 30000, "response": { "response": { "fullResponse": true, "neverError": true } } } }, "id": "36baeac7-13c0-4aaa-9d02-e1286cef3ae5" }, { "name": "Mock Contact Page", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ -280, 480 ], "parameters": { "mode": "runOnceForAllItems", "jsCode": "/**\n * NODE: \"Mock Contact Page\" (Code node, Run Once for All Items)\n * -------------------------------------------------------------\n * Stands in for the CRM's paginated contact endpoint so the whole audit runs\n * with no credentials and no client account. Same response shape as the live\n * call, same cursor semantics, so nothing downstream can tell the difference.\n *\n * The fixture list is 24 contacts from a CRM that has been running for three\n * years: five real duplicate clusters, two near-misses that must NOT be merged,\n * dead email addresses, un-textable numbers, a DND record, and a contact with no\n * reachable channel at all. If a dedupe tool gets this list right it will get a\n * real one right.\n */\n\nconst MOCK_CONTACTS = [\n // --- cluster A: same person, three records, three spellings of the email ---\n { id: 'ct_1001', firstName: 'Robert', lastName: 'Chen', email: 'rob.chen@northsidehvac.example.com',\n phone: '+14155550143', companyName: 'Northside HVAC', city: 'San Francisco', postalCode: '94110',\n country: 'US', tags: ['source:website_form'], dnd: false, emailStatus: 'valid',\n createdAt: '2023-02-11T10:00:00Z', lastActivityAt: '2026-07-30T14:22:00Z' },\n { id: 'ct_1002', firstName: 'Bob', lastName: 'Chen', email: 'r.chen@northsidehvac.example.com',\n phone: '415-555-0143', companyName: 'Northside HVAC Inc', city: 'San Francisco', postalCode: '94110',\n country: 'US', tags: ['trade-show-2024'], dnd: false, emailStatus: 'valid',\n createdAt: '2024-05-02T09:30:00Z', lastActivityAt: '2025-11-02T09:00:00Z' },\n { id: 'ct_1003', firstName: 'Robert', lastName: 'Chen', email: '',\n phone: '(415) 555 0143', companyName: '', city: '', postalCode: '',\n country: 'US', tags: [], dnd: false, emailStatus: '',\n createdAt: '2025-01-19T16:45:00Z', lastActivityAt: null },\n\n // --- cluster B: one Gmail inbox, two \"different\" addresses -----------------\n { id: 'ct_1004', firstName: 'Sarah', lastName: 'Okonkwo', email: 'sarah.okonkwo@gmail.com',\n phone: '', companyName: '', city: 'Manchester', postalCode: 'M1 4BT',\n country: 'GB', tags: ['source:facebook_lead_ads'], dnd: false, emailStatus: 'valid',\n createdAt: '2024-08-01T11:00:00Z', lastActivityAt: '2026-06-14T08:10:00Z' },\n { id: 'ct_1005', firstName: 'Sarah', lastName: 'Okonkwo', email: 's.arah.okonkwo+forms@googlemail.com',\n phone: '+447700900461', companyName: '', city: 'Manchester', postalCode: 'M1 4BT',\n country: 'GB', tags: ['newsletter'], dnd: false, emailStatus: 'valid',\n createdAt: '2025-03-22T13:05:00Z', lastActivityAt: '2026-08-01T19:40:00Z' },\n\n // --- cluster C: same number, written two ways -----------------------------\n { id: 'ct_1006', firstName: 'Aminath', lastName: 'Rasheed', email: 'aminath.r@example.com',\n phone: '+9607712345', companyName: '', city: 'Mal\u00e9', postalCode: '20026',\n country: 'MV', tags: ['existing-client'], dnd: false, emailStatus: 'valid',\n createdAt: '2023-09-14T07:00:00Z', lastActivityAt: '2026-08-05T06:30:00Z' },\n { id: 'ct_1007', firstName: 'Aminath', lastName: '', email: '',\n phone: '00960 771 2345', companyName: 'Rasheed Trading', city: '', postalCode: '',\n country: '', tags: [], dnd: false, emailStatus: '',\n createdAt: '2024-01-08T05:20:00Z', lastActivityAt: null },\n\n // --- cluster D: nickname + full name, one mobile --------------------------\n { id: 'ct_1008', firstName: 'Mike', lastName: 'Johnson', email: 'mike.johnson@example.com',\n phone: '+14155550188', companyName: '', city: 'Oakland', postalCode: '94607',\n country: 'US', tags: [], dnd: false, emailStatus: 'valid',\n createdAt: '2023-11-30T18:00:00Z', lastActivityAt: '2025-02-11T12:00:00Z' },\n { id: 'ct_1009', firstName: 'Michael', lastName: 'Johnson', email: 'mjohnson@example.com',\n phone: '4155550188', companyName: 'Johnson Plumbing', city: 'Oakland', postalCode: '94607',\n country: 'US', tags: ['quote-requested'], dnd: false, emailStatus: 'valid',\n createdAt: '2025-06-04T10:15:00Z', lastActivityAt: '2026-07-19T15:45:00Z' },\n\n // --- cluster E: typo'd surname, identical email ---------------------------\n { id: 'ct_1010', firstName: 'Jennifer', lastName: 'Nakamura', email: 'jen.nakamura@brightpath.example.com',\n phone: '+61412345678', companyName: 'Brightpath', city: 'Sydney', postalCode: '2000',\n country: 'AU', tags: ['vip'], dnd: false, emailStatus: 'valid',\n createdAt: '2023-04-17T08:00:00Z', lastActivityAt: '2026-08-09T02:00:00Z' },\n { id: 'ct_1011', firstName: 'Jennifer', lastName: 'Nakamurra', email: 'jen.nakamura@brightpath.example.com',\n phone: '', companyName: '', city: '', postalCode: '',\n country: 'AU', tags: [], dnd: false, emailStatus: 'valid',\n createdAt: '2025-10-01T00:00:00Z', lastActivityAt: null },\n\n // --- near-misses: similar people who are NOT the same person --------------\n { id: 'ct_1012', firstName: 'David', lastName: 'Smith', email: 'dsmith@acmedental.example.com',\n phone: '+14155550201', companyName: 'Acme Dental', city: 'San Jose', postalCode: '95112',\n country: 'US', tags: [], dnd: false, emailStatus: 'valid',\n createdAt: '2024-02-20T09:00:00Z', lastActivityAt: '2026-05-30T11:00:00Z' },\n { id: 'ct_1013', firstName: 'David', lastName: 'Smyth', email: 'dsmyth@acmedental.example.com',\n phone: '+14155550202', companyName: 'Acme Dental', city: 'San Jose', postalCode: '95112',\n country: 'US', tags: [], dnd: false, emailStatus: 'valid',\n createdAt: '2024-02-20T09:05:00Z', lastActivityAt: '2026-05-30T11:05:00Z' },\n // Shared family address, two real people.\n { id: 'ct_1014', firstName: 'Anna', lastName: 'Petrova', email: 'petrov.family@example.net',\n phone: '+14155550210', companyName: '', city: 'Daly City', postalCode: '94014',\n country: 'US', tags: [], dnd: false, emailStatus: 'valid',\n createdAt: '2024-07-07T07:07:00Z', lastActivityAt: '2026-03-01T10:00:00Z' },\n { id: 'ct_1015', firstName: 'Igor', lastName: 'Petrov', email: 'petrov.family@example.net',\n phone: '+14155550211', companyName: '', city: 'Daly City', postalCode: '94014',\n country: 'US', tags: [], dnd: false, emailStatus: 'valid',\n createdAt: '2024-07-07T07:09:00Z', lastActivityAt: '2026-03-01T10:02:00Z' },\n\n // --- deliverability problems ---------------------------------------------\n { id: 'ct_1016', firstName: 'Info', lastName: '', email: 'info@westsideclinic.example.com',\n phone: '+14155550222', companyName: 'Westside Clinic', city: 'San Francisco', postalCode: '94115',\n country: 'US', tags: [], dnd: false, emailStatus: 'valid',\n createdAt: '2023-06-01T12:00:00Z', lastActivityAt: '2026-01-15T09:00:00Z' },\n { id: 'ct_1017', firstName: 'Test', lastName: 'Test', email: 'test@mailinator.com',\n phone: '5555555555', companyName: '', city: '', postalCode: '',\n country: 'US', tags: [], dnd: false, emailStatus: 'valid',\n createdAt: '2025-04-04T04:04:00Z', lastActivityAt: null },\n { id: 'ct_1018', firstName: 'Karen', lastName: 'Whitfield', email: 'karen.whitfield@gmial.com',\n phone: '+14155550233', companyName: '', city: 'Sacramento', postalCode: '95814',\n country: 'US', tags: [], dnd: false, emailStatus: 'bounced',\n createdAt: '2023-01-05T08:00:00Z', lastActivityAt: '2023-03-01T08:00:00Z' },\n { id: 'ct_1019', firstName: 'Daniel', lastName: 'Okafor', email: 'daniel.okafor@example.com',\n phone: '+2348012345678', companyName: '', city: 'Lagos', postalCode: '',\n country: '', tags: ['unsubscribed'], dnd: true, emailStatus: 'unsubscribed',\n createdAt: '2024-09-09T09:09:00Z', lastActivityAt: '2024-10-01T09:00:00Z' },\n { id: 'ct_1020', firstName: '', lastName: '', email: 'not-an-email',\n phone: '123', companyName: '', city: '', postalCode: '',\n country: 'US', tags: [], dnd: false, emailStatus: '',\n createdAt: '2025-12-25T00:00:00Z', lastActivityAt: null },\n\n // --- clean singles --------------------------------------------------------\n { id: 'ct_1021', firstName: 'Priya', lastName: 'Venkatesan', email: 'priya.v@brightpathdental.example.com',\n phone: '+919845012345', companyName: 'Brightpath Dental', city: 'Bengaluru', postalCode: '560001',\n country: 'IN', tags: ['referral'], dnd: false, emailStatus: 'valid',\n createdAt: '2025-08-14T16:40:00Z', lastActivityAt: '2026-08-10T12:00:00Z' },\n { id: 'ct_1022', firstName: 'Lucas', lastName: 'van der Berg', email: 'lucas@vanderberg-bouw.example.com',\n phone: '', companyName: 'Van der Berg Bouw', city: 'Utrecht', postalCode: '3511',\n country: '', tags: [], dnd: false, emailStatus: 'valid',\n createdAt: '2026-08-14T00:00:00Z', lastActivityAt: '2026-08-14T00:00:00Z' },\n { id: 'ct_1023', firstName: 'Dana', lastName: 'Reyes', email: '',\n phone: '+14155550177', companyName: '', city: '', postalCode: '',\n country: 'US', tags: ['referral'], dnd: false, emailStatus: '',\n createdAt: '2026-08-14T15:15:00Z', lastActivityAt: '2026-08-14T15:15:00Z' },\n // Nobody has touched this record in three years.\n { id: 'ct_1024', firstName: 'Harold', lastName: 'Beckett', email: 'h.beckett@example.org',\n phone: '+14155550244', companyName: '', city: 'Fresno', postalCode: '93701',\n country: 'US', tags: [], dnd: false, emailStatus: 'valid',\n createdAt: '2022-05-05T05:05:00Z', lastActivityAt: '2023-01-20T05:05:00Z' },\n];\n\n/** Mimics a cursor-paginated endpoint. The cursor is opaque on purpose \u2014 the\n * loop must not be able to cheat by doing arithmetic on it. */\nfunction fetchPage(cursor, limit) {\n const offset = cursor ? Number(Buffer.from(String(cursor), 'base64').toString('utf8')) || 0 : 0;\n const size = Math.max(1, Number(limit) || 10);\n const slice = MOCK_CONTACTS.slice(offset, offset + size);\n const nextOffset = offset + slice.length;\n const hasMore = nextOffset < MOCK_CONTACTS.length;\n\n return {\n contacts: slice,\n meta: {\n total: MOCK_CONTACTS.length,\n returned: slice.length,\n nextCursor: hasMore ? Buffer.from(String(nextOffset), 'utf8').toString('base64') : null,\n source: 'MOCK_MODE',\n },\n };\n}\n\nfunction processAll(items) {\n return items.map((item, index) => {\n const request = (item.json && item.json.request) || {};\n return {\n json: { ...item.json, response: fetchPage(request.cursor, request.limit) },\n pairedItem: index,\n };\n });\n}\n\n// --- exported for the local test harness; n8n ignores this block -----------\nif (typeof module !== 'undefined' && module.exports) {\n module.exports = { MOCK_CONTACTS, fetchPage, processAll };\n}\n\n// --- n8n entry point -------------------------------------------------------\nif (typeof $input !== 'undefined') {\n return processAll($input.all());\n}" }, "id": "aecf03a6-5adb-45d0-853e-57b708745903" }, { "name": "Classify Fetch Error", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ -40, 20 ], "parameters": { "mode": "runOnceForAllItems", "jsCode": "/**\n * NODE: \"Classify API Error\" (Code node, Run Once for All Items)\n * --------------------------------------------------------------\n * Wired to the non-2xx branch of the paginated fetch. Identical to the file in demo 01 apart\n * from this header \u2014 the same three-way decision applies to any API in any workflow, which\n * is exactly why it lives in one reviewable file instead of being retyped.\n * Turns a raw failure into a decision: retry after N seconds, alert a human, or\n * give up and queue the page for manual handling \u2014 never \"fail silently\" and\n * never \"retry forever\".\n *\n * Why this is a Code node and not the HTTP node's built-in retry:\n *\n * n8n's node-level \"Retry On Fail\" is ignored when \"On Error\" is set to either\n * Continue option \u2014 the node continues down the error branch on the first\n * failure instead of retrying (n8n issue #10763). So if you want retries *and*\n * a graceful error path, you have to own the retry loop yourself. This node\n * plus a Wait node is that loop.\n *\n * It also does the thing most retry code skips: it reads Retry-After. A 429 that\n * says \"wait 30 seconds\" and gets retried in 2 seconds is not a retry, it is a\n * second offence, and providers extend the block for it.\n */\n\nconst RETRY = {\n maxAttempts: 5,\n baseDelaySeconds: 2, // 2, 4, 8, 16, 32 before jitter\n maxDelaySeconds: 120,\n jitterRatio: 0.3, // \u00b130%, so parallel executions do not retry in lockstep\n};\n\n// Transport-level failures worth retrying \u2014 the request never reached the app.\nconst RETRYABLE_NETWORK_CODES = [\n 'ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'ENOTFOUND', 'EAI_AGAIN',\n 'EPIPE', 'ESOCKETTIMEDOUT', 'ERR_SOCKET_CONNECTION_TIMEOUT',\n];\n\nconst RETRYABLE_STATUSES = [408, 425, 429, 500, 502, 503, 504, 522, 524];\n\n/**\n * n8n wraps HTTP failures differently depending on node version, whether the\n * response was JSON, and whether the failure was transport or application level.\n * Dig the useful bits out of all of them rather than assuming one shape.\n */\nfunction extractError(json) {\n // Preferred shape: the HTTP node is configured Full Response + Never Error, so\n // a 429 arrives as an ordinary item with statusCode, headers and body intact.\n // That is the only configuration in which Retry-After is readable at all \u2014 on\n // the error output, n8n hands you the error, not the response envelope.\n const fullResponse = typeof json.statusCode === 'number' && json.headers !== undefined;\n const container = fullResponse ? json : (json.error || json);\n\n const status = Number(\n container.statusCode ?? container.status ?? container.httpCode ??\n (container.response && (container.response.status ?? container.response.statusCode)) ??\n (container.context && container.context.statusCode) ??\n (typeof container.code === 'number' ? container.code : undefined) ?? 0\n ) || 0;\n\n const headers =\n container.headers ||\n (container.response && container.response.headers) ||\n (container.context && container.context.headers) || {};\n\n const body =\n (fullResponse ? container.body : undefined) ??\n (container.response && (container.response.body ?? container.response.data)) ??\n container.body ?? container.data ?? null;\n\n const networkCode = typeof container.code === 'string' ? container.code\n : typeof container.errno === 'string' ? container.errno : '';\n\n const message = String(\n container.message || container.description ||\n (body && (body.message || body.error || body.msg)) ||\n 'Unknown error'\n ).slice(0, 500);\n\n return { status, headers, body, networkCode, message };\n}\n\n/** Header lookup that does not care about casing, because proxies do not either. */\nfunction header(headers, name) {\n if (!headers) return undefined;\n const target = name.toLowerCase();\n for (const [key, value] of Object.entries(headers)) {\n if (String(key).toLowerCase() === target) return value;\n }\n return undefined;\n}\n\n/**\n * Retry-After is legal in two forms: delay-seconds (\"120\") and an HTTP-date\n * (\"Wed, 21 Oct 2026 07:28:00 GMT\"). Both appear in the wild. Also handles the\n * x-ratelimit-reset variants, which some APIs give as a unix timestamp.\n */\nfunction retryAfterSeconds(headers, nowMs) {\n const raw = header(headers, 'retry-after');\n if (raw !== undefined && raw !== null && String(raw).trim() !== '') {\n const text = String(raw).trim();\n if (/^\\d+$/.test(text)) return Number(text);\n const date = new Date(text);\n if (!isNaN(date.getTime())) {\n return Math.max(0, Math.ceil((date.getTime() - nowMs) / 1000));\n }\n }\n\n const reset = header(headers, 'x-ratelimit-reset') ?? header(headers, 'ratelimit-reset');\n if (reset !== undefined && /^\\d+$/.test(String(reset))) {\n const value = Number(reset);\n // Ten digits or more is an absolute unix timestamp; anything less is a delta.\n const seconds = String(value).length >= 10\n ? Math.ceil((value * 1000 - nowMs) / 1000)\n : value;\n if (seconds > 0) return seconds;\n }\n\n return null;\n}\n\nfunction backoffSeconds(attempt) {\n const exponential = RETRY.baseDelaySeconds * Math.pow(2, Math.max(0, attempt - 1));\n const capped = Math.min(exponential, RETRY.maxDelaySeconds);\n const jitter = capped * RETRY.jitterRatio * (Math.random() * 2 - 1);\n return Math.max(1, Math.round(capped + jitter));\n}\n\n/**\n * Returns a decision object. `disposition` is the only field the IF nodes read:\n * 'retry' -> Wait node, then back to the HTTP node\n * 'alert' -> notify a human immediately; the whole integration is down\n * 'quarantine' -> this one lead is bad; park it, keep processing the rest\n */\nfunction classify(json, nowMs, runIndex) {\n const state = json.retry || { attempt: 0, maxAttempts: RETRY.maxAttempts, history: [] };\n\n // Attempt counting has to survive the item losing its state: when an HTTP node\n // fails, what arrives on the error output is the error, not necessarily the\n // item that went in. n8n's $runIndex counts how many times this node has run\n // inside the current loop, so it is the reliable floor for \"which attempt is\n // this\". Without it, a retry loop that loses state retries forever.\n const floor = Number.isFinite(Number(runIndex)) ? Number(runIndex) : 0;\n const attempt = Math.max(Number(state.attempt || 0), floor) + 1;\n const maxAttempts = Number(state.maxAttempts || RETRY.maxAttempts);\n\n const error = extractError(json);\n const { status, networkCode } = error;\n\n let category;\n let disposition;\n let operatorMessage;\n let retryable = false;\n\n if (status === 401 || status === 403) {\n category = 'auth';\n disposition = 'alert';\n operatorMessage = status === 401\n ? 'CRM rejected the credentials (401). The token has expired or been revoked \u2014 reconnect the credential in n8n. Nothing will sync until this is fixed.'\n : 'CRM accepted the credentials but refused the action (403). The connected user is missing a scope or the location ID is wrong.';\n } else if (status === 429) {\n category = 'rate_limit';\n retryable = true;\n operatorMessage = 'Rate limited by the CRM. Backing off and retrying.';\n } else if (status === 422 || status === 400) {\n category = 'validation';\n disposition = 'quarantine';\n operatorMessage = 'The CRM rejected this payload (' + status + '). This is a data problem with one lead, not an outage \u2014 parked for review. Detail: ' + error.message;\n } else if (status === 404) {\n category = 'not_found';\n disposition = 'quarantine';\n operatorMessage = 'Endpoint or record not found (404). Check the URL and the location/subaccount ID.';\n } else if (status === 409) {\n category = 'conflict';\n disposition = 'quarantine';\n operatorMessage = 'Conflict (409) \u2014 the CRM believes this contact already exists. Re-run the search rather than retrying the write.';\n } else if (RETRYABLE_STATUSES.includes(status)) {\n category = 'server';\n retryable = true;\n operatorMessage = 'CRM returned ' + status + '. Transient server-side failure, retrying.';\n } else if (RETRYABLE_NETWORK_CODES.includes(networkCode)) {\n category = 'network';\n retryable = true;\n operatorMessage = 'Network failure (' + networkCode + ') before the CRM answered. Retrying.';\n } else if (status >= 400 && status < 500) {\n category = 'client';\n disposition = 'quarantine';\n operatorMessage = 'CRM returned ' + status + '. Not retryable \u2014 the request itself is wrong.';\n } else {\n category = 'unknown';\n retryable = true;\n operatorMessage = 'Unclassified failure: ' + error.message + '. Treating as transient.';\n }\n\n let waitSeconds = 0;\n if (retryable) {\n if (attempt >= maxAttempts) {\n disposition = 'quarantine';\n operatorMessage = 'Gave up after ' + maxAttempts + ' attempts. Last failure: ' + operatorMessage;\n } else {\n disposition = 'retry';\n // Honour the server's own instruction when it gives one; it knows better\n // than our exponential curve does.\n const serverHint = retryAfterSeconds(error.headers, nowMs);\n waitSeconds = serverHint !== null\n ? Math.min(serverHint, RETRY.maxDelaySeconds)\n : backoffSeconds(attempt);\n if (serverHint !== null) {\n operatorMessage += ' Server asked for ' + serverHint + 's, honouring it.';\n }\n }\n }\n\n const history = Array.isArray(state.history) ? state.history.slice(-9) : [];\n history.push({\n attempt,\n at: new Date(nowMs).toISOString(),\n status,\n category,\n waitSeconds,\n });\n\n return {\n ...json,\n retry: { attempt, maxAttempts, waitSeconds, history },\n failure: {\n category,\n disposition,\n status,\n networkCode,\n message: error.message,\n operatorMessage,\n // Body is capped: a 500 from a CRM sometimes returns an entire HTML error\n // page, and stuffing that into every execution log is how you fill a disk.\n responseSnippet: error.body\n ? String(typeof error.body === 'string' ? error.body : JSON.stringify(error.body)).slice(0, 600)\n : null,\n },\n shouldRetry: disposition === 'retry',\n shouldAlert: disposition === 'alert',\n isQuarantined: disposition === 'quarantine',\n };\n}\n\nfunction processAll(items, runIndex) {\n const nowMs = Date.now();\n return items.map((item, index) => ({\n json: classify(item.json || {}, nowMs, runIndex || 0),\n pairedItem: index,\n }));\n}\n\n// --- exported for the local test harness; n8n ignores this block -----------\nif (typeof module !== 'undefined' && module.exports) {\n module.exports = {\n RETRY, extractError, header, retryAfterSeconds, backoffSeconds, classify, processAll,\n };\n}\n\n// --- n8n entry point -------------------------------------------------------\nif (typeof $input !== 'undefined') {\n return processAll($input.all(), typeof $runIndex !== 'undefined' ? $runIndex : 0);\n}" }, "id": "231412fc-e490-40fd-894f-7a1b9ab9da37" }, { "name": "Retry Fetch?", "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 180, 20 ], "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "loose", "version": 2 }, "conditions": [ { "id": "should-retry", "leftValue": "={{ $json.shouldRetry }}", "rightValue": "", "operator": { "type": "boolean", "operation": "true", "singleValue": true } } ], "combinator": "and" }, "looseTypeValidation": true, "options": {} }, "id": "b39c773e-8141-4e41-94a0-a33b8d1044b6" }, { "name": "Backoff Wait", "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 180, -180 ], "parameters": { "amount": "={{ $json.retry.waitSeconds }}", "unit": "seconds" }, "id": "af21aea2-9060-4c62-ba86-7ac793e07d0d" }, { "name": "Accumulate Page", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 200, 350 ], "parameters": { "mode": "runOnceForAllItems", "jsCode": "/**\n * NODE: \"Accumulate Page\" (Code node, Run Once for All Items)\n * -----------------------------------------------------------\n * Folds one fetched page into the running audit state and decides whether the\n * loop continues.\n *\n * Four stop conditions, all of them necessary:\n * 1. the API says there is no next cursor\n * 2. the page came back empty\n * 3. maxPages \u2014 a paginator with no ceiling will happily run until the\n * execution times out, and a cursor bug turns that into an infinite loop\n * 4. maxContacts \u2014 this node holds every contact in memory so the clustering\n * step can see the whole set at once. That is fine for tens of thousands\n * and wrong for millions; see the README for the batched variant.\n *\n * It also detects a stalled cursor (the API returning the same cursor twice),\n * which is the specific way pagination loops usually fail in production.\n */\n\nfunction accumulate(state, response) {\n const config = state.config;\n const audit = state.audit;\n\n const contacts = (response && (response.contacts || response.data || response.items)) || [];\n const meta = (response && response.meta) || {};\n const nextCursor = meta.nextCursor ?? meta.next_cursor ?? meta.cursor ?? null;\n\n const merged = audit.contacts.concat(contacts);\n const page = audit.page + 1;\n\n let hasMore = Boolean(nextCursor) && contacts.length > 0;\n let stopReason = null;\n\n if (contacts.length === 0) {\n hasMore = false;\n stopReason = 'empty_page';\n } else if (!nextCursor) {\n hasMore = false;\n stopReason = 'no_next_cursor';\n } else if (nextCursor === audit.cursor) {\n // The API handed back the cursor we just used. Continuing means fetching\n // the same page forever.\n hasMore = false;\n stopReason = 'cursor_did_not_advance';\n } else if (page >= config.maxPages) {\n hasMore = false;\n stopReason = 'max_pages_reached';\n } else if (merged.length >= config.maxContacts) {\n hasMore = false;\n stopReason = 'max_contacts_reached';\n }\n\n const partial = Boolean(stopReason) &&\n !['no_next_cursor', 'empty_page'].includes(stopReason);\n\n return {\n config,\n retry: { attempt: 0, maxAttempts: 5, waitSeconds: 0, history: [] },\n audit: {\n ...audit,\n page,\n cursor: nextCursor,\n hasMore,\n contacts: merged,\n pagesFetched: page,\n partial: audit.partial || partial,\n partialReason: audit.partialReason || (partial ? stopReason : null),\n lastStopReason: stopReason,\n total: meta.total ?? null,\n },\n };\n}\n\n// --- exported for the local test harness; n8n ignores this block -----------\nif (typeof module !== 'undefined' && module.exports) {\n module.exports = { accumulate };\n}\n\n// --- n8n entry point -------------------------------------------------------\nif (typeof $input !== 'undefined') {\n const item = $input.all()[0];\n const json = item.json || {};\n\n // Mock path: the item still carries config + audit + response.\n // Live path: the HTTP node replaced the item with the raw API response, so\n // the loop state comes back from the node that built the request.\n const state = json.audit && json.config\n ? json\n : $('Build Page Request').all()[0].json;\n // Mock: { response }. Live with Full Response on: { statusCode, headers, body }.\n const response = json.response !== undefined ? json.response\n : json.body !== undefined ? json.body : json;\n\n return [{ json: accumulate(state, response) }];\n}" }, "id": "93bad2b4-4670-4d0b-b002-2e56373ac9d7" }, { "name": "More Pages?", "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 400, 350 ], "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "loose", "version": 2 }, "conditions": [ { "id": "has-more", "leftValue": "={{ $json.audit.hasMore }}", "rightValue": "", "operator": { "type": "boolean", "operation": "true", "singleValue": true } } ], "combinator": "and" }, "looseTypeValidation": true, "options": {} }, "id": "a866b3e8-56cf-4679-bb67-7fe3e607fc2a" }, { "name": "Cluster & Score Contacts", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 640, 460 ], "parameters": { "mode": "runOnceForAllItems", "jsCode": "/**\n * NODE: \"Cluster & Score Contacts\" (Code node, Run Once for All Items)\n * --------------------------------------------------------------------\n * The whole audit happens here. Input is every contact the paginator collected;\n * output is one item containing a merge plan, a suppression list, per-contact\n * deliverability scores and a summary.\n *\n * Three things in here are worth reading properly:\n *\n * 1. BLOCKING. Comparing every contact to every other contact is O(n\u00b2) \u2014 on\n * 40,000 contacts that is 800 million comparisons and the execution dies.\n * Instead each contact is filed under a few cheap keys (last 9 phone digits,\n * normalised email, soundex of surname + postcode) and only contacts sharing\n * a key are ever compared. That turns it into roughly O(n).\n *\n * 2. NEGATIVE EVIDENCE. Most dedupe tools only look for reasons to merge. The\n * expensive mistakes come from ignoring reasons not to: two people sharing a\n * family email address, a husband and wife at one company, \"David Smith\" and\n * \"David Smyth\" who are two different dentists at the same practice. Every\n * pair here is scored for conflict as well as similarity, and a conflicted\n * pair is sent for review rather than merged. Merging two real customers is\n * not recoverable; leaving a duplicate is.\n *\n * 3. NOTHING IS ASSERTED THAT HAS NOT BEEN CHECKED. This node does not claim an\n * address is deliverable \u2014 it has not sent anything. It reports syntax,\n * known-bad patterns, and the CRM's own bounce/unsubscribe flags, and marks\n * everything else \"unverified\". A hygiene report that overstates what it\n * knows is worse than no report.\n */\n\n// ---------------------------------------------------------------------------\n// Phone / email normalisation (kept local: an n8n Code node cannot import)\n// ---------------------------------------------------------------------------\nconst COUNTRY_RULES = {\n US: { cc: '1', nsnLengths: [10], trunk: '1' }, CA: { cc: '1', nsnLengths: [10], trunk: '1' },\n GB: { cc: '44', nsnLengths: [10, 9], trunk: '0' }, IE: { cc: '353', nsnLengths: [9, 8], trunk: '0' },\n AU: { cc: '61', nsnLengths: [9], trunk: '0' }, NZ: { cc: '64', nsnLengths: [9, 8], trunk: '0' },\n IN: { cc: '91', nsnLengths: [10], trunk: '0' }, AE: { cc: '971', nsnLengths: [9], trunk: '0' },\n SG: { cc: '65', nsnLengths: [8], trunk: '' }, ZA: { cc: '27', nsnLengths: [9], trunk: '0' },\n DE: { cc: '49', nsnLengths: [10, 11], trunk: '0' }, NG: { cc: '234', nsnLengths: [10], trunk: '0' },\n NL: { cc: '31', nsnLengths: [9], trunk: '0' }, MV: { cc: '960', nsnLengths: [7], trunk: '' },\n};\n\nconst CC_LOOKUP = Object.entries(COUNTRY_RULES)\n .sort((a, b) => b[1].cc.length - a[1].cc.length)\n .map(([iso, rule]) => ({ iso, cc: rule.cc }));\n\nconst DISPOSABLE_DOMAINS = ['mailinator.com', 'guerrillamail.com', '10minutemail.com',\n 'tempmail.com', 'yopmail.com', 'trashmail.com', 'sharklasers.com', 'getnada.com'];\n\nconst ROLE_LOCAL_PARTS = ['info', 'admin', 'support', 'sales', 'contact', 'hello',\n 'office', 'billing', 'noreply', 'no-reply', 'help', 'team', 'enquiries'];\n\nconst TYPO_DOMAINS = { 'gmial.com': 'gmail.com', 'gmai.com': 'gmail.com', 'gmail.co': 'gmail.com',\n 'yaho.com': 'yahoo.com', 'hotmial.com': 'hotmail.com', 'outlok.com': 'outlook.com' };\n\nfunction normalizePhone(raw, defaultCountry) {\n const text = String(raw ?? '').trim();\n if (!text) return { e164: '', valid: false, reason: 'missing' };\n\n const hadPlus = text.startsWith('+');\n let digits = text.replace(/\\D/g, '');\n if (!digits) return { e164: '', valid: false, reason: 'no_digits' };\n\n let international = hadPlus;\n if (!international && digits.startsWith('00')) { digits = digits.slice(2); international = true; }\n\n let cc = '';\n let nsn = '';\n if (international) {\n const match = CC_LOOKUP.find((entry) => digits.startsWith(entry.cc));\n if (!match) return { e164: '+' + digits, valid: digits.length >= 8 && digits.length <= 15, reason: 'unknown_country' };\n cc = match.cc;\n nsn = digits.slice(cc.length);\n } else {\n const rule = COUNTRY_RULES[defaultCountry] || COUNTRY_RULES.US;\n cc = rule.cc;\n nsn = digits;\n if (rule.trunk && nsn.startsWith(rule.trunk) && !rule.nsnLengths.includes(nsn.length)) {\n const stripped = nsn.slice(rule.trunk.length);\n if (rule.nsnLengths.includes(stripped.length)) nsn = stripped;\n }\n }\n\n const iso = (CC_LOOKUP.find((entry) => entry.cc === cc) || {}).iso || defaultCountry;\n const rule = COUNTRY_RULES[iso];\n if (rule && rule.trunk && nsn.startsWith(rule.trunk) &&\n !rule.nsnLengths.includes(nsn.length) &&\n rule.nsnLengths.includes(nsn.length - rule.trunk.length)) {\n nsn = nsn.slice(rule.trunk.length);\n }\n\n const lengthOk = rule ? rule.nsnLengths.includes(nsn.length) : (nsn.length >= 7 && nsn.length <= 12);\n const placeholder = /^(\\d)\\1+$/.test(nsn) ||\n ['1234567890', '0123456789', '5555555555'].includes(nsn);\n const nanpBad = cc === '1' && nsn.length === 10 && (/^[01]/.test(nsn) || /^[01]/.test(nsn.slice(3)));\n\n const valid = lengthOk && !placeholder && !nanpBad;\n return {\n e164: valid ? '+' + cc + nsn : '',\n national: nsn,\n country: iso,\n valid,\n reason: !lengthOk ? 'bad_length' : placeholder ? 'placeholder' : nanpBad ? 'invalid_nanp' : 'ok',\n };\n}\n\nfunction normalizeEmail(raw) {\n const text = String(raw ?? '').trim().toLowerCase().replace(/\\s+/g, '');\n if (!text) return { value: '', key: '', valid: false, reason: 'missing', domain: '', flags: {} };\n\n const parts = text.split('@');\n if (parts.length !== 2 || !parts[0] || !parts[1]) {\n return { value: text, key: '', valid: false, reason: 'unparseable', domain: '', flags: {} };\n }\n\n let [local, domain] = parts;\n const typo = Boolean(TYPO_DOMAINS[domain]);\n\n const valid = /^[a-z0-9!#$%&'*+/=?^_`{|}~.-]+@[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+$/\n .test(text) && !/\\.\\./.test(text);\n\n const bare = local.split('+')[0];\n let keyLocal = bare;\n let keyDomain = domain === 'googlemail.com' ? 'gmail.com' : domain;\n if (keyDomain === 'gmail.com') keyLocal = keyLocal.replace(/\\./g, '');\n\n return {\n value: text,\n key: valid ? keyLocal + '@' + keyDomain : '',\n domain: keyDomain,\n valid,\n reason: valid ? 'ok' : 'bad_syntax',\n flags: {\n role: ROLE_LOCAL_PARTS.includes(bare),\n disposable: DISPOSABLE_DOMAINS.includes(domain),\n typoDomain: typo,\n suggestion: typo ? bare + '@' + TYPO_DOMAINS[domain] : '',\n },\n };\n}\n\n// ---------------------------------------------------------------------------\n// Name handling\n// ---------------------------------------------------------------------------\nconst NICKNAMES = {\n bob: 'robert', rob: 'robert', bobby: 'robert', robbie: 'robert',\n mike: 'michael', mick: 'michael', micky: 'michael',\n jen: 'jennifer', jenny: 'jennifer', jenn: 'jennifer',\n dave: 'david', dan: 'daniel', danny: 'daniel', tom: 'thomas', tommy: 'thomas',\n bill: 'william', will: 'william', billy: 'william', liz: 'elizabeth', beth: 'elizabeth',\n kate: 'katherine', katie: 'katherine', cathy: 'catherine', sue: 'susan',\n chris: 'christopher', steve: 'stephen', jim: 'james', jimmy: 'james',\n tony: 'anthony', nick: 'nicholas', alex: 'alexander', sam: 'samuel',\n matt: 'matthew', greg: 'gregory', pat: 'patrick', ed: 'edward', ted: 'edward',\n};\n\nfunction canonicalName(value) {\n return String(value ?? '').toLowerCase().normalize('NFD')\n .replace(/[\\u0300-\\u036f]/g, '') // strip accents: \"Mal\u00e9\" -> \"male\"\n .replace(/[^a-z\\s'-]/g, ' ')\n .replace(/\\s+/g, ' ')\n .trim();\n}\n\nfunction expandNickname(name) {\n const canonical = canonicalName(name);\n return NICKNAMES[canonical] || canonical;\n}\n\n/** Soundex, used only as a blocking key \u2014 cheap, phonetic, and good enough to\n * put \"Nakamura\" and \"Nakamurra\" in the same bucket. */\nfunction soundex(value) {\n const text = canonicalName(value).replace(/[^a-z]/g, '');\n if (!text) return '';\n const codes = { b: 1, f: 1, p: 1, v: 1, c: 2, g: 2, j: 2, k: 2, q: 2, s: 2, x: 2, z: 2,\n d: 3, t: 3, l: 4, m: 5, n: 5, r: 6 };\n let result = text[0].toUpperCase();\n let previous = codes[text[0]] || 0;\n for (let i = 1; i < text.length && result.length < 4; i++) {\n const code = codes[text[i]] || 0;\n if (code && code !== previous) result += code;\n if (!'hw'.includes(text[i])) previous = code;\n }\n return (result + '000').slice(0, 4);\n}\n\n/** Jaro-Winkler similarity, 0-1. Better than Levenshtein for short human names:\n * it rewards a shared prefix, which is exactly how names get mistyped. */\nfunction jaroWinkler(a, b) {\n const s1 = canonicalName(a);\n const s2 = canonicalName(b);\n if (!s1 || !s2) return 0;\n if (s1 === s2) return 1;\n\n const matchWindow = Math.max(0, Math.floor(Math.max(s1.length, s2.length) / 2) - 1);\n const s1Matched = new Array(s1.length).fill(false);\n const s2Matched = new Array(s2.length).fill(false);\n let matches = 0;\n\n for (let i = 0; i < s1.length; i++) {\n const start = Math.max(0, i - matchWindow);\n const end = Math.min(i + matchWindow + 1, s2.length);\n for (let j = start; j < end; j++) {\n if (s2Matched[j] || s1[i] !== s2[j]) continue;\n s1Matched[i] = true;\n s2Matched[j] = true;\n matches += 1;\n break;\n }\n }\n if (matches === 0) return 0;\n\n let transpositions = 0;\n let k = 0;\n for (let i = 0; i < s1.length; i++) {\n if (!s1Matched[i]) continue;\n while (!s2Matched[k]) k += 1;\n if (s1[i] !== s2[k]) transpositions += 1;\n k += 1;\n }\n transpositions /= 2;\n\n const jaro = (matches / s1.length + matches / s2.length +\n (matches - transpositions) / matches) / 3;\n\n let prefix = 0;\n while (prefix < 4 && prefix < s1.length && prefix < s2.length && s1[prefix] === s2[prefix]) {\n prefix += 1;\n }\n return jaro + prefix * 0.1 * (1 - jaro);\n}\n\n// ---------------------------------------------------------------------------\n// Union-Find, so that A~B and B~C put A, B and C in one cluster\n// ---------------------------------------------------------------------------\nfunction createUnionFind(size) {\n const parent = Array.from({ length: size }, (unused, index) => index);\n const rank = new Array(size).fill(0);\n\n function find(index) {\n while (parent[index] !== index) {\n parent[index] = parent[parent[index]]; // path compression\n index = parent[index];\n }\n return index;\n }\n\n function union(a, b) {\n const rootA = find(a);\n const rootB = find(b);\n if (rootA === rootB) return;\n if (rank[rootA] < rank[rootB]) parent[rootA] = rootB;\n else if (rank[rootA] > rank[rootB]) parent[rootB] = rootA;\n else { parent[rootB] = rootA; rank[rootA] += 1; }\n }\n\n return { find, union };\n}\n\n// ---------------------------------------------------------------------------\n// Per-contact preparation\n// ---------------------------------------------------------------------------\nfunction prepareContact(contact, config, nowMs) {\n const phone = normalizePhone(contact.phone, contact.country || config.defaultCountry);\n const email = normalizeEmail(contact.email);\n\n const first = canonicalName(contact.firstName);\n const last = canonicalName(contact.lastName);\n const firstCanonical = expandNickname(first);\n\n const lastActivityMs = contact.lastActivityAt ? Date.parse(contact.lastActivityAt) : NaN;\n const ageDays = Number.isNaN(lastActivityMs) ? null\n : Math.floor((nowMs - lastActivityMs) / 86400000);\n\n // Completeness: how much of this record is actually filled in. Drives which\n // record survives a merge.\n const fields = [contact.firstName, contact.lastName, contact.email, contact.phone,\n contact.companyName, contact.city, contact.postalCode, contact.country];\n const completeness = fields.filter((value) => String(value ?? '').trim() !== '').length;\n\n return {\n raw: contact,\n id: String(contact.id),\n phone,\n email,\n first,\n firstCanonical,\n last,\n fullName: [first, last].filter(Boolean).join(' '),\n postalCode: String(contact.postalCode ?? '').toUpperCase().replace(/\\s+/g, ''),\n company: canonicalName(contact.companyName),\n completeness,\n ageDays,\n createdMs: Date.parse(contact.createdAt) || 0,\n lastActivityMs: Number.isNaN(lastActivityMs) ? 0 : lastActivityMs,\n };\n}\n\n// ---------------------------------------------------------------------------\n// Pair comparison\n// ---------------------------------------------------------------------------\n/**\n * Returns { verdict, confidence, signals[], conflicts[] }.\n * verdict is one of 'merge' | 'review' | 'separate'.\n */\nfunction comparePair(a, b, config) {\n const signals = [];\n const conflicts = [];\n let confidence = 0;\n\n const bothPhones = a.phone.e164 && b.phone.e164;\n const bothEmails = a.email.key && b.email.key;\n\n // --- positive evidence -------------------------------------------------\n if (bothEmails && a.email.key === b.email.key) {\n confidence = Math.max(confidence, 95);\n signals.push('exact_email');\n }\n if (bothPhones && a.phone.e164 === b.phone.e164) {\n confidence = Math.max(confidence, 90);\n signals.push('exact_phone');\n }\n if (bothPhones && a.phone.e164 !== b.phone.e164 &&\n a.phone.national && a.phone.national === b.phone.national) {\n confidence = Math.max(confidence, 80);\n signals.push('same_national_number');\n }\n\n const nameSimilarity = jaroWinkler(a.fullName, b.fullName);\n const firstMatch = a.firstCanonical && a.firstCanonical === b.firstCanonical;\n const lastSimilarity = jaroWinkler(a.last, b.last);\n\n const secondSignal =\n (a.postalCode && a.postalCode === b.postalCode) ? 'same_postcode'\n : (a.company && a.company === b.company) ? 'same_company'\n : (a.email.domain && a.email.domain === b.email.domain &&\n !['gmail.com', 'yahoo.com', 'hotmail.com', 'outlook.com', 'icloud.com'].includes(a.email.domain))\n ? 'same_email_domain'\n : null;\n\n if (nameSimilarity >= config.nameSimilarityThreshold && secondSignal) {\n confidence = Math.max(confidence, 70);\n signals.push('similar_name');\n signals.push(secondSignal);\n }\n\n if (firstMatch && lastSimilarity >= 0.92 && secondSignal) {\n confidence = Math.max(confidence, 65);\n signals.push('nickname_or_typo_name');\n }\n\n if (confidence === 0) return { verdict: 'separate', confidence: 0, signals, conflicts, nameSimilarity };\n\n // --- negative evidence -------------------------------------------------\n // Two records that each carry a phone number, and the numbers differ, are\n // usually two people. This is the single most valuable check in the file.\n if (bothPhones && a.phone.e164 !== b.phone.e164 && !signals.includes('same_national_number')) {\n conflicts.push('different_phone_numbers');\n }\n if (bothEmails && a.email.key !== b.email.key && bothPhones &&\n a.phone.e164 !== b.phone.e164) {\n conflicts.push('different_email_and_phone');\n }\n // Shared inbox, different humans: \"petrov.family@example.net\".\n if (a.first && b.first && !firstMatch && jaroWinkler(a.first, b.first) < 0.70) {\n conflicts.push('different_first_names');\n }\n if (a.last && b.last && lastSimilarity < 0.80) {\n conflicts.push('different_last_names');\n }\n\n let verdict;\n if (conflicts.length === 0 && confidence >= 90) verdict = 'merge';\n else if (conflicts.length === 0 && confidence >= 65) verdict = 'review';\n else if (conflicts.length && confidence >= 90) verdict = 'review'; // strong match, real doubt\n else verdict = 'separate';\n\n return { verdict, confidence, signals, conflicts, nameSimilarity };\n}\n\n// ---------------------------------------------------------------------------\n// Blocking: only compare contacts that share a cheap key\n// ---------------------------------------------------------------------------\nfunction buildBlocks(prepared) {\n const blocks = new Map();\n\n const add = (key, index) => {\n if (!key) return;\n if (!blocks.has(key)) blocks.set(key, []);\n blocks.get(key).push(index);\n };\n\n prepared.forEach((contact, index) => {\n if (contact.phone.national) add('p:' + contact.phone.national.slice(-9), index);\n if (contact.email.key) add('e:' + contact.email.key, index);\n if (contact.last) add('n:' + soundex(contact.last) + ':' + (contact.postalCode || contact.email.domain || ''), index);\n if (contact.last && contact.firstCanonical) add('f:' + contact.firstCanonical + ':' + soundex(contact.last), index);\n });\n\n // A block containing most of the list is not a block \u2014 it is a full scan with\n // extra steps. Usually caused by thousands of contacts sharing a blank field.\n const MAX_BLOCK = 200;\n const usable = [];\n const oversized = [];\n for (const [key, members] of blocks) {\n if (members.length < 2) continue;\n if (members.length > MAX_BLOCK) { oversized.push({ key, size: members.length }); continue; }\n usable.push(members);\n }\n\n return { blocks: usable, oversized };\n}\n\n// ---------------------------------------------------------------------------\n// Deliverability\n// ---------------------------------------------------------------------------\nfunction scoreDeliverability(contact, config) {\n const issues = [];\n const raw = contact.raw;\n const status = String(raw.emailStatus ?? '').toLowerCase();\n\n if (!contact.email.value) issues.push('EMAIL_MISSING');\n else if (!contact.email.valid) issues.push('EMAIL_INVALID_SYNTAX');\n if (contact.email.flags.role) issues.push('EMAIL_ROLE_ADDRESS');\n if (contact.email.flags.disposable) issues.push('EMAIL_DISPOSABLE');\n if (contact.email.flags.typoDomain) issues.push('EMAIL_LIKELY_TYPO_DOMAIN');\n if (status === 'bounced') issues.push('EMAIL_HARD_BOUNCED');\n if (status === 'unsubscribed') issues.push('EMAIL_UNSUBSCRIBED');\n\n if (!raw.phone) issues.push('PHONE_MISSING');\n else if (!contact.phone.valid) issues.push('PHONE_INVALID_' + contact.phone.reason.toUpperCase());\n if (raw.dnd === true) issues.push('DND_SET');\n\n if (contact.ageDays === null) issues.push('NO_RECORDED_ACTIVITY');\n else if (contact.ageDays > config.staleAfterDays) issues.push('STALE_' + contact.ageDays + 'D');\n\n const mailable = contact.email.valid && !contact.email.flags.disposable &&\n status !== 'bounced' && status !== 'unsubscribed' && raw.dnd !== true;\n const textable = contact.phone.valid && raw.dnd !== true;\n\n return {\n id: contact.id,\n name: [raw.firstName, raw.lastName].filter(Boolean).join(' '),\n email: contact.email.value,\n emailSuggestion: contact.email.flags.suggestion || '',\n phone: contact.phone.e164 || raw.phone || '',\n mailable,\n textable,\n // Deliberately not \"verified\". Nothing here has been sent or looked up over\n // the network, so nothing here can honestly claim an address exists.\n emailVerification: 'unverified_syntax_only',\n reachable: mailable || textable,\n issues,\n };\n}\n\n// ---------------------------------------------------------------------------\n// Main\n// ---------------------------------------------------------------------------\nfunction audit(contacts, config, nowMs) {\n const prepared = contacts.map((contact) => prepareContact(contact, config, nowMs));\n const { blocks, oversized } = buildBlocks(prepared);\n\n const unionFind = createUnionFind(prepared.length);\n const pairs = [];\n const reviewPairs = [];\n const comparedKeys = new Set();\n let comparisons = 0;\n\n for (const members of blocks) {\n for (let i = 0; i < members.length; i++) {\n for (let j = i + 1; j < members.length; j++) {\n const a = members[i];\n const b = members[j];\n const key = a < b ? a + ':' + b : b + ':' + a;\n if (comparedKeys.has(key)) continue; // blocks overlap; compare once\n comparedKeys.add(key);\n comparisons += 1;\n\n const result = comparePair(prepared[a], prepared[b], config);\n if (result.verdict === 'merge') {\n unionFind.union(a, b);\n pairs.push({ a: prepared[a].id, b: prepared[b].id, ...result });\n } else if (result.verdict === 'review') {\n reviewPairs.push({ a: prepared[a].id, b: prepared[b].id, ...result });\n }\n }\n }\n }\n\n // --- group into clusters -----------------------------------------------\n const clusterMap = new Map();\n prepared.forEach((contact, index) => {\n const root = unionFind.find(index);\n if (!clusterMap.has(root)) clusterMap.set(root, []);\n clusterMap.get(root).push(contact);\n });\n\n const mergePlan = [];\n for (const members of clusterMap.values()) {\n if (members.length < 2) continue;\n\n // Survivor: most complete record, then most recently active, then oldest\n // (keeping the original id preserves the account's own history), then id\n // order so the plan is byte-identical on every run.\n const ordered = members.slice().sort((a, b) =>\n b.completeness - a.completeness ||\n b.lastActivityMs - a.lastActivityMs ||\n a.createdMs - b.createdMs ||\n a.id.localeCompare(b.id));\n\n const survivor = ordered[0];\n const losers = ordered.slice(1);\n\n // What the survivor actually gains. If the answer is \"nothing\", the merge is\n // still worth doing \u2014 but the client should see that it is a pure cleanup.\n const gains = {};\n for (const field of ['firstName', 'lastName', 'email', 'phone', 'companyName',\n 'city', 'postalCode', 'country']) {\n if (String(survivor.raw[field] ?? '').trim() !== '') continue;\n const donor = losers.find((loser) => String(loser.raw[field] ?? '').trim() !== '');\n if (donor) gains[field] = donor.raw[field];\n }\n\n const tags = Array.from(new Set(members.flatMap((member) => member.raw.tags || [])));\n const extraEmails = Array.from(new Set(members.map((member) => member.email.value)\n .filter((value) => value && value !== survivor.email.value)));\n const extraPhones = Array.from(new Set(members.map((member) => member.phone.e164)\n .filter((value) => value && value !== survivor.phone.e164)));\n\n const clusterPairs = pairs.filter((pair) =>\n members.some((member) => member.id === pair.a) &&\n members.some((member) => member.id === pair.b));\n\n mergePlan.push({\n clusterId: 'cl_' + survivor.id,\n survivorId: survivor.id,\n survivorName: [survivor.raw.firstName, survivor.raw.lastName].filter(Boolean).join(' '),\n mergeIds: losers.map((loser) => loser.id),\n size: members.length,\n confidence: Math.min(...clusterPairs.map((pair) => pair.confidence)),\n signals: Array.from(new Set(clusterPairs.flatMap((pair) => pair.signals))),\n fieldsGained: gains,\n unionTags: tags,\n secondaryEmails: extraEmails,\n secondaryPhones: extraPhones,\n });\n }\n\n mergePlan.sort((a, b) => a.survivorId.localeCompare(b.survivorId));\n\n // --- deliverability + suppression --------------------------------------\n const scores = prepared.map((contact) => scoreDeliverability(contact, config));\n const mergedAwayIds = new Set(mergePlan.flatMap((plan) => plan.mergeIds));\n\n const suppression = scores\n .filter((score) => !score.reachable || score.issues.some((issue) =>\n ['EMAIL_HARD_BOUNCED', 'EMAIL_UNSUBSCRIBED', 'DND_SET', 'EMAIL_DISPOSABLE'].includes(issue)))\n .map((score) => ({\n id: score.id,\n name: score.name,\n email: score.email,\n phone: score.phone,\n reason: score.issues[0] || 'UNREACHABLE',\n allIssues: score.issues,\n }));\n\n const summary = {\n contactsAudited: prepared.length,\n comparisonsRun: comparisons,\n comparisonsAvoided: (prepared.length * (prepared.length - 1)) / 2 - comparisons,\n duplicateClusters: mergePlan.length,\n contactsToMerge: mergedAwayIds.size,\n pairsNeedingHumanReview: reviewPairs.length,\n mailable: scores.filter((score) => score.mailable).length,\n textable: scores.filter((score) => score.textable).length,\n unreachable: scores.filter((score) => !score.reachable).length,\n onSuppressionList: suppression.length,\n staleOverThreshold: scores.filter((score) =>\n score.issues.some((issue) => issue.startsWith('STALE_'))).length,\n typoDomainsFound: scores.filter((score) => score.emailSuggestion).length,\n oversizedBlocksSkipped: oversized.length,\n };\n\n return { summary, mergePlan, reviewPairs, suppression, contactScores: scores, oversized };\n}\n\n// --- exported for the local test harness; n8n ignores this block -----------\nif (typeof module !== 'undefined' && module.exports) {\n module.exports = {\n normalizePhone, normalizeEmail, canonicalName, expandNickname, soundex,\n jaroWinkler, createUnionFind, prepareContact, comparePair, buildBlocks,\n scoreDeliverability, audit,\n };\n}\n\n// --- n8n entry point -------------------------------------------------------\nif (typeof $input !== 'undefined') {\n const state = $input.all()[0].json || {};\n // This node has two upstreams: the finished pagination loop, and the partial\n // -audit recovery path. Neither is allowed to crash it \u2014 an audit of zero\n // contacts that says \"the fetch failed\" is a useful answer; a TypeError is not.\n const auditState = state.audit || {};\n const config = state.config || {\n defaultCountry: 'US', staleAfterDays: 730,\n nameSimilarityThreshold: 0.90, dryRun: true,\n };\n const collected = Array.isArray(auditState.contacts) ? auditState.contacts : [];\n\n const result = audit(collected, config, Date.now());\n\n return [{\n json: {\n config,\n runId: auditState.runId || null,\n startedAt: auditState.startedAt || null,\n finishedAt: new Date().toISOString(),\n pagesFetched: auditState.pagesFetched || 0,\n partial: auditState.partial === true,\n partialReason: auditState.partialReason || null,\n fetchFailure: state.fetchFailure || null,\n ...result,\n },\n }];\n}" }, "id": "5159f85b-e6dd-42f6-8797-209a33ff492e" }, { "name": "Build Reports", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 880, 580 ], "parameters": { "mode": "runOnceForAllItems", "jsCode": "/**\n * NODE: \"Build Reports\" (Code node, Run Once for All Items)\n * ---------------------------------------------------------\n * Turns the audit result into files a human can act on, attached to the item as\n * real binary data \u2014 no Convert to File node needed, and no dependency on which\n * spreadsheet operations a given n8n build happens to ship with.\n *\n * merge-plan.csv one row per duplicate cluster: keep this id, merge these\n * suppression.csv contacts to stop mailing or texting, with the reason\n * needs-review.csv pairs that look like duplicates but have real conflicts\n *\n * The CSV writer is 15 lines and handles the three things that break naive\n * string concatenation: embedded commas, embedded quotes, and the leading\n * =/+/-/@ that turns a cell into a formula when the client opens it in Excel.\n */\n\n/** RFC 4180 quoting, plus formula-injection defusing. */\nfunction csvCell(value) {\n if (value === null || value === undefined) return '';\n let text = Array.isArray(value) ? value.join('; ')\n : typeof value === 'object' ? JSON.stringify(value)\n : String(value);\n\n // A cell starting with = + - @ is executed as a formula by Excel and Sheets.\n // Prefixing a single quote is the standard defusal and is invisible to a user.\n if (/^[=+\\-@\\t\\r]/.test(text)) text = \"'\" + text;\n\n if (/[\",\\n\\r]/.test(text)) text = '\"' + text.replace(/\"/g, '\"\"') + '\"';\n return text;\n}\n\nfunction toCsv(rows, columns) {\n const header = columns.map((column) => csvCell(column.label)).join(',');\n const body = rows.map((row) =>\n columns.map((column) => csvCell(\n typeof column.value === 'function' ? column.value(row) : row[column.key]\n )).join(',')\n );\n // CRLF, because Excel on Windows still treats a lone LF as one giant row.\n return [header, ...body].join('\\r\\n') + '\\r\\n';\n}\n\nfunction toBinary(csv, fileName) {\n const buffer = Buffer.from(csv, 'utf8');\n return {\n // BOM-free UTF-8. If the client's Excel mangles accented names, prepend\n // a UTF-8 BOM (\\ufeff) to the csv string \u2014 that is the only reliable fix on Windows.\n data: buffer.toString('base64'),\n mimeType: 'text/csv',\n fileName,\n fileExtension: 'csv',\n fileSize: buffer.length + ' B',\n };\n}\n\nfunction buildReports(audit) {\n const mergeCsv = toCsv(audit.mergePlan, [\n { label: 'cluster_id', key: 'clusterId' },\n { label: 'keep_contact_id', key: 'survivorId' },\n { label: 'keep_name', key: 'survivorName' },\n { label: 'merge_contact_ids', key: 'mergeIds' },\n { label: 'records_in_cluster', key: 'size' },\n { label: 'confidence', key: 'confidence' },\n { label: 'matched_on', key: 'signals' },\n { label: 'fields_survivor_gains', value: (row) => Object.keys(row.fieldsGained).join('; ') },\n { label: 'secondary_emails', key: 'secondaryEmails' },\n { label: 'secondary_phones', key: 'secondaryPhones' },\n { label: 'tags_after_merge', key: 'unionTags' },\n ]);\n\n const suppressionCsv = toCsv(audit.suppression, [\n { label: 'contact_id', key: 'id' },\n { label: 'name', key: 'name' },\n { label: 'email', key: 'email' },\n { label: 'phone', key: 'phone' },\n { label: 'primary_reason', key: 'reason' },\n { label: 'all_issues', key: 'allIssues' },\n ]);\n\n const reviewCsv = toCsv(audit.reviewPairs, [\n { label: 'contact_a', key: 'a' },\n { label: 'contact_b', key: 'b' },\n { label: 'confidence', key: 'confidence' },\n { label: 'matched_on', key: 'signals' },\n { label: 'conflicts', key: 'conflicts' },\n { label: 'name_similarity', value: (row) => row.nameSimilarity.toFixed(3) },\n ]);\n\n const fixesCsv = toCsv(\n audit.contactScores.filter((score) => score.emailSuggestion),\n [\n { label: 'contact_id', key: 'id' },\n { label: 'name', key: 'name' },\n { label: 'current_email', key: 'email' },\n { label: 'suggested_email', key: 'emailSuggestion' },\n ]\n );\n\n return { mergeCsv, suppressionCsv, reviewCsv, fixesCsv };\n}\n\n/** The bit an agency owner actually reads. Plain sentences, no jargon. */\nfunction plainEnglishSummary(audit) {\n const s = audit.summary;\n const lines = [];\n\n lines.push(s.contactsAudited + ' contacts audited across ' + audit.pagesFetched + ' page(s).');\n\n if (s.duplicateClusters === 0) {\n lines.push('No confident duplicates found.');\n } else {\n lines.push(s.duplicateClusters + ' duplicate group(s) found, covering '\n + (s.duplicateClusters + s.contactsToMerge) + ' records. Merging them removes '\n + s.contactsToMerge + ' contact(s).');\n }\n\n if (s.pairsNeedingHumanReview > 0) {\n lines.push(s.pairsNeedingHumanReview + ' pair(s) look similar but have conflicting details '\n + '(different phone numbers, different first names on a shared email). These are NOT in the '\n + 'merge plan \u2014 a person should look at them. Merging two real customers cannot be undone.');\n }\n\n lines.push(s.mailable + ' contact(s) are mailable and ' + s.textable + ' are textable. '\n + s.unreachable + ' have no usable channel at all.');\n\n if (s.onSuppressionList > 0) {\n lines.push(s.onSuppressionList + ' contact(s) should be suppressed: hard bounces, unsubscribes, '\n + 'do-not-disturb flags and disposable addresses. Continuing to send to these is what damages '\n + 'the sending reputation for everyone else on the list.');\n }\n if (s.typoDomainsFound > 0) {\n lines.push(s.typoDomainsFound + ' email(s) have an obvious typo in the domain (gmial.com and '\n + 'friends). Corrections are in email-fixes.csv \u2014 these are recoverable contacts.');\n }\n if (s.staleOverThreshold > 0) {\n lines.push(s.staleOverThreshold + ' contact(s) have had no activity beyond the staleness '\n + 'threshold. Worth a re-engagement campaign before they are archived.');\n }\n\n lines.push('Duplicate detection ran ' + s.comparisonsRun + ' comparisons instead of '\n + (s.comparisonsRun + s.comparisonsAvoided) + ' \u2014 blocking skipped '\n + s.comparisonsAvoided + ' pairs that could not possibly match.');\n\n lines.push(audit.config.dryRun\n ? 'DRY RUN: nothing was written to the CRM. Review the merge plan, then set dryRun to false.'\n : 'LIVE RUN: the merge plan was applied to the CRM.');\n\n if (audit.partial) {\n lines.push('WARNING: this audit is incomplete (' + audit.partialReason\n + '). Treat the numbers as a floor, not a total.');\n }\n\n lines.push('Email addresses were checked for syntax and known-bad patterns only. '\n + 'Nothing was sent and no mail server was contacted, so \"mailable\" means '\n + '\"not obviously broken\", not \"verified\".');\n\n return lines;\n}\n\n// --- exported for the local test harness; n8n ignores this block -----------\nif (typeof module !== 'undefined' && module.exports) {\n module.exports = { csvCell, toCsv, toBinary, buildReports, plainEnglishSummary };\n}\n\n// --- n8n entry point -------------------------------------------------------\nif (typeof $input !== 'undefined') {\n const audit = $input.all()[0].json;\n const reports = buildReports(audit);\n\n return [{\n json: {\n runId: audit.runId,\n finishedAt: audit.finishedAt,\n dryRun: audit.config.dryRun === true,\n partial: audit.partial === true,\n summary: audit.summary,\n readThis: plainEnglishSummary(audit),\n mergePlan: audit.mergePlan,\n needsReview: audit.reviewPairs,\n files: ['merge-plan.csv', 'suppression.csv', 'needs-review.csv', 'email-fixes.csv'],\n },\n binary: {\n merge_plan: toBinary(reports.mergeCsv, 'merge-plan.csv'),\n suppression: toBinary(reports.suppressionCsv, 'suppression.csv'),\n needs_review: toBinary(reports.reviewCsv, 'needs-review.csv'),\n email_fixes: toBinary(reports.fixesCsv, 'email-fixes.csv'),\n },\n }];\n}" }, "id": "e9554131-bfc2-4073-9aad-bca871c030d0" }, { "name": "Apply the Merges?", "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 880, 340 ], "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "loose", "version": 2 }, "conditions": [ { "id": "not-dry-run", "leftValue": "={{ $json.config.dryRun !== true && $json.mergePlan.length > 0 }}", "rightValue": "", "operator": { "type": "boolean", "operation": "true", "singleValue": true } } ], "combinator": "and" }, "looseTypeValidation": true, "options": {} }, "id": "d66cb622-3fec-4a63-8187-7c38ff770cab" }, { "name": "Dry Run \u2014 Nothing Written", "type": "n8n-nodes-base.noOp", "typeVersion": 1, "position": [ 1120, 440 ], "parameters": {}, "id": "88a9cbec-67bc-44f6-ad76-8fbab5b7d855" }, { "name": "Prepare Merge Batches", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 1120, 220 ], "parameters": { "mode": "runOnceForAllItems", "jsCode": "/**\n * NODE: \"Prepare Merge Batches\" (Code node, Run Once for All Items)\n * -----------------------------------------------------------------\n * Only runs when dryRun is off. Turns the merge plan into one item per API call,\n * with three protections that matter when you are about to modify a client's\n * production CRM:\n *\n * 1. A CEILING. maxMergesPerRun caps how much damage a bad plan can do in one\n * execution. If the plan is bigger, it is truncated and the remainder is\n * reported, not silently dropped.\n * 2. A THROTTLE. Each item carries the delay the workflow should wait before\n * issuing it, derived from the API's documented rate limit, so the merge\n * job does not become the reason the client's other automations start\n * getting 429s.\n * 3. A CONFIDENCE FLOOR. Anything below minMergeConfidence never gets issued,\n * even if it made it into the plan.\n */\n\nconst SAFETY = {\n maxMergesPerRun: 100,\n minMergeConfidence: 90,\n requestsPerSecond: 5, // GoHighLevel v2 burst limit at time of writing\n};\n\nfunction prepare(audit) {\n const plan = (audit.mergePlan || []).filter(\n (entry) => entry.confidence >= SAFETY.minMergeConfidence\n );\n\n const held = (audit.mergePlan || []).length - plan.length;\n const capped = plan.slice(0, SAFETY.maxMergesPerRun);\n const deferred = plan.length - capped.length;\n\n const gapMs = Math.ceil(1000 / SAFETY.requestsPerSecond);\n\n return capped.map((entry, index) => ({\n runId: audit.runId,\n clusterId: entry.clusterId,\n survivorId: entry.survivorId,\n mergeIds: entry.mergeIds,\n confidence: entry.confidence,\n // The body shape depends on the CRM. This is the common one: keep the\n // survivor, fold the others in, and carry across the union of tags.\n payload: {\n primaryContactId: entry.survivorId,\n mergeContactIds: entry.mergeIds,\n tags: entry.unionTags,\n fields: entry.fieldsGained,\n },\n // Advisory schedule. The workflow enforces the actual pacing with the HTTP\n // node's Batching option (5 requests, 1s apart); this is here so the plan is\n // self-describing when it is exported or handed to a different runner.\n throttle: {\n index,\n delayMs: index * gapMs,\n requestsPerSecond: SAFETY.requestsPerSecond,\n },\n batchInfo: {\n issued: capped.length,\n heldBelowConfidence: held,\n deferredToNextRun: deferred,\n },\n }));\n}\n\n// --- exported for the local test harness; n8n ignores this block -----------\nif (typeof module !== 'undefined' && module.exports) {\n module.exports = { SAFETY, prepare };\n}\n\n// --- n8n entry point -------------------------------------------------------\nif (typeof $input !== 'undefined') {\n const audit = $input.all()[0].json;\n const batches = prepare(audit);\n\n if (batches.length === 0) {\n return [{ json: { runId: audit.runId, nothingToMerge: true,\n reason: 'No cluster met the minimum merge confidence of ' + SAFETY.minMergeConfidence + '.' } }];\n }\n return batches.map((batch) => ({ json: batch }));\n}" }, "id": "4d4c595b-6e9c-4605-841e-5f2b56de1c61" }, { "name": "CRM: Merge Contacts", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 1340, 220 ], "parameters": { "method": "POST", "url": "https://services.leadconnectorhq.com/contacts/merge", "authentication": "genericCredentialType", "genericAuthType": "httpHeaderAuth", "sendHeaders": true, "specifyHeaders": "keypair", "headerParameters": { "parameters": [ { "name": "Version", "value": "2021-07-28" }, { "name": "Accept", "value": "application/json" } ] }, "sendBody": true, "contentType": "json", "specifyBody": "json", "jsonBody": "={{ JSON.stringify($json.payload) }}", "options": { "timeout": 30000, "batching": { "batch": { "batchSize": 5, "batchInterval": 1000 } }, "response": { "response": { "fullResponse": true, "neverError": true } } } }, "id": "9e692115-9da9-4da3-b56b-c0bf41a33f3b" }, { "name": "Record Merge Results", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 1580, 220 ], "parameters": { "mode": "runOnceForAllItems", "jsCode": "/**\n * NODE: \"Record Merge Results\" (Code node, Run Once for All Items)\n * ----------------------------------------------------------------\n * The merge HTTP node is set to continue on error rather than stop, so this node\n * receives both successes and failures. It produces the one artefact you want\n * after a destructive bulk operation: a per-cluster ledger of what actually\n * happened, so a partial failure can be resumed instead of re-run from scratch.\n *\n * Re-running a merge job blindly is how you turn one bad afternoon into two.\n */\n\nfunction summariseResult(json) {\n const failed = Boolean(json.error) || json.success === false ||\n (json.statusCode !== undefined && Number(json.statusCode) >= 400);\n\n return {\n clusterId: json.clusterId || null,\n survivorId: json.survivorId || (json.contact && json.contact.id) || null,\n mergedIds: json.mergeIds || [],\n status: failed ? 'failed' : 'merged',\n detail: failed\n ? String((json.error && (json.error.message || json.error)) || json.message || 'Unknown error').slice(0, 300)\n : 'ok',\n };\n}\n\nfunction processAll(items, nowIso) {\n const results = items.map((item) => summariseResult(item.json || {}));\n\n const merged = results.filter((result) => result.status === 'merged');\n const failed = results.filter((result) => result.status === 'failed');\n\n return [{\n json: {\n completedAt: nowIso,\n attempted: results.length,\n merged: merged.length,\n failed: failed.length,\n // The exact clusters to retry. Feed this back in rather than re-running\n // the whole audit \u2014 the second run would see a CRM that has already\n // changed underneath it.\n retryClusters: failed.map((result) => result.clusterId).filter(Boolean),\n ledger: results,\n },\n }];\n}\n\n// --- exported for the local test harness; n8n ignores this block -----------\nif (typeof module !== 'undefined' && module.exports) {\n module.exports = { summariseResult, processAll };\n}\n\n// --- n8n entry point -------------------------------------------------------\nif (typeof $input !== 'undefined') {\n // The merge HTTP node replaced each item with the API response, which does not\n // contain the cluster id. Pair each result back to the batch that produced it,\n // otherwise `retryClusters` is always empty and the \"resumable\" claim is a lie.\n const incoming = $input.all().map((item, index) => {\n const json = item.json || {};\n if (json.clusterId) return item;\n try {\n const batch = $('Prepare Merge Batches').itemMatching(index).json;\n return { json: { ...batch, ...json } };\n } catch (error) {\n return { json: { ...json, pairingError: error.message } };\n }\n });\n return processAll(incoming, new Date().toISOString());\n}" }, "id": "eb3404e4-6230-42a5-9d38-95dc0056a806" }, { "name": "README", "type": "n8n-nodes-base.stickyNote", "typeVersion": 1, "position": [ -1200, -80 ], "parameters": { "width": 480, "height": 300, "color": 4, "content": "## CRM List Hygiene Audit\n\nWalks the whole contact list, finds real duplicates, scores every record for deliverability, and produces a merge plan as a CSV.\n\n**Dry run by default.** `dryRun: true` in **Init Audit Run** means it writes nothing \u2014 it produces a plan for a human to approve. `mockMode: true` means it needs no credentials at all: hit **Test workflow** and it runs against 24 built-in fixture contacts.\n\nThe interesting node is **Cluster & Score Contacts**. Open it." }, "id": "26f37130-35fa-46ff-b1bd-2edcd4a5b4b8" }, { "name": "Pagination note", "type": "n8n-nodes-base.stickyNote", "typeVersion": 1, "position": [ -760, -260 ], "parameters": { "width": 420, "height": 220, "color": 5, "content": "## The loop\n\n**Build Page Request** is the loop head, and it exists because an HTTP node's error output does not carry the item that went in. Anything downstream of a retry has to be able to rebuild the loop state \u2014 this does it in one place.\n\nFour stop conditions: no cursor, empty page, page ceiling, contact ceiling. Plus a stalled-cursor check, which is how paginators actually fail in production." }, "id": "3b37e005-8c58-4798-a20c-ec8c1afbe403" }, { "name": "Safety note", "type": "n8n-nodes-base.stickyNote", "typeVersion": 1, "position": [ 900, -80 ], "parameters": { "width": 420, "height": 240, "color": 3, "content": "## Before it writes anything\n\nThree gates between a plan and a destructive bulk operation:\n\n1. `dryRun` must be explicitly turned off\n2. confidence floor of 90 \u2014 anything below never gets issued\n3. a hard ceiling of 100 merges per run, throttled to 5 req/s\n\nPairs with conflicting evidence (different phones, different first names on a shared email) go to **needs-review.csv**, never to the merge plan. Merging two real customers is not reversible." }, "id": "0a208b68-aadd-4339-9f1a-e047140a8237" }, { "name": "Page Fetched OK?", "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ -40, 220 ], "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "loose", "version": 2 }, "conditions": [ { "id": "page-fetched-ok", "leftValue": "={{ $json.statusCode >= 200 && $json.statusCode < 300 }}", "rightValue": "", "operator": { "type": "boolean", "operation": "true", "singleValue": true } } ], "combinator": "and" }, "looseTypeValidation": true, "options": {} }, "id": "30d32fbb-c710-4a3d-a903-428e6331c608" }, { "name": "Report Partial Audit", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 400, 20 ], "parameters": { "mode": "runOnceForAllItems", "jsCode": "/**\n * NODE: \"Report Partial Audit\" (Code node, Run Once for All Items)\n * ----------------------------------------------------------------\n * Runs when the paginated fetch failed in a way that is not worth retrying \u2014 an\n * expired token, a bad location id, or a retry budget that ran out.\n *\n * The wrong thing to do here is throw. A crashed execution tells the operator\n * \"something broke\" and nothing else. This rebuilds whatever pages did come back,\n * stamps the run as partial with the reason, and hands it to the clustering step\n * so the operator still gets an audit \u2014 clearly labelled as incomplete \u2014 plus a\n * plain sentence explaining what to fix.\n *\n * A partial audit that says so is useful. A silent one is dangerous, because the\n * merge plan would look like it covered the whole list.\n */\n\nfunction recover(failure, state) {\n const audit = (state && state.audit) || {\n runId: 'audit_recovered', startedAt: new Date().toISOString(),\n page: 0, cursor: null, contacts: [], pagesFetched: 0,\n };\n\n const category = (failure && failure.category) || 'unknown';\n const advice = {\n auth: 'The CRM rejected the credentials. Reconnect the Header Auth credential on '\n + '\"CRM: Fetch Contact Page\". Nothing will sync until this is fixed.',\n not_found: 'The endpoint or location id was not found. Check locationId in \"Init Audit Run\".',\n validation: 'The CRM rejected the request itself. Check the query parameters on the fetch node.',\n rate_limit: 'Rate limited for longer than the retry budget allows. Re-run later, or lower pageSize.',\n server: 'The CRM was returning server errors for the whole retry budget. Re-run later.',\n network: 'The request never reached the CRM. Check outbound network access from this n8n instance.',\n }[category] || 'Unclassified failure. See failure.operatorMessage.';\n\n return {\n config: state && state.config ? state.config : null,\n audit: {\n ...audit,\n hasMore: false,\n partial: true,\n partialReason: 'FETCH_FAILED_' + String(category).toUpperCase(),\n lastStopReason: 'fetch_failed',\n },\n fetchFailure: {\n category,\n status: failure ? failure.status : null,\n message: failure ? failure.operatorMessage : 'No failure detail available.',\n whatToDo: advice,\n },\n };\n}\n\n// --- exported for the local test harness; n8n ignores this block -----------\nif (typeof module !== 'undefined' && module.exports) {\n module.exports = { recover };\n}\n\n// --- n8n entry point -------------------------------------------------------\nif (typeof $input !== 'undefined') {\n const json = $input.all()[0].json || {};\n\n // Whatever pages did come back are held by the accumulator's last run.\n let state = json.config && json.audit ? json : null;\n if (!state) {\n for (const nodeName of ['Accumulate Page', 'Init Audit Run']) {\n try {\n const runs = $(nodeName).all();\n const candidate = runs[runs.length - 1].json;\n if (candidate && candidate.config && candidate.audit) { state = candidate; break; }\n } catch (error) {\n // That node did not run on this path \u2014 expected when page 1 failed.\n }\n }\n }\n\n return [{ json: recover(json.failure, state) }];\n}" }, "id": "99b38bd6-3fbe-4b01-8f0a-9165a274fc60" } ], "connections": { "Run Audit Now": { "main": [ [ { "node": "Init Audit Run", "type": "main", "index": 0 } ] ] }, "Weekly Audit": { "main": [ [ { "node": "Init Audit Run", "type": "main", "index": 0 } ] ] }, "Init Audit Run": { "main": [ [ { "node": "Build Page Request", "type": "main", "index": 0 } ] ] }, "Build Page Request": { "main": [ [ { "node": "Use Live CRM?", "type": "main", "index": 0 } ] ] }, "Use Live CRM?": { "main": [ [ { "node": "CRM: Fetch Contact Page", "type": "main", "index": 0 } ], [ { "node": "Mock Contact Page", "type": "main", "index": 0 } ] ] }, "CRM: Fetch Contact Page": { "main": [ [ { "node": "Page Fetched OK?", "type": "main", "index": 0 } ] ] }, "Mock Contact Page": { "main": [ [ { "node": "Accumulate Page", "type": "main", "index": 0 } ] ] }, "Classify Fetch Error": { "main": [ [ { "node": "Retry Fetch?", "type": "main", "index": 0 } ] ] }, "Retry Fetch?": { "main": [ [ { "node": "Backoff Wait", "type": "main", "index": 0 } ], [ { "node": "Report Partial Audit", "type": "main", "index": 0 } ] ] }, "Backoff Wait": { "main": [ [ { "node": "Build Page Request", "type": "main", "index": 0 } ] ] }, "Accumulate Page": { "main": [ [ { "node": "More Pages?", "type": "main", "index": 0 } ] ] }, "More Pages?": { "main": [ [ { "node": "Build Page Request", "type": "main", "index": 0 } ], [ { "node": "Cluster & Score Contacts", "type": "main", "index": 0 } ] ] }, "Cluster & Score Contacts": { "main": [ [ { "node": "Build Reports", "type": "main", "index": 0 }, { "node": "Apply the Merges?", "type": "main", "index": 0 } ] ] }, "Apply the Merges?": { "main": [ [ { "node": "Prepare Merge Batches", "type": "main", "index": 0 } ], [ { "node": "Dry Run \u2014 Nothing Written", "type": "main", "index": 0 } ] ] }, "Prepare Merge Batches": { "main": [ [ { "node": "CRM: Merge Contacts", "type": "main", "index": 0 } ] ] }, "CRM: Merge Contacts": { "main": [ [ { "node": "Record Merge Results", "type": "main", "index": 0 } ] ] }, "Page Fetched OK?": { "main": [ [ { "node": "Accumulate Page", "type": "main", "index": 0 } ], [ { "node": "Classify Fetch Error", "type": "main", "index": 0 } ] ] }, "Report Partial Audit": { "main": [ [ { "node": "Cluster & Score Contacts", "type": "main", "index": 0 } ] ] } } }