{ "id": "rW0wVKb45UbwYE9K", "meta": { "templateCredsSetupCompleted": true }, "name": "Automated Data Quality Monitoring & Reporting (SQL + Email + Google Sheets)", "tags": [], "nodes": [ { "id": "52fa31e9-1811-4999-8254-4516e56610e7", "name": "Schedule Trigger", "type": "n8n-nodes-base.scheduleTrigger", "position": [ 112, 400 ], "parameters": { "rule": { "interval": [ { "field": "hours", "hoursInterval": 24 } ] } }, "typeVersion": 1.2 }, { "id": "8db194eb-88ae-4ff5-bee9-46da8c912288", "name": "Config", "type": "n8n-nodes-base.set", "position": [ 336, 400 ], "parameters": { "options": {}, "assignments": { "assignments": [ { "id": "cfg-01", "name": "tableName", "type": "string", "value": "restaurant_orders" }, { "id": "cfg-02", "name": "nullCheckCol", "type": "string", "value": "restaurant_orders" }, { "id": "cfg-03", "name": "duplicateCheckCol", "type": "string", "value": "order_id" }, { "id": "cfg-04", "name": "outlierCol", "type": "string", "value": "order_id" }, { "id": "cfg-05", "name": "outlierMin", "type": "number", "value": 0 }, { "id": "cfg-06", "name": "outlierMax", "type": "number", "value": 150 }, { "id": "cfg-07", "name": "rowCountMin", "type": "number", "value": 500 }, { "id": "cfg-08", "name": "rowCountMax", "type": "number", "value": 100000 }, { "id": "cfg-09", "name": "nullThresholdPct", "type": "number", "value": 5 }, { "id": "cfg-10", "name": "dupThresholdPct", "type": "number", "value": 1 }, { "id": "25a47144-53ab-4923-ae70-78d297745dd9", "name": "Distribution list", "type": "string", "value": "user@example.com" } ] } }, "typeVersion": 3.4 }, { "id": "d809a9c0-0dd8-45b8-be46-91132ecb234c", "name": "Null Check", "type": "n8n-nodes-base.postgres", "position": [ 560, 112 ], "parameters": { "query": "SELECT 'null_check' AS check_type, COUNT(*) AS total_rows, SUM(CASE WHEN {{ $('Config').item.json.nullCheckCol }} IS NULL OR CAST({{ $('Config').item.json.nullCheckCol }} AS TEXT) = '' THEN 1 ELSE 0 END) AS null_count, ROUND(SUM(CASE WHEN {{ $('Config').item.json.nullCheckCol }} IS NULL OR CAST({{ $('Config').item.json.nullCheckCol }} AS TEXT) = '' THEN 1 ELSE 0 END) * 100.0 / NULLIF(COUNT(*), 0), 2) AS null_pct FROM {{ $('Config').item.json.tableName }}", "options": {}, "operation": "executeQuery" }, "credentials": { "postgres": { "name": "" } }, "typeVersion": 2.5 }, { "id": "bbf08a47-ce4b-49fa-8557-1227f6b30dd6", "name": "Duplicate Check", "type": "n8n-nodes-base.postgres", "position": [ 560, 304 ], "parameters": { "query": "SELECT 'dup_check' AS check_type, COUNT(*) AS total_rows, COUNT(*) - COUNT(DISTINCT {{ $('Config').item.json.duplicateCheckCol }}) AS dup_count, ROUND((COUNT(*) - COUNT(DISTINCT {{ $('Config').item.json.duplicateCheckCol }})) * 100.0 / NULLIF(COUNT(*), 0), 2) AS dup_pct FROM {{ $('Config').item.json.tableName }}", "options": {}, "operation": "executeQuery" }, "credentials": { "postgres": { "name": "" } }, "typeVersion": 2.5 }, { "id": "4ec88710-8d0b-480a-8239-ed25e0d118c6", "name": "Row Count", "type": "n8n-nodes-base.postgres", "position": [ 560, 496 ], "parameters": { "query": "SELECT 'row_count' AS check_type, COUNT(*) AS row_count FROM {{ $('Config').item.json.tableName }}", "options": {}, "operation": "executeQuery" }, "credentials": { "postgres": { "name": "" } }, "typeVersion": 2.5 }, { "id": "f3d04639-05a9-4b69-ba7c-87ac6d9fafc2", "name": "Outlier Check", "type": "n8n-nodes-base.postgres", "position": [ 560, 688 ], "parameters": { "query": "SELECT 'outlier_check' AS check_type, COUNT(*) AS total_rows, SUM(CASE WHEN {{ $('Config').item.json.outlierCol }} < {{ $('Config').item.json.outlierMin }} OR {{ $('Config').item.json.outlierCol }} > {{ $('Config').item.json.outlierMax }} THEN 1 ELSE 0 END) AS outlier_count, ROUND(SUM(CASE WHEN {{ $('Config').item.json.outlierCol }} < {{ $('Config').item.json.outlierMin }} OR {{ $('Config').item.json.outlierCol }} > {{ $('Config').item.json.outlierMax }} THEN 1 ELSE 0 END) * 100.0 / NULLIF(COUNT(*), 0), 2) AS outlier_pct FROM {{ $('Config').item.json.tableName }}", "options": {}, "operation": "executeQuery" }, "credentials": { "postgres": { "name": "" } }, "typeVersion": 2.5 }, { "id": "cea373b4-c913-4d9f-b566-90b9ddb35879", "name": "Merge", "type": "n8n-nodes-base.merge", "position": [ 832, 352 ], "parameters": { "numberInputs": 5 }, "typeVersion": 3 }, { "id": "0b323db9-8b30-453b-a396-ca74ce1b0308", "name": "Evaluate & Format", "type": "n8n-nodes-base.code", "position": [ 1008, 400 ], "parameters": { "jsCode": "const cfg = $('Config').item.json;\nconst items = $input.all();\n\nfunction findCheck(type) {\n return (items.find(i => i.json.check_type === type) || { json: {} }).json;\n}\n\nconst nullR = findCheck('null_check');\nconst dupR = findCheck('dup_check');\nconst countR = findCheck('row_count');\nconst outlierR = findCheck('outlier_check');\n\nfunction evalStatus(value, threshold, mode) {\n if (mode === 'above') {\n if (value === 0) return { label: 'PASS', color: '#16a34a' };\n if (value <= threshold) return { label: 'WARN', color: '#d97706' };\n return { label: 'FAIL', color: '#dc2626' };\n }\n if (mode === 'range') {\n if (value >= cfg.rowCountMin && value <= cfg.rowCountMax) return { label: 'PASS', color: '#16a34a' };\n if (value < cfg.rowCountMin * 0.8 || value > cfg.rowCountMax * 1.2) return { label: 'FAIL', color: '#dc2626' };\n return { label: 'WARN', color: '#d97706' };\n }\n}\n\nconst nullPct = parseFloat(nullR.null_pct ?? 0);\nconst dupPct = parseFloat(dupR.dup_pct ?? 0);\nconst rowCount = parseInt(countR.row_count ?? 0, 10);\nconst outlierPct = parseFloat(outlierR.outlier_pct ?? 0);\n\nconst checks = [\n { name: 'Null / missing values', col: cfg.nullCheckCol, value: `${nullPct}%`, threshold: `< ${cfg.nullThresholdPct}%`, st: evalStatus(nullPct, cfg.nullThresholdPct, 'above') },\n { name: 'Duplicate rows', col: cfg.duplicateCheckCol, value: `${dupPct}%`, threshold: `< ${cfg.dupThresholdPct}%`, st: evalStatus(dupPct, cfg.dupThresholdPct, 'above') },\n { name: 'Row count anomaly', col: cfg.tableName, value: rowCount.toLocaleString(), threshold: `${cfg.rowCountMin} \u2013 ${cfg.rowCountMax}`, st: evalStatus(rowCount, null, 'range') },\n { name: 'Outlier values', col: cfg.outlierCol, value: `${outlierPct}%`, threshold: `${cfg.outlierMin} \u2013 ${cfg.outlierMax}`, st: evalStatus(outlierPct, 5, 'above') },\n];\n\nconst hasFail = checks.some(c => c.st.label === 'FAIL');\nconst hasWarn = checks.some(c => c.st.label === 'WARN');\nconst overall = hasFail ? 'FAIL' : hasWarn ? 'WARN' : 'PASS';\nconst oc = { PASS: '#16a34a', WARN: '#d97706', FAIL: '#dc2626' }[overall];\nconst now = new Date().toISOString().slice(0, 19).replace('T', ' ') + ' UTC';\n\nconst badge = (label, color) =>\n `${label}`;\n\nconst tableRow = c =>\n `` +\n `${c.name}` +\n `${c.col}` +\n `${c.value}` +\n `${c.threshold}` +\n `${badge(c.st.label, c.st.color)}` +\n ``;\n\nconst htmlReport =\n `` +\n `
` +\n `
` +\n `
` +\n `
Data Quality Report
` +\n `
Table: ${cfg.tableName}  ·  ${now}
` +\n `
${badge(overall, oc)}
` +\n `` +\n `` +\n `` +\n `` +\n `` +\n `` +\n `` +\n `` +\n `${checks.map(tableRow).join('')}` +\n `
CheckColumnValueThresholdStatus
` +\n `
` +\n `Generated by n8n Data Quality Bot  ·  Thresholds configured in the Config (Set) node` +\n `
`;\n\nreturn [{\n json: {\n overall,\n htmlReport,\n emailSubject: `[${overall}] Data Quality \u2014 ${cfg.tableName} \u2014 ${now}`,\n reportEmail: cfg.reportEmail,\n sheet_timestamp: now,\n sheet_table: cfg.tableName,\n sheet_null_pct: nullPct,\n sheet_dup_pct: dupPct,\n sheet_row_count: rowCount,\n sheet_outlier_pct: outlierPct,\n sheet_status: overall\n }\n}];" }, "typeVersion": 2 }, { "id": "f678cd51-4d24-4318-993d-2795fad9bce4", "name": "Log to Google Sheets", "type": "n8n-nodes-base.googleSheets", "position": [ 1232, 496 ], "parameters": { "columns": { "value": { "Dup %": "={{ $json.sheet_dup_pct }}", "Table": "={{ $json.sheet_table }}", "Null %": "={{ $json.sheet_null_pct }}", "Status": "={{ $json.sheet_status }}", "overall": "={{ $json.overall }}", "Outlier %": "={{ $json.sheet_outlier_pct }}", "Row Count": "={{ $json.sheet_row_count }}", "Timestamp": "={{ $json.sheet_timestamp }}" }, "schema": [ { "id": "Timestamp", "type": "string", "display": true, "removed": false, "required": false, "displayName": "Timestamp", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "Table", "type": "string", "display": true, "removed": false, "required": false, "displayName": "Table", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "Null %", "type": "string", "display": true, "removed": false, "required": false, "displayName": "Null %", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "Dup %", "type": "string", "display": true, "removed": false, "required": false, "displayName": "Dup %", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "Row Count", "type": "string", "display": true, "removed": false, "required": false, "displayName": "Row Count", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "Outlier %", "type": "string", "display": true, "removed": false, "required": false, "displayName": "Outlier %", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "Status", "type": "string", "display": true, "removed": false, "required": false, "displayName": "Status", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "overall", "type": "string", "display": true, "removed": false, "required": false, "displayName": "overall", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "htmlReport", "type": "string", "display": true, "removed": true, "required": false, "displayName": "htmlReport", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "emailSubject", "type": "string", "display": true, "removed": true, "required": false, "displayName": "emailSubject", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "sheet_timestamp", "type": "string", "display": true, "removed": true, "required": false, "displayName": "sheet_timestamp", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "sheet_table", "type": "string", "display": true, "removed": true, "required": false, "displayName": "sheet_table", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "sheet_null_pct", "type": "string", "display": true, "removed": true, "required": false, "displayName": "sheet_null_pct", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "sheet_dup_pct", "type": "string", "display": true, "removed": true, "required": false, "displayName": "sheet_dup_pct", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "sheet_row_count", "type": "string", "display": true, "removed": true, "required": false, "displayName": "sheet_row_count", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "sheet_outlier_pct", "type": "string", "display": true, "removed": true, "required": false, "displayName": "sheet_outlier_pct", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "sheet_status", "type": "string", "display": true, "removed": true, "required": false, "displayName": "sheet_status", "defaultMatch": false, "canBeUsedToMatch": true } ], "mappingMode": "defineBelow", "matchingColumns": [], "attemptToConvertTypes": false, "convertFieldsToString": false }, "options": {}, "operation": "append", "sheetName": { "__rl": true, "mode": "list", "value": "gid=0", "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1zIzM4gtIvtEj6t9YmBCQtieYLZLRmXj_37BoBtVTWw8/edit#gid=0", "cachedResultName": "Sheet1" }, "documentId": { "__rl": true, "mode": "list", "value": "1zIzM4gtIvtEj6t9YmBCQtieYLZLRmXj_37BoBtVTWw8", "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1zIzM4gtIvtEj6t9YmBCQtieYLZLRmXj_37BoBtVTWw8/edit?usp=drivesdk", "cachedResultName": "Log for order" } }, "credentials": { "googleSheetsOAuth2Api": { "name": "" } }, "typeVersion": 4.4 }, { "id": "51ba3e03-248d-4e7a-8df2-7c11a3a4eca5", "name": "Send a message", "type": "n8n-nodes-base.gmail", "position": [ 1232, 304 ], "parameters": { "sendTo": "={{ $('Config').item.json['Distribution list'] }}", "message": "={{ $json.htmlReport }}", "options": {}, "subject": "={{ $json.emailSubject }}" }, "credentials": { "gmailOAuth2": { "name": "" } }, "typeVersion": 2.2 }, { "id": "80b53380-1912-4945-add4-4e4d976bad67", "name": "Sticky Note", "type": "n8n-nodes-base.stickyNote", "position": [ -672, 16 ], "parameters": { "width": 528, "height": 976, "content": "\ud83d\udcca Automated Data Quality Report Bot\n\nThis workflow automatically monitors data quality for a selected database table and generates a structured report with actionable insights. It performs four key checks\u2014null values, duplicates, row count anomalies, and outliers\u2014then evaluates results against configurable thresholds to determine overall data health (PASS / WARN / FAIL). The output is sent via email and logged into Google Sheets for historical tracking and auditing.\n\nHow it works\n\nThe workflow runs on a schedule trigger (daily by default). It reads configuration values (table name, columns, thresholds) from the Config node and executes four SQL checks in parallel. Results are merged and processed in a code node, which evaluates each metric and generates a formatted HTML report. Finally, results are sent via email and appended to a Google Sheet.\n\nSetup\n## \ud83d\udee0\ufe0f Setup Guide\n\n**Step 1 \u2014 Config node**\nEdit the `Config` node to set your table name, column names, thresholds and Email Distribution list\n\n**Step 2 \u2014 DB credentials**\nOpen each of the 4 teal query nodes and attach your Postgres credential. Swap the node type if using MySQL or another DB.\n\n**Step 3 \u2014 Email**\nConnect your SMTP or Gmail credential to the `Send Email` node.\n\n**Step 4 \u2014 Google Sheets**\nConnect your Google account to `Log to Google Sheets`. Create a sheet with columns: Timestamp, Table, Null_Pct, Dup_Pct, Row_Count, Outlier_Pct, Status.\n\n**Step 5 \u2014 Activate**\nToggle the workflow on. It runs daily at 8am by default.\nCustomization\nAdd more checks by duplicating query nodes.\nAdjust thresholds for stricter/looser validation.\nExtend reporting (Slack, Teams, dashboards).\n\n**Customization**\nAdd more checks by duplicating query nodes.\nAdjust thresholds for stricter/looser validation.\nExtend reporting (Slack, Teams, dashboards)." }, "typeVersion": 1 }, { "id": "ebb0c98d-0995-40e4-b753-87b683c4230c", "name": "Sticky Note1", "type": "n8n-nodes-base.stickyNote", "position": [ 48, 272 ], "parameters": { "color": 7, "width": 400, "height": 288, "content": "\u23f0 Trigger & Configuration\n\nControls workflow execution and defines all dynamic inputs like table name, columns, thresholds, and email recipients." }, "typeVersion": 1 }, { "id": "a0e34fa9-e962-4c7d-ad42-45e9be249169", "name": "Sticky Note2", "type": "n8n-nodes-base.stickyNote", "position": [ 480, -80 ], "parameters": { "color": 7, "height": 928, "content": "\ud83e\uddea Data Quality Checks (SQL)\n\nRuns four parallel checks:\n\nNull values\nDuplicate records\nRow count validation\nOutlier detection" }, "typeVersion": 1 }, { "id": "c0b454bc-1fb1-4af2-8ba1-96c351d98402", "name": "Sticky Note3", "type": "n8n-nodes-base.stickyNote", "position": [ 768, 192 ], "parameters": { "color": 7, "width": 160, "height": 416, "content": "\ud83d\udd17 Merge Results\n\nCombines outputs from all SQL checks into a single dataset for evaluation." }, "typeVersion": 1 }, { "id": "933b2425-3eb3-4821-91e3-f53719cd0ba1", "name": "Sticky Note4", "type": "n8n-nodes-base.stickyNote", "position": [ 976, 176 ], "parameters": { "color": 7, "width": 160, "height": 352, "content": "\ud83e\udde0 Evaluate & Format Report\n\nApplies threshold logic (PASS/WARN/FAIL) and generates a clean HTML report with status indicators." }, "typeVersion": 1 }, { "id": "26018a1a-1fb3-4c75-9665-71e11095d902", "name": "Sticky Note5", "type": "n8n-nodes-base.stickyNote", "position": [ 1168, 144 ], "parameters": { "color": 7, "height": 528, "content": "\ud83d\udce4 Output & Notifications\n\nSends report via email and logs results into Google Sheets for tracking and auditing." }, "typeVersion": 1 }, { "id": "e33474a1-3dfb-45a3-ae4f-8cc71850f245", "name": "Sticky Note6", "type": "n8n-nodes-base.stickyNote", "position": [ 288, 384 ], "parameters": { "color": "#D82222", "width": 192, "height": 368, "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\u26a0\ufe0f Critical Setup\n\nEnsure all column names in the Config node match your database schema exactly. Incorrect names will break SQL queries." }, "typeVersion": 1 } ], "active": false, "settings": { "binaryMode": "separate", "availableInMCP": false, "executionOrder": "v1" }, "versionId": "88272d2c-f833-4962-9606-9c39c27a2208", "connections": { "Merge": { "main": [ [ { "node": "Evaluate & Format", "type": "main", "index": 0 } ] ] }, "Config": { "main": [ [ { "node": "Null Check", "type": "main", "index": 0 }, { "node": "Duplicate Check", "type": "main", "index": 0 }, { "node": "Row Count", "type": "main", "index": 0 }, { "node": "Outlier Check", "type": "main", "index": 0 } ] ] }, "Row Count": { "main": [ [ { "node": "Merge", "type": "main", "index": 3 } ] ] }, "Null Check": { "main": [ [ { "node": "Merge", "type": "main", "index": 0 } ] ] }, "Outlier Check": { "main": [ [ { "node": "Merge", "type": "main", "index": 4 } ] ] }, "Duplicate Check": { "main": [ [ { "node": "Merge", "type": "main", "index": 1 } ] ] }, "Schedule Trigger": { "main": [ [ { "node": "Config", "type": "main", "index": 0 } ] ] }, "Evaluate & Format": { "main": [ [ { "node": "Log to Google Sheets", "type": "main", "index": 0 }, { "node": "Send a message", "type": "main", "index": 0 } ] ] } } }