{ "id": "G9psPUm3yJXIKj9m", "meta": { "templateCredsSetupCompleted": true }, "name": "\ud83c\udf31 Check Data Freshness", "tags": [], "nodes": [ { "id": "01241280-49c4-47d1-a09c-3277e5313d13", "name": "Schedule Trigger", "type": "n8n-nodes-base.scheduleTrigger", "position": [ -160, 432 ], "parameters": { "rule": { "interval": [ {} ] } }, "typeVersion": 1.2 }, { "id": "3c5924ab-1d15-4747-b18c-a047d1f1e26b", "name": "Loop Over Items", "type": "n8n-nodes-base.splitInBatches", "position": [ 304, 432 ], "parameters": { "options": {} }, "typeVersion": 3 }, { "id": "4fd6e105-9bc9-4494-964f-755d86940117", "name": "Aggregate Stale Tables", "type": "n8n-nodes-base.aggregate", "position": [ 1120, 272 ], "parameters": { "options": {}, "fieldsToAggregate": { "fieldToAggregate": [ { "renameField": true, "outputFieldName": "tables", "fieldToAggregate": "table" } ] } }, "typeVersion": 1 }, { "id": "ee5378e3-3907-419b-8ca5-53e26abeb295", "name": "Get most recent row from table", "type": "n8n-nodes-base.postgres", "position": [ 528, 432 ], "parameters": { "sort": { "values": [ { "column": "={{ $json.timestampColumn }}", "direction": "DESC" } ] }, "limit": 1, "table": { "__rl": true, "mode": "name", "value": "={{ $json.tableName }}" }, "schema": { "__rl": true, "mode": "list", "value": "public" }, "options": {}, "operation": "select" }, "credentials": { "postgres": { "name": "" } }, "typeVersion": 2.6 }, { "id": "41b21d6f-0134-424e-87af-05cd38a8997b", "name": "Calculate lag", "type": "n8n-nodes-base.dateTime", "position": [ 704, 432 ], "parameters": { "endDate": "={{ $now }}", "options": {}, "operation": "getTimeBetweenDates", "startDate": "={{ $json[$('Produce tables + date columns').item.json.timestampColumn] }}" }, "typeVersion": 2 }, { "id": "3b09f150-585a-41aa-9343-d68ca08c94c2", "name": "Add back table name", "type": "n8n-nodes-base.set", "position": [ 864, 432 ], "parameters": { "options": {}, "assignments": { "assignments": [ { "id": "8f72670f-9e0e-425b-a5a6-0986d2e475d1", "name": "daysOld", "type": "number", "value": "={{ $json.timeDifference.days }}" }, { "id": "1d5bd3ec-a1b5-4d40-94ca-15d22696d5f0", "name": "table", "type": "string", "value": "={{ $('Produce tables + date columns').item.json.tableName }}" } ] } }, "typeVersion": 3.4 }, { "id": "e8749421-ced2-4951-9b28-d003513aefd7", "name": "Sticky Note", "type": "n8n-nodes-base.stickyNote", "position": [ 64, 208 ], "parameters": { "width": 976, "height": 432, "content": "## Find tables with stale data" }, "typeVersion": 1 }, { "id": "b6c620c4-16c0-402b-b18e-047ed9a7da82", "name": "Remove fresh tables", "type": "n8n-nodes-base.filter", "position": [ 528, 272 ], "parameters": { "options": {}, "conditions": { "options": { "version": 2, "leftValue": "", "caseSensitive": true, "typeValidation": "strict" }, "combinator": "and", "conditions": [ { "id": "dc7e2a08-be0b-4226-b2f8-cd7609b1d61e", "operator": { "type": "number", "operation": "gte" }, "leftValue": "={{ $json.daysOld }}", "rightValue": 3 } ] } }, "typeVersion": 2.2 }, { "id": "99b1aa9a-ab2c-437a-959a-bffc86d5e153", "name": "Sticky Note1", "type": "n8n-nodes-base.stickyNote", "position": [ -672, -528 ], "parameters": { "width": 656, "height": 928, "content": "# Monitor Postgres Data Freshness and Email Alert If Stale\nThis template monitors a set of tables inside a Postgres database to ensure they're getting updated.\n\nIf the table hasn't been updated in 3 days (configurable), an email alert is sent containing the tables that are stale.\n\n## Requirements\nYou must have a Postgres database containing one or more tables that you'd like to monitor.\n\nEach table to monitor must have a date or timestamp column that tracks when data was pushed.\n\nFor example, this might be:\n\n- A `timestamp` column if your table holds event/timeseries data\n- A `last_updated` column if your rows are expected to be modified\n\n## Usage\n1. Use this template\n1. Add your Postgres and email credentials\n1. Adjust the `Produce tables + date columns` node to produce pairs of `[table, date_column]` that should be monitored for freshness\n - \ud83d\udc81\u200d\u2642\ufe0f Note that a timestamp column also works\n1. (Optional) Adjust the `Remove fresh tables` node for your desired staleness window (default is 3 days, but you can adjust as you please)\n1. (Optional) Customize the `Send alerts` node to call whichever alerting workflow you please (I recommend [my alerting workflow](https://creators.n8n.io/workflows/6189) for easiest plug-and-play)\n\n## How it works\nThis template works by:\n\n1. Pulling the most recent row for each table\n1. Calculating how out-of-date each table is, in days\n1. Dropping fresh tables that have been updated within the past 3 days\n1. Sending an email alert with the stale tables that haven't been updated within the past 3 days\n" }, "typeVersion": 1 }, { "id": "9f2c4f7e-afee-4293-acc9-a3b854814986", "name": "Produce tables + date columns", "type": "n8n-nodes-base.code", "position": [ 112, 432 ], "parameters": { "jsCode": "const VALUE_TUPLES = [\n [\"sleep_summary\",\"sleep_date\"],\n [\"mac_screentime_sessions\", \"ts\"],\n]\n\nlet result = []\nfor (const [tableName, timestampCol] of VALUE_TUPLES) {\n result.push({\n \"tableName\": tableName,\n \"timestampColumn\": timestampCol,\n })\n}\nreturn result;" }, "typeVersion": 2 }, { "id": "11b1618a-eb84-4f09-8678-51914758e3c5", "name": "Send alerts", "type": "n8n-nodes-base.executeWorkflow", "position": [ 1328, 272 ], "parameters": { "options": {}, "workflowId": { "__rl": true, "mode": "list", "value": "17yjHH5i669dVRGk", "cachedResultName": "\u2757 Send Alert" }, "workflowInputs": { "value": { "body": "=The following quantified life tables are stale (>=3 days old):\n{{ $json.tables.join(\"\\n\") }}", "subject": "Stale Quantified Life Data" }, "schema": [ { "id": "subject", "type": "string", "display": true, "required": false, "displayName": "subject", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "body", "type": "string", "display": true, "removed": false, "required": false, "displayName": "body", "defaultMatch": false, "canBeUsedToMatch": true } ], "mappingMode": "defineBelow", "matchingColumns": [], "attemptToConvertTypes": false, "convertFieldsToString": true } }, "typeVersion": 1.2 }, { "id": "ffa03d08-8596-42f3-82cf-0741373832c3", "name": "Sticky Note2", "type": "n8n-nodes-base.stickyNote", "position": [ 1344, -32 ], "parameters": { "color": 3, "width": 262, "height": 336, "content": "# (Optional)\n# Customize the alerting workflow\nI recommend [my alerting workflow](https://creators.n8n.io/workflows/6189)\n# \ud83d\udd3d\n" }, "typeVersion": 1 }, { "id": "08108247-6969-4544-90d1-ceaaa5091a9d", "name": "Sticky Note3", "type": "n8n-nodes-base.stickyNote", "position": [ 544, -16 ], "parameters": { "color": 3, "width": 230, "height": 304, "content": "# (Optional)\n# Customize the stale threshold\n# \ud83d\udd3d\n" }, "typeVersion": 1 } ], "active": true, "settings": { "callerPolicy": "workflowsFromSameOwner", "errorWorkflow": "94eLbjCAIMla9lSa", "executionOrder": "v1" }, "versionId": "55232ec1-8d2f-4639-bf66-e86c0b01f9fa", "connections": { "Calculate lag": { "main": [ [ { "node": "Add back table name", "type": "main", "index": 0 } ] ] }, "Loop Over Items": { "main": [ [ { "node": "Remove fresh tables", "type": "main", "index": 0 } ], [ { "node": "Get most recent row from table", "type": "main", "index": 0 } ] ] }, "Schedule Trigger": { "main": [ [ { "node": "Produce tables + date columns", "type": "main", "index": 0 } ] ] }, "Add back table name": { "main": [ [ { "node": "Loop Over Items", "type": "main", "index": 0 } ] ] }, "Remove fresh tables": { "main": [ [ { "node": "Aggregate Stale Tables", "type": "main", "index": 0 } ] ] }, "Aggregate Stale Tables": { "main": [ [ { "node": "Send alerts", "type": "main", "index": 0 } ] ] }, "Produce tables + date columns": { "main": [ [ { "node": "Loop Over Items", "type": "main", "index": 0 } ] ] }, "Get most recent row from table": { "main": [ [ { "node": "Calculate lag", "type": "main", "index": 0 } ] ] } } }