{ "name": "wf_scrape_reddit_posts", "nodes": [ { "parameters": { "resource": "webScrapper", "operation": "monitorProgressSnapshot", "snapshot_id": "={{ $json.snapshot_id }}", "requestOptions": {} }, "type": "@brightdata/n8n-nodes-brightdata.brightData", "typeVersion": 1, "position": [ 912, -640 ], "id": "db71b778-2dc9-4fc8-996d-f8407e9af347", "name": "check_batch", "credentials": { "brightdataApi": { "name": "" } }, "onError": "continueErrorOutput", "notes": "This should be removed after verification that the notify as expected" }, { "parameters": { "amount": 15 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ 1360, -384 ], "id": "f65f8120-1e2e-499a-8442-f1347c6e2992", "name": "delay_15_sec" }, { "parameters": { "resource": "webScrapper", "operation": "downloadSnapshot", "batch_size": "=1000", "snapshot_id": "={{ $json.snapshot_id }}", "requestOptions": {} }, "type": "@brightdata/n8n-nodes-brightdata.brightData", "typeVersion": 1, "position": [ 1584, -544 ], "id": "e12e6d44-8a89-4cef-b11e-922faa770097", "name": "download_snapshots", "credentials": { "brightdataApi": { "name": "" } }, "onError": "continueErrorOutput" }, { "parameters": { "operation": "appendOrUpdate", "documentId": { "__rl": true, "value": "1Ij6RVvdsF2yUdJfz2_p9gAwRz2LmySUWxlxajQyY-XE", "mode": "list", "cachedResultName": "underfoot_subreddit_cache", "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1Ij6RVvdsF2yUdJfz2_p9gAwRz2LmySUWxlxajQyY-XE/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "value": "gid=0", "mode": "list", "cachedResultName": "Shee1", "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1Ij6RVvdsF2yUdJfz2_p9gAwRz2LmySUWxlxajQyY-XE/edit#gid=0" }, "columns": { "mappingMode": "autoMapInputData", "value": {}, "matchingColumns": [ "url" ], "schema": [ { "id": "url", "displayName": "url", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "title", "displayName": "title", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "description", "displayName": "description", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "num_comments", "displayName": "num_comments", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "date_posted", "displayName": "date_posted", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "num_upvotes", "displayName": "num_upvotes", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "photos", "displayName": "photos", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "embedded_links", "displayName": "embedded_links", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "created_at_iso", "displayName": "created_at_iso", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false } ], "attemptToConvertTypes": false, "convertFieldsToString": false }, "options": { "useAppend": true } }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ 2032, -448 ], "id": "8da4cadd-3d1d-4af3-a7bf-95cb54c28821", "name": "update_cache", "credentials": { "googleSheetsOAuth2Api": { "name": "" } }, "onError": "continueErrorOutput" }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "ddc2c830-0d39-4d4c-94cc-9c647a42686c", "leftValue": "={{ $json.records }}", "rightValue": 0, "operator": { "type": "number", "operation": "gt" } } ], "combinator": "and" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 1360, -608 ], "id": "60052ae1-0314-4a67-a378-53f7f81e5e72", "name": "has_records" }, { "parameters": { "workflowId": { "__rl": true, "value": "wRCvCmRGcILIoLjy", "mode": "list", "cachedResultName": "Underfoot \u2014 wf_error_notifications" }, "workflowInputs": { "mappingMode": "defineBelow", "value": { "json": "={{ $json }}", "callingWorkflow": "={{ $workflow }}" }, "matchingColumns": [ "json" ], "schema": [ { "id": "json", "displayName": "json", "required": false, "defaultMatch": false, "display": true, "canBeUsedToMatch": true, "type": "object", "removed": false }, { "id": "callingWorkflow", "displayName": "callingWorkflow", "required": false, "defaultMatch": false, "display": true, "canBeUsedToMatch": true, "type": "object", "removed": false } ], "attemptToConvertTypes": false, "convertFieldsToString": true }, "mode": "each", "options": {} }, "type": "n8n-nodes-base.executeWorkflow", "typeVersion": 1.2, "position": [ 2256, -944 ], "id": "173c7e6e-a1a5-4ef5-8213-0488bba0dae2", "name": "call_error_notifier", "retryOnFail": false, "waitBetweenTries": 1500, "onError": "continueRegularOutput" }, { "parameters": { "inputSource": "jsonExample", "jsonExample": "{\n \"intent\": \"an interest\",\n \"location\": \"a place to look for events\"\n}" }, "type": "n8n-nodes-base.executeWorkflowTrigger", "typeVersion": 1.1, "position": [ 16, -880 ], "id": "55f19696-4b73-49df-afef-8bd049d307d8", "name": "workflow_trigger", "onError": "continueRegularOutput" }, { "parameters": { "jsCode": "// n8n Code node (Run Once for Each Item)\n// Input: { user_intent, user_location }\n// Output: array of keyword objects ready for Bright Data\n\nconst intent = ($input.first().json.intent || \"\").trim();\nconst loc = ($input.first().json.location || \"\").trim();\n\nconst parts = loc.split(\",\").map(s => s.trim()).filter(Boolean);\nconst city = parts[0] || \"\";\nconst state = parts[1] || \"\";\nconst country = parts[2] || \"\";\n\nconst combos = [];\nif (city) combos.push(`${intent} ${city}`);\nif (city && state) combos.push(`${intent} ${city} ${state}`);\nif (city && state && country) combos.push(`${intent} ${city} ${state} ${country}`);\n\nconst uniq = arr => [...new Set(arr.filter(Boolean))];\n\nconst results = [];\nfor (const kw of uniq(combos)) {\n results.push({ keyword: kw, date: \"Past month\", num_of_posts: 10, sort_by: \"Top\" });\n results.push({ keyword: kw, date: \"Past week\", num_of_posts: 10, sort_by: \"Hot\" });\n}\n\nreturn results;" }, "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 240, -880 ], "id": "cba6ae5d-22b4-435f-bd67-13a69f7eb097", "name": "generate_discovery_urls", "onError": "continueErrorOutput" }, { "parameters": { "method": "POST", "url": "https://api.brightdata.com/datasets/v3/trigger", "authentication": "genericCredentialType", "genericAuthType": "httpBearerAuth", "sendQuery": true, "queryParameters": { "parameters": [ { "name": "dataset_id", "value": "={{ $vars.REDDIT_POSTS_BY_KEYWORD_DATASET_ID }}" }, { "name": "include_errors", "value": "true" }, { "name": "type", "value": "discover_new" }, { "name": "discover_by", "value": "keyword" }, { "name": "notify", "value": "={{ $vars.DISCORD_BRIGHTDATA_NOTIFY_WEBHOOK }}" } ] }, "sendHeaders": true, "headerParameters": { "parameters": [ { "name": "Content-Type", "value": "application/json" } ] }, "sendBody": true, "specifyBody": "json", "jsonBody": "={{ $json }}", "options": { "timeout": 12000 } }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 688, -800 ], "id": "e4e73758-8fac-47c7-b4cf-bff70ca28ddb", "name": "discover_reddit_posts_by_keyword", "executeOnce": false, "credentials": { "httpBearerAuth": { "name": "" } }, "onError": "continueErrorOutput" }, { "parameters": { "options": {} }, "type": "n8n-nodes-base.removeDuplicates", "typeVersion": 2, "position": [ 464, -800 ], "id": "bd40e50c-3ffe-40c7-a6db-3b4d81d3f5ca", "name": "remove_duplicates", "onError": "continueRegularOutput", "notes": "This is really extra because I wanted to see how it works. In reality, I have complete control already over what's inserted into the list. There would be no reason to do it after the fact." }, { "parameters": { "rules": { "values": [ { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "leftValue": "={{ $json.status }}", "rightValue": "ready", "operator": { "type": "string", "operation": "equals" }, "id": "154ccc45-69c3-407a-a5fb-7c41edba0d42" } ], "combinator": "and" }, "renameOutput": true, "outputKey": "ready" }, { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "548371ce-824a-410d-b1e9-ac53cfa2226c", "leftValue": "={{ $json.status }}", "rightValue": "running", "operator": { "type": "string", "operation": "equals", "name": "filter.operator.equals" } } ], "combinator": "and" }, "renameOutput": true, "outputKey": "running" }, { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 2 }, "conditions": [ { "id": "d80a18d1-3ee4-4e49-a06f-bffbf9f38ae5", "leftValue": "={{ $json.status }}", "rightValue": "failed", "operator": { "type": "string", "operation": "equals", "name": "filter.operator.equals" } } ], "combinator": "and" }, "renameOutput": true, "outputKey": "failed" } ] }, "options": {} }, "type": "n8n-nodes-base.switch", "typeVersion": 3.2, "position": [ 1136, -656 ], "id": "0c5f643f-67cc-48ba-838e-30dc30f88ae8", "name": "check_status", "onError": "continueRegularOutput" }, { "parameters": { "jsCode": "// Run Once for All Items\nconst toISO = (v) => { try { return v ? new Date(v).toISOString() : null; } catch { return null; } };\nconst num = (v) => {\n if (v === null || v === undefined || v === '') return null;\n const n = Number(v);\n return Number.isFinite(n) ? n : null;\n};\n\nconst user_intent = $('workflow_trigger').first().json.intent ?? null;\nconst user_location = $('workflow_trigger').first().json?.location ?? null;\n\n\nreturn $input.all().map(({ json: p }) => {\n const embedded_links = Array.isArray(p.related_posts)\n ? p.related_posts.map(r => ({ url: r?.url ?? null, title: r?.title ?? null }))\n : null;\n\n return {\n url: p.url ?? null,\n title: p.title ?? null,\n description: p.description ?? null,\n num_comments: num(p.num_comments),\n date_posted: toISO(p.date_posted),\n num_upvotes: num(p.num_upvotes),\n photos: Array.isArray(p.photos) ? p.photos : (p.photos ? [p.photos] : null),\n embedded_links,\n created_at_iso: toISO(p.timestamp) || toISO(p.date_posted),\n user_intent,\n user_location\n };\n});" }, "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 1808, -448 ], "id": "80157cbd-2d68-4bdc-93d3-8d61edf37ce8", "name": "normalize", "onError": "continueErrorOutput" }, { "parameters": { "method": "POST", "url": "https://uqvwaiexsgprdbdecoxx.supabase.co/functions/v1/merge-cache", "authentication": "genericCredentialType", "genericAuthType": "httpBearerAuth", "sendBody": true, "specifyBody": "json", "jsonBody": "={\n \"title\": {{ $json.title }},\n \"description\": {{ $json.description }},\n \"source_type\": {{ $workflow.name }},\n \"external_id\": {{ $execution.id }},\n \"url\": {{ $json.url }},\n \"image_url\": {{ $json.image_url }},\n \"data\": {{ $json }}\n}", "options": { "timeout": 120000 } }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 2032, -256 ], "id": "56806290-271d-46bb-b196-5a6cb6e886c3", "name": "update_cache_supabase", "credentials": { "httpBearerAuth": { "name": "" } } } ], "connections": { "check_batch": { "main": [ [ { "node": "check_status", "type": "main", "index": 0 } ], [ { "node": "call_error_notifier", "type": "main", "index": 0 } ] ] }, "delay_15_sec": { "main": [ [ { "node": "check_batch", "type": "main", "index": 0 } ] ] }, "download_snapshots": { "main": [ [ { "node": "normalize", "type": "main", "index": 0 } ], [ { "node": "call_error_notifier", "type": "main", "index": 0 } ] ] }, "has_records": { "main": [ [ { "node": "download_snapshots", "type": "main", "index": 0 } ], [ { "node": "call_error_notifier", "type": "main", "index": 0 } ] ] }, "update_cache": { "main": [ [], [ { "node": "call_error_notifier", "type": "main", "index": 0 } ] ] }, "workflow_trigger": { "main": [ [ { "node": "generate_discovery_urls", "type": "main", "index": 0 } ] ] }, "generate_discovery_urls": { "main": [ [ { "node": "remove_duplicates", "type": "main", "index": 0 } ], [ { "node": "call_error_notifier", "type": "main", "index": 0 } ] ] }, "discover_reddit_posts_by_keyword": { "main": [ [ { "node": "check_batch", "type": "main", "index": 0 } ], [ { "node": "call_error_notifier", "type": "main", "index": 0 } ] ] }, "remove_duplicates": { "main": [ [ { "node": "discover_reddit_posts_by_keyword", "type": "main", "index": 0 } ] ] }, "check_status": { "main": [ [ { "node": "has_records", "type": "main", "index": 0 } ], [ { "node": "delay_15_sec", "type": "main", "index": 0 } ], [ { "node": "call_error_notifier", "type": "main", "index": 0 } ] ] }, "normalize": { "main": [ [ { "node": "update_cache", "type": "main", "index": 0 }, { "node": "update_cache_supabase", "type": "main", "index": 0 } ], [ { "node": "call_error_notifier", "type": "main", "index": 0 } ] ] } }, "active": false, "settings": { "executionOrder": "v1", "callerPolicy": "workflowsFromSameOwner", "errorWorkflow": "g0RA1ILyuSMHbt6q" }, "versionId": "5f018cce-2920-473e-bbef-cb5694c17d2b", "meta": { "templateCredsSetupCompleted": true }, "id": "ctgclUQYW6ckB5ou", "tags": [] }