{ "nodes": [ { "id": "61e7b101-ae69-4a2a-a073-9701141e6817", "name": "Sticky Note", "type": "n8n-nodes-base.stickyNote", "position": [ 880, -288 ], "parameters": { "width": 480, "height": 480, "content": "### How it works\n\n1. The workflow triggers manually to initiate data processing.\n2. Environment variables are set for HTTP requests and data handling.\n3. Data is fetched from DataForSEO and parsed.\n4. SQL queries are executed to prepare, filter, and merge data.\n5. Information is extracted and validated using AI models.\n6. Results are formatted and finalized for output.\n\n### Setup steps\n\n- [ ] Ensure DataForSEO and database credentials are configured.\n- [ ] Set up OpenAI credentials for AI model usage.\n- [ ] Verify environment variables like target and language are correctly set.\n\n### Customization\n\nThe AI model prompts and data parsing logic can be customized based on specific SEO metrics and outputs." }, "typeVersion": 1 }, { "id": "c8fa3ba8-9e32-47c4-a365-653f8ea94729", "name": "Sticky Note1", "type": "n8n-nodes-base.stickyNote", "position": [ 1392, -112 ], "parameters": { "color": 7, "width": 400, "height": 304, "content": "## Manual trigger setup\n\nInitiates the workflow and sets environment variables." }, "typeVersion": 1 }, { "id": "4c38f6de-10a7-42fb-a43b-b597108615f4", "name": "Sticky Note2", "type": "n8n-nodes-base.stickyNote", "position": [ 1824, -96 ], "parameters": { "color": 7, "width": 864, "height": 272, "content": "## Fetch and prepare data\n\nHandles HTTP requests and prepares items for processing." }, "typeVersion": 1 }, { "id": "d37053c2-dd3b-4998-bd77-85447a6585e4", "name": "Sticky Note3", "type": "n8n-nodes-base.stickyNote", "position": [ 2720, -208 ], "parameters": { "color": 7, "width": 416, "height": 496, "content": "## Extract and merge data\n\nParses SERP data and merges with existing data through SQL." }, "typeVersion": 1 }, { "id": "aeae40db-bee5-48ae-ad0e-d9241a92b4a7", "name": "Sticky Note4", "type": "n8n-nodes-base.stickyNote", "position": [ 3168, -128 ], "parameters": { "color": 7, "width": 528, "height": 320, "content": "## Check conditions and optimize\n\nEvaluates audit status and optimizes prompts for AI processing." }, "typeVersion": 1 }, { "id": "73838fdb-c329-43b7-a735-9c582ec20cca", "name": "Sticky Note5", "type": "n8n-nodes-base.stickyNote", "position": [ 3712, -192 ], "parameters": { "color": 7, "width": 496, "height": 560, "content": "## AI-based information extraction\n\nUtilizes AI to extract and verify information." }, "typeVersion": 1 }, { "id": "edcdfbf8-c19d-41e6-9858-1f5e90e7d438", "name": "Sticky Note6", "type": "n8n-nodes-base.stickyNote", "position": [ 4256, -112 ], "parameters": { "color": 7, "width": 768, "height": 416, "content": "## Conditional check and final output\n\nChecks results and formats for final output or further SQL processing." }, "typeVersion": 1 }, { "id": "6bd8029e-fbf6-4422-b139-33e9e90206a4", "name": "Get SERP Data for SEO", "type": "n8n-nodes-dataforseo.dataForSeo", "position": [ 2544, 16 ], "parameters": { "os": "ios", "device": "mobile", "keyword": "={{ $json.keyword }}", "resource": "serp", "se_domain": "google.com", "language_name": "={{ $json.market_language }}", "location_name": "={{ $json.market_location }}" }, "credentials": { "dataForSeoApi": { "name": "" } }, "typeVersion": 1 }, { "id": "1600f493-f0bd-4250-85b3-9ba1c8fbcf23", "name": "Manual Start Trigger", "type": "n8n-nodes-base.manualTrigger", "position": [ 1424, 16 ], "parameters": {}, "typeVersion": 1 }, { "id": "471a2bd6-5840-460a-918b-0991e7f0c0a1", "name": "Parse Domain and Path", "type": "n8n-nodes-base.code", "position": [ 2768, 112 ], "parameters": { "jsCode": "// Flatten DataForSEO results \u2014 robust string parsing for n8n sandbox\n\nconst input = $input.first().json;\n\n// Support both:\n// 1) top-level object\n// 2) top-level array with first object\nconst root = Array.isArray(input) ? input[0] : input;\n\nconst items = root?.tasks?.[0]?.result?.[0]?.items || [];\n\n// \ud83d\udea8 THE FIX: Count the total items right here\nconst totalCount = items.length;\n\nreturn items.map(item => {\n const rawUrl = typeof item.url === 'string' ? item.url : '';\n\n let host = '';\n let domain = '';\n let path = '/';\n let query = '';\n let fragment = '';\n let parse_status = 'ok';\n\n try {\n if (rawUrl) {\n // Remove protocol\n let clean = rawUrl.replace(/^https?:\\/\\//i, '');\n\n // Extract fragment\n const hashIndex = clean.indexOf('#');\n if (hashIndex > -1) {\n fragment = clean.substring(hashIndex + 1);\n clean = clean.substring(0, hashIndex);\n }\n\n // Extract query\n const queryIndex = clean.indexOf('?');\n if (queryIndex > -1) {\n query = clean.substring(queryIndex + 1);\n clean = clean.substring(0, queryIndex);\n }\n\n // Split host/path\n const slashIndex = clean.indexOf('/');\n if (slashIndex > -1) {\n host = clean.substring(0, slashIndex);\n path = clean.substring(slashIndex) || '/';\n } else {\n host = clean;\n path = '/';\n }\n\n // Normalize domain\n domain = host.replace(/^www\\./i, '');\n } else {\n parse_status = 'empty-url';\n }\n } catch (e) {\n parse_status = 'parse-error';\n host = '';\n domain = '';\n path = '/';\n }\n\n return {\n json: {\n total_serp_results: totalCount, // \ud83d\udea8 Passed along for the LLM\n rank: item.rank_absolute || 0,\n rank_group: item.rank_group || 0,\n page: item.page || 0,\n serp_type: item.type || '',\n title: item.title || '',\n description: item.description || '',\n full_url: rawUrl,\n host,\n domain,\n path,\n query,\n fragment,\n parse_status\n }\n };\n});" }, "typeVersion": 2 }, { "id": "7c5c06cd-2f29-4910-b14b-a3b9d1f006e2", "name": "SQL-Based Data Merge", "type": "n8n-nodes-base.merge", "position": [ 2992, 16 ], "parameters": { "mode": "combineBySql", "query": "SELECT \n serp.rank, \n serp.title, \n serp.full_url, \n serp.domain,\n rules.rule_id, \n COALESCE(rules.goggle_id, (SELECT MAX(goggle_id) FROM input1)) AS goggle_id,\n rules.action AS existing_action, \n rules.strength AS existing_strength,\n CASE \n WHEN rules.rule_id IS NOT NULL THEN rules.action \n ELSE '\ud83d\udea8 NIEUW' \n END AS audit_status\nFROM input2 AS serp\nLEFT JOIN input1 AS rules\n ON serp.domain = rules.rule_target\nORDER BY serp.rank ASC;", "options": { "emptyQueryResult": "success" } }, "typeVersion": 3.2 }, { "id": "599409ea-4c5e-4c19-8558-bc741314e01a", "name": "Execute Postgres Query", "type": "n8n-nodes-base.postgres", "position": [ 2768, -80 ], "parameters": { "query": "SELECT \n g.id AS goggle_id, \n r.id AS rule_id,\n r.action, \n r.strength, \n r.target AS rule_target, \n r.path_pattern \nFROM goggles g\nLEFT JOIN goggle_rules r ON g.id = r.goggle_id\n-- Pakt automatisch je actieve Goggle (voorkomt problemen als je er later meer toevoegt)\nWHERE g.name = 'StuccoOS Vakkennis';", "options": {}, "operation": "executeQuery" }, "credentials": { "postgres": { "name": "" } }, "typeVersion": 2.6 }, { "id": "c3d718a5-6531-4609-92b7-1ece5eeceb0e", "name": "Use OpenAI GPT-4o", "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi", "position": [ 3840, 224 ], "parameters": { "model": { "__rl": true, "mode": "list", "value": "o4-mini", "cachedResultName": "o4-mini" }, "options": { "reasoningEffort": "medium" }, "responsesApiEnabled": false }, "credentials": { "openAiApi": { "name": "" } }, "typeVersion": 1.3 }, { "id": "308e6f50-cc17-44c8-9d35-19d21efb5ecf", "name": "Post to DataForSEO API", "type": "n8n-nodes-base.httpRequest", "position": [ 1872, 16 ], "parameters": { "url": "https://api.dataforseo.com/v3/dataforseo_labs/google/domain_intersection/live", "method": "POST", "options": {}, "jsonBody": "=[\n {\n \"target1\": \"{{ $json.target1 }}\",\n \"target2\": \"{{ $json.target2 }}\",\n \"language_code\": \"{{ $json.language_code }}\",\n \"location_code\": {{ $json.location_code }},\n \"intersections\": {{ $json.intersections }},\n \"include_serp_info\": {{ $json.include_serp_info }},\n \"limit\": {{ $json.limit }},\n \"order_by\": {{ JSON.stringify($json.order_by) }}\n }\n]", "sendBody": true, "sendHeaders": true, "specifyBody": "json", "authentication": "predefinedCredentialType", "headerParameters": { "parameters": [ { "name": "Content-Type", "value": "application/json" } ] }, "nodeCredentialType": "dataForSeoApi" }, "credentials": { "dataForSeoApi": { "name": "" } }, "typeVersion": 4.4 }, { "id": "8284e61e-04ab-437c-8cea-3d0d23c7a039", "name": "Process Intersection Results", "type": "n8n-nodes-base.code", "position": [ 2096, 16 ], "parameters": { "jsCode": "// 1. Fetch the items from the DataForSEO intersection results\nconst items = $input.first().json.tasks[0].result[0].items;\n\n// 2. Fetch the \"Dashboard\" settings from your env_ node\nconst env = $('Set Environment Variables').item.json;\n\n// 3. Map and Inject: Attach context to every keyword\nreturn items.map(item => {\n return {\n json: {\n keyword: item.keyword_data.keyword,\n search_volume: item.keyword_data.keyword_info.search_volume,\n intent: item.keyword_data.search_intent_info.main_intent,\n source: `Intersection: ${env.target1} vs ${env.target2}`,\n discovery_priority: item.keyword_data.keyword_info.search_volume > 1000 ? 'high' : 'medium',\n \n // Accessible env settings for downstream nodes (SERP & AI)\n market_location: env.Location,\n market_language: env.Language,\n market_language_code: env.language_code,\n // We pass this so the AI knows which industry it's auditing\n industry_context: env.industry_context || 'plastering, ceiling, and wall finishing'\n }\n };\n});" }, "typeVersion": 2 }, { "id": "d75960be-9826-4b54-915e-d96576012ad4", "name": "Run SQL Data Fetch", "type": "n8n-nodes-base.postgres", "position": [ 4800, 0 ], "parameters": { "query": "INSERT INTO goggle_rules (\n goggle_id, \n action, \n strength, \n target, \n target_type, \n path_pattern, \n confidence, \n site_type, \n reasoning, \n source, \n updated_at\n)\nVALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, NOW()\n)\nON CONFLICT (goggle_id, target, target_type, path_pattern)\nDO UPDATE SET\n action = CASE \n WHEN EXCLUDED.confidence = 'high' OR goggle_rules.confidence != 'high' THEN EXCLUDED.action \n ELSE goggle_rules.action \n END,\n strength = CASE \n WHEN EXCLUDED.confidence = 'high' OR goggle_rules.confidence != 'high' THEN EXCLUDED.strength \n ELSE goggle_rules.strength \n END,\n confidence = CASE \n WHEN EXCLUDED.confidence = 'high' OR goggle_rules.confidence != 'high' THEN EXCLUDED.confidence \n ELSE goggle_rules.confidence \n END,\n reasoning = CASE \n WHEN EXCLUDED.confidence = 'high' OR goggle_rules.confidence != 'high' THEN EXCLUDED.reasoning \n ELSE goggle_rules.reasoning \n END,\n updated_at = NOW();", "options": { "queryBatching": "transaction", "queryReplacement": "={{ [$json.goggle_id, $json.action, $json.strength, $json.target, $json.target_type, $json.path_pattern, $json.confidence, $json.site_type, $json.reasoning, $json.source] }}" }, "operation": "executeQuery" }, "credentials": { "postgres": { "name": "" } }, "typeVersion": 2.6 }, { "id": "c96d71ab-00f1-4bb8-884b-40285a3fa3bf", "name": "Information Extraction Agent", "type": "@n8n/n8n-nodes-langchain.informationExtractor", "position": [ 3776, 0 ], "parameters": { "text": "=### CRITICAL TASK\nYou are provided with a batch of EXACTLY {{ $json.item_count }} domains.\nYour output 'rules' array MUST contain EXACTLY {{ $json.item_count }} entries. \n\n### DOMAIN DATA\n{{ $json.domain_list_string }}", "options": { "systemPromptTemplate": "You are a strict SEO Domain Auditor specializing in the Dutch plastering, ceiling, and wall finishing industry.\n\nYOUR MISSION:\nYou will receive a list of domains and their context. You MUST extract and classify EVERY SINGLE DOMAIN into the `rules` array. Do not skip any.\n\nCLASSIFICATION RULES:\n\n1. SITE TYPE:\n- `specialist`: Manufacturers (e.g., Knauf, Gyproc), dedicated plastering contractors, industry associations (NOA), technical blogs.\n- `retailer`: DIY hardware stores (e.g., Gamma, Hornbach, Praxis), general construction webshops.\n- `aggregator`: Lead-generation platforms (e.g., Werkspot, Trustoo), quotation aggregators, social media, forums, and legal/news sites.\n\n2. ACTION & STRENGTH (Strict Integers 1-5):\n- `boost` (Strength 2-5): For specialists and manufacturers. Top manufacturers get 5. Local specialists get 2-4.\n- `downrank` (Strength 2-4): For retailers and aggregators.\n- `discard` (Strength 1): For social media (YouTube, Facebook), marketplaces (Marktplaats), and entirely unrelated sites (e.g., agriculture, law).\n\n3. BATHROOM NUANCE (Badkamers):\n- Do not automatically discard bathroom sites. If they focus on \"Beton Cir\u00e9\", \"Microcement\", or waterproof plastering, classify them as `specialist` and `boost`." }, "schemaType": "manual", "inputSchema": "{\n \"type\": \"object\",\n \"properties\": {\n \"rules\": {\n \"type\": \"array\",\n \"description\": \"An array containing the classification for EVERY domain provided in the input. You MUST NOT skip any domain.\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"domain\": {\n \"type\": \"string\",\n \"description\": \"The exact domain name from the input list (e.g., 'knauf.nl').\"\n },\n \"reasoning\": {\n \"type\": \"string\",\n \"description\": \"THINKING STEP: Write a concise 1-2 sentence analysis in Dutch. Explain WHY you chose the site_type and action. Look for specialist keywords versus generic retail or lead-gen signals.\"\n },\n \"site_type\": {\n \"type\": \"string\",\n \"enum\": [\"specialist\", \"retailer\", \"aggregator\"],\n \"description\": \"Categorize the domain. 'specialist' = plastering professionals, manufacturers, or technical knowledge bases. 'retailer' = DIY stores, general hardware webshops. 'aggregator' = lead-generation (Werkspot), social media, directories, forums.\"\n },\n \"action\": {\n \"type\": \"string\",\n \"enum\": [\"boost\", \"downrank\", \"discard\"],\n \"description\": \"'boost' = high quality/relevant specialists. 'downrank' = retailers or aggregators with mixed value. 'discard' = social media, unrelated noise, purely local promotion.\"\n },\n \"strength\": {\n \"type\": \"integer\",\n \"minimum\": 1,\n \"maximum\": 5,\n \"description\": \"Assign a strict integer weight from 1 to 5. 5 = top authority/manufacturer. 2-4 = specialist/retailer. 1 = discard or very low value. DO NOT use decimals.\"\n }\n },\n \"required\": [\"domain\", \"reasoning\", \"site_type\", \"action\", \"strength\"]\n }\n }\n },\n \"required\": [\"rules\"]\n}" }, "typeVersion": 1.2 }, { "id": "d22e292b-953e-4991-93ad-8683deb9e658", "name": "If Audit Status Pending", "type": "n8n-nodes-base.if", "position": [ 3216, 16 ], "parameters": { "options": {}, "conditions": { "options": { "version": 3, "leftValue": "", "caseSensitive": true, "typeValidation": "loose" }, "combinator": "and", "conditions": [ { "id": "f456723d-d00e-4756-b709-11a04fc80944", "operator": { "type": "string", "operation": "equals" }, "leftValue": "={{ $json.audit_status }}", "rightValue": "=\ud83d\udea8 NIEUW" } ] }, "looseTypeValidation": true }, "typeVersion": 2.3 }, { "id": "34e46dee-c0e5-42bc-b21d-45f3e3e99d39", "name": "Split Rules", "type": "n8n-nodes-base.splitOut", "position": [ 4496, 0 ], "parameters": { "options": {}, "fieldToSplitOut": "rules" }, "typeVersion": 1 }, { "id": "b1b78d85-0306-450c-9a55-57e8519ffcdb", "name": "Format Data for SQL", "type": "n8n-nodes-base.code", "position": [ 4640, 0 ], "parameters": { "jsCode": "const allItems = $input.all();\n\nconst goggleId = $('Execute Postgres Query').first().json.goggle_id;\n\n// 2. Harde stop als de ID mist (Geen fallback ID's meer!)\nif (!goggleId) {\n throw new Error(\"FATAL: Geen Goggle ID gevonden. Controleer of 'Run SQL Query' een geldige ID uit de database heeft gehaald.\");\n}\n\nreturn allItems.map(item => {\n const rule = item.json;\n \n // Normalisatie\n const domain = rule.domain ? String(rule.domain).toLowerCase().trim() : 'geen-domein';\n const actionStr = rule.action ? String(rule.action).toLowerCase().trim() : 'discard';\n const finalAction = ['boost', 'downrank', 'discard'].includes(actionStr) ? actionStr : 'discard';\n const allowedTypes = ['specialist', 'retailer', 'aggregator'];\n const finalType = allowedTypes.includes(rule.site_type) ? rule.site_type : 'aggregator';\n\n return {\n json: {\n goggle_id: goggleId, // 100% dynamisch\n target: domain,\n action: finalAction,\n strength: parseInt(rule.strength) || 1,\n target_type: 'site',\n path_pattern: '',\n confidence: 'low',\n site_type: finalType,\n reasoning: rule.reasoning || 'SERP Quick Scan',\n source: 'serp_audit'\n }\n };\n});" }, "typeVersion": 2 }, { "id": "9062350f-b486-474d-ac71-16d083280bc4", "name": "Optimize AI Prompts", "type": "n8n-nodes-base.code", "position": [ 3552, 0 ], "parameters": { "jsCode": "const items = $input.all();\n\n// 1. Safety Guard: If no new domains are found, stop the workflow here to save credits.\nif (items.length === 0) return [];\n\n// 2. Fetch goggle_id from the incoming data stream (SQL Merge passed it here)\nconst goggleId = items[0].json.goggle_id;\n\n// 3. Strip tokens: Only keep domain and title (the context)\nconst optimizedList = items.map(item => {\n return {\n domain: item.json.domain,\n context: item.json.title\n };\n});\n\n// 4. Create the Master AI Task\nreturn [{\n json: {\n goggle_id: goggleId, // Passed through for the final SORT node\n item_count: optimizedList.length,\n domain_list_string: JSON.stringify(optimizedList, null, 2)\n }\n}];" }, "typeVersion": 2 }, { "id": "2dc2ef4e-7788-40af-abd7-9cfc2b3a972c", "name": "Verify AI Process", "type": "n8n-nodes-base.code", "position": [ 4080, 0 ], "parameters": { "jsCode": "// 1. Specifically look back at the Information Extractor node, ignoring the SQL output\nconst aiOutput = $('Information Extraction Agent').first().json.output.rules || [];\n\n// 2. The rest of your logic remains the same\nconst expectedCount = $('Optimize AI Prompts').first().json.item_count;\nlet currentRetries = $input.first().json.retry_count || 0;\n\nconst isComplete = (aiOutput.length === expectedCount) || (currentRetries >= 3);\n\nreturn [{\n json: {\n rules: aiOutput,\n is_complete: isComplete,\n retry_count: currentRetries + 1,\n actual_count: aiOutput.length,\n expected_count: expectedCount\n }\n}];" }, "typeVersion": 2 }, { "id": "636a9198-0b64-4f97-bcc3-f7ec52f5636b", "name": "If Complete Status Is True", "type": "n8n-nodes-base.if", "position": [ 4304, 112 ], "parameters": { "options": {}, "conditions": { "options": { "version": 3, "leftValue": "", "caseSensitive": true, "typeValidation": "strict" }, "combinator": "and", "conditions": [ { "id": "c1d2e3f4", "operator": { "type": "boolean", "operation": "true", "singleValue": true }, "leftValue": "={{ $json.is_complete }}", "rightValue": "={{ true }}" } ] } }, "typeVersion": 2.3 }, { "id": "7ecd5d7b-1efe-4950-b54f-9e1455c82f77", "name": "Set Environment Variables", "type": "n8n-nodes-base.set", "position": [ 1600, 16 ], "parameters": { "options": {}, "assignments": { "assignments": [ { "id": "target1-id", "name": "target1", "type": "string", "value": "knauf.com" }, { "id": "target2-id", "name": "target2", "type": "string", "value": "gyproc.nl" }, { "id": "246737d1-726b-4405-984a-591b886890ea", "name": "Language", "type": "string", "value": "Dutch" }, { "id": "lang-id", "name": "language_code", "type": "string", "value": "nl" }, { "id": "5c4b2b6c-b537-40a3-8c42-bb857a8ce1bd", "name": "Location", "type": "string", "value": "Netherlands" }, { "id": "loc-id", "name": "location_code", "type": "number", "value": 2528 }, { "id": "inter-id", "name": "intersections", "type": "boolean", "value": true }, { "id": "serp-id", "name": "include_serp_info", "type": "boolean", "value": true }, { "id": "limit-id", "name": "limit", "type": "number", "value": 10 }, { "id": "order-id", "name": "order_by", "type": "array", "value": "={{ [\"keyword_data.keyword_info.search_volume,desc\"] }}" } ] } }, "typeVersion": 3.4 }, { "id": "09c19744-c6c6-49be-8e63-e11c89d970ef", "name": "Batch Process Items in Tens", "type": "n8n-nodes-base.splitInBatches", "position": [ 2304, 16 ], "parameters": { "options": {} }, "typeVersion": 3 }, { "id": "77aee493-aa2c-4856-87df-2d1e7889b050", "name": "Fetch All Goggle Entries", "type": "n8n-nodes-base.postgres", "position": [ 1888, 1072 ], "parameters": { "query": "SELECT \n *, \n '{{ $json.mode }}' as run_mode \nFROM {{ $json.table }}\nWHERE {{ $json.setting_L }} = '{{ $json.setting_R }}';", "options": { "queryReplacement": "" }, "operation": "executeQuery" }, "credentials": { "postgres": { "name": "" } }, "typeVersion": 2.6 }, { "id": "8cc09057-b8fd-451e-9ac8-124b098de558", "name": "Batch Process Goggles", "type": "n8n-nodes-base.splitInBatches", "position": [ 2544, 720 ], "parameters": { "options": {} }, "typeVersion": 3 }, { "id": "f5fcf5c2-5259-4633-a091-844a35a144b1", "name": "Check Gist ID Presence", "type": "n8n-nodes-base.if", "position": [ 2768, 640 ], "parameters": { "options": {}, "conditions": { "options": { "version": 1, "leftValue": "", "caseSensitive": true, "typeValidation": "strict" }, "combinator": "and", "conditions": [ { "id": "has-gist-id", "operator": { "type": "string", "operation": "notEmpty", "singleValue": true }, "leftValue": "={{ $json.github_gist_id }}", "rightValue": "" } ] } }, "typeVersion": 2 }, { "id": "8dcd8f20-d488-4c54-bc66-2b387a03d621", "name": "Patch GitHub Gist", "type": "n8n-nodes-base.httpRequest", "position": [ 3280, 544 ], "parameters": { "url": "=https://api.github.com/gists/{{ $json.github_gist_id }}", "method": "PATCH", "options": { "response": { "response": { "neverError": true } } }, "jsonBody": "={{ JSON.stringify({\n \"description\": $json.goggle_name || 'stephan-koning',\n \"files\": {\n \"stucco-search.goggle\": {\n \"content\": $json.full_goggle_text\n }\n }\n}) }}", "sendBody": true, "specifyBody": "json", "authentication": "predefinedCredentialType", "nodeCredentialType": "githubApi" }, "credentials": { "githubApi": { "name": "" } }, "typeVersion": 4.4 }, { "id": "d25c6416-4a7b-43e1-bd0b-f8dcb71d5d59", "name": "Post New GitHub Gist", "type": "n8n-nodes-base.httpRequest", "position": [ 2992, 736 ], "parameters": { "url": "https://api.github.com/gists", "method": "POST", "options": {}, "jsonBody": "={{ JSON.stringify({\n \"description\": $json.goggle_name || 'stephan-koning',\n \"files\": {\n \"stucco-search.goggle\": {\n \"content\": $json.full_goggle_text\n }\n }\n}) }}", "sendBody": true, "specifyBody": "json", "authentication": "predefinedCredentialType", "nodeCredentialType": "githubApi" }, "credentials": { "githubApi": { "name": "" } }, "typeVersion": 4.4 }, { "id": "117d4ada-345a-4e63-b973-e79fd944770d", "name": "Store Gist ID in Database", "type": "n8n-nodes-base.postgres", "position": [ 3280, 736 ], "parameters": { "table": { "__rl": true, "mode": "list", "value": "goggles", "cachedResultName": "goggles" }, "schema": { "__rl": true, "mode": "list", "value": "public" }, "columns": { "value": { "id": "={{ $('Check Gist ID Presence').item.json.goggle_id }}", "author": "StuccoOS", "github_gist_id": "={{ $json.id }}" }, "schema": [ { "id": "id", "type": "string", "display": true, "removed": false, "required": true, "displayName": "id", "defaultMatch": true, "canBeUsedToMatch": true }, { "id": "name", "type": "string", "display": true, "removed": false, "required": true, "displayName": "name", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "description", "type": "string", "display": true, "removed": false, "required": false, "displayName": "description", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "author", "type": "string", "display": true, "removed": false, "required": false, "displayName": "author", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "public", "type": "boolean", "display": true, "removed": false, "required": false, "displayName": "public", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "avatar", "type": "string", "display": true, "removed": false, "required": false, "displayName": "avatar", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "homepage", "type": "string", "display": true, "removed": false, "required": false, "displayName": "homepage", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "github_gist_id", "type": "string", "display": true, "removed": false, "required": false, "displayName": "github_gist_id", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "created_at", "type": "dateTime", "display": true, "removed": false, "required": false, "displayName": "created_at", "defaultMatch": false, "canBeUsedToMatch": true }, { "id": "updated_at", "type": "dateTime", "display": true, "removed": false, "required": false, "displayName": "updated_at", "defaultMatch": false, "canBeUsedToMatch": true } ], "mappingMode": "defineBelow", "matchingColumns": [ "id" ], "attemptToConvertTypes": false, "convertFieldsToString": false }, "options": {}, "operation": "update" }, "credentials": { "postgres": { "name": "" } }, "typeVersion": 2.6 }, { "id": "d786624d-9913-4e53-9247-bce5cb0d3558", "name": "Wait 3 Seconds", "type": "n8n-nodes-base.wait", "position": [ 3568, 736 ], "parameters": { "amount": 3 }, "typeVersion": 1.1 }, { "id": "d27fcdfb-5f97-4476-b6ea-3b5de03d089a", "name": "Batch Process Domains", "type": "n8n-nodes-base.splitInBatches", "position": [ 2336, 1488 ], "parameters": { "options": { "reset": false } }, "typeVersion": 3 }, { "id": "c503071b-4285-42de-b158-b0ea6f622f66", "name": "Trigger Every 6 Hours", "type": "n8n-nodes-base.scheduleTrigger", "position": [ 1440, 976 ], "parameters": { "rule": { "interval": [ { "field": "hours", "hoursInterval": 6 } ] } }, "typeVersion": 1.2 }, { "id": "0629e52a-8671-44a9-a474-3de3b81101b6", "name": "Trigger Weekly on Monday", "type": "n8n-nodes-base.scheduleTrigger", "position": [ 1440, 1168 ], "parameters": { "rule": { "interval": [ { "field": "weeks", "triggerAtDay": [ 1 ] } ] } }, "typeVersion": 1.2 }, { "id": "3bfcfd7e-7a05-4a1e-b1e5-13ff86b6b49c", "name": "Set Update Mode Fields", "type": "n8n-nodes-base.set", "position": [ 1664, 976 ], "parameters": { "options": {}, "assignments": { "assignments": [ { "id": "goggle-id", "name": "goggle_id", "type": "string", "value": "a7b8c9d0-1234-4567-89ab-cdef01234567" }, { "id": "mode", "name": "mode", "type": "string", "value": "update" }, { "id": "d25d9431-553d-44c5-837b-a1521bd3068a", "name": "table", "type": "string", "value": "v_goggle_full_preview" }, { "id": "4871acf7-8e1f-4e98-b1bc-f47a0db2fc69", "name": "setting_L", "type": "string", "value": "goggle_id" }, { "id": "42beceb3-7b7e-464f-a1c8-52a03c098a5e", "name": "setting_R", "type": "string", "value": "a7b8c9d0-1234-4567-89ab-cdef01234567" } ] } }, "typeVersion": 3.4 }, { "id": "5daf11e5-26a4-4902-be49-f27b956ced47", "name": "Switch by Mode", "type": "n8n-nodes-base.switch", "position": [ 2112, 1072 ], "parameters": { "rules": { "values": [ { "outputKey": "\ud83d\udd04 Update Gist Only", "conditions": { "options": { "version": 3, "leftValue": "", "caseSensitive": true, "typeValidation": "strict" }, "combinator": "and", "conditions": [ { "id": "is-update", "operator": { "type": "string", "operation": "equals" }, "leftValue": "={{ $json.run_mode }}", "rightValue": "update" } ] }, "renameOutput": true }, { "outputKey": "\ud83d\udd0d Run FireCrawl + AI", "conditions": { "options": { "version": 3, "leftValue": "", "caseSensitive": true, "typeValidation": "strict" }, "combinator": "and", "conditions": [ { "id": "is-discovery", "operator": { "type": "string", "operation": "equals" }, "leftValue": "={{ $json.run_mode }}", "rightValue": "discovery" } ] }, "renameOutput": true } ] }, "options": {} }, "typeVersion": 3.4 }, { "id": "d87a55d6-c648-4e40-a5b7-fc550f47262e", "name": "OpenAI GPT-4o", "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi", "position": [ 3088, 1296 ], "parameters": { "model": { "__rl": true, "mode": "list", "value": "gpt-4o", "cachedResultName": "gpt-4o" }, "options": { "temperature": 0.3 }, "builtInTools": {} }, "credentials": { "openAiApi": { "name": "" } }, "typeVersion": 1.3 }, { "id": "73c5bdd7-915d-40a3-a327-4cc619c457a8", "name": "Map Firecrawl Data", "type": "@mendable/n8n-nodes-firecrawl.firecrawl", "onError": "continueErrorOutput", "position": [ 2560, 1312 ], "parameters": { "url": "=https://{{ $json.target }}", "limit": 500, "timeout": 30000, "resource": "MapSearch", "operation": "map", "requestOptions": {} }, "credentials": { "firecrawlApi": { "name": "" } }, "executeOnce": false, "retryOnFail": true, "typeVersion": 1, "alwaysOutputData": false, "waitBetweenTries": 5000 }, { "id": "b7be2714-ff5e-4aa2-a9ad-46bbe3f0bc03", "name": "Set Discovery Mode Fields", "type": "n8n-nodes-base.set", "position": [ 1664, 1168 ], "parameters": { "options": {}, "assignments": { "assignments": [ { "id": "goggle-id", "name": "goggle_id", "type": "string", "value": "a7b8c9d0-1234-4567-89ab-cdef01234567" }, { "id": "mode", "name": "mode", "type": "string", "value": "discovery" }, { "id": "d25d9431-553d-44c5-837b-a1521bd3068a", "name": "table", "type": "string", "value": "goggle_rules" }, { "id": "4871acf7-8e1f-4e98-b1bc-f47a0db2fc69", "name": "setting_L", "type": "string", "value": "confidence" }, { "id": "42beceb3-7b7e-464f-a1c8-52a03c098a5e", "name": "setting_R", "type": "string", "value": "=low" } ] } }, "typeVersion": 3.4 }, { "id": "15a1af19-e2b2-4189-8ce1-85f8b8720a03", "name": "Parse Domain Structures", "type": "n8n-nodes-base.code", "position": [ 2784, 1072 ], "parameters": { "jsCode": "const items = $input.all();\n\n// Safety check for empty data\nif (!items || items.length === 0) {\n return [{ json: { domain: 'unknown', error: 'No data' } }];\n}\n\n// Helper function (buiten de loop gehouden voor snelheid)\nconst parseUrlManual = (urlStr) => {\n try {\n let clean = String(urlStr || '');\n if (clean.startsWith('http://')) clean = clean.substring(7);\n if (clean.startsWith('https://')) clean = clean.substring(8);\n if (clean.startsWith('www.')) clean = clean.substring(4);\n let parts = clean.split('/');\n let hostname = parts[0] || 'unknown';\n let pathname = '/' + parts.slice(1).join('/').split('?')[0].split('#')[0];\n if (pathname.length > 1 && pathname.endsWith('/')) pathname = pathname.slice(0, -1);\n if (!pathname) pathname = '/';\n return { hostname, pathname, original: urlStr };\n } catch (e) { \n return { hostname: 'error', pathname: '/', original: urlStr }; \n }\n};\n\nconst VAKKENNIS_KWS = ['expertise', 'trainingen', 'technisch', 'bestek', 'richtlijn', 'instructie', 'kennis', 'bim', 'stappenplan', 'projecten'];\nconst RETAIL_KWS = ['/product/', '/product-categorie/', '/shop/', '/merk/', 'winkelwagen', 'checkout', 'cart', 'bestellen'];\n\n// \ud83d\udea8 BATCH LOOP START: Verwerk ELK item in de batch apart\nreturn items.map((item) => {\n const mapData = item.json;\n const urlsToProcess = mapData.links || [];\n const totalScanned = urlsToProcess.length;\n\n // \ud83d\udea8 ROCKSOLID FIX: Haal goggle_id direct uit de input stroom of gebruik fallback. Geen storingsgevoelige $('Node') references meer!\n const goggleId = mapData.goggle_id || \"a7b8c9d0-1234-4567-89ab-cdef01234567\";\n \n const rawDomain = urlsToProcess[0]?.url || mapData.url || mapData.target || '';\n const domain = parseUrlManual(rawDomain).hostname;\n\n let pathCounts = {};\n let retailHits = 0;\n let cleanUrlsLog = [];\n\n urlsToProcess.forEach(urlItem => {\n const urlString = typeof urlItem === 'object' ? urlItem.url : urlItem;\n if (!urlString) return;\n \n const parsed = parseUrlManual(urlString);\n const pathLower = parsed.pathname.toLowerCase();\n \n const isJunk = pathLower.includes('sitemap') || pathLower.includes('.xml') || pathLower.includes('/tag/');\n if (isJunk) return;\n\n if (cleanUrlsLog.length < 100) cleanUrlsLog.push(parsed.original);\n \n if (RETAIL_KWS.some(kw => pathLower.includes(kw))) retailHits++;\n\n const segments = parsed.pathname.split('/').filter(s => s.length > 0);\n let currentPath = '';\n \n for (let i = 0; i < Math.min(segments.length, 3); i++) {\n currentPath += '/' + segments[i];\n if (!pathCounts[currentPath]) {\n pathCounts[currentPath] = { count: 0, vakkennisHits: 0, samples: [] };\n }\n pathCounts[currentPath].count++;\n \n VAKKENNIS_KWS.forEach(kw => {\n if (pathLower.includes(kw)) pathCounts[currentPath].vakkennisHits++;\n });\n \n if (pathCounts[currentPath].samples.length < 3) {\n pathCounts[currentPath].samples.push(parsed.original);\n }\n }\n });\n\n // Calculate ratios safely (prevent NaN if totalScanned is 0)\n const retailRatio = totalScanned > 0 ? (retailHits / totalScanned) : 0;\n const isRetailer = retailRatio > 0.2 || retailHits > 20;\n\n let powerFolders = Object.entries(pathCounts)\n .map(([path, data]) => {\n const isExpertise = data.vakkennisHits > 0 && (data.vakkennisHits / data.count >= 0.1);\n return { path, count: data.count, isExpertise, examples: data.samples };\n })\n .filter(f => f.count >= 3)\n .sort((a, b) => b.count - a.count);\n\n const finalFolders = powerFolders.filter((folder, index, self) => {\n const isRetailFolder = RETAIL_KWS.some(kw => folder.path.toLowerCase().includes(kw));\n const isRoot = folder.count > (totalScanned * 0.9) && folder.path.split('/').length <= 2 && !folder.isExpertise && !isRetailFolder;\n \n const hasSpecificChild = self.some(other => \n other.path !== folder.path && other.path.startsWith(folder.path + '/') && other.count >= (folder.count * 0.8)\n );\n return !isRoot && !hasSpecificChild;\n }).slice(0, 15);\n\n const retailRatioPercent = Math.round(retailRatio * 100);\n\n // auditText wordt nu netjes IN de loop gedeclareerd en toegewezen\n const auditText = [\n `## DOMAIN AUDIT REQUEST: ${domain}`,\n `Totaal URLs gescand: ${totalScanned}`,\n `Retail/Product URL ratio: ${retailRatioPercent}%`,\n `Harde Retailer Indicatie: ${isRetailer ? 'JA (Grote catalogus/shop gevonden)' : 'NEE'}`,\n ``,\n `### Power Folders (Inhoudsanalyse):`,\n ...finalFolders.map(f => `- ${f.path} (${f.count} pagina's) ${f.isExpertise ? '[VAKKENNIS GEDETECTEERD]' : ''}\\n Voorbeelden: ${f.examples.join(' | ')}`)\n ].join('\\n');\n\n // Stuur het resultaat voor dit specifieke domein terug\n return { \n json: { \n domain, \n goggle_id: goggleId, \n audit_text: auditText, \n debug_log: { \n scanned_total: totalScanned, \n retail_hits: retailHits, \n is_retailer: isRetailer, \n power_folders: finalFolders \n } \n } \n };\n});" }, "executeOnce": true, "typeVersion": 2 }, { "id": "dd735944-5281-4e36-afe4-1b29b55da9ac", "name": "Build Database Rules", "type": "n8n-nodes-base.code", "position": [ 3328, 1072 ], "parameters": { "jsCode": "// Pak ALLE items die van de AI-node komen\nconst allItems = $input.all();\n\nreturn allItems.map((item, index) => {\n const ai = item.json.output || {};\n \n // \ud83d\udea8 FIX: We gebruiken hier de EXACTE naam van de node in je workflow\n const upstream = $items('Parse Domain Structures')[index].json;\n\n \n return {\n json: {\n // We zorgen dat er altijd een ID is, anders pakt hij de default\n goggle_id: upstream.goggle_id || \"a7b8c9d0-1234-4567-89ab-cdef01234567\",\n action: ai.action || 'boost',\n strength: parseInt(ai.strength) || 3,\n target: upstream.domain, \n target_type: 'site',\n path_pattern: '',\n confidence: 'high',\n site_type: ai.site_type || 'unknown',\n reasoning: ai.reasoning || '',\n source: 'deep_audit'\n }\n };\n});" }, "typeVersion": 2 }, { "id": "0e838cf2-260b-4424-90d7-b1a3b8ef9e13", "name": "Insert Rule into Database", "type": "n8n-nodes-base.postgres", "position": [ 3520, 1072 ], "parameters": { "query": "INSERT INTO goggle_rules (\n goggle_id,\n action,\n strength,\n target,\n target_type,\n path_pattern, -- Keep this!\n confidence,\n site_type,\n reasoning,\n source,\n updated_at\n)\nVALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, NOW()\n)\nON CONFLICT (goggle_id, target, target_type, path_pattern)\nDO UPDATE SET\n action = EXCLUDED.action,\n strength = EXCLUDED.strength,\n confidence = EXCLUDED.confidence,\n site_type = EXCLUDED.site_type,\n reasoning = EXCLUDED.reasoning,\n source = EXCLUDED.source,\n updated_at = NOW();", "options": { "queryBatching": "transaction", "queryReplacement": "={{ [\n $json.goggle_id || \"\", \n $json.action || \"boost\", \n $json.strength || 3, \n $json.target || \"unknown\", \n $json.target_type || \"site\", \n $json.path_pattern || \"\", // The Fix: if empty, send empty string\n $json.confidence || \"low\", \n $json.site_type || \"unknown\", \n $json.reasoning || \"\", \n $json.source || \"audit\"\n] }}" }, "operation": "executeQuery" }, "credentials": { "postgres": { "name": "" } }, "typeVersion": 2.6 }, { "id": "d8c5a87e-31ee-4058-b4e1-5f782ffa73a2", "name": "Information Extractor Agent", "type": "@n8n/n8n-nodes-langchain.informationExtractor", "position": [ 3008, 1072 ], "parameters": { "text": "={{ $json.audit_text }}", "options": { "systemPromptTemplate": "Je bent een Senior Domein-Auditor voor de Nederlandse stukadoors-, plafond-, wandafwerking- en afbouwsector.\n\nJe krijgt een domein-audit tekst met signalen zoals:\n- domeinnaam\n- type site\n- webshop-indicatoren\n- Power Folders\n- URL-voorbeelden\n- technische content\n- projectcases\n- vakkennis-signalen\n\nJOUW DOEL:\nClassificeer het domein streng, consistent en bruikbaar voor ranking-doeleinden binnen de afbouwsector.\n\n====================\nSTAP 1 \u2014 SITE TYPE\n====================\n\nKies exact \u00e9\u00e9n `site_type` uit:\n- manufacturer\n- specialist\n- retailer\n- aggregator\n- unknown\n\nDefinities:\n\n1. manufacturer\nEigen merk, fabrikant of systeemleverancier met technische documentatie, datasheets, bestekteksten, systeemopbouwen, certificaten, downloads, BIM/CAD, productbladen of duidelijke verwerkingsinstructies.\n\n2. specialist\nUitvoerend specialist of nichebedrijf met aantoonbare vakkennis in stucwerk, afbouw, plafonds, wandafwerking, decoratieve afwerking, buitengevelafwerking, akoestiek of verwante uitvoering. Signalen: projectcases, eigen expertise, niche-diensten, verwerkingsadvies, probleemoplossing.\n\n3. retailer\nWebshop, bouwmarkt of productcatalogus-site met shop-first signalen zoals prijzen, categoriepagina\u2019s, filters, winkelwagen, brede productverkoop of bestelgerichte architectuur.\n\n4. aggregator\nVergelijker, bedrijvengids, offerteplatform, lead-gen platform, directory of verzamelsite zonder eigen diepe vakkennis of eigen technische autoriteit.\n\n5. unknown\nOnvoldoende bewijs, te weinig inhoud, onduidelijk, mixed signals zonder duidelijke hoofdidentiteit, of een slechte/lege scrape.\n\n====================\nSTAP 2 \u2014 ACTION\n====================\n\nKies exact \u00e9\u00e9n `action`:\n- boost\n- downrank\n- discard\n\nRegels:\n- manufacturer => altijd boost\n- specialist => meestal boost\n- retailer => altijd downrank\n- aggregator => meestal downrank, soms discard\n- unknown => meestal discard\n\nGebruik `discard` als de site:\n- ruis is\n- nauwelijks relevant is voor afbouw/stukadoors/plafonds/wanden\n- geen eigen vakkennis heeft\n- social/forum/UGC-achtig is\n- of te weinig betrouwbare signalen bevat\n\n====================\nSTAP 3 \u2014 STRENGTH\n====================\n\nKies een integer 1 t/m 5.\n\nHarde regels:\n- manufacturer => alleen 4 of 5\n- specialist => alleen 2, 3 of 4\n- retailer => alleen 1, 2 of 3\n- aggregator => alleen 1 of 2\n- unknown => alleen 1\n- discard => altijd 1\n- boost => nooit 1\n- strength moet logisch passen bij site_type en action\n\nInterpretatie:\n- 5 = uitzonderlijk sterke fabrikant / top technische autoriteit\n- 4 = sterke fabrikant of sterke nichespecialist\n- 3 = degelijke specialist of relevante retailer met enkele nuttige kenniszones\n- 2 = beperkte specialistische waarde / zwakke maar nog relevante site\n- 1 = discard, ruis, unknown of zeer lage waarde\n\n====================\nSTAP 4 \u2014 CONFIDENCE\n====================\n\nKies exact \u00e9\u00e9n `confidence`:\n- high\n- medium\n- low\n\nRegels:\n- high = duidelijke en consistente signalen, weinig twijfel\n- medium = redelijk duidelijk maar niet volledig sluitend\n- low = zwakke scrape, gemengde signalen of onvoldoende bewijs\n\nGebruik high alleen als de input echt duidelijke bewijsstukken bevat.\n\n===========================\nSTAP 5 \u2014 PROMISING PATHS\n===========================\n\nVul `promising_paths` als simpele string in.\n\nRegels:\n- Neem alleen paden op die letterlijk in de input staan.\n- Neem alleen paden op die duidelijke vakkennis/expertise bevatten.\n- Gebruik vooral promising_paths wanneer het domein een downrank krijgt, maar sommige kennisfolders toch waardevol zijn.\n- Als de hele site al een boost krijgt, vul meestal \"geen\" in.\n- Als er geen duidelijke uitzonderingspaden zijn, vul \"geen\" in.\n- Gebruik komma-gescheiden paden, bijvoorbeeld: \"/projecten, /downloads\"\n- Geef NOOIT een array.\n- Verzin nooit paden die niet in de input staan.\n- Neem geen shop-, cart-, checkout-, contact-, login-, account- of pure productlisting-paden op tenzij de input expliciet aantoont dat ze technische expertise bevatten.\n- Geef alleen exacte foldernamen of paden terug, geen uitleg in dit veld.\n\n===========================\nSTAP 6 \u2014 EVIDENCE SIGNALS\n===========================\n\nVul `evidence_signals` als simpele string in.\n\nRegels:\n- Geef 2 tot 5 concrete signalen uit de input.\n- Gebruik komma-gescheiden termen of korte frases.\n- Alleen signalen die echt in de input ondersteund worden.\n- Niet te algemeen formuleren.\n- Voorbeelden:\n - \"datasheets, technische documentatie, systeemopbouw, geen webshop\"\n - \"winkelwagen, productcategorieen, prijzen, brede catalogus\"\n - \"projectcases, niche-diensten, eigen foto's, verwerkingsadvies\"\n\n====================\nSTAP 7 \u2014 REASONING\n====================\n\nGeef `reasoning` in maximaal 2 korte zinnen in het Nederlands.\nNoem:\n1. waarom dit site_type en action logisch zijn\n2. waarom promising_paths wel of niet gekozen zijn\n\n====================\nBESLISLOGICA SAMENGEVAT\n====================\n\nGebruik deze logica strikt:\n- manufacturer + duidelijke technische autoriteit => boost + 4/5 + meestal high confidence\n- specialist + echte niche/vakkennis => boost + 2/3/4\n- retailer + webshopstructuur => downrank + 1/2/3\n- aggregator zonder eigen expertise => downrank of discard + 1/2\n- unknown of slechte scrape => discard + 1 + low confidence\n\n====================\nHARDE OUTPUTREGELS\n====================\n\n1. Geef UITSLUITEND raw JSON terug.\n2. GEEN markdown.\n3. GEEN backticks.\n4. Begin direct met { en eindig met }.\n5. `promising_paths` moet altijd een simpele string zijn.\n6. `evidence_signals` moet altijd een simpele string zijn.\n7. Gebruik exact de toegestane labels.\n8. Geen extra velden toevoegen." }, "attributes": { "attributes": [ { "name": "site_type", "required": true, "description": "Kies exact \u00e9\u00e9n: manufacturer, specialist, retailer, aggregator, unknown." }, { "name": "action", "required": true, "description": "Kies exact \u00e9\u00e9n: boost, downrank, discard." }, { "name": "strength", "type": "number", "required": true, "description": "Integer 1-5. manufacturer alleen 4-5, specialist alleen 2-4, retailer alleen 1-3, aggregator alleen 1-2, unknown alleen 1. discard altijd 1. boost nooit 1." }, { "name": "confidence", "required": true, "description": "Kies exact \u00e9\u00e9n: high, medium, low." }, { "name": "promising_paths", "required": true, "description": "Simpele string met komma-gescheiden paden die letterlijk in de input staan en duidelijke vakkennis bevatten. Geen array. Gebruik 'geen' als er geen uitzonderingspaden zijn." }, { "name": "evidence_signals", "required": true, "description": "Korte string met 2-5 concrete signalen uit de input die de classificatie onderbouwen, gescheiden door komma's. Bijvoorbeeld: 'datasheets, technische documentatie, geen webshop'." }, { "name": "reasoning", "required": true, "description": "Max 2 korte zinnen in het Nederlands. Leg uit waarom dit site_type/action is gekozen en waarom promising_paths wel of niet zijn geselecteerd." } ] } }, "typeVersion": 1.2 }, { "id": "2010ce1b-4e4d-49e0-b354-6f201fb61115", "name": "Run Secondary SQL Query", "type": "n8n-nodes-base.postgres", "position": [ 3536, 1328 ], "parameters": { "query": "INSERT INTO goggle_audit_trail (\n goggle_id, \n target, \n site_type, \n full_log\n)\nVALUES (\n $1, $2, $3, $4\n);", "options": { "queryReplacement": "={{ [\n $('Batch Process Domains').item.json.goggle_id, \n $('Batch Process Domains').item.json.target,\n 'unknown',\n $json \n] }}" }, "operation": "executeQuery" }, "credentials": { "postgres": { "name": "" } }, "typeVersion": 2.6 }, { "id": "9b1075bf-f107-4136-b9e7-2cebfdcdcab0", "name": "Execute JavaScript Code", "type": "n8n-nodes-base.code", "position": [ 3824, 1456 ], "parameters": { "jsCode": "const items = $input.all();\n\nreturn items.map(item => {\n const errorObj = item.json;\n\n // 1. DYNAMIC DATA RETRIEVAL\n // We look for the data in the current item (if passed through)\n // or via the pairedItem (the data from 'Process Domains in Batches')\n const goggleId = item.json.goggle_id || item.pairedItem?.json?.goggle_id;\n const targetUrl = item.json.target || item.json.url || item.pairedItem?.json?.target || \"unknown-url\";\n\n // 2. CLEAN DOMAIN EXTRACTION\n const domain = targetUrl\n .replace('https://', '')\n .replace('http://', '')\n .split('/')[0];\n\n return {\n json: {\n // 3. NO HARDCODED UUIDs\n goggle_id: goggleId,\n target: domain,\n site_type: 'error_timeout', \n full_log: { \n error: true, \n message: errorObj.message || \"Execution Error\", \n details: errorObj.errorDetails || errorObj || \"No further details\"\n }\n }\n };\n});" }, "executeOnce": true, "typeVersion": 2 }, { "id": "b72282fe-f8cd-4c11-b18e-90f5bd030adf", "name": "Log Error Postgres", "type": "n8n-nodes-base.postgres", "position": [ 4048, 1456 ], "parameters": { "query": "INSERT INTO goggle_audit_trail (\n goggle_id, \n target, \n site_type, \n full_log\n)\nVALUES (\n $1, $2, $3, $4\n);", "options": { "queryReplacement": "={{ [\n $json.goggle_id, \n $json.target, \n $json.site_type, \n $json.full_log \n] }}" }, "operation": "executeQuery" }, "credentials": { "postgres": { "name": "" } }, "typeVersion": 2.6 }, { "id": "d372989c-7a78-43ef-8074-572d2463f9c8", "name": "Wait 5 Seconds", "type": "n8n-nodes-base.wait", "position": [ 4272, 1360 ], "parameters": {}, "typeVersion": 1.1 }, { "id": "3ac4d480-c0b0-44b0-bcaa-a727fe1ef199", "name": "Merge", "type": "n8n-nodes-base.merge", "position": [ 3888, 1280 ], "parameters": { "mode": "combine", "options": {}, "combineBy": "combineAll" }, "typeVersion": 3.2 }, { "id": "b9e000d3-7fca-4540-b7fb-97328070fb7f", "name": "Sticky Note8", "type": "n8n-nodes-base.stickyNote", "position": [ 880, 768 ], "parameters": { "width": 480, "height": 448, "content": "## Goggle - Power Folder Pipeline\n\n### How it works\n\n1. Schedules determine the mode of operation (update or discovery).\n2. Retrieves entries from the database and routes them based on mode.\n3. Processes entries and interacts with GitHub Gists.\n4. Executes additional operations based on analyzed data.\n5. Formats and adds new rules to the database based on AI analysis.\n\n### Setup steps\n\n- [ ] Set up GitHub API credentials for Gist operations.\n- [ ] Ensure the Firecrawl API is accessible for domain mapping.\n\n### Customization\n\nCustomize the trigger schedules to fit the specific timing requirements of your use case." }, "typeVersion": 1 }, { "id": "ee90f897-2562-4ff0-80ec-988171521202", "name": "Sticky Note9", "type": "n8n-nodes-base.stickyNote", "position": [ 1392, 768 ], "parameters": { "color": 7, "height": 544, "content": "## Mode configuration triggers\n\nSchedules triggering either update mode or discovery mode." }, "typeVersion": 1 }, { "id": "b1db47a1-ef33-4ed5-b329-eafa017958f2", "name": "Sticky Note10", "type": "n8n-nodes-base.stickyNote", "position": [ 1840, 928 ], "parameters": { "color": 7, "width": 416, "height": 304, "content": "## Retrieve and route entries\n\nFetches all relevant entries from the database and routes them based on mode." }, "typeVersion": 1 }, { "id": "0c6ad6da-081f-45be-a475-9f055d225ec8", "name": "Sticky Note11", "type": "n8n-nodes-base.stickyNote", "position": [ 2496, 432 ], "parameters": { "color": 7, "width": 1216, "height": 464, "content": "## Process Goggle data\n\nHandles Goggle entries, updating or creating Gists as needed." }, "typeVersion": 1 }, { "id": "4cc46dc0-d27c-479a-bd57-5f40b901232e", "name": "Sticky Note12", "type": "n8n-nodes-base.stickyNote", "position": [ 2288, 944 ], "parameters": { "color": 7, "width": 1440, "height": 688, "content": "## Domain data processing\n\nProcesses domain data in batches, analyzes structure, and decides actions with AI assistance." }, "typeVersion": 1 }, { "id": "3efa573a-9552-413a-bcbb-0b58a79ac3c2", "name": "Sticky Note13", "type": "n8n-nodes-base.stickyNote", "position": [ 3776, 1152 ], "parameters": { "color": 7, "width": 640, "height": 464, "content": "## Execute and handle SQL and errors\n\nRuns secondary queries and manages errors, ensuring continuity." }, "typeVersion": 1 }, { "id": "0ab4db5d-660d-4ecf-918b-8aee97674a9c", "name": "Step 0 Documentation1", "type": "n8n-nodes-base.stickyNote", "position": [ 272, -448 ], "parameters": { "color": "#FFE431", "width": 520, "height": 548, "content": "\n### How it works\n\n\nExecute this node MANUALLY ONCE before starting any discovery or update loops. It is safe to re-run for maintenance.\n\n1. Tables: Automatically creates the core infrastructure (goggles, goggle_rules, audit_trail).\n2. Logic: Creates the SQL View that formats your data into valid Brave Goggle code.\n\n### Setup Steps\n- [ ] Postgres: Connect your database credentials to this node.\n- [ ] Identity: Verify the Goggle Name in the SQL script.\n- [ ] API Keys: Ensure DataForSEO and OpenAI are set in downstream nodes." }, "typeVersion": 1 }, { "id": "e557794a-9256-4bbe-a027-59dda3da897d", "name": "Setup Goggle Identity", "type": "n8n-nodes-base.set", "position": [ 336, -144 ], "parameters": { "options": {}, "assignments": { "assignments": [ { "id": "g-id", "name": "goggle_id", "type": "string", "value": "a7b8c9d0-1234-4567-89ab-cdef01234567" }, { "id": "g-name", "name": "goggle_name", "type": "string", "value": "StuccoOS Vakkennis" }, { "id": "g-desc", "name": "goggle_description", "type": "string", "value": "Boost technical plastering sources, filter lead-gen noise" }, { "id": "g-author", "name": "goggle_author", "type": "string", "value": "StuccoOS" }, { "id": "g-public", "name": "goggle_public", "type": "boolean", "value": true } ] } }, "typeVersion": 3.4 }, { "id": "02ed549d-25f8-4f28-9ed2-540bd1b4a222", "name": "Execute SQL Query", "type": "n8n-nodes-base.postgres", "position": [ 544, -144 ], "parameters": { "query": "-- 1. CORE TABLES\nCREATE TABLE IF NOT EXISTS goggles (\n id UUID DEFAULT gen_random_uuid() PRIMARY KEY,\n name VARCHAR(255) NOT NULL,\n description TEXT,\n author VARCHAR(255) DEFAULT 'StuccoOS',\n public BOOLEAN DEFAULT true,\n avatar TEXT,\n homepage TEXT,\n github_gist_id VARCHAR(255),\n created_at TIMESTAMPTZ DEFAULT NOW(),\n updated_at TIMESTAMPTZ DEFAULT NOW()\n);\n\nCREATE TABLE IF NOT EXISTS goggle_rules (\n id UUID DEFAULT gen_random_uuid() PRIMARY KEY,\n goggle_id UUID NOT NULL REFERENCES goggles(id) ON DELETE CASCADE,\n action VARCHAR(20) NOT NULL CHECK (action IN ('boost', 'downrank', 'discard')),\n strength INTEGER CHECK (strength >= 1 AND strength <= 10),\n target VARCHAR(255),\n target_type VARCHAR(20) DEFAULT 'site' CHECK (target_type IN ('site', 'pattern')),\n path_pattern VARCHAR(255),\n confidence VARCHAR(20) DEFAULT 'medium' CHECK (confidence IN ('high', 'medium', 'low')),\n reasoning TEXT,\n site_type TEXT,\n source TEXT,\n created_at TIMESTAMPTZ DEFAULT NOW(),\n updated_at TIMESTAMPTZ DEFAULT NOW()\n);\n\nCREATE TABLE IF NOT EXISTS workflow_errors (\n id UUID DEFAULT gen_random_uuid() PRIMARY KEY,\n workflow_name TEXT,\n node_name TEXT,\n error_message TEXT,\n input_payload JSONB,\n created_at TIMESTAMPTZ DEFAULT NOW()\n);\n\nCREATE TABLE IF NOT EXISTS goggle_audit_trail (\n id UUID DEFAULT gen_random_uuid() PRIMARY KEY,\n goggle_id UUID REFERENCES goggles(id) ON DELETE CASCADE,\n target VARCHAR(255),\n site_type VARCHAR(50),\n full_log JSONB,\n created_at TIMESTAMPTZ DEFAULT NOW()\n);\n\n-- 2. INDEXES & CONSTRAINTS\nCREATE INDEX IF NOT EXISTS idx_goggle_rules_goggle_id ON goggle_rules(goggle_id);\nCREATE INDEX IF NOT EXISTS idx_goggle_rules_action ON goggle_rules(action);\nCREATE INDEX IF NOT EXISTS idx_goggle_rules_target ON goggle_rules(target);\nCREATE INDEX IF NOT EXISTS idx_goggle_rules_composite ON goggle_rules(goggle_id, target);\nCREATE INDEX IF NOT EXISTS idx_audit_target ON goggle_audit_trail(target);\n\n-- Drop old constraints if they exist and create the standardized one\nALTER TABLE goggle_rules DROP CONSTRAINT IF EXISTS unique_rule_identity;\nALTER TABLE goggle_rules DROP CONSTRAINT IF EXISTS unique_goggle_rule_identity;\nALTER TABLE goggle_rules \nADD CONSTRAINT unique_goggle_rule_identity \nUNIQUE NULLS NOT DISTINCT (goggle_id, target, target_type, path_pattern);\n\n-- 3. VIEWS\nDROP VIEW IF EXISTS v_goggle_full_preview CASCADE;\nCREATE OR REPLACE VIEW v_goggle_full_preview AS\nWITH rule_stats AS (\n SELECT \n goggle_id,\n COUNT(*) as total_rules,\n COUNT(*) FILTER (WHERE action = 'boost') as boost_count,\n COUNT(*) FILTER (WHERE action = 'downrank') as downrank_count,\n COUNT(*) FILTER (WHERE action = 'discard') as discard_count,\n ROUND(AVG(strength)::numeric, 1) as avg_strength,\n MAX(updated_at) as last_updated\n FROM goggle_rules\n GROUP BY goggle_id\n),\nformatted_rules AS (\n SELECT \n gr.goggle_id,\n string_agg(\n CASE \n WHEN gr.target_type = 'site' AND gr.path_pattern IS NOT NULL AND gr.action = 'discard' THEN '*' || gr.path_pattern || '*$discard,site=' || gr.target || ',inurl'\n WHEN gr.target_type = 'site' AND gr.path_pattern IS NOT NULL THEN '*' || gr.path_pattern || '*$' || gr.action || '=' || gr.strength || ',site=' || gr.target || ',inurl'\n WHEN gr.target_type = 'site' AND gr.action = 'discard' THEN '$discard,site=' || gr.target\n WHEN gr.target_type = 'site' THEN '$' || gr.action || '=' || gr.strength || ',site=' || gr.target\n WHEN gr.target_type = 'pattern' AND gr.action = 'discard' THEN '*' || gr.path_pattern || '*$discard,inurl'\n WHEN gr.target_type = 'pattern' THEN '*' || gr.path_pattern || '*$' || gr.action || '=' || gr.strength || ',inurl'\n ELSE '! unknown rule format'\n END, E'\\n'\n ORDER BY \n CASE gr.action WHEN 'boost' THEN 1 WHEN 'downrank' THEN 2 WHEN 'discard' THEN 3 END,\n gr.strength DESC NULLS LAST, gr.target ASC\n ) as rules_text\n FROM goggle_rules gr\n GROUP BY gr.goggle_id\n)\nSELECT \n g.id as goggle_id, g.name as goggle_name, g.description, g.author, g.public,\n g.github_gist_id, g.homepage, g.avatar,\n COALESCE(rs.total_rules, 0) as total_rules,\n COALESCE(rs.boost_count, 0) as boost_count,\n COALESCE(rs.downrank_count, 0) as downrank_count,\n COALESCE(rs.discard_count, 0) as discard_count,\n COALESCE(rs.avg_strength, 0) as avg_strength,\n rs.last_updated,\n CONCAT(\n '! name: ', g.name, E'\\n',\n '! description: ', COALESCE(g.description, 'Custom search filter'), E'\\n',\n '! author: ', COALESCE(g.author, 'StuccoOS'), E'\\n',\n '! public: ', CASE WHEN g.public THEN 'true' ELSE 'false' END, E'\\n',\n CASE WHEN g.avatar IS NOT NULL THEN '! avatar: ' || g.avatar || E'\\n' ELSE '' END,\n CASE WHEN g.homepage IS NOT NULL THEN '! homepage: ' || g.homepage || E'\\n' ELSE '' END,\n E'\\n! === RULES (', COALESCE(rs.total_rules, 0)::text, ') ===\\n\\n',\n COALESCE(fr.rules_text, '! No rules defined yet'),\n E'\\n\\n! Generated: ', NOW()::text\n ) as full_goggle_text,\n COALESCE(fr.rules_text, '') as rules_only\nFROM goggles g\nLEFT JOIN rule_stats rs ON rs.goggle_id = g.id\nLEFT JOIN formatted_rules fr ON fr.goggle_id = g.id;\n\n-- 4. INSERT FIRST GOGGLE DYNAMICALLY (Fresh Install)\nINSERT INTO goggles (id, name, description, author, public)\nVALUES (\n '{{ $json.goggle_id }}',\n '{{ $json.goggle_name }}',\n '{{ $json.goggle_description }}',\n '{{ $json.goggle_author }}',\n {{ $json.goggle_public }}\n)\nON CONFLICT (id) DO UPDATE SET \n name = EXCLUDED.name, \n description = EXCLUDED.description,\n author = EXCLUDED.author,\n public = EXCLUDED.public,\n updated_at = NOW();", "options": {}, "operation": "executeQuery" }, "credentials": { "postgres": { "name": "" } }, "typeVersion": 2.6 }, { "id": "0354d743-89df-44da-946a-46a98b04db3e", "name": "Sticky Note7", "type": "n8n-nodes-base.stickyNote", "position": [ 256, -592 ], "parameters": { "color": 6, "width": 320, "height": 80, "content": "## Step 0: SQL Foundation" }, "typeVersion": 1 }, { "id": "99ba11b5-4550-4dd3-83f0-855ac717067e", "name": "Sticky Note14", "type": "n8n-nodes-base.stickyNote", "position": [ 880, -400 ], "parameters": { "color": 6, "width": 320, "height": 80, "content": "## Step 1: The Setup flow\n" }, "typeVersion": 1 }, { "id": "08ff2928-c77a-481e-bd5d-d1daaa34d455", "name": "Sticky Note15", "type": "n8n-nodes-base.stickyNote", "position": [ 880, 672 ], "parameters": { "color": 6, "width": 320, "height": 80, "content": "## Step 3: The Trigger flow" }, "typeVersion": 1 } ], "connections": { "Merge": { "main": [ [ { "node": "Wait 5 Seconds", "type": "main", "index": 0 } ] ] }, "Split Rules": { "main": [ [ { "node": "Format Data for SQL", "type": "main", "index": 0 } ] ] }, "OpenAI GPT-4o": { "ai_languageModel": [ [ { "node": "Information Extractor Agent", "type": "ai_languageModel", "index": 0 } ] ] }, "Switch by Mode": { "main": [ [ { "node": "Batch Process Goggles", "type": "main", "index": 0 } ], [ { "node": "Batch Process Domains", "type": "main", "index": 0 } ] ] }, "Wait 3 Seconds": { "main": [ [ { "node": "Batch Process Goggles", "type": "main", "index": 0 } ] ] }, "Wait 5 Seconds": { "main": [ [ { "node": "Batch Process Domains", "type": "main", "index": 0 } ] ] }, "Patch GitHub Gist": { "main": [ [ { "node": "Wait 3 Seconds", "type": "main", "index": 0 } ] ] }, "Use OpenAI GPT-4o": { "ai_languageModel": [ [ { "node": "Information Extraction Agent", "type": "ai_languageModel", "index": 0 } ] ] }, "Verify AI Process": { "main": [ [ { "node": "If Complete Status Is True", "type": "main", "index": 0 } ] ] }, "Log Error Postgres": { "main": [ [ { "node": "Wait 5 Seconds", "type": "main", "index": 0 } ] ] }, "Map Firecrawl Data": { "main": [ [ { "node": "Parse Domain Structures", "type": "main", "index": 0 }, { "node": "Run Secondary SQL Query", "type": "main", "index": 0 } ], [ { "node": "Execute JavaScript Code", "type": "main", "index": 0 } ] ] }, "Run SQL Data Fetch": { "main": [ [ { "node": "Batch Process Items in Tens", "type": "main", "index": 0 } ] ] }, "Format Data for SQL": { "main": [ [ { "node": "Run SQL Data Fetch", "type": "main", "index": 0 } ] ] }, "Optimize AI Prompts": { "main": [ [ { "node": "Information Extraction Agent", "type": "main", "index": 0 } ] ] }, "Build Database Rules": { "main": [ [ { "node": "Insert Rule into Database", "type": "main", "index": 0 } ] ] }, "Manual Start Trigger": { "main": [ [ { "node": "Set Environment Variables", "type": "main", "index": 0 } ] ] }, "Post New GitHub Gist": { "main": [ [ { "node": "Store Gist ID in Database", "type": "main", "index": 0 } ] ] }, "SQL-Based Data Merge": { "main": [ [ { "node": "If Audit Status Pending", "type": "main", "index": 0 } ] ] }, "Batch Process Domains": { "main": [ [], [ { "node": "Map Firecrawl Data", "type": "main", "index": 0 } ] ] }, "Batch Process Goggles": { "main": [ [], [ { "node": "Check Gist ID Presence", "type": "main", "index": 0 } ] ] }, "Get SERP Data for SEO": { "main": [ [ { "node": "Parse Domain and Path", "type": "main", "index": 0 }, { "node": "Execute Postgres Query", "type": "main", "index": 0 } ] ] }, "Parse Domain and Path": { "main": [ [ { "node": "SQL-Based Data Merge", "type": "main", "index": 1 } ] ] }, "Setup Goggle Identity": { "main": [ [ { "node": "Execute SQL Query", "type": "main", "index": 0 } ] ] }, "Trigger Every 6 Hours": { "main": [ [ { "node": "Set Update Mode Fields", "type": "main", "index": 0 } ] ] }, "Check Gist ID Presence": { "main": [ [ { "node": "Patch GitHub Gist", "type": "main", "index": 0 } ], [ { "node": "Post New GitHub Gist", "type": "main", "index": 0 } ] ] }, "Execute Postgres Query": { "main": [ [ { "node": "SQL-Based Data Merge", "type": "main", "index": 0 } ] ] }, "Post to DataForSEO API": { "main": [ [ { "node": "Process Intersection Results", "type": "main", "index": 0 } ] ] }, "Set Update Mode Fields": { "main": [ [ { "node": "Fetch All Goggle Entries", "type": "main", "index": 0 } ] ] }, "Execute JavaScript Code": { "main": [ [ { "node": "Log Error Postgres", "type": "main", "index": 0 } ] ] }, "If Audit Status Pending": { "main": [ [ { "node": "Optimize AI Prompts", "type": "main", "index": 0 } ] ] }, "Parse Domain Structures": { "main": [ [ { "node": "Information Extractor Agent", "type": "main", "index": 0 } ] ] }, "Run Secondary SQL Query": { "main": [ [ { "node": "Merge", "type": "main", "index": 1 } ] ] }, "Fetch All Goggle Entries": { "main": [ [ { "node": "Switch by Mode", "type": "main", "index": 0 } ] ] }, "Trigger Weekly on Monday": { "main": [ [ { "node": "Set Discovery Mode Fields", "type": "main", "index": 0 } ] ] }, "Insert Rule into Database": { "main": [ [ { "node": "Merge", "type": "main", "index": 0 } ] ] }, "Set Discovery Mode Fields": { "main": [ [ { "node": "Fetch All Goggle Entries", "type": "main", "index": 0 } ] ] }, "Set Environment Variables": { "main": [ [ { "node": "Post to DataForSEO API", "type": "main", "index": 0 } ] ] }, "Store Gist ID in Database": { "main": [ [ { "node": "Wait 3 Seconds", "type": "main", "index": 0 } ] ] }, "If Complete Status Is True": { "main": [ [ { "node": "Split Rules", "type": "main", "index": 0 } ], [ { "node": "Information Extraction Agent", "type": "main", "index": 0 } ] ] }, "Batch Process Items in Tens": { "main": [ [], [ { "node": "Get SERP Data for SEO", "type": "main", "index": 0 } ] ] }, "Information Extractor Agent": { "main": [ [ { "node": "Build Database Rules", "type": "main", "index": 0 } ] ] }, "Information Extraction Agent": { "main": [ [ { "node": "Verify AI Process", "type": "main", "index": 0 } ] ] }, "Process Intersection Results": { "main": [ [ { "node": "Batch Process Items in Tens", "type": "main", "index": 0 } ] ] } } }