Wf-20 Process_urls

The workflow JSON

Copy or download the full n8n JSON below. Paste it into a new n8n workflow, add your credentials, activate. Full import guide →

{
  "name": "WF-20 process_urls",
  "nodes": [
    {
      "id": "trigger",
      "name": "WF Input",
      "type": "n8n-nodes-base.executeWorkflowTrigger",
      "typeVersion": 1.1,
      "position": [
        0,
        0
      ],
      "parameters": {
        "workflowInputs": {
          "values": [
            {
              "name": "urls",
              "type": "array"
            },
            {
              "name": "trigger",
              "type": "string"
            },
            {
              "name": "scope",
              "type": "string"
            },
            {
              "name": "query",
              "type": "string"
            }
          ]
        }
      }
    },
    {
      "id": "createrun",
      "name": "Create run",
      "type": "n8n-nodes-base.postgres",
      "typeVersion": 2.4,
      "position": [
        200,
        0
      ],
      "parameters": {
        "operation": "executeQuery",
        "query": "insert into runs (trigger, scope, urls_found, query) values ($1, $2, $3::int, nullif($4,'')) returning id as run_id, (select s.extra_prompt from queries q join skills s on s.name = q.skill where lower(q.text) = lower(nullif($4,''))) as skill_prompt, (select s.extra_schema::text from queries q join skills s on s.name = q.skill where lower(q.text) = lower(nullif($4,''))) as skill_schema, (select q.skill from queries q where lower(q.text) = lower(nullif($4,''))) as skill_name",
        "options": {
          "queryReplacement": "={{ $json.trigger }},{{ $json.scope }},{{ ($json.urls || []).length }},{{ $json.query || '' }}"
        }
      },
      "credentials": {
        "postgres": {
          "name": "<your credential>"
        }
      },
      "executeOnce": true
    },
    {
      "id": "canon",
      "name": "Canonicalize",
      "type": "n8n-nodes-base.code",
      "typeVersion": 2,
      "position": [
        400,
        0
      ],
      "parameters": {
        "mode": "runOnceForAllItems",
        "jsCode": "// Canonical URL form used for dedupe (spec \u00a75.2).\n//\n// This file is the single source of truth: scripts/build-wf20.js inlines the\n// function body into WF-20's Canonicalize node, so the tests in\n// tests/canonicalize.test.js exercise exactly what runs in production.\n//\n// No URL/URLSearchParams \u2014 the n8n task-runner sandbox has neither.\n\n// `src` joins the list for the same reason as `source`: it is overwhelmingly a\n// referrer tag (?src=twitter), and leaving it in split one page into two feed\n// items during testing.\nconst STRIP = /^(utm_.*|fbclid|gclid|ref|source|src|mc_cid|mc_eid|igshid)$/i;\n\nfunction canon(rawUrl) {\n  try {\n    const s = String(rawUrl).trim();\n    const m = s.match(/^([a-zA-Z][a-zA-Z0-9+.-]*):\\/\\/([^/?#]+)([^?#]*)(\\?[^#]*)?(#.*)?$/);\n    if (!m) return null;\n    const scheme = m[1].toLowerCase();\n    let hostport = m[2];\n    let path = m[3] || '/';\n    const query = m[4] ? m[4].slice(1) : '';\n    let userinfo = '';\n    const at = hostport.lastIndexOf('@');\n    if (at !== -1) { userinfo = hostport.slice(0, at + 1); hostport = hostport.slice(at + 1); }\n    let host = hostport;\n    let port = '';\n    const ci = hostport.lastIndexOf(':');\n    if (ci !== -1 && /^\\d+$/.test(hostport.slice(ci + 1))) { host = hostport.slice(0, ci); port = hostport.slice(ci); }\n    host = host.toLowerCase();\n    if (host.startsWith('amp.')) host = host.slice(4);\n    if ((scheme === 'http' && port === ':80') || (scheme === 'https' && port === ':443')) port = '';\n    path = path.replace(/\\/amp(\\/|$)/, '$1');\n    path = path.replace(/\\/{2,}/g, '/');\n    if (path.length > 1 && path.endsWith('/')) path = path.slice(0, -1);\n    if (!path) path = '/';\n    const params = query ? query.split('&').filter(p => p !== '' && !STRIP.test(p.split('=')[0])) : [];\n    const qs = params.length ? '?' + params.join('&') : '';\n    return scheme + '://' + userinfo + host + port + path + qs;\n  } catch (e) { return null; }\n}\n\nlet lastErr = null;\nconst runId = $input.first().json.run_id;\nconst urls = $('WF Input').first().json.urls || [];\nconst out = [];\nfor (const item of urls) {\n  const obj = typeof item === 'string' ? { url: item } : (item || {});\n  const c = obj.url ? canon(obj.url) : null;\n  if (!c) continue;\n  out.push({ json: { url: obj.url, title: obj.title || null, source: obj.source || null,\n    published_at: obj.published_at || null, canonical_url: c, run_id: runId } });\n}\nif (!out.length) return [{ json: { __no_new: true, run_id: runId, debug: lastErr } }];\nreturn out;"
      }
    },
    {
      "id": "builddq",
      "name": "Build dedupe query",
      "type": "n8n-nodes-base.code",
      "typeVersion": 2,
      "position": [
        600,
        0
      ],
      "parameters": {
        "mode": "runOnceForAllItems",
        "jsCode": "const first = $input.first().json;\nif (first.__no_new) return [{ json: { query: 'select null as canonical_url where false', __no_new: true } }];\nconst esc = (u) => String(u).replace(/'/g, \"''\");\nconst urls = $input.all().map(i => i.json.canonical_url);\nconst query = 'select canonical_url from seen_urls where canonical_url in (' +\n  urls.map(u => \"'\" + esc(u) + \"'\").join(',') + ')';\nreturn [{ json: { query } }];"
      }
    },
    {
      "id": "findseen",
      "name": "Find seen",
      "type": "n8n-nodes-base.postgres",
      "typeVersion": 2.4,
      "position": [
        800,
        0
      ],
      "parameters": {
        "operation": "executeQuery",
        "query": "={{ $json.query }}",
        "options": {}
      },
      "credentials": {
        "postgres": {
          "name": "<your credential>"
        }
      },
      "executeOnce": true,
      "alwaysOutputData": true
    },
    {
      "id": "filternew",
      "name": "Filter new",
      "type": "n8n-nodes-base.code",
      "typeVersion": 2,
      "position": [
        1000,
        0
      ],
      "parameters": {
        "mode": "runOnceForAllItems",
        "jsCode": "const runId = $('Create run').first().json.run_id;\nconst seen = new Set($input.all().map(i => i.json && i.json.canonical_url).filter(Boolean));\nlet candidates = [];\ntry { candidates = $('Canonicalize').all().map(i => i.json).filter(j => !j.__no_new); } catch (e) {}\nconst byCanon = new Map();\nfor (const c of candidates) if (!seen.has(c.canonical_url) && !byCanon.has(c.canonical_url)) byCanon.set(c.canonical_url, c);\nconst fresh = [...byCanon.values()];\nif (!fresh.length) return [{ json: { __no_new: true, run_id: runId } }];\nreturn fresh.map(j => ({ json: j }));"
      }
    },
    {
      "id": "anynew",
      "name": "Any new?",
      "type": "n8n-nodes-base.if",
      "typeVersion": 2,
      "position": [
        1200,
        0
      ],
      "parameters": {
        "options": {},
        "conditions": {
          "options": {
            "caseSensitive": true,
            "leftValue": "",
            "typeValidation": "loose"
          },
          "combinator": "and",
          "conditions": [
            {
              "id": "c1",
              "leftValue": "={{ Boolean($json.__no_new) }}",
              "rightValue": false,
              "operator": {
                "type": "boolean",
                "operation": "equals"
              }
            }
          ]
        }
      }
    },
    {
      "id": "scrape",
      "name": "Scrape",
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 4.2,
      "position": [
        1400,
        -100
      ],
      "parameters": {
        "method": "POST",
        "url": "http://crawl4ai:11235/crawl",
        "authentication": "genericCredentialType",
        "genericAuthType": "httpBearerAuth",
        "sendBody": true,
        "specifyBody": "json",
        "jsonBody": "={{ JSON.stringify({ urls: [$json.canonical_url], crawler_config: { type: 'CrawlerRunConfig', params: { markdown_generator: { type: 'DefaultMarkdownGenerator', params: { content_filter: { type: 'PruningContentFilter', params: {} } } } } } }) }}",
        "options": {
          "timeout": 20000,
          "batching": {
            "batch": {
              "batchSize": 4,
              "batchInterval": 1000
            }
          }
        }
      },
      "credentials": {
        "httpBearerAuth": {
          "name": "<your credential>"
        }
      },
      "retryOnFail": true,
      "maxTries": 2,
      "waitBetweenTries": 2000,
      "onError": "continueRegularOutput"
    },
    {
      "id": "scrapeok",
      "name": "Scrape OK?",
      "type": "n8n-nodes-base.if",
      "typeVersion": 2,
      "position": [
        1600,
        -100
      ],
      "parameters": {
        "options": {},
        "conditions": {
          "options": {
            "caseSensitive": true,
            "leftValue": "",
            "typeValidation": "loose"
          },
          "combinator": "and",
          "conditions": [
            {
              "id": "c1",
              "leftValue": "={{ ($json.results?.[0]?.markdown?.fit_markdown || $json.results?.[0]?.markdown?.raw_markdown || '').length }}",
              "rightValue": 400,
              "operator": {
                "type": "number",
                "operation": "gte"
              }
            }
          ]
        }
      }
    },
    {
      "id": "ispdf",
      "name": "Is PDF?",
      "type": "n8n-nodes-base.if",
      "typeVersion": 2,
      "position": [
        1300,
        -100
      ],
      "parameters": {
        "options": {},
        "conditions": {
          "options": {
            "caseSensitive": true,
            "leftValue": "",
            "typeValidation": "loose"
          },
          "combinator": "and",
          "conditions": [
            {
              "id": "c1",
              "leftValue": "={{ $json.canonical_url.toLowerCase().split('?')[0].endsWith('.pdf') }}",
              "rightValue": true,
              "operator": {
                "type": "boolean",
                "operation": "equals"
              }
            }
          ]
        }
      }
    },
    {
      "id": "firecrawl",
      "name": "Firecrawl",
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 4.2,
      "position": [
        1700,
        50
      ],
      "parameters": {
        "method": "POST",
        "url": "https://api.firecrawl.dev/v1/scrape",
        "authentication": "genericCredentialType",
        "genericAuthType": "httpBearerAuth",
        "sendBody": true,
        "specifyBody": "json",
        "jsonBody": "={{ JSON.stringify({ url: $('Filter new').item.json.canonical_url, formats: ['markdown'] }) }}",
        "options": {
          "timeout": 45000,
          "batching": {
            "batch": {
              "batchSize": 1,
              "batchInterval": 6000
            }
          }
        }
      },
      "credentials": {
        "httpBearerAuth": {
          "name": "<your credential>"
        }
      },
      "retryOnFail": true,
      "maxTries": 2,
      "waitBetweenTries": 5000,
      "onError": "continueRegularOutput"
    },
    {
      "id": "fcok",
      "name": "Firecrawl OK?",
      "type": "n8n-nodes-base.if",
      "typeVersion": 2,
      "position": [
        1850,
        50
      ],
      "parameters": {
        "options": {},
        "conditions": {
          "options": {
            "caseSensitive": true,
            "leftValue": "",
            "typeValidation": "loose"
          },
          "combinator": "and",
          "conditions": [
            {
              "id": "c1",
              "leftValue": "={{ ($json.data?.markdown || '').length }}",
              "rightValue": 200,
              "operator": {
                "type": "number",
                "operation": "gte"
              }
            }
          ]
        }
      }
    },
    {
      "id": "logfail",
      "name": "Log failure",
      "type": "n8n-nodes-base.code",
      "typeVersion": 2,
      "position": [
        2000,
        150
      ],
      "parameters": {
        "mode": "runOnceForEachItem",
        "jsCode": "const orig = $('Filter new').item.json;\nlet reason = 'both_scrapers_failed_or_short_markdown';\nif ($json.error) {\n  const msg = (typeof $json.error === 'object' && $json.error.message) ? $json.error.message : String($json.error);\n  reason = 'firecrawl_error: ' + String(msg).slice(0, 120);\n} else if ($json.data) reason = 'firecrawl_short_markdown';\nreturn { json: { failed: true, url: orig.url, canonical_url: orig.canonical_url, reason, run_id: orig.run_id } };"
      }
    },
    {
      "id": "prepdoc",
      "name": "Prep doc",
      "type": "n8n-nodes-base.code",
      "typeVersion": 2,
      "position": [
        1800,
        -200
      ],
      "parameters": {
        "mode": "runOnceForEachItem",
        "jsCode": "// Triage system prompt + response schema, shared by the build scripts so WF-20\n// (live pipeline) and WF-30 (backlog drain) can never drift apart.\n//\n// v2 (2026-07-26): two judgement axes replace the single `relevance` score,\n// which collapsed \u2014 on a real query (\"AI infrastructure costs\") 10 of 10 items\n// scored 5, so min_relevance filtered nothing. `relevance` is still emitted\n// alongside for A/B comparison and will be dropped once the axes are trusted.\n//\n// Deliberately NOT an axis: novelty. exa and brave return a publication date on\n// ~100% of results, so recency is exact SQL arithmetic on published_at rather\n// than a 9B model's guess. The prompt still receives the dates so it does not\n// invent recency inside the summary.\n//\n// The beat \u2014 what the writer actually covers \u2014 is personal editorial\n// configuration and stays out of the repo. beat.md (gitignored; one paragraph\n// of plain prose, no quotes/backticks/backslashes) is substituted for\n// __BEAT__ by scripts/deploy.js. Copy beat.example.md to beat.md to start.\nconst BEAT = '__BEAT__';\n\nconst TRIAGE_SYSTEM = [\n  'You are a research triage assistant for a writer covering: ' + BEAT,\n  'The audience is practitioners \u2014 people who build and operate these systems,',\n  'not general readers. Score every article against that beat, not general',\n  'interest.',\n  '',\n  'You read one scraped article and return ONLY a JSON object with exactly these',\n  'fields, in this order:',\n  '',\n  'summary: 2-3 sentences on what the article actually says. Substance only \u2014 no',\n  'meta-commentary about the article itself.',\n  '',\n  'angle: one sentence naming the specific, publishable hook this gives the writer',\n  'for that audience. Use null if there is no defensible angle beyond restating',\n  'the article.',\n  '',\n  'relevance (integer 1-5): overall usefulness to that beat.',\n  '',\n  'specificity (integer 1-5): how well anchored the claims are.',\n  '  5 = concrete data points, named companies and people, quantified outcomes,',\n  '      verifiable claims.',\n  '  3 = a mix \u2014 some claims backed, some asserted.',\n  '  1 = entirely abstract commentary, opinion, or speculation with nothing to',\n  '      check. Vendor content marketing and SEO listicles that recycle public',\n  '      figures without sourcing belong at 1-2.',\n  '',\n  'angle_strength (integer 1-5): how good the angle you just wrote actually is.',\n  '  5 = supports a contrarian, non-obvious, or counterintuitive take that most',\n  '      commentators would miss.',\n  '  3 = a viable angle, but the obvious one most people would write.',\n  '  1 = no defensible angle. If angle is null, angle_strength is 1.',\n  '',\n  'tags: array of 1-5 short lowercase kebab-case topic tags (e.g. \"ai-agents\",',\n  '\"frontier-markets\", \"inference-cost\"). No spaces, no punctuation.',\n  '',\n  'Scoring rules:',\n  '- specificity and angle_strength are independent. Judge each on its own',\n  '  evidence and do not let one pull the other. An article often deserves a high',\n  '  score on one and a low score on the other; that spread is the point.',\n  '- Most articles are not 5s. A typical useful article lands at 2-4. Reserve 5',\n  '  for cases that clearly meet its anchor.',\n  '- If the content is unusable \u2014 a paywall stub, error page, login wall, or a',\n  '  navigation/index/listing page with no article body \u2014 set angle to null, set',\n  '  all scores to 1, and say so in the summary.',\n  '',\n  'Return nothing except the JSON object.'\n].join('\\n');\n\n// Property order is load-bearing: grammar-constrained decoding emits fields in\n// schema order, so summary forces a real read before any number is committed,\n// and angle_strength comes after the angle it rates.\nconst TRIAGE_SCHEMA = {\n  type: 'json_schema',\n  json_schema: {\n    name: 'triage', strict: true,\n    schema: {\n      type: 'object',\n      properties: {\n        summary: { type: 'string' },\n        angle: { type: ['string', 'null'] },\n        relevance: { type: 'integer', minimum: 1, maximum: 5 },\n        specificity: { type: 'integer', minimum: 1, maximum: 5 },\n        angle_strength: { type: 'integer', minimum: 1, maximum: 5 },\n        tags: { type: 'array', items: { type: 'string' }, minItems: 1, maxItems: 5 }\n      },\n      required: ['summary', 'angle', 'relevance', 'specificity', 'angle_strength', 'tags'],\n      additionalProperties: false\n    }\n  }\n};\n\nconst TRIAGE_MODEL = 'qwen/qwen3.5-9b';\n\n// Per-source triage profiles.\n//\n// MEASURED, 2026-07-27 \u2014 read this before tuning, it is counter-intuitive:\n// triage time tracks the model's REASONING OUTPUT, not how much you feed it.\n// Same document at three read lengths: 2500 chars/2835 tokens/38.7s,\n// 6000/4113/47.4s, 8000/2779/32.3s. The longest read was the fastest.\n// Qwen spends ~3000 tokens thinking to produce ~120 tokens of JSON, and that\n// is where the ~33s goes.\n//\n// Suppressing the thinking does not work on this model: enable_thinking via\n// chat_template_kwargs, a /no_think suffix, and reasoning_effort:minimal were\n// all tried and all ignored (2900-3300 tokens, 32-38s, no change).\n//\n// So `content_chars` does NOT buy speed. It buys judgement quality and context\n// budget, which is still worth varying: a feed blurb has nothing after 2500\n// chars, a hand-picked URL deserves the full read.\n//\n// The one real speed/cost lever is `model`. LM Studio serves every loaded\n// model on the same endpoint with the same key, so pointing a source at a\n// different model needs no new credential or URL \u2014 just `lms load <id>` first.\n// If the named model is not loaded the request fails, retries, and falls back\n// to TRIAGE_FAILED, so a missing model degrades rather than breaks.\n//\n// max_tokens is a ceiling, not a target \u2014 the model stops on its own well\n// below it. Keep it above ~2500 or the reasoning strands the JSON.\nconst TRIAGE_PROFILES = {\n  // Feed items are short and mostly low-signal; a smaller read is plenty.\n  rss:     { model: TRIAGE_MODEL, content_chars: 2500, max_tokens: 3000 },\n  // Search hits on the standing queries are the material worth thinking about.\n  exa:     { model: TRIAGE_MODEL, content_chars: 6000, max_tokens: 6000 },\n  tavily:  { model: TRIAGE_MODEL, content_chars: 6000, max_tokens: 6000 },\n  brave:   { model: TRIAGE_MODEL, content_chars: 6000, max_tokens: 6000 },\n  // A URL pasted by hand was chosen deliberately \u2014 read it properly.\n  manual:  { model: TRIAGE_MODEL, content_chars: 8000, max_tokens: 6000 },\n  default: { model: TRIAGE_MODEL, content_chars: 6000, max_tokens: 6000 }\n};\n\nfunction profileFor(source) {\n  const key = String(source || '').toLowerCase().trim();\n  return TRIAGE_PROFILES[key] || TRIAGE_PROFILES.default;\n}\n\n\n// Validation of the triage model's reply. Inlined into WF-21 by\n// scripts/build-wf20.js; covered by tests/triage-validate.test.js.\n\n// Qwen puts chain-of-thought in reasoning_content. When it thinks past the\n// token budget, `content` comes back empty with the JSON stranded in there \u2014\n// recover it rather than burning a retry.\nfunction parseContent(j) {\n  try {\n    const msg = (j && j.choices && j.choices[0] && j.choices[0].message) || {};\n    let c = msg.content;\n    if (!c && msg.reasoning_content) {\n      const m = String(msg.reasoning_content).match(/\\{[\\s\\S]*\\}/);\n      if (m) c = m[0];\n    }\n    if (!c) return null;\n    c = String(c).trim().replace(/^```(json)?/i, '').replace(/```$/, '').trim();\n    const o = JSON.parse(c);\n    const ok = (v) => Number.isInteger(v) && v >= 1 && v <= 5;\n    if (o && typeof o.summary === 'string' && ok(o.relevance) &&\n        ok(o.specificity) && ok(o.angle_strength) &&\n        Array.isArray(o.tags) && o.tags.length >= 1) return o;\n    return null;\n  } catch (e) { return null; }\n}\n\n// Postgres array literal. Tags arrive from a model, so treat them as hostile:\n// braces, quotes, commas and backslashes would all corrupt the literal.\nfunction tagsPg(tags) {\n  const clean = (tags || []).slice(0, 5)\n    .map(t => String(t).toLowerCase().replace(/[{}\",\\\\]/g, '').trim())\n    .filter(Boolean);\n  return '{' + clean.join(',') + '}';\n}\n\n// The six fields every triage returns, skill or no skill.\nconst BASE_FIELDS = ['summary', 'angle', 'relevance', 'specificity', 'angle_strength', 'tags'];\n\n// A skill widens the response schema, so the reply carries extra properties.\n// Split them off into their own object: the fixed columns stay columns, the\n// skill's fields go to feed_items.structured as JSON.\nfunction splitStructured(o) {\n  if (!o || typeof o !== 'object') return { structured: null };\n  const structured = {};\n  let any = false;\n  for (const k of Object.keys(o)) {\n    if (BASE_FIELDS.includes(k)) continue;\n    structured[k] = o[k];\n    any = true;\n  }\n  return { structured: any ? structured : null };\n}\n\n// Merge a skill's extra properties into the base response schema. strict mode\n// requires every declared property to be listed as required, so add both.\nfunction withSkill(baseSchema, extraProps) {\n  const s = JSON.parse(JSON.stringify(baseSchema));\n  if (!extraProps || typeof extraProps !== 'object') return s;\n  const target = s.json_schema.schema;\n  for (const [k, v] of Object.entries(extraProps)) {\n    if (BASE_FIELDS.includes(k)) continue; // a skill may not redefine the core\n    target.properties[k] = v;\n    if (!target.required.includes(k)) target.required.push(k);\n  }\n  return s;\n}\n\n\nconst orig = $('Filter new').item.json;\nlet markdown = '';\nlet title = '';\nlet scraper = 'crawl4ai';\nif ($json.results) {\n  const r = ($json.results && $json.results[0]) || {};\n  const md = r.markdown || {};\n  const fit = (typeof md === 'object' ? md.fit_markdown : md) || '';\n  const raw = (typeof md === 'object' ? md.raw_markdown : '') || '';\n  markdown = fit.length >= 400 ? fit : raw;\n  title = (r.metadata && r.metadata.title) || orig.title || '';\n} else {\n  scraper = 'firecrawl';\n  const d = $json.data || {};\n  markdown = d.markdown || '';\n  title = (d.metadata && d.metadata.title) || orig.title || '';\n}\n// A skill (resolved from the run's query) appends to the prompt and widens the\n// schema \u2014 one model call still, standard fields unchanged.\nconst run = $('Create run').first().json;\nconst sys = TRIAGE_SYSTEM + (run.skill_prompt ? '\\n\\nADDITIONAL EXTRACTION\\n' + run.skill_prompt : '');\nlet schema = TRIAGE_SCHEMA;\nif (run.skill_schema) {\n  try { schema = withSkill(TRIAGE_SCHEMA, JSON.parse(run.skill_schema)); } catch (e) {}\n}\nconst prof = profileFor(orig.source);\n// Dates are stated, never inferred: the sandbox clock is UTC and the model has\n// none, so without these it invents recency inside the summary.\nconst dhaka = new Date(Date.now() + 6 * 3600 * 1000).toISOString().slice(0, 10);\nconst pub = orig.published_at ? String(orig.published_at).slice(0, 10) : 'unknown';\nconst user = 'TODAY: ' + dhaka + '\\n' + 'PUBLISHED: ' + pub + '\\n' +\n  'TITLE: ' + title + '\\n' + 'URL: ' + orig.canonical_url +\n  '\\n\\nCONTENT:\\n' + markdown.slice(0, prof.content_chars);\nconst triage_body = JSON.stringify({ model: prof.model, temperature: 0.2, max_tokens: prof.max_tokens,\n  response_format: schema,\n  messages: [ { role: 'system', content: sys }, { role: 'user', content: user } ] });\nreturn { json: { canonical_url: orig.canonical_url, source: orig.source || '', title,\n  markdown, scraper, run_id: orig.run_id, triage_body, triage_profile: prof.model + '/' + prof.content_chars,\n  published_at: orig.published_at || '' } };"
      }
    },
    {
      "id": "insertdoc",
      "name": "Insert doc",
      "type": "n8n-nodes-base.postgres",
      "typeVersion": 2.4,
      "position": [
        2000,
        -200
      ],
      "parameters": {
        "operation": "executeQuery",
        "query": "with s as (insert into seen_urls (canonical_url, source) values ($1, nullif($2,'')) on conflict (canonical_url) do nothing) insert into raw_docs (canonical_url, title, markdown, scraper, run_id, published_at) values ($1, nullif($3,''), $4, $6, $5::bigint, nullif($7,'')::timestamptz) returning id as raw_doc_id",
        "options": {
          "queryReplacement": "={{ $json.canonical_url }},{{ $json.source }},{{ $json.title }},{{ $json.markdown }},{{ $json.run_id }},{{ $json.scraper }},{{ $json.published_at }}"
        }
      },
      "credentials": {
        "postgres": {
          "name": "<your credential>"
        }
      }
    },
    {
      "id": "runtriage",
      "name": "Run triage",
      "type": "n8n-nodes-base.executeWorkflow",
      "typeVersion": 1.2,
      "position": [
        2200,
        -200
      ],
      "parameters": {
        "mode": "each",
        "workflowId": {
          "__rl": true,
          "value": "HISagmvZYi6O7N5u",
          "mode": "id"
        },
        "workflowInputs": {
          "mappingMode": "defineBelow",
          "value": {
            "triage_body": "={{ $('Prep doc').item.json.triage_body }}",
            "raw_doc_id": "={{ $json.raw_doc_id }}",
            "canonical_url": "={{ $('Prep doc').item.json.canonical_url }}",
            "title": "={{ $('Prep doc').item.json.title }}",
            "published_at": "={{ $('Prep doc').item.json.published_at }}",
            "run_id": "={{ $('Prep doc').item.json.run_id }}"
          },
          "matchingColumns": [],
          "schema": [
            {
              "id": "triage_body",
              "displayName": "triage_body",
              "required": false,
              "defaultMatch": false,
              "display": true,
              "canBeUsedToMatch": true,
              "type": "string"
            },
            {
              "id": "raw_doc_id",
              "displayName": "raw_doc_id",
              "required": false,
              "defaultMatch": false,
              "display": true,
              "canBeUsedToMatch": true,
              "type": "string"
            },
            {
              "id": "canonical_url",
              "displayName": "canonical_url",
              "required": false,
              "defaultMatch": false,
              "display": true,
              "canBeUsedToMatch": true,
              "type": "string"
            },
            {
              "id": "title",
              "displayName": "title",
              "required": false,
              "defaultMatch": false,
              "display": true,
              "canBeUsedToMatch": true,
              "type": "string"
            },
            {
              "id": "published_at",
              "displayName": "published_at",
              "required": false,
              "defaultMatch": false,
              "display": true,
              "canBeUsedToMatch": true,
              "type": "string"
            },
            {
              "id": "run_id",
              "displayName": "run_id",
              "required": false,
              "defaultMatch": false,
              "display": true,
              "canBeUsedToMatch": true,
              "type": "string"
            }
          ]
        },
        "options": {
          "waitForSubWorkflow": true
        }
      },
      "onError": "continueRegularOutput"
    },
    {
      "id": "mergestats",
      "name": "Merge for stats",
      "type": "n8n-nodes-base.merge",
      "typeVersion": 3,
      "position": [
        3600,
        0
      ],
      "parameters": {
        "mode": "append",
        "numberInputs": 3
      }
    },
    {
      "id": "stats",
      "name": "Collect stats",
      "type": "n8n-nodes-base.code",
      "typeVersion": 2,
      "position": [
        3800,
        0
      ],
      "parameters": {
        "mode": "runOnceForAllItems",
        "jsCode": "const runId = $('Create run').first().json.run_id;\nlet fresh = 0; try { fresh = $('Filter new').all().map(i => i.json).filter(j => !j.__no_new).length; } catch (e) {}\nconst fails = [];\nfor (let r = 0; r < 50; r++) {\n  let items;\n  try { items = $('Log failure').all(0, r); } catch (e) { break; }\n  for (const it of items) fails.push({ url: it.json.url, reason: it.json.reason });\n}\n// full detail -> runs.error (DB); short host list -> Telegram summary\nconst error = fails.map(f => f.url + ' (' + f.reason + ')').join('; ').slice(0, 2000);\nconst hosts = [...new Set(fails.map(f => String(f.url).replace(/^https?:\\/\\//, '').split('/')[0]))];\nconst error_brief = fails.length\n  ? fails.length + ' failed: ' + hosts.slice(0, 5).join(', ') + (hosts.length > 5 ? ', \u2026' : '')\n  : '';\nreturn [{ json: { run_id: runId, urls_new: fresh, error, error_brief } }];"
      }
    },
    {
      "id": "finalize",
      "name": "Finalize run",
      "type": "n8n-nodes-base.postgres",
      "typeVersion": 2.4,
      "position": [
        4000,
        0
      ],
      "parameters": {
        "operation": "executeQuery",
        "query": "update runs set finished_at = now(), urls_new = $2::int, urls_scraped = (select count(*) from raw_docs where run_id = $1::bigint), items_created = (select count(*) from feed_items where run_id = $1::bigint), error = nullif($3,''), status = case when $2::int = 0 and $3 = '' then 'ok' when (select count(*) from raw_docs where run_id = $1::bigint) = 0 and $2::int > 0 then 'failed' when (select count(*) from raw_docs where run_id = $1::bigint) < $2::int then 'partial' when (select count(*) from feed_items where run_id = $1::bigint) < (select count(*) from raw_docs where run_id = $1::bigint) then 'partial' else 'ok' end where id = $1::bigint returning id as run_id, status, scope, urls_found, urls_new, urls_scraped, items_created, error, (select title from feed_items where run_id = $1::bigint order by score desc, relevance desc nulls last, id desc limit 1) as top_title",
        "options": {
          "queryReplacement": "={{ $json.run_id }},{{ $json.urls_new }},{{ $json.error }}"
        }
      },
      "credentials": {
        "postgres": {
          "name": "<your credential>"
        }
      },
      "executeOnce": true
    },
    {
      "id": "compose",
      "name": "Compose summary",
      "type": "n8n-nodes-base.code",
      "typeVersion": 2,
      "position": [
        4180,
        0
      ],
      "parameters": {
        "mode": "runOnceForAllItems",
        "jsCode": "const r = $('Finalize run').first().json;\nlet brief = '';\ntry { brief = $('Collect stats').first().json.error_brief || ''; } catch (e) {}\nconst esc = (s) => String(s).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');\nconst made = Number(r.items_created) || 0;\nconst failed = Number((brief.match(/^(\\d+) failed/) || [])[1] || 0);\nconst plural = (n, w) => n + ' ' + w + (n === 1 ? '' : 's');\n\nlet text;\nif (made === 0 && failed === 0) {\n  text = '\ud83d\udd2d Nothing new \u2014 everything found was already in your feed.';\n} else if (made === 0) {\n  text = '\ud83d\udd2d Nothing added. ' + plural(failed, 'page') + \" couldn't be read.\";\n} else {\n  text = '\ud83d\udd2d Found ' + plural(made, 'new page') + ', all summarized.';\n  if (failed) text += ' ' + plural(failed, 'other') + \" couldn't be read.\";\n}\nif (r.top_title) text += '\\nTop: ' + esc(r.top_title);\nif (brief) {\n  const hosts = brief.replace(/^\\d+ failed: /, '');\n  text += '\\n\u26a0 ' + esc(hosts);\n}\nreturn [{ json: { text } }];"
      }
    },
    {
      "id": "tgsummary",
      "name": "Telegram summary",
      "type": "n8n-nodes-base.telegram",
      "typeVersion": 1.2,
      "position": [
        4380,
        0
      ],
      "parameters": {
        "chatId": "__TG_CHAT__",
        "text": "={{ $json.text }}",
        "additionalFields": {
          "appendAttribution": false,
          "parse_mode": "HTML"
        }
      },
      "credentials": {
        "telegramApi": {
          "name": "<your credential>"
        }
      },
      "onError": "continueRegularOutput"
    },
    {
      "id": "retsummary",
      "name": "Return summary",
      "type": "n8n-nodes-base.code",
      "typeVersion": 2,
      "position": [
        4400,
        0
      ],
      "parameters": {
        "mode": "runOnceForAllItems",
        "jsCode": "return $('Finalize run').all();"
      }
    }
  ],
  "connections": {
    "WF Input": {
      "main": [
        [
          {
            "node": "Create run",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Create run": {
      "main": [
        [
          {
            "node": "Canonicalize",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Canonicalize": {
      "main": [
        [
          {
            "node": "Build dedupe query",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Build dedupe query": {
      "main": [
        [
          {
            "node": "Find seen",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Find seen": {
      "main": [
        [
          {
            "node": "Filter new",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Filter new": {
      "main": [
        [
          {
            "node": "Any new?",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Any new?": {
      "main": [
        [
          {
            "node": "Is PDF?",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "Merge for stats",
            "type": "main",
            "index": 2
          }
        ]
      ]
    },
    "Is PDF?": {
      "main": [
        [
          {
            "node": "Firecrawl",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "Scrape",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Scrape": {
      "main": [
        [
          {
            "node": "Scrape OK?",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Scrape OK?": {
      "main": [
        [
          {
            "node": "Prep doc",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "Firecrawl",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Firecrawl": {
      "main": [
        [
          {
            "node": "Firecrawl OK?",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Firecrawl OK?": {
      "main": [
        [
          {
            "node": "Prep doc",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "Log failure",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Prep doc": {
      "main": [
        [
          {
            "node": "Insert doc",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Insert doc": {
      "main": [
        [
          {
            "node": "Run triage",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Run triage": {
      "main": [
        [
          {
            "node": "Merge for stats",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Log failure": {
      "main": [
        [
          {
            "node": "Merge for stats",
            "type": "main",
            "index": 1
          }
        ]
      ]
    },
    "Merge for stats": {
      "main": [
        [
          {
            "node": "Collect stats",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Collect stats": {
      "main": [
        [
          {
            "node": "Finalize run",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Finalize run": {
      "main": [
        [
          {
            "node": "Compose summary",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Compose summary": {
      "main": [
        [
          {
            "node": "Telegram summary",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Telegram summary": {
      "main": [
        [
          {
            "node": "Return summary",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  },
  "settings": {
    "executionOrder": "v1"
  }
}

Credentials you'll need

Each integration node will prompt for credentials when you import. We strip credential IDs before publishing — you'll add your own.

• httpBearerAuth
• postgres
• telegramApi

About this workflow

WF-20 process_urls. Uses executeWorkflowTrigger, postgres, httpRequest, telegram. Event-driven trigger; 22 nodes.

Source: https://github.com/killerwaz/research-overseer/blob/master/workflows/wf20-process-urls.json — original creator credit. Request a take-down →

More Slack & Telegram workflows → · Browse all categories →

Related workflows

Workflows that share integrations, category, or trigger type with this one. All free to copy and import.