This workflow follows the Execute Workflow Trigger → HTTP Request recipe pattern — see all workflows that pair these two integrations.
The workflow JSON
Copy or download the full n8n JSON below. Paste it into a new n8n workflow, add your credentials, activate. Full import guide →
{
"name": "[FC Sub] Crawl Site (Bounded)",
"settings": {
"executionOrder": "v1"
},
"nodes": [
{
"parameters": {
"content": "## [FC Sub] Crawl Site (Bounded)\n**Purpose:** Recursively crawls a site via Firecrawl `/v1/crawl`. Enforces hard caps (max 50 pages, max depth 3) to keep credit burn bounded. Polls the async job up to ~40s and returns whatever's ready.\n\n**Called by:** main agent's `crawl_site` tool.\n\n**Inputs:**\n- `url` (required) \u2014 site root or starting URL\n- `max_pages?` \u2014 default 25, hard-capped to 50\n- `max_depth?` \u2014 default 2, hard-capped to 3\n- `include_patterns?` \u2014 array of URL-glob patterns to include\n- `exclude_patterns?` \u2014 array of URL-globs to exclude\n- `session_id?`\n\n**Credit cost:** 1 credit per page actually scraped.\n\n**Winning pattern:** The agent's system prompt prefers `map_site` + `batch_scrape` over `crawl_site` \u2014 this tool is the fallback when the user explicitly wants exhaustive traversal.\n\n**Flow:** Trigger \u2192 Prep (enforce caps) \u2192 POST `/v1/crawl` \u2192 Wait 10s \u2192 GET status \u2192 IF not done: Wait 30s \u2192 GET status \u2192 Shape \u2192 Ledger \u2192 Return.\n\n**Credentials:** Firecrawl API, Postgres RW.",
"height": 660,
"width": 660,
"color": 6
},
"type": "n8n-nodes-base.stickyNote",
"typeVersion": 1,
"position": [
-40,
-700
],
"id": "sticky-fc-crawl",
"name": "README"
},
{
"parameters": {
"inputSource": "passthrough"
},
"type": "n8n-nodes-base.executeWorkflowTrigger",
"typeVersion": 1.1,
"position": [
0,
0
],
"id": "fc-crawl-trigger",
"name": "When Executed by Another Workflow"
},
{
"parameters": {
"jsCode": "const input = $input.first().json;\nconst q = (input.query && typeof input.query === 'object') ? input.query : {};\nconst pick = (k, def) => {\n if (input[k] !== undefined) return input[k];\n if (q[k] !== undefined) return q[k];\n return def;\n};\nconst parseIfString = (v, fb) => {\n if (v === undefined || v === null) return fb;\n if (Array.isArray(v)) return v;\n if (typeof v !== 'string') return v;\n try { return JSON.parse(v); } catch { return fb; }\n};\n\nlet url = (pick('url') || '').toString().trim();\nif (!url) throw new Error('crawl_site requires a `url` parameter.');\nif (!/^https?:\\/\\//i.test(url)) url = 'https://' + url;\ntry { new URL(url); } catch { throw new Error(`Invalid URL: ${url}`); }\n\nlet max_pages = Number(pick('max_pages', 25));\nif (!Number.isFinite(max_pages) || max_pages < 1) max_pages = 25;\nif (max_pages > 50) max_pages = 50; // hard cap\n\nlet max_depth = Number(pick('max_depth', 2));\nif (!Number.isFinite(max_depth) || max_depth < 1) max_depth = 2;\nif (max_depth > 3) max_depth = 3; // hard cap\n\nconst include_patterns = parseIfString(pick('include_patterns'), []);\nconst exclude_patterns = parseIfString(pick('exclude_patterns'), []);\nconst session_id = (pick('session_id') || $execution.id || 'no-session').toString();\n\nreturn [{ json: {\n url,\n max_pages,\n max_depth,\n include_patterns,\n exclude_patterns,\n session_id,\n execution_id: $execution.id\n} }];"
},
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
220,
0
],
"id": "fc-crawl-prep",
"name": "Prep + Enforce Caps"
},
{
"parameters": {
"method": "POST",
"url": "https://api.firecrawl.dev/v1/crawl",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"sendBody": true,
"specifyBody": "json",
"jsonBody": "={\n \"url\": {{ JSON.stringify($json.url) }},\n \"limit\": {{ $json.max_pages }},\n \"maxDepth\": {{ $json.max_depth }}\n {{ $json.include_patterns && $json.include_patterns.length ? (', \"includePaths\": ' + JSON.stringify($json.include_patterns)) : '' }}\n {{ $json.exclude_patterns && $json.exclude_patterns.length ? (', \"excludePaths\": ' + JSON.stringify($json.exclude_patterns)) : '' }},\n \"scrapeOptions\": { \"formats\": [\"markdown\"], \"onlyMainContent\": true }\n}",
"options": {
"response": {
"response": {
"responseFormat": "json"
}
},
"timeout": 30000
}
},
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [
440,
0
],
"id": "fc-crawl-start",
"name": "Start Crawl Job"
},
{
"parameters": {
"amount": 10,
"unit": "seconds"
},
"type": "n8n-nodes-base.wait",
"typeVersion": 1.1,
"position": [
660,
0
],
"id": "fc-crawl-wait1",
"name": "Wait 10s"
},
{
"parameters": {
"method": "GET",
"url": "=https://api.firecrawl.dev/v1/crawl/{{ $('Start Crawl Job').first().json.id }}",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"options": {
"response": {
"response": {
"responseFormat": "json"
}
},
"timeout": 30000
}
},
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [
880,
0
],
"id": "fc-crawl-poll1",
"name": "Poll Status (10s)"
},
{
"parameters": {
"conditions": {
"options": {
"caseSensitive": true,
"leftValue": "",
"typeValidation": "loose"
},
"conditions": [
{
"id": "status-completed-1",
"leftValue": "={{ $json.status }}",
"rightValue": "completed",
"operator": {
"type": "string",
"operation": "equals"
}
}
],
"combinator": "and"
},
"options": {}
},
"type": "n8n-nodes-base.if",
"typeVersion": 2.2,
"position": [
1100,
0
],
"id": "fc-crawl-if-done1",
"name": "IF Completed (1st poll)"
},
{
"parameters": {
"amount": 30,
"unit": "seconds"
},
"type": "n8n-nodes-base.wait",
"typeVersion": 1.1,
"position": [
1320,
200
],
"id": "fc-crawl-wait2",
"name": "Wait 30s"
},
{
"parameters": {
"method": "GET",
"url": "=https://api.firecrawl.dev/v1/crawl/{{ $('Start Crawl Job').first().json.id }}",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"options": {
"response": {
"response": {
"responseFormat": "json"
}
},
"timeout": 30000
}
},
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [
1540,
200
],
"id": "fc-crawl-poll2",
"name": "Poll Status (40s)"
},
{
"parameters": {
"jsCode": "const prep = $('Prep + Enforce Caps').first().json;\nconst start = $('Start Crawl Job').first().json;\nconst resp = $input.first().json;\nconst MAX_PAGES_RETURN = prep.max_pages; // already capped to 50\nconst MAX_MD = 12000; // per-page markdown cap\n\nconst status = resp.status || 'unknown';\nconst pages = Array.isArray(resp.data) ? resp.data.slice(0, MAX_PAGES_RETURN) : [];\n\nconst shaped = pages.map(p => ({\n url: p.url || p.metadata?.sourceURL || '',\n title: (p.metadata && p.metadata.title) || '',\n markdown: ((p.markdown || '').length > MAX_MD)\n ? p.markdown.slice(0, MAX_MD) + '\\n\\n[...truncated]'\n : (p.markdown || '')\n}));\n\nconst credits_used = shaped.length; // 1 credit per scraped page\nconst complete = status === 'completed';\nconst note = complete\n ? undefined\n : `Crawl still running after ~40s (status: ${status}, ${shaped.length} of ${resp.total || '?'} pages so far). Re-call with the same url or use batch_scrape on map_site results for faster targeted crawling.`;\n\nreturn [{ json: {\n site: prep.url,\n status,\n complete,\n job_id: start.id || start.jobId || null,\n pages: shaped,\n page_count: shaped.length,\n total_expected: resp.total || null,\n credits_used,\n note,\n session_id: prep.session_id,\n execution_id: prep.execution_id\n} }];"
},
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
1760,
100
],
"id": "fc-crawl-shape",
"name": "Shape Response"
},
{
"parameters": {
"operation": "executeQuery",
"query": "INSERT INTO public.firecrawl_credit_ledger (session_id, execution_id, operation, credits_used, url, status, metadata)\nVALUES ($1, $2, 'crawl', $3, $4, $5, $6::jsonb);",
"options": {
"queryReplacement": "={{ $json.session_id }}, {{ $json.execution_id }}, {{ $json.credits_used }}, {{ $json.site }}, {{ $json.complete ? 'ok' : 'partial' }}, {{ JSON.stringify({ status: $json.status, page_count: $json.page_count, total_expected: $json.total_expected, job_id: $json.job_id }) }}"
}
},
"type": "n8n-nodes-base.postgres",
"typeVersion": 2.6,
"position": [
1980,
100
],
"id": "fc-crawl-ledger",
"name": "Log Credit Ledger"
},
{
"parameters": {
"assignments": {
"assignments": [
{
"id": "c1",
"name": "site",
"value": "={{ $('Shape Response').first().json.site }}",
"type": "string"
},
{
"id": "c2",
"name": "status",
"value": "={{ $('Shape Response').first().json.status }}",
"type": "string"
},
{
"id": "c3",
"name": "complete",
"value": "={{ $('Shape Response').first().json.complete }}",
"type": "boolean"
},
{
"id": "c4",
"name": "pages",
"value": "={{ $('Shape Response').first().json.pages }}",
"type": "array"
},
{
"id": "c5",
"name": "page_count",
"value": "={{ $('Shape Response').first().json.page_count }}",
"type": "number"
},
{
"id": "c6",
"name": "total_expected",
"value": "={{ $('Shape Response').first().json.total_expected }}",
"type": "number"
},
{
"id": "c7",
"name": "credits_used",
"value": "={{ $('Shape Response').first().json.credits_used }}",
"type": "number"
},
{
"id": "c8",
"name": "note",
"value": "={{ $('Shape Response').first().json.note }}",
"type": "string"
},
{
"id": "c9",
"name": "job_id",
"value": "={{ $('Shape Response').first().json.job_id }}",
"type": "string"
}
]
},
"options": {}
},
"type": "n8n-nodes-base.set",
"typeVersion": 3.4,
"position": [
2200,
100
],
"id": "fc-crawl-return",
"name": "Return Crawl"
}
],
"connections": {
"When Executed by Another Workflow": {
"main": [
[
{
"node": "Prep + Enforce Caps",
"type": "main",
"index": 0
}
]
]
},
"Prep + Enforce Caps": {
"main": [
[
{
"node": "Start Crawl Job",
"type": "main",
"index": 0
}
]
]
},
"Start Crawl Job": {
"main": [
[
{
"node": "Wait 10s",
"type": "main",
"index": 0
}
]
]
},
"Wait 10s": {
"main": [
[
{
"node": "Poll Status (10s)",
"type": "main",
"index": 0
}
]
]
},
"Poll Status (10s)": {
"main": [
[
{
"node": "IF Completed (1st poll)",
"type": "main",
"index": 0
}
]
]
},
"IF Completed (1st poll)": {
"main": [
[
{
"node": "Shape Response",
"type": "main",
"index": 0
}
],
[
{
"node": "Wait 30s",
"type": "main",
"index": 0
}
]
]
},
"Wait 30s": {
"main": [
[
{
"node": "Poll Status (40s)",
"type": "main",
"index": 0
}
]
]
},
"Poll Status (40s)": {
"main": [
[
{
"node": "Shape Response",
"type": "main",
"index": 0
}
]
]
},
"Shape Response": {
"main": [
[
{
"node": "Log Credit Ledger",
"type": "main",
"index": 0
}
]
]
},
"Log Credit Ledger": {
"main": [
[
{
"node": "Return Crawl",
"type": "main",
"index": 0
}
]
]
}
}
}
Pro
For the full experience including quality scoring and batch install features for each workflow upgrade to Pro
About this workflow
[FC Sub] Crawl Site (Bounded). Uses executeWorkflowTrigger, httpRequest, postgres. Event-driven trigger; 12 nodes.
Source: https://github.com/MinaSaad1/n8n-firecrawl-web-crawler-agent/blob/main/workflows/06-sub-crawl-bounded.json — original creator credit. Request a take-down →
Related workflows
Workflows that share integrations, category, or trigger type with this one. All free to copy and import.
[FC Sub] Scrape URL with 24h Cache. Uses executeWorkflowTrigger, postgres, httpRequest. Event-driven trigger; 14 nodes.
Execute Workflow Trigger, Postgres, HTTP Request
[FC Sub] Batch Scrape URLs. Uses executeWorkflowTrigger, httpRequest, postgres. Event-driven trigger; 12 nodes.
Execute Workflow Trigger, HTTP Request, Postgres
[FC Sub] Extract Structured Data. Uses executeWorkflowTrigger, httpRequest, postgres. Event-driven trigger; 12 nodes.
Execute Workflow Trigger, HTTP Request, Postgres
[FC Sub] Search the Web. Uses executeWorkflowTrigger, httpRequest, postgres. Event-driven trigger; 7 nodes.
Execute Workflow Trigger, HTTP Request, Postgres
[FC Sub] Map a Website. Uses executeWorkflowTrigger, httpRequest, postgres. Event-driven trigger; 7 nodes.
Execute Workflow Trigger, HTTP Request, Postgres