This workflow follows the Execute Workflow Trigger → HTTP Request recipe pattern — see all workflows that pair these two integrations.
The workflow JSON
Copy or download the full n8n JSON below. Paste it into a new n8n workflow, add your credentials, activate. Full import guide →
{
"name": "[FC Sub] Batch Scrape URLs",
"settings": {
"executionOrder": "v1"
},
"nodes": [
{
"parameters": {
"content": "## [FC Sub] Batch Scrape URLs\n**Purpose:** Scrapes up to 25 URLs in one call via Firecrawl `/v1/batch/scrape`. More efficient than looping `scrape_url` \u2014 Firecrawl parallelizes internally.\n\n**Called by:** main agent's `batch_scrape` tool.\n\n**Inputs:**\n- `urls[]` (required) \u2014 array of URLs, hard-capped to 25 per call\n- `formats?` \u2014 default `['markdown']`\n- `only_main_content?` \u2014 default true\n- `session_id?`\n\n**Credit cost:** 1 credit per URL.\n\n**Winning pattern:** Combine with `map_site` \u2014 map \u2192 filter URLs via `think` \u2192 batch_scrape the filtered list. This is the canonical way to extract content from a site section while keeping credit burn bounded.\n\n**Flow:** Trigger \u2192 Prep (cap at 25) \u2192 POST `/v1/batch/scrape` \u2192 Wait 15s \u2192 Poll status \u2192 Optional second poll \u2192 Shape \u2192 Ledger \u2192 Return.\n\n**Credentials:** Firecrawl API, Postgres RW.",
"height": 560,
"width": 640,
"color": 6
},
"type": "n8n-nodes-base.stickyNote",
"typeVersion": 1,
"position": [
-40,
-600
],
"id": "sticky-fc-batch",
"name": "README"
},
{
"parameters": {
"inputSource": "passthrough"
},
"type": "n8n-nodes-base.executeWorkflowTrigger",
"typeVersion": 1.1,
"position": [
0,
0
],
"id": "fc-batch-trigger",
"name": "When Executed by Another Workflow"
},
{
"parameters": {
"jsCode": "const input = $input.first().json;\nconst q = (input.query && typeof input.query === 'object') ? input.query : {};\nconst pick = (k, def) => {\n if (input[k] !== undefined) return input[k];\n if (q[k] !== undefined) return q[k];\n return def;\n};\nconst parseIfString = (v, fb) => {\n if (v === undefined || v === null) return fb;\n if (Array.isArray(v)) return v;\n if (typeof v !== 'string') return v;\n try { return JSON.parse(v); } catch { return fb; }\n};\n\nlet urls = parseIfString(pick('urls'), null);\nif (!Array.isArray(urls) || urls.length === 0) {\n throw new Error('batch_scrape requires `urls` \u2014 an array of URL strings.');\n}\n\nurls = urls\n .map(u => (typeof u === 'string' ? u.trim() : ''))\n .filter(u => u.length > 0)\n .map(u => /^https?:\\/\\//i.test(u) ? u : 'https://' + u);\n\nurls = Array.from(new Set(urls)); // dedupe\nif (urls.length > 25) urls = urls.slice(0, 25); // hard cap for credit protection\n\nconst formats = parseIfString(pick('formats'), ['markdown']);\nconst only_main_content = pick('only_main_content', true) !== false;\nconst session_id = (pick('session_id') || $execution.id || 'no-session').toString();\n\nreturn [{ json: {\n urls,\n url_count: urls.length,\n formats,\n only_main_content,\n session_id,\n execution_id: $execution.id\n} }];"
},
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
220,
0
],
"id": "fc-batch-prep",
"name": "Prep + Cap at 25"
},
{
"parameters": {
"method": "POST",
"url": "https://api.firecrawl.dev/v1/batch/scrape",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"sendBody": true,
"specifyBody": "json",
"jsonBody": "={\n \"urls\": {{ JSON.stringify($json.urls) }},\n \"formats\": {{ JSON.stringify($json.formats) }},\n \"onlyMainContent\": {{ $json.only_main_content }}\n}",
"options": {
"response": {
"response": {
"responseFormat": "json"
}
},
"timeout": 30000
}
},
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [
440,
0
],
"id": "fc-batch-start",
"name": "Start Batch Job"
},
{
"parameters": {
"amount": 15,
"unit": "seconds"
},
"type": "n8n-nodes-base.wait",
"typeVersion": 1.1,
"position": [
660,
0
],
"id": "fc-batch-wait1",
"name": "Wait 15s"
},
{
"parameters": {
"method": "GET",
"url": "=https://api.firecrawl.dev/v1/batch/scrape/{{ $('Start Batch Job').first().json.id }}",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"options": {
"response": {
"response": {
"responseFormat": "json"
}
},
"timeout": 30000
}
},
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [
880,
0
],
"id": "fc-batch-poll1",
"name": "Poll Status (15s)"
},
{
"parameters": {
"conditions": {
"options": {
"caseSensitive": true,
"leftValue": "",
"typeValidation": "loose"
},
"conditions": [
{
"id": "batch-status-1",
"leftValue": "={{ $json.status }}",
"rightValue": "completed",
"operator": {
"type": "string",
"operation": "equals"
}
}
],
"combinator": "and"
},
"options": {}
},
"type": "n8n-nodes-base.if",
"typeVersion": 2.2,
"position": [
1100,
0
],
"id": "fc-batch-if-done1",
"name": "IF Completed"
},
{
"parameters": {
"amount": 20,
"unit": "seconds"
},
"type": "n8n-nodes-base.wait",
"typeVersion": 1.1,
"position": [
1320,
200
],
"id": "fc-batch-wait2",
"name": "Wait 20s"
},
{
"parameters": {
"method": "GET",
"url": "=https://api.firecrawl.dev/v1/batch/scrape/{{ $('Start Batch Job').first().json.id }}",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"options": {
"response": {
"response": {
"responseFormat": "json"
}
},
"timeout": 30000
}
},
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [
1540,
200
],
"id": "fc-batch-poll2",
"name": "Poll Status (35s)"
},
{
"parameters": {
"jsCode": "const prep = $('Prep + Cap at 25').first().json;\nconst start = $('Start Batch Job').first().json;\nconst resp = $input.first().json;\nconst MAX_MD = 10000;\n\nconst status = resp.status || 'unknown';\nconst complete = status === 'completed';\nconst pages = Array.isArray(resp.data) ? resp.data : [];\n\nconst results = pages.map(p => ({\n url: p.url || p.metadata?.sourceURL || '',\n title: (p.metadata && p.metadata.title) || '',\n markdown: ((p.markdown || '').length > MAX_MD)\n ? p.markdown.slice(0, MAX_MD) + '\\n\\n[...truncated]'\n : (p.markdown || '')\n}));\n\nconst succeeded = results.filter(r => r.markdown).length;\nconst failed = prep.url_count - succeeded;\nconst credits_used = succeeded; // only successful scrapes charge credits\n\nreturn [{ json: {\n urls_submitted: prep.urls,\n url_count: prep.url_count,\n status,\n complete,\n job_id: start.id || start.jobId || null,\n results,\n succeeded,\n failed,\n credits_used,\n note: complete ? undefined : `Batch still processing after ~35s (status: ${status}). ${succeeded} of ${prep.url_count} URLs done. Re-run with the same urls or reduce the list.`,\n session_id: prep.session_id,\n execution_id: prep.execution_id\n} }];"
},
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
1760,
100
],
"id": "fc-batch-shape",
"name": "Shape Response"
},
{
"parameters": {
"operation": "executeQuery",
"query": "INSERT INTO public.firecrawl_credit_ledger (session_id, execution_id, operation, credits_used, status, metadata)\nVALUES ($1, $2, 'batch_scrape', $3, $4, $5::jsonb);",
"options": {
"queryReplacement": "={{ $json.session_id }}, {{ $json.execution_id }}, {{ $json.credits_used }}, {{ $json.complete ? 'ok' : 'partial' }}, {{ JSON.stringify({ status: $json.status, submitted: $json.url_count, succeeded: $json.succeeded, failed: $json.failed, job_id: $json.job_id }) }}"
}
},
"type": "n8n-nodes-base.postgres",
"typeVersion": 2.6,
"position": [
1980,
100
],
"id": "fc-batch-ledger",
"name": "Log Credit Ledger"
},
{
"parameters": {
"assignments": {
"assignments": [
{
"id": "ba1",
"name": "status",
"value": "={{ $('Shape Response').first().json.status }}",
"type": "string"
},
{
"id": "ba2",
"name": "complete",
"value": "={{ $('Shape Response').first().json.complete }}",
"type": "boolean"
},
{
"id": "ba3",
"name": "results",
"value": "={{ $('Shape Response').first().json.results }}",
"type": "array"
},
{
"id": "ba4",
"name": "succeeded",
"value": "={{ $('Shape Response').first().json.succeeded }}",
"type": "number"
},
{
"id": "ba5",
"name": "failed",
"value": "={{ $('Shape Response').first().json.failed }}",
"type": "number"
},
{
"id": "ba6",
"name": "credits_used",
"value": "={{ $('Shape Response').first().json.credits_used }}",
"type": "number"
},
{
"id": "ba7",
"name": "note",
"value": "={{ $('Shape Response').first().json.note }}",
"type": "string"
},
{
"id": "ba8",
"name": "job_id",
"value": "={{ $('Shape Response').first().json.job_id }}",
"type": "string"
}
]
},
"options": {}
},
"type": "n8n-nodes-base.set",
"typeVersion": 3.4,
"position": [
2200,
100
],
"id": "fc-batch-return",
"name": "Return Batch"
}
],
"connections": {
"When Executed by Another Workflow": {
"main": [
[
{
"node": "Prep + Cap at 25",
"type": "main",
"index": 0
}
]
]
},
"Prep + Cap at 25": {
"main": [
[
{
"node": "Start Batch Job",
"type": "main",
"index": 0
}
]
]
},
"Start Batch Job": {
"main": [
[
{
"node": "Wait 15s",
"type": "main",
"index": 0
}
]
]
},
"Wait 15s": {
"main": [
[
{
"node": "Poll Status (15s)",
"type": "main",
"index": 0
}
]
]
},
"Poll Status (15s)": {
"main": [
[
{
"node": "IF Completed",
"type": "main",
"index": 0
}
]
]
},
"IF Completed": {
"main": [
[
{
"node": "Shape Response",
"type": "main",
"index": 0
}
],
[
{
"node": "Wait 20s",
"type": "main",
"index": 0
}
]
]
},
"Wait 20s": {
"main": [
[
{
"node": "Poll Status (35s)",
"type": "main",
"index": 0
}
]
]
},
"Poll Status (35s)": {
"main": [
[
{
"node": "Shape Response",
"type": "main",
"index": 0
}
]
]
},
"Shape Response": {
"main": [
[
{
"node": "Log Credit Ledger",
"type": "main",
"index": 0
}
]
]
},
"Log Credit Ledger": {
"main": [
[
{
"node": "Return Batch",
"type": "main",
"index": 0
}
]
]
}
}
}
Pro
For the full experience including quality scoring and batch install features for each workflow upgrade to Pro
About this workflow
[FC Sub] Batch Scrape URLs. Uses executeWorkflowTrigger, httpRequest, postgres. Event-driven trigger; 12 nodes.
Source: https://github.com/MinaSaad1/n8n-firecrawl-web-crawler-agent/blob/main/workflows/07-sub-batch-scrape.json — original creator credit. Request a take-down →
Related workflows
Workflows that share integrations, category, or trigger type with this one. All free to copy and import.
[FC Sub] Scrape URL with 24h Cache. Uses executeWorkflowTrigger, postgres, httpRequest. Event-driven trigger; 14 nodes.
Execute Workflow Trigger, Postgres, HTTP Request
[FC Sub] Crawl Site (Bounded). Uses executeWorkflowTrigger, httpRequest, postgres. Event-driven trigger; 12 nodes.
Execute Workflow Trigger, HTTP Request, Postgres
[FC Sub] Extract Structured Data. Uses executeWorkflowTrigger, httpRequest, postgres. Event-driven trigger; 12 nodes.
Execute Workflow Trigger, HTTP Request, Postgres
[FC Sub] Search the Web. Uses executeWorkflowTrigger, httpRequest, postgres. Event-driven trigger; 7 nodes.
Execute Workflow Trigger, HTTP Request, Postgres
[FC Sub] Map a Website. Uses executeWorkflowTrigger, httpRequest, postgres. Event-driven trigger; 7 nodes.
Execute Workflow Trigger, HTTP Request, Postgres