{
  "nodes": [
    {
      "id": "38b06683-0f7b-4f40-9e5b-ac85b561b04a",
      "name": "Sticky Note",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -1248,
        -256
      ],
      "parameters": {
        "width": 480,
        "height": 896,
        "content": "## Start a ScraperAPI crawler job from an incoming API request\n\n### How it works\n\n1. Initializes by setting up storage and creating necessary tables.\n2. Receives crawl requests via a webhook and builds parameters for the crawl.\n3. Starts a scraping job using the defined parameters and responds with the job details.\n4. Receives crawl results through a webhook, parses the data, and stores it in a database.\n5. Triggers an email alert in case of any workflow errors.\n\n### Setup steps\n\n- [ ] Configure webhook endpoints for receiving crawl requests and results.\n- [ ] Set up ScraperAPI credentials for initiating scraping jobs.\n- [ ] Configure email settings for error alerts.\n\n### Customization\n\nAdjust webhook URLs and email recipients as needed for different environments or requirements."
      },
      "typeVersion": 1
    },
    {
      "id": "6f4af3b2-cead-4707-b3a7-706cd8035591",
      "name": "Sticky Note1",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -688,
        -256
      ],
      "parameters": {
        "color": 7,
        "width": 448,
        "height": 304,
        "content": "## Setup Storage\n\nCreates storage setup and initializes required tables."
      },
      "typeVersion": 1
    },
    {
      "id": "37270bed-4549-4134-8251-c02a7179f852",
      "name": "Sticky Note2",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -688,
        128
      ],
      "parameters": {
        "color": 7,
        "width": 928,
        "height": 272,
        "content": "## Handle crawl request\n\nReceives a crawl request, builds parameters, and starts the crawl job."
      },
      "typeVersion": 1
    },
    {
      "id": "06023c5b-f785-42dd-b595-629066b9a369",
      "name": "Sticky Note3",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -688,
        480
      ],
      "parameters": {
        "color": 7,
        "width": 688,
        "height": 272,
        "content": "## Process crawl results\n\nReceives and processes crawl results, storing them in a database."
      },
      "typeVersion": 1
    },
    {
      "id": "setup-trigger",
      "name": "Set up storage",
      "type": "n8n-nodes-base.manualTrigger",
      "position": [
        -640,
        -120
      ],
      "parameters": {},
      "typeVersion": 1
    },
    {
      "id": "create-table",
      "name": "Create crawled-pages table",
      "type": "n8n-nodes-base.dataTable",
      "position": [
        -380,
        -120
      ],
      "parameters": {
        "columns": {
          "column": [
            {
              "name": "url",
              "type": "string"
            },
            {
              "name": "content",
              "type": "string"
            },
            {
              "name": "status_code",
              "type": "number"
            },
            {
              "name": "scraped_at",
              "type": "string"
            }
          ]
        },
        "options": {
          "createIfNotExists": true
        },
        "resource": "table",
        "operation": "create",
        "tableName": "crawl_results"
      },
      "typeVersion": 1.1
    },
    {
      "id": "request-webhook",
      "name": "Receive crawl request",
      "type": "n8n-nodes-base.webhook",
      "position": [
        -640,
        240
      ],
      "parameters": {
        "path": "start-crawl",
        "options": {},
        "httpMethod": "POST",
        "responseMode": "responseNode"
      },
      "typeVersion": 2
    },
    {
      "id": "build-params",
      "name": "Build crawl parameters",
      "type": "n8n-nodes-base.code",
      "position": [
        -400,
        240
      ],
      "parameters": {
        "jsCode": "// On-demand crawler API \u2014 turns an incoming HTTP request into a ScraperAPI\n// crawler job. POST a JSON body to the \"Receive crawl request\" webhook:\n//\n//   {\n//     \"startUrl\": \"https://example.com/blog\",        // required\n//     \"includeRegexp\": \"https://example\\\\.com/blog.*\", // optional\n//     \"maxDepth\": 1,                                   // optional\n//     \"countryCode\": \"us\",                            // optional\n//     \"render\": false                                  // optional\n//   }\n//\n// The crawler API requires either a max depth or a crawl budget, so this\n// node always sends one. It defaults maxDepth to 1 (start URL + one level\n// of links) \u2014 the cheapest option and the only depth allowed on the Free\n// plan; depth > 1 needs a paid ScraperAPI plan.\n//\n// IMPORTANT: paste the \"Receive crawl results\" webhook Production URL into\n// CALLBACK_URL below before going live \u2014 that is where ScraperAPI streams\n// each crawled page.\n\nconst CALLBACK_URL = 'https://YOUR-N8N-HOST/webhook/crawl-results';\n\nconst body = $json.body || {};\nconst startUrl = body.startUrl;\nif (!startUrl) {\n  throw new Error('Request body must include \"startUrl\".');\n}\n\n// Default the include filter to \"anything under the start URL\" so a bare\n// { \"startUrl\": ... } request stays scoped instead of crawling the whole\n// domain. Override by passing your own includeRegexp.\nconst escaped = startUrl.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');\n\nreturn [{\n  json: {\n    startUrl,\n    includeRegexp: body.includeRegexp || (escaped + '.*'),\n    maxDepth: body.maxDepth != null ? Number(body.maxDepth) : 1,\n    countryCode: body.countryCode || 'us',\n    render: body.render === true,\n    callbackUrl: CALLBACK_URL,\n  },\n  pairedItem: { item: 0 },\n}];",
        "language": "javaScript"
      },
      "typeVersion": 2
    },
    {
      "id": "start-crawl-job",
      "name": "Start crawler job",
      "type": "n8n-nodes-scraperapi-official.scraperApi",
      "maxTries": 2,
      "position": [
        -140,
        240
      ],
      "parameters": {
        "resource": "crawler",
        "operation": "crawlerJobCreate",
        "crawlerMaxDepth": "={{ $json.maxDepth }}",
        "crawlerStartUrl": "={{ $json.startUrl }}",
        "crawlerCallbackUrl": "={{ $json.callbackUrl }}",
        "crawlerUrlRegexpInclude": "={{ $json.includeRegexp }}",
        "crawlerOptionalParameters": {
          "crawlerApiParameters": {
            "crawlerApiRender": "={{ $json.render }}",
            "crawlerApiCountryCode": "={{ $json.countryCode }}",
            "crawlerApiOutputFormat": "markdown"
          },
          "crawlerScheduleInterval": "once"
        }
      },
      "retryOnFail": true,
      "typeVersion": 1,
      "waitBetweenTries": 2000
    },
    {
      "id": "respond-job",
      "name": "Respond with crawler job",
      "type": "n8n-nodes-base.respondToWebhook",
      "position": [
        100,
        240
      ],
      "parameters": {
        "options": {},
        "respondWith": "json",
        "responseBody": "={{ $json }}"
      },
      "typeVersion": 1.1
    },
    {
      "id": "results-webhook",
      "name": "Receive crawl results",
      "type": "n8n-nodes-base.webhook",
      "position": [
        -640,
        600
      ],
      "parameters": {
        "path": "crawl-results",
        "options": {},
        "httpMethod": "POST",
        "responseMode": "onReceived"
      },
      "typeVersion": 2
    },
    {
      "id": "parse-results",
      "name": "Parse crawled pages",
      "type": "n8n-nodes-base.code",
      "position": [
        -400,
        600
      ],
      "parameters": {
        "jsCode": "// ScraperAPI streams one POST per crawled page to this webhook, plus a\n// final job-summary POST when the crawl finishes. Keep only successful\n// per-page results; drop the summary and any failed fetches.\n\nconst payload = $json.body || $json;\nconst results = Array.isArray(payload) ? payload : [payload];\n\nconst out = [];\nfor (const r of results) {\n  const resp = r && r.response;\n  const status = resp && resp.statusCode;\n  // A page result has a url and a response.body string; the job summary\n  // (jobState / finished counters) and failed fetches are skipped.\n  if (!r || !r.url || !resp || typeof resp.body !== 'string') continue;\n  if (status && status >= 400) continue;\n  out.push({\n    json: {\n      url: r.url,\n      content: resp.body,\n      status_code: status || 200,\n      scraped_at: new Date().toISOString(),\n    },\n    pairedItem: { item: 0 },\n  });\n}\n\nreturn out;",
        "language": "javaScript"
      },
      "typeVersion": 2
    },
    {
      "id": "store-results",
      "name": "Store crawled pages",
      "type": "n8n-nodes-base.dataTable",
      "position": [
        -140,
        600
      ],
      "parameters": {
        "columns": {
          "value": {
            "url": "={{ $json.url }}",
            "content": "={{ $json.content }}",
            "scraped_at": "={{ $json.scraped_at }}",
            "status_code": "={{ $json.status_code }}"
          },
          "mappingMode": "defineBelow"
        },
        "filters": {
          "conditions": [
            {
              "keyName": "url",
              "keyValue": "={{ $json.url }}",
              "condition": "eq"
            }
          ]
        },
        "options": {},
        "resource": "row",
        "matchType": "anyCondition",
        "operation": "upsert",
        "dataTableId": {
          "__rl": true,
          "mode": "name",
          "value": "crawl_results"
        }
      },
      "typeVersion": 1.1
    }
  ],
  "connections": {
    "Set up storage": {
      "main": [
        [
          {
            "node": "Create crawled-pages table",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Start crawler job": {
      "main": [
        [
          {
            "node": "Respond with crawler job",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Parse crawled pages": {
      "main": [
        [
          {
            "node": "Store crawled pages",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Receive crawl request": {
      "main": [
        [
          {
            "node": "Build crawl parameters",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Receive crawl results": {
      "main": [
        [
          {
            "node": "Parse crawled pages",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Build crawl parameters": {
      "main": [
        [
          {
            "node": "Start crawler job",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}