{
  "name": "Bulk site URL discovery with Firecrawl Map and Google Sheets",
  "nodes": [
    {
      "id": "7249c312-0d90-4505-bdfd-8d9a44f99c0d",
      "name": "Sticky Note",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -1088,
        416
      ],
      "parameters": {
        "width": 480,
        "height": 896,
        "content": "## Bulk site URL discovery with Firecrawl Map and Google Sheets\n\n### How it works\n\n1. Trigger the workflow manually to start processing.\n2. Retrieve site URLs from a Google Sheet.\n3. Filter the list to remove processed sites.\n4. Loop through each site to map and save results.\n5. Export mapped URLs to the Google Sheet and handle errors.\n\n### Setup steps\n\n- [ ] Set up Google Sheets with necessary fields for site URLs.\n- [ ] Configure Firecrawl API access for site mapping.\n- [ ] Ensure write permissions to Google Sheets for saving results.\n- [ ] Set up error logging in Google Sheets.\n\n### Customization\n\nYou may adjust the Firecrawl API URL for different endpoints or modify the batch size for processing."
      },
      "typeVersion": 1
    },
    {
      "id": "3efed257-0eac-4b0d-8803-2da8142dbc41",
      "name": "Sticky Note1",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -528,
        832
      ],
      "parameters": {
        "color": 7,
        "width": 240,
        "height": 304,
        "content": "## Workflow initiation\n\nStarts the workflow when manually executed."
      },
      "typeVersion": 1
    },
    {
      "id": "0cf68bf9-0868-4804-896d-d33ed9a49736",
      "name": "Sticky Note2",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -256,
        832
      ],
      "parameters": {
        "color": 7,
        "width": 416,
        "height": 304,
        "content": "## Retrieve and filter sites\n\nFetches sites from Google Sheets and filters out already processed ones."
      },
      "typeVersion": 1
    },
    {
      "id": "54c7a1a1-264d-47d6-9d1b-42dbc169af85",
      "name": "Sticky Note3",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        192,
        752
      ],
      "parameters": {
        "color": 7,
        "width": 416,
        "height": 384,
        "content": "## Site processing loop\n\nHandles iteration over each site for processing and completion check."
      },
      "typeVersion": 1
    },
    {
      "id": "9a916c78-158d-4b85-869a-9c02e7d928c2",
      "name": "Sticky Note4",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        368,
        416
      ],
      "parameters": {
        "color": 7,
        "width": 416,
        "height": 304,
        "content": "## Prepare and map data\n\nPrepares site data and sends it to Firecrawl for mapping."
      },
      "typeVersion": 1
    },
    {
      "id": "886b4608-fb7a-4600-81a4-d3a66e209139",
      "name": "Sticky Note5",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        816,
        464
      ],
      "parameters": {
        "color": 7,
        "width": 416,
        "height": 304,
        "content": "## Process mapping results\n\nFilters mapped URLs and saves them in Google Sheets."
      },
      "typeVersion": 1
    },
    {
      "id": "71c6f824-380a-4505-a472-e1208e6b0482",
      "name": "Sticky Note6",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        816,
        800
      ],
      "parameters": {
        "color": 7,
        "width": 864,
        "height": 544,
        "content": "## Mark and pause\n\nMarks site as processed or logs errors, then pauses if needed."
      },
      "typeVersion": 1
    },
    {
      "id": "b1000000-0000-4000-8000-000000000001",
      "name": "When clicking 'Execute workflow'",
      "type": "n8n-nodes-base.manualTrigger",
      "position": [
        -480,
        960
      ],
      "parameters": {},
      "typeVersion": 1
    },
    {
      "id": "b1000000-0000-4000-8000-000000000002",
      "name": "Get Sites from Sheet",
      "type": "n8n-nodes-base.googleSheets",
      "position": [
        -208,
        960
      ],
      "parameters": {
        "options": {},
        "sheetName": {
          "__rl": true,
          "mode": "name",
          "value": "Sites"
        },
        "documentId": {
          "__rl": true,
          "mode": "id",
          "value": "YOUR_GOOGLE_SHEET_ID"
        }
      },
      "typeVersion": 4.7
    },
    {
      "id": "b1000000-0000-4000-8000-000000000003",
      "name": "Keep Unprocessed Sites",
      "type": "n8n-nodes-base.filter",
      "position": [
        16,
        960
      ],
      "parameters": {
        "options": {},
        "conditions": {
          "options": {
            "version": 3,
            "leftValue": "",
            "caseSensitive": true,
            "typeValidation": "strict"
          },
          "combinator": "and",
          "conditions": [
            {
              "id": "6d04720a-8f28-4546-8db9-e3eabbb5571f",
              "operator": {
                "type": "string",
                "operation": "empty",
                "singleValue": true
              },
              "leftValue": "={{ $json.Status }}",
              "rightValue": ""
            }
          ]
        }
      },
      "typeVersion": 2.3
    },
    {
      "id": "b1000000-0000-4000-8000-000000000004",
      "name": "Loop Over Sites",
      "type": "n8n-nodes-base.splitInBatches",
      "position": [
        240,
        960
      ],
      "parameters": {
        "options": {}
      },
      "typeVersion": 3
    },
    {
      "id": "b1000000-0000-4000-8000-000000000005",
      "name": "Workflow Complete",
      "type": "n8n-nodes-base.noOp",
      "position": [
        464,
        880
      ],
      "parameters": {},
      "typeVersion": 1
    },
    {
      "id": "b1000000-0000-4000-8000-000000000006",
      "name": "Prepare Site Data",
      "type": "n8n-nodes-base.set",
      "position": [
        416,
        544
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "7930e307-5a71-406f-8794-401dfe1693f7",
              "name": "site_name",
              "type": "string",
              "value": "={{ $json.Name }}"
            },
            {
              "id": "eaf46fb8-61cb-40c7-9233-1e56da8fde4b",
              "name": "site_url",
              "type": "string",
              "value": "={{ $json.Website.replace('www.', '') }}"
            }
          ]
        }
      },
      "typeVersion": 3.4
    },
    {
      "id": "b1000000-0000-4000-8000-000000000007",
      "name": "Firecrawl: Map Site",
      "type": "n8n-nodes-base.httpRequest",
      "onError": "continueErrorOutput",
      "maxTries": 3,
      "position": [
        640,
        544
      ],
      "parameters": {
        "url": "https://api.firecrawl.dev/v2/map",
        "method": "POST",
        "options": {
          "timeout": 120000,
          "response": {
            "response": {
              "neverError": false
            }
          }
        },
        "jsonBody": "={{ JSON.stringify({\n  url: $json.site_url,\n  sitemap: 'skip',\n  includeSubdomains: true,\n  limit: 5000\n}) }}",
        "sendBody": true,
        "specifyBody": "json",
        "authentication": "genericCredentialType",
        "genericAuthType": "httpHeaderAuth"
      },
      "retryOnFail": true,
      "typeVersion": 4.2,
      "alwaysOutputData": false,
      "waitBetweenTries": 5000
    },
    {
      "id": "b1000000-0000-4000-8000-000000000008",
      "name": "Filter Project URLs",
      "type": "n8n-nodes-base.code",
      "position": [
        864,
        592
      ],
      "parameters": {
        "jsCode": "// ---------------------------------------------------------------\n// Filter raw Firecrawl links down to likely project / content pages.\n// Tune the three config blocks below for your own use case.\n// ---------------------------------------------------------------\n\n// URLs containing any of these fragments are discarded.\nconst SKIP_FRAGMENTS = [\n  'sitemap', '/feed', '/wp-json',\n  '/privacy', '/terms', '/cookie',\n  '/contact', '/about', '/team',\n  '/career', '/job',\n  '/subscribe', '/newsletter',\n  '/login', '/account', '/cart',\n  '/news/', '/blog/', '/event', '/press',\n  '/reel', '/shop', '/store',\n];\n\n// File extensions to discard.\nconst SKIP_EXTENSIONS = ['.xml', '.xsl'];\n\n// URLs containing any of these are treated as explicit work/project pages.\nconst WORK_FRAGMENTS = [\n  '/work/', '/project', '/portfolio',\n  '/case', '/client', '/campaign',\n];\n\n// Minimum number of path segments required for an explicit work URL to count\n// as an individual project page (e.g. example.com/work/project-slug).\nconst MIN_SEGMENTS_FOR_PROJECT = 4;\n\n// ---------------------------------------------------------------\n\nconst siteName = $('Prepare Site Data').item.json.site_name;\nconst baseUrl = $('Prepare Site Data').item.json.site_url;\n\nconst out = [];\n\nfor (const item of $input.all()) {\n  const links = item?.json?.links;\n  if (!Array.isArray(links)) continue;\n\n  for (const link of links) {\n    if (!link?.url) continue;\n\n    const url = link.url.toLowerCase();\n    const path = url.replace(/https?:\\/\\/[^\\/]+/, ''); // path only\n\n    // Skip non-content URLs\n    if (SKIP_FRAGMENTS.some((f) => url.includes(f))) continue;\n    if (SKIP_EXTENSIONS.some((e) => url.endsWith(e))) continue;\n\n    // Skip the homepage\n    if (path === '' || path === '/') continue;\n\n    // Explicit work path with a project slug on the end\n    const isExplicitWorkPage = WORK_FRAGMENTS.some((f) => url.includes(f));\n    const hasProjectSlug =\n      url.split('/').filter(Boolean).length >= MIN_SEGMENTS_FOR_PROJECT;\n\n    // Flat project URL: a single slug off the root, e.g. /spring-campaign-2024\n    const pathParts = path.split('/').filter(Boolean);\n    const isFlatProjectUrl =\n      pathParts.length === 1 &&\n      pathParts[0].length > 3 &&\n      !pathParts[0].includes('.');\n\n    if ((isExplicitWorkPage && hasProjectSlug) || isFlatProjectUrl) {\n      out.push({\n        json: {\n          site_name: siteName,\n          site_url: baseUrl,\n          url: link.url,\n        },\n      });\n    }\n  }\n}\n\nreturn out;"
      },
      "typeVersion": 2,
      "alwaysOutputData": true
    },
    {
      "id": "b1000000-0000-4000-8000-000000000009",
      "name": "Save Mapped URLs",
      "type": "n8n-nodes-base.googleSheets",
      "position": [
        1088,
        592
      ],
      "parameters": {
        "columns": {
          "value": {
            "Name": "={{ $('Prepare Site Data').item.json.site_name }}",
            "Mapped URL": "={{ $json.url }}",
            "Home Domain": "={{ $('Prepare Site Data').item.json.site_url }}"
          },
          "schema": [
            {
              "id": "Name",
              "type": "string",
              "display": true,
              "removed": false,
              "required": false,
              "displayName": "Name",
              "defaultMatch": false,
              "canBeUsedToMatch": true
            },
            {
              "id": "Home Domain",
              "type": "string",
              "display": true,
              "removed": false,
              "required": false,
              "displayName": "Home Domain",
              "defaultMatch": false,
              "canBeUsedToMatch": true
            },
            {
              "id": "Mapped URL",
              "type": "string",
              "display": true,
              "removed": false,
              "required": false,
              "displayName": "Mapped URL",
              "defaultMatch": false,
              "canBeUsedToMatch": true
            },
            {
              "id": "Error Message",
              "type": "string",
              "display": true,
              "removed": false,
              "required": false,
              "displayName": "Error Message",
              "defaultMatch": false,
              "canBeUsedToMatch": true
            }
          ],
          "mappingMode": "defineBelow",
          "matchingColumns": [
            "Mapped URL"
          ],
          "attemptToConvertTypes": false,
          "convertFieldsToString": false
        },
        "options": {},
        "operation": "appendOrUpdate",
        "sheetName": {
          "__rl": true,
          "mode": "name",
          "value": "Mapped URLs"
        },
        "documentId": {
          "__rl": true,
          "mode": "id",
          "value": "YOUR_GOOGLE_SHEET_ID"
        }
      },
      "typeVersion": 4.7
    },
    {
      "id": "b1000000-0000-4000-8000-000000000010",
      "name": "Mark Site as Done",
      "type": "n8n-nodes-base.googleSheets",
      "position": [
        1312,
        912
      ],
      "parameters": {
        "columns": {
          "value": {
            "Name": "={{ $('Prepare Site Data').item.json.site_name }}",
            "Status": "Done"
          },
          "schema": [
            {
              "id": "Name",
              "type": "string",
              "display": true,
              "removed": false,
              "required": false,
              "displayName": "Name",
              "defaultMatch": false,
              "canBeUsedToMatch": true
            },
            {
              "id": "Website",
              "type": "string",
              "display": true,
              "removed": true,
              "required": false,
              "displayName": "Website",
              "defaultMatch": false,
              "canBeUsedToMatch": true
            },
            {
              "id": "Status",
              "type": "string",
              "display": true,
              "removed": false,
              "required": false,
              "displayName": "Status",
              "defaultMatch": false,
              "canBeUsedToMatch": true
            }
          ],
          "mappingMode": "defineBelow",
          "matchingColumns": [
            "Name"
          ],
          "attemptToConvertTypes": false,
          "convertFieldsToString": false
        },
        "options": {},
        "operation": "appendOrUpdate",
        "sheetName": {
          "__rl": true,
          "mode": "name",
          "value": "Sites"
        },
        "documentId": {
          "__rl": true,
          "mode": "id",
          "value": "YOUR_GOOGLE_SHEET_ID"
        }
      },
      "typeVersion": 4.7
    },
    {
      "id": "b1000000-0000-4000-8000-000000000011",
      "name": "Log Mapping Error",
      "type": "n8n-nodes-base.googleSheets",
      "position": [
        864,
        1104
      ],
      "parameters": {
        "columns": {
          "value": {
            "Name": "={{ $('Prepare Site Data').item.json.site_name }}",
            "Home Domain": "={{ $('Prepare Site Data').item.json.site_url }}",
            "Error Message": "={{ $json.error || $json.message || 'Firecrawl map failed' }}"
          },
          "schema": [
            {
              "id": "Name",
              "type": "string",
              "display": true,
              "required": false,
              "displayName": "Name",
              "defaultMatch": false,
              "canBeUsedToMatch": true
            },
            {
              "id": "Home Domain",
              "type": "string",
              "display": true,
              "removed": false,
              "required": false,
              "displayName": "Home Domain",
              "defaultMatch": false,
              "canBeUsedToMatch": true
            },
            {
              "id": "Mapped URL",
              "type": "string",
              "display": true,
              "removed": true,
              "required": false,
              "displayName": "Mapped URL",
              "defaultMatch": false,
              "canBeUsedToMatch": true
            },
            {
              "id": "Error Message",
              "type": "string",
              "display": true,
              "removed": false,
              "required": false,
              "displayName": "Error Message",
              "defaultMatch": false,
              "canBeUsedToMatch": true
            }
          ],
          "mappingMode": "defineBelow",
          "matchingColumns": [],
          "attemptToConvertTypes": false,
          "convertFieldsToString": false
        },
        "options": {},
        "operation": "append",
        "sheetName": {
          "__rl": true,
          "mode": "name",
          "value": "Mapped URLs"
        },
        "documentId": {
          "__rl": true,
          "mode": "id",
          "value": "YOUR_GOOGLE_SHEET_ID"
        }
      },
      "typeVersion": 4.7
    },
    {
      "id": "b1000000-0000-4000-8000-000000000012",
      "name": "Mark Site as Error",
      "type": "n8n-nodes-base.googleSheets",
      "position": [
        1088,
        1104
      ],
      "parameters": {
        "columns": {
          "value": {
            "Name": "={{ $('Prepare Site Data').item.json.site_name }}",
            "Status": "Error"
          },
          "schema": [
            {
              "id": "Name",
              "type": "string",
              "display": true,
              "removed": false,
              "required": false,
              "displayName": "Name",
              "defaultMatch": false,
              "canBeUsedToMatch": true
            },
            {
              "id": "Website",
              "type": "string",
              "display": true,
              "removed": true,
              "required": false,
              "displayName": "Website",
              "defaultMatch": false,
              "canBeUsedToMatch": true
            },
            {
              "id": "Status",
              "type": "string",
              "display": true,
              "removed": false,
              "required": false,
              "displayName": "Status",
              "defaultMatch": false,
              "canBeUsedToMatch": true
            }
          ],
          "mappingMode": "defineBelow",
          "matchingColumns": [
            "Name"
          ],
          "attemptToConvertTypes": false,
          "convertFieldsToString": false
        },
        "options": {},
        "operation": "appendOrUpdate",
        "sheetName": {
          "__rl": true,
          "mode": "name",
          "value": "Sites"
        },
        "documentId": {
          "__rl": true,
          "mode": "id",
          "value": "YOUR_GOOGLE_SHEET_ID"
        }
      },
      "typeVersion": 4.7
    },
    {
      "id": "b1000000-0000-4000-8000-000000000013",
      "name": "Rate Limit Pause",
      "type": "n8n-nodes-base.wait",
      "position": [
        1536,
        1184
      ],
      "parameters": {
        "amount": 20
      },
      "typeVersion": 1.1
    },
    {
      "id": "b1000000-0000-4000-8000-000000000014",
      "name": "Rate Limit Pause (Error)",
      "type": "n8n-nodes-base.wait",
      "position": [
        1312,
        1104
      ],
      "parameters": {
        "amount": 20
      },
      "typeVersion": 1.1
    }
  ],
  "settings": {
    "executionOrder": "v1"
  },
  "_fileName": "Bulk_site_URL_discovery_with_Firecrawl_Map_and_Google_Sheets.json",
  "connections": {
    "Loop Over Sites": {
      "main": [
        [
          {
            "node": "Workflow Complete",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "Prepare Site Data",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Rate Limit Pause": {
      "main": [
        [
          {
            "node": "Loop Over Sites",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Save Mapped URLs": {
      "main": [
        [
          {
            "node": "Mark Site as Done",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Log Mapping Error": {
      "main": [
        [
          {
            "node": "Mark Site as Error",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Mark Site as Done": {
      "main": [
        [
          {
            "node": "Rate Limit Pause",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Prepare Site Data": {
      "main": [
        [
          {
            "node": "Firecrawl: Map Site",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Mark Site as Error": {
      "main": [
        [
          {
            "node": "Rate Limit Pause (Error)",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Filter Project URLs": {
      "main": [
        [
          {
            "node": "Save Mapped URLs",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Firecrawl: Map Site": {
      "main": [
        [
          {
            "node": "Filter Project URLs",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "Log Mapping Error",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Get Sites from Sheet": {
      "main": [
        [
          {
            "node": "Keep Unprocessed Sites",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Keep Unprocessed Sites": {
      "main": [
        [
          {
            "node": "Loop Over Sites",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Rate Limit Pause (Error)": {
      "main": [
        [
          {
            "node": "Loop Over Sites",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "When clicking 'Execute workflow'": {
      "main": [
        [
          {
            "node": "Get Sites from Sheet",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}