{
  "id": "F5Vc6P90dSHIJ9Iv",
  "meta": {
    "templateCredsSetupCompleted": true
  },
  "name": "Robots.txt Generator for SEO & GEO",
  "tags": [],
  "nodes": [
    {
      "id": "685f802b-b2a7-458f-8b86-ea556152fd60",
      "name": "If",
      "type": "n8n-nodes-base.if",
      "position": [
        -96,
        -48
      ],
      "parameters": {
        "options": {},
        "conditions": {
          "options": {
            "version": 3,
            "leftValue": "",
            "caseSensitive": true,
            "typeValidation": "strict"
          },
          "combinator": "and",
          "conditions": [
            {
              "id": "f1c8ee1a-b701-4dd2-98a1-3e2fe20e4a5a",
              "operator": {
                "type": "number",
                "operation": "equals"
              },
              "leftValue": "={{ $json.statusCode }}",
              "rightValue": 200
            }
          ]
        }
      },
      "typeVersion": 2.3
    },
    {
      "id": "931808e9-16fc-41f4-9808-252896157b00",
      "name": "robots.txt",
      "type": "n8n-nodes-base.httpRequest",
      "position": [
        -336,
        -48
      ],
      "parameters": {
        "url": "={{ $json.website_url }}/robots.txt",
        "options": {
          "response": {
            "response": {
              "fullResponse": true
            }
          }
        },
        "sendHeaders": true,
        "headerParameters": {
          "parameters": [
            {
              "name": "User-Agent",
              "value": "Mozilla/5.0"
            },
            {
              "name": "Accept",
              "value": "text/plain,text/html"
            }
          ]
        }
      },
      "typeVersion": 4.4
    },
    {
      "id": "6c020f9c-f263-4acc-ba95-1272f188ebbb",
      "name": "No Operation, do nothing",
      "type": "n8n-nodes-base.noOp",
      "position": [
        192,
        224
      ],
      "parameters": {},
      "typeVersion": 1
    },
    {
      "id": "76ed553f-7571-49f5-bd98-fb36ca5a066d",
      "name": "When clicking \u2018Execute workflow\u2019",
      "type": "n8n-nodes-base.manualTrigger",
      "position": [
        -880,
        -48
      ],
      "parameters": {},
      "typeVersion": 1
    },
    {
      "id": "9bd4cbe7-9575-4339-84b1-eeaab4776c18",
      "name": "Set Website Url",
      "type": "n8n-nodes-base.set",
      "position": [
        -624,
        -48
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "a03c93f9-4a91-4535-9e25-d2ec5bc61a73",
              "name": "website_url",
              "type": "string",
              "value": "https://n3w.it"
            },
            {
              "id": "40c3d988-9e79-478c-a390-585e30c69820",
              "name": "sitemap",
              "type": "string",
              "value": "https://n3w.it/sitemap.xml"
            }
          ]
        }
      },
      "typeVersion": 3.4
    },
    {
      "id": "40f14918-1263-4ec6-85da-2ecd25d7dab2",
      "name": "Get HTML",
      "type": "n8n-nodes-base.httpRequest",
      "position": [
        192,
        -256
      ],
      "parameters": {
        "url": "={{ $('Set Website Url').item.json.website_url }}",
        "options": {}
      },
      "typeVersion": 4.4
    },
    {
      "id": "556f1ba4-1b90-4e0d-80b9-b7630d37f38a",
      "name": "Discover CMS",
      "type": "@n8n/n8n-nodes-langchain.chainLlm",
      "position": [
        464,
        -256
      ],
      "parameters": {
        "text": "={{ $json.data }}",
        "batching": {},
        "messages": {
          "messageValues": [
            {
              "message": "=You are a CMS detection assistant.\n\nYou will receive the HTML source code of a website homepage.\n\nYour task is to detect the CMS used by the website (e.g. WordPress, Joomla, Drupal, Shopify, Magento, Wix, Squarespace, PrestaShop, Webflow, Ghost, Blogger, etc.).\n\nReturn ONLY a valid JSON object in this format:\n\n{\n  \"cms\": \"WordPress\"\n}\n\nIf no common CMS can be confidently identified, return:\n\n{\n  \"cms\": \"not detected\"\n}\n\nDo not include explanations, markdown, or any additional text. Output only the JSON."
            }
          ]
        },
        "promptType": "define"
      },
      "typeVersion": 1.9
    },
    {
      "id": "70908f48-5bbe-4b1e-be56-65a89422208a",
      "name": "OpenAI Chat Model",
      "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
      "position": [
        848,
        -16
      ],
      "parameters": {
        "model": {
          "__rl": true,
          "mode": "list",
          "value": "gpt-5.4-mini",
          "cachedResultName": "gpt-5.4-mini"
        },
        "options": {},
        "builtInTools": {}
      },
      "credentials": {
        "openAiApi": {
          "name": "<your credential>"
        }
      },
      "typeVersion": 1.3
    },
    {
      "id": "9b780136-785f-4f04-9e5f-e2cad9a445d4",
      "name": "Parser",
      "type": "n8n-nodes-base.code",
      "position": [
        832,
        -256
      ],
      "parameters": {
        "jsCode": "return items.map(item => {\n  return {\n    json: JSON.parse(item.json.text)\n  };\n});"
      },
      "typeVersion": 2
    },
    {
      "id": "8b9e95e3-722d-4195-ba54-6e0e2944ab98",
      "name": "Robots.txt Generator",
      "type": "@n8n/n8n-nodes-langchain.chainLlm",
      "position": [
        1120,
        -256
      ],
      "parameters": {
        "text": "=robots.txt: {{ $('robots.txt').item.json.data }}\nSitemap url: {{ $('Set Website Url').item.json.sitemap }}\nCMS: {{ $json.cms }}",
        "batching": {},
        "messages": {
          "messageValues": [
            {
              "message": "=You are an expert Technical SEO and GEO specialist.\n\nYou will receive:\n- The current robots.txt content\n- The sitemap URL\n- The detected CMS\n\nInput:\n- current robots.txt\n- Sitemap url\n- CMS\n\nYour task is to generate an optimized robots.txt file for SEO and GEO.\n\nRules:\n- Adapt the robots.txt to the detected CMS when possible.\n- If CMS is \"not detected\", generate a safe generic robots.txt.\n- Do not block important public pages, assets, CSS, JS, images, or sitemap access.\n- Block only useless, duplicate, private, admin, internal search, cart, checkout, account, login, filter, and tracking URLs when appropriate for the CMS.\n- Allow major search engines and AI/LLM crawlers when useful for GEO visibility.\n- Include the Sitemap directive using the provided sitemap URL.\n- Keep the file clean, valid, and production-ready.\n- Use only standard robots.txt syntax: User-agent, Allow, Disallow, Sitemap.\n- Return ONLY the final robots.txt content.\n- Do not include explanations, markdown, JSON, comments outside the robots.txt file, or extra text."
            }
          ]
        },
        "promptType": "define"
      },
      "typeVersion": 1.9
    },
    {
      "id": "720940e0-0c37-4106-9132-978e122a266b",
      "name": "Convert to File",
      "type": "n8n-nodes-base.convertToFile",
      "position": [
        1536,
        -256
      ],
      "parameters": {
        "options": {
          "fileName": "robots.txt"
        },
        "operation": "toText",
        "sourceProperty": "text"
      },
      "typeVersion": 1.1
    },
    {
      "id": "36fa0c43-f820-4563-b684-21f8058a8ba7",
      "name": "Upload file",
      "type": "n8n-nodes-base.googleDrive",
      "position": [
        1760,
        -256
      ],
      "parameters": {
        "name": "={{$binary.data.fileName}}",
        "driveId": {
          "__rl": true,
          "mode": "list",
          "value": "My Drive"
        },
        "options": {},
        "folderId": {
          "__rl": true,
          "mode": "list",
          "value": "root",
          "cachedResultName": "/ (Root folder)"
        }
      },
      "credentials": {
        "googleDriveOAuth2Api": {
          "name": "<your credential>"
        }
      },
      "typeVersion": 3
    },
    {
      "id": "dbee1971-0c32-4549-86c0-a5ca33758739",
      "name": "Sticky Note11",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        1056,
        -1216
      ],
      "parameters": {
        "color": 7,
        "width": 736,
        "height": 784,
        "content": "## MY NEW YOUTUBE CHANNEL\n\ud83d\udc49 [Subscribe to my new **YouTube channel**](https://youtube.com/@n3witalia). Here I\u2019ll share videos and Shorts with practical tutorials and **FREE templates for n8n**.\n\n[![image](https://n3wstorage.b-cdn.net/n3witalia/youtube-n8n-cover.jpg)](https://youtube.com/@n3witalia)"
      },
      "typeVersion": 1
    },
    {
      "id": "16cbf623-aa2d-4569-b460-599e06051519",
      "name": "Sticky Note",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        160,
        -1072
      ],
      "parameters": {
        "width": 864,
        "height": 640,
        "content": "# Robots.txt Generator for SEO & GEO\n\n### How it works\n\n1. The workflow starts manually and sets the target website URL and sitemap values.\n2. It checks whether the site already has a robots.txt file and branches accordingly.\n3. If needed, it fetches the site HTML, uses an LLM to identify the CMS, and parses the result.\\n4. It generates a robots.txt file with another LLM chain, converts it to a file, and uploads it to Google Drive.\n\n### Setup steps\n- [ ] Configure the manual trigger or replace it with your preferred input trigger.\n- [ ] Set the target website URL and any sitemap value in the Set Website Url node.\n- [ ] Ensure the HTTP Request nodes can access the target site without blocking or authentication issues.\n- [ ] Connect and authorize the OpenAI credentials used by the LLM chain nodes.\n- [ ] Review the robot generation prompt and parser logic so the output matches your SEO/GEO rules\n- [ ] Configure Google Drive credentials and choose the destination folder for the generated file.\n\n### Customization\nYou can customize the website URL input, the CMS detection logic, the robots.txt rules generated by the LLM, and the Google Drive upload destination."
      },
      "typeVersion": 1
    },
    {
      "id": "5393cbbf-50ec-4fe8-868c-e867c5249237",
      "name": "Sticky Note1",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -688,
        -240
      ],
      "parameters": {
        "color": 7,
        "width": 256,
        "height": 368,
        "content": "## Manual site input\nStarts the workflow and initializes the website URL and sitemap values used downstream."
      },
      "typeVersion": 1
    },
    {
      "id": "0907927c-0e1d-4cf5-962b-60c650734f77",
      "name": "Sticky Note2",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -400,
        -240
      ],
      "parameters": {
        "color": 7,
        "height": 368,
        "content": "## Check existing robots\nFetches the site's robots.txt file and branches based on whether it should continue with generation or stop."
      },
      "typeVersion": 1
    },
    {
      "id": "8286cf0c-7167-4c3b-8978-c5c0a2789b19",
      "name": "Sticky Note3",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        160,
        -400
      ],
      "parameters": {
        "color": 7,
        "width": 864,
        "height": 336,
        "content": "## Detect CMS and parse\nRetrieves the homepage HTML, infers the site CMS with an LLM chain, and normalizes the output with custom code."
      },
      "typeVersion": 1
    },
    {
      "id": "582dae63-52bc-4a93-b6b8-e3a60dffc903",
      "name": "Sticky Note4",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        1056,
        -400
      ],
      "parameters": {
        "color": 7,
        "width": 880,
        "height": 336,
        "content": "## Generate and upload file\nCreates the final robots.txt content, converts it into a file, and uploads it to Google Drive."
      },
      "typeVersion": 1
    },
    {
      "id": "ba1006e8-ab51-4175-8b4d-62954682a919",
      "name": "Sticky Note5",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        160,
        96
      ],
      "parameters": {
        "color": 7,
        "width": 880,
        "height": 272,
        "content": "## Existing file exit\n\nStops the workflow on the branch where no further action is required, leaving the existing robots.txt untouched."
      },
      "typeVersion": 1
    }
  ],
  "active": false,
  "settings": {
    "binaryMode": "separate",
    "executionOrder": "v1"
  },
  "versionId": "70e83d62-c50b-4a1d-bef0-5eb9c0acd83c",
  "nodeGroups": [],
  "connections": {
    "If": {
      "main": [
        [
          {
            "node": "Get HTML",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "No Operation, do nothing",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Parser": {
      "main": [
        [
          {
            "node": "Robots.txt Generator",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Get HTML": {
      "main": [
        [
          {
            "node": "Discover CMS",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "robots.txt": {
      "main": [
        [
          {
            "node": "If",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Discover CMS": {
      "main": [
        [
          {
            "node": "Parser",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Convert to File": {
      "main": [
        [
          {
            "node": "Upload file",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Set Website Url": {
      "main": [
        [
          {
            "node": "robots.txt",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "OpenAI Chat Model": {
      "ai_languageModel": [
        [
          {
            "node": "Discover CMS",
            "type": "ai_languageModel",
            "index": 0
          },
          {
            "node": "Robots.txt Generator",
            "type": "ai_languageModel",
            "index": 0
          }
        ]
      ]
    },
    "Robots.txt Generator": {
      "main": [
        [
          {
            "node": "Convert to File",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "When clicking \u2018Execute workflow\u2019": {
      "main": [
        [
          {
            "node": "Set Website Url",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}