{
  "id": "eGHpM6RNrSQcTJRt",
  "meta": {
    "templateCredsSetupCompleted": true
  },
  "name": "ideabrowser-scraper",
  "tags": [],
  "nodes": [
    {
      "id": "a19357ec-0bff-4f07-b403-fa18f82abf86",
      "name": "Markdown1",
      "type": "n8n-nodes-base.markdown",
      "position": [
        2040,
        800
      ],
      "parameters": {
        "html": "={{ $json.data }}",
        "options": {}
      },
      "typeVersion": 1
    },
    {
      "id": "7fbca587-d982-4800-b559-6e01008af711",
      "name": "Get the links",
      "type": "n8n-nodes-base.code",
      "position": [
        580,
        800
      ],
      "parameters": {
        "jsCode": "for (const item of $input.all()) {\n    const rawData = item.json.data;\n\n    // Step 1: Extract base path for idea\n    const basePathRegex = /\\/idea\\/[^\"\\/]+/;\n    const basePathMatch = rawData.match(basePathRegex);\n\n    if (!basePathMatch) {\n        item.json.error = \"No valid idea path found in rawData\";\n        continue;\n    }\n\n    const basePath = basePathMatch[0]; // e.g., /idea/ai-powered-historical-ad-style-generator\n\n    // Step 2: Generate insight links\n    const insightPages = [\n        \"\",\n        \"/value-ladder\",\n        \"/why-now\",\n        \"/proof-signals\",\n        \"/market-gap\",\n        \"/execution-plan\",\n        \"/value-equation\",\n        \"/value-matrix\",\n        \"/acp\",\n        \"/community-signals\",\n        \"/keywords\"\n    ];\n\n    const insightLinks = insightPages.map(page => {\n        return `https://www.ideabrowser.com${basePath}${page}`; \n    });\n\n    // Step 3: Extract only readable text by removing HTML tags\n    const cleanText = rawData.replace(/<[^>]+>/g, '').trim();\n\n    // Step 4: Update item with results\n    item.json.insightLinks = insightLinks;\n    item.json.ideaPath = basePath;\n    item.json.cleanText = cleanText; // \ud83d\udc48 This is the new key with only readable text\n}\n\nreturn $input.all();"
      },
      "typeVersion": 2
    },
    {
      "id": "10bfccb0-c5d3-4002-ad31-9e6d00d710a3",
      "name": "Get URL data of idea",
      "type": "n8n-nodes-base.httpRequest",
      "position": [
        380,
        800
      ],
      "parameters": {
        "url": "https://ideabrowser.com/idea-of-the-day ",
        "options": {},
        "authentication": "genericCredentialType",
        "genericAuthType": "httpHeaderAuth"
      },
      "credentials": {
        "httpHeaderAuth": {
          "name": "<your credential>"
        }
      },
      "typeVersion": 4.2
    },
    {
      "id": "2ac9f703-006b-494d-b53f-372c8dbd8233",
      "name": "Create google doc",
      "type": "n8n-nodes-base.googleDocs",
      "position": [
        780,
        580
      ],
      "parameters": {
        "title": "={{ $today.toLocaleString().concat(\"-\",$json.ideaPath.split(\"/\")[2] )}}",
        "folderId": "10Lxy7-UbLzEGcTnDk12n8pOiWyyu1EE4"
      },
      "credentials": {
        "googleDocsOAuth2Api": {
          "name": "<your credential>"
        }
      },
      "typeVersion": 2
    },
    {
      "id": "bf9d318c-310e-4c49-bbb0-3cfbc1e16f7d",
      "name": "Merge the data",
      "type": "n8n-nodes-base.merge",
      "position": [
        1000,
        800
      ],
      "parameters": {
        "mode": "combine",
        "options": {},
        "combineBy": "combineAll"
      },
      "typeVersion": 3.2
    },
    {
      "id": "286f31e3-d9e5-4a3a-8590-7c11e82c6e78",
      "name": "Split the url",
      "type": "n8n-nodes-base.splitOut",
      "position": [
        1260,
        800
      ],
      "parameters": {
        "options": {},
        "fieldToSplitOut": "insightLinks"
      },
      "typeVersion": 1
    },
    {
      "id": "7ac224d8-4eb9-429a-bae3-607e49f2d498",
      "name": "Loop over URL",
      "type": "n8n-nodes-base.splitInBatches",
      "position": [
        1580,
        780
      ],
      "parameters": {
        "options": {}
      },
      "typeVersion": 3
    },
    {
      "id": "82e59148-8aa9-4eb7-a65b-37c375d63ef0",
      "name": "Get URL content of each page",
      "type": "n8n-nodes-base.httpRequest",
      "position": [
        1820,
        800
      ],
      "parameters": {
        "url": "={{ $json.insightLinks }}",
        "options": {},
        "authentication": "genericCredentialType",
        "genericAuthType": "httpHeaderAuth"
      },
      "credentials": {
        "httpHeaderAuth": {
          "name": "<your credential>"
        }
      },
      "typeVersion": 4.2
    },
    {
      "id": "c5902af1-187e-4471-8833-8c3b68732d36",
      "name": "Update the google docs with the data",
      "type": "n8n-nodes-base.googleDocs",
      "position": [
        2240,
        800
      ],
      "parameters": {
        "actionsUi": {
          "actionFields": [
            {
              "text": "={{ $json.data }}",
              "action": "insert"
            }
          ]
        },
        "operation": "update",
        "documentURL": "={{ $('Create google doc').item.json.id }}"
      },
      "credentials": {
        "googleDocsOAuth2Api": {
          "name": "<your credential>"
        }
      },
      "typeVersion": 2
    },
    {
      "id": "0cd82b9e-1b64-4853-aaab-3036fe33aa5d",
      "name": "Schedule Trigger",
      "type": "n8n-nodes-base.scheduleTrigger",
      "position": [
        200,
        800
      ],
      "parameters": {
        "rule": {
          "interval": [
            {
              "triggerAtHour": 8
            }
          ]
        }
      },
      "typeVersion": 1.2
    }
  ],
  "active": false,
  "settings": {
    "timezone": "Asia/Kolkata",
    "callerPolicy": "workflowsFromSameOwner",
    "executionOrder": "v1",
    "executionTimeout": -1
  },
  "versionId": "a1e9d613-2210-4565-a36c-c483d51130fb",
  "connections": {
    "Markdown1": {
      "main": [
        [
          {
            "node": "Update the google docs with the data",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Get the links": {
      "main": [
        [
          {
            "node": "Create google doc",
            "type": "main",
            "index": 0
          },
          {
            "node": "Merge the data",
            "type": "main",
            "index": 1
          }
        ]
      ]
    },
    "Loop over URL": {
      "main": [
        [],
        [
          {
            "node": "Get URL content of each page",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Split the url": {
      "main": [
        [
          {
            "node": "Loop over URL",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Merge the data": {
      "main": [
        [
          {
            "node": "Split the url",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Schedule Trigger": {
      "main": [
        [
          {
            "node": "Get URL data of idea",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Create google doc": {
      "main": [
        [
          {
            "node": "Merge the data",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Get URL data of idea": {
      "main": [
        [
          {
            "node": "Get the links",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Get URL content of each page": {
      "main": [
        [
          {
            "node": "Markdown1",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Update the google docs with the data": {
      "main": [
        [
          {
            "node": "Loop over URL",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}