{ "name": "arxiv_ingestion_pipeline", "nodes": [ { "parameters": { "rule": { "interval": [ { "field": "minutes", "minutesInterval": 8 } ] } }, "type": "n8n-nodes-base.scheduleTrigger", "typeVersion": 1.3, "position": [ -1440, 880 ], "id": "cf148930-b60c-4b7f-99f3-04bdfbdab796", "name": "CRON: 8m Interval" }, { "parameters": { "documentId": { "__rl": true, "value": "1jJm8dEdfcf3DDEiiSQVGXNUW5Yf5gDxl5F_OnfRR5Ac", "mode": "list", "cachedResultName": "sheet5", "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1jJm8dEdfcf3DDEiiSQVGXNUW5Yf5gDxl5F_OnfRR5Ac/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "value": "gid=0", "mode": "list", "cachedResultName": "Sheet1", "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1jJm8dEdfcf3DDEiiSQVGXNUW5Yf5gDxl5F_OnfRR5Ac/edit#gid=0" }, "filtersUI": { "values": [ { "lookupColumn": "ID", "lookupValue": "1" } ] }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ -1216, 880 ], "id": "0550df15-7332-4c33-8c1a-2db4b21f1746", "name": "Read State: Last Token", "credentials": { "googleSheetsOAuth2Api": { "name": "" } } }, { "parameters": { "url": "=http://export.arxiv.org/oai2", "sendQuery": true, "queryParameters": { "parameters": [ { "name": "verb", "value": "ListRecords" }, { "name": "metadataPrefix", "value": "arXiv" }, { "name": "from", "value": "2020-01-01" } ] }, "sendHeaders": true, "headerParameters": { "parameters": [ { "name": "Agent", "value": "ResearchBot/3.0 (melinmerlin89@gmail.com)" } ] }, "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.4, "position": [ -768, 784 ], "id": "64862b75-4b97-4591-8a8d-26cc85e5c11a", "name": "API: Initial Fetch (Date/Category)", "retryOnFail": true, "waitBetweenTries": 5000, "maxTries": 5 }, { "parameters": { "url": "=http://export.arxiv.org/oai2", "sendQuery": true, "queryParameters": { "parameters": [ { "name": "verb", "value": "ListRecords" }, { "name": "resumptionToken", "value": "={{ $json.Token_Manager.trim()}}" } ] }, "sendHeaders": true, "headerParameters": { "parameters": [ { "name": "Agent", "value": "ResearchBot/3.0 (melinmerlin89@gmail.com)" } ] }, "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.4, "position": [ -768, 976 ], "id": "99d3f926-0d18-4ade-900e-fb26b8eeb185", "name": "API: Resume Fetch (Token)", "retryOnFail": true, "waitBetweenTries": 5000, "maxTries": 5 }, { "parameters": { "options": {} }, "type": "n8n-nodes-base.xml", "typeVersion": 1, "position": [ -544, 880 ], "id": "033bc474-003c-4f8f-bda4-a3fe0be2669b", "name": "Parse: OAI-PMH XML to JSON" }, { "parameters": { "fieldToSplitOut": "['OAI-PMH'].ListRecords.record", "include": "selectedOtherFields", "fieldsToInclude": "resumptionToken", "options": {} }, "type": "n8n-nodes-base.splitOut", "typeVersion": 1, "position": [ -320, 592 ], "id": "d02620da-7dfa-48b9-857e-de1642e01e90", "name": "Batch: Extract Papers Array" }, { "parameters": { "conditions": { "options": { "caseSensitive": false, "leftValue": "", "typeValidation": "strict", "version": 3 }, "conditions": [ { "id": "29f25975-5782-482c-8da3-37dc8ba3f997", "leftValue": "={{ $json['[\\'OAI-PMH\\'].ListRecords.record'].metadata.arXiv.title }}", "rightValue": "=/(perovskite solar cells|photovoltaic|concentrated solar power|quantum dot solar|multi-junction solar|nuclear fusion|tokamak|stellarator|magnetic confinement|plasma heating|nuclear fission|small modular reactor|wind turbine aerodynamics|offshore wind|hydrokinetic|tidal energy|wave energy converter|geothermal power|piezoelectric nanogenerator|thermoelectric generator|triboelectric nanogenerator|hydrogen fuel cell|solid oxide fuel cell|proton exchange membrane|solid-state batteries|lithium-sulfur batteries|supercapacitors|thermal energy storage|biomass gasification|biofuel cell|energy harvesting|wireless power transfer|smart grid|microgrid|power electronics|energy conversion efficiency|permanent magnet generator|electromagnetic induction|magnetic levitation generator|magnetic energy harvesting)/i", "operator": { "type": "string", "operation": "regex" } }, { "id": "b35623fb-c1d7-4b3e-a6df-ed9d3253a005", "leftValue": "={{ $json['[\\'OAI-PMH\\'].ListRecords.record'].metadata.arXiv.abstract }}", "rightValue": "=/(perovskite solar cells|photovoltaic|concentrated solar power|quantum dot solar|multi-junction solar|nuclear fusion|tokamak|stellarator|magnetic confinement|plasma heating|nuclear fission|small modular reactor|wind turbine aerodynamics|offshore wind|hydrokinetic|tidal energy|wave energy converter|geothermal power|piezoelectric nanogenerator|thermoelectric generator|triboelectric nanogenerator|hydrogen fuel cell|solid oxide fuel cell|proton exchange membrane|solid-state batteries|lithium-sulfur batteries|supercapacitors|thermal energy storage|biomass gasification|biofuel cell|energy harvesting|wireless power transfer|smart grid|microgrid|power electronics|energy conversion efficiency|permanent magnet generator|electromagnetic induction|magnetic levitation generator|magnetic energy harvesting)/i", "operator": { "type": "string", "operation": "regex" } } ], "combinator": "or" }, "options": { "ignoreCase": true } }, "id": "ac75d090-6708-4882-ade7-742ff17b3496", "name": "Filter: Target Physics & Energy Topics", "type": "n8n-nodes-base.filter", "typeVersion": 2.3, "position": [ -96, 592 ] }, { "parameters": { "operation": "append", "documentId": { "__rl": true, "value": "1zynKp2284M_nGBNSiu_fGzmE3lyzLtSrWk-e1LOBagY", "mode": "list", "cachedResultName": "energy data", "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1zynKp2284M_nGBNSiu_fGzmE3lyzLtSrWk-e1LOBagY/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "value": "gid=0", "mode": "list", "cachedResultName": "Sheet1", "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1zynKp2284M_nGBNSiu_fGzmE3lyzLtSrWk-e1LOBagY/edit#gid=0" }, "columns": { "mappingMode": "defineBelow", "value": { "Year": "={{ $json['[\\'OAI-PMH\\'].ListRecords.record'].header.datestamp }}", "Title ": "={{ $json['[\\'OAI-PMH\\'].ListRecords.record'].metadata.arXiv.title }}", "abstract": "={{ $json['[\\'OAI-PMH\\'].ListRecords.record'].metadata.arXiv.abstract }}", "Download URL": "={{ 'https://arxiv.org/pdf/' +$json['[\\'OAI-PMH\\'].ListRecords.record'].metadata.arXiv.id + '.pdf'}} ", "Paper ID ": "={{'http://arxiv.org/abs/' + $json['[\\'OAI-PMH\\'].ListRecords.record'].metadata.arXiv.id }}", "Citation Count": "0" }, "matchingColumns": [], "schema": [ { "id": "Paper ID ", "displayName": "Paper ID ", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "Title ", "displayName": "Title ", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "Year", "displayName": "Year", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "Citation Count", "displayName": "Citation Count", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "Download URL", "displayName": "Download URL", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "abstract", "displayName": "abstract", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true } ], "attemptToConvertTypes": false, "convertFieldsToString": false }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ 128, 592 ], "id": "6ff1b7be-fd35-4837-8524-0b33063b5cfa", "name": "DB: Save Target Papers", "credentials": { "googleSheetsOAuth2Api": { "name": "" } } }, { "parameters": { "operation": "update", "documentId": { "__rl": true, "value": "1jJm8dEdfcf3DDEiiSQVGXNUW5Yf5gDxl5F_OnfRR5Ac", "mode": "list", "cachedResultName": "sheet5", "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1jJm8dEdfcf3DDEiiSQVGXNUW5Yf5gDxl5F_OnfRR5Ac/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "value": "gid=0", "mode": "list", "cachedResultName": "Sheet1", "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1jJm8dEdfcf3DDEiiSQVGXNUW5Yf5gDxl5F_OnfRR5Ac/edit#gid=0" }, "columns": { "mappingMode": "defineBelow", "value": { "Token_Manager": "={{ $json[\"OAI-PMH\"][\"ListRecords\"][\"resumptionToken\"][\"_\"] }} ", "ID": "1" }, "matchingColumns": [ "ID" ], "schema": [ { "id": "ID", "displayName": "ID", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true, "removed": false }, { "id": "Token_Manager", "displayName": "Token_Manager", "required": false, "defaultMatch": false, "display": true, "type": "string", "canBeUsedToMatch": true }, { "id": "row_number", "displayName": "row_number", "required": false, "defaultMatch": false, "display": true, "type": "number", "canBeUsedToMatch": true, "readOnly": true, "removed": true } ], "attemptToConvertTypes": false, "convertFieldsToString": false }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ -320, 784 ], "id": "6f2e395e-c250-4469-8e50-9d75b4dc2393", "name": "State: Update Resumption Token", "credentials": { "googleSheetsOAuth2Api": { "name": "" } } }, { "parameters": { "chatId": "5071054083", "text": "={{ $json.Token_Manager.trim()}}", "additionalFields": { "appendAttribution": false } }, "type": "n8n-nodes-base.telegram", "typeVersion": 1.2, "position": [ -96, 784 ], "id": "7324a161-ecca-4a33-a6d3-061c22cffc70", "name": "Telegram Alert: Batch Processed", "executeOnce": true, "credentials": { "telegramApi": { "name": "" } } }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 3 }, "conditions": [ { "id": "889088f9-f09a-4653-b913-975559d48b83", "leftValue": "={{ $runIndex}}", "rightValue": 7, "operator": { "type": "number", "operation": "lt" } } ], "combinator": "and" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.3, "position": [ -320, 976 ], "id": "636261f5-b6dd-48c5-8810-816b7081b9d6", "name": "Guard: RAM Protection" }, { "parameters": { "amount": 7 }, "type": "n8n-nodes-base.wait", "typeVersion": 1.1, "position": [ -96, 976 ], "id": "4a517c94-3685-4cb0-b621-8eb20e48224c", "name": "Rate Limit: API Cooldown" }, { "parameters": { "documentId": { "__rl": true, "value": "1jJm8dEdfcf3DDEiiSQVGXNUW5Yf5gDxl5F_OnfRR5Ac", "mode": "list", "cachedResultName": "sheet5", "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1jJm8dEdfcf3DDEiiSQVGXNUW5Yf5gDxl5F_OnfRR5Ac/edit?usp=drivesdk" }, "sheetName": { "__rl": true, "value": "gid=0", "mode": "list", "cachedResultName": "Sheet1", "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1jJm8dEdfcf3DDEiiSQVGXNUW5Yf5gDxl5F_OnfRR5Ac/edit#gid=0" }, "filtersUI": { "values": [ { "lookupColumn": "ID", "lookupValue": "1" } ] }, "options": {} }, "type": "n8n-nodes-base.googleSheets", "typeVersion": 4.7, "position": [ 128, 976 ], "id": "f5d7e36c-c7f6-42c3-bfb8-c7e73d9d9348", "name": "Read State: Next Page Token", "credentials": { "googleSheetsOAuth2Api": { "name": "" } } }, { "parameters": { "url": "=http://export.arxiv.org/oai2", "sendQuery": true, "queryParameters": { "parameters": [ { "name": "verb", "value": "ListRecords" }, { "name": "resumptionToken", "value": "={{ $json.Token_Manager.trim()}}" } ] }, "sendHeaders": true, "headerParameters": { "parameters": [ { "name": "Agent", "value": "ResearchBot/3.0 (melinmerlin89@gmail.com)" } ] }, "options": {} }, "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.4, "position": [ 352, 1056 ], "id": "30a4d797-327e-48a8-96cd-91e0712244f2", "name": "API: Fetch Next Page", "retryOnFail": true, "waitBetweenTries": 5000, "maxTries": 5 }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict", "version": 3 }, "conditions": [ { "id": "925d0149-8d3d-4462-9910-e4f9cc12afd3", "leftValue": "={{ $json.Token_Manager.trim()}}", "rightValue": "", "operator": { "type": "string", "operation": "empty", "singleValue": true } } ], "combinator": "and" }, "options": {} }, "type": "n8n-nodes-base.if", "typeVersion": 2.3, "position": [ -992, 880 ], "id": "4a7436cc-f39e-4b14-b894-446213f1c372", "name": "Check Token Value" } ], "connections": { "CRON: 8m Interval": { "main": [ [ { "node": "Read State: Last Token", "type": "main", "index": 0 } ] ] }, "Read State: Last Token": { "main": [ [ { "node": "Check Token Value", "type": "main", "index": 0 } ] ] }, "API: Initial Fetch (Date/Category)": { "main": [ [ { "node": "Parse: OAI-PMH XML to JSON", "type": "main", "index": 0 } ] ] }, "API: Resume Fetch (Token)": { "main": [ [ { "node": "Parse: OAI-PMH XML to JSON", "type": "main", "index": 0 } ] ] }, "Parse: OAI-PMH XML to JSON": { "main": [ [ { "node": "Batch: Extract Papers Array", "type": "main", "index": 0 }, { "node": "State: Update Resumption Token", "type": "main", "index": 0 }, { "node": "Guard: RAM Protection", "type": "main", "index": 0 } ] ] }, "Batch: Extract Papers Array": { "main": [ [ { "node": "Filter: Target Physics & Energy Topics", "type": "main", "index": 0 } ] ] }, "Filter: Target Physics & Energy Topics": { "main": [ [ { "node": "DB: Save Target Papers", "type": "main", "index": 0 } ] ] }, "State: Update Resumption Token": { "main": [ [ { "node": "Telegram Alert: Batch Processed", "type": "main", "index": 0 } ] ] }, "Guard: RAM Protection": { "main": [ [ { "node": "Rate Limit: API Cooldown", "type": "main", "index": 0 } ] ] }, "Rate Limit: API Cooldown": { "main": [ [ { "node": "Read State: Next Page Token", "type": "main", "index": 0 } ] ] }, "Read State: Next Page Token": { "main": [ [ { "node": "API: Fetch Next Page", "type": "main", "index": 0 } ] ] }, "API: Fetch Next Page": { "main": [ [ { "node": "Parse: OAI-PMH XML to JSON", "type": "main", "index": 0 } ] ] }, "Check Token Value": { "main": [ [ { "node": "API: Initial Fetch (Date/Category)", "type": "main", "index": 0 } ], [ { "node": "API: Resume Fetch (Token)", "type": "main", "index": 0 } ] ] } }, "active": false, "settings": { "executionOrder": "v1", "binaryMode": "separate", "availableInMCP": false }, "versionId": "ec47aa27-8e28-47d7-bace-aa3ed5ec9d09", "meta": { "templateCredsSetupCompleted": true }, "id": "kpM27XX7h5bmUREW0WgXM", "tags": [] }