{ "name": "6ixo - Kijiji Hamilton Sync to CSV", "nodes": [ { "parameters": {}, "id": "1", "name": "Manual Trigger", "type": "n8n-nodes-base.manualTrigger", "typeVersion": 1, "position": [ 0, 0 ] }, { "parameters": { "rule": { "interval": [ { "field": "hours", "hoursInterval": 1 } ] } }, "id": "2", "name": "Every Hour", "type": "n8n-nodes-base.scheduleTrigger", "typeVersion": 1.2, "position": [ 0, 180 ] }, { "parameters": { "assignments": { "assignments": [ { "id": "githubToken", "name": "githubToken", "value": "={{ $env.GITHUB_TOKEN || 'PASTE_GITHUB_TOKEN_HERE' }}", "type": "string" }, { "id": "githubOwner", "name": "githubOwner", "value": "={{ $env.GITHUB_OWNER || 'bisco401' }}", "type": "string" }, { "id": "githubRepo", "name": "githubRepo", "value": "={{ $env.GITHUB_REPO || '6ixo' }}", "type": "string" }, { "id": "githubBranch", "name": "githubBranch", "value": "={{ $env.GITHUB_BRANCH || 'main' }}", "type": "string" }, { "id": "csvPath", "name": "csvPath", "value": "={{ $env.SIXO_CSV_PATH || 'data/scraped-listings.csv' }}", "type": "string" }, { "id": "crawl4aiUrl", "name": "crawl4aiUrl", "value": "={{ $env.CRAWL4AI_URL || 'http://crawl4ai:11235/crawl' }}", "type": "string" }, { "id": "defaultImportStatus", "name": "defaultImportStatus", "value": "published", "type": "string" }, { "id": "hideUnavailableListings", "name": "hideUnavailableListings", "value": "true", "type": "string" }, { "id": "availabilityMaxRows", "name": "availabilityMaxRows", "value": "24", "type": "string" }, { "id": "availabilityBatchSize", "name": "availabilityBatchSize", "value": "6", "type": "string" }, { "id": "availabilityFreshHours", "name": "availabilityFreshHours", "value": "12", "type": "string" }, { "id": "sourcesJson", "name": "sourcesJson", "value": "[\n {\n \"name\": \"Kijiji Hamilton Recent Ads\",\n \"enabled\": true,\n \"list_url\": \"https://www.kijiji.ca/b-hamilton/l80014?sort=dateDesc\",\n \"city\": \"Hamilton\",\n \"country\": \"Canada\",\n \"strict_city_match\": true,\n \"max_listings\": 50\n }\n]", "type": "string" } ] }, "options": {} }, "id": "3", "name": "Set Config Here", "type": "n8n-nodes-base.set", "typeVersion": 3.4, "position": [ 260, 90 ] }, { "parameters": { "method": "POST", "url": "={{ $json.crawl4aiUrl }}", "sendBody": true, "contentType": "json", "specifyBody": "json", "jsonBody": "={{ { urls: JSON.parse($json.sourcesJson).filter(source => source && source.enabled !== false && (source.list_url || source.url)).map(source => source.list_url || source.url), browser_config: { headless: true, viewport: { width: 1440, height: 2200 }, verbose: false }, crawler_config: { stream: false, cache_mode: \"bypass\", wait_until: \"load\", wait_for: \"css:body\", page_timeout: 45000, delay_before_return_html: 1, scan_full_page: true, remove_overlay_elements: true, remove_consent_popups: true, flatten_shadow_dom: true } } }}", "options": {} }, "id": "4", "name": "Crawl Kijiji List Pages", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 520, 180 ] }, { "parameters": { "mode": "combine", "combineBy": "combineByPosition", "options": {} }, "id": "5", "name": "Merge Config + Crawl Result", "type": "n8n-nodes-base.merge", "typeVersion": 3, "position": [ 760, 90 ] }, { "parameters": { "jsCode": "const input = $input.first().json;\nconst OWNER = input.githubOwner || 'bisco401';\nconst REPO = input.githubRepo || '6ixo';\nconst BRANCH = input.githubBranch || 'main';\nconst CSV_PATH = input.csvPath || 'data/scraped-listings.csv';\nconst TOKEN = input.githubToken;\nconst CRAWL4AI_URL = input.crawl4aiUrl || 'http://crawl4ai:11235/crawl';\nconst DEFAULT_STATUS = input.defaultImportStatus || 'published';\nconst HIDE_UNAVAILABLE = String(input.hideUnavailableListings ?? 'true').toLowerCase() === 'true';\nconst AVAILABILITY_MAX_ROWS = Math.max(0, Number(input.availabilityMaxRows || 24));\nconst AVAILABILITY_BATCH_SIZE = Math.max(1, Number(input.availabilityBatchSize || 6));\nconst AVAILABILITY_FRESH_HOURS = Math.max(0, Number(input.availabilityFreshHours || 12));\nconst nowIso = new Date().toISOString();\nconst nowMs = Date.parse(nowIso);\n\nlet sources = [];\ntry {\n sources = typeof input.sourcesJson === 'string' ? JSON.parse(input.sourcesJson || '[]') : (input.sourcesJson || []);\n} catch (error) {\n throw new Error('sourcesJson is not valid JSON. Fix the Set Config Here node first.');\n}\n\nif (!TOKEN || TOKEN === 'PASTE_GITHUB_TOKEN_HERE') throw new Error('Paste your GitHub token into the Set Config Here node, or set GITHUB_TOKEN in n8n.');\nif (!Array.isArray(sources) || !sources.length) throw new Error('Add at least one Kijiji source in sourcesJson.');\n\nconst csvHeaders = [\n 'id','status','target_surface','app_category','app_subcategory','title','price_text','price_value','currency','city','country','seller','phone','description','image_urls','source_site','source_url','scraped_at','make','model','trim','year','condition','transmission','color','mileage_km','attributes','source_availability','source_availability_checked_at','source_http_status','source_unavailable_reason','source_last_seen_at','source_resolved_url'\n];\nconst ghHeaders = { authorization: `Bearer ${TOKEN}`, accept: 'application/vnd.github+json', 'x-github-api-version': '2022-11-28' };\n\nconst httpRequest = async (options) => {\n const request = {\n method: options.method || 'GET',\n uri: options.url,\n url: options.url,\n headers: options.headers || {},\n body: options.body,\n json: options.json === true,\n resolveWithFullResponse: true,\n simple: false\n };\n try {\n const response = await this.helpers.httpRequest(request);\n const status = response.statusCode || response.status || 200;\n return { status, ok: status >= 200 && status < 300, body: response.body ?? response };\n } catch (error) {\n const status = error.statusCode || error.status || error.response?.status || error.response?.statusCode || 500;\n const body = error.response?.body || error.response?.data || error.message || '';\n return { status, ok: status >= 200 && status < 300, body };\n }\n};\n\nconst csvEscape = (value = '') => {\n const text = String(value ?? '');\n return /[\",\\n\\r]/.test(text) ? `\"${text.replace(/\"/g, '\"\"')}\"` : text;\n};\n\nconst parseCsv = (text = '') => {\n const rows = [];\n let row = [];\n let cell = '';\n let quoted = false;\n const pushCell = () => { row.push(cell); cell = ''; };\n const pushRow = () => { pushCell(); if (row.some((v) => String(v || '').trim())) rows.push(row); row = []; };\n const inputText = String(text || '').replace(/^\\uFEFF/, '');\n for (let i = 0; i < inputText.length; i += 1) {\n const char = inputText[i];\n const next = inputText[i + 1];\n if (quoted) {\n if (char === '\"' && next === '\"') { cell += '\"'; i += 1; }\n else if (char === '\"') quoted = false;\n else cell += char;\n } else if (char === '\"') quoted = true;\n else if (char === ',') pushCell();\n else if (char === '\\n') pushRow();\n else if (char !== '\\r') cell += char;\n }\n if (cell || row.length) pushRow();\n const headers = (rows.shift() || csvHeaders).map((header) => String(header || '').trim());\n return rows.map((values) => headers.reduce((acc, header, index) => {\n if (header) acc[header] = values[index] || '';\n return acc;\n }, {}));\n};\n\nconst toCsv = (rows = []) => [\n csvHeaders.join(','),\n ...rows.map((row) => csvHeaders.map((header) => csvEscape(row[header] || '')).join(','))\n].join('\\n') + '\\n';\n\nconst normalizeUrl = (value = '') => {\n const raw = String(value || '').trim();\n if (!raw) return '';\n try {\n const url = new URL(raw, 'https://www.kijiji.ca');\n url.hash = '';\n return url.toString().replace(/\\/$/, '');\n } catch {\n return raw.replace(/#.*$/, '').replace(/\\/$/, '');\n }\n};\n\nconst decodeHtml = (value = '') => String(value || '')\n .replace(/ /g, ' ')\n .replace(/&/g, '&')\n .replace(/"/g, '\"')\n .replace(/"/g, '\"')\n .replace(/'/g, \"'\")\n .replace(/</g, '<')\n .replace(/>/g, '>');\n\nconst cleanText = (value = '') => decodeHtml(String(value || ''))\n .replace(//gi, ' ')\n .replace(//gi, ' ')\n .replace(/<[^>]+>/g, ' ')\n .replace(/\\s+/g, ' ')\n .trim();\n\nconst getGithubCsv = async () => {\n const url = `https://api.github.com/repos/${OWNER}/${REPO}/contents/${encodeURIComponent(CSV_PATH).replace(/%2F/g, '/')}?ref=${BRANCH}`;\n const res = await httpRequest({ url, headers: ghHeaders, json: true });\n if (res.status === 404) return { sha: null, text: csvHeaders.join(',') + '\\n' };\n if (!res.ok) throw new Error(`Could not read ${CSV_PATH}: ${res.status} ${typeof res.body === 'string' ? res.body : JSON.stringify(res.body)}`);\n const body = typeof res.body === 'string' ? JSON.parse(res.body) : res.body;\n return { sha: body.sha, text: Buffer.from(body.content || '', 'base64').toString('utf8') };\n};\n\nconst putGithubCsv = async (text, sha) => {\n const url = `https://api.github.com/repos/${OWNER}/${REPO}/contents/${encodeURIComponent(CSV_PATH).replace(/%2F/g, '/')}`;\n const body = {\n message: 'Sync Kijiji Hamilton listings CSV',\n branch: BRANCH,\n content: Buffer.from(text, 'utf8').toString('base64'),\n ...(sha ? { sha } : {})\n };\n const res = await httpRequest({ url, method: 'PUT', headers: { ...ghHeaders, 'content-type': 'application/json' }, body, json: true });\n if (!res.ok) throw new Error(`Could not update ${CSV_PATH}: ${res.status} ${typeof res.body === 'string' ? res.body : JSON.stringify(res.body)}`);\n return typeof res.body === 'string' ? JSON.parse(res.body) : res.body;\n};\n\nconst normalizeCrawlItems = (value) => {\n if (!value) return [];\n if (Array.isArray(value)) return value.flatMap(normalizeCrawlItems);\n if (Array.isArray(value.results)) return value.results;\n if (Array.isArray(value.data)) return value.data;\n if (value.body) return normalizeCrawlItems(value.body);\n return [value];\n};\n\nconst normalizeImageUrl = (value = '') => {\n const raw = decodeHtml(String(value || '').trim());\n if (!raw || /{{|}}|imageMessage|photoapparat|fb\\/jacars/i.test(raw)) return '';\n try {\n const url = new URL(raw, 'https://www.kijiji.ca');\n if (!/^https?:$/i.test(url.protocol)) return '';\n if (/media.kijiji.ca/i.test(url.hostname) && url.searchParams.has('rule')) {\n url.searchParams.set('rule', 'kijijica-640-webp');\n }\n return url.toString();\n } catch {\n return raw.startsWith('http') ? raw : '';\n }\n};\n\nconst imageUrlFromValue = (value) => {\n if (!value) return [];\n if (Array.isArray(value)) return value.flatMap(imageUrlFromValue);\n if (typeof value === 'object') return imageUrlFromValue(value.contentUrl || value.url || value.src || value.image);\n const normalized = normalizeImageUrl(value);\n return normalized ? [normalized] : [];\n};\n\nconst extractImageUrls = (html = '') => {\n const found = [];\n const seen = new Set();\n const add = (value) => {\n for (const imageUrl of imageUrlFromValue(value)) {\n if (!imageUrl || seen.has(imageUrl)) continue;\n seen.add(imageUrl);\n found.push(imageUrl);\n }\n };\n\n for (const block of extractScriptJson(html)) {\n const stack = Array.isArray(block) ? [...block] : [block];\n while (stack.length) {\n const item = stack.shift();\n if (!item || typeof item !== 'object') continue;\n if (item.image) add(item.image);\n if (item.contentUrl) add(item.contentUrl);\n if (Array.isArray(item.itemListElement)) stack.push(...item.itemListElement.map((entry) => entry?.item || entry));\n if (Array.isArray(item['@graph'])) stack.push(...item['@graph']);\n }\n }\n\n const mediaMatches = String(html || '').match(/https?:\\/\\/media\\.kijiji\\.ca[^\"'\\s<>]+/gi) || [];\n mediaMatches.forEach((url) => add(url.replace(/\\u002F/g, '/').replace(/\\u0026/g, '&')));\n return found.slice(0, 12);\n};\n\nconst extractScriptJson = (html = '', id = '') => {\n const pattern = id\n ? new RegExp(`]+id=[\"']${id}[\"'][^>]*>([\\\\s\\\\S]*?)<\\\\/script>`, 'i')\n : /]*type=[\"']application\\/(?:ld\\+json|json)[\"'][^>]*>([\\s\\S]*?)<\\/script>/gi;\n if (id) {\n const match = String(html || '').match(pattern);\n if (!match) return null;\n try { return JSON.parse(decodeHtml(match[1])); } catch { return null; }\n }\n return [...String(html || '').matchAll(pattern)].map((match) => {\n try { return JSON.parse(decodeHtml(match[1])); } catch { return null; }\n }).filter(Boolean);\n};\n\nconst priceTextFromListing = (listing = {}) => {\n const amount = Number(listing?.price?.amount);\n const currency = String(listing?.price?.currency || 'CAD').trim() || 'CAD';\n if (!Number.isFinite(amount)) return '';\n const major = amount > 999 ? amount / 100 : amount;\n return currency === 'CAD' ? `CA$ ${major.toLocaleString('en-CA', { minimumFractionDigits: 2, maximumFractionDigits: 2 })}` : `${currency} ${major}`;\n};\n\nconst priceValueFromText = (value = '') => {\n const parsed = Number(String(value || '').replace(/[^0-9.]/g, ''));\n return Number.isFinite(parsed) ? String(parsed) : '';\n};\n\nconst normalizeCondition = (value = '') => {\n const text = String(value || '').toLowerCase();\n if (/like new/.test(text)) return 'like_new';\n if (/excellent/.test(text)) return 'excellent';\n if (/good/.test(text)) return 'good';\n if (/fair/.test(text)) return 'fair';\n if (/brand new|\\bnew\\b/.test(text)) return 'new';\n if (/used|pre-owned|preowned/.test(text)) return 'used';\n return '';\n};\n\nconst phonePattern = /(? {\n const found = [];\n const seen = new Set();\n for (const value of values) {\n const text = cleanText(value);\n let match;\n while ((match = phonePattern.exec(text))) {\n const raw = match[0];\n const start = Math.max(0, match.index - 45);\n const end = Math.min(text.length, match.index + raw.length + 25);\n const hasSeparator = /[\\s().-]/.test(raw.trim());\n if (!hasSeparator && !phoneContextPattern.test(text.slice(start, end))) continue;\n const phone = `${match[1]}${match[2]}${match[3]}`;\n if (!seen.has(phone)) {\n seen.add(phone);\n found.push(phone);\n }\n }\n }\n return found.join(' | ');\n};\n\nconst extractApolloListings = (html = '') => {\n const nextData = extractScriptJson(html, '__NEXT_DATA__');\n const apollo = nextData?.props?.pageProps?.__APOLLO_STATE__ || nextData?.props?.pageProps?.apolloState || {};\n return Object.values(apollo)\n .filter((entry) => entry && entry.__typename === 'StandardListing' && entry.url)\n .map((entry) => {\n const attrs = Array.isArray(entry?.attributes?.all) ? entry.attributes.all : [];\n const attrMap = new Map(attrs.map((item) => [String(item?.canonicalName || '').trim(), item]));\n const conditionAttr = attrMap.get('condition');\n const priceText = priceTextFromListing(entry);\n const description = cleanText(entry.description || '');\n const title = cleanText(entry.title || '');\n return {\n listingId: String(entry.id || '').trim(),\n title,\n description,\n sourceUrl: normalizeUrl(entry.url || ''),\n imageUrls: Array.isArray(entry.imageUrls) ? entry.imageUrls.flatMap(imageUrlFromValue).filter(Boolean) : [],\n priceText,\n priceValue: priceValueFromText(priceText),\n currency: String(entry?.price?.currency || 'CAD').trim() || 'CAD',\n city: cleanText(entry?.location?.name || ''),\n locationAddress: cleanText(entry?.location?.address || ''),\n seller: cleanText(entry?.posterInfo?.name || '') || 'Unknown',\n sellerId: String(entry?.posterInfo?.posterId || '').trim(),\n sellerType: cleanText(entry?.posterInfo?.sellerType || ''),\n condition: normalizeCondition((conditionAttr?.values || conditionAttr?.canonicalValues || [])[0] || `${title} ${description}`),\n postedAt: String(entry.activationDate || entry.sortingDate || '').trim(),\n sortingDate: String(entry.sortingDate || entry.activationDate || '').trim(),\n phone: extractPhones(title, description)\n };\n });\n};\n\nconst extractJsonLdListings = (html = '') => {\n const blocks = extractScriptJson(html);\n const itemLists = blocks.flatMap((item) => Array.isArray(item) ? item : [item]).filter((item) => item?.['@type'] === 'ItemList' && Array.isArray(item.itemListElement));\n return itemLists.flatMap((list) => list.itemListElement.map((entry) => {\n const item = entry?.item || entry || {};\n const sourceUrl = normalizeUrl(item.url || '');\n const price = String(item?.offers?.price || '').trim();\n return {\n listingId: sourceUrl.match(/\\/(\\d+)(?:[/?#]|$)/)?.[1] || '',\n title: cleanText(item.name || ''),\n description: cleanText(item.description || ''),\n sourceUrl,\n imageUrls: item.image ? imageUrlFromValue(item.image) : [],\n priceText: price ? `$${price}` : '',\n priceValue: price,\n currency: String(item?.offers?.priceCurrency || 'CAD').trim() || 'CAD',\n city: '',\n locationAddress: '',\n seller: 'Unknown',\n sellerId: '',\n sellerType: '',\n condition: '',\n postedAt: '',\n sortingDate: '',\n phone: extractPhones(item.name, item.description)\n };\n })).filter((row) => row.title && row.sourceUrl);\n};\n\nconst inferCategory = (listing = {}, source = {}) => {\n if (source.app_category || source.target_surface) {\n return {\n targetSurface: source.target_surface || (source.app_category === 'vehicles' ? 'vehicles' : 'marketplace'),\n appCategory: source.app_category || 'electronics',\n appSubcategory: source.app_subcategory || 'other'\n };\n }\n const text = `${listing.sourceUrl || ''} ${listing.title || ''} ${listing.description || ''}`.toLowerCase();\n if (/\\b(tires?|rims?|wheels?|auto-parts|car-parts|vehicle-parts|parting-out|motorcycle|brake|transmission|engine)\\b/.test(text)) {\n return { targetSurface: 'vehicles', appCategory: 'vehicles', appSubcategory: /\\b(tires?|rims?|wheels?)\\b/.test(text) ? 'tires_rims' : 'auto_parts' };\n }\n if (/\\b(service|scrap metal|pick up|pickup|repair|cleaning|moving|delivery)\\b/.test(text)) {\n return { targetSurface: 'marketplace', appCategory: 'services', appSubcategory: 'other' };\n }\n if (/\\b(phone|iphone|samsung|laptop|tablet|camera|lens|stereo|speaker|yamaha|canon|dewalt|battery|charger|computer|headphone|tv)\\b/.test(text)) {\n let subcategory = 'other';\n if (/\\b(phone|iphone|samsung)\\b/.test(text)) subcategory = 'phones_accessories';\n else if (/\\b(laptop|tablet|computer)\\b/.test(text)) subcategory = 'computers_tablets';\n else if (/\\b(camera|lens|canon)\\b/.test(text)) subcategory = 'cameras_photography';\n else if (/\\b(stereo|speaker|yamaha|headphone)\\b/.test(text)) subcategory = 'audio_headphones';\n return { targetSurface: 'marketplace', appCategory: 'electronics', appSubcategory: subcategory };\n }\n return { targetSurface: 'marketplace', appCategory: 'electronics', appSubcategory: 'other' };\n};\n\nconst matchesCity = (listing, source) => {\n const strict = source.strict_city_match === true || String(source.strict_city_match || '').toLowerCase() === 'true';\n if (!strict) return true;\n const city = cleanText(source.city || '').toLowerCase();\n if (!city) return true;\n const hay = [listing.city, listing.locationAddress, listing.sourceUrl].map((value) => cleanText(value).toLowerCase()).join(' ');\n return hay.includes(city.toLowerCase());\n};\n\nconst rowsFromCrawl = (crawlValue, activeSources) => {\n const crawlItems = normalizeCrawlItems(crawlValue || {});\n const crawlByUrl = new Map();\n for (const item of crawlItems) {\n const key = normalizeUrl(item.url || item.redirected_url || '');\n if (key) crawlByUrl.set(key, item);\n }\n const output = [];\n const sourceSummaries = [];\n for (const source of activeSources) {\n const listUrl = normalizeUrl(source.list_url || source.url || '');\n const crawlItem = crawlByUrl.get(listUrl) || Array.from(crawlByUrl.values()).find((entry) => normalizeUrl(entry.url || entry.redirected_url || '') === listUrl);\n const sourceName = source.name || source.city || listUrl;\n if (!crawlItem) {\n sourceSummaries.push({ source: sourceName, status: 'failed', fetched: 0, error: 'Crawl4AI did not return this URL.' });\n continue;\n }\n const raw = crawlItem.html || crawlItem.cleaned_html || crawlItem.fit_html || crawlItem.markdown?.raw_markdown || crawlItem.markdown || crawlItem.fit_markdown || '';\n if (!raw) {\n sourceSummaries.push({ source: sourceName, status: 'failed', fetched: 0, error: crawlItem.error_message || 'No HTML returned.' });\n continue;\n }\n let listings = extractApolloListings(raw);\n if (!listings.length) listings = extractJsonLdListings(raw);\n const maxListings = Math.max(1, Number(source.max_listings || 50));\n const seen = new Set();\n const normalized = listings\n .filter((listing) => listing && listing.title && listing.sourceUrl && !seen.has(listing.sourceUrl) && (seen.add(listing.sourceUrl) || true))\n .filter((listing) => matchesCity(listing, source))\n .filter((listing) => String(listing.phone || '').trim())\n .slice(0, maxListings)\n .map((listing) => {\n const inferred = inferCategory(listing, source);\n const id = listing.listingId ? `kijiji-${listing.listingId}` : `kijiji-${Math.abs([...listing.sourceUrl].reduce((a, c) => ((a << 5) - a + c.charCodeAt(0)) | 0, 0))}`;\n return {\n id,\n status: source.default_status || DEFAULT_STATUS,\n target_surface: inferred.targetSurface,\n app_category: inferred.appCategory,\n app_subcategory: inferred.appSubcategory,\n title: listing.title,\n price_text: listing.priceText,\n price_value: listing.priceValue,\n currency: listing.currency || 'CAD',\n city: listing.city || source.city || '',\n country: source.country || 'Canada',\n seller: listing.seller || 'Unknown',\n phone: listing.phone || '',\n description: listing.description,\n image_urls: (listing.imageUrls || []).join('|'),\n source_site: 'Kijiji',\n source_url: listing.sourceUrl,\n scraped_at: listing.sortingDate || listing.postedAt || nowIso,\n make: '',\n model: '',\n trim: '',\n year: String(`${listing.title} ${listing.description}`.match(/\\b(19|20)\\d{2}\\b/)?.[0] || ''),\n condition: listing.condition || '',\n transmission: '',\n color: '',\n mileage_km: '',\n attributes: JSON.stringify({ parser: 'kijiji_crawl4ai_sync', listUrl, sourceName, locationAddress: listing.locationAddress, sellerId: listing.sellerId, sellerType: listing.sellerType }),\n source_availability: 'active',\n source_availability_checked_at: nowIso,\n source_http_status: String(crawlItem.status_code || crawlItem.statusCode || 200),\n source_unavailable_reason: '',\n source_last_seen_at: nowIso,\n source_resolved_url: listing.sourceUrl\n };\n });\n output.push(...normalized);\n sourceSummaries.push({ source: sourceName, status: normalized.length ? 'success' : 'empty', fetched: normalized.length });\n }\n return { rows: output, sourceSummaries };\n};\n\nconst crawlDetailUrls = async (urls) => {\n const results = new Map();\n for (let i = 0; i < urls.length; i += AVAILABILITY_BATCH_SIZE) {\n const batch = urls.slice(i, i + AVAILABILITY_BATCH_SIZE);\n const response = await httpRequest({\n url: CRAWL4AI_URL,\n method: 'POST',\n json: true,\n body: {\n urls: batch,\n browser_config: { headless: true, viewport: { width: 1280, height: 1800 }, verbose: false },\n crawler_config: { stream: false, cache_mode: 'bypass', wait_until: 'domcontentloaded', wait_for: 'css:body', page_timeout: 20000, delay_before_return_html: 0.5, remove_overlay_elements: true, remove_consent_popups: true }\n }\n });\n const items = normalizeCrawlItems(response.body || response);\n for (let index = 0; index < items.length; index += 1) {\n const item = items[index] || {};\n const fallbackUrl = batch[index] || '';\n const keys = [item.url, item.redirected_url, fallbackUrl].map(normalizeUrl).filter(Boolean);\n const raw = item.html || item.cleaned_html || item.fit_html || item.markdown?.raw_markdown || item.markdown || item.fit_markdown || '';\n const text = cleanText(raw).toLowerCase();\n const statusCode = Number(item.status_code || item.statusCode || item.response?.status || response.status || 0);\n let availability = 'unknown';\n let reason = '';\n if (statusCode === 404 || statusCode === 410) {\n availability = 'gone';\n reason = `HTTP ${statusCode}`;\n } else if (/\\b(sold|item has sold|listing sold)\\b/i.test(text)) {\n availability = 'sold';\n reason = 'Source page says sold.';\n } else if (/ad is no longer available|listing is no longer available|no longer available|has been removed|ad has expired|this ad is unavailable|page not found|does not exist/i.test(text)) {\n availability = 'unavailable';\n reason = 'Source page says removed, expired, or unavailable.';\n } else if (raw && /kijiji/i.test(raw)) {\n availability = 'active';\n } else if (item.error_message || item.error) {\n reason = String(item.error_message || item.error || '').slice(0, 160);\n } else if (!raw) {\n reason = 'No detail page content returned.';\n }\n const result = { availability, reason, statusCode: statusCode ? String(statusCode) : '', resolvedUrl: normalizeUrl(item.redirected_url || item.url || fallbackUrl), imageUrls: extractImageUrls(raw) };\n keys.forEach((key) => results.set(key, result));\n }\n }\n return results;\n};\n\nconst activeSources = sources.filter((source) => source && source.enabled !== false);\nconst existing = await getGithubCsv();\nconst existingRows = parseCsv(existing.text);\nconst rowsByUrl = new Map(existingRows.map((row) => [normalizeUrl(row.source_url || ''), row]).filter(([key]) => key));\n\nconst { rows: scrapedRows, sourceSummaries } = rowsFromCrawl(input.crawl4aiResult || input, activeSources);\nconst scrapedDetailUrls = scrapedRows.map((row) => normalizeUrl(row.source_url || '')).filter(Boolean).slice(0, AVAILABILITY_MAX_ROWS);\nconst scrapedDetailsByUrl = scrapedDetailUrls.length ? await crawlDetailUrls.call(this, scrapedDetailUrls) : new Map();\nfor (const row of scrapedRows) {\n const detail = scrapedDetailsByUrl.get(normalizeUrl(row.source_url || ''));\n if (detail?.imageUrls?.length) {\n const currentImages = String(row.image_urls || '').split('|').map(normalizeImageUrl).filter(Boolean);\n row.image_urls = Array.from(new Set([...detail.imageUrls, ...currentImages])).slice(0, 12).join('|');\n }\n}\nlet inserted = 0;\nlet updated = 0;\nfor (const row of scrapedRows) {\n const key = normalizeUrl(row.source_url || '');\n if (!key) continue;\n const previous = rowsByUrl.get(key) || {};\n if (previous.source_url) updated += 1;\n else inserted += 1;\n rowsByUrl.set(key, { ...previous, ...row });\n}\n\nconst scrapedUrlSet = new Set(scrapedRows.map((row) => normalizeUrl(row.source_url || '')).filter(Boolean));\nconst configuredCities = activeSources.map((source) => cleanText(source.city || '').toLowerCase()).filter(Boolean);\nconst checkedAgeHours = (row) => {\n const checkedAt = Date.parse(row.source_availability_checked_at || '');\n if (!Number.isFinite(checkedAt)) return Number.POSITIVE_INFINITY;\n return Math.max(0, (nowMs - checkedAt) / 36e5);\n};\nconst shouldCheckRow = (row) => {\n const url = normalizeUrl(row.source_url || '');\n if (!/kijiji\\.ca/i.test(url) && String(row.source_site || '').toLowerCase() !== 'kijiji') return false;\n if (!configuredCities.length) return true;\n const hay = [row.city, row.source_url, row.attributes].map((value) => cleanText(value).toLowerCase()).join(' ');\n return configuredCities.some((city) => hay.includes(city));\n};\n\nconst availabilityCandidates = Array.from(rowsByUrl.values())\n .filter(shouldCheckRow)\n .filter((row) => {\n const url = normalizeUrl(row.source_url || '');\n if (scrapedUrlSet.has(url)) return false;\n return checkedAgeHours(row) >= AVAILABILITY_FRESH_HOURS;\n })\n .sort((a, b) => checkedAgeHours(b) - checkedAgeHours(a) || String(b.scraped_at || '').localeCompare(String(a.scraped_at || '')))\n .slice(0, AVAILABILITY_MAX_ROWS);\nconst candidateUrls = availabilityCandidates.map((row) => normalizeUrl(row.source_url || '')).filter(Boolean);\nconst availabilityByUrl = candidateUrls.length ? await crawlDetailUrls.call(this, candidateUrls) : new Map();\nlet active = 0;\nlet sold = 0;\nlet hidden = 0;\nlet unavailable = 0;\nlet unknown = 0;\nfor (const row of availabilityCandidates) {\n const key = normalizeUrl(row.source_url || '');\n const result = availabilityByUrl.get(key);\n if (!result) continue;\n row.source_availability = result.availability;\n row.source_availability_checked_at = nowIso;\n row.source_http_status = result.statusCode;\n row.source_unavailable_reason = result.reason;\n row.source_resolved_url = result.resolvedUrl || row.source_url;\n if (result.availability === 'active') {\n row.source_last_seen_at = nowIso;\n active += 1;\n } else if (result.availability === 'sold') {\n sold += 1;\n } else if (['unavailable', 'gone'].includes(result.availability)) {\n unavailable += 1;\n if (HIDE_UNAVAILABLE) {\n row.status = 'rejected';\n hidden += 1;\n }\n } else {\n unknown += 1;\n }\n}\n\nconst nextRows = Array.from(rowsByUrl.values()).sort((a, b) => String(b.scraped_at || '').localeCompare(String(a.scraped_at || '')));\nawait putGithubCsv(toCsv(nextRows), existing.sha);\n\nreturn [{\n json: {\n status: 'completed',\n csvPath: CSV_PATH,\n sources: sourceSummaries,\n scraped: scrapedRows.length,\n inserted,\n updated,\n detailImagesChecked: scrapedDetailUrls.length,\n availabilityChecked: availabilityCandidates.length,\n availabilityFreshHours: AVAILABILITY_FRESH_HOURS,\n active,\n sold,\n unavailable,\n unknown,\n hidden,\n hideUnavailableListings: HIDE_UNAVAILABLE\n }\n}];" }, "id": "6", "name": "Sync New Updated Sold Rows", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 1020, 90 ] } ], "connections": { "Manual Trigger": { "main": [ [ { "node": "Set Config Here", "type": "main", "index": 0 } ] ] }, "Every Hour": { "main": [ [ { "node": "Set Config Here", "type": "main", "index": 0 } ] ] }, "Set Config Here": { "main": [ [ { "node": "Crawl Kijiji List Pages", "type": "main", "index": 0 }, { "node": "Merge Config + Crawl Result", "type": "main", "index": 0 } ] ] }, "Crawl Kijiji List Pages": { "main": [ [ { "node": "Merge Config + Crawl Result", "type": "main", "index": 1 } ] ] }, "Merge Config + Crawl Result": { "main": [ [ { "node": "Sync New Updated Sold Rows", "type": "main", "index": 0 } ] ] } }, "active": false, "settings": { "executionOrder": "v1" }, "versionId": "", "meta": { "templateCredsSetupCompleted": true }, "id": "", "tags": [] }