6ixo - Kijiji Hamilton Sync to CSV

The workflow JSON

Copy or download the full n8n JSON below. Paste it into a new n8n workflow, add your credentials, activate. Full import guide →

{
  "name": "6ixo - Kijiji Hamilton Sync to CSV",
  "nodes": [
    {
      "parameters": {},
      "id": "1",
      "name": "Manual Trigger",
      "type": "n8n-nodes-base.manualTrigger",
      "typeVersion": 1,
      "position": [
        0,
        0
      ]
    },
    {
      "parameters": {
        "rule": {
          "interval": [
            {
              "field": "hours",
              "hoursInterval": 1
            }
          ]
        }
      },
      "id": "2",
      "name": "Every Hour",
      "type": "n8n-nodes-base.scheduleTrigger",
      "typeVersion": 1.2,
      "position": [
        0,
        180
      ]
    },
    {
      "parameters": {
        "assignments": {
          "assignments": [
            {
              "id": "githubToken",
              "name": "githubToken",
              "value": "={{ $env.GITHUB_TOKEN || 'PASTE_GITHUB_TOKEN_HERE' }}",
              "type": "string"
            },
            {
              "id": "githubOwner",
              "name": "githubOwner",
              "value": "={{ $env.GITHUB_OWNER || 'bisco401' }}",
              "type": "string"
            },
            {
              "id": "githubRepo",
              "name": "githubRepo",
              "value": "={{ $env.GITHUB_REPO || '6ixo' }}",
              "type": "string"
            },
            {
              "id": "githubBranch",
              "name": "githubBranch",
              "value": "={{ $env.GITHUB_BRANCH || 'main' }}",
              "type": "string"
            },
            {
              "id": "csvPath",
              "name": "csvPath",
              "value": "={{ $env.SIXO_CSV_PATH || 'data/scraped-listings.csv' }}",
              "type": "string"
            },
            {
              "id": "crawl4aiUrl",
              "name": "crawl4aiUrl",
              "value": "={{ $env.CRAWL4AI_URL || 'http://crawl4ai:11235/crawl' }}",
              "type": "string"
            },
            {
              "id": "defaultImportStatus",
              "name": "defaultImportStatus",
              "value": "published",
              "type": "string"
            },
            {
              "id": "hideUnavailableListings",
              "name": "hideUnavailableListings",
              "value": "true",
              "type": "string"
            },
            {
              "id": "availabilityMaxRows",
              "name": "availabilityMaxRows",
              "value": "24",
              "type": "string"
            },
            {
              "id": "availabilityBatchSize",
              "name": "availabilityBatchSize",
              "value": "6",
              "type": "string"
            },
            {
              "id": "availabilityFreshHours",
              "name": "availabilityFreshHours",
              "value": "12",
              "type": "string"
            },
            {
              "id": "sourcesJson",
              "name": "sourcesJson",
              "value": "[\n  {\n    \"name\": \"Kijiji Hamilton Recent Ads\",\n    \"enabled\": true,\n    \"list_url\": \"https://www.kijiji.ca/b-hamilton/l80014?sort=dateDesc\",\n    \"city\": \"Hamilton\",\n    \"country\": \"Canada\",\n    \"strict_city_match\": true,\n    \"max_listings\": 50\n  }\n]",
              "type": "string"
            }
          ]
        },
        "options": {}
      },
      "id": "3",
      "name": "Set Config Here",
      "type": "n8n-nodes-base.set",
      "typeVersion": 3.4,
      "position": [
        260,
        90
      ]
    },
    {
      "parameters": {
        "method": "POST",
        "url": "={{ $json.crawl4aiUrl }}",
        "sendBody": true,
        "contentType": "json",
        "specifyBody": "json",
        "jsonBody": "={{ { urls: JSON.parse($json.sourcesJson).filter(source => source && source.enabled !== false && (source.list_url || source.url)).map(source => source.list_url || source.url), browser_config: { headless: true, viewport: { width: 1440, height: 2200 }, verbose: false }, crawler_config: { stream: false, cache_mode: \"bypass\", wait_until: \"load\", wait_for: \"css:body\", page_timeout: 45000, delay_before_return_html: 1, scan_full_page: true, remove_overlay_elements: true, remove_consent_popups: true, flatten_shadow_dom: true } } }}",
        "options": {}
      },
      "id": "4",
      "name": "Crawl Kijiji List Pages",
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 4.2,
      "position": [
        520,
        180
      ]
    },
    {
      "parameters": {
        "mode": "combine",
        "combineBy": "combineByPosition",
        "options": {}
      },
      "id": "5",
      "name": "Merge Config + Crawl Result",
      "type": "n8n-nodes-base.merge",
      "typeVersion": 3,
      "position": [
        760,
        90
      ]
    },
    {
      "parameters": {
        "jsCode": "const input = $input.first().json;\nconst OWNER = input.githubOwner || 'bisco401';\nconst REPO = input.githubRepo || '6ixo';\nconst BRANCH = input.githubBranch || 'main';\nconst CSV_PATH = input.csvPath || 'data/scraped-listings.csv';\nconst TOKEN = input.githubToken;\nconst CRAWL4AI_URL = input.crawl4aiUrl || 'http://crawl4ai:11235/crawl';\nconst DEFAULT_STATUS = input.defaultImportStatus || 'published';\nconst HIDE_UNAVAILABLE = String(input.hideUnavailableListings ?? 'true').toLowerCase() === 'true';\nconst AVAILABILITY_MAX_ROWS = Math.max(0, Number(input.availabilityMaxRows || 24));\nconst AVAILABILITY_BATCH_SIZE = Math.max(1, Number(input.availabilityBatchSize || 6));\nconst AVAILABILITY_FRESH_HOURS = Math.max(0, Number(input.availabilityFreshHours || 12));\nconst nowIso = new Date().toISOString();\nconst nowMs = Date.parse(nowIso);\n\nlet sources = [];\ntry {\n  sources = typeof input.sourcesJson === 'string' ? JSON.parse(input.sourcesJson || '[]') : (input.sourcesJson || []);\n} catch (error) {\n  throw new Error('sourcesJson is not valid JSON. Fix the Set Config Here node first.');\n}\n\nif (!TOKEN || TOKEN === 'PASTE_GITHUB_TOKEN_HERE') throw new Error('Paste your GitHub token into the Set Config Here node, or set GITHUB_TOKEN in n8n.');\nif (!Array.isArray(sources) || !sources.length) throw new Error('Add at least one Kijiji source in sourcesJson.');\n\nconst csvHeaders = [\n  'id','status','target_surface','app_category','app_subcategory','title','price_text','price_value','currency','city','country','seller','phone','description','image_urls','source_site','source_url','scraped_at','make','model','trim','year','condition','transmission','color','mileage_km','attributes','source_availability','source_availability_checked_at','source_http_status','source_unavailable_reason','source_last_seen_at','source_resolved_url'\n];\nconst ghHeaders = { authorization: `Bearer ${TOKEN}`, accept: 'application/vnd.github+json', 'x-github-api-version': '2022-11-28' };\n\nconst httpRequest = async (options) => {\n  const request = {\n    method: options.method || 'GET',\n    uri: options.url,\n    url: options.url,\n    headers: options.headers || {},\n    body: options.body,\n    json: options.json === true,\n    resolveWithFullResponse: true,\n    simple: false\n  };\n  try {\n    const response = await this.helpers.httpRequest(request);\n    const status = response.statusCode || response.status || 200;\n    return { status, ok: status >= 200 && status < 300, body: response.body ?? response };\n  } catch (error) {\n    const status = error.statusCode || error.status || error.response?.status || error.response?.statusCode || 500;\n    const body = error.response?.body || error.response?.data || error.message || '';\n    return { status, ok: status >= 200 && status < 300, body };\n  }\n};\n\nconst csvEscape = (value = '') => {\n  const text = String(value ?? '');\n  return /[\",\\n\\r]/.test(text) ? `\"${text.replace(/\"/g, '\"\"')}\"` : text;\n};\n\nconst parseCsv = (text = '') => {\n  const rows = [];\n  let row = [];\n  let cell = '';\n  let quoted = false;\n  const pushCell = () => { row.push(cell); cell = ''; };\n  const pushRow = () => { pushCell(); if (row.some((v) => String(v || '').trim())) rows.push(row); row = []; };\n  const inputText = String(text || '').replace(/^\\uFEFF/, '');\n  for (let i = 0; i < inputText.length; i += 1) {\n    const char = inputText[i];\n    const next = inputText[i + 1];\n    if (quoted) {\n      if (char === '\"' && next === '\"') { cell += '\"'; i += 1; }\n      else if (char === '\"') quoted = false;\n      else cell += char;\n    } else if (char === '\"') quoted = true;\n    else if (char === ',') pushCell();\n    else if (char === '\\n') pushRow();\n    else if (char !== '\\r') cell += char;\n  }\n  if (cell || row.length) pushRow();\n  const headers = (rows.shift() || csvHeaders).map((header) => String(header || '').trim());\n  return rows.map((values) => headers.reduce((acc, header, index) => {\n    if (header) acc[header] = values[index] || '';\n    return acc;\n  }, {}));\n};\n\nconst toCsv = (rows = []) => [\n  csvHeaders.join(','),\n  ...rows.map((row) => csvHeaders.map((header) => csvEscape(row[header] || '')).join(','))\n].join('\\n') + '\\n';\n\nconst normalizeUrl = (value = '') => {\n  const raw = String(value || '').trim();\n  if (!raw) return '';\n  try {\n    const url = new URL(raw, 'https://www.kijiji.ca');\n    url.hash = '';\n    return url.toString().replace(/\\/$/, '');\n  } catch {\n    return raw.replace(/#.*$/, '').replace(/\\/$/, '');\n  }\n};\n\nconst decodeHtml = (value = '') => String(value || '')\n  .replace(/&nbsp;/g, ' ')\n  .replace(/&amp;/g, '&')\n  .replace(/&quot;/g, '\"')\n  .replace(/&#34;/g, '\"')\n  .replace(/&#39;/g, \"'\")\n  .replace(/&lt;/g, '<')\n  .replace(/&gt;/g, '>');\n\nconst cleanText = (value = '') => decodeHtml(String(value || ''))\n  .replace(/<script[\\s\\S]*?<\\/script>/gi, ' ')\n  .replace(/<style[\\s\\S]*?<\\/style>/gi, ' ')\n  .replace(/<[^>]+>/g, ' ')\n  .replace(/\\s+/g, ' ')\n  .trim();\n\nconst getGithubCsv = async () => {\n  const url = `https://api.github.com/repos/${OWNER}/${REPO}/contents/${encodeURIComponent(CSV_PATH).replace(/%2F/g, '/')}?ref=${BRANCH}`;\n  const res = await httpRequest({ url, headers: ghHeaders, json: true });\n  if (res.status === 404) return { sha: null, text: csvHeaders.join(',') + '\\n' };\n  if (!res.ok) throw new Error(`Could not read ${CSV_PATH}: ${res.status} ${typeof res.body === 'string' ? res.body : JSON.stringify(res.body)}`);\n  const body = typeof res.body === 'string' ? JSON.parse(res.body) : res.body;\n  return { sha: body.sha, text: Buffer.from(body.content || '', 'base64').toString('utf8') };\n};\n\nconst putGithubCsv = async (text, sha) => {\n  const url = `https://api.github.com/repos/${OWNER}/${REPO}/contents/${encodeURIComponent(CSV_PATH).replace(/%2F/g, '/')}`;\n  const body = {\n    message: 'Sync Kijiji Hamilton listings CSV',\n    branch: BRANCH,\n    content: Buffer.from(text, 'utf8').toString('base64'),\n    ...(sha ? { sha } : {})\n  };\n  const res = await httpRequest({ url, method: 'PUT', headers: { ...ghHeaders, 'content-type': 'application/json' }, body, json: true });\n  if (!res.ok) throw new Error(`Could not update ${CSV_PATH}: ${res.status} ${typeof res.body === 'string' ? res.body : JSON.stringify(res.body)}`);\n  return typeof res.body === 'string' ? JSON.parse(res.body) : res.body;\n};\n\nconst normalizeCrawlItems = (value) => {\n  if (!value) return [];\n  if (Array.isArray(value)) return value.flatMap(normalizeCrawlItems);\n  if (Array.isArray(value.results)) return value.results;\n  if (Array.isArray(value.data)) return value.data;\n  if (value.body) return normalizeCrawlItems(value.body);\n  return [value];\n};\n\nconst normalizeImageUrl = (value = '') => {\n  const raw = decodeHtml(String(value || '').trim());\n  if (!raw || /{{|}}|imageMessage|photoapparat|fb\\/jacars/i.test(raw)) return '';\n  try {\n    const url = new URL(raw, 'https://www.kijiji.ca');\n    if (!/^https?:$/i.test(url.protocol)) return '';\n    if (/media.kijiji.ca/i.test(url.hostname) && url.searchParams.has('rule')) {\n      url.searchParams.set('rule', 'kijijica-640-webp');\n    }\n    return url.toString();\n  } catch {\n    return raw.startsWith('http') ? raw : '';\n  }\n};\n\nconst imageUrlFromValue = (value) => {\n  if (!value) return [];\n  if (Array.isArray(value)) return value.flatMap(imageUrlFromValue);\n  if (typeof value === 'object') return imageUrlFromValue(value.contentUrl || value.url || value.src || value.image);\n  const normalized = normalizeImageUrl(value);\n  return normalized ? [normalized] : [];\n};\n\nconst extractImageUrls = (html = '') => {\n  const found = [];\n  const seen = new Set();\n  const add = (value) => {\n    for (const imageUrl of imageUrlFromValue(value)) {\n      if (!imageUrl || seen.has(imageUrl)) continue;\n      seen.add(imageUrl);\n      found.push(imageUrl);\n    }\n  };\n\n  for (const block of extractScriptJson(html)) {\n    const stack = Array.isArray(block) ? [...block] : [block];\n    while (stack.length) {\n      const item = stack.shift();\n      if (!item || typeof item !== 'object') continue;\n      if (item.image) add(item.image);\n      if (item.contentUrl) add(item.contentUrl);\n      if (Array.isArray(item.itemListElement)) stack.push(...item.itemListElement.map((entry) => entry?.item || entry));\n      if (Array.isArray(item['@graph'])) stack.push(...item['@graph']);\n    }\n  }\n\n  const mediaMatches = String(html || '').match(/https?:\\/\\/media\\.kijiji\\.ca[^\"'\\s<>]+/gi) || [];\n  mediaMatches.forEach((url) => add(url.replace(/\\u002F/g, '/').replace(/\\u0026/g, '&')));\n  return found.slice(0, 12);\n};\n\nconst extractScriptJson = (html = '', id = '') => {\n  const pattern = id\n    ? new RegExp(`<script[^>]+id=[\"']${id}[\"'][^>]*>([\\\\s\\\\S]*?)<\\\\/script>`, 'i')\n    : /<script[^>]*type=[\"']application\\/(?:ld\\+json|json)[\"'][^>]*>([\\s\\S]*?)<\\/script>/gi;\n  if (id) {\n    const match = String(html || '').match(pattern);\n    if (!match) return null;\n    try { return JSON.parse(decodeHtml(match[1])); } catch { return null; }\n  }\n  return [...String(html || '').matchAll(pattern)].map((match) => {\n    try { return JSON.parse(decodeHtml(match[1])); } catch { return null; }\n  }).filter(Boolean);\n};\n\nconst priceTextFromListing = (listing = {}) => {\n  const amount = Number(listing?.price?.amount);\n  const currency = String(listing?.price?.currency || 'CAD').trim() || 'CAD';\n  if (!Number.isFinite(amount)) return '';\n  const major = amount > 999 ? amount / 100 : amount;\n  return currency === 'CAD' ? `CA$ ${major.toLocaleString('en-CA', { minimumFractionDigits: 2, maximumFractionDigits: 2 })}` : `${currency} ${major}`;\n};\n\nconst priceValueFromText = (value = '') => {\n  const parsed = Number(String(value || '').replace(/[^0-9.]/g, ''));\n  return Number.isFinite(parsed) ? String(parsed) : '';\n};\n\nconst normalizeCondition = (value = '') => {\n  const text = String(value || '').toLowerCase();\n  if (/like new/.test(text)) return 'like_new';\n  if (/excellent/.test(text)) return 'excellent';\n  if (/good/.test(text)) return 'good';\n  if (/fair/.test(text)) return 'fair';\n  if (/brand new|\\bnew\\b/.test(text)) return 'new';\n  if (/used|pre-owned|preowned/.test(text)) return 'used';\n  return '';\n};\n\nconst phonePattern = /(?<!\\d)(?:\\+?1[\\s.-]*)?(?:\\(?((?:226|249|289|343|365|416|437|519|548|647|705|742|807|905))\\)?[\\s.-]*)(\\d{3})[\\s.-]*(\\d{4})(?!\\d)/g;\nconst phoneContextPattern = /\\b(?:call|text|txt|phone|cell|tel|contact|whatsapp|message)\\b/i;\nconst extractPhones = (...values) => {\n  const found = [];\n  const seen = new Set();\n  for (const value of values) {\n    const text = cleanText(value);\n    let match;\n    while ((match = phonePattern.exec(text))) {\n      const raw = match[0];\n      const start = Math.max(0, match.index - 45);\n      const end = Math.min(text.length, match.index + raw.length + 25);\n      const hasSeparator = /[\\s().-]/.test(raw.trim());\n      if (!hasSeparator && !phoneContextPattern.test(text.slice(start, end))) continue;\n      const phone = `${match[1]}${match[2]}${match[3]}`;\n      if (!seen.has(phone)) {\n        seen.add(phone);\n        found.push(phone);\n      }\n    }\n  }\n  return found.join(' | ');\n};\n\nconst extractApolloListings = (html = '') => {\n  const nextData = extractScriptJson(html, '__NEXT_DATA__');\n  const apollo = nextData?.props?.pageProps?.__APOLLO_STATE__ || nextData?.props?.pageProps?.apolloState || {};\n  return Object.values(apollo)\n    .filter((entry) => entry && entry.__typename === 'StandardListing' && entry.url)\n    .map((entry) => {\n      const attrs = Array.isArray(entry?.attributes?.all) ? entry.attributes.all : [];\n      const attrMap = new Map(attrs.map((item) => [String(item?.canonicalName || '').trim(), item]));\n      const conditionAttr = attrMap.get('condition');\n      const priceText = priceTextFromListing(entry);\n      const description = cleanText(entry.description || '');\n      const title = cleanText(entry.title || '');\n      return {\n        listingId: String(entry.id || '').trim(),\n        title,\n        description,\n        sourceUrl: normalizeUrl(entry.url || ''),\n        imageUrls: Array.isArray(entry.imageUrls) ? entry.imageUrls.flatMap(imageUrlFromValue).filter(Boolean) : [],\n        priceText,\n        priceValue: priceValueFromText(priceText),\n        currency: String(entry?.price?.currency || 'CAD').trim() || 'CAD',\n        city: cleanText(entry?.location?.name || ''),\n        locationAddress: cleanText(entry?.location?.address || ''),\n        seller: cleanText(entry?.posterInfo?.name || '') || 'Unknown',\n        sellerId: String(entry?.posterInfo?.posterId || '').trim(),\n        sellerType: cleanText(entry?.posterInfo?.sellerType || ''),\n        condition: normalizeCondition((conditionAttr?.values || conditionAttr?.canonicalValues || [])[0] || `${title} ${description}`),\n        postedAt: String(entry.activationDate || entry.sortingDate || '').trim(),\n        sortingDate: String(entry.sortingDate || entry.activationDate || '').trim(),\n        phone: extractPhones(title, description)\n      };\n    });\n};\n\nconst extractJsonLdListings = (html = '') => {\n  const blocks = extractScriptJson(html);\n  const itemLists = blocks.flatMap((item) => Array.isArray(item) ? item : [item]).filter((item) => item?.['@type'] === 'ItemList' && Array.isArray(item.itemListElement));\n  return itemLists.flatMap((list) => list.itemListElement.map((entry) => {\n    const item = entry?.item || entry || {};\n    const sourceUrl = normalizeUrl(item.url || '');\n    const price = String(item?.offers?.price || '').trim();\n    return {\n      listingId: sourceUrl.match(/\\/(\\d+)(?:[/?#]|$)/)?.[1] || '',\n      title: cleanText(item.name || ''),\n      description: cleanText(item.description || ''),\n      sourceUrl,\n      imageUrls: item.image ? imageUrlFromValue(item.image) : [],\n      priceText: price ? `$${price}` : '',\n      priceValue: price,\n      currency: String(item?.offers?.priceCurrency || 'CAD').trim() || 'CAD',\n      city: '',\n      locationAddress: '',\n      seller: 'Unknown',\n      sellerId: '',\n      sellerType: '',\n      condition: '',\n      postedAt: '',\n      sortingDate: '',\n      phone: extractPhones(item.name, item.description)\n    };\n  })).filter((row) => row.title && row.sourceUrl);\n};\n\nconst inferCategory = (listing = {}, source = {}) => {\n  if (source.app_category || source.target_surface) {\n    return {\n      targetSurface: source.target_surface || (source.app_category === 'vehicles' ? 'vehicles' : 'marketplace'),\n      appCategory: source.app_category || 'electronics',\n      appSubcategory: source.app_subcategory || 'other'\n    };\n  }\n  const text = `${listing.sourceUrl || ''} ${listing.title || ''} ${listing.description || ''}`.toLowerCase();\n  if (/\\b(tires?|rims?|wheels?|auto-parts|car-parts|vehicle-parts|parting-out|motorcycle|brake|transmission|engine)\\b/.test(text)) {\n    return { targetSurface: 'vehicles', appCategory: 'vehicles', appSubcategory: /\\b(tires?|rims?|wheels?)\\b/.test(text) ? 'tires_rims' : 'auto_parts' };\n  }\n  if (/\\b(service|scrap metal|pick up|pickup|repair|cleaning|moving|delivery)\\b/.test(text)) {\n    return { targetSurface: 'marketplace', appCategory: 'services', appSubcategory: 'other' };\n  }\n  if (/\\b(phone|iphone|samsung|laptop|tablet|camera|lens|stereo|speaker|yamaha|canon|dewalt|battery|charger|computer|headphone|tv)\\b/.test(text)) {\n    let subcategory = 'other';\n    if (/\\b(phone|iphone|samsung)\\b/.test(text)) subcategory = 'phones_accessories';\n    else if (/\\b(laptop|tablet|computer)\\b/.test(text)) subcategory = 'computers_tablets';\n    else if (/\\b(camera|lens|canon)\\b/.test(text)) subcategory = 'cameras_photography';\n    else if (/\\b(stereo|speaker|yamaha|headphone)\\b/.test(text)) subcategory = 'audio_headphones';\n    return { targetSurface: 'marketplace', appCategory: 'electronics', appSubcategory: subcategory };\n  }\n  return { targetSurface: 'marketplace', appCategory: 'electronics', appSubcategory: 'other' };\n};\n\nconst matchesCity = (listing, source) => {\n  const strict = source.strict_city_match === true || String(source.strict_city_match || '').toLowerCase() === 'true';\n  if (!strict) return true;\n  const city = cleanText(source.city || '').toLowerCase();\n  if (!city) return true;\n  const hay = [listing.city, listing.locationAddress, listing.sourceUrl].map((value) => cleanText(value).toLowerCase()).join(' ');\n  return hay.includes(city.toLowerCase());\n};\n\nconst rowsFromCrawl = (crawlValue, activeSources) => {\n  const crawlItems = normalizeCrawlItems(crawlValue || {});\n  const crawlByUrl = new Map();\n  for (const item of crawlItems) {\n    const key = normalizeUrl(item.url || item.redirected_url || '');\n    if (key) crawlByUrl.set(key, item);\n  }\n  const output = [];\n  const sourceSummaries = [];\n  for (const source of activeSources) {\n    const listUrl = normalizeUrl(source.list_url || source.url || '');\n    const crawlItem = crawlByUrl.get(listUrl) || Array.from(crawlByUrl.values()).find((entry) => normalizeUrl(entry.url || entry.redirected_url || '') === listUrl);\n    const sourceName = source.name || source.city || listUrl;\n    if (!crawlItem) {\n      sourceSummaries.push({ source: sourceName, status: 'failed', fetched: 0, error: 'Crawl4AI did not return this URL.' });\n      continue;\n    }\n    const raw = crawlItem.html || crawlItem.cleaned_html || crawlItem.fit_html || crawlItem.markdown?.raw_markdown || crawlItem.markdown || crawlItem.fit_markdown || '';\n    if (!raw) {\n      sourceSummaries.push({ source: sourceName, status: 'failed', fetched: 0, error: crawlItem.error_message || 'No HTML returned.' });\n      continue;\n    }\n    let listings = extractApolloListings(raw);\n    if (!listings.length) listings = extractJsonLdListings(raw);\n    const maxListings = Math.max(1, Number(source.max_listings || 50));\n    const seen = new Set();\n    const normalized = listings\n      .filter((listing) => listing && listing.title && listing.sourceUrl && !seen.has(listing.sourceUrl) && (seen.add(listing.sourceUrl) || true))\n      .filter((listing) => matchesCity(listing, source))\n      .filter((listing) => String(listing.phone || '').trim())\n      .slice(0, maxListings)\n      .map((listing) => {\n        const inferred = inferCategory(listing, source);\n        const id = listing.listingId ? `kijiji-${listing.listingId}` : `kijiji-${Math.abs([...listing.sourceUrl].reduce((a, c) => ((a << 5) - a + c.charCodeAt(0)) | 0, 0))}`;\n        return {\n          id,\n          status: source.default_status || DEFAULT_STATUS,\n          target_surface: inferred.targetSurface,\n          app_category: inferred.appCategory,\n          app_subcategory: inferred.appSubcategory,\n          title: listing.title,\n          price_text: listing.priceText,\n          price_value: listing.priceValue,\n          currency: listing.currency || 'CAD',\n          city: listing.city || source.city || '',\n          country: source.country || 'Canada',\n          seller: listing.seller || 'Unknown',\n          phone: listing.phone || '',\n          description: listing.description,\n          image_urls: (listing.imageUrls || []).join('|'),\n          source_site: 'Kijiji',\n          source_url: listing.sourceUrl,\n          scraped_at: listing.sortingDate || listing.postedAt || nowIso,\n          make: '',\n          model: '',\n          trim: '',\n          year: String(`${listing.title} ${listing.description}`.match(/\\b(19|20)\\d{2}\\b/)?.[0] || ''),\n          condition: listing.condition || '',\n          transmission: '',\n          color: '',\n          mileage_km: '',\n          attributes: JSON.stringify({ parser: 'kijiji_crawl4ai_sync', listUrl, sourceName, locationAddress: listing.locationAddress, sellerId: listing.sellerId, sellerType: listing.sellerType }),\n          source_availability: 'active',\n          source_availability_checked_at: nowIso,\n          source_http_status: String(crawlItem.status_code || crawlItem.statusCode || 200),\n          source_unavailable_reason: '',\n          source_last_seen_at: nowIso,\n          source_resolved_url: listing.sourceUrl\n        };\n      });\n    output.push(...normalized);\n    sourceSummaries.push({ source: sourceName, status: normalized.length ? 'success' : 'empty', fetched: normalized.length });\n  }\n  return { rows: output, sourceSummaries };\n};\n\nconst crawlDetailUrls = async (urls) => {\n  const results = new Map();\n  for (let i = 0; i < urls.length; i += AVAILABILITY_BATCH_SIZE) {\n    const batch = urls.slice(i, i + AVAILABILITY_BATCH_SIZE);\n    const response = await httpRequest({\n      url: CRAWL4AI_URL,\n      method: 'POST',\n      json: true,\n      body: {\n        urls: batch,\n        browser_config: { headless: true, viewport: { width: 1280, height: 1800 }, verbose: false },\n        crawler_config: { stream: false, cache_mode: 'bypass', wait_until: 'domcontentloaded', wait_for: 'css:body', page_timeout: 20000, delay_before_return_html: 0.5, remove_overlay_elements: true, remove_consent_popups: true }\n      }\n    });\n    const items = normalizeCrawlItems(response.body || response);\n    for (let index = 0; index < items.length; index += 1) {\n      const item = items[index] || {};\n      const fallbackUrl = batch[index] || '';\n      const keys = [item.url, item.redirected_url, fallbackUrl].map(normalizeUrl).filter(Boolean);\n      const raw = item.html || item.cleaned_html || item.fit_html || item.markdown?.raw_markdown || item.markdown || item.fit_markdown || '';\n      const text = cleanText(raw).toLowerCase();\n      const statusCode = Number(item.status_code || item.statusCode || item.response?.status || response.status || 0);\n      let availability = 'unknown';\n      let reason = '';\n      if (statusCode === 404 || statusCode === 410) {\n        availability = 'gone';\n        reason = `HTTP ${statusCode}`;\n      } else if (/\\b(sold|item has sold|listing sold)\\b/i.test(text)) {\n        availability = 'sold';\n        reason = 'Source page says sold.';\n      } else if (/ad is no longer available|listing is no longer available|no longer available|has been removed|ad has expired|this ad is unavailable|page not found|does not exist/i.test(text)) {\n        availability = 'unavailable';\n        reason = 'Source page says removed, expired, or unavailable.';\n      } else if (raw && /kijiji/i.test(raw)) {\n        availability = 'active';\n      } else if (item.error_message || item.error) {\n        reason = String(item.error_message || item.error || '').slice(0, 160);\n      } else if (!raw) {\n        reason = 'No detail page content returned.';\n      }\n      const result = { availability, reason, statusCode: statusCode ? String(statusCode) : '', resolvedUrl: normalizeUrl(item.redirected_url || item.url || fallbackUrl), imageUrls: extractImageUrls(raw) };\n      keys.forEach((key) => results.set(key, result));\n    }\n  }\n  return results;\n};\n\nconst activeSources = sources.filter((source) => source && source.enabled !== false);\nconst existing = await getGithubCsv();\nconst existingRows = parseCsv(existing.text);\nconst rowsByUrl = new Map(existingRows.map((row) => [normalizeUrl(row.source_url || ''), row]).filter(([key]) => key));\n\nconst { rows: scrapedRows, sourceSummaries } = rowsFromCrawl(input.crawl4aiResult || input, activeSources);\nconst scrapedDetailUrls = scrapedRows.map((row) => normalizeUrl(row.source_url || '')).filter(Boolean).slice(0, AVAILABILITY_MAX_ROWS);\nconst scrapedDetailsByUrl = scrapedDetailUrls.length ? await crawlDetailUrls.call(this, scrapedDetailUrls) : new Map();\nfor (const row of scrapedRows) {\n  const detail = scrapedDetailsByUrl.get(normalizeUrl(row.source_url || ''));\n  if (detail?.imageUrls?.length) {\n    const currentImages = String(row.image_urls || '').split('|').map(normalizeImageUrl).filter(Boolean);\n    row.image_urls = Array.from(new Set([...detail.imageUrls, ...currentImages])).slice(0, 12).join('|');\n  }\n}\nlet inserted = 0;\nlet updated = 0;\nfor (const row of scrapedRows) {\n  const key = normalizeUrl(row.source_url || '');\n  if (!key) continue;\n  const previous = rowsByUrl.get(key) || {};\n  if (previous.source_url) updated += 1;\n  else inserted += 1;\n  rowsByUrl.set(key, { ...previous, ...row });\n}\n\nconst scrapedUrlSet = new Set(scrapedRows.map((row) => normalizeUrl(row.source_url || '')).filter(Boolean));\nconst configuredCities = activeSources.map((source) => cleanText(source.city || '').toLowerCase()).filter(Boolean);\nconst checkedAgeHours = (row) => {\n  const checkedAt = Date.parse(row.source_availability_checked_at || '');\n  if (!Number.isFinite(checkedAt)) return Number.POSITIVE_INFINITY;\n  return Math.max(0, (nowMs - checkedAt) / 36e5);\n};\nconst shouldCheckRow = (row) => {\n  const url = normalizeUrl(row.source_url || '');\n  if (!/kijiji\\.ca/i.test(url) && String(row.source_site || '').toLowerCase() !== 'kijiji') return false;\n  if (!configuredCities.length) return true;\n  const hay = [row.city, row.source_url, row.attributes].map((value) => cleanText(value).toLowerCase()).join(' ');\n  return configuredCities.some((city) => hay.includes(city));\n};\n\nconst availabilityCandidates = Array.from(rowsByUrl.values())\n  .filter(shouldCheckRow)\n  .filter((row) => {\n    const url = normalizeUrl(row.source_url || '');\n    if (scrapedUrlSet.has(url)) return false;\n    return checkedAgeHours(row) >= AVAILABILITY_FRESH_HOURS;\n  })\n  .sort((a, b) => checkedAgeHours(b) - checkedAgeHours(a) || String(b.scraped_at || '').localeCompare(String(a.scraped_at || '')))\n  .slice(0, AVAILABILITY_MAX_ROWS);\nconst candidateUrls = availabilityCandidates.map((row) => normalizeUrl(row.source_url || '')).filter(Boolean);\nconst availabilityByUrl = candidateUrls.length ? await crawlDetailUrls.call(this, candidateUrls) : new Map();\nlet active = 0;\nlet sold = 0;\nlet hidden = 0;\nlet unavailable = 0;\nlet unknown = 0;\nfor (const row of availabilityCandidates) {\n  const key = normalizeUrl(row.source_url || '');\n  const result = availabilityByUrl.get(key);\n  if (!result) continue;\n  row.source_availability = result.availability;\n  row.source_availability_checked_at = nowIso;\n  row.source_http_status = result.statusCode;\n  row.source_unavailable_reason = result.reason;\n  row.source_resolved_url = result.resolvedUrl || row.source_url;\n  if (result.availability === 'active') {\n    row.source_last_seen_at = nowIso;\n    active += 1;\n  } else if (result.availability === 'sold') {\n    sold += 1;\n  } else if (['unavailable', 'gone'].includes(result.availability)) {\n    unavailable += 1;\n    if (HIDE_UNAVAILABLE) {\n      row.status = 'rejected';\n      hidden += 1;\n    }\n  } else {\n    unknown += 1;\n  }\n}\n\nconst nextRows = Array.from(rowsByUrl.values()).sort((a, b) => String(b.scraped_at || '').localeCompare(String(a.scraped_at || '')));\nawait putGithubCsv(toCsv(nextRows), existing.sha);\n\nreturn [{\n  json: {\n    status: 'completed',\n    csvPath: CSV_PATH,\n    sources: sourceSummaries,\n    scraped: scrapedRows.length,\n    inserted,\n    updated,\n    detailImagesChecked: scrapedDetailUrls.length,\n    availabilityChecked: availabilityCandidates.length,\n    availabilityFreshHours: AVAILABILITY_FRESH_HOURS,\n    active,\n    sold,\n    unavailable,\n    unknown,\n    hidden,\n    hideUnavailableListings: HIDE_UNAVAILABLE\n  }\n}];"
      },
      "id": "6",
      "name": "Sync New Updated Sold Rows",
      "type": "n8n-nodes-base.code",
      "typeVersion": 2,
      "position": [
        1020,
        90
      ]
    }
  ],
  "connections": {
    "Manual Trigger": {
      "main": [
        [
          {
            "node": "Set Config Here",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Every Hour": {
      "main": [
        [
          {
            "node": "Set Config Here",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Set Config Here": {
      "main": [
        [
          {
            "node": "Crawl Kijiji List Pages",
            "type": "main",
            "index": 0
          },
          {
            "node": "Merge Config + Crawl Result",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Crawl Kijiji List Pages": {
      "main": [
        [
          {
            "node": "Merge Config + Crawl Result",
            "type": "main",
            "index": 1
          }
        ]
      ]
    },
    "Merge Config + Crawl Result": {
      "main": [
        [
          {
            "node": "Sync New Updated Sold Rows",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  },
  "active": false,
  "settings": {
    "executionOrder": "v1"
  },
  "versionId": "",
  "meta": {
    "templateCredsSetupCompleted": true
  },
  "id": "",
  "tags": []
}

About this workflow

6ixo - Kijiji Hamilton Sync to CSV. Uses httpRequest. Event-driven trigger; 6 nodes.

Source: https://github.com/bisco401/6ixo/blob/a53d9ab3886258a04bceca3c18d9474321aa85ff/automations/n8n/6ixo-kijiji-hamilton-sync-to-csv.json — original creator credit. Request a take-down →

More Data & Sheets workflows → · Browse all categories →

Related workflows

Workflows that share integrations, category, or trigger type with this one. All free to copy and import.