1	'use strict';
2	// MAHALLY — Salla's directory of Saudi online stores (mahally.com). Discovery via product search
3	// (/ar/browse?query=<category>), which surfaces merchant store links. Each store page exposes
4	// name, city, rating, review count, product count, and (often) the CR number + maroof.sa link
5	// pulled out of the free-text description. Bot-protected (Cloudflare-style human check on a bare
6	// fetch), so this goes through the same proxy + stealth browser as Maps — images/media/fonts are
7	// already blocked in browser.js's newPage() to save proxy GB.
8	// Usage: node src/scrape-mahally.js [outFile] [targetCount]
9	const fs = require('fs'), path = require('path');
10	(function () {
11	  const p = path.join(__dirname, '..', '.env');
12	  if (!fs.existsSync(p)) return;
13	  for (const l of fs.readFileSync(p, 'utf8').split('\n')) {
14	    const t = l.trim(); if (!t || t.startsWith('#')) continue;
15	    const i = t.indexOf('='); if (i > -1 && !(t.slice(0, i).trim() in process.env)) process.env[t.slice(0, i).trim()] = t.slice(i + 1).trim();
16	  }
17	})();
18	const { launchBrowser, newPage } = require('./browser');
19
20	const OUT_FILE = process.argv[2] || path.join(__dirname, '..', 'data', 'mahally_leads.csv');
21	const TARGET = parseInt(process.argv[3] || '100', 10);
22
23	// Real category names pulled from Mahally's own nav (found live on the browse page)
24	const CATEGORIES = [
25	  'القهوة', 'مأكولات', 'أزياء', 'جمال وعناية', 'إلكترونيات', 'المنزل',
26	  'قرطاسية', 'نظارات', 'كتب', 'بأيدي محلية', 'الحيوانات', 'ألعاب',
27	];
28
29	async function discoverStoreLinks(browser) {
30	  const found = new Set();
31	  for (const cat of CATEGORIES) {
32	    const page = await newPage(browser);
33	    try {
34	      const url = `https://mahally.com/ar/browse?query=${encodeURIComponent(cat)}`;
35	      await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
36	      await new Promise((r) => setTimeout(r, 3000));
37	      const links = await page.evaluate(() =>
38	        [...new Set(Array.from(document.querySelectorAll('a[href*="/stores/"]')).map((a) => a.href))]
39	      );
40	      for (const l of links) found.add(l.split('?')[0].replace(/\/$/, ''));
41	      process.stdout.write(`  [${cat}] +${links.length} (total unique: ${found.size})\n`);
42	    } catch (e) {
43	      process.stdout.write(`  [${cat}] failed: ${e.message}\n`);
44	    } finally {
45	      await page.close().catch(() => {});
46	    }
47	  }
48	  return [...found];
49	}
50
51	function parseStorePage(text) {
52	  if (/المتجر غير موجود/.test(text)) return null;
53	  const lines = text.split('\n').map((l) => l.trim()).filter(Boolean);
54	  // Pattern (from live sample): name, name(again), rating, (reviews), city, description...
55	  const name = lines[0] || null;
56	  let rating = null, reviews = null, city = null, idx = 1;
57	  // second occurrence of name is skipped if repeated
58	  if (lines[1] === name) idx = 2;
59	  if (/^[\d.]+$/.test(lines[idx] || '')) { rating = parseFloat(lines[idx]); idx++; }
60	  const reviewMatch = (lines[idx] || '').match(/^\((\d+)\)$/);
61	  if (reviewMatch) { reviews = parseInt(reviewMatch[1], 10); idx++; }
62	  city = lines[idx] || null;
63	  const description = lines.slice(idx + 1, idx + 4).join(' ');
64	  const crMatch = text.match(/رقم\s*(\d{6,})/);
65	  const productMatch = text.match(/(\d+)\s*منتج في المتجر/);
66	  const maroofMatch = text.match(/https?:\/\/maroof\.sa\/\d+/);
67	  return {
68	    name,
69	    city,
70	    rating,
71	    reviews,
72	    product_count: productMatch ? parseInt(productMatch[1], 10) : null,
73	    cr_number: crMatch ? crMatch[1] : null,
74	    maroof_url: maroofMatch ? maroofMatch[0] : null,
75	    description: description.slice(0, 200),
76	  };
77	}
78
79	async function fetchStoreDetails(browser, storeUrl) {
80	  const page = await newPage(browser);
81	  try {
82	    await page.goto(storeUrl + '/', { waitUntil: 'domcontentloaded', timeout: 45000 });
83	    await new Promise((r) => setTimeout(r, 2000));
84	    const text = await page.evaluate(() => document.body.innerText);
85	    const parsed = parseStorePage(text);
86	    if (!parsed) return null;
87	    return { url: storeUrl, ...parsed };
88	  } catch (e) {
89	    return null;
90	  } finally {
91	    await page.close().catch(() => {});
92	  }
93	}
94
95	function toCsvRow(vals) {
96	  return vals.map((v) => {
97	    const s = (v === null || v === undefined) ? '' : String(v);
98	    return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
99	  }).join(',');
100	}
101
102	(async () => {
103	  const browser = await launchBrowser();
104	  console.log(`Discovering stores across ${CATEGORIES.length} categories...`);
105	  const links = await discoverStoreLinks(browser);
106	  console.log(`Discovered ${links.length} unique candidate stores. Fetching details...`);
107
108	  const CONC = 6;
109	  const results = [];
110	  let i = 0;
111	  async function worker() {
112	    while (i < links.length) {
113	      const link = links[i++];
114	      const r = await fetchStoreDetails(browser, link);
115	      if (r) results.push(r);
116	      if (results.length % 20 === 0) process.stdout.write(`  fetched ${i}/${links.length}, ${results.length} valid\n`);
117	    }
118	  }
119	  await Promise.all(Array.from({ length: CONC }, worker));
120	  await browser.close();
121
122	  const valid = results.filter((r) => r.reviews !== null && r.reviews > 0);
123	  valid.sort((a, b) => a.reviews - b.reviews);
124	  console.log(`\n${valid.length} stores with review data. Review count range: ${valid[0]?.reviews} - ${valid[valid.length - 1]?.reviews}`);
125
126	  // "Mid-level" = the middle band by review count (not brand-new/zero, not mega-established)
127	  const loCut = Math.floor(valid.length * 0.3);
128	  const hiCut = Math.floor(valid.length * 0.8);
129	  let midBand = valid.slice(loCut, hiCut);
130	  if (midBand.length > TARGET) midBand = midBand.slice(0, TARGET);
131
132	  const header = ['store_name', 'city', 'rating', 'review_count', 'product_count', 'cr_number', 'maroof_url', 'description', 'mahally_url'];
133	  const rows = [toCsvRow(header)];
134	  for (const r of midBand) {
135	    rows.push(toCsvRow([r.name, r.city, r.rating, r.reviews, r.product_count, r.cr_number, r.maroof_url, r.description, r.url]));
136	  }
137	  fs.mkdirSync(path.dirname(OUT_FILE), { recursive: true });
138	  fs.writeFileSync(OUT_FILE, rows.join('\n'));
139	  console.log(`\nWrote ${midBand.length} mid-level stores to ${OUT_FILE}`);
140	})();
141

'use strict';
// MAHALLY — Salla's directory of Saudi online stores (mahally.com). Discovery via product search
// (/ar/browse?query=<category>), which surfaces merchant store links. Each store page exposes
// name, city, rating, review count, product count, and (often) the CR number + maroof.sa link
// pulled out of the free-text description. Bot-protected (Cloudflare-style human check on a bare
// fetch), so this goes through the same proxy + stealth browser as Maps — images/media/fonts are
// already blocked in browser.js's newPage() to save proxy GB.
// Usage: node src/scrape-mahally.js [outFile] [targetCount]
const fs = require('fs'), path = require('path');
(function () {
  const p = path.join(__dirname, '..', '.env');
  if (!fs.existsSync(p)) return;
  for (const l of fs.readFileSync(p, 'utf8').split('\n')) {
    const t = l.trim(); if (!t || t.startsWith('#')) continue;
    const i = t.indexOf('='); if (i > -1 && !(t.slice(0, i).trim() in process.env)) process.env[t.slice(0, i).trim()] = t.slice(i + 1).trim();
  }
})();
const { launchBrowser, newPage } = require('./browser');

const OUT_FILE = process.argv[2] || path.join(__dirname, '..', 'data', 'mahally_leads.csv');
const TARGET = parseInt(process.argv[3] || '100', 10);

// Real category names pulled from Mahally's own nav (found live on the browse page)
const CATEGORIES = [
  'القهوة', 'مأكولات', 'أزياء', 'جمال وعناية', 'إلكترونيات', 'المنزل',
  'قرطاسية', 'نظارات', 'كتب', 'بأيدي محلية', 'الحيوانات', 'ألعاب',
];

async function discoverStoreLinks(browser) {
  const found = new Set();
  for (const cat of CATEGORIES) {
    const page = await newPage(browser);
    try {
      const url = `https://mahally.com/ar/browse?query=${encodeURIComponent(cat)}`;
      await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
      await new Promise((r) => setTimeout(r, 3000));
      const links = await page.evaluate(() =>
        [...new Set(Array.from(document.querySelectorAll('a[href*="/stores/"]')).map((a) => a.href))]
      );
      for (const l of links) found.add(l.split('?')[0].replace(/\/$/, ''));
      process.stdout.write(`  [${cat}] +${links.length} (total unique: ${found.size})\n`);
    } catch (e) {
      process.stdout.write(`  [${cat}] failed: ${e.message}\n`);
    } finally {
      await page.close().catch(() => {});
    }
  }
  return [...found];
}

function parseStorePage(text) {
  if (/المتجر غير موجود/.test(text)) return null;
  const lines = text.split('\n').map((l) => l.trim()).filter(Boolean);
  // Pattern (from live sample): name, name(again), rating, (reviews), city, description...
  const name = lines[0] || null;
  let rating = null, reviews = null, city = null, idx = 1;
  // second occurrence of name is skipped if repeated
  if (lines[1] === name) idx = 2;
  if (/^[\d.]+$/.test(lines[idx] || '')) { rating = parseFloat(lines[idx]); idx++; }
  const reviewMatch = (lines[idx] || '').match(/^\((\d+)\)$/);
  if (reviewMatch) { reviews = parseInt(reviewMatch[1], 10); idx++; }
  city = lines[idx] || null;
  const description = lines.slice(idx + 1, idx + 4).join(' ');
  const crMatch = text.match(/رقم\s*(\d{6,})/);
  const productMatch = text.match(/(\d+)\s*منتج في المتجر/);
  const maroofMatch = text.match(/https?:\/\/maroof\.sa\/\d+/);
  return {
    name,
    city,
    rating,
    reviews,
    product_count: productMatch ? parseInt(productMatch[1], 10) : null,
    cr_number: crMatch ? crMatch[1] : null,
    maroof_url: maroofMatch ? maroofMatch[0] : null,
    description: description.slice(0, 200),
  };
}

async function fetchStoreDetails(browser, storeUrl) {
  const page = await newPage(browser);
  try {
    await page.goto(storeUrl + '/', { waitUntil: 'domcontentloaded', timeout: 45000 });
    await new Promise((r) => setTimeout(r, 2000));
    const text = await page.evaluate(() => document.body.innerText);
    const parsed = parseStorePage(text);
    if (!parsed) return null;
    return { url: storeUrl, ...parsed };
  } catch (e) {
    return null;
  } finally {
    await page.close().catch(() => {});
  }
}

function toCsvRow(vals) {
  return vals.map((v) => {
    const s = (v === null || v === undefined) ? '' : String(v);
    return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
  }).join(',');
}

(async () => {
  const browser = await launchBrowser();
  console.log(`Discovering stores across ${CATEGORIES.length} categories...`);
  const links = await discoverStoreLinks(browser);
  console.log(`Discovered ${links.length} unique candidate stores. Fetching details...`);

  const CONC = 6;
  const results = [];
  let i = 0;
  async function worker() {
    while (i < links.length) {
      const link = links[i++];
      const r = await fetchStoreDetails(browser, link);
      if (r) results.push(r);
      if (results.length % 20 === 0) process.stdout.write(`  fetched ${i}/${links.length}, ${results.length} valid\n`);
    }
  }
  await Promise.all(Array.from({ length: CONC }, worker));
  await browser.close();

  const valid = results.filter((r) => r.reviews !== null && r.reviews > 0);
  valid.sort((a, b) => a.reviews - b.reviews);
  console.log(`\n${valid.length} stores with review data. Review count range: ${valid[0]?.reviews} - ${valid[valid.length - 1]?.reviews}`);

  // "Mid-level" = the middle band by review count (not brand-new/zero, not mega-established)
  const loCut = Math.floor(valid.length * 0.3);
  const hiCut = Math.floor(valid.length * 0.8);
  let midBand = valid.slice(loCut, hiCut);
  if (midBand.length > TARGET) midBand = midBand.slice(0, TARGET);

  const header = ['store_name', 'city', 'rating', 'review_count', 'product_count', 'cr_number', 'maroof_url', 'description', 'mahally_url'];
  const rows = [toCsvRow(header)];
  for (const r of midBand) {
    rows.push(toCsvRow([r.name, r.city, r.rating, r.reviews, r.product_count, r.cr_number, r.maroof_url, r.description, r.url]));
  }
  fs.mkdirSync(path.dirname(OUT_FILE), { recursive: true });
  fs.writeFileSync(OUT_FILE, rows.join('\n'));
  console.log(`\nWrote ${midBand.length} mid-level stores to ${OUT_FILE}`);
})();


Write

'use strict';
// MAHALLY — Salla's directory of Saudi online stores (mahally.com). Discovery via product search
// (/ar/browse?query=<category>), which surfaces merchant store links. Each store page exposes
// name, city, rating, review count, product count, and (often) the CR number + maroof.sa link
// pulled out of the free-text description. Bot-protected (Cloudflare-style human check on a bare
// fetch), so this goes through the same proxy + stealth browser as Maps — images/media/fonts are
// already blocked in browser.js's newPage() to save proxy GB.
// Usage: node src/scrape-mahally.js [outFile] [targetCount]
const fs = require('fs'), path = require('path');
(function () {
  const p = path.join(__dirname, '..', '.env');
  if (!fs.existsSync(p)) return;
  for (const l of fs.readFileSync(p, 'utf8').split('\n')) {
    const t = l.trim(); if (!t || t.startsWith('#')) continue;
    const i = t.indexOf('='); if (i > -1 && !(t.slice(0, i).trim() in process.env)) process.env[t.slice(0, i).trim()] = t.slice(i + 1).trim();
  }
})();
const { launchBrowser, newPage } = require('./browser');

const OUT_FILE = process.argv[2] || path.join(__dirname, '..', 'data', 'mahally_leads.csv');
const TARGET = parseInt(process.argv[3] || '100', 10);
const RAW_FILE = OUT_FILE.replace(/\.csv$/, '') + '.raw.jsonl';

// Real category names pulled from Mahally's own nav (found live on the browse page)
const CATEGORIES = [
  'القهوة', 'مأكولات', 'أزياء', 'جمال وعناية', 'إلكترونيات', 'المنزل',
  'قرطاسية', 'نظارات', 'كتب', 'بأيدي محلية', 'الحيوانات', 'ألعاب',
];

async function discoverStoreLinks(browser) {
  const found = new Set();
  for (const cat of CATEGORIES) {
    const page = await newPage(browser);
    try {
      const url = `https://mahally.com/ar/browse?query=${encodeURIComponent(cat)}`;
      await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
      await new Promise((r) => setTimeout(r, 3000));
      const links = await page.evaluate(() =>
        [...new Set(Array.from(document.querySelectorAll('a[href*="/stores/"]')).map((a) => a.href))]
      );
      for (const l of links) found.add(l.split('?')[0].replace(/\/$/, ''));
      process.stdout.write(`  [${cat}] +${links.length} (total unique: ${found.size})\n`);
    } catch (e) {
      process.stdout.write(`  [${cat}] failed: ${e.message}\n`);
    } finally {
      await page.close().catch(() => {});
    }
  }
  return [...found];
}

function parseStorePage(text) {
  if (/المتجر غير موجود/.test(text)) return null;
  const lines = text.split('\n').map((l) => l.trim()).filter(Boolean);
  const name = lines[0] || null;
  let rating = null, reviews = null, city = null, idx = 1;
  if (lines[1] === name) idx = 2;
  if (/^[\d.]+$/.test(lines[idx] || '')) { rating = parseFloat(lines[idx]); idx++; }
  const reviewMatch = (lines[idx] || '').match(/^\((\d+)\)$/);
  if (reviewMatch) { reviews = parseInt(reviewMatch[1], 10); idx++; }
  city = lines[idx] || null;
  const description = lines.slice(idx + 1, idx + 4).join(' ');
  const crMatch = text.match(/رقم\s*(\d{6,})/);
  const productMatch = text.match(/(\d+)\s*منتج في المتجر/);
  const maroofMatch = text.match(/https?:\/\/maroof\.sa\/\d+/);
  return {
    name, city, rating, reviews,
    product_count: productMatch ? parseInt(productMatch[1], 10) : null,
    cr_number: crMatch ? crMatch[1] : null,
    maroof_url: maroofMatch ? maroofMatch[0] : null,
    description: description.slice(0, 200),
  };
}

async function fetchStoreDetails(browser, storeUrl) {
  const page = await newPage(browser);
  try {
    await page.goto(storeUrl + '/', { waitUntil: 'domcontentloaded', timeout: 45000 });
    await new Promise((r) => setTimeout(r, 2000));
    const text = await page.evaluate(() => document.body.innerText);
    const parsed = parseStorePage(text);
    if (!parsed) return null;
    return { url: storeUrl, ...parsed };
  } catch (e) {
    return null;
  } finally {
    await page.close().catch(() => {});
  }
}

function toCsvRow(vals) {
  return vals.map((v) => {
    const s = (v === null || v === undefined) ? '' : String(v);
    return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
  }).join(',');
}

(async () => {
  let browser = await launchBrowser();
  console.log(`Discovering stores across ${CATEGORIES.length} categories...`);
  const links = await discoverStoreLinks(browser);
  console.log(`Discovered ${links.length} unique candidate stores. Fetching details (checkpointing to ${RAW_FILE})...`);

  const rawStream = fs.createWriteStream(RAW_FILE, { flags: 'w' });
  const CONC = 3; // kept low so this doesn't fight the live Maps scraper for CPU/RAM
  let validCount = 0, i = 0;

  async function worker() {
    while (i < links.length) {
      const link = links[i++];
      let r;
      try {
        r = await fetchStoreDetails(browser, link);
      } catch (e) {
        // Browser itself likely died (Protocol error / disconnected) — relaunch and retry this link once.
        if (!browser.isConnected()) {
          console.log(`  browser disconnected, relaunching...`);
          try { await browser.close(); } catch (_) {}
          browser = await launchBrowser();
        }
        r = await fetchStoreDetails(browser, link).catch(() => null);
      }
      if (r) {
        validCount++;
        rawStream.write(JSON.stringify(r) + '\n');
      }
      if (i % 20 === 0) process.stdout.write(`  fetched ${i}/${links.length}, ${validCount} valid\n`);
    }
  }
  await Promise.all(Array.from({ length: CONC }, worker));
  await new Promise((resolve) => rawStream.end(resolve));
  await browser.close().catch(() => {});

  // Build final CSV from the checkpointed raw file (survives any crash after this point too)
  const results = fs.readFileSync(RAW_FILE, 'utf8').trim().split('\n').filter(Boolean).map((l) => JSON.parse(l));
  const valid = results.filter((r) => r.reviews !== null && r.reviews > 0);
  valid.sort((a, b) => a.reviews - b.reviews);
  console.log(`\n${valid.length} stores with review data. Review count range: ${valid[0]?.reviews} - ${valid[valid.length - 1]?.reviews}`);

  // "Mid-level" = the middle band by review count (not brand-new/zero, not mega-established)
  const loCut = Math.floor(valid.length * 0.3);
  const hiCut = Math.floor(valid.length * 0.8);
  let midBand = valid.slice(loCut, hiCut);
  if (midBand.length > TARGET) midBand = midBand.slice(0, TARGET);

  const header = ['store_name', 'city', 'rating', 'review_count', 'product_count', 'cr_number', 'maroof_url', 'description', 'mahally_url'];
  const rows = [toCsvRow(header)];
  for (const r of midBand) {
    rows.push(toCsvRow([r.name, r.city, r.rating, r.reviews, r.product_count, r.cr_number, r.maroof_url, r.description, r.url]));
  }
  fs.mkdirSync(path.dirname(OUT_FILE), { recursive: true });
  fs.writeFileSync(OUT_FILE, rows.join('\n'));
  console.log(`\nWrote ${midBand.length} mid-level stores to ${OUT_FILE}`);
})();


The file /root/ultron/src/scrape-mahally.js has been updated successfully. (file state is current in your context — no need to Read it back)

'use strict';
// MAHALLY — Salla's directory of Saudi online stores (mahally.com). Discovery via product search
// (/ar/browse?query=<category>), which surfaces merchant store links. Each store page exposes
// name, city, rating, review count, product count, and (often) the CR number + maroof.sa link
// pulled out of the free-text description. Bot-protected (Cloudflare-style human check on a bare
// fetch), so this goes through the same proxy + stealth browser as Maps — images/media/fonts are
// already blocked in browser.js's newPage() to save proxy GB.
// Usage: node src/scrape-mahally.js [outFile] [targetCount]
const fs = require('fs'), path = require('path');
(function () {
  const p = path.join(__dirname, '..', '.env');
  if (!fs.existsSync(p)) return;
  for (const l of fs.readFileSync(p, 'utf8').split('\n')) {
    const t = l.trim(); if (!t || t.startsWith('#')) continue;
    const i = t.indexOf('='); if (i > -1 && !(t.slice(0, i).trim() in process.env)) process.env[t.slice(0, i).trim()] = t.slice(i + 1).trim();
  }
})();
const { launchBrowser, newPage } = require('./browser');

const OUT_FILE = process.argv[2] || path.join(__dirname, '..', 'data', 'mahally_leads.csv');
const TARGET = parseInt(process.argv[3] || '100', 10);
const RAW_FILE = OUT_FILE.replace(/\.csv$/, '') + '.raw.jsonl';

// Real category names pulled from Mahally's own nav (found live on the browse page)
const CATEGORIES = [
  'القهوة', 'مأكولات', 'أزياء', 'جمال وعناية', 'إلكترونيات', 'المنزل',
  'قرطاسية', 'نظارات', 'كتب', 'بأيدي محلية', 'الحيوانات', 'ألعاب',
];

async function discoverStoreLinks(browser) {
  const found = new Set();
  for (const cat of CATEGORIES) {
    const page = await newPage(browser);
    try {
      const url = `https://mahally.com/ar/browse?query=${encodeURIComponent(cat)}`;
      await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
      await new Promise((r) => setTimeout(r, 3000));
      const links = await page.evaluate(() =>
        [...new Set(Array.from(document.querySelectorAll('a[href*="/stores/"]')).map((a) => a.href))]
      );
      for (const l of links) found.add(l.split('?')[0].replace(/\/$/, ''));
      process.stdout.write(`  [${cat}] +${links.length} (total unique: ${found.size})\n`);
    } catch (e) {
      process.stdout.write(`  [${cat}] failed: ${e.message}\n`);
    } finally {
      await page.close().catch(() => {});
    }
  }
  return [...found];
}

function parseStorePage(text) {
  if (/المتجر غير موجود/.test(text)) return null;
  const lines = text.split('\n').map((l) => l.trim()).filter(Boolean);
  const name = lines[0] || null;
  let rating = null, reviews = null, city = null, idx = 1;
  if (lines[1] === name) idx = 2;
  if (/^[\d.]+$/.test(lines[idx] || '')) { rating = parseFloat(lines[idx]); idx++; }
  const reviewMatch = (lines[idx] || '').match(/^\((\d+)\)$/);
  if (reviewMatch) { reviews = parseInt(reviewMatch[1], 10); idx++; }
  city = lines[idx] || null;
  const description = lines.slice(idx + 1, idx + 4).join(' ');
  const crMatch = text.match(/رقم\s*(\d{6,})/);
  const productMatch = text.match(/(\d+)\s*منتج في المتجر/);
  const maroofMatch = text.match(/https?:\/\/maroof\.sa\/\d+/);
  return {
    name, city, rating, reviews,
    product_count: productMatch ? parseInt(productMatch[1], 10) : null,
    cr_number: crMatch ? crMatch[1] : null,
    maroof_url: maroofMatch ? maroofMatch[0] : null,
    description: description.slice(0, 200),
  };
}

async function fetchStoreDetails(browser, storeUrl) {
  const page = await newPage(browser);
  try {
    await page.goto(storeUrl + '/', { waitUntil: 'domcontentloaded', timeout: 45000 });
    await new Promise((r) => setTimeout(r, 2000));
    const text = await page.evaluate(() => document.body.innerText);
    const parsed = parseStorePage(text);
    if (!parsed) return null;
    return { url: storeUrl, ...parsed };
  } catch (e) {
    return null;
  } finally {
    await page.close().catch(() => {});
  }
}

function toCsvRow(vals) {
  return vals.map((v) => {
    const s = (v === null || v === undefined) ? '' : String(v);
    return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
  }).join(',');
}

(async () => {
  let browser = await launchBrowser();
  console.log(`Discovering stores across ${CATEGORIES.length} categories...`);
  const links = await discoverStoreLinks(browser);
  console.log(`Discovered ${links.length} unique candidate stores. Fetching details (checkpointing to ${RAW_FILE})...`);

  const rawStream = fs.createWriteStream(RAW_FILE, { flags: 'w' });
  const CONC = 3; // kept low so this doesn't fight the live Maps scraper for CPU/RAM
  let validCount = 0, i = 0;

  async function worker() {
    while (i < links.length) {
      const link = links[i++];
      let r;
      try {
        r = await fetchStoreDetails(browser, link);
      } catch (e) {
        // Browser itself likely died (Protocol error / disconnected) — relaunch and retry this link once.
        if (!browser.isConnected()) {
          console.log(`  browser disconnected, relaunching...`);
          try { await browser.close(); } catch (_) {}
          browser = await launchBrowser();
        }
        r = await fetchStoreDetails(browser, link).catch(() => null);
      }
      if (r) {
        validCount++;
        rawStream.write(JSON.stringify(r) + '\n');
      }
      if (i % 20 === 0) process.stdout.write(`  fetched ${i}/${links.length}, ${validCount} valid\n`);
    }
  }
  await Promise.all(Array.from({ length: CONC }, worker));
  await new Promise((resolve) => rawStream.end(resolve));
  await browser.close().catch(() => {});

  // Build final CSV from the checkpointed raw file (survives any crash after this point too)
  const results = fs.readFileSync(RAW_FILE, 'utf8').trim().split('\n').filter(Boolean).map((l) => JSON.parse(l));
  const valid = results.filter((r) => r.reviews !== null && r.reviews > 0);
  valid.sort((a, b) => a.reviews - b.reviews);
  console.log(`\n${valid.length} stores with review data. Review count range: ${valid[0]?.reviews} - ${valid[valid.length - 1]?.reviews}`);

  // "Mid-level" = the middle band by review count (not brand-new/zero, not mega-established)
  const loCut = Math.floor(valid.length * 0.3);
  const hiCut = Math.floor(valid.length * 0.8);
  let midBand = valid.slice(loCut, hiCut);
  if (midBand.length > TARGET) midBand = midBand.slice(0, TARGET);

  const header = ['store_name', 'city', 'rating', 'review_count', 'product_count', 'cr_number', 'maroof_url', 'description', 'mahally_url'];
  const rows = [toCsvRow(header)];
  for (const r of midBand) {
    rows.push(toCsvRow([r.name, r.city, r.rating, r.reviews, r.product_count, r.cr_number, r.maroof_url, r.description, r.url]));
  }
  fs.mkdirSync(path.dirname(OUT_FILE), { recursive: true });
  fs.writeFileSync(OUT_FILE, rows.join('\n'));
  console.log(`\nWrote ${midBand.length} mid-level stores to ${OUT_FILE}`);
})();

+const RAW_FILE = OUT_FILE.replace(/\.csv$/, '') + '.raw.jsonl';

-  // Pattern (from live sample): name, name(again), rating, (reviews), city, description...

-  // second occurrence of name is skipped if repeated

-  const browser = await launchBrowser();

-  console.log(`Discovered ${links.length} unique candidate stores. Fetching details...`);

+  console.log(`Discovered ${links.length} unique candidate stores. Fetching details (checkpointing to ${RAW_FILE})...`);

+  const rawStream = fs.createWriteStream(RAW_FILE, { flags: 'w' });

+  const CONC = 3; // kept low so this doesn't fight the live Maps scraper for CPU/RAM

-      const r = await fetchStoreDetails(browser, link);

-      if (results.length % 20 === 0) process.stdout.write(`  fetched ${i}/${links.length}, ${results.length} valid\n`);

+        r = await fetchStoreDetails(browser, link);

+        // Browser itself likely died (Protocol error / disconnected) — relaunch and retry this link once.

+          console.log(`  browser disconnected, relaunching...`);

+          try { await browser.close(); } catch (_) {}

+          browser = await launchBrowser();

+        r = await fetchStoreDetails(browser, link).catch(() => null);

+        rawStream.write(JSON.stringify(r) + '\n');

+      if (i % 20 === 0) process.stdout.write(`  fetched ${i}/${links.length}, ${validCount} valid\n`);

+  await new Promise((resolve) => rawStream.end(resolve));

+  await browser.close().catch(() => {});

+  // Build final CSV from the checkpointed raw file (survives any crash after this point too)

+  const results = fs.readFileSync(RAW_FILE, 'utf8').trim().split('\n').filter(Boolean).map((l) => JSON.parse(l));

'use strict';
// MAHALLY — Salla's directory of Saudi online stores (mahally.com). Discovery via product search
// (/ar/browse?query=<category>), which surfaces merchant store links. Each store page exposes
// name, city, rating, review count, product count, and (often) the CR number + maroof.sa link
// pulled out of the free-text description. Bot-protected (Cloudflare-style human check on a bare
// fetch), so this goes through the same proxy + stealth browser as Maps — images/media/fonts are
// already blocked in browser.js's newPage() to save proxy GB.
// Usage: node src/scrape-mahally.js [outFile] [targetCount]
const fs = require('fs'), path = require('path');
(function () {
  const p = path.join(__dirname, '..', '.env');
  if (!fs.existsSync(p)) return;
  for (const l of fs.readFileSync(p, 'utf8').split('\n')) {
    const t = l.trim(); if (!t || t.startsWith('#')) continue;
    const i = t.indexOf('='); if (i > -1 && !(t.slice(0, i).trim() in process.env)) process.env[t.slice(0, i).trim()] = t.slice(i + 1).trim();
  }
})();
const { launchBrowser, newPage } = require('./browser');

const OUT_FILE = process.argv[2] || path.join(__dirname, '..', 'data', 'mahally_leads.csv');
const TARGET = parseInt(process.argv[3] || '100', 10);

// Real category names pulled from Mahally's own nav (found live on the browse page)
const CATEGORIES = [
  'القهوة', 'مأكولات', 'أزياء', 'جمال وعناية', 'إلكترونيات', 'المنزل',
  'قرطاسية', 'نظارات', 'كتب', 'بأيدي محلية', 'الحيوانات', 'ألعاب',
];

async function discoverStoreLinks(browser) {
  const found = new Set();
  for (const cat of CATEGORIES) {
    const page = await newPage(browser);
    try {
      const url = `https://mahally.com/ar/browse?query=${encodeURIComponent(cat)}`;
      await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
      await new Promise((r) => setTimeout(r, 3000));
      const links = await page.evaluate(() =>
        [...new Set(Array.from(document.querySelectorAll('a[href*="/stores/"]')).map((a) => a.href))]
      );
      for (const l of links) found.add(l.split('?')[0].replace(/\/$/, ''));
      process.stdout.write(`  [${cat}] +${links.length} (total unique: ${found.size})\n`);
    } catch (e) {
      process.stdout.write(`  [${cat}] failed: ${e.message}\n`);
    } finally {
      await page.close().catch(() => {});
    }
  }
  return [...found];
}

function parseStorePage(text) {
  if (/المتجر غير موجود/.test(text)) return null;
  const lines = text.split('\n').map((l) => l.trim()).filter(Boolean);
  // Pattern (from live sample): name, name(again), rating, (reviews), city, description...
  const name = lines[0] || null;
  let rating = null, reviews = null, city = null, idx = 1;
  // second occurrence of name is skipped if repeated
  if (lines[1] === name) idx = 2;
  if (/^[\d.]+$/.test(lines[idx] || '')) { rating = parseFloat(lines[idx]); idx++; }
  const reviewMatch = (lines[idx] || '').match(/^\((\d+)\)$/);
  if (reviewMatch) { reviews = parseInt(reviewMatch[1], 10); idx++; }
  city = lines[idx] || null;
  const description = lines.slice(idx + 1, idx + 4).join(' ');
  const crMatch = text.match(/رقم\s*(\d{6,})/);
  const productMatch = text.match(/(\d+)\s*منتج في المتجر/);
  const maroofMatch = text.match(/https?:\/\/maroof\.sa\/\d+/);
  return {
    name,
    city,
    rating,
    reviews,
    product_count: productMatch ? parseInt(productMatch[1], 10) : null,
    cr_number: crMatch ? crMatch[1] : null,
    maroof_url: maroofMatch ? maroofMatch[0] : null,
    description: description.slice(0, 200),
  };
}

async function fetchStoreDetails(browser, storeUrl) {
  const page = await newPage(browser);
  try {
    await page.goto(storeUrl + '/', { waitUntil: 'domcontentloaded', timeout: 45000 });
    await new Promise((r) => setTimeout(r, 2000));
    const text = await page.evaluate(() => document.body.innerText);
    const parsed = parseStorePage(text);
    if (!parsed) return null;
    return { url: storeUrl, ...parsed };
  } catch (e) {
    return null;
  } finally {
    await page.close().catch(() => {});
  }
}

function toCsvRow(vals) {
  return vals.map((v) => {
    const s = (v === null || v === undefined) ? '' : String(v);
    return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
  }).join(',');
}

(async () => {
  const browser = await launchBrowser();
  console.log(`Discovering stores across ${CATEGORIES.length} categories...`);
  const links = await discoverStoreLinks(browser);
  console.log(`Discovered ${links.length} unique candidate stores. Fetching details...`);

  const CONC = 6;
  const results = [];
  let i = 0;
  async function worker() {
    while (i < links.length) {
      const link = links[i++];
      const r = await fetchStoreDetails(browser, link);
      if (r) results.push(r);
      if (results.length % 20 === 0) process.stdout.write(`  fetched ${i}/${links.length}, ${results.length} valid\n`);
    }
  }
  await Promise.all(Array.from({ length: CONC }, worker));
  await browser.close();

  const valid = results.filter((r) => r.reviews !== null && r.reviews > 0);
  valid.sort((a, b) => a.reviews - b.reviews);
  console.log(`\n${valid.length} stores with review data. Review count range: ${valid[0]?.reviews} - ${valid[valid.length - 1]?.reviews}`);

  // "Mid-level" = the middle band by review count (not brand-new/zero, not mega-established)
  const loCut = Math.floor(valid.length * 0.3);
  const hiCut = Math.floor(valid.length * 0.8);
  let midBand = valid.slice(loCut, hiCut);
  if (midBand.length > TARGET) midBand = midBand.slice(0, TARGET);

  const header = ['store_name', 'city', 'rating', 'review_count', 'product_count', 'cr_number', 'maroof_url', 'description', 'mahally_url'];
  const rows = [toCsvRow(header)];
  for (const r of midBand) {
    rows.push(toCsvRow([r.name, r.city, r.rating, r.reviews, r.product_count, r.cr_number, r.maroof_url, r.description, r.url]));
  }
  fs.mkdirSync(path.dirname(OUT_FILE), { recursive: true });
  fs.writeFileSync(OUT_FILE, rows.join('\n'));
  console.log(`\nWrote ${midBand.length} mid-level stores to ${OUT_FILE}`);
})();


Now let's rerun it in the background with the fixes.


Bash

nohup node src/scrape-mahally.js data/mahally_leads.csv 100 > data/mahally_run.log 2>&1 &
echo "started pid $!"


started pid 131027


I'll monitor this run in the background and check back periodically.


until ! ps -p 131027 > /dev/null 2>&1; do sleep 15; done; echo mahally_scrape_done


Command running in background with ID: b2h9ry2ag. Output is being written to: /tmp/claude-0/-root-ultron/e44a47d2-ef9c-4c03-9995-d56770ba2ff7/tasks/b2h9ry2ag.output. You will be notified when it completes. To check interim output, use Read on that file path.


Discovering stores across 12 categories...
  [القهوة] +39 (total unique: 39)


Fixed and running again (lower concurrency, checkpointing to disk as it goes so a crash can't wipe progress this time). Discovery + ~467 store detail fetches will take a while — I'll ping you the sheet the moment it's done, no need to check in.


sent (id: 135)

sent (id: 135)


I'll wait for the background scrape to finish before checking results.


<channel source="plugin:telegram:telegram" chat_id="798935024" message_id="136" user="798935024" user_id="798935024" ts="2026-07-03T21:31:41.000Z">
Cool cool lower concerncy and try again and no need to fetch that amout 50 is good enough for test and then give them to me as a sheet
</channel>


reacted

reacted


tail -5 /root/ultron/data/mahally_run.log; ps aux | grep "[s]crape-mahally"


Discovering stores across 12 categories...
  [القهوة] +39 (total unique: 39)
  [مأكولات] +71 (total unique: 110)
  [أزياء] +30 (total unique: 140)
root      131027  3.9  1.2 1171316 104732 ?      Sl   21:31   0:01 node src/scrape-mahally.js data/mahally_leads.csv 100


Killing the current run and restarting with lower concurrency and a capped fetch count for a faster test.


kill -9 131027 2>/dev/null; sleep 1; ps aux | grep "[s]crape-mahally"


(Bash completed with no output)


<task-notification>
<task-id>b2h9ry2ag</task-id>
<tool-use-id>toolu_01CVkEXSzHnYCfruPvw9ryh4</tool-use-id>
<output-file>/tmp/claude-0/-root-ultron/e44a47d2-ef9c-4c03-9995-d56770ba2ff7/tasks/b2h9ry2ag.output</output-file>
<status>completed</status>
<summary>Background command "until ! ps -p 131027 > /dev/null 2>&1; do sleep 15; done; echo mahally_scrape_done" completed (exit code 0)</summary>
</task-notification>


102	  console.log(`Discovered ${links.length} unique candidate stores. Fetching details (checkpointing to ${RAW_FILE})...`);
103
104	  const rawStream = fs.createWriteStream(RAW_FILE, { flags: 'w' });
105	  const CONC = 3; // kept low so this doesn't fight the live Maps scraper for CPU/RAM
106	  let validCount = 0, i = 0;
107
108	  async function worker() {
109	    while (i < links.length) {
110	      const link = links[i++];
111	      let r;
112	      try {
113	        r = await fetchStoreDetails(browser, link);
114	      } catch (e) {
115	        // Browser itself likely died (Protocol error / disconnected) — relaunch and retry this link once.
116	        if (!browser.isConnected()) {

  console.log(`Discovered ${links.length} unique candidate stores. Fetching details (checkpointing to ${RAW_FILE})...`);

  const rawStream = fs.createWriteStream(RAW_FILE, { flags: 'w' });
  const CONC = 3; // kept low so this doesn't fight the live Maps scraper for CPU/RAM
  let validCount = 0, i = 0;

  async function worker() {
    while (i < links.length) {
      const link = links[i++];
      let r;
      try {
        r = await fetchStoreDetails(browser, link);
      } catch (e) {
        // Browser itself likely died (Protocol error / disconnected) — relaunch and retry this link once.
        if (!browser.isConnected()) {


Prompt:
<task-notification>
<task-id>b2h9ry2ag</task-id>
<tool-use-id>toolu_01CVkEXSzHnYCfruPvw9ryh4</tool-use-id>
<output-file>/tmp/claude-0/-root-ultron/e44a47d2-ef9c-4c03-9995-d56770ba2ff7/tasks/b2h9ry2ag.output</output-file>
<status>completed</status>
<summary>Background command "until ! ps -p 131027 > /dev/null 2>&1; do sleep 15; done; echo mahally_scrape_done" completed (exit code 0)</summary>
</task-notification>


Edit

const OUT_FILE = process.argv[2] || path.join(__dirname, '..', 'data', 'mahally_leads.csv');
const TARGET = parseInt(process.argv[3] || '100', 10);
const RAW_FILE = OUT_FILE.replace(/\.csv$/, '') + '.raw.jsonl';

const OUT_FILE = process.argv[2] || path.join(__dirname, '..', 'data', 'mahally_leads.csv');
const TARGET = parseInt(process.argv[3] || '100', 10);
const MAX_FETCH = parseInt(process.argv[4] || '0', 10) || Infinity; // cap on store detail pages fetched
const RAW_FILE = OUT_FILE.replace(/\.csv$/, '') + '.raw.jsonl';


const OUT_FILE = process.argv[2] || path.join(__dirname, '..', 'data', 'mahally_leads.csv');
const TARGET = parseInt(process.argv[3] || '100', 10);
const MAX_FETCH = parseInt(process.argv[4] || '0', 10) || Infinity; // cap on store detail pages fetched
const RAW_FILE = OUT_FILE.replace(/\.csv$/, '') + '.raw.jsonl';

'use strict';
// MAHALLY — Salla's directory of Saudi online stores (mahally.com). Discovery via product search
// (/ar/browse?query=<category>), which surfaces merchant store links. Each store page exposes
// name, city, rating, review count, product count, and (often) the CR number + maroof.sa link
// pulled out of the free-text description. Bot-protected (Cloudflare-style human check on a bare
// fetch), so this goes through the same proxy + stealth browser as Maps — images/media/fonts are
// already blocked in browser.js's newPage() to save proxy GB.
// Usage: node src/scrape-mahally.js [outFile] [targetCount]
const fs = require('fs'), path = require('path');
(function () {
  const p = path.join(__dirname, '..', '.env');
  if (!fs.existsSync(p)) return;
  for (const l of fs.readFileSync(p, 'utf8').split('\n')) {
    const t = l.trim(); if (!t || t.startsWith('#')) continue;
    const i = t.indexOf('='); if (i > -1 && !(t.slice(0, i).trim() in process.env)) process.env[t.slice(0, i).trim()] = t.slice(i + 1).trim();
  }
})();
const { launchBrowser, newPage } = require('./browser');

const OUT_FILE = process.argv[2] || path.join(__dirname, '..', 'data', 'mahally_leads.csv');
const TARGET = parseInt(process.argv[3] || '100', 10);
const RAW_FILE = OUT_FILE.replace(/\.csv$/, '') + '.raw.jsonl';

// Real category names pulled from Mahally's own nav (found live on the browse page)
const CATEGORIES = [
  'القهوة', 'مأكولات', 'أزياء', 'جمال وعناية', 'إلكترونيات', 'المنزل',
  'قرطاسية', 'نظارات', 'كتب', 'بأيدي محلية', 'الحيوانات', 'ألعاب',
];

async function discoverStoreLinks(browser) {
  const found = new Set();
  for (const cat of CATEGORIES) {
    const page = await newPage(browser);
    try {
      const url = `https://mahally.com/ar/browse?query=${encodeURIComponent(cat)}`;
      await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
      await new Promise((r) => setTimeout(r, 3000));
      const links = await page.evaluate(() =>
        [...new Set(Array.from(document.querySelectorAll('a[href*="/stores/"]')).map((a) => a.href))]
      );
      for (const l of links) found.add(l.split('?')[0].replace(/\/$/, ''));
      process.stdout.write(`  [${cat}] +${links.length} (total unique: ${found.size})\n`);
    } catch (e) {
      process.stdout.write(`  [${cat}] failed: ${e.message}\n`);
    } finally {
      await page.close().catch(() => {});
    }
  }
  return [...found];
}

function parseStorePage(text) {
  if (/المتجر غير موجود/.test(text)) return null;
  const lines = text.split('\n').map((l) => l.trim()).filter(Boolean);
  const name = lines[0] || null;
  let rating = null, reviews = null, city = null, idx = 1;
  if (lines[1] === name) idx = 2;
  if (/^[\d.]+$/.test(lines[idx] || '')) { rating = parseFloat(lines[idx]); idx++; }
  const reviewMatch = (lines[idx] || '').match(/^\((\d+)\)$/);
  if (reviewMatch) { reviews = parseInt(reviewMatch[1], 10); idx++; }
  city = lines[idx] || null;
  const description = lines.slice(idx + 1, idx + 4).join(' ');
  const crMatch = text.match(/رقم\s*(\d{6,})/);
  const productMatch = text.match(/(\d+)\s*منتج في المتجر/);
  const maroofMatch = text.match(/https?:\/\/maroof\.sa\/\d+/);
  return {
    name, city, rating, reviews,
    product_count: productMatch ? parseInt(productMatch[1], 10) : null,
    cr_number: crMatch ? crMatch[1] : null,
    maroof_url: maroofMatch ? maroofMatch[0] : null,
    description: description.slice(0, 200),
  };
}

async function fetchStoreDetails(browser, storeUrl) {
  const page = await newPage(browser);
  try {
    await page.goto(storeUrl + '/', { waitUntil: 'domcontentloaded', timeout: 45000 });
    await new Promise((r) => setTimeout(r, 2000));
    const text = await page.evaluate(() => document.body.innerText);
    const parsed = parseStorePage(text);
    if (!parsed) return null;
    return { url: storeUrl, ...parsed };
  } catch (e) {
    return null;
  } finally {
    await page.close().catch(() => {});
  }
}

function toCsvRow(vals) {
  return vals.map((v) => {
    const s = (v === null || v === undefined) ? '' : String(v);
    return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
  }).join(',');
}

(async () => {
  let browser = await launchBrowser();
  console.log(`Discovering stores across ${CATEGORIES.length} categories...`);
  const links = await discoverStoreLinks(browser);
  console.log(`Discovered ${links.length} unique candidate stores. Fetching details (checkpointing to ${RAW_FILE})...`);

  const rawStream = fs.createWriteStream(RAW_FILE, { flags: 'w' });
  const CONC = 3; // kept low so this doesn't fight the live Maps scraper for CPU/RAM
  let validCount = 0, i = 0;

  async function worker() {
    while (i < links.length) {
      const link = links[i++];
      let r;
      try {
        r = await fetchStoreDetails(browser, link);
      } catch (e) {
        // Browser itself likely died (Protocol error / disconnected) — relaunch and retry this link once.
        if (!browser.isConnected()) {
          console.log(`  browser disconnected, relaunching...`);
          try { await browser.close(); } catch (_) {}
          browser = await launchBrowser();
        }
        r = await fetchStoreDetails(browser, link).catch(() => null);
      }
      if (r) {
        validCount++;
        rawStream.write(JSON.stringify(r) + '\n');
      }
      if (i % 20 === 0) process.stdout.write(`  fetched ${i}/${links.length}, ${validCount} valid\n`);
    }
  }
  await Promise.all(Array.from({ length: CONC }, worker));
  await new Promise((resolve) => rawStream.end(resolve));
  await browser.close().catch(() => {});

  // Build final CSV from the checkpointed raw file (survives any crash after this point too)
  const results = fs.readFileSync(RAW_FILE, 'utf8').trim().split('\n').filter(Boolean).map((l) => JSON.parse(l));
  const valid = results.filter((r) => r.reviews !== null && r.reviews > 0);
  valid.sort((a, b) => a.reviews - b.reviews);
  console.log(`\n${valid.length} stores with review data. Review count range: ${valid[0]?.reviews} - ${valid[valid.length - 1]?.reviews}`);

  // "Mid-level" = the middle band by review count (not brand-new/zero, not mega-established)
  const loCut = Math.floor(valid.length * 0.3);
  const hiCut = Math.floor(valid.length * 0.8);
  let midBand = valid.slice(loCut, hiCut);
  if (midBand.length > TARGET) midBand = midBand.slice(0, TARGET);

  const header = ['store_name', 'city', 'rating', 'review_count', 'product_count', 'cr_number', 'maroof_url', 'description', 'mahally_url'];
  const rows = [toCsvRow(header)];
  for (const r of midBand) {
    rows.push(toCsvRow([r.name, r.city, r.rating, r.reviews, r.product_count, r.cr_number, r.maroof_url, r.description, r.url]));
  }
  fs.mkdirSync(path.dirname(OUT_FILE), { recursive: true });
  fs.writeFileSync(OUT_FILE, rows.join('\n'));
  console.log(`\nWrote ${midBand.length} mid-level stores to ${OUT_FILE}`);
})();

+const MAX_FETCH = parseInt(process.argv[4] || '0', 10) || Infinity; // cap on store detail pages fetched


  const links = await discoverStoreLinks(browser);
  console.log(`Discovered ${links.length} unique candidate stores. Fetching details (checkpointing to ${RAW_FILE})...`);

  const rawStream = fs.createWriteStream(RAW_FILE, { flags: 'w' });
  const CONC = 3; // kept low so this doesn't fight the live Maps scraper for CPU/RAM

  let links = await discoverStoreLinks(browser);
  if (links.length > MAX_FETCH) links = links.slice(0, MAX_FETCH);
  console.log(`Discovered ${links.length} candidate stores (capped at ${MAX_FETCH}). Fetching details (checkpointing to ${RAW_FILE})...`);

  const rawStream = fs.createWriteStream(RAW_FILE, { flags: 'w' });
  const CONC = 2; // kept low so this doesn't fight the live Maps scraper for CPU/RAM


  let links = await discoverStoreLinks(browser);
  if (links.length > MAX_FETCH) links = links.slice(0, MAX_FETCH);
  console.log(`Discovered ${links.length} candidate stores (capped at ${MAX_FETCH}). Fetching details (checkpointing to ${RAW_FILE})...`);

  const rawStream = fs.createWriteStream(RAW_FILE, { flags: 'w' });
  const CONC = 2; // kept low so this doesn't fight the live Maps scraper for CPU/RAM

'use strict';
// MAHALLY — Salla's directory of Saudi online stores (mahally.com). Discovery via product search
// (/ar/browse?query=<category>), which surfaces merchant store links. Each store page exposes
// name, city, rating, review count, product count, and (often) the CR number + maroof.sa link
// pulled out of the free-text description. Bot-protected (Cloudflare-style human check on a bare
// fetch), so this goes through the same proxy + stealth browser as Maps — images/media/fonts are
// already blocked in browser.js's newPage() to save proxy GB.
// Usage: node src/scrape-mahally.js [outFile] [targetCount]
const fs = require('fs'), path = require('path');
(function () {
  const p = path.join(__dirname, '..', '.env');
  if (!fs.existsSync(p)) return;
  for (const l of fs.readFileSync(p, 'utf8').split('\n')) {
    const t = l.trim(); if (!t || t.startsWith('#')) continue;
    const i = t.indexOf('='); if (i > -1 && !(t.slice(0, i).trim() in process.env)) process.env[t.slice(0, i).trim()] = t.slice(i + 1).trim();
  }
})();
const { launchBrowser, newPage } = require('./browser');

const OUT_FILE = process.argv[2] || path.join(__dirname, '..', 'data', 'mahally_leads.csv');
const TARGET = parseInt(process.argv[3] || '100', 10);
const MAX_FETCH = parseInt(process.argv[4] || '0', 10) || Infinity; // cap on store detail pages fetched
const RAW_FILE = OUT_FILE.replace(/\.csv$/, '') + '.raw.jsonl';

// Real category names pulled from Mahally's own nav (found live on the browse page)
const CATEGORIES = [
  'القهوة', 'مأكولات', 'أزياء', 'جمال وعناية', 'إلكترونيات', 'المنزل',
  'قرطاسية', 'نظارات', 'كتب', 'بأيدي محلية', 'الحيوانات', 'ألعاب',
];

async function discoverStoreLinks(browser) {
  const found = new Set();
  for (const cat of CATEGORIES) {
    const page = await newPage(browser);
    try {
      const url = `https://mahally.com/ar/browse?query=${encodeURIComponent(cat)}`;
      await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
      await new Promise((r) => setTimeout(r, 3000));
      const links = await page.evaluate(() =>
        [...new Set(Array.from(document.querySelectorAll('a[href*="/stores/"]')).map((a) => a.href))]
      );
      for (const l of links) found.add(l.split('?')[0].replace(/\/$/, ''));
      process.stdout.write(`  [${cat}] +${links.length} (total unique: ${found.size})\n`);
    } catch (e) {
      process.stdout.write(`  [${cat}] failed: ${e.message}\n`);
    } finally {
      await page.close().catch(() => {});
    }
  }
  return [...found];
}

function parseStorePage(text) {
  if (/المتجر غير موجود/.test(text)) return null;
  const lines = text.split('\n').map((l) => l.trim()).filter(Boolean);
  const name = lines[0] || null;
  let rating = null, reviews = null, city = null, idx = 1;
  if (lines[1] === name) idx = 2;
  if (/^[\d.]+$/.test(lines[idx] || '')) { rating = parseFloat(lines[idx]); idx++; }
  const reviewMatch = (lines[idx] || '').match(/^\((\d+)\)$/);
  if (reviewMatch) { reviews = parseInt(reviewMatch[1], 10); idx++; }
  city = lines[idx] || null;
  const description = lines.slice(idx + 1, idx + 4).join(' ');
  const crMatch = text.match(/رقم\s*(\d{6,})/);
  const productMatch = text.match(/(\d+)\s*منتج في المتجر/);
  const maroofMatch = text.match(/https?:\/\/maroof\.sa\/\d+/);
  return {
    name, city, rating, reviews,
    product_count: productMatch ? parseInt(productMatch[1], 10) : null,
    cr_number: crMatch ? crMatch[1] : null,
    maroof_url: maroofMatch ? maroofMatch[0] : null,
    description: description.slice(0, 200),
  };
}

async function fetchStoreDetails(browser, storeUrl) {
  const page = await newPage(browser);
  try {
    await page.goto(storeUrl + '/', { waitUntil: 'domcontentloaded', timeout: 45000 });
    await new Promise((r) => setTimeout(r, 2000));
    const text = await page.evaluate(() => document.body.innerText);
    const parsed = parseStorePage(text);
    if (!parsed) return null;
    return { url: storeUrl, ...parsed };
  } catch (e) {
    return null;
  } finally {
    await page.close().catch(() => {});
  }
}

function toCsvRow(vals) {
  return vals.map((v) => {
    const s = (v === null || v === undefined) ? '' : String(v);
    return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
  }).join(',');
}

(async () => {
  let browser = await launchBrowser();
  console.log(`Discovering stores across ${CATEGORIES.length} categories...`);
  const links = await discoverStoreLinks(browser);
  console.log(`Discovered ${links.length} unique candidate stores. Fetching details (checkpointing to ${RAW_FILE})...`);

  const rawStream = fs.createWriteStream(RAW_FILE, { flags: 'w' });
  const CONC = 3; // kept low so this doesn't fight the live Maps scraper for CPU/RAM
  let validCount = 0, i = 0;

  async function worker() {
    while (i < links.length) {
      const link = links[i++];
      let r;
      try {
        r = await fetchStoreDetails(browser, link);
      } catch (e) {
        // Browser itself likely died (Protocol error / disconnected) — relaunch and retry this link once.
        if (!browser.isConnected()) {
          console.log(`  browser disconnected, relaunching...`);
          try { await browser.close(); } catch (_) {}
          browser = await launchBrowser();
        }
        r = await fetchStoreDetails(browser, link).catch(() => null);
      }
      if (r) {
        validCount++;
        rawStream.write(JSON.stringify(r) + '\n');
      }
      if (i % 20 === 0) process.stdout.write(`  fetched ${i}/${links.length}, ${validCount} valid\n`);
    }
  }
  await Promise.all(Array.from({ length: CONC }, worker));
  await new Promise((resolve) => rawStream.end(resolve));
  await browser.close().catch(() => {});

  // Build final CSV from the checkpointed raw file (survives any crash after this point too)
  const results = fs.readFileSync(RAW_FILE, 'utf8').trim().split('\n').filter(Boolean).map((l) => JSON.parse(l));
  const valid = results.filter((r) => r.reviews !== null && r.reviews > 0);
  valid.sort((a, b) => a.reviews - b.reviews);
  console.log(`\n${valid.length} stores with review data. Review count range: ${valid[0]?.reviews} - ${valid[valid.length - 1]?.reviews}`);

  // "Mid-level" = the middle band by review count (not brand-new/zero, not mega-established)
  const loCut = Math.floor(valid.length * 0.3);
  const hiCut = Math.floor(valid.length * 0.8);
  let midBand = valid.slice(loCut, hiCut);
  if (midBand.length > TARGET) midBand = midBand.slice(0, TARGET);

  const header = ['store_name', 'city', 'rating', 'review_count', 'product_count', 'cr_number', 'maroof_url', 'description', 'mahally_url'];
  const rows = [toCsvRow(header)];
  for (const r of midBand) {
    rows.push(toCsvRow([r.name, r.city, r.rating, r.reviews, r.product_count, r.cr_number, r.maroof_url, r.description, r.url]));
  }
  fs.mkdirSync(path.dirname(OUT_FILE), { recursive: true });
  fs.writeFileSync(OUT_FILE, rows.join('\n'));
  console.log(`\nWrote ${midBand.length} mid-level stores to ${OUT_FILE}`);
})();

-  const links = await discoverStoreLinks(browser);

-  console.log(`Discovered ${links.length} unique candidate stores. Fetching details (checkpointing to ${RAW_FILE})...`);

+  let links = await discoverStoreLinks(browser);

+  if (links.length > MAX_FETCH) links = links.slice(0, MAX_FETCH);

+  console.log(`Discovered ${links.length} candidate stores (capped at ${MAX_FETCH}). Fetching details (checkpointing to ${RAW_FILE})...`);

-  const CONC = 3; // kept low so this doesn't fight the live Maps scraper for CPU/RAM

+  const CONC = 2; // kept low so this doesn't fight the live Maps scraper for CPU/RAM


Now rerunning with target=50, capped at 150 store fetches for a faster test.


Bash

rm -f /root/ultron/data/mahally_leads.csv /root/ultron/data/mahally_leads.raw.jsonl
nohup node src/scrape-mahally.js data/mahally_leads.csv 50 150 > data/mahally_run.log 2>&1 &
echo "started pid $!"


started pid 131388


until ! ps -p 131388 > /dev/null 2>&1; do sleep 10; done; echo mahally_scrape_v2_done


Command running in background with ID: b9b8d68ys. Output is being written to: /tmp/claude-0/-root-ultron/e44a47d2-ef9c-4c03-9995-d56770ba2ff7/tasks/b9b8d68ys.output. You will be notified when it completes. To check interim output, use Read on that file path.


Got it — restarted with concurrency 2 (down from 3) and capped at 150 stores fetched, targeting 50 mid-level ones for the sheet. Should finish faster than the last attempt. Will send the sheet once it's done.


sent (id: 137)

sent (id: 137)


Restarted with concurrency 2 and capped at 150 stores (targeting 50 mid-level for the sheet), as requested. I'll wait for it to finish and send the CSV.


<task-notification>
<task-id>b9b8d68ys</task-id>
<tool-use-id>toolu_01Ew4jgfWVjdSr6AxtXLmKkn</tool-use-id>
<output-file>/tmp/claude-0/-root-ultron/e44a47d2-ef9c-4c03-9995-d56770ba2ff7/tasks/b9b8d68ys.output</output-file>
<status>completed</status>
<summary>Background command "until ! ps -p 131388 > /dev/null 2>&1; do sleep 10; done; echo mahally_scrape_v2_done" completed (exit code 0)</summary>
</task-notification>


tail -30 /root/ultron/data/mahally_run.log
echo "---"
ls -la /root/ultron/data/mahally_leads.csv 2>&1
wc -l /root/ultron/data/mahally_leads.csv 2>&1


Discovering stores across 12 categories...
  [القهوة] +0 (total unique: 0)
  [مأكولات] +0 (total unique: 0)
  [أزياء] +0 (total unique: 0)
  [جمال وعناية] +42 (total unique: 42)
  [إلكترونيات] +0 (total unique: 42)
  [المنزل] +47 (total unique: 87)
  [قرطاسية] +7 (total unique: 94)
  [نظارات] +0 (total unique: 94)
  [كتب] +0 (total unique: 94)
  [بأيدي محلية] +0 (total unique: 94)
  [الحيوانات] +0 (total unique: 94)
  [ألعاب] +39 (total unique: 132)
Discovered 132 candidate stores (capped at 150). Fetching details (checkpointing to data/mahally_leads.raw.jsonl)...
  fetched 20/132, 19 valid
  fetched 40/132, 39 valid
  fetched 60/132, 59 valid
  fetched 80/132, 79 valid
  fetched 100/132, 99 valid
  fetched 120/132, 119 valid

0 stores with review data. Review count range: undefined - undefined

Wrote 0 mid-level stores to data/mahally_leads.csv
---
-rw-r--r-- 1 root root 94 Jul  3 21:40 /root/ultron/data/mahally_leads.csv
0 /root/ultron/data/mahally_leads.csv


head -5 /root/ultron/data/mahally_leads.raw.jsonl
echo "---count---"
wc -l /root/ultron/data/mahally_leads.raw.jsonl


{"url":"https://mahally.com/stores/1831524028","name":"Error 1101 Ray ID: a1590282ececad9d • 2026-07-03 21:33:39 UTC","city":"Worker threw exception","rating":null,"reviews":null,"product_count":null,"cr_number":null,"maroof_url":null,"description":"What happened? You've requested a page on a website (mahally.com) that is on the Cloudflare network. An unknown error occurred while rendering the page. What can I do?"}
{"url":"https://mahally.com/stores/1700005346","name":"فعّل الإشعارات لاستلام آخر العروض والأخبار من محلي.","city":"تفعيل الإشعارات","rating":null,"reviews":null,"product_count":40660,"cr_number":null,"maroof_url":null,"description":"لاحقاً تسجيل الدخول الرئيسية"}
{"url":"https://mahally.com/stores/488210590","name":"فعّل الإشعارات لاستلام آخر العروض والأخبار من محلي.","city":"تفعيل الإشعارات","rating":null,"reviews":null,"product_count":121,"cr_number":null,"maroof_url":null,"description":"لاحقاً تسجيل الدخول الرئيسية"}
{"url":"https://mahally.com/stores/1021723745","name":"Error 1101 Ray ID: a15902b14fa0ad9d • 2026-07-03 21:33:47 UTC","city":"Worker threw exception","rating":null,"reviews":null,"product_count":null,"cr_number":null,"maroof_url":null,"description":"What happened? You've requested a page on a website (mahally.com) that is on the Cloudflare network. An unknown error occurred while rendering the page. What can I do?"}
{"url":"https://mahally.com/stores/1515115134","name":"Error 1101 Ray ID: a15902be7c25ad9d • 2026-07-03 21:33:49 UTC","city":"Worker threw exception","rating":null,"reviews":null,"product_count":null,"cr_number":null,"maroof_url":null,"description":"What happened? You've requested a page on a website (mahally.com) that is on the Cloudflare network. An unknown error occurred while rendering the page. What can I do?"}
---count---
132 /root/ultron/data/mahally_leads.raw.jsonl


52
53	function parseStorePage(text) {
54	  if (/المتجر غير موجود/.test(text)) return null;
55	  const lines = text.split('\n').map((l) => l.trim()).filter(Boolean);
56	  const name = lines[0] || null;
57	  let rating = null, reviews = null, city = null, idx = 1;
58	  if (lines[1] === name) idx = 2;
59	  if (/^[\d.]+$/.test(lines[idx] || '')) { rating = parseFloat(lines[idx]); idx++; }
60	  const reviewMatch = (lines[idx] || '').match(/^\((\d+)\)$/);
61	  if (reviewMatch) { reviews = parseInt(reviewMatch[1], 10); idx++; }
62	  city = lines[idx] || null;
63	  const description = lines.slice(idx + 1, idx + 4).join(' ');
64	  const crMatch = text.match(/رقم\s*(\d{6,})/);
65	  const productMatch = text.match(/(\d+)\s*منتج في المتجر/);
66	  const maroofMatch = text.match(/https?:\/\/maroof\.sa\/\d+/);
67	  return {
68	    name, city, rating, reviews,
69	    product_count: productMatch ? parseInt(productMatch[1], 10) : null,
70	    cr_number: crMatch ? crMatch[1] : null,
71	    maroof_url: maroofMatch ? maroofMatch[0] : null,
72	    description: description.slice(0, 200),
73	  };
74	}
75
76	async function fetchStoreDetails(browser, storeUrl) {
77	  const page = await newPage(browser);
78	  try {
79	    await page.goto(storeUrl + '/', { waitUntil: 'domcontentloaded', timeout: 45000 });
80	    await new Promise((r) => setTimeout(r, 2000));
81	    const text = await page.evaluate(() => document.body.innerText);

function parseStorePage(text) {
  if (/المتجر غير موجود/.test(text)) return null;
  const lines = text.split('\n').map((l) => l.trim()).filter(Boolean);
  const name = lines[0] || null;
  let rating = null, reviews = null, city = null, idx = 1;
  if (lines[1] === name) idx = 2;
  if (/^[\d.]+$/.test(lines[idx] || '')) { rating = parseFloat(lines[idx]); idx++; }
  const reviewMatch = (lines[idx] || '').match(/^\((\d+)\)$/);
  if (reviewMatch) { reviews = parseInt(reviewMatch[1], 10); idx++; }
  city = lines[idx] || null;
  const description = lines.slice(idx + 1, idx + 4).join(' ');
  const crMatch = text.match(/رقم\s*(\d{6,})/);
  const productMatch = text.match(/(\d+)\s*منتج في المتجر/);
  const maroofMatch = text.match(/https?:\/\/maroof\.sa\/\d+/);
  return {
    name, city, rating, reviews,
    product_count: productMatch ? parseInt(productMatch[1], 10) : null,
    cr_number: crMatch ? crMatch[1] : null,
    maroof_url: maroofMatch ? maroofMatch[0] : null,
    description: description.slice(0, 200),
  };
}

async function fetchStoreDetails(browser, storeUrl) {
  const page = await newPage(browser);
  try {
    await page.goto(storeUrl + '/', { waitUntil: 'domcontentloaded', timeout: 45000 });
    await new Promise((r) => setTimeout(r, 2000));
    const text = await page.evaluate(() => document.body.innerText);


Edit

function parseStorePage(text) {
  if (/المتجر غير موجود/.test(text)) return null;
  const lines = text.split('\n').map((l) => l.trim()).filter(Boolean);
  const name = lines[0] || null;
  let rating = null, reviews = null, city = null, idx = 1;
  if (lines[1] === name) idx = 2;
  if (/^[\d.]+$/.test(lines[idx] || '')) { rating = parseFloat(lines[idx]); idx++; }
  const reviewMatch = (lines[idx] || '').match(/^\((\d+)\)$/);
  if (reviewMatch) { reviews = parseInt(reviewMatch[1], 10); idx++; }
  city = lines[idx] || null;
  const description = lines.slice(idx + 1, idx + 4).join(' ');

// Boilerplate lines that show up on some loads (notification-permission banner) and shift
// position-based parsing if not stripped first.
const BOILERPLATE = new Set([
  'فعّل الإشعارات لاستلام آخر العروض والأخبار من محلي.',
  'تفعيل الإشعارات', 'لاحقاً', 'تسجيل الدخول', 'الرئيسية',
]);

function parseStorePage(text) {
  if (/المتجر غير موجود/.test(text)) return null;
  if (/Error 1101|Ray ID|Cloudflare/i.test(text)) return { cloudflareError: true };
  const lines = text.split('\n').map((l) => l.trim()).filter((l) => l && !BOILERPLATE.has(l));
  const name = lines[0] || null;
  let rating = null, reviews = null, city = null, idx = 1;
  if (lines[1] === name) idx = 2;
  if (/^[\d.]+$/.test(lines[idx] || '')) { rating = parseFloat(lines[idx]); idx++; }
  const reviewMatch = (lines[idx] || '').match(/^\((\d+)\)$/);
  if (reviewMatch) { reviews = parseInt(reviewMatch[1], 10); idx++; }
  city = lines[idx] || null;
  const description = lines.slice(idx + 1, idx + 4).join(' ');


// Boilerplate lines that show up on some loads (notification-permission banner) and shift
// position-based parsing if not stripped first.
const BOILERPLATE = new Set([
  'فعّل الإشعارات لاستلام آخر العروض والأخبار من محلي.',
  'تفعيل الإشعارات', 'لاحقاً', 'تسجيل الدخول', 'الرئيسية',
]);

function parseStorePage(text) {
  if (/المتجر غير موجود/.test(text)) return null;
  if (/Error 1101|Ray ID|Cloudflare/i.test(text)) return { cloudflareError: true };
  const lines = text.split('\n').map((l) => l.trim()).filter((l) => l && !BOILERPLATE.has(l));
  const name = lines[0] || null;
  let rating = null, reviews = null, city = null, idx = 1;
  if (lines[1] === name) idx = 2;
  if (/^[\d.]+$/.test(lines[idx] || '')) { rating = parseFloat(lines[idx]); idx++; }
  const reviewMatch = (lines[idx] || '').match(/^\((\d+)\)$/);
  if (reviewMatch) { reviews = parseInt(reviewMatch[1], 10); idx++; }
  city = lines[idx] || null;
  const description = lines.slice(idx + 1, idx + 4).join(' ');

'use strict';
// MAHALLY — Salla's directory of Saudi online stores (mahally.com). Discovery via product search
// (/ar/browse?query=<category>), which surfaces merchant store links. Each store page exposes
// name, city, rating, review count, product count, and (often) the CR number + maroof.sa link
// pulled out of the free-text description. Bot-protected (Cloudflare-style human check on a bare
// fetch), so this goes through the same proxy + stealth browser as Maps — images/media/fonts are
// already blocked in browser.js's newPage() to save proxy GB.
// Usage: node src/scrape-mahally.js [outFile] [targetCount]
const fs = require('fs'), path = require('path');
(function () {
  const p = path.join(__dirname, '..', '.env');
  if (!fs.existsSync(p)) return;
  for (const l of fs.readFileSync(p, 'utf8').split('\n')) {
    const t = l.trim(); if (!t || t.startsWith('#')) continue;
    const i = t.indexOf('='); if (i > -1 && !(t.slice(0, i).trim() in process.env)) process.env[t.slice(0, i).trim()] = t.slice(i + 1).trim();
  }
})();
const { launchBrowser, newPage } = require('./browser');

const OUT_FILE = process.argv[2] || path.join(__dirname, '..', 'data', 'mahally_leads.csv');
const TARGET = parseInt(process.argv[3] || '100', 10);
const MAX_FETCH = parseInt(process.argv[4] || '0', 10) || Infinity; // cap on store detail pages fetched
const RAW_FILE = OUT_FILE.replace(/\.csv$/, '') + '.raw.jsonl';

// Real category names pulled from Mahally's own nav (found live on the browse page)
const CATEGORIES = [
  'القهوة', 'مأكولات', 'أزياء', 'جمال وعناية', 'إلكترونيات', 'المنزل',
  'قرطاسية', 'نظارات', 'كتب', 'بأيدي محلية', 'الحيوانات', 'ألعاب',
];

async function discoverStoreLinks(browser) {
  const found = new Set();
  for (const cat of CATEGORIES) {
    const page = await newPage(browser);
    try {
      const url = `https://mahally.com/ar/browse?query=${encodeURIComponent(cat)}`;
      await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
      await new Promise((r) => setTimeout(r, 3000));
      const links = await page.evaluate(() =>
        [...new Set(Array.from(document.querySelectorAll('a[href*="/stores/"]')).map((a) => a.href))]
      );
      for (const l of links) found.add(l.split('?')[0].replace(/\/$/, ''));
      process.stdout.write(`  [${cat}] +${links.length} (total unique: ${found.size})\n`);
    } catch (e) {
      process.stdout.write(`  [${cat}] failed: ${e.message}\n`);
    } finally {
      await page.close().catch(() => {});
    }
  }
  return [...found];
}

function parseStorePage(text) {
  if (/المتجر غير موجود/.test(text)) return null;
  const lines = text.split('\n').map((l) => l.trim()).filter(Boolean);
  const name = lines[0] || null;
  let rating = null, reviews = null, city = null, idx = 1;
  if (lines[1] === name) idx = 2;
  if (/^[\d.]+$/.test(lines[idx] || '')) { rating = parseFloat(lines[idx]); idx++; }
  const reviewMatch = (lines[idx] || '').match(/^\((\d+)\)$/);
  if (reviewMatch) { reviews = parseInt(reviewMatch[1], 10); idx++; }
  city = lines[idx] || null;
  const description = lines.slice(idx + 1, idx + 4).join(' ');
  const crMatch = text.match(/رقم\s*(\d{6,})/);
  const productMatch = text.match(/(\d+)\s*منتج في المتجر/);
  const maroofMatch = text.match(/https?:\/\/maroof\.sa\/\d+/);
  return {
    name, city, rating, reviews,
    product_count: productMatch ? parseInt(productMatch[1], 10) : null,
    cr_number: crMatch ? crMatch[1] : null,
    maroof_url: maroofMatch ? maroofMatch[0] : null,
    description: description.slice(0, 200),
  };
}

async function fetchStoreDetails(browser, storeUrl) {
  const page = await newPage(browser);
  try {
    await page.goto(storeUrl + '/', { waitUntil: 'domcontentloaded', timeout: 45000 });
    await new Promise((r) => setTimeout(r, 2000));
    const text = await page.evaluate(() => document.body.innerText);
    const parsed = parseStorePage(text);
    if (!parsed) return null;
    return { url: storeUrl, ...parsed };
  } catch (e) {
    return null;
  } finally {
    await page.close().catch(() => {});
  }
}

function toCsvRow(vals) {
  return vals.map((v) => {
    const s = (v === null || v === undefined) ? '' : String(v);
    return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
  }).join(',');
}

(async () => {
  let browser = await launchBrowser();
  console.log(`Discovering stores across ${CATEGORIES.length} categories...`);
  let links = await discoverStoreLinks(browser);
  if (links.length > MAX_FETCH) links = links.slice(0, MAX_FETCH);
  console.log(`Discovered ${links.length} candidate stores (capped at ${MAX_FETCH}). Fetching details (checkpointing to ${RAW_FILE})...`);

  const rawStream = fs.createWriteStream(RAW_FILE, { flags: 'w' });
  const CONC = 2; // kept low so this doesn't fight the live Maps scraper for CPU/RAM
  let validCount = 0, i = 0;

  async function worker() {
    while (i < links.length) {
      const link = links[i++];
      let r;
      try {
        r = await fetchStoreDetails(browser, link);
      } catch (e) {
        // Browser itself likely died (Protocol error / disconnected) — relaunch and retry this link once.
        if (!browser.isConnected()) {
          console.log(`  browser disconnected, relaunching...`);
          try { await browser.close(); } catch (_) {}
          browser = await launchBrowser();
        }
        r = await fetchStoreDetails(browser, link).catch(() => null);
      }
      if (r) {
        validCount++;
        rawStream.write(JSON.stringify(r) + '\n');
      }
      if (i % 20 === 0) process.stdout.write(`  fetched ${i}/${links.length}, ${validCount} valid\n`);
    }
  }
  await Promise.all(Array.from({ length: CONC }, worker));
  await new Promise((resolve) => rawStream.end(resolve));
  await browser.close().catch(() => {});

  // Build final CSV from the checkpointed raw file (survives any crash after this point too)
  const results = fs.readFileSync(RAW_FILE, 'utf8').trim().split('\n').filter(Boolean).map((l) => JSON.parse(l));
  const valid = results.filter((r) => r.reviews !== null && r.reviews > 0);
  valid.sort((a, b) => a.reviews - b.reviews);
  console.log(`\n${valid.length} stores with review data. Review count range: ${valid[0]?.reviews} - ${valid[valid.length - 1]?.reviews}`);

  // "Mid-level" = the middle band by review count (not brand-new/zero, not mega-established)
  const loCut = Math.floor(valid.length * 0.3);
  const hiCut = Math.floor(valid.length * 0.8);
  let midBand = valid.slice(loCut, hiCut);
  if (midBand.length > TARGET) midBand = midBand.slice(0, TARGET);

  const header = ['store_name', 'city', 'rating', 'review_count', 'product_count', 'cr_number', 'maroof_url', 'description', 'mahally_url'];
  const rows = [toCsvRow(header)];
  for (const r of midBand) {
    rows.push(toCsvRow([r.name, r.city, r.rating, r.reviews, r.product_count, r.cr_number, r.maroof_url, r.description, r.url]));
  }
  fs.mkdirSync(path.dirname(OUT_FILE), { recursive: true });
  fs.writeFileSync(OUT_FILE, rows.join('\n'));
  console.log(`\nWrote ${midBand.length} mid-level stores to ${OUT_FILE}`);
})();

+// Boilerplate lines that show up on some loads (notification-permission banner) and shift

+// position-based parsing if not stripped first.

+  'فعّل الإشعارات لاستلام آخر العروض والأخبار من محلي.',

+  'تفعيل الإشعارات', 'لاحقاً', 'تسجيل الدخول', 'الرئيسية',

-  const lines = text.split('\n').map((l) => l.trim()).filter(Boolean);

+  if (/Error 1101|Ray ID|Cloudflare/i.test(text)) return { cloudflareError: true };

+  const lines = text.split('\n').map((l) => l.trim()).filter((l) => l && !BOILERPLATE.has(l));


Now let me update `fetchStoreDetails` to retry once on a Cloudflare error page.


82	}
83
84	async function fetchStoreDetails(browser, storeUrl) {
85	  const page = await newPage(browser);
86	  try {
87	    await page.goto(storeUrl + '/', { waitUntil: 'domcontentloaded', timeout: 45000 });
88	    await new Promise((r) => setTimeout(r, 2000));
89	    const text = await page.evaluate(() => document.body.innerText);
90	    const parsed = parseStorePage(text);
91	    if (!parsed) return null;
92	    return { url: storeUrl, ...parsed };
93	  } catch (e) {
94	    return null;
95	  } finally {
96	    await page.close().catch(() => {});

}

async function fetchStoreDetails(browser, storeUrl) {
  const page = await newPage(browser);
  try {
    await page.goto(storeUrl + '/', { waitUntil: 'domcontentloaded', timeout: 45000 });
    await new Promise((r) => setTimeout(r, 2000));
    const text = await page.evaluate(() => document.body.innerText);
    const parsed = parseStorePage(text);
    if (!parsed) return null;
    return { url: storeUrl, ...parsed };
  } catch (e) {
    return null;
  } finally {
    await page.close().catch(() => {});


async function fetchStoreDetails(browser, storeUrl) {
  const page = await newPage(browser);
  try {
    await page.goto(storeUrl + '/', { waitUntil: 'domcontentloaded', timeout: 45000 });
    await new Promise((r) => setTimeout(r, 2000));
    const text = await page.evaluate(() => document.body.innerText);
    const parsed = parseStorePage(text);
    if (!parsed) return null;
    return { url: storeUrl, ...parsed };
  } catch (e) {
    return null;
  } finally {
    await page.close().catch(() => {});

async function fetchStoreDetails(browser, storeUrl, retryOnCloudflare = true) {
  const page = await newPage(browser);
  try {
    await page.goto(storeUrl + '/', { waitUntil: 'domcontentloaded', timeout: 45000 });
    await new Promise((r) => setTimeout(r, 2000));
    const text = await page.evaluate(() => document.body.innerText);
    const parsed = parseStorePage(text);
    if (!parsed) return null;
    if (parsed.cloudflareError) {
      await page.close().catch(() => {});
      if (!retryOnCloudflare) return null;
      await new Promise((r) => setTimeout(r, 4000)); // let a fresh proxy connection/edge node take over
      return fetchStoreDetails(browser, storeUrl, false);
    }
    return { url: storeUrl, ...parsed };
  } catch (e) {
    return null;
  } finally {
    await page.close().catch(() => {});


The file /root/ultron/src/scrape-mahally.js has been updated successfully. (file state is current in your context — no need to Read it back)

async function fetchStoreDetails(browser, storeUrl, retryOnCloudflare = true) {
  const page = await newPage(browser);
  try {
    await page.goto(storeUrl + '/', { waitUntil: 'domcontentloaded', timeout: 45000 });
    await new Promise((r) => setTimeout(r, 2000));
    const text = await page.evaluate(() => document.body.innerText);
    const parsed = parseStorePage(text);
    if (!parsed) return null;
    if (parsed.cloudflareError) {
      await page.close().catch(() => {});
      if (!retryOnCloudflare) return null;
      await new Promise((r) => setTimeout(r, 4000)); // let a fresh proxy connection/edge node take over
      return fetchStoreDetails(browser, storeUrl, false);
    }
    return { url: storeUrl, ...parsed };
  } catch (e) {
    return null;
  } finally {
    await page.close().catch(() => {});

'use strict';
// MAHALLY — Salla's directory of Saudi online stores (mahally.com). Discovery via product search
// (/ar/browse?query=<category>), which surfaces merchant store links. Each store page exposes
// name, city, rating, review count, product count, and (often) the CR number + maroof.sa link
// pulled out of the free-text description. Bot-protected (Cloudflare-style human check on a bare
// fetch), so this goes through the same proxy + stealth browser as Maps — images/media/fonts are
// already blocked in browser.js's newPage() to save proxy GB.
// Usage: node src/scrape-mahally.js [outFile] [targetCount]
const fs = require('fs'), path = require('path');
(function () {
  const p = path.join(__dirname, '..', '.env');
  if (!fs.existsSync(p)) return;
  for (const l of fs.readFileSync(p, 'utf8').split('\n')) {
    const t = l.trim(); if (!t || t.startsWith('#')) continue;
    const i = t.indexOf('='); if (i > -1 && !(t.slice(0, i).trim() in process.env)) process.env[t.slice(0, i).trim()] = t.slice(i + 1).trim();
  }
})();
const { launchBrowser, newPage } = require('./browser');

const OUT_FILE = process.argv[2] || path.join(__dirname, '..', 'data', 'mahally_leads.csv');
const TARGET = parseInt(process.argv[3] || '100', 10);
const MAX_FETCH = parseInt(process.argv[4] || '0', 10) || Infinity; // cap on store detail pages fetched
const RAW_FILE = OUT_FILE.replace(/\.csv$/, '') + '.raw.jsonl';

// Real category names pulled from Mahally's own nav (found live on the browse page)
const CATEGORIES = [
  'القهوة', 'مأكولات', 'أزياء', 'جمال وعناية', 'إلكترونيات', 'المنزل',
  'قرطاسية', 'نظارات', 'كتب', 'بأيدي محلية', 'الحيوانات', 'ألعاب',
];

async function discoverStoreLinks(browser) {
  const found = new Set();
  for (const cat of CATEGORIES) {
    const page = await newPage(browser);
    try {
      const url = `https://mahally.com/ar/browse?query=${encodeURIComponent(cat)}`;
      await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
      await new Promise((r) => setTimeout(r, 3000));
      const links = await page.evaluate(() =>
        [...new Set(Array.from(document.querySelectorAll('a[href*="/stores/"]')).map((a) => a.href))]
      );
      for (const l of links) found.add(l.split('?')[0].replace(/\/$/, ''));
      process.stdout.write(`  [${cat}] +${links.length} (total unique: ${found.size})\n`);
    } catch (e) {
      process.stdout.write(`  [${cat}] failed: ${e.message}\n`);
    } finally {
      await page.close().catch(() => {});
    }
  }
  return [...found];
}

// Boilerplate lines that show up on some loads (notification-permission banner) and shift
// position-based parsing if not stripped first.
const BOILERPLATE = new Set([
  'فعّل الإشعارات لاستلام آخر العروض والأخبار من محلي.',
  'تفعيل الإشعارات', 'لاحقاً', 'تسجيل الدخول', 'الرئيسية',
]);

function parseStorePage(text) {
  if (/المتجر غير موجود/.test(text)) return null;
  if (/Error 1101|Ray ID|Cloudflare/i.test(text)) return { cloudflareError: true };
  const lines = text.split('\n').map((l) => l.trim()).filter((l) => l && !BOILERPLATE.has(l));
  const name = lines[0] || null;
  let rating = null, reviews = null, city = null, idx = 1;
  if (lines[1] === name) idx = 2;
  if (/^[\d.]+$/.test(lines[idx] || '')) { rating = parseFloat(lines[idx]); idx++; }
  const reviewMatch = (lines[idx] || '').match(/^\((\d+)\)$/);
  if (reviewMatch) { reviews = parseInt(reviewMatch[1], 10); idx++; }
  city = lines[idx] || null;
  const description = lines.slice(idx + 1, idx + 4).join(' ');
  const crMatch = text.match(/رقم\s*(\d{6,})/);
  const productMatch = text.match(/(\d+)\s*منتج في المتجر/);
  const maroofMatch = text.match(/https?:\/\/maroof\.sa\/\d+/);
  return {
    name, city, rating, reviews,
    product_count: productMatch ? parseInt(productMatch[1], 10) : null,
    cr_number: crMatch ? crMatch[1] : null,
    maroof_url: maroofMatch ? maroofMatch[0] : null,
    description: description.slice(0, 200),
  };
}

async function fetchStoreDetails(browser, storeUrl) {
  const page = await newPage(browser);
  try {
    await page.goto(storeUrl + '/', { waitUntil: 'domcontentloaded', timeout: 45000 });
    await new Promise((r) => setTimeout(r, 2000));
    const text = await page.evaluate(() => document.body.innerText);
    const parsed = parseStorePage(text);
    if (!parsed) return null;
    return { url: storeUrl, ...parsed };
  } catch (e) {
    return null;
  } finally {
    await page.close().catch(() => {});
  }
}

function toCsvRow(vals) {
  return vals.map((v) => {
    const s = (v === null || v === undefined) ? '' : String(v);
    return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
  }).join(',');
}

(async () => {
  let browser = await launchBrowser();
  console.log(`Discovering stores across ${CATEGORIES.length} categories...`);
  let links = await discoverStoreLinks(browser);
  if (links.length > MAX_FETCH) links = links.slice(0, MAX_FETCH);
  console.log(`Discovered ${links.length} candidate stores (capped at ${MAX_FETCH}). Fetching details (checkpointing to ${RAW_FILE})...`);

  const rawStream = fs.createWriteStream(RAW_FILE, { flags: 'w' });
  const CONC = 2; // kept low so this doesn't fight the live Maps scraper for CPU/RAM
  let validCount = 0, i = 0;

  async function worker() {
    while (i < links.length) {
      const link = links[i++];
      let r;
      try {
        r = await fetchStoreDetails(browser, link);
      } catch (e) {
        // Browser itself likely died (Protocol error / disconnected) — relaunch and retry this link once.
        if (!browser.isConnected()) {
          console.log(`  browser disconnected, relaunching...`);
          try { await browser.close(); } catch (_) {}
          browser = await launchBrowser();
        }
        r = await fetchStoreDetails(browser, link).catch(() => null);
      }
      if (r) {
        validCount++;
        rawStream.write(JSON.stringify(r) + '\n');
      }
      if (i % 20 === 0) process.stdout.write(`  fetched ${i}/${links.length}, ${validCount} valid\n`);
    }
  }
  await Promise.all(Array.from({ length: CONC }, worker));
  await new Promise((resolve) => rawStream.end(resolve));
  await browser.close().catch(() => {});

  // Build final CSV from the checkpointed raw file (survives any crash after this point too)
  const results = fs.readFileSync(RAW_FILE, 'utf8').trim().split('\n').filter(Boolean).map((l) => JSON.parse(l));
  const valid = results.filter((r) => r.reviews !== null && r.reviews > 0);
  valid.sort((a, b) => a.reviews - b.reviews);
  console.log(`\n${valid.length} stores with review data. Review count range: ${valid[0]?.reviews} - ${valid[valid.length - 1]?.reviews}`);

  // "Mid-level" = the middle band by review count (not brand-new/zero, not mega-established)
  const loCut = Math.floor(valid.length * 0.3);
  const hiCut = Math.floor(valid.length * 0.8);
  let midBand = valid.slice(loCut, hiCut);
  if (midBand.length > TARGET) midBand = midBand.slice(0, TARGET);

  const header = ['store_name', 'city', 'rating', 'review_count', 'product_count', 'cr_number', 'maroof_url', 'description', 'mahally_url'];
  const rows = [toCsvRow(header)];
  for (const r of midBand) {
    rows.push(toCsvRow([r.name, r.city, r.rating, r.reviews, r.product_count, r.cr_number, r.maroof_url, r.description, r.url]));
  }
  fs.mkdirSync(path.dirname(OUT_FILE), { recursive: true });
  fs.writeFileSync(OUT_FILE, rows.join('\n'));
  console.log(`\nWrote ${midBand.length} mid-level stores to ${OUT_FILE}`);
})();

-async function fetchStoreDetails(browser, storeUrl) {

+async function fetchStoreDetails(browser, storeUrl, retryOnCloudflare = true) {

+      await page.close().catch(() => {});

+      if (!retryOnCloudflare) return null;

+      await new Promise((r) => setTimeout(r, 4000)); // let a fresh proxy connection/edge node take over

+      return fetchStoreDetails(browser, storeUrl, false);


84	async function fetchStoreDetails(browser, storeUrl, retryOnCloudflare = true) {
85	  const page = await newPage(browser);
86	  try {
87	    await page.goto(storeUrl + '/', { waitUntil: 'domcontentloaded', timeout: 45000 });
88	    await new Promise((r) => setTimeout(r, 2000));
89	    const text = await page.evaluate(() => document.body.innerText);
90	    const parsed = parseStorePage(text);
91	    if (!parsed) return null;
92	    if (parsed.cloudflareError) {
93	      await page.close().catch(() => {});
94	      if (!retryOnCloudflare) return null;
95	      await new Promise((r) => setTimeout(r, 4000)); // let a fresh proxy connection/edge node take over
96	      return fetchStoreDetails(browser, storeUrl, false);
97	    }
98	    return { url: storeUrl, ...parsed };
99	  } catch (e) {
100	    return null;
101	  } finally {
102	    await page.close().catch(() => {});
103	  }

async function fetchStoreDetails(browser, storeUrl, retryOnCloudflare = true) {
  const page = await newPage(browser);
  try {
    await page.goto(storeUrl + '/', { waitUntil: 'domcontentloaded', timeout: 45000 });
    await new Promise((r) => setTimeout(r, 2000));
    const text = await page.evaluate(() => document.body.innerText);
    const parsed = parseStorePage(text);
    if (!parsed) return null;
    if (parsed.cloudflareError) {
      await page.close().catch(() => {});
      if (!retryOnCloudflare) return null;
      await new Promise((r) => setTimeout(r, 4000)); // let a fresh proxy connection/edge node take over
      return fetchStoreDetails(browser, storeUrl, false);
    }
    return { url: storeUrl, ...parsed };
  } catch (e) {
    return null;
  } finally {
    await page.close().catch(() => {});
  }


That redundant close is harmless (both are no-op safe), but let me clean it up anyway.


    if (parsed.cloudflareError) {
      await page.close().catch(() => {});
      if (!retryOnCloudflare) return null;
      await new Promise((r) => setTimeout(r, 4000)); // let a fresh proxy connection/edge node take over
      return fetchStoreDetails(browser, storeUrl, false);
    }

    if (parsed.cloudflareError) {
      if (!retryOnCloudflare) return null;
      await new Promise((r) => setTimeout(r, 4000)); // let a fresh proxy connection/edge node take over
      return fetchStoreDetails(browser, storeUrl, false);
    }


    if (parsed.cloudflareError) {
      if (!retryOnCloudflare) return null;
      await new Promise((r) => setTimeout(r, 4000)); // let a fresh proxy connection/edge node take over
      return fetchStoreDetails(browser, storeUrl, false);
    }

'use strict';
// MAHALLY — Salla's directory of Saudi online stores (mahally.com). Discovery via product search
// (/ar/browse?query=<category>), which surfaces merchant store links. Each store page exposes
// name, city, rating, review count, product count, and (often) the CR number + maroof.sa link
// pulled out of the free-text description. Bot-protected (Cloudflare-style human check on a bare
// fetch), so this goes through the same proxy + stealth browser as Maps — images/media/fonts are
// already blocked in browser.js's newPage() to save proxy GB.
// Usage: node src/scrape-mahally.js [outFile] [targetCount]
const fs = require('fs'), path = require('path');
(function () {
  const p = path.join(__dirname, '..', '.env');
  if (!fs.existsSync(p)) return;
  for (const l of fs.readFileSync(p, 'utf8').split('\n')) {
    const t = l.trim(); if (!t || t.startsWith('#')) continue;
    const i = t.indexOf('='); if (i > -1 && !(t.slice(0, i).trim() in process.env)) process.env[t.slice(0, i).trim()] = t.slice(i + 1).trim();
  }
})();
const { launchBrowser, newPage } = require('./browser');

const OUT_FILE = process.argv[2] || path.join(__dirname, '..', 'data', 'mahally_leads.csv');
const TARGET = parseInt(process.argv[3] || '100', 10);
const MAX_FETCH = parseInt(process.argv[4] || '0', 10) || Infinity; // cap on store detail pages fetched
const RAW_FILE = OUT_FILE.replace(/\.csv$/, '') + '.raw.jsonl';

// Real category names pulled from Mahally's own nav (found live on the browse page)
const CATEGORIES = [
  'القهوة', 'مأكولات', 'أزياء', 'جمال وعناية', 'إلكترونيات', 'المنزل',
  'قرطاسية', 'نظارات', 'كتب', 'بأيدي محلية', 'الحيوانات', 'ألعاب',
];

async function discoverStoreLinks(browser) {
  const found = new Set();
  for (const cat of CATEGORIES) {
    const page = await newPage(browser);
    try {
      const url = `https://mahally.com/ar/browse?query=${encodeURIComponent(cat)}`;
      await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
      await new Promise((r) => setTimeout(r, 3000));
      const links = await page.evaluate(() =>
        [...new Set(Array.from(document.querySelectorAll('a[href*="/stores/"]')).map((a) => a.href))]
      );
      for (const l of links) found.add(l.split('?')[0].replace(/\/$/, ''));
      process.stdout.write(`  [${cat}] +${links.length} (total unique: ${found.size})\n`);
    } catch (e) {
      process.stdout.write(`  [${cat}] failed: ${e.message}\n`);
    } finally {
      await page.close().catch(() => {});
    }
  }
  return [...found];
}

// Boilerplate lines that show up on some loads (notification-permission banner) and shift
// position-based parsing if not stripped first.
const BOILERPLATE = new Set([
  'فعّل الإشعارات لاستلام آخر العروض والأخبار من محلي.',
  'تفعيل الإشعارات', 'لاحقاً', 'تسجيل الدخول', 'الرئيسية',
]);

function parseStorePage(text) {
  if (/المتجر غير موجود/.test(text)) return null;
  if (/Error 1101|Ray ID|Cloudflare/i.test(text)) return { cloudflareError: true };
  const lines = text.split('\n').map((l) => l.trim()).filter((l) => l && !BOILERPLATE.has(l));
  const name = lines[0] || null;
  let rating = null, reviews = null, city = null, idx = 1;
  if (lines[1] === name) idx = 2;
  if (/^[\d.]+$/.test(lines[idx] || '')) { rating = parseFloat(lines[idx]); idx++; }
  const reviewMatch = (lines[idx] || '').match(/^\((\d+)\)$/);
  if (reviewMatch) { reviews = parseInt(reviewMatch[1], 10); idx++; }
  city = lines[idx] || null;
  const description = lines.slice(idx + 1, idx + 4).join(' ');
  const crMatch = text.match(/رقم\s*(\d{6,})/);
  const productMatch = text.match(/(\d+)\s*منتج في المتجر/);
  const maroofMatch = text.match(/https?:\/\/maroof\.sa\/\d+/);
  return {
    name, city, rating, reviews,
    product_count: productMatch ? parseInt(productMatch[1], 10) : null,
    cr_number: crMatch ? crMatch[1] : null,
    maroof_url: maroofMatch ? maroofMatch[0] : null,
    description: description.slice(0, 200),
  };
}

async function fetchStoreDetails(browser, storeUrl, retryOnCloudflare = true) {
  const page = await newPage(browser);
  try {
    await page.goto(storeUrl + '/', { waitUntil: 'domcontentloaded', timeout: 45000 });
    await new Promise((r) => setTimeout(r, 2000));
    const text = await page.evaluate(() => document.body.innerText);
    const parsed = parseStorePage(text);
    if (!parsed) return null;
    if (parsed.cloudflareError) {
      await page.close().catch(() => {});
      if (!retryOnCloudflare) return null;
      await new Promise((r) => setTimeout(r, 4000)); // let a fresh proxy connection/edge node take over
      return fetchStoreDetails(browser, storeUrl, false);
    }
    return { url: storeUrl, ...parsed };
  } catch (e) {
    return null;
  } finally {
    await page.close().catch(() => {});
  }
}

function toCsvRow(vals) {
  return vals.map((v) => {
    const s = (v === null || v === undefined) ? '' : String(v);
    return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
  }).join(',');
}

(async () => {
  let browser = await launchBrowser();
  console.log(`Discovering stores across ${CATEGORIES.length} categories...`);
  let links = await discoverStoreLinks(browser);
  if (links.length > MAX_FETCH) links = links.slice(0, MAX_FETCH);
  console.log(`Discovered ${links.length} candidate stores (capped at ${MAX_FETCH}). Fetching details (checkpointing to ${RAW_FILE})...`);

  const rawStream = fs.createWriteStream(RAW_FILE, { flags: 'w' });
  const CONC = 2; // kept low so this doesn't fight the live Maps scraper for CPU/RAM
  let validCount = 0, i = 0;

  async function worker() {
    while (i < links.length) {
      const link = links[i++];
      let r;
      try {
        r = await fetchStoreDetails(browser, link);
      } catch (e) {
        // Browser itself likely died (Protocol error / disconnected) — relaunch and retry this link once.
        if (!browser.isConnected()) {
          console.log(`  browser disconnected, relaunching...`);
          try { await browser.close(); } catch (_) {}
          browser = await launchBrowser();
        }
        r = await fetchStoreDetails(browser, link).catch(() => null);
      }
      if (r) {
        validCount++;
        rawStream.write(JSON.stringify(r) + '\n');
      }
      if (i % 20 === 0) process.stdout.write(`  fetched ${i}/${links.length}, ${validCount} valid\n`);
    }
  }
  await Promise.all(Array.from({ length: CONC }, worker));
  await new Promise((resolve) => rawStream.end(resolve));
  await browser.close().catch(() => {});

  // Build final CSV from the checkpointed raw file (survives any crash after this point too)
  const results = fs.readFileSync(RAW_FILE, 'utf8').trim().split('\n').filter(Boolean).map((l) => JSON.parse(l));
  const valid = results.filter((r) => r.reviews !== null && r.reviews > 0);
  valid.sort((a, b) => a.reviews - b.reviews);
  console.log(`\n${valid.length} stores with review data. Review count range: ${valid[0]?.reviews} - ${valid[valid.length - 1]?.reviews}`);

  // "Mid-level" = the middle band by review count (not brand-new/zero, not mega-established)
  const loCut = Math.floor(valid.length * 0.3);
  const hiCut = Math.floor(valid.length * 0.8);
  let midBand = valid.slice(loCut, hiCut);
  if (midBand.length > TARGET) midBand = midBand.slice(0, TARGET);

  const header = ['store_name', 'city', 'rating', 'review_count', 'product_count', 'cr_number', 'maroof_url', 'description', 'mahally_url'];
  const rows = [toCsvRow(header)];
  for (const r of midBand) {
    rows.push(toCsvRow([r.name, r.city, r.rating, r.reviews, r.product_count, r.cr_number, r.maroof_url, r.description, r.url]));
  }
  fs.mkdirSync(path.dirname(OUT_FILE), { recursive: true });
  fs.writeFileSync(OUT_FILE, rows.join('\n'));
  console.log(`\nWrote ${midBand.length} mid-level stores to ${OUT_FILE}`);
})();

-      await page.close().catch(() => {});


29	];
30
31	async function discoverStoreLinks(browser) {
32	  const found = new Set();
33	  for (const cat of CATEGORIES) {
34	    const page = await newPage(browser);
35	    try {
36	      const url = `https://mahally.com/ar/browse?query=${encodeURIComponent(cat)}`;
37	      await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
38	      await new Promise((r) => setTimeout(r, 3000));
39	      const links = await page.evaluate(() =>
40	        [...new Set(Array.from(document.querySelectorAll('a[href*="/stores/"]')).map((a) => a.href))]
41	      );
42	      for (const l of links) found.add(l.split('?')[0].replace(/\/$/, ''));
43	      process.stdout.write(`  [${cat}] +${links.length} (total unique: ${found.size})\n`);
44	    } catch (e) {
45	      process.stdout.write(`  [${cat}] failed: ${e.message}\n`);
46	    } finally {
47	      await page.close().catch(() => {});
48	    }
49	  }

];

async function discoverStoreLinks(browser) {
  const found = new Set();
  for (const cat of CATEGORIES) {
    const page = await newPage(browser);
    try {
      const url = `https://mahally.com/ar/browse?query=${encodeURIComponent(cat)}`;
      await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
      await new Promise((r) => setTimeout(r, 3000));
      const links = await page.evaluate(() =>
        [...new Set(Array.from(document.querySelectorAll('a[href*="/stores/"]')).map((a) => a.href))]
      );
      for (const l of links) found.add(l.split('?')[0].replace(/\/$/, ''));
      process.stdout.write(`  [${cat}] +${links.length} (total unique: ${found.size})\n`);
    } catch (e) {
      process.stdout.write(`  [${cat}] failed: ${e.message}\n`);
    } finally {
      await page.close().catch(() => {});
    }
  }


async function discoverStoreLinks(browser) {
  const found = new Set();
  for (const cat of CATEGORIES) {
    const page = await newPage(browser);
    try {
      const url = `https://mahally.com/ar/browse?query=${encodeURIComponent(cat)}`;
      await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
      await new Promise((r) => setTimeout(r, 3000));
      const links = await page.evaluate(() =>
        [...new Set(Array.from(document.querySelectorAll('a[href*="/stores/"]')).map((a) => a.href))]
      );
      for (const l of links) found.add(l.split('?')[0].replace(/\/$/, ''));
      process.stdout.write(`  [${cat}] +${links.length} (total unique: ${found.size})\n`);
    } catch (e) {
      process.stdout.write(`  [${cat}] failed: ${e.message}\n`);
    } finally {
      await page.close().catch(() => {});
    }
  }

async function discoverOne(browser, cat) {
  const page = await newPage(browser);
  try {
    const url = `https://mahally.com/ar/browse?query=${encodeURIComponent(cat)}`;
    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
    await new Promise((r) => setTimeout(r, 3000));
    const { links, cloudflareError } = await page.evaluate(() => ({
      links: [...new Set(Array.from(document.querySelectorAll('a[href*="/stores/"]')).map((a) => a.href))],
      cloudflareError: /Error 1101|Ray ID/i.test(document.body.innerText),
    }));
    if (cloudflareError) return { links: [], cloudflareError: true };
    return { links, cloudflareError: false };
  } finally {
    await page.close().catch(() => {});
  }
}

async function discoverStoreLinks(browser) {
  const found = new Set();
  for (const cat of CATEGORIES) {
    try {
      let { links, cloudflareError } = await discoverOne(browser, cat);
      if (cloudflareError || links.length === 0) {
        // Bot-check flake or empty page render — one retry after a short pause is usually enough.
        await new Promise((r) => setTimeout(r, 4000));
        ({ links } = await discoverOne(browser, cat));
      }
      for (const l of links) found.add(l.split('?')[0].replace(/\/$/, ''));
      process.stdout.write(`  [${cat}] +${links.length} (total unique: ${found.size})\n`);
    } catch (e) {
      process.stdout.write(`  [${cat}] failed: ${e.message}\n`);
    }
  }