Baseline commit of the existing Maquina Comercial Pessoal project

Tracks the app that was already in place (capture, treatment, outreach,
WhatsApp integration, server, frontend, tests, docs) so future feature
work can branch and use isolated worktrees.

src/sources/cnpj-base.mjs

@@ -0,0 +1,182 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { cleanText } from '../lead-engine.mjs';
+import { onlyDigits } from '../scoring.mjs';
+import { streamCsvRows } from '../util/csv.mjs';
+import { buildLeadFromPlace, UA_HEADERS } from './common.mjs';
+import { fetchJsonWithTimeout } from '../util/fetch.mjs';
+
+async function brasilApiCnpj(baseUrl, cnpj) {
+  if (!cnpj || cnpj.length !== 14) return null;
+  const { response, data } = await fetchJsonWithTimeout(`${baseUrl}/cnpj/v1/${cnpj}`, { headers: UA_HEADERS });
+  if (!response.ok) return null;
+  return data;
+}
+
+function mergeBrasilApi(place, data) {
+  if (!data) return place;
+  return {
+    ...place,
+    name: data.nome_fantasia || data.razao_social || place.name,
+    legal_name: data.razao_social || place.legal_name,
+    phone: [data.ddd_telefone_1, data.ddd_telefone_2].filter(Boolean)[0] || place.phone,
+    city: data.municipio || place.city,
+    state: data.uf || place.state,
+    cnae: data.cnae_fiscal ? String(data.cnae_fiscal) : place.cnae,
+    address: [data.logradouro, data.numero, data.bairro].filter(Boolean).join(', ') || place.address
+  };
+}
+
+function normalizeHeader(header) {
+  return cleanText(header).replace(/\s+/g, '_');
+}
+
+function getAny(row, keys) {
+  for (const key of keys) {
+    const normalized = normalizeHeader(key);
+    if (row[normalized]) return row[normalized];
+  }
+  return '';
+}
+
+function cnpjMatchesSegment(row, segment, textCache) {
+  const cnae = onlyDigits(getAny(row, ['cnae', 'cnae_fiscal', 'cnae principal', 'cnae_principal']));
+  const cnaeCodes = segment._cnaeCodes;
+
+  for (const code of cnaeCodes) {
+    if (cnae.startsWith(code)) return true;
+  }
+
+  if (textCache.value === null) {
+    textCache.value = cleanText([
+      getAny(row, ['nome fantasia', 'nome_fantasia', 'fantasia']),
+      getAny(row, ['razao social', 'razao_social', 'nome empresarial']),
+      getAny(row, ['atividade principal', 'atividade_principal', 'descricao cnae', 'descricao_cnae'])
+    ].join(' '));
+  }
+
+  const text = textCache.value;
+  if (!text) return false;
+  return segment._queryTerms.some((term) => text.includes(term));
+}
+
+function cnpjRowToPlace(row, segment, city, state) {
+  const cnpj = onlyDigits(getAny(row, ['cnpj', 'cnpj_basico', 'cnpj completo', 'cnpj_completo']));
+  const phone = [
+    getAny(row, ['ddd 1', 'ddd1', 'ddd']),
+    getAny(row, ['telefone 1', 'telefone1', 'telefone', 'whatsapp'])
+  ].filter(Boolean).join('');
+  const name = getAny(row, ['nome fantasia', 'nome_fantasia', 'fantasia'])
+    || getAny(row, ['razao social', 'razao_social', 'nome empresarial']);
+
+  if (!name) return null;
+
+  return {
+    name,
+    legal_name: getAny(row, ['razao social', 'razao_social', 'nome empresarial']),
+    phone,
+    city: getAny(row, ['municipio', 'cidade']) || city,
+    state: getAny(row, ['uf', 'estado']) || state,
+    segment: segment.name,
+    website: '',
+    cnpj,
+    cnae: getAny(row, ['cnae', 'cnae_fiscal', 'cnae principal', 'cnae_principal']),
+    address: [
+      getAny(row, ['logradouro']),
+      getAny(row, ['numero']),
+      getAny(row, ['bairro'])
+    ].filter(Boolean).join(', '),
+    query: `cnpj-base ${segment.name} em ${city} ${state}`
+  };
+}
+
+// Fonte cnpj-base: le os CSVs locais extraidos da Receita Federal
+// (prospeccao/cnpj/estabelecimentos-*.csv) em streaming, filtrando por
+// UF + cidades e casando cada linha com o primeiro nicho compativel.
+export async function* collect(params, ctx) {
+  const { uf, cities, segments } = params;
+  const allCities = !cities || !cities.length || cities.includes('*');
+  const citySet = allCities ? null : new Set(cities.map((c) => cleanText(c)));
+  const ufClean = cleanText(uf);
+
+  // Pre-processa termos por segmento uma unica vez (o loop interno roda
+  // milhoes de vezes na base de uma UF inteira).
+  const preparedSegments = segments.map((segment) => ({
+    ...segment,
+    _cnaeCodes: (segment.cnaeCodes || []).map(onlyDigits).filter(Boolean),
+    _queryTerms: (segment.queries || []).map((term) => cleanText(term)).filter(Boolean)
+  }));
+
+  const exact = path.join(ctx.cnpjDir, `estabelecimentos-${String(uf).toUpperCase()}.csv`);
+  const legacy = path.join(ctx.cnpjDir, 'estabelecimentos-bahia.csv');
+  let files = [];
+  try {
+    await fs.access(exact);
+    files = [exact];
+  } catch (error) {
+    if (error.code !== 'ENOENT') throw error;
+    if (String(uf).toUpperCase() === 'BA') {
+      try {
+        await fs.access(legacy);
+        files = [legacy];
+      } catch (legacyError) {
+        if (legacyError.code !== 'ENOENT') throw legacyError;
+      }
+    }
+  }
+  if (!files.length) {
+    ctx.log(`base CNPJ de ${uf} nao encontrada em ${ctx.cnpjDir} — fonte cnpj-base pulada`);
+    return;
+  }
+
+  let batch = [];
+  let scanned = 0;
+
+  for (const file of files) {
+    ctx.log(`cnpj-base: lendo ${path.basename(file)}`);
+
+    for await (const row of streamCsvRows(file, { normalizeHeader })) {
+      if (ctx.isCancelled()) return;
+      scanned += 1;
+
+      if (scanned % 100000 === 0) {
+        ctx.emit({ phase: 'cnpj-base', message: `${scanned} linhas lidas`, detail: path.basename(file) });
+      }
+
+      const rowState = cleanText(getAny(row, ['uf', 'estado']));
+      if (rowState && ufClean && rowState !== ufClean) continue;
+
+      const rowCity = cleanText(getAny(row, ['municipio', 'cidade']));
+      if (citySet && !citySet.has(rowCity)) continue;
+
+      const textCache = { value: null };
+      for (const segment of preparedSegments) {
+        if (!cnpjMatchesSegment(row, segment, textCache)) continue;
+
+        let place = cnpjRowToPlace(row, segment, rowCity, uf);
+        if (place) {
+          if (params.enrichBrasilApi && place.cnpj) {
+            try {
+              place = mergeBrasilApi(place, await brasilApiCnpj(ctx.endpoints.brasilApiUrl, place.cnpj));
+            } catch {
+              // enriquecimento e opcional: falha de rede nao derruba a captura
+            }
+          }
+          const origin = params.enrichBrasilApi ? 'cnpj-base+brasilapi' : 'cnpj-base';
+          batch.push(buildLeadFromPlace(place, segment, place.city, uf, origin));
+        }
+        break;
+      }
+
+      if (batch.length >= 2000) {
+        yield { origin: 'cnpj-base', label: path.basename(file), leads: batch };
+        batch = [];
+      }
+    }
+  }
+
+  if (batch.length) {
+    yield { origin: 'cnpj-base', label: 'cnpj-base', leads: batch };
+  }
+  ctx.log(`cnpj-base: ${scanned} linhas da base local processadas`);
+}

src/sources/common.mjs

@@ -0,0 +1,74 @@
+import { normalizePhone } from '../lead-engine.mjs';
+
+export const UA_HEADERS = {
+  accept: 'application/json,text/plain,*/*',
+  'user-agent': 'Maquina-Comercial-Pessoal/1.0'
+};
+
+export const UF_NAMES = {
+  AC: 'Acre', AL: 'Alagoas', AP: 'Amapa', AM: 'Amazonas', BA: 'Bahia',
+  CE: 'Ceara', DF: 'Distrito Federal', ES: 'Espirito Santo', GO: 'Goias',
+  MA: 'Maranhao', MT: 'Mato Grosso', MS: 'Mato Grosso do Sul', MG: 'Minas Gerais',
+  PA: 'Para', PB: 'Paraiba', PR: 'Parana', PE: 'Pernambuco', PI: 'Piaui',
+  RJ: 'Rio de Janeiro', RN: 'Rio Grande do Norte', RS: 'Rio Grande do Sul',
+  RO: 'Rondonia', RR: 'Roraima', SC: 'Santa Catarina', SP: 'Sao Paulo',
+  SE: 'Sergipe', TO: 'Tocantins'
+};
+
+export const UFS = Object.keys(UF_NAMES);
+
+export function splitCityState(cityText = '') {
+  const parts = String(cityText).trim().split(/\s+/);
+  const last = parts.at(-1);
+  if (last && last.length === 2 && UF_NAMES[last.toUpperCase()]) {
+    return { city: parts.slice(0, -1).join(' '), state: last.toUpperCase() };
+  }
+  return { city: String(cityText).trim(), state: '' };
+}
+
+// Monta o payload padrao de lead (mesmo formato consumido por processLead)
+// a partir de um estabelecimento/lugar encontrado em qualquer fonte.
+export function buildLeadFromPlace(place, segment, city, state, origin) {
+  const phone = normalizePhone(place.phone || place.formatted_phone_number || place.international_phone_number || '');
+  const segmentName = place.segment || segment.name;
+
+  return {
+    source: origin,
+    contact: {
+      name: 'Responsavel comercial',
+      phone
+    },
+    company: {
+      name: place.name,
+      city: place.city || city,
+      state: place.state || state,
+      segment: segmentName
+    },
+    interest: segment.interest,
+    pains: segment.pains,
+    urgency: 'media',
+    message: `Lead prospectado automaticamente em ${origin}. Segmento: ${segmentName}. Website: ${place.website || 'nao informado'}.`,
+    consent: false,
+    send_external_whatsapp: false,
+    prospecting: {
+      source: origin,
+      query: place.query,
+      place_id: place.place_id || '',
+      osm_id: place.osm_id || '',
+      osm_type: place.osm_type || '',
+      website: place.website || '',
+      cnpj: place.cnpj || '',
+      cnae: place.cnae || '',
+      legal_name: place.legal_name || '',
+      address: place.formatted_address || place.address || '',
+      rating: place.rating || '',
+      user_ratings_total: place.user_ratings_total || '',
+      affinity: segment.affinity || 'media',
+      products: segment.products || []
+    }
+  };
+}
+
+export function applyLimits(items, limit) {
+  return limit > 0 ? items.slice(0, limit) : items;
+}

src/sources/google-places.mjs

@@ -0,0 +1,102 @@
+import { sleep } from '../util/collections.mjs';
+import { buildLeadFromPlace } from './common.mjs';
+import { fetchJsonWithTimeout } from '../util/fetch.mjs';
+
+async function searchPage(apiKey, params) {
+  const url = new URL('https://maps.googleapis.com/maps/api/place/textsearch/json');
+  for (const [key, value] of Object.entries(params)) {
+    if (value) url.searchParams.set(key, value);
+  }
+  url.searchParams.set('key', apiKey);
+  url.searchParams.set('language', 'pt-BR');
+  url.searchParams.set('region', 'br');
+
+  const { response, data } = await fetchJsonWithTimeout(url);
+  if (!response.ok) throw new Error(`Google Places HTTP ${response.status}`);
+
+  if (data.status && !['OK', 'ZERO_RESULTS'].includes(data.status)) {
+    throw new Error(`Google Places ${data.status}: ${data.error_message || 'sem detalhes'}`);
+  }
+  return data;
+}
+
+async function search(query, apiKey, { maxPerQuery, pages }) {
+  const places = [];
+  let page = 0;
+  let pageToken = '';
+
+  while (page < pages && places.length < maxPerQuery) {
+    if (pageToken) await sleep(2500);
+    const data = await searchPage(apiKey, pageToken ? { pagetoken: pageToken } : { query });
+    // Resultados por pagina sao no maximo 20 — spread aqui e seguro.
+    places.push(...(data.results || []));
+    page += 1;
+    pageToken = data.next_page_token || '';
+    if (!pageToken) break;
+  }
+
+  return places.slice(0, maxPerQuery);
+}
+
+async function placeDetails(placeId, apiKey) {
+  const url = new URL('https://maps.googleapis.com/maps/api/place/details/json');
+  url.searchParams.set('place_id', placeId);
+  url.searchParams.set('key', apiKey);
+  url.searchParams.set('language', 'pt-BR');
+  url.searchParams.set('fields', 'place_id,name,formatted_address,formatted_phone_number,international_phone_number,website,rating,user_ratings_total');
+
+  const { response, data } = await fetchJsonWithTimeout(url);
+  if (!response.ok) throw new Error(`Google Details HTTP ${response.status}`);
+  return data.result || {};
+}
+
+// Fonte google-places: exige GOOGLE_PLACES_API_KEY (paga). Traz telefone e
+// website com boa cobertura.
+export async function* collect(params, ctx) {
+  const apiKey = process.env.GOOGLE_PLACES_API_KEY;
+  if (!apiKey || apiKey === 'preencher_google_maps_api_key') {
+    throw new Error('GOOGLE_PLACES_API_KEY ausente: configure a chave para usar a fonte Google Places');
+  }
+
+  const { uf, cities, segments, limits } = params;
+  const pages = Math.min(3, Math.max(1, limits.googlePages || 3));
+  let executedQueries = 0;
+
+  for (const city of cities || []) {
+    const label = `${city} ${uf}`;
+    const cityLeads = [];
+
+    for (const segment of segments) {
+      for (const queryTerm of segment.queries || []) {
+        if (ctx.isCancelled()) return;
+        if (limits.queryLimit > 0 && executedQueries >= limits.queryLimit) {
+          if (cityLeads.length) yield { origin: 'google-places', label, leads: cityLeads };
+          return;
+        }
+
+        const query = `${queryTerm} em ${label}`;
+        ctx.emit({ phase: 'google-places', message: query });
+        executedQueries += 1;
+
+        let places = [];
+        try {
+          places = await search(query, apiKey, { maxPerQuery: limits.maxPerQuery || 10, pages });
+        } catch (error) {
+          ctx.log(`google-places falhou em ${query}: ${error.message}`);
+          continue;
+        }
+
+        for (const place of places) {
+          try {
+            const details = place.place_id ? await placeDetails(place.place_id, apiKey) : {};
+            cityLeads.push(buildLeadFromPlace({ ...place, ...details, query }, segment, city, uf, 'google-places'));
+          } catch (error) {
+            ctx.log(`google-places falhou ao detalhar "${place.name || place.place_id}": ${error.message}`);
+          }
+        }
+      }
+    }
+
+    if (cityLeads.length) yield { origin: 'google-places', label, leads: cityLeads };
+  }
+}

src/sources/index.mjs

@@ -0,0 +1,89 @@
+import * as cnpjBase from './cnpj-base.mjs';
+import * as overpass from './overpass.mjs';
+import * as nominatim from './nominatim.mjs';
+import * as wikidata from './wikidata.mjs';
+import * as googlePlaces from './google-places.mjs';
+import * as mock from './mock.mjs';
+
+// Registro central de fontes de captura. `describe()` alimenta a tela de
+// captura (o front nunca conhece detalhes das fontes, so este catalogo).
+export const SOURCES = [
+  {
+    id: 'cnpj-base',
+    label: 'Base CNPJ (Receita Federal)',
+    descricao: 'Base oficial local: todas as empresas ativas da UF com telefone. Fonte principal.',
+    gratuita: true,
+    recomendada: true,
+    collect: cnpjBase.collect
+  },
+  {
+    id: 'brasilapi',
+    label: 'Base CNPJ + BrasilAPI',
+    descricao: 'Mesma base CNPJ, enriquecida online pela BrasilAPI (mais lenta).',
+    gratuita: true,
+    recomendada: false,
+    collect: (params, ctx) => cnpjBase.collect({ ...params, enrichBrasilApi: true }, ctx)
+  },
+  {
+    id: 'overpass',
+    label: 'OpenStreetMap (Overpass)',
+    descricao: 'Estabelecimentos mapeados no OpenStreetMap, com telefone/site quando cadastrados.',
+    gratuita: true,
+    recomendada: true,
+    collect: overpass.collect
+  },
+  {
+    id: 'nominatim',
+    label: 'OpenStreetMap (Nominatim)',
+    descricao: 'Busca textual no OpenStreetMap por termo de nicho.',
+    gratuita: true,
+    recomendada: false,
+    collect: nominatim.collect
+  },
+  {
+    id: 'wikidata',
+    label: 'Wikidata',
+    descricao: 'Empresas registradas no Wikidata (volume baixo, util para sites).',
+    gratuita: true,
+    recomendada: false,
+    collect: wikidata.collect
+  },
+  {
+    id: 'google-places',
+    label: 'Google Places',
+    descricao: 'Google Places (paga, exige GOOGLE_PLACES_API_KEY no .env).',
+    gratuita: false,
+    recomendada: false,
+    needsKey: 'GOOGLE_PLACES_API_KEY',
+    collect: googlePlaces.collect
+  },
+  {
+    id: 'mock',
+    label: 'Teste (mock)',
+    descricao: '3 leads ficticios para testar o fluxo sem tocar em APIs.',
+    gratuita: true,
+    recomendada: false,
+    oculta: true,
+    collect: mock.collect
+  }
+];
+
+export function getSource(id) {
+  return SOURCES.find((source) => source.id === id) || null;
+}
+
+export function describeSources() {
+  return SOURCES.filter((s) => !s.oculta).map((source) => {
+    const keyMissing = source.needsKey
+      && (!process.env[source.needsKey] || process.env[source.needsKey].startsWith('preencher'));
+    return {
+      id: source.id,
+      label: source.label,
+      descricao: source.descricao,
+      gratuita: source.gratuita,
+      recomendada: source.recomendada,
+      disponivel: !keyMissing,
+      motivo: keyMissing ? `configure ${source.needsKey} no arquivo .env` : ''
+    };
+  });
+}

src/sources/mock.mjs

@@ -0,0 +1,49 @@
+import { buildLeadFromPlace } from './common.mjs';
+
+// Fonte mock: 3 leads ficticios para testar o fluxo completo (captura ->
+// banco -> tratamento -> exportacao) sem tocar em nenhuma API externa.
+const MOCK_PLACES = [
+  {
+    name: 'Mercado Central SAJ',
+    phone: '(75) 99999-0001',
+    city: 'Santo Antonio de Jesus',
+    state: 'BA',
+    segment: 'mercados_de_bairro',
+    website: 'https://mercadocentral.example'
+  },
+  {
+    name: 'Clinica Vida Plena',
+    phone: '(75) 99999-0002',
+    city: 'Santo Antonio de Jesus',
+    state: 'BA',
+    segment: 'clinicas_medicas',
+    website: ''
+  },
+  {
+    name: 'Restaurante Sabor da Terra',
+    phone: '(75) 99999-0003',
+    city: 'Santo Antonio de Jesus',
+    state: 'BA',
+    segment: 'restaurantes',
+    website: ''
+  }
+];
+
+export async function* collect(params, ctx) {
+  const segments = params.segments || [];
+  const leads = MOCK_PLACES.map((place) => {
+    const segment = segments.find((item) => item.name === place.segment)
+      || segments[0]
+      || { name: place.segment, interest: 'ERP / sistema de gestao', pains: [] };
+    return buildLeadFromPlace(
+      { ...place, query: `mock ${place.segment}` },
+      segment,
+      place.city,
+      place.state,
+      'mock'
+    );
+  });
+
+  ctx.emit({ phase: 'mock', message: `${leads.length} leads de teste gerados` });
+  yield { origin: 'mock', label: 'mock', leads };
+}

src/sources/nominatim.mjs

@@ -0,0 +1,103 @@
+import { cleanText } from '../lead-engine.mjs';
+import { sleep } from '../util/collections.mjs';
+import { UA_HEADERS, UF_NAMES, buildLeadFromPlace } from './common.mjs';
+import { fetchTextWithTimeout } from '../util/fetch.mjs';
+
+async function nominatimSearch(baseUrl, query, limit) {
+  const url = new URL(baseUrl);
+  url.searchParams.set('format', 'jsonv2');
+  url.searchParams.set('q', query);
+  url.searchParams.set('countrycodes', 'br');
+  url.searchParams.set('addressdetails', '1');
+  url.searchParams.set('limit', String(Math.max(1, limit)));
+
+  const { response, text } = await fetchTextWithTimeout(url, { headers: UA_HEADERS });
+  if (!response.ok) {
+    throw new Error(`Nominatim HTTP ${response.status}: ${text.slice(0, 300)}`);
+  }
+  return JSON.parse(text);
+}
+
+function isResultInTarget(result, city, uf) {
+  const address = result.address || {};
+  const resultCity = address.city || address.town || address.village || address.municipality || '';
+  const resultState = cleanText(address.state || '');
+  const expectedState = cleanText(UF_NAMES[uf] || uf);
+  const currentCity = cleanText(resultCity);
+  const expectedCity = cleanText(city);
+
+  return resultState === expectedState
+    && (currentCity === expectedCity || currentCity.includes(expectedCity) || expectedCity.includes(currentCity));
+}
+
+function resultToPlace(result, segment, city, uf, query) {
+  const name = result.name || result.display_name?.split(',')[0];
+  if (!name) return null;
+
+  const address = result.address || {};
+
+  return {
+    name,
+    phone: '',
+    city: address.city || address.town || address.village || city,
+    state: uf,
+    segment: segment.name,
+    website: '',
+    address: result.display_name || '',
+    formatted_address: result.display_name || '',
+    query,
+    osm_id: String(result.osm_id || ''),
+    osm_type: result.osm_type || ''
+  };
+}
+
+// Fonte nominatim (OpenStreetMap): busca textual termo a termo por cidade,
+// validando que o resultado pertence mesmo a cidade/UF pedida.
+export async function* collect(params, ctx) {
+  const { uf, cities, segments, limits } = params;
+  if (!cities || !cities.length) {
+    ctx.log('nominatim: nenhuma cidade selecionada — fonte pulada');
+    return;
+  }
+
+  let executedQueries = 0;
+
+  for (const city of cities) {
+    const label = `${city} ${uf}`;
+    const cityLeads = [];
+
+    for (const segment of segments) {
+      for (const queryTerm of segment.queries || []) {
+        if (ctx.isCancelled()) return;
+        if (limits.queryLimit > 0 && executedQueries >= limits.queryLimit) {
+          if (cityLeads.length) yield { origin: 'nominatim', label, leads: cityLeads };
+          return;
+        }
+
+        const query = `${queryTerm} ${city} ${uf}`;
+        executedQueries += 1;
+        ctx.emit({ phase: 'nominatim', message: query });
+
+        let results = [];
+        try {
+          results = await nominatimSearch(ctx.endpoints.nominatimUrl, query, limits.maxPerQuery || 10);
+        } catch (error) {
+          ctx.log(`nominatim falhou em ${query}: ${error.message}`);
+          await sleep(Math.max(ctx.delays.overpassDelayMs, 5000));
+          continue;
+        }
+
+        for (const result of results) {
+          if (!isResultInTarget(result, city, uf)) continue;
+          const place = resultToPlace(result, segment, city, uf, query);
+          if (!place) continue;
+          cityLeads.push(buildLeadFromPlace(place, segment, city, uf, 'nominatim'));
+        }
+
+        await sleep(Math.max(ctx.delays.overpassDelayMs, 1200));
+      }
+    }
+
+    if (cityLeads.length) yield { origin: 'nominatim', label, leads: cityLeads };
+  }
+}

src/sources/overpass.mjs

@@ -0,0 +1,225 @@
+import { cleanText } from '../lead-engine.mjs';
+import { sleep } from '../util/collections.mjs';
+import { UA_HEADERS, buildLeadFromPlace } from './common.mjs';
+import { fetchTextWithTimeout } from '../util/fetch.mjs';
+
+const OVERPASS_COOLDOWN_MS = 60000;
+
+function overpassSelector(tag, bbox) {
+  const [south, west, north, east] = bbox;
+  const key = String(tag.key).replace(/"/g, '\\"');
+  const value = String(tag.value).replace(/"/g, '\\"');
+  const selector = `["${key}"="${value}"](${south},${west},${north},${east});`;
+  return [`node${selector}`, `way${selector}`, `relation${selector}`].join('\n');
+}
+
+export function escapeOverpassRegex(value) {
+  return String(value).replace(/[\\^$.*+?()[\]{}|]/g, '\\$&').replace(/"/g, '\\"');
+}
+
+function overpassTextSelector(key, regex, bbox) {
+  const [south, west, north, east] = bbox;
+  const selector = `["${key}"~"${regex}",i](${south},${west},${north},${east});`;
+  return [`node${selector}`, `way${selector}`].join('\n');
+}
+
+function segmentTextRegex(segment) {
+  return (segment.queries || [])
+    .filter((term) => String(term).trim().length >= 6)
+    .slice(0, 8)
+    .map(escapeOverpassRegex)
+    .join('|');
+}
+
+function buildCombinedQuery(segments, bbox, includeText) {
+  const seen = new Set();
+  const selectors = [];
+
+  for (const segment of segments) {
+    for (const tag of segment.overpassTags || []) {
+      const key = `tag:${tag.key}=${tag.value}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      selectors.push(overpassSelector(tag, bbox));
+    }
+  }
+
+  if (includeText) {
+    for (const segment of segments) {
+      const textRegex = segmentTextRegex(segment);
+      const key = `text:${segment.name}`;
+      if (!textRegex || seen.has(key)) continue;
+      seen.add(key);
+      selectors.push(overpassTextSelector('name', textRegex, bbox));
+      selectors.push(overpassTextSelector('brand', textRegex, bbox));
+      selectors.push(overpassTextSelector('operator', textRegex, bbox));
+    }
+  }
+
+  return ['[out:json][timeout:40];', '(', selectors.join('\n'), ');', 'out body center;'].join('\n');
+}
+
+function matchSegmentForElement(tags, segments, includeText) {
+  for (const segment of segments) {
+    for (const tag of segment.overpassTags || []) {
+      if (tags[tag.key] === tag.value) return segment;
+    }
+  }
+
+  if (includeText) {
+    const haystack = cleanText([tags.name, tags.brand, tags.operator].filter(Boolean).join(' '));
+    if (!haystack) return null;
+
+    for (const segment of segments) {
+      const terms = (segment.queries || []).filter((term) => String(term).trim().length >= 6);
+      if (terms.some((term) => haystack.includes(cleanText(term)))) return segment;
+    }
+  }
+
+  return null;
+}
+
+function valueFromTags(tags = {}, keys = []) {
+  for (const key of keys) {
+    if (tags[key]) return tags[key];
+  }
+  return '';
+}
+
+function elementToPlace(element, segment, city, uf, query) {
+  const tags = element.tags || {};
+  const name = tags.name || tags.brand || tags.operator;
+  if (!name) return null;
+
+  const street = valueFromTags(tags, ['addr:street']);
+  const housenumber = valueFromTags(tags, ['addr:housenumber']);
+  const address = [street, housenumber].filter(Boolean).join(', ');
+
+  return {
+    name,
+    phone: valueFromTags(tags, ['contact:phone', 'phone', 'mobile', 'contact:mobile']),
+    city: valueFromTags(tags, ['addr:city']) || city,
+    state: uf,
+    segment: segment.name,
+    website: valueFromTags(tags, ['contact:website', 'website', 'url']),
+    address,
+    formatted_address: address,
+    query,
+    osm_id: String(element.id || ''),
+    osm_type: element.type || ''
+  };
+}
+
+export function createOverpassClient({ urls, timeoutMs }) {
+  const failures = new Map();
+
+  return async function request(query) {
+    let lastError;
+
+    for (const apiUrl of urls) {
+      const state = failures.get(apiUrl);
+      if (state?.until && Date.now() < state.until) continue;
+
+      const recordFailure = () => {
+        const current = failures.get(apiUrl) || { count: 0, until: 0 };
+        const count = current.count + 1;
+        failures.set(apiUrl, { count, until: count >= 3 ? Date.now() + OVERPASS_COOLDOWN_MS : 0 });
+      };
+
+      const result = await fetchTextWithTimeout(apiUrl, {
+          method: 'POST',
+          headers: {
+            ...UA_HEADERS,
+            'content-type': 'application/x-www-form-urlencoded;charset=UTF-8'
+          },
+          body: new URLSearchParams({ data: query })
+        }, timeoutMs, { maxBytes: 50 * 1024 * 1024 }).catch((error) => {
+        lastError = /timeout/i.test(error.message)
+          ? new Error(`Overpass timeout apos ${timeoutMs}ms em ${apiUrl}`)
+          : error;
+        recordFailure();
+        return null;
+      });
+
+      if (!result) continue;
+      const { response, text } = result;
+
+      if (!response.ok) {
+        lastError = new Error(`Overpass HTTP ${response.status} em ${apiUrl}: ${text.slice(0, 300)}`);
+        recordFailure();
+        if (response.status === 429) await sleep(5000);
+        continue;
+      }
+
+      const data = JSON.parse(text);
+      failures.set(apiUrl, { count: 0, until: 0 });
+      return data.elements || [];
+    }
+
+    throw lastError || new Error('Overpass indisponivel: todos os endpoints em cooldown');
+  };
+}
+
+// Fonte overpass (OpenStreetMap): uma consulta combinada por cidade cobrindo
+// todos os nichos selecionados, com failover entre endpoints e cooldown.
+export async function* collect(params, ctx) {
+  const { uf, cities, segments, limits } = params;
+  if (!cities || !cities.length) {
+    ctx.log('overpass: nenhuma cidade selecionada — fonte pulada');
+    return;
+  }
+
+  const request = createOverpassClient({
+    urls: ctx.endpoints.overpassUrls,
+    timeoutMs: ctx.delays.overpassTimeoutMs
+  });
+
+  for (const city of cities) {
+    if (ctx.isCancelled()) return;
+    const label = `${city} ${uf}`;
+
+    let bbox = null;
+    try {
+      bbox = await ctx.geocode(uf, city);
+    } catch (error) {
+      ctx.log(`overpass: geocode falhou em ${label}: ${error.message}`);
+    }
+    if (!bbox) {
+      ctx.log(`overpass: sem bounding box para ${label} — cidade pulada`);
+      continue;
+    }
+
+    ctx.emit({ phase: 'overpass', message: `consultando ${label}` });
+
+    let elements = [];
+    try {
+      elements = await request(buildCombinedQuery(segments, bbox, ctx.overpassTextSearch));
+    } catch (error) {
+      ctx.log(`overpass falhou em ${label}: ${error.message}`);
+      await sleep(Math.max(ctx.delays.overpassDelayMs, 15000));
+      continue;
+    }
+
+    const perSegmentCount = new Map();
+    const cityLeads = [];
+
+    for (const element of elements) {
+      const segment = matchSegmentForElement(element.tags || {}, segments, ctx.overpassTextSearch);
+      if (!segment) continue;
+
+      if (limits.maxPerQuery > 0) {
+        const count = perSegmentCount.get(segment.name) || 0;
+        if (count >= limits.maxPerQuery) continue;
+        perSegmentCount.set(segment.name, count + 1);
+      }
+
+      const place = elementToPlace(element, segment, city, uf, `${segment.name} em ${label}`);
+      if (!place) continue;
+      cityLeads.push(buildLeadFromPlace(place, segment, city, uf, 'overpass'));
+    }
+
+    ctx.log(`overpass: ${elements.length} elementos brutos, ${cityLeads.length} leads em ${label}`);
+    if (cityLeads.length) yield { origin: 'overpass', label, leads: cityLeads };
+    await sleep(ctx.delays.overpassDelayMs);
+  }
+}

src/sources/wikidata.mjs

@@ -0,0 +1,103 @@
+import { cleanText } from '../lead-engine.mjs';
+import { UA_HEADERS, buildLeadFromPlace } from './common.mjs';
+import { escapeOverpassRegex } from './overpass.mjs';
+import { fetchJsonWithTimeout } from '../util/fetch.mjs';
+
+async function findWikidataCityId(city) {
+  // Busca por rotulo exato em SPARQL falha para nomes sem acento (base usa
+  // "Santo Antonio", Wikidata registra "Santo Antônio"). wbsearchentities
+  // e tolerante a acentos.
+  try {
+    const url = new URL('https://www.wikidata.org/w/api.php');
+    url.searchParams.set('action', 'wbsearchentities');
+    url.searchParams.set('search', city);
+    url.searchParams.set('language', 'pt');
+    url.searchParams.set('type', 'item');
+    url.searchParams.set('format', 'json');
+
+    const { response, data } = await fetchJsonWithTimeout(url, { headers: UA_HEADERS });
+    if (!response.ok) return '';
+
+    const match = (data.search || []).find((item) => {
+      const description = cleanText(item.description || '');
+      return description.includes('municipality') || description.includes('municipio') || description.includes('munic');
+    });
+
+    return match?.id || '';
+  } catch {
+    return '';
+  }
+}
+
+// Fonte wikidata: empresas registradas no Wikidata por municipio (SPARQL).
+// Volume baixo, mas gratuito e util para achar empresas com website.
+export async function* collect(params, ctx) {
+  const { uf, cities, segments, limits } = params;
+  let executedQueries = 0;
+
+  for (const city of cities || []) {
+    if (ctx.isCancelled()) return;
+
+    const cityId = await findWikidataCityId(city);
+    if (!cityId) {
+      ctx.log(`wikidata: municipio nao encontrado para ${city} — pulando`);
+      continue;
+    }
+
+    const label = `${city} ${uf}`;
+    const cityLeads = [];
+
+    for (const segment of segments) {
+      if (ctx.isCancelled()) return;
+      if (limits.queryLimit > 0 && executedQueries >= limits.queryLimit) break;
+
+      const regex = (segment.queries || [])
+        .filter((term) => term.length >= 4)
+        .slice(0, 10)
+        .map(escapeOverpassRegex)
+        .join('|');
+      if (!regex) continue;
+
+      const sparql = `
+        SELECT ?item ?itemLabel ?website WHERE {
+          ?item wdt:P31/wdt:P279* wd:Q4830453.
+          { ?item wdt:P131 wd:${cityId}. } UNION { ?item wdt:P159/wdt:P131* wd:${cityId}. }
+          OPTIONAL { ?item wdt:P856 ?website. }
+          SERVICE wikibase:label { bd:serviceParam wikibase:language "pt,en". }
+          FILTER(REGEX(?itemLabel, "${regex}", "i"))
+        }
+        LIMIT ${limits.maxPerQuery || 10}
+      `;
+
+      ctx.emit({ phase: 'wikidata', message: `${segment.name} em ${label}` });
+      executedQueries += 1;
+
+      try {
+        const url = new URL(ctx.endpoints.wikidataUrl);
+        url.searchParams.set('query', sparql);
+        url.searchParams.set('format', 'json');
+        const { response, data } = await fetchJsonWithTimeout(url, {
+          headers: { ...UA_HEADERS, accept: 'application/sparql-results+json' }
+        });
+        if (!response.ok) throw new Error(`Wikidata HTTP ${response.status}`);
+
+        for (const result of data.results?.bindings || []) {
+          const place = {
+            name: result.itemLabel?.value || '',
+            city,
+            state: uf,
+            segment: segment.name,
+            website: result.website?.value || '',
+            query: `wikidata ${segment.name} em ${label}`,
+            place_id: result.item?.value || ''
+          };
+          if (place.name) cityLeads.push(buildLeadFromPlace(place, segment, city, uf, 'wikidata'));
+        }
+      } catch (error) {
+        ctx.log(`wikidata falhou em ${segment.name} ${label}: ${error.message}`);
+      }
+    }
+
+    if (cityLeads.length) yield { origin: 'wikidata', label, leads: cityLeads };
+  }
+}