COMERCIAL-GTI/maquina-comercial-pessoal/src/sources/overpass.mjs

import { cleanText } from '../lead-engine.mjs';
import { sleep } from '../util/collections.mjs';
import { UA_HEADERS, buildLeadFromPlace } from './common.mjs';
import { fetchTextWithTimeout } from '../util/fetch.mjs';

const OVERPASS_COOLDOWN_MS = 60000;

function overpassSelector(tag, bbox) {
  const [south, west, north, east] = bbox;
  const key = String(tag.key).replace(/"/g, '\\"');
  const value = String(tag.value).replace(/"/g, '\\"');
  const selector = `["${key}"="${value}"](${south},${west},${north},${east});`;
  return [`node${selector}`, `way${selector}`, `relation${selector}`].join('\n');
}

export function escapeOverpassRegex(value) {
  return String(value).replace(/[\\^$.*+?()[\]{}|]/g, '\\$&').replace(/"/g, '\\"');
}

function overpassTextSelector(key, regex, bbox) {
  const [south, west, north, east] = bbox;
  const selector = `["${key}"~"${regex}",i](${south},${west},${north},${east});`;
  return [`node${selector}`, `way${selector}`].join('\n');
}

function segmentTextRegex(segment) {
  return (segment.queries || [])
    .filter((term) => String(term).trim().length >= 6)
    .slice(0, 8)
    .map(escapeOverpassRegex)
    .join('|');
}

function buildCombinedQuery(segments, bbox, includeText) {
  const seen = new Set();
  const selectors = [];

  for (const segment of segments) {
    for (const tag of segment.overpassTags || []) {
      const key = `tag:${tag.key}=${tag.value}`;
      if (seen.has(key)) continue;
      seen.add(key);
      selectors.push(overpassSelector(tag, bbox));
    }
  }

  if (includeText) {
    for (const segment of segments) {
      const textRegex = segmentTextRegex(segment);
      const key = `text:${segment.name}`;
      if (!textRegex || seen.has(key)) continue;
      seen.add(key);
      selectors.push(overpassTextSelector('name', textRegex, bbox));
      selectors.push(overpassTextSelector('brand', textRegex, bbox));
      selectors.push(overpassTextSelector('operator', textRegex, bbox));
    }
  }

  return ['[out:json][timeout:40];', '(', selectors.join('\n'), ');', 'out body center;'].join('\n');
}

function matchSegmentForElement(tags, segments, includeText) {
  for (const segment of segments) {
    for (const tag of segment.overpassTags || []) {
      if (tags[tag.key] === tag.value) return segment;
    }
  }

  if (includeText) {
    const haystack = cleanText([tags.name, tags.brand, tags.operator].filter(Boolean).join(' '));
    if (!haystack) return null;

    for (const segment of segments) {
      const terms = (segment.queries || []).filter((term) => String(term).trim().length >= 6);
      if (terms.some((term) => haystack.includes(cleanText(term)))) return segment;
    }
  }

  return null;
}

function valueFromTags(tags = {}, keys = []) {
  for (const key of keys) {
    if (tags[key]) return tags[key];
  }
  return '';
}

function elementToPlace(element, segment, city, uf, query) {
  const tags = element.tags || {};
  const name = tags.name || tags.brand || tags.operator;
  if (!name) return null;

  const street = valueFromTags(tags, ['addr:street']);
  const housenumber = valueFromTags(tags, ['addr:housenumber']);
  const address = [street, housenumber].filter(Boolean).join(', ');

  return {
    name,
    phone: valueFromTags(tags, ['contact:phone', 'phone', 'mobile', 'contact:mobile']),
    city: valueFromTags(tags, ['addr:city']) || city,
    state: uf,
    segment: segment.name,
    website: valueFromTags(tags, ['contact:website', 'website', 'url']),
    address,
    formatted_address: address,
    query,
    osm_id: String(element.id || ''),
    osm_type: element.type || ''
  };
}

export function createOverpassClient({ urls, timeoutMs }) {
  const failures = new Map();

  return async function request(query) {
    let lastError;

    for (const apiUrl of urls) {
      const state = failures.get(apiUrl);
      if (state?.until && Date.now() < state.until) continue;

      const recordFailure = () => {
        const current = failures.get(apiUrl) || { count: 0, until: 0 };
        const count = current.count + 1;
        failures.set(apiUrl, { count, until: count >= 3 ? Date.now() + OVERPASS_COOLDOWN_MS : 0 });
      };

      const result = await fetchTextWithTimeout(apiUrl, {
          method: 'POST',
          headers: {
            ...UA_HEADERS,
            'content-type': 'application/x-www-form-urlencoded;charset=UTF-8'
          },
          body: new URLSearchParams({ data: query })
        }, timeoutMs, { maxBytes: 50 * 1024 * 1024 }).catch((error) => {
        lastError = /timeout/i.test(error.message)
          ? new Error(`Overpass timeout apos ${timeoutMs}ms em ${apiUrl}`)
          : error;
        recordFailure();
        return null;
      });

      if (!result) continue;
      const { response, text } = result;

      if (!response.ok) {
        lastError = new Error(`Overpass HTTP ${response.status} em ${apiUrl}: ${text.slice(0, 300)}`);
        recordFailure();
        if (response.status === 429) await sleep(5000);
        continue;
      }

      const data = JSON.parse(text);
      failures.set(apiUrl, { count: 0, until: 0 });
      return data.elements || [];
    }

    throw lastError || new Error('Overpass indisponivel: todos os endpoints em cooldown');
  };
}

// Fonte overpass (OpenStreetMap): uma consulta combinada por cidade cobrindo
// todos os nichos selecionados, com failover entre endpoints e cooldown.
export async function* collect(params, ctx) {
  const { uf, cities, segments, limits } = params;
  if (!cities || !cities.length) {
    ctx.log('overpass: nenhuma cidade selecionada — fonte pulada');
    return;
  }

  const request = createOverpassClient({
    urls: ctx.endpoints.overpassUrls,
    timeoutMs: ctx.delays.overpassTimeoutMs
  });

  for (const city of cities) {
    if (ctx.isCancelled()) return;
    const label = `${city} ${uf}`;

    let bbox = null;
    try {
      bbox = await ctx.geocode(uf, city);
    } catch (error) {
      ctx.log(`overpass: geocode falhou em ${label}: ${error.message}`);
    }
    if (!bbox) {
      ctx.log(`overpass: sem bounding box para ${label} — cidade pulada`);
      continue;
    }

    ctx.emit({ phase: 'overpass', message: `consultando ${label}` });

    let elements = [];
    try {
      elements = await request(buildCombinedQuery(segments, bbox, ctx.overpassTextSearch));
    } catch (error) {
      ctx.log(`overpass falhou em ${label}: ${error.message}`);
      await sleep(Math.max(ctx.delays.overpassDelayMs, 15000));
      continue;
    }

    const perSegmentCount = new Map();
    const cityLeads = [];

    for (const element of elements) {
      const segment = matchSegmentForElement(element.tags || {}, segments, ctx.overpassTextSearch);
      if (!segment) continue;

      if (limits.maxPerQuery > 0) {
        const count = perSegmentCount.get(segment.name) || 0;
        if (count >= limits.maxPerQuery) continue;
        perSegmentCount.set(segment.name, count + 1);
      }

      const place = elementToPlace(element, segment, city, uf, `${segment.name} em ${label}`);
      if (!place) continue;
      cityLeads.push(buildLeadFromPlace(place, segment, city, uf, 'overpass'));
    }

    ctx.log(`overpass: ${elements.length} elementos brutos, ${cityLeads.length} leads em ${label}`);
    if (cityLeads.length) yield { origin: 'overpass', label, leads: cityLeads };
    await sleep(ctx.delays.overpassDelayMs);
  }
}