chore: import maquina-comercial-pessoal history via subtree

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> git-subtree-dir: maquina-comercial-pessoal git-subtree-mainline: f5af038e19 git-subtree-split: a9ef4d776d
2026-09-28 15:50:17 -03:00
parent f5af038e19 a9ef4d776d
commit 417da56bd6
150 changed files with 33449 additions and 0 deletions
@@ -0,0 +1,225 @@
+import { cleanText } from '../lead-engine.mjs';
+import { sleep } from '../util/collections.mjs';
+import { UA_HEADERS, buildLeadFromPlace } from './common.mjs';
+import { fetchTextWithTimeout } from '../util/fetch.mjs';
+
+const OVERPASS_COOLDOWN_MS = 60000;
+
+function overpassSelector(tag, bbox) {
+  const [south, west, north, east] = bbox;
+  const key = String(tag.key).replace(/"/g, '\\"');
+  const value = String(tag.value).replace(/"/g, '\\"');
+  const selector = `["${key}"="${value}"](${south},${west},${north},${east});`;
+  return [`node${selector}`, `way${selector}`, `relation${selector}`].join('\n');
+}
+
+export function escapeOverpassRegex(value) {
+  return String(value).replace(/[\\^$.*+?()[\]{}|]/g, '\\$&').replace(/"/g, '\\"');
+}
+
+function overpassTextSelector(key, regex, bbox) {
+  const [south, west, north, east] = bbox;
+  const selector = `["${key}"~"${regex}",i](${south},${west},${north},${east});`;
+  return [`node${selector}`, `way${selector}`].join('\n');
+}
+
+function segmentTextRegex(segment) {
+  return (segment.queries || [])
+    .filter((term) => String(term).trim().length >= 6)
+    .slice(0, 8)
+    .map(escapeOverpassRegex)
+    .join('|');
+}
+
+function buildCombinedQuery(segments, bbox, includeText) {
+  const seen = new Set();
+  const selectors = [];
+
+  for (const segment of segments) {
+    for (const tag of segment.overpassTags || []) {
+      const key = `tag:${tag.key}=${tag.value}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      selectors.push(overpassSelector(tag, bbox));
+    }
+  }
+
+  if (includeText) {
+    for (const segment of segments) {
+      const textRegex = segmentTextRegex(segment);
+      const key = `text:${segment.name}`;
+      if (!textRegex || seen.has(key)) continue;
+      seen.add(key);
+      selectors.push(overpassTextSelector('name', textRegex, bbox));
+      selectors.push(overpassTextSelector('brand', textRegex, bbox));
+      selectors.push(overpassTextSelector('operator', textRegex, bbox));
+    }
+  }
+
+  return ['[out:json][timeout:40];', '(', selectors.join('\n'), ');', 'out body center;'].join('\n');
+}
+
+function matchSegmentForElement(tags, segments, includeText) {
+  for (const segment of segments) {
+    for (const tag of segment.overpassTags || []) {
+      if (tags[tag.key] === tag.value) return segment;
+    }
+  }
+
+  if (includeText) {
+    const haystack = cleanText([tags.name, tags.brand, tags.operator].filter(Boolean).join(' '));
+    if (!haystack) return null;
+
+    for (const segment of segments) {
+      const terms = (segment.queries || []).filter((term) => String(term).trim().length >= 6);
+      if (terms.some((term) => haystack.includes(cleanText(term)))) return segment;
+    }
+  }
+
+  return null;
+}
+
+function valueFromTags(tags = {}, keys = []) {
+  for (const key of keys) {
+    if (tags[key]) return tags[key];
+  }
+  return '';
+}
+
+function elementToPlace(element, segment, city, uf, query) {
+  const tags = element.tags || {};
+  const name = tags.name || tags.brand || tags.operator;
+  if (!name) return null;
+
+  const street = valueFromTags(tags, ['addr:street']);
+  const housenumber = valueFromTags(tags, ['addr:housenumber']);
+  const address = [street, housenumber].filter(Boolean).join(', ');
+
+  return {
+    name,
+    phone: valueFromTags(tags, ['contact:phone', 'phone', 'mobile', 'contact:mobile']),
+    city: valueFromTags(tags, ['addr:city']) || city,
+    state: uf,
+    segment: segment.name,
+    website: valueFromTags(tags, ['contact:website', 'website', 'url']),
+    address,
+    formatted_address: address,
+    query,
+    osm_id: String(element.id || ''),
+    osm_type: element.type || ''
+  };
+}
+
+export function createOverpassClient({ urls, timeoutMs }) {
+  const failures = new Map();
+
+  return async function request(query) {
+    let lastError;
+
+    for (const apiUrl of urls) {
+      const state = failures.get(apiUrl);
+      if (state?.until && Date.now() < state.until) continue;
+
+      const recordFailure = () => {
+        const current = failures.get(apiUrl) || { count: 0, until: 0 };
+        const count = current.count + 1;
+        failures.set(apiUrl, { count, until: count >= 3 ? Date.now() + OVERPASS_COOLDOWN_MS : 0 });
+      };
+
+      const result = await fetchTextWithTimeout(apiUrl, {
+          method: 'POST',
+          headers: {
+            ...UA_HEADERS,
+            'content-type': 'application/x-www-form-urlencoded;charset=UTF-8'
+          },
+          body: new URLSearchParams({ data: query })
+        }, timeoutMs, { maxBytes: 50 * 1024 * 1024 }).catch((error) => {
+        lastError = /timeout/i.test(error.message)
+          ? new Error(`Overpass timeout apos ${timeoutMs}ms em ${apiUrl}`)
+          : error;
+        recordFailure();
+        return null;
+      });
+
+      if (!result) continue;
+      const { response, text } = result;
+
+      if (!response.ok) {
+        lastError = new Error(`Overpass HTTP ${response.status} em ${apiUrl}: ${text.slice(0, 300)}`);
+        recordFailure();
+        if (response.status === 429) await sleep(5000);
+        continue;
+      }
+
+      const data = JSON.parse(text);
+      failures.set(apiUrl, { count: 0, until: 0 });
+      return data.elements || [];
+    }
+
+    throw lastError || new Error('Overpass indisponivel: todos os endpoints em cooldown');
+  };
+}
+
+// Fonte overpass (OpenStreetMap): uma consulta combinada por cidade cobrindo
+// todos os nichos selecionados, com failover entre endpoints e cooldown.
+export async function* collect(params, ctx) {
+  const { uf, cities, segments, limits } = params;
+  if (!cities || !cities.length) {
+    ctx.log('overpass: nenhuma cidade selecionada — fonte pulada');
+    return;
+  }
+
+  const request = createOverpassClient({
+    urls: ctx.endpoints.overpassUrls,
+    timeoutMs: ctx.delays.overpassTimeoutMs
+  });
+
+  for (const city of cities) {
+    if (ctx.isCancelled()) return;
+    const label = `${city} ${uf}`;
+
+    let bbox = null;
+    try {
+      bbox = await ctx.geocode(uf, city);
+    } catch (error) {
+      ctx.log(`overpass: geocode falhou em ${label}: ${error.message}`);
+    }
+    if (!bbox) {
+      ctx.log(`overpass: sem bounding box para ${label} — cidade pulada`);
+      continue;
+    }
+
+    ctx.emit({ phase: 'overpass', message: `consultando ${label}` });
+
+    let elements = [];
+    try {
+      elements = await request(buildCombinedQuery(segments, bbox, ctx.overpassTextSearch));
+    } catch (error) {
+      ctx.log(`overpass falhou em ${label}: ${error.message}`);
+      await sleep(Math.max(ctx.delays.overpassDelayMs, 15000));
+      continue;
+    }
+
+    const perSegmentCount = new Map();
+    const cityLeads = [];
+
+    for (const element of elements) {
+      const segment = matchSegmentForElement(element.tags || {}, segments, ctx.overpassTextSearch);
+      if (!segment) continue;
+
+      if (limits.maxPerQuery > 0) {
+        const count = perSegmentCount.get(segment.name) || 0;
+        if (count >= limits.maxPerQuery) continue;
+        perSegmentCount.set(segment.name, count + 1);
+      }
+
+      const place = elementToPlace(element, segment, city, uf, `${segment.name} em ${label}`);
+      if (!place) continue;
+      cityLeads.push(buildLeadFromPlace(place, segment, city, uf, 'overpass'));
+    }
+
+    ctx.log(`overpass: ${elements.length} elementos brutos, ${cityLeads.length} leads em ${label}`);
+    if (cityLeads.length) yield { origin: 'overpass', label, leads: cityLeads };
+    await sleep(ctx.delays.overpassDelayMs);
+  }
+}