import { cleanText } from '../lead-engine.mjs'; import { sleep } from '../util/collections.mjs'; import { UA_HEADERS, buildLeadFromPlace } from './common.mjs'; import { fetchTextWithTimeout } from '../util/fetch.mjs'; const OVERPASS_COOLDOWN_MS = 60000; function overpassSelector(tag, bbox) { const [south, west, north, east] = bbox; const key = String(tag.key).replace(/"/g, '\\"'); const value = String(tag.value).replace(/"/g, '\\"'); const selector = `["${key}"="${value}"](${south},${west},${north},${east});`; return [`node${selector}`, `way${selector}`, `relation${selector}`].join('\n'); } export function escapeOverpassRegex(value) { return String(value).replace(/[\\^$.*+?()[\]{}|]/g, '\\$&').replace(/"/g, '\\"'); } function overpassTextSelector(key, regex, bbox) { const [south, west, north, east] = bbox; const selector = `["${key}"~"${regex}",i](${south},${west},${north},${east});`; return [`node${selector}`, `way${selector}`].join('\n'); } function segmentTextRegex(segment) { return (segment.queries || []) .filter((term) => String(term).trim().length >= 6) .slice(0, 8) .map(escapeOverpassRegex) .join('|'); } function buildCombinedQuery(segments, bbox, includeText) { const seen = new Set(); const selectors = []; for (const segment of segments) { for (const tag of segment.overpassTags || []) { const key = `tag:${tag.key}=${tag.value}`; if (seen.has(key)) continue; seen.add(key); selectors.push(overpassSelector(tag, bbox)); } } if (includeText) { for (const segment of segments) { const textRegex = segmentTextRegex(segment); const key = `text:${segment.name}`; if (!textRegex || seen.has(key)) continue; seen.add(key); selectors.push(overpassTextSelector('name', textRegex, bbox)); selectors.push(overpassTextSelector('brand', textRegex, bbox)); selectors.push(overpassTextSelector('operator', textRegex, bbox)); } } return ['[out:json][timeout:40];', '(', selectors.join('\n'), ');', 'out body center;'].join('\n'); } function matchSegmentForElement(tags, segments, includeText) { for (const segment of segments) { for (const tag of segment.overpassTags || []) { if (tags[tag.key] === tag.value) return segment; } } if (includeText) { const haystack = cleanText([tags.name, tags.brand, tags.operator].filter(Boolean).join(' ')); if (!haystack) return null; for (const segment of segments) { const terms = (segment.queries || []).filter((term) => String(term).trim().length >= 6); if (terms.some((term) => haystack.includes(cleanText(term)))) return segment; } } return null; } function valueFromTags(tags = {}, keys = []) { for (const key of keys) { if (tags[key]) return tags[key]; } return ''; } function elementToPlace(element, segment, city, uf, query) { const tags = element.tags || {}; const name = tags.name || tags.brand || tags.operator; if (!name) return null; const street = valueFromTags(tags, ['addr:street']); const housenumber = valueFromTags(tags, ['addr:housenumber']); const address = [street, housenumber].filter(Boolean).join(', '); return { name, phone: valueFromTags(tags, ['contact:phone', 'phone', 'mobile', 'contact:mobile']), city: valueFromTags(tags, ['addr:city']) || city, state: uf, segment: segment.name, website: valueFromTags(tags, ['contact:website', 'website', 'url']), address, formatted_address: address, query, osm_id: String(element.id || ''), osm_type: element.type || '' }; } export function createOverpassClient({ urls, timeoutMs }) { const failures = new Map(); return async function request(query) { let lastError; for (const apiUrl of urls) { const state = failures.get(apiUrl); if (state?.until && Date.now() < state.until) continue; const recordFailure = () => { const current = failures.get(apiUrl) || { count: 0, until: 0 }; const count = current.count + 1; failures.set(apiUrl, { count, until: count >= 3 ? Date.now() + OVERPASS_COOLDOWN_MS : 0 }); }; const result = await fetchTextWithTimeout(apiUrl, { method: 'POST', headers: { ...UA_HEADERS, 'content-type': 'application/x-www-form-urlencoded;charset=UTF-8' }, body: new URLSearchParams({ data: query }) }, timeoutMs, { maxBytes: 50 * 1024 * 1024 }).catch((error) => { lastError = /timeout/i.test(error.message) ? new Error(`Overpass timeout apos ${timeoutMs}ms em ${apiUrl}`) : error; recordFailure(); return null; }); if (!result) continue; const { response, text } = result; if (!response.ok) { lastError = new Error(`Overpass HTTP ${response.status} em ${apiUrl}: ${text.slice(0, 300)}`); recordFailure(); if (response.status === 429) await sleep(5000); continue; } const data = JSON.parse(text); failures.set(apiUrl, { count: 0, until: 0 }); return data.elements || []; } throw lastError || new Error('Overpass indisponivel: todos os endpoints em cooldown'); }; } // Fonte overpass (OpenStreetMap): uma consulta combinada por cidade cobrindo // todos os nichos selecionados, com failover entre endpoints e cooldown. export async function* collect(params, ctx) { const { uf, cities, segments, limits } = params; if (!cities || !cities.length) { ctx.log('overpass: nenhuma cidade selecionada — fonte pulada'); return; } const request = createOverpassClient({ urls: ctx.endpoints.overpassUrls, timeoutMs: ctx.delays.overpassTimeoutMs }); for (const city of cities) { if (ctx.isCancelled()) return; const label = `${city} ${uf}`; let bbox = null; try { bbox = await ctx.geocode(uf, city); } catch (error) { ctx.log(`overpass: geocode falhou em ${label}: ${error.message}`); } if (!bbox) { ctx.log(`overpass: sem bounding box para ${label} — cidade pulada`); continue; } ctx.emit({ phase: 'overpass', message: `consultando ${label}` }); let elements = []; try { elements = await request(buildCombinedQuery(segments, bbox, ctx.overpassTextSearch)); } catch (error) { ctx.log(`overpass falhou em ${label}: ${error.message}`); await sleep(Math.max(ctx.delays.overpassDelayMs, 15000)); continue; } const perSegmentCount = new Map(); const cityLeads = []; for (const element of elements) { const segment = matchSegmentForElement(element.tags || {}, segments, ctx.overpassTextSearch); if (!segment) continue; if (limits.maxPerQuery > 0) { const count = perSegmentCount.get(segment.name) || 0; if (count >= limits.maxPerQuery) continue; perSegmentCount.set(segment.name, count + 1); } const place = elementToPlace(element, segment, city, uf, `${segment.name} em ${label}`); if (!place) continue; cityLeads.push(buildLeadFromPlace(place, segment, city, uf, 'overpass')); } ctx.log(`overpass: ${elements.length} elementos brutos, ${cityLeads.length} leads em ${label}`); if (cityLeads.length) yield { origin: 'overpass', label, leads: cityLeads }; await sleep(ctx.delays.overpassDelayMs); } }