import path from 'node:path'; import { loadDotEnv, paths } from '../env.mjs'; import { openDb } from '../db.mjs'; import { extractUf } from './rfb-extractor.mjs'; import { downloadRfbDataset, validateRfbRawFiles } from './rfb-downloader.mjs'; import { createJobIO } from '../jobs/protocol.mjs'; import { randomUUID } from 'node:crypto'; import { createJobLease } from '../jobs/lease.mjs'; // Processo filho de extracao RFB: le a UF da linha em extractions e roda a // extracao completa, reportando progresso via stdout. await loadDotEnv(); const args = new Map( process.argv.slice(2).map((arg) => { const [key, value = 'true'] = arg.replace(/^--/, '').split('='); return [key, value]; }) ); const io = createJobIO(); async function main() { const extractionId = Number(args.get('extraction-id')); if (!extractionId) throw new Error('extract-job: --extraction-id obrigatorio'); const db = openDb(); let lease; try { const extraction = db.prepare('SELECT * FROM extractions WHERE id = ?').get(extractionId); if (!extraction) throw new Error(`extraction #${extractionId} nao encontrada`); const uf = extraction.uf; lease = createJobLease({ db, table: 'extractions', id: extractionId, ownerToken: args.get('owner-token') || randomUUID() }); const outCsv = path.join(paths.cnpjDir, `estabelecimentos-${uf}.csv`); const checkpointPath = path.join(paths.cnpjDir, `checkpoint-estabelecimentos-${uf}.ndjson`); const download = await downloadRfbDataset({ rawDir: paths.rfbRaw, isCancelled: io.isCancelled, onProgress: (progress) => { const mapped = { ...progress, pct: Math.round((progress.pct || 0) * 0.45) }; io.emit(mapped); db.prepare('UPDATE extractions SET progress_json = ? WHERE id = ?') .run(JSON.stringify(mapped), extractionId); } }); if (download.cancelled) { db.prepare("UPDATE extractions SET status = 'cancelled', finished_at = datetime('now') WHERE id = ?").run(extractionId); io.done({ cancelled: true, snapshot: download.snapshot, downloaded: download.downloaded.length }); return; } await validateRfbRawFiles(paths.rfbRaw); const result = await extractUf({ uf, rawDir: paths.rfbRaw, outCsv, checkpointPath, db, log: (m) => io.log(m), isCancelled: io.isCancelled, onProgress: (p) => { const mapped = { ...p, pct: 45 + Math.round((p.pct || 0) * 0.55) }; io.emit(mapped); db.prepare('UPDATE extractions SET progress_json = ? WHERE id = ?') .run(JSON.stringify(mapped), extractionId); } }); if (result.cancelled) { db.prepare("UPDATE extractions SET status = 'cancelled', finished_at = datetime('now') WHERE id = ?") .run(extractionId); io.done({ cancelled: true }); } else { db.prepare( `UPDATE extractions SET status = 'done', csv_path = ?, total_rows = ?, with_phone = ?, municipios_count = ?, finished_at = datetime('now') WHERE id = ?` ).run(result.csvPath, result.totalRows, result.withPhone, result.municipiosCount, extractionId); io.done({ snapshot: download.snapshot, downloaded: download.downloaded.length, skipped: download.skipped.length, totalRows: result.totalRows, withPhone: result.withPhone, municipios: result.municipiosCount }); } } catch (error) { db.prepare("UPDATE extractions SET status = 'error', error = ?, finished_at = datetime('now') WHERE id = ? AND status IN ('pending','running')") .run(String(error.message || error), extractionId); throw error; } finally { lease?.close(); db.close(); } } main().catch((error) => { io.fail(error); process.exit(1); });