Gustavo
d330bb5ff0
Tracks the app that was already in place (capture, treatment, outreach, WhatsApp integration, server, frontend, tests, docs) so future feature work can branch and use isolated worktrees.
91 lines
3.3 KiB
JavaScript
91 lines
3.3 KiB
JavaScript
import path from 'node:path';
|
|
import { loadDotEnv, paths } from '../src/env.mjs';
|
|
import { openDb, ensureSeeds } from '../src/db.mjs';
|
|
import { extractUf } from '../src/extract/rfb-extractor.mjs';
|
|
import { downloadRfbDataset, validateRfbRawFiles } from '../src/extract/rfb-downloader.mjs';
|
|
import { randomUUID } from 'node:crypto';
|
|
import { createJobLease } from '../src/jobs/lease.mjs';
|
|
|
|
// Extrai a base CNPJ de uma UF inteira a partir dos zips da Receita Federal:
|
|
// node scripts/extrair-base-cnpj.mjs --uf=BA
|
|
// Por padrao baixa/retoma os arquivos necessarios antes de extrair. Use
|
|
// --no-download somente quando a base local completa ja estiver disponivel.
|
|
|
|
await loadDotEnv();
|
|
|
|
const args = new Map(
|
|
process.argv.slice(2).map((arg) => {
|
|
const [key, value = 'true'] = arg.replace(/^--/, '').split('=');
|
|
return [key, value];
|
|
})
|
|
);
|
|
|
|
const uf = (args.get('uf') || '').toUpperCase();
|
|
if (!/^[A-Z]{2}$/.test(uf)) {
|
|
console.error('Uso: node scripts/extrair-base-cnpj.mjs --uf=BA');
|
|
process.exit(1);
|
|
}
|
|
|
|
const rawDir = args.get('raw-dir') || paths.rfbRaw;
|
|
const outCsv = path.join(paths.cnpjDir, `estabelecimentos-${uf}.csv`);
|
|
const checkpointPath = path.join(paths.cnpjDir, `checkpoint-estabelecimentos-${uf}.ndjson`);
|
|
|
|
async function main() {
|
|
const db = openDb();
|
|
await ensureSeeds(db, { log: (m) => console.log(`[extrair] ${m}`) });
|
|
|
|
const active = db.prepare("SELECT id, uf FROM extractions WHERE status IN ('pending','running')").get();
|
|
if (active) throw new Error(`ja existe uma extracao em andamento (#${active.id}, ${active.uf})`);
|
|
const inserted = db.prepare(
|
|
"INSERT INTO extractions (uf, status, started_at) VALUES (?, 'running', datetime('now'))"
|
|
).run(uf);
|
|
const extractionId = Number(inserted.lastInsertRowid);
|
|
const lease = createJobLease({ db, table: 'extractions', id: extractionId, ownerToken: randomUUID() });
|
|
|
|
try {
|
|
if (!args.has('no-download')) {
|
|
await downloadRfbDataset({
|
|
rawDir,
|
|
onProgress: (p) => console.log(`[download] ${p.pct ?? 0}% ${p.message}`)
|
|
});
|
|
}
|
|
await validateRfbRawFiles(rawDir);
|
|
const result = await extractUf({
|
|
uf,
|
|
rawDir,
|
|
outCsv,
|
|
checkpointPath,
|
|
db,
|
|
log: (m) => console.log(`[extrair] ${m}`),
|
|
onProgress: (p) => {
|
|
console.log(`[extrair] ${p.pct}% ${p.phase}: ${p.message}`);
|
|
db.prepare('UPDATE extractions SET progress_json = ? WHERE id = ?')
|
|
.run(JSON.stringify(p), extractionId);
|
|
}
|
|
});
|
|
|
|
if (result.cancelled) {
|
|
db.prepare("UPDATE extractions SET status = 'cancelled', finished_at = datetime('now') WHERE id = ?")
|
|
.run(extractionId);
|
|
} else {
|
|
db.prepare(
|
|
`UPDATE extractions SET status = 'done', csv_path = ?, total_rows = ?, with_phone = ?,
|
|
municipios_count = ?, finished_at = datetime('now') WHERE id = ?`
|
|
).run(result.csvPath, result.totalRows, result.withPhone, result.municipiosCount, extractionId);
|
|
}
|
|
} catch (error) {
|
|
db.prepare(
|
|
"UPDATE extractions SET status = 'error', error = ?, finished_at = datetime('now') WHERE id = ?"
|
|
).run(String(error.message || error), extractionId);
|
|
throw error;
|
|
} finally {
|
|
lease.close();
|
|
db.close();
|
|
}
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error(`[extrair] erro: ${error.stack || error.message}`);
|
|
process.exit(1);
|
|
});
|