chore: import maquina-comercial-pessoal history via subtree
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> git-subtree-dir: maquina-comercial-pessoal git-subtree-mainline:f5af038e19git-subtree-split:a9ef4d776d
This commit is contained in:
@@ -0,0 +1,100 @@
|
||||
import path from 'node:path';
|
||||
import { loadDotEnv, paths } from '../env.mjs';
|
||||
import { openDb } from '../db.mjs';
|
||||
import { extractUf } from './rfb-extractor.mjs';
|
||||
import { downloadRfbDataset, validateRfbRawFiles } from './rfb-downloader.mjs';
|
||||
import { createJobIO } from '../jobs/protocol.mjs';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { createJobLease } from '../jobs/lease.mjs';
|
||||
|
||||
// Processo filho de extracao RFB: le a UF da linha em extractions e roda a
|
||||
// extracao completa, reportando progresso via stdout.
|
||||
|
||||
await loadDotEnv();
|
||||
|
||||
const args = new Map(
|
||||
process.argv.slice(2).map((arg) => {
|
||||
const [key, value = 'true'] = arg.replace(/^--/, '').split('=');
|
||||
return [key, value];
|
||||
})
|
||||
);
|
||||
|
||||
const io = createJobIO();
|
||||
|
||||
async function main() {
|
||||
const extractionId = Number(args.get('extraction-id'));
|
||||
if (!extractionId) throw new Error('extract-job: --extraction-id obrigatorio');
|
||||
|
||||
const db = openDb();
|
||||
let lease;
|
||||
try {
|
||||
const extraction = db.prepare('SELECT * FROM extractions WHERE id = ?').get(extractionId);
|
||||
if (!extraction) throw new Error(`extraction #${extractionId} nao encontrada`);
|
||||
const uf = extraction.uf;
|
||||
lease = createJobLease({ db, table: 'extractions', id: extractionId, ownerToken: args.get('owner-token') || randomUUID() });
|
||||
const outCsv = path.join(paths.cnpjDir, `estabelecimentos-${uf}.csv`);
|
||||
const checkpointPath = path.join(paths.cnpjDir, `checkpoint-estabelecimentos-${uf}.ndjson`);
|
||||
const download = await downloadRfbDataset({
|
||||
rawDir: paths.rfbRaw,
|
||||
isCancelled: io.isCancelled,
|
||||
onProgress: (progress) => {
|
||||
const mapped = { ...progress, pct: Math.round((progress.pct || 0) * 0.45) };
|
||||
io.emit(mapped);
|
||||
db.prepare('UPDATE extractions SET progress_json = ? WHERE id = ?')
|
||||
.run(JSON.stringify(mapped), extractionId);
|
||||
}
|
||||
});
|
||||
if (download.cancelled) {
|
||||
db.prepare("UPDATE extractions SET status = 'cancelled', finished_at = datetime('now') WHERE id = ?").run(extractionId);
|
||||
io.done({ cancelled: true, snapshot: download.snapshot, downloaded: download.downloaded.length });
|
||||
return;
|
||||
}
|
||||
await validateRfbRawFiles(paths.rfbRaw);
|
||||
const result = await extractUf({
|
||||
uf,
|
||||
rawDir: paths.rfbRaw,
|
||||
outCsv,
|
||||
checkpointPath,
|
||||
db,
|
||||
log: (m) => io.log(m),
|
||||
isCancelled: io.isCancelled,
|
||||
onProgress: (p) => {
|
||||
const mapped = { ...p, pct: 45 + Math.round((p.pct || 0) * 0.55) };
|
||||
io.emit(mapped);
|
||||
db.prepare('UPDATE extractions SET progress_json = ? WHERE id = ?')
|
||||
.run(JSON.stringify(mapped), extractionId);
|
||||
}
|
||||
});
|
||||
|
||||
if (result.cancelled) {
|
||||
db.prepare("UPDATE extractions SET status = 'cancelled', finished_at = datetime('now') WHERE id = ?")
|
||||
.run(extractionId);
|
||||
io.done({ cancelled: true });
|
||||
} else {
|
||||
db.prepare(
|
||||
`UPDATE extractions SET status = 'done', csv_path = ?, total_rows = ?, with_phone = ?,
|
||||
municipios_count = ?, finished_at = datetime('now') WHERE id = ?`
|
||||
).run(result.csvPath, result.totalRows, result.withPhone, result.municipiosCount, extractionId);
|
||||
io.done({
|
||||
snapshot: download.snapshot,
|
||||
downloaded: download.downloaded.length,
|
||||
skipped: download.skipped.length,
|
||||
totalRows: result.totalRows,
|
||||
withPhone: result.withPhone,
|
||||
municipios: result.municipiosCount
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
db.prepare("UPDATE extractions SET status = 'error', error = ?, finished_at = datetime('now') WHERE id = ? AND status IN ('pending','running')")
|
||||
.run(String(error.message || error), extractionId);
|
||||
throw error;
|
||||
} finally {
|
||||
lease?.close();
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
io.fail(error);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user