COMERCIAL-GTI/maquina-comercial-pessoal/tests/rfb-downloader.test.mjs

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import {
  REQUIRED_RFB_FILES,
  parseLatestSnapshot,
  validateRfbRawFiles,
  downloadRfbDataset
} from '../src/extract/rfb-downloader.mjs';

const zip = Buffer.concat([Buffer.from('PK\x05\x06'), Buffer.alloc(18)]);

async function tempDir(t) {
  const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'rfb-test-'));
  t.after(() => fs.rm(dir, { recursive: true, force: true }));
  return dir;
}

function response(body, { status = 200, headers = {} } = {}) {
  return new Response(body, { status, headers });
}

test('lista somente os 21 arquivos necessarios e escolhe o snapshot mais recente', () => {
  assert.equal(REQUIRED_RFB_FILES.length, 21);
  assert.deepEqual(REQUIRED_RFB_FILES.slice(0, 3), ['Municipios.zip', 'Empresas0.zip', 'Empresas1.zip']);
  assert.equal(REQUIRED_RFB_FILES.at(-1), 'Estabelecimentos9.zip');
  assert.equal(parseLatestSnapshot('<a href="2026-06-01/">x</a><a href="2026-07-12/">y</a>'), '2026-07-12');
});

test('ignora datas de modificacao que nao sao links de snapshot', () => {
  const html = '<a href="2026-07-12/">2026-07-12/</a> 2026-07-20 18:11 -';
  assert.equal(parseLatestSnapshot(html), '2026-07-12');
});

test('validacao rejeita conjunto incompleto, HTML e ZIP menor que o minimo', async (t) => {
  const dir = await tempDir(t);
  await fs.writeFile(path.join(dir, 'Municipios.zip'), Buffer.from('<html>erro</html>'));
  await assert.rejects(validateRfbRawFiles(dir, { minZipBytes: 4 }), /21 arquivos|ausentes|invalido/i);
});

test('validacao rejeita ZIP com assinatura inicial mas sem EOCD', async (t) => {
  const dir = await tempDir(t);
  for (const name of REQUIRED_RFB_FILES) await fs.writeFile(path.join(dir, name), Buffer.from('PK\x03\x04sem-diretorio-central'));
  await assert.rejects(validateRfbRawFiles(dir, { minZipBytes: 4 }), /EOCD/i);
});

test('validacao rejeita PK com EOCD falso estruturalmente inconsistente', async (t) => {
  const dir = await tempDir(t);
  const falseZip = Buffer.concat([
    Buffer.from('PKjunkPK\x05\x06'),
    Buffer.from([0, 0, 0, 0, 1, 0, 1, 0]),
    Buffer.alloc(8),
    Buffer.from([0, 0])
  ]);
  for (const name of REQUIRED_RFB_FILES) await fs.writeFile(path.join(dir, name), falseZip);
  await assert.rejects(validateRfbRawFiles(dir, { minZipBytes: 4 }), /diretorio central|EOCD/i);
});

test('validacao rejeita sentinelas ZIP64 com erro claro', async (t) => {
  const dir = await tempDir(t);
  const zip64 = Buffer.from(zip);
  zip64.writeUInt16LE(0xffff, 8);
  for (const name of REQUIRED_RFB_FILES) await fs.writeFile(path.join(dir, name), zip64);
  await assert.rejects(validateRfbRawFiles(dir, { minZipBytes: 4 }), /ZIP64/i);
});

test('download descobre snapshot, baixa os necessarios e depois pula arquivos validos', async (t) => {
  const dir = await tempDir(t);
  let downloads = 0;
  const mockFetch = async (url) => {
    if (String(url).endsWith('/arquivos/')) {
      return response('<a href="2026-07-12/">snapshot</a>', { headers: { 'content-type': 'text/html' } });
    }
    downloads += 1;
    return response(zip, { headers: { 'content-length': String(zip.length) } });
  };

  const first = await downloadRfbDataset({
    rawDir: dir,
    indexUrl: 'https://example.test/arquivos/',
    fetch: mockFetch,
    minZipBytes: 4
  });
  assert.equal(first.snapshot, '2026-07-12');
  assert.equal(first.downloaded.length, 21);
  assert.equal(downloads, 21);
  await validateRfbRawFiles(dir, { minZipBytes: 4 });

  const second = await downloadRfbDataset({
    rawDir: dir,
    indexUrl: 'https://example.test/arquivos/',
    fetch: mockFetch,
    minZipBytes: 4
  });
  assert.equal(second.skipped.length, 21);
  assert.equal(downloads, 21);
});

test('download rejeita HTTP 404 e resposta HTML sem promover o .part', async (t) => {
  const dir = await tempDir(t);
  const index = response('<a href="2026-07-12/">snapshot</a>');
  let caseName = '404';
  const mockFetch = async (url) => {
    if (String(url).endsWith('/arquivos/')) return index.clone();
    if (caseName === '404') return response('not found', { status: 404 });
    return response('<html>bloqueado</html>', { headers: { 'content-length': '22' } });
  };
  await assert.rejects(downloadRfbDataset({ rawDir: dir, indexUrl: 'https://x/arquivos/', fetch: mockFetch, minZipBytes: 4 }), /HTTP 404/);
  caseName = 'html';
  await assert.rejects(downloadRfbDataset({ rawDir: dir, indexUrl: 'https://x/arquivos/', fetch: mockFetch, minZipBytes: 4 }), /ZIP|assinatura/i);
  await assert.rejects(fs.stat(path.join(dir, 'Municipios.zip')));
});

test('download retoma .part com Range e preserva parcial ao cancelar', async (t) => {
  const dir = await tempDir(t);
  await fs.writeFile(path.join(dir, '.rfb-snapshot.json'), JSON.stringify({
    snapshot: '2026-07-12', complete: false, files: {},
    partials: { 'Municipios.zip': { validator: '"v1"', etag: '"v1"', lastModified: null, totalSize: zip.length } }
  }));
  await fs.writeFile(path.join(dir, 'Municipios.zip.part'), zip.subarray(0, 5));
  let range;
  let ifRange;
  let cancelled = false;
  const mockFetch = async (url, options = {}) => {
    if (String(url).endsWith('/arquivos/')) return response('<a href="2026-07-12/">snapshot</a>');
    range ||= options.headers?.Range || options.headers?.range;
    ifRange ||= options.headers?.['If-Range'] || options.headers?.['if-range'];
    if (String(url).endsWith('/Municipios.zip')) {
      return response(zip.subarray(5), {
        status: 206,
        headers: { etag: '"v1"', 'content-range': `bytes 5-${zip.length - 1}/${zip.length}`, 'content-length': String(zip.length - 5) }
      });
    }
    cancelled = true;
    return response(zip, { headers: { 'content-length': String(zip.length) } });
  };
  const result = await downloadRfbDataset({
    rawDir: dir,
    indexUrl: 'https://x/arquivos/',
    fetch: mockFetch,
    minZipBytes: 4,
    isCancelled: () => cancelled
  });
  assert.equal(range, 'bytes=5-');
  assert.equal(ifRange, '"v1"');
  assert.equal(result.cancelled, true);
  assert.deepEqual(await fs.readFile(path.join(dir, 'Municipios.zip')), zip);
});

test('resposta 206 divergente descarta parcial e reinicia uma vez do zero', async (t) => {
  const dir = await tempDir(t);
  await fs.writeFile(path.join(dir, '.rfb-snapshot.json'), JSON.stringify({
    snapshot: '2026-07-12', complete: false, files: {},
    partials: { 'Municipios.zip': { validator: '"v1"', etag: '"v1"', totalSize: zip.length } }
  }));
  await fs.writeFile(path.join(dir, 'Municipios.zip.part'), zip.subarray(0, 5));
  let municipiosCalls = 0;
  let cancelled = false;
  const mockFetch = async (url, options = {}) => {
    if (String(url).endsWith('/arquivos/')) return response('<a href="2026-07-12/">snapshot</a>');
    if (String(url).endsWith('/Municipios.zip')) {
      municipiosCalls += 1;
      if (options.headers?.Range) return response(zip.subarray(5), {
        status: 206,
        headers: { etag: '"v2"', 'content-range': `bytes 5-${zip.length - 1}/${zip.length}`, 'content-length': String(zip.length - 5) }
      });
      return response(zip, { headers: { etag: '"v2"', 'content-length': String(zip.length) } });
    }
    cancelled = true;
    return response(zip, { headers: { 'content-length': String(zip.length) } });
  };
  const result = await downloadRfbDataset({ rawDir: dir, indexUrl: 'https://x/arquivos/', fetch: mockFetch, minZipBytes: 4, isCancelled: () => cancelled });
  assert.equal(result.cancelled, true);
  assert.equal(municipiosCalls, 2);
  assert.deepEqual(await fs.readFile(path.join(dir, 'Municipios.zip')), zip);
});

test('progresso de muitos chunks emite inicio e fim sem evento por chunk', async (t) => {
  const dir = await tempDir(t);
  const chunks = Array.from({ length: 100 }, (_, index) => zip.subarray(index % zip.length, (index % zip.length) + 1));
  const body = new ReadableStream({
    start(controller) {
      for (const chunk of chunks) controller.enqueue(chunk);
      controller.enqueue(zip);
      controller.close();
    }
  });
  const total = chunks.length + zip.length;
  let cancelled = false;
  const events = [];
  const mockFetch = async (url) => {
    if (String(url).endsWith('/arquivos/')) return response('<a href="2026-07-12/">snapshot</a>');
    if (String(url).endsWith('/Municipios.zip')) return response(body, { headers: { 'content-length': String(total) } });
    cancelled = true;
    return response(zip, { headers: { 'content-length': String(zip.length) } });
  };
  await downloadRfbDataset({ rawDir: dir, indexUrl: 'https://x/arquivos/', fetch: mockFetch, minZipBytes: 4, isCancelled: () => cancelled, onProgress: (event) => events.push(event) });
  const fileEvents = events.filter((event) => event.file === 'Municipios.zip');
  assert.ok(fileEvents.length >= 2);
  assert.ok(fileEvents.length <= 3, `eventos por arquivo: ${fileEvents.length}`);
  assert.equal(fileEvents[0].bytes, 0);
  assert.equal(fileEvents.at(-1).bytes, total);
});

test('ENOSPC preserva ZIP e manifesto anteriores byte a byte', async (t) => {
  const dir = await tempDir(t);
  const manifest = Buffer.from(JSON.stringify({ snapshot: '2026-06-01', complete: true, files: { 'Municipios.zip': zip.length } }));
  await fs.writeFile(path.join(dir, '.rfb-snapshot.json'), manifest);
  await fs.writeFile(path.join(dir, 'Municipios.zip'), zip);
  const mockFetch = async (url) => response('<a href="2026-07-12/">snapshot</a>');
  await assert.rejects(downloadRfbDataset({
    rawDir: dir,
    indexUrl: 'https://x/arquivos/',
    fetch: mockFetch,
    minZipBytes: 4,
    minimumFreeBytes: 100,
    statfs: async () => ({ bavail: 1, bsize: 1 })
  }), (error) => error.code === 'ENOSPC');
  assert.deepEqual(await fs.readFile(path.join(dir, 'Municipios.zip')), zip);
  assert.deepEqual(await fs.readFile(path.join(dir, '.rfb-snapshot.json')), manifest);
});

test('download rejeita Content-Range que nao continua o .part', async (t) => {
  const dir = await tempDir(t);
  await fs.writeFile(path.join(dir, '.rfb-snapshot.json'), JSON.stringify({
    snapshot: '2026-07-12', complete: false, files: {},
    partials: { 'Municipios.zip': { validator: '"v1"', etag: '"v1"', totalSize: zip.length } }
  }));
  await fs.writeFile(path.join(dir, 'Municipios.zip.part'), zip.subarray(0, 5));
  const mockFetch = async (url) => String(url).endsWith('/arquivos/')
    ? response('<a href="2026-07-12/">snapshot</a>')
    : response(zip.subarray(5), {
      status: 206,
      headers: { etag: '"v1"', 'content-range': `bytes 4-${zip.length - 2}/${zip.length}`, 'content-length': String(zip.length - 5) }
    });
  await assert.rejects(
    downloadRfbDataset({ rawDir: dir, indexUrl: 'https://x/arquivos/', fetch: mockFetch, minZipBytes: 4 }),
    /Content-Range/i
  );
});

test('arquivos sem manifesto nao sao atribuidos ao snapshot mais recente', async (t) => {
  const dir = await tempDir(t);
  for (const name of REQUIRED_RFB_FILES) await fs.writeFile(path.join(dir, name), zip);
  let downloads = 0;
  const mockFetch = async (url) => {
    if (String(url).endsWith('/arquivos/')) return response('<a href="2026-07-12/">snapshot</a>');
    downloads += 1;
    return response(zip, { headers: { 'content-length': String(zip.length) } });
  };

  const result = await downloadRfbDataset({
    rawDir: dir,
    indexUrl: 'https://x/arquivos/',
    fetch: mockFetch,
    minZipBytes: 4
  });

  assert.equal(result.downloaded.length, REQUIRED_RFB_FILES.length);
  assert.equal(downloads, REQUIRED_RFB_FILES.length);
});

test('manifesto objeto sem tamanho esperado nao autoriza pular ZIP final', async (t) => {
  const dir = await tempDir(t);
  for (const name of REQUIRED_RFB_FILES) await fs.writeFile(path.join(dir, name), zip);
  await fs.writeFile(path.join(dir, '.rfb-snapshot.json'), JSON.stringify({
    snapshot: '2026-07-12', complete: true,
    files: Object.fromEntries(REQUIRED_RFB_FILES.map((name) => [name, { etag: '"v1"' }]))
  }));
  let downloads = 0;
  const mockFetch = async (url) => {
    if (String(url).endsWith('/arquivos/')) return response('<a href="2026-07-12/">snapshot</a>');
    downloads += 1;
    return response(zip, { headers: { etag: '"v1"', 'content-length': String(zip.length) } });
  };
  await downloadRfbDataset({ rawDir: dir, indexUrl: 'https://x/arquivos/', fetch: mockFetch, minZipBytes: 4 });
  assert.equal(downloads, REQUIRED_RFB_FILES.length);
});

test('download interrompe corpo sem atividade e preserva o arquivo parcial', async (t) => {
  const dir = await tempDir(t);
  const stalledBody = new ReadableStream({
    start(controller) {
      controller.enqueue(zip.subarray(0, 4));
    }
  });
  const mockFetch = async (url) => String(url).endsWith('/arquivos/')
    ? response('<a href="2026-07-12/">snapshot</a>')
    : response(stalledBody, { headers: { 'content-length': String(zip.length) } });

  await assert.rejects(
    downloadRfbDataset({
      rawDir: dir,
      indexUrl: 'https://x/arquivos/',
      fetch: mockFetch,
      minZipBytes: 4,
      idleTimeoutMs: 10
    }),
    /sem dados|timeout/i
  );
  assert.equal((await fs.stat(path.join(dir, 'Municipios.zip.part'))).size, 4);
});

test('snapshot completo pode ser reutilizado sem espaco extra e com indice offline', async (t) => {
  const dir = await tempDir(t);
  const files = {};
  for (const name of REQUIRED_RFB_FILES) {
    await fs.writeFile(path.join(dir, name), zip);
    files[name] = { size: zip.length, etag: '"v1"', lastModified: null };
  }
  await fs.writeFile(path.join(dir, '.rfb-snapshot.json'), JSON.stringify({
    snapshot: '2026-07-12', complete: true, files, partials: {}
  }));

  const result = await downloadRfbDataset({
    rawDir: dir,
    indexUrl: 'https://offline.test/arquivos/',
    fetch: async () => { throw new Error('sem rede'); },
    minZipBytes: 4,
    minimumFreeBytes: Number.MAX_SAFE_INTEGER,
    statfs: async () => ({ bavail: 0n, bsize: 1n })
  });

  assert.equal(result.snapshot, '2026-07-12');
  assert.equal(result.offline, true);
  assert.equal(result.skipped.length, REQUIRED_RFB_FILES.length);
});

test('snapshot completo com validator faz HEAD e baixa novamente se o arquivo mudou', async (t) => {
  const dir = await tempDir(t);
  const files = {};
  for (const name of REQUIRED_RFB_FILES) {
    await fs.writeFile(path.join(dir, name), zip);
    files[name] = { size: zip.length, etag: '"v1"', lastModified: null };
  }
  await fs.writeFile(path.join(dir, '.rfb-snapshot.json'), JSON.stringify({ snapshot: '2026-07-12', complete: true, files, partials: {} }));
  const methods = [];
  const mockFetch = async (url, options = {}) => {
    if (String(url).endsWith('/arquivos/')) return response('<a href="2026-07-12/">snapshot</a>');
    methods.push(options.method || 'GET');
    if (options.method === 'HEAD') return response(null, { headers: { etag: '"v2"', 'content-length': String(zip.length) } });
    return response(zip, { headers: { etag: '"v2"', 'content-length': String(zip.length) } });
  };
  const result = await downloadRfbDataset({ rawDir: dir, indexUrl: 'https://x/arquivos/', fetch: mockFetch, minZipBytes: 4 });
  assert.equal(methods.filter((method) => method === 'HEAD').length, 1);
  assert.equal(result.downloaded.length, REQUIRED_RFB_FILES.length);
  assert.equal(methods.filter((method) => method === 'GET').length, REQUIRED_RFB_FILES.length);
});

test('snapshot completo sem validator nao faz HEAD', async (t) => {
  const dir = await tempDir(t);
  const files = {};
  for (const name of REQUIRED_RFB_FILES) {
    await fs.writeFile(path.join(dir, name), zip);
    files[name] = { size: zip.length, etag: null, lastModified: null };
  }
  await fs.writeFile(path.join(dir, '.rfb-snapshot.json'), JSON.stringify({ snapshot: '2026-07-12', complete: true, files, partials: {} }));
  let heads = 0;
  const result = await downloadRfbDataset({
    rawDir: dir,
    indexUrl: 'https://x/arquivos/',
    fetch: async (url, options = {}) => {
      if (options.method === 'HEAD') heads += 1;
      return response('<a href="2026-07-12/">snapshot</a>');
    },
    minZipBytes: 4
  });
  assert.equal(heads, 0);
  assert.equal(result.skipped.length, REQUIRED_RFB_FILES.length);
});

test('falha de rede no HEAD usa snapshot local estruturalmente validado', async (t) => {
  const dir = await tempDir(t);
  const files = {};
  for (const name of REQUIRED_RFB_FILES) {
    await fs.writeFile(path.join(dir, name), zip);
    files[name] = { size: zip.length, etag: '"v1"', lastModified: null };
  }
  await fs.writeFile(path.join(dir, '.rfb-snapshot.json'), JSON.stringify({ snapshot: '2026-07-12', complete: true, files, partials: {} }));
  const result = await downloadRfbDataset({
    rawDir: dir,
    indexUrl: 'https://x/arquivos/',
    fetch: async (url, options = {}) => {
      if (String(url).endsWith('/arquivos/')) return response('<a href="2026-07-12/">snapshot</a>');
      if (options.method === 'HEAD') throw new TypeError('rede indisponivel');
      throw new Error('GET inesperado');
    },
    minZipBytes: 4
  });
  assert.equal(result.offline, true);
  assert.equal(result.skipped.length, REQUIRED_RFB_FILES.length);
});

test('retomada exige somente restante ou margem de extracao, nao 8 GiB', async (t) => {
  const dir = await tempDir(t);
  const oneGiB = 1024 * 1024 * 1024;
  const partials = {};
  for (const name of REQUIRED_RFB_FILES.slice(0, 7)) {
    const part = path.join(dir, `${name}.part`);
    await fs.writeFile(part, Buffer.alloc(1));
    await fs.truncate(part, oneGiB);
    partials[name] = { validator: '"v1"', etag: '"v1"', totalSize: oneGiB + 1 };
  }
  await fs.writeFile(path.join(dir, '.rfb-snapshot.json'), JSON.stringify({
    snapshot: '2026-07-12', complete: false,
    files: {}, partials
  }));
  let cancelled = false;
  const result = await downloadRfbDataset({
    rawDir: dir,
    indexUrl: 'https://x/arquivos/',
    fetch: async (url) => {
      if (String(url).endsWith('/arquivos/')) return response('<a href="2026-07-12/">snapshot</a>');
      cancelled = true;
      return response(zip, { headers: { 'content-length': String(zip.length) } });
    },
    minZipBytes: 4,
    isCancelled: () => cancelled,
    statfs: async () => ({ bavail: 3n * 1024n * 1024n * 1024n, bsize: 1n })
  });
  assert.equal(result.cancelled, true);
});

test('indice offline restaura backup completo quando o snapshot atual esta incompleto', async (t) => {
  const dir = await tempDir(t);
  const backup = path.join(dir, '.rfb-backup');
  await fs.mkdir(backup);
  const files = {};
  for (const name of REQUIRED_RFB_FILES) {
    await fs.writeFile(path.join(backup, name), zip);
    files[name] = { size: zip.length, etag: null, lastModified: null };
  }
  await fs.writeFile(path.join(backup, '.rfb-snapshot.json'), JSON.stringify({ snapshot: '2026-06-01', complete: true, files, partials: {} }));
  await fs.writeFile(path.join(dir, '.rfb-snapshot.json'), JSON.stringify({ snapshot: '2026-07-12', complete: false, files: {}, partials: {} }));
  await fs.writeFile(path.join(dir, 'Municipios.zip'), Buffer.from('quebrado'));
  const result = await downloadRfbDataset({ rawDir: dir, fetch: async () => { throw new Error('offline'); }, minZipBytes: 4 });
  assert.equal(result.offline, true);
  assert.equal(result.snapshot, '2026-06-01');
  assert.deepEqual(await fs.readFile(path.join(dir, 'Municipios.zip')), zip);
  assert.ok((await fs.stat(path.join(dir, '.rfb-incomplete'))).isDirectory());
});

test('backup valido existente sobrevive a snapshot intermediario incompleto', async (t) => {
  const dir = await tempDir(t);
  const backup = path.join(dir, '.rfb-backup');
  await fs.mkdir(backup);
  const files = Object.fromEntries(REQUIRED_RFB_FILES.map((name) => [name, { size: zip.length }]));
  for (const name of REQUIRED_RFB_FILES) await fs.writeFile(path.join(backup, name), zip);
  const backupManifest = JSON.stringify({ snapshot: '2026-06-01', complete: true, files, partials: {} });
  await fs.writeFile(path.join(backup, '.rfb-snapshot.json'), backupManifest);
  await fs.writeFile(path.join(dir, '.rfb-snapshot.json'), JSON.stringify({ snapshot: '2026-07-01', complete: false, files: {}, partials: {} }));
  let cancelled = false;
  await downloadRfbDataset({
    rawDir: dir,
    indexUrl: 'https://x/arquivos/',
    fetch: async (url) => {
      if (String(url).endsWith('/arquivos/')) return response('<a href="2026-07-12/">snapshot</a>');
      cancelled = true;
      return response(zip, { headers: { 'content-length': String(zip.length) } });
    },
    minZipBytes: 4,
    isCancelled: () => cancelled
  });
  assert.equal(await fs.readFile(path.join(backup, '.rfb-snapshot.json'), 'utf8'), backupManifest);
});

test('snapshot A completo vira backup durante falha de B e e restaurado offline', async (t) => {
  const dir = await tempDir(t);
  const files = {};
  for (const name of REQUIRED_RFB_FILES) {
    await fs.writeFile(path.join(dir, name), zip);
    files[name] = { size: zip.length, etag: null, lastModified: null };
  }
  await fs.writeFile(path.join(dir, '.rfb-snapshot.json'), JSON.stringify({ snapshot: '2026-06-01', complete: true, files, partials: {} }));
  await assert.rejects(downloadRfbDataset({
    rawDir: dir,
    indexUrl: 'https://x/arquivos/',
    fetch: async (url) => String(url).endsWith('/arquivos/')
      ? response('<a href="2026-07-01/">snapshot</a>')
      : response('<html>falha</html>', { headers: { 'content-length': '18' } }),
    minZipBytes: 4
  }), /ZIP|EOCD|assinatura/i);
  assert.ok((await fs.stat(path.join(dir, '.rfb-backup'))).isDirectory());

  const restored = await downloadRfbDataset({ rawDir: dir, fetch: async () => { throw new Error('offline'); }, minZipBytes: 4 });
  assert.equal(restored.offline, true);
  assert.equal(restored.snapshot, '2026-06-01');
  assert.deepEqual(await fs.readFile(path.join(dir, 'Municipios.zip')), zip);
});