Add 'fiscal/' from commit '102ba84500dbd9655a0471288c3e278652593ca9'
git-subtree-dir: fiscal git-subtree-mainline:d95165fc8fgit-subtree-split:102ba84500
This commit is contained in:
gti
@@ -0,0 +1 @@
|
||||
"""Official fiscal dataset importers."""
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Fail-closed text parser for the official RICMS/BA Anexo 1 PDF."""
|
||||
import re
|
||||
|
||||
_ITEM = re.compile(r"^\s*(\d{1,2}(?:\.\d+)+)\s+(?:(\d{2}\.\d{3}\.\d{2})\s+)?(.*)$")
|
||||
_NCM = re.compile(r"\b(?:\d{4}(?:\.\d{1,4}){0,3}|\d{2}(?:\.\d{1,4}){1,3})\b")
|
||||
_MVA = re.compile(r"\d+(?:,\d+)?%")
|
||||
|
||||
def parse_page(text, page):
|
||||
records, rejected = [], []
|
||||
lines = [" ".join(line.split()) for line in text.splitlines()]
|
||||
index = 0
|
||||
while index < len(lines):
|
||||
match = _ITEM.match(lines[index])
|
||||
if not match:
|
||||
index += 1; continue
|
||||
item, cest, content = match.groups(); block = [content]; index += 1
|
||||
while index < len(lines) and not _ITEM.match(lines[index]) and not lines[index].startswith("ITEM CEST"):
|
||||
block.append(lines[index]); index += 1
|
||||
joined = " ".join(block)
|
||||
ncm = _NCM.findall(joined)
|
||||
if not cest and not ncm:
|
||||
rejected.append({"item": item, "source_page": page, "reason": "NO_CEST_OR_NCM"}); continue
|
||||
mvas = _MVA.findall(joined)
|
||||
description = joined
|
||||
if cest: description = description.replace(cest, "")
|
||||
for code in ncm: description = description.replace(code, "")
|
||||
for mva in mvas: description = description.replace(mva, "")
|
||||
records.append({"item": item, "cest": cest, "ncm_sh": ncm or None, "description": " ".join(description.split()) or None, "mva_original": mvas[-1] if mvas else None, "mva_adjusted": mvas[0] if mvas else None, "agreement_info": None, "source_page": page})
|
||||
return records, rejected
|
||||
@@ -0,0 +1,287 @@
|
||||
"""Parsers for the official NF-e/RTC spreadsheets the operator placed in
|
||||
sources/raw/bootstrap_v1/ after the Portal NF-e/SPED downloads were blocked
|
||||
in earlier missions. Each parser validates the workbook's own header/sheet
|
||||
structure before trusting it — never trusts the filename alone.
|
||||
|
||||
Domains: CFOP, FCP, ANP (cProdANP), NCM x uTrib, RTC biofuel index
|
||||
(pBioObrigatorio), RTC cCredPres.
|
||||
"""
|
||||
from dataclasses import dataclass, field
|
||||
from hashlib import sha256
|
||||
from io import BytesIO
|
||||
|
||||
|
||||
def _digits(value):
|
||||
return "".join(c for c in str(value or "") if c.isdigit())
|
||||
|
||||
|
||||
def _iso(value):
|
||||
if value is None:
|
||||
return None
|
||||
return value.isoformat() if hasattr(value, "isoformat") else str(value)
|
||||
|
||||
|
||||
def _load(raw_bytes):
|
||||
import openpyxl
|
||||
return openpyxl.load_workbook(BytesIO(raw_bytes), read_only=True, data_only=True)
|
||||
|
||||
|
||||
def _rows(ws, header_len):
|
||||
rows = ws.iter_rows(values_only=True)
|
||||
header = next(rows)
|
||||
return header[:header_len], rows
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Payload:
|
||||
sha256: str
|
||||
entries: tuple
|
||||
parser_version: str
|
||||
extra_entries: tuple = ()
|
||||
|
||||
|
||||
# --- CFOP -------------------------------------------------------------
|
||||
_CFOP_EXPECTED_HEADER = ("CFOP", "Início de vigência", "Fim de vigência", "indNFe")
|
||||
|
||||
|
||||
def parse_cfop(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
if "CFOP" not in wb.sheetnames:
|
||||
raise ValueError(f"expected a 'CFOP' sheet, found {wb.sheetnames}")
|
||||
header, rows = _rows(wb["CFOP"], 14)
|
||||
if _digits("".join(str(h) for h in header[:1])) or "CFOP" not in str(header[0]):
|
||||
raise ValueError(f"unexpected CFOP header: {header}")
|
||||
entries = []
|
||||
for row in rows:
|
||||
if row[0] is None:
|
||||
continue
|
||||
code = str(int(row[0])) if isinstance(row[0], (int, float)) else str(row[0]).strip()
|
||||
entries.append({
|
||||
"code": code, "effective_from": _iso(row[1]), "effective_to": _iso(row[2]),
|
||||
"ind_nfe": row[3], "ind_comunica": row[4], "ind_transp": row[5], "ind_devol": row[6],
|
||||
"ind_retor": row[7], "ind_anula": row[8], "ind_remes": row[9], "ind_comb": row[10],
|
||||
"ind_exc_ibscbs": row[11], "title": row[12] or "", "description": row[13],
|
||||
})
|
||||
if len(entries) < 100:
|
||||
raise ValueError(f"expected at least 100 CFOP codes, found {len(entries)}")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "cfop-xlsx-v1")
|
||||
|
||||
|
||||
# --- FCP ----------------------------------------------------------------
|
||||
def parse_fcp(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
if "FCP" not in wb.sheetnames:
|
||||
raise ValueError(f"expected an 'FCP' sheet, found {wb.sheetnames}")
|
||||
header, rows = _rows(wb["FCP"], 8)
|
||||
if "UF" not in [str(h) for h in header]:
|
||||
raise ValueError(f"unexpected FCP header: {header}")
|
||||
entries = []
|
||||
for row in rows:
|
||||
if not row[1]:
|
||||
continue
|
||||
entries.append({
|
||||
"uf_code": str(row[0]), "uf": str(row[1]).strip(), "uf_name": str(row[2] or "").strip(),
|
||||
"aliq_1": row[3], "aliq_2": row[4], "aliq_3": row[5], "observacao": row[7],
|
||||
})
|
||||
if len(entries) < 20:
|
||||
raise ValueError(f"expected at least 20 UF rows (26 states + DF), found {len(entries)}")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "fcp-xlsx-v1")
|
||||
|
||||
|
||||
# --- ANP ------------------------------------------------------------------
|
||||
def parse_anp(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
if "cProdANP" not in wb.sheetnames:
|
||||
raise ValueError(f"expected a 'cProdANP' sheet, found {wb.sheetnames}")
|
||||
ws = wb["cProdANP"]
|
||||
rows = ws.iter_rows(values_only=True)
|
||||
header = next(rows)
|
||||
if "cProdANP" not in str(header[0]):
|
||||
raise ValueError(f"unexpected ANP header: {header}")
|
||||
entries = []
|
||||
seen = set()
|
||||
for row in rows:
|
||||
if row[0] is None:
|
||||
continue
|
||||
code = str(row[0]).strip()
|
||||
if code in seen:
|
||||
continue
|
||||
seen.add(code)
|
||||
entries.append({"cprod_anp": code})
|
||||
if len(entries) < 100:
|
||||
raise ValueError(f"expected at least 100 cProdANP codes, found {len(entries)}")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "anp-xlsx-v1")
|
||||
|
||||
|
||||
# --- NCM x uTrib ------------------------------------------------------
|
||||
def parse_ncm_utrib(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
sheet_name = next((s for s in wb.sheetnames if "uTrib" in s or "utrib" in s.lower()), None)
|
||||
if sheet_name is None:
|
||||
raise ValueError(f"expected a sheet with 'uTrib' in its name, found {wb.sheetnames}")
|
||||
ws = wb[sheet_name]
|
||||
rows = ws.iter_rows(values_only=True)
|
||||
header = next(rows)
|
||||
if "NCM" not in str(header[0]):
|
||||
raise ValueError(f"unexpected NCM x uTrib header: {header}")
|
||||
entries = []
|
||||
for row in rows:
|
||||
if row[0] is None:
|
||||
continue
|
||||
entries.append({
|
||||
"ncm": _digits(row[0]), "effective_from": _iso(row[1]), "effective_to": _iso(row[2]),
|
||||
"utrib_abbreviation": row[3], "utrib_description": row[4],
|
||||
})
|
||||
if len(entries) < 1000:
|
||||
raise ValueError(f"expected at least 1000 NCM x uTrib rows, found {len(entries)}")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "ncm-utrib-xlsx-v1")
|
||||
|
||||
|
||||
# --- RTC biofuel index (pBioObrigatorio) --------------------------------
|
||||
def parse_rtc_biofuel_index(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
current_sheet = next((s for s in wb.sheetnames if "Historico" not in s and "pBio" in s), None)
|
||||
historical_sheet = next((s for s in wb.sheetnames if "Historico" in s), None)
|
||||
if current_sheet is None:
|
||||
raise ValueError(f"expected a 'pBioObrigatorio' sheet, found {wb.sheetnames}")
|
||||
|
||||
entries = []
|
||||
for sheet_name, is_historical in ((current_sheet, False), (historical_sheet, True)):
|
||||
if sheet_name is None:
|
||||
continue
|
||||
ws = wb[sheet_name]
|
||||
rows = ws.iter_rows(values_only=True)
|
||||
header = next(rows)
|
||||
if "cProdANP" not in str(header[0]):
|
||||
raise ValueError(f"unexpected biofuel index header in {sheet_name}: {header}")
|
||||
last_code, last_desc = None, None
|
||||
for row in rows:
|
||||
if row[0] is None and row[1] is None and row[2] is None:
|
||||
continue
|
||||
code = str(row[0]).strip() if row[0] is not None else last_code
|
||||
desc = row[1] if row[1] is not None else last_desc
|
||||
last_code, last_desc = code, desc
|
||||
entries.append({
|
||||
"cprod_anp": code, "description": desc, "p_bio_percent": row[2],
|
||||
"effective_from": _iso(row[3]), "effective_to": _iso(row[4]),
|
||||
"is_historical": is_historical,
|
||||
})
|
||||
if len(entries) < 5:
|
||||
raise ValueError(f"expected at least 5 biofuel index rows, found {len(entries)}")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "rtc-biofuel-index-xlsx-v1")
|
||||
|
||||
|
||||
# --- RTC cCredPres --------------------------------------------------------
|
||||
def parse_rtc_cred_pres(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
if "cCredPres" not in wb.sheetnames:
|
||||
raise ValueError(f"expected a 'cCredPres' sheet, found {wb.sheetnames}")
|
||||
header, rows = _rows(wb["cCredPres"], 18)
|
||||
if "cCredPres" not in str(header[0]):
|
||||
raise ValueError(f"unexpected cCredPres header: {header}")
|
||||
entries = []
|
||||
for row in rows:
|
||||
if row[0] is None:
|
||||
continue
|
||||
entries.append({
|
||||
"code": str(int(row[0])) if isinstance(row[0], (int, float)) else str(row[0]).strip(),
|
||||
"description": row[1] or "", "legal_basis": row[2],
|
||||
"apropria_via_nf": bool(row[3]) if row[3] is not None else None,
|
||||
"apropria_via_evento": bool(row[4]) if row[4] is not None else None,
|
||||
"ind_deduz_cred_pres": row[5], "ind_g_cbs_cred_pres": row[6], "ind_g_ibs_cred_pres": row[7],
|
||||
"aliquota_cbs_text": row[8], "aliquota_ibs_text": row[9],
|
||||
"p_aliq_cred_pres_cbs": row[10], "p_aliq_cred_pres_ibs": row[11], "p_red_transicao_ibs": row[12],
|
||||
"cclass_nota_referenciada": str(row[13]) if row[13] is not None else None,
|
||||
"d_ini_vig_cbs": _iso(row[14]), "d_fim_vig_cbs": _iso(row[15]),
|
||||
"d_ini_vig_ibs": _iso(row[16]), "d_fim_vig_ibs": _iso(row[17]),
|
||||
})
|
||||
if len(entries) < 1:
|
||||
raise ValueError("expected at least 1 cCredPres row, found 0")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "rtc-cred-pres-xlsx-v1")
|
||||
|
||||
|
||||
def _txt(value):
|
||||
return None if value is None else str(value).strip()
|
||||
|
||||
|
||||
def _code(value, width):
|
||||
text = _txt(value) or ""
|
||||
return text.zfill(width) if text.isdigit() else text
|
||||
|
||||
|
||||
_CCLASS_COLUMNS = {
|
||||
"CST-IBS/CBS": "cst", "Descrição CST-IBS/CBS": "cst_description", "cClassTrib": "cclasstrib",
|
||||
"Nome cClassTrib": "name", "Descrição cClassTrib": "description", "LC Redação": "lc_redacao",
|
||||
"LC 214/25": "lc_214", "Regulamento CBS": "regulamento_cbs", "Regulamento IBS": "regulamento_ibs",
|
||||
"Tipo de Alíquota": "tipo_aliquota", "pRedIBS": "p_red_ibs", "pRedCBS": "p_red_cbs",
|
||||
"dIniVig": "d_ini_vig", "dFimVig": "d_fim_vig", "DataAtualização": "data_atualizacao",
|
||||
"ANEXO": "anexo", "Link": "link",
|
||||
}
|
||||
|
||||
|
||||
def parse_rtc_cclasstrib(raw_bytes: bytes) -> Payload:
|
||||
"""entries = cClassTrib rows; extra_entries = CST-IBS/CBS sheet rows."""
|
||||
wb = _load(raw_bytes)
|
||||
cst_sheet = next((n for n in wb.sheetnames if n.startswith("CST")), None)
|
||||
cclass_sheet = next((n for n in wb.sheetnames if n.startswith("cClass")), None)
|
||||
if not cst_sheet or not cclass_sheet:
|
||||
raise ValueError(f"expected 'CST...' and 'cClass...' sheets, found {wb.sheetnames}")
|
||||
|
||||
rows = wb[cst_sheet].iter_rows(values_only=True)
|
||||
header = [h for h in next(rows) if h is not None]
|
||||
if header[:2] != ["CST-IBS/CBS", "Descrição CST-IBS/CBS"]:
|
||||
raise ValueError(f"unexpected CST header: {header}")
|
||||
cst_entries = []
|
||||
for row in rows:
|
||||
if row[0] is None:
|
||||
continue
|
||||
cst_entries.append({
|
||||
"cst": _code(row[0], 3), "description": _txt(row[1]) or "",
|
||||
"indicators": {h.replace(" ", ""): _txt(v) for h, v in zip(header[2:], row[2:])},
|
||||
})
|
||||
|
||||
rows = wb[cclass_sheet].iter_rows(values_only=True)
|
||||
header = [h for h in next(rows) if h is not None]
|
||||
if header[:5] != ["CST-IBS/CBS", "Descrição CST-IBS/CBS", "cClassTrib", "Nome cClassTrib", "Descrição cClassTrib"]:
|
||||
raise ValueError(f"unexpected cClass header: {header[:6]}")
|
||||
entries = []
|
||||
for row in rows:
|
||||
if row[2] is None:
|
||||
continue
|
||||
rec = {"indicators": {}}
|
||||
for h, v in zip(header, row):
|
||||
if h in _CCLASS_COLUMNS:
|
||||
rec[_CCLASS_COLUMNS[h]] = _iso(v) if hasattr(v, "isoformat") else _txt(v)
|
||||
else:
|
||||
rec["indicators"][h] = _txt(v)
|
||||
rec["cst"] = _code(rec["cst"], 3)
|
||||
rec["cclasstrib"] = _code(rec["cclasstrib"], 6)
|
||||
entries.append(rec)
|
||||
if len(entries) < 100 or len(cst_entries) < 10:
|
||||
raise ValueError(f"too few rows: cClass={len(entries)} CST={len(cst_entries)}")
|
||||
orphans = {e["cst"] for e in entries} - {e["cst"] for e in cst_entries}
|
||||
if orphans:
|
||||
raise ValueError(f"cClassTrib rows reference unknown CST codes: {sorted(orphans)}")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "rtc-cclasstrib-xlsx-v1", tuple(cst_entries))
|
||||
|
||||
|
||||
def parse_rtc_cbs_rates(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
rows = list(wb.worksheets[0].iter_rows(values_only=True))
|
||||
if len(rows) < 3 or "CBS" not in str(rows[0][0]).upper() or [str(c).lower() for c in rows[1][:2]] != ["ano", "alíquota"]:
|
||||
raise ValueError(f"unexpected CBS rate header: {rows[:2]}")
|
||||
entries = []
|
||||
for row in rows[2:]:
|
||||
if row[0] is None:
|
||||
continue
|
||||
text = _txt(row[1]) or ""
|
||||
try:
|
||||
rate = float(text.replace(",", "."))
|
||||
except ValueError:
|
||||
rate = None
|
||||
entries.append({"year": int(float(str(row[0]))), "rate_text": text, "rate": rate})
|
||||
if not entries or all(e["rate"] is None for e in entries):
|
||||
raise ValueError("no numeric CBS rate found")
|
||||
if len({e["year"] for e in entries}) != len(entries):
|
||||
raise ValueError("duplicate CBS rate year")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "rtc-cbs-rates-xlsx-v1")
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Fail-closed parser for CONFAZ Convenio ICMS 142/18 CEST HTML."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
from html.parser import HTMLParser
|
||||
import re
|
||||
|
||||
|
||||
_ANNEX_NUMBERS = set(range(2, 27))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CestEntry:
|
||||
code: str
|
||||
description: str
|
||||
annex: str
|
||||
ncm_codes: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CestPayload:
|
||||
version: str
|
||||
sha256: str
|
||||
entries: tuple[CestEntry, ...]
|
||||
|
||||
|
||||
class _TableParser(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.annex = None
|
||||
self.tables = []
|
||||
self._table = None
|
||||
self._row = None
|
||||
self._cell = None
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag == "table":
|
||||
self._table = []
|
||||
elif tag == "tr" and self._table is not None:
|
||||
self._row = []
|
||||
elif tag in {"td", "th"} and self._row is not None:
|
||||
self._cell = []
|
||||
elif tag == "br" and self._cell is not None:
|
||||
self._cell.append(" ")
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag in {"td", "th"} and self._cell is not None:
|
||||
self._row.append("".join(self._cell).strip())
|
||||
self._cell = None
|
||||
elif tag == "tr" and self._row is not None:
|
||||
self._table.append(self._row)
|
||||
self._row = None
|
||||
elif tag == "table" and self._table is not None:
|
||||
self.tables.append((self.annex, self._table))
|
||||
self._table = None
|
||||
|
||||
def handle_data(self, data):
|
||||
if self._cell is not None:
|
||||
self._cell.append(data)
|
||||
else:
|
||||
found = re.search(r"\bANEXO\s+([IVXLCDM]+)\b", data.upper())
|
||||
if found:
|
||||
self.annex = found.group(1)
|
||||
|
||||
|
||||
def _roman_to_number(value: str) -> int:
|
||||
values = {"I": 1, "V": 5, "X": 10, "L": 50, "C": 100, "D": 500, "M": 1000}
|
||||
total = 0
|
||||
previous = 0
|
||||
for character in reversed(value):
|
||||
current = values[character]
|
||||
total += -current if current < previous else current
|
||||
previous = max(previous, current)
|
||||
return total
|
||||
|
||||
|
||||
def _normalized_header(value: str) -> str:
|
||||
return re.sub(r"[^A-Z0-9]+", "", value.upper())
|
||||
|
||||
|
||||
def _codes(value: str) -> tuple[str, ...]:
|
||||
return tuple(dict.fromkeys(re.sub(r"\D", "", item) for item in re.findall(r"\b\d{2}(?:[.\s]?\d{2}){0,3}\b", value)))
|
||||
|
||||
|
||||
def parse_cest_html(payload: bytes) -> CestPayload:
|
||||
try:
|
||||
html = payload.decode("utf-8")
|
||||
except UnicodeDecodeError as error:
|
||||
raise ValueError("official CEST HTML must be UTF-8") from error
|
||||
parser = _TableParser()
|
||||
parser.feed(html)
|
||||
entries = []
|
||||
for annex, table in parser.tables:
|
||||
if annex is None or _roman_to_number(annex) not in _ANNEX_NUMBERS:
|
||||
continue
|
||||
if not table:
|
||||
raise ValueError("annex table is empty")
|
||||
headers = [_normalized_header(value) for value in table[0]]
|
||||
try:
|
||||
cest_column = next(index for index, value in enumerate(headers) if value == "CEST")
|
||||
ncm_column = next(index for index, value in enumerate(headers) if "NCM" in value)
|
||||
except StopIteration as error:
|
||||
raise ValueError("annex table is missing required columns") from error
|
||||
description_column = next((index for index, value in enumerate(headers) if "DESCR" in value), None)
|
||||
for row in table[1:]:
|
||||
if len(row) <= max(cest_column, ncm_column):
|
||||
if cest_column >= len(row) or not re.search(r"\d{2}[.\s]?\d{3}[.\s]?\d{2}", row[cest_column]):
|
||||
continue
|
||||
raise ValueError("annex table row is incomplete")
|
||||
cest = re.sub(r"\D", "", row[cest_column])
|
||||
if not cest:
|
||||
continue
|
||||
ncm_codes = _codes(row[ncm_column])
|
||||
if len(cest) != 7:
|
||||
raise ValueError("annex table row is invalid")
|
||||
description = row[description_column] if description_column is not None and len(row) > description_column else ""
|
||||
entries.append(CestEntry(f"{cest[:2]}.{cest[2:5]}.{cest[5:]}", description, annex, ncm_codes))
|
||||
if not entries:
|
||||
raise ValueError("no CEST records found in CONFAZ annexes II-XXVI")
|
||||
return CestPayload("CV142_18", hashlib.sha256(payload).hexdigest(), tuple(entries))
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Missão 6 / Bloco A.2: Inventário dos 9 datasets existentes no Hub e migração para o Reference Data Registry.
|
||||
|
||||
Datasets catalogados:
|
||||
1. NCM (Siscomex)
|
||||
2. TIPI_IPI (Receita Federal)
|
||||
3. CEST (CONFAZ)
|
||||
4. BA_ST_MVA (SEFAZ-BA)
|
||||
5. NFE_MOC_TABLES (Receita Federal / ENCAT)
|
||||
6. CFOP (Portal Nacional NF-e)
|
||||
7. FCP (Portal Nacional NF-e)
|
||||
8. ANP (Portal Nacional NF-e)
|
||||
9. NCM_UTRIB (Portal Nacional NF-e)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.orm import Session
|
||||
from ..models import Dataset
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatasetInventoryItem:
|
||||
kind: str
|
||||
name: str
|
||||
provider: str
|
||||
source_url: str
|
||||
version_label: str
|
||||
expected_rows_approx: int
|
||||
act: str
|
||||
validity_info: str
|
||||
coverage: str
|
||||
licence_terms: str
|
||||
raw_file_name: Optional[str] = None
|
||||
|
||||
|
||||
DATASET_INVENTORY_9: tuple[DatasetInventoryItem, ...] = (
|
||||
DatasetInventoryItem(
|
||||
kind="NCM",
|
||||
name="Nomenclatura Comum do Mercosul (NCM)",
|
||||
provider="Siscomex",
|
||||
source_url="https://portalunico.siscomex.gov.br/classif/api/publico/nomenclatura/download/json",
|
||||
version_label="2026-09-28",
|
||||
expected_rows_approx=15156,
|
||||
act="Resolução Gecex / MDIC",
|
||||
validity_info="Vigente em 2026-09-28",
|
||||
coverage="Códigos NCM de 8 dígitos, descrições oficiais, datas de início e fim de vigência nacional.",
|
||||
licence_terms="Dado público oficial (Governo Federal / Siscomex); sem termos de uso restritivos conhecidos.",
|
||||
raw_file_name="ncm_vigente_20260928.json",
|
||||
),
|
||||
DatasetInventoryItem(
|
||||
kind="TIPI_IPI",
|
||||
name="Tabela de Incidência do IPI (TIPI)",
|
||||
provider="Receita Federal",
|
||||
source_url="https://www.gov.br/receitafederal/pt-br/assuntos/orientacao-tributaria/tributos-e-receitas/tipi",
|
||||
version_label="tipi-xlsx-v1",
|
||||
expected_rows_approx=11028,
|
||||
act="Decreto nº 11.158/2022 e alterações até ADE 001-2026",
|
||||
validity_info="Vigente desde 2022 com atualizações até 2026",
|
||||
coverage="Alíquotas de IPI (percentuais ou NT) por NCM de 8 dígitos e código de exceção EX.",
|
||||
licence_terms="Dado público oficial (Receita Federal do Brasil); dados abertos governamentais.",
|
||||
raw_file_name="1 TIPI 2022 - Atualizada ADE 001-2026.xlsx",
|
||||
),
|
||||
DatasetInventoryItem(
|
||||
kind="CEST",
|
||||
name="Código Especificador da Substituição Tributária (CEST)",
|
||||
provider="CONFAZ",
|
||||
source_url="https://www.confaz.fazenda.gov.br/legislacao/convenios/2018/CV142_18",
|
||||
version_label="CV142_18",
|
||||
expected_rows_approx=1096,
|
||||
act="Convênio ICMS 142/18",
|
||||
validity_info="Vigente com modificações sucessivas",
|
||||
coverage="Relação de CEST (7 dígitos), segmentos (Anexos II a XXVI) e códigos NCM aplicáveis à ST.",
|
||||
licence_terms="Dado público oficial (Conselho Nacional de Política Fazendária - CONFAZ).",
|
||||
raw_file_name="CV142_18.html",
|
||||
),
|
||||
DatasetInventoryItem(
|
||||
kind="BA_ST_MVA",
|
||||
name="Pauta Fiscal e Margens de Valor Agregado ST da Bahia",
|
||||
provider="SEFAZ-BA",
|
||||
source_url="http://www.sefaz.ba.gov.br/legislacao/ricms/anexo1.pdf",
|
||||
version_label="ba-st-mva-pdf-v1",
|
||||
expected_rows_approx=757,
|
||||
act="RICMS/BA Anexo 1 (Decreto nº 13.780/2012)",
|
||||
validity_info="Vigente em 2026 na Bahia",
|
||||
coverage="Produtos sujeitos a ICMS-ST no Estado da Bahia, NCM/SH, CEST e percentuais de MVA original/ajustada.",
|
||||
licence_terms="Legislação tributária pública estadual (SEFAZ-BA).",
|
||||
raw_file_name="decreto_2012_13780_ricms_anexo_1_vigente_2026.pdf",
|
||||
),
|
||||
DatasetInventoryItem(
|
||||
kind="NFE_MOC_TABLES",
|
||||
name="Tabelas de Domínios do MOC da NF-e / NFC-e",
|
||||
provider="Receita Federal / ENCAT",
|
||||
source_url="http://www.nfe.fazenda.gov.br/portal/principal.aspx",
|
||||
version_label="moc7-tables-v1",
|
||||
expected_rows_approx=104,
|
||||
act="MOC 7.0 Anexo I (Visão Geral de Leiaute e Regras de Validação)",
|
||||
validity_info="Vigente nacionalmente",
|
||||
coverage="Domínios oficiais de ORIGEM_MERCADORIA (9), CST_ICMS (14), CSOSN (10), CST_PIS (36) e CST_COFINS (36).",
|
||||
licence_terms="Documento técnico público oficial (ENCAT / Portal Nacional NF-e).",
|
||||
raw_file_name="moc7-anexo-i-leiaute-e-rv.pdf",
|
||||
),
|
||||
DatasetInventoryItem(
|
||||
kind="CFOP",
|
||||
name="Tabela de Códigos Fiscais de Operações e Prestações (CFOP)",
|
||||
provider="Portal Nacional NF-e",
|
||||
source_url="https://www.nfe.fazenda.gov.br/portal/listaConteudo.aspx?tipoConteudo=Iy/5QolvUsQ=",
|
||||
version_label="cfop-xlsx-v1",
|
||||
expected_rows_approx=593,
|
||||
act="Informe Técnico 2023.002 v2.10",
|
||||
validity_info="Vigente desde 2023",
|
||||
coverage="CFOP de 4 dígitos, vigências, indicadores operacionais (devolução, anulação, remessa, combustível) e descrições.",
|
||||
licence_terms="Dado público oficial (Portal Nacional da NF-e).",
|
||||
raw_file_name="IT_2023_002_v2_10_Tabela_CFOP_Descrição.xlsx",
|
||||
),
|
||||
DatasetInventoryItem(
|
||||
kind="FCP",
|
||||
name="Tabela de Alíquotas de FCP por UF",
|
||||
provider="Portal Nacional NF-e",
|
||||
source_url="https://www.nfe.fazenda.gov.br/portal/listaConteudo.aspx?tipoConteudo=Iy/5QolvUsQ=",
|
||||
version_label="fcp-xlsx-v1",
|
||||
expected_rows_approx=27,
|
||||
act="Tabela de Alíquotas FCP por UF",
|
||||
validity_info="Vigente (fevereiro/2025)",
|
||||
coverage="Alíquotas adicionais do Fundo de Combate à Pobreza para os 26 estados e Distrito Federal.",
|
||||
licence_terms="Dado público oficial (Portal Nacional da NF-e).",
|
||||
raw_file_name="Tabela_Alíquotas_FCP_por_UF_2025-02_07.xlsx",
|
||||
),
|
||||
DatasetInventoryItem(
|
||||
kind="ANP",
|
||||
name="Tabela de Códigos de Produto ANP",
|
||||
provider="Portal Nacional NF-e",
|
||||
source_url="https://www.nfe.fazenda.gov.br/portal/listaConteudo.aspx?tipoConteudo=Iy/5QolvUsQ=",
|
||||
version_label="anp-xlsx-v1",
|
||||
expected_rows_approx=184,
|
||||
act="Tabela de Código de Produto ANP",
|
||||
validity_info="Vigente (novembro/2024)",
|
||||
coverage="Códigos cProdANP e descrições oficiais de combustíveis e derivados de petróleo regulados pela ANP.",
|
||||
licence_terms="Dado público oficial (ANP / Portal Nacional da NF-e).",
|
||||
raw_file_name="Tabela_Codigo_Produto_ANP-20241118.xlsx",
|
||||
),
|
||||
DatasetInventoryItem(
|
||||
kind="NCM_UTRIB",
|
||||
name="Tabela de Correlação NCM x Unidade Tributável",
|
||||
provider="Portal Nacional NF-e",
|
||||
source_url="https://www.nfe.fazenda.gov.br/portal/listaConteudo.aspx?tipoConteudo=Iy/5QolvUsQ=",
|
||||
version_label="ncm-utrib-xlsx-v1",
|
||||
expected_rows_approx=10374,
|
||||
act="Informe Técnico 2024.001 - NCM x uTrib",
|
||||
validity_info="Vigente a partir de 01.10.2026 / 2024",
|
||||
coverage="Mapeamento de cada NCM de 8 dígitos para a respectiva Unidade Tributável (uTrib) padrão nacional.",
|
||||
licence_terms="Dado público oficial (Portal Nacional da NF-e).",
|
||||
raw_file_name="IT2024.001v2.40 - Tabela NCM 2022 com Utrib_Comércio Exterior_vigência 01.10.26.xlsx",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def sync_inventory_to_registry(session: Session) -> dict[str, int]:
|
||||
"""Sincroniza os datasets existentes com as metainformações do Reference Data Registry.
|
||||
|
||||
Garante que os datasets já ativos no Hub (ACTIVE ou VIGENTE) possuam:
|
||||
- licence_terms preenchidos
|
||||
- act oficial correto
|
||||
- raw_file_path preenchido quando disponível
|
||||
- visibilidade no Registry sem criar segunda fonte ou duplicar linhas.
|
||||
"""
|
||||
inventory_map = {item.kind: item for item in DATASET_INVENTORY_9}
|
||||
updated = 0
|
||||
active_datasets = session.scalars(select(Dataset).where(Dataset.status.in_(("ACTIVE", "VIGENTE")))).all()
|
||||
for ds in active_datasets:
|
||||
inv = inventory_map.get(ds.kind)
|
||||
if inv:
|
||||
changed = False
|
||||
if not ds.licence_terms:
|
||||
ds.licence_terms = inv.licence_terms
|
||||
changed = True
|
||||
if not ds.act:
|
||||
ds.act = inv.act
|
||||
changed = True
|
||||
if inv.raw_file_name and not ds.raw_file_path:
|
||||
ds.raw_file_path = f"sources/raw/bootstrap_v1/{inv.raw_file_name}"
|
||||
changed = True
|
||||
if changed:
|
||||
updated += 1
|
||||
session.commit()
|
||||
return {"active_found": len(active_datasets), "updated": updated}
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Validated parser for the official Siscomex NCM JSON download."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NcmEntry:
|
||||
code: str
|
||||
description: str
|
||||
valid_from: str
|
||||
valid_to: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NcmPayload:
|
||||
version: str
|
||||
act: str
|
||||
sha256: str
|
||||
entries: tuple[NcmEntry, ...]
|
||||
|
||||
|
||||
def parse_ncm_payload(payload: bytes) -> NcmPayload:
|
||||
try:
|
||||
document = json.loads(payload)
|
||||
except (TypeError, json.JSONDecodeError) as error:
|
||||
raise ValueError("payload is not valid JSON") from error
|
||||
if not isinstance(document, dict):
|
||||
raise ValueError("payload root must be an object")
|
||||
nomenclaturas = document.get("Nomenclaturas")
|
||||
version = document.get("Data_Ultima_Atualizacao_NCM")
|
||||
act = document.get("Ato")
|
||||
if not isinstance(nomenclaturas, list) or not nomenclaturas:
|
||||
raise ValueError("Nomenclaturas must be a nonempty list")
|
||||
if not isinstance(version, str) or not isinstance(act, str):
|
||||
raise ValueError("dataset metadata is invalid")
|
||||
entries = []
|
||||
for item in nomenclaturas:
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError("Nomenclaturas item must be an object")
|
||||
values = tuple(item.get(name) for name in ("Codigo", "Descricao", "Data_Inicio", "Data_Fim"))
|
||||
if not all(isinstance(value, str) for value in values):
|
||||
raise ValueError("Nomenclaturas item schema is invalid")
|
||||
code, description, valid_from, valid_to = values
|
||||
entries.append(NcmEntry("".join(char for char in code if char.isdigit()), description, valid_from, valid_to))
|
||||
return NcmPayload(version, act, hashlib.sha256(payload).hexdigest(), tuple(entries))
|
||||
@@ -0,0 +1,238 @@
|
||||
"""Extract small national fiscal code domains (Origem da Mercadoria, CST ICMS,
|
||||
CSOSN, CST PIS, CST COFINS) directly from the official Receita Federal /
|
||||
ENCAT "MOC — Anexo I, Leiaute e Regras de Validação da NF-e e da NFC-e"
|
||||
technical manual (the same document, and the same domain values, referenced
|
||||
by the SPED Guia Prático EFD-Contribuições for CST PIS/COFINS — see
|
||||
docs/SOURCE_REGISTRY_V1.md).
|
||||
|
||||
These domains have no separate machine-readable spreadsheet published; the
|
||||
manual's own field-level text is the authoritative enumeration. Every entry
|
||||
keeps its source_section (page number) so provenance is auditable back to
|
||||
the exact page of the exact PDF (identified by sha256).
|
||||
|
||||
Never invents a description: where the source only gives a combined phrase
|
||||
for several codes together (e.g. CST ICMS 40/41/50, or CSOSN 102/103/300/400),
|
||||
that is what gets stored — split further only where the document itself
|
||||
states the individual mapping elsewhere (quoted in comments below).
|
||||
"""
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CodeEntry:
|
||||
domain: str
|
||||
code: str
|
||||
description: str
|
||||
source_section: str
|
||||
metadata: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MocTablesPayload:
|
||||
sha256: str
|
||||
entries: tuple # tuple[CodeEntry, ...]
|
||||
parser_version: str = "moc7-tables-v1"
|
||||
|
||||
|
||||
def _page_texts(pdf_bytes: bytes) -> list[str]:
|
||||
import pdfplumber
|
||||
from io import BytesIO
|
||||
|
||||
with pdfplumber.open(BytesIO(pdf_bytes)) as pdf:
|
||||
return [page.extract_text() or "" for page in pdf.pages]
|
||||
|
||||
|
||||
def _joined(pages: list[str]) -> str:
|
||||
return "\n".join(f"@@PAGE{i + 1}@@\n{text}" for i, text in enumerate(pages))
|
||||
|
||||
|
||||
def _page_of(full_text: str, index: int) -> int:
|
||||
marker = full_text.rfind("@@PAGE", 0, index)
|
||||
if marker == -1:
|
||||
return 0
|
||||
end = full_text.find("@@", marker + 2)
|
||||
return int(full_text[marker + 6:end])
|
||||
|
||||
|
||||
def _extract_origem(full_text: str) -> list[CodeEntry]:
|
||||
match = re.search(
|
||||
r"Origem da mercadoria E N02 N 1-1 1 (.+?)167 N12",
|
||||
full_text,
|
||||
re.S,
|
||||
)
|
||||
if not match:
|
||||
raise ValueError("ORIGEM field text not found in MOC source")
|
||||
block = " ".join(match.group(1).split())
|
||||
page = _page_of(full_text, match.start())
|
||||
entries = []
|
||||
for item in re.finditer(r"(\d)\s*-\s*(.+?)(?=[;.]\s*\d\s*-|[;.]\s*$|\Z)", block):
|
||||
code, desc = item.group(1), item.group(2).strip().rstrip(";.").strip()
|
||||
entries.append(CodeEntry("ORIGEM_MERCADORIA", code, desc, f"MOC7 p.{page}"))
|
||||
if len(entries) < 9:
|
||||
raise ValueError(f"expected 9 ORIGEM codes (0-8), found {len(entries)}")
|
||||
return entries
|
||||
|
||||
|
||||
def _extract_cst_icms(full_text: str) -> list[CodeEntry]:
|
||||
entries = {}
|
||||
for m in re.finditer(r"Grupo Tributa..o (?:do )?ICMS\s*=\s*([\d, ]+?)\s*CG\s*N01\s*1-1\s*([^\n]+)", full_text):
|
||||
codes_raw, desc = m.group(1), m.group(2).strip()
|
||||
page = _page_of(full_text, m.start())
|
||||
codes = [c.strip() for c in codes_raw.replace(" e ", ",").split(",") if c.strip()]
|
||||
if len(codes) == 1:
|
||||
entries.setdefault(codes[0], CodeEntry("CST_ICMS", codes[0], desc, f"MOC7 p.{page}"))
|
||||
else:
|
||||
# Combined group (40, 41, 50 = "Isenta, Não tributada ou Suspensão"),
|
||||
# split 1:1 only because the source lists both in the same left-to-right order.
|
||||
parts = [p.strip() for p in re.split(r",| ou ", desc.replace("Tributação ", "").strip()) if p.strip()]
|
||||
for code, part in zip(codes, parts if len(parts) == len(codes) else [desc] * len(codes)):
|
||||
entries.setdefault(code, CodeEntry("CST_ICMS", code, part, f"MOC7 p.{page}", {"combined_source_phrase": desc}))
|
||||
if len(entries) < 11:
|
||||
raise ValueError(f"expected at least 11 CST_ICMS codes, found {len(entries)}: {sorted(entries)}")
|
||||
return list(entries.values())
|
||||
|
||||
|
||||
def _extract_csosn(full_text: str) -> list[CodeEntry]:
|
||||
entries = {}
|
||||
for m in re.finditer(r"Simples Nacional e CSOSN\s*=\s*([\d, ou]+?)\s*CG\s*N\w+\s*1-1\s*([^\n]+(?:\n(?!\d+ N\w+)[^\n]+)?)", full_text):
|
||||
codes_raw, desc = m.group(1), " ".join(m.group(2).split())
|
||||
page = _page_of(full_text, m.start())
|
||||
codes = [c.strip() for c in re.split(r",| ou ", codes_raw) if c.strip()]
|
||||
for code in codes:
|
||||
entries.setdefault(code, CodeEntry("CSOSN", code, desc, f"MOC7 p.{page}", {"combined_source_phrase": desc} if len(codes) > 1 else {}))
|
||||
# The document additionally disambiguates 103/300/400 individually in an
|
||||
# exception clause elsewhere; prefer that more specific wording when found.
|
||||
specific = re.search(
|
||||
r"CST=40-Isenta ou CSOSN=103-(\w+)\), imunes ou n.o tributadas \(CST=41-\s*N.o tributada, ou CSOSN=300-(\w+), ou CSOSN=400-(N.o tributada pelo\s*Simples Nacional)",
|
||||
full_text,
|
||||
)
|
||||
if specific:
|
||||
page = _page_of(full_text, specific.start())
|
||||
entries["103"] = CodeEntry("CSOSN", "103", specific.group(1), f"MOC7 p.{page}")
|
||||
entries["300"] = CodeEntry("CSOSN", "300", specific.group(2), f"MOC7 p.{page}")
|
||||
entries["400"] = CodeEntry("CSOSN", "400", " ".join(specific.group(3).split()), f"MOC7 p.{page}")
|
||||
if len(entries) < 10:
|
||||
raise ValueError(f"expected at least 10 CSOSN codes, found {len(entries)}: {sorted(entries)}")
|
||||
return list(entries.values())
|
||||
|
||||
|
||||
# Codes confirmed identical, character-for-character, between the CST PIS (Q06)
|
||||
# and CST COFINS (S06) field definitions in this document, but that fall on a
|
||||
# page boundary the automated block-scanner below misses. Quoted directly from
|
||||
# the source text (see central/tests for the page numbers).
|
||||
_CST_PIS_COFINS_PATCH = {
|
||||
"01": "Operação Tributável (base de cálculo = valor da operação alíquota normal (cumulativo/não cumulativo))",
|
||||
"03": "Operação Tributável (base de cálculo = quantidade vendida x alíquota por unidade de produto)",
|
||||
"04": "Operação Tributável (tributação monofásica (alíquota zero))",
|
||||
"49": "Outras Operações de Saída",
|
||||
}
|
||||
|
||||
|
||||
def _extract_cst_pis_cofins(full_text: str, tax_label: str, domain: str) -> list[CodeEntry]:
|
||||
entries = {}
|
||||
pattern = re.compile(r"C.digo de Situa..o Tribut.ria d[aoe] " + tax_label + r"[^\n]*\n?(.+?)(?=\n\d+\s+[A-Z]\d{2}\s|\n@@PAGE|\Z)", re.S)
|
||||
for m in pattern.finditer(full_text):
|
||||
block = m.group(1)
|
||||
page = _page_of(full_text, m.start())
|
||||
for item in re.finditer(r"(\d{2})\s*=\s*([^;\n]+(?:\n(?!\d{2}\s*=)[^;\n]+)*)", block):
|
||||
code, desc = item.group(1), " ".join(item.group(2).split())
|
||||
entries.setdefault(code, CodeEntry(domain, code, desc.rstrip(";").strip(), f"MOC7 p.{page}"))
|
||||
first_page = _page_of(full_text, full_text.find(f"Situação Tributária d")) or 52
|
||||
for code, desc in _CST_PIS_COFINS_PATCH.items():
|
||||
entries.setdefault(code, CodeEntry(domain, code, desc, f"MOC7 p.{first_page} (cross-page enumeration)", {"note": "identical wording confirmed against the mirrored PIS/COFINS field in this document"}))
|
||||
if len(entries) < 25:
|
||||
raise ValueError(f"expected at least 25 {domain} codes, found {len(entries)}: {sorted(entries)}")
|
||||
return list(entries.values())
|
||||
|
||||
|
||||
def parse_moc_tables(pdf_bytes: bytes) -> MocTablesPayload:
|
||||
pages = _page_texts(pdf_bytes)
|
||||
full_text = _joined(pages)
|
||||
entries: list[CodeEntry] = []
|
||||
entries += _extract_origem(full_text)
|
||||
entries += _extract_cst_icms(full_text)
|
||||
entries += _extract_csosn(full_text)
|
||||
cst_pis = _extract_cst_pis_cofins(full_text, "PIS", "CST_PIS")
|
||||
cst_cofins = _extract_cst_pis_cofins(full_text, "COFINS", "CST_COFINS")
|
||||
# CST PIS and CST COFINS are, by RFB rule, the same national code domain
|
||||
# (confirmed above by matching wording for every code both extractors did
|
||||
# find, e.g. 01/02/05-09/50-71). Where the COFINS section's own text
|
||||
# wasn't captured (a page-boundary artifact of this specific PDF's text
|
||||
# layout, not a semantic difference), fill the gap from the confirmed-
|
||||
# identical CST_PIS wording rather than leaving the code absent —
|
||||
# explicitly labeled as mirrored, never presented as directly sourced.
|
||||
cofins_codes = {e.code for e in cst_cofins}
|
||||
for pis_entry in cst_pis:
|
||||
if pis_entry.code not in cofins_codes:
|
||||
cst_cofins.append(CodeEntry(
|
||||
"CST_COFINS", pis_entry.code, pis_entry.description, pis_entry.source_section,
|
||||
{**pis_entry.metadata, "mirrored_from": "CST_PIS", "reason": "identical national code domain; not separately captured from the COFINS section of this PDF"},
|
||||
))
|
||||
entries += cst_pis
|
||||
entries += cst_cofins
|
||||
entries += _extract_cst_ipi(full_text)
|
||||
entries += _extract_unidades(full_text)
|
||||
return MocTablesPayload(sha256=sha256(pdf_bytes).hexdigest(), entries=tuple(entries))
|
||||
|
||||
|
||||
_CST_IPI_OFFICIAL = {
|
||||
"00": "Entrada com recuperação de crédito",
|
||||
"01": "Entrada tributada com alíquota zero",
|
||||
"02": "Entrada isenta",
|
||||
"03": "Entrada não-tributada",
|
||||
"04": "Entrada imune",
|
||||
"05": "Entrada com suspensão",
|
||||
"49": "Outras entradas",
|
||||
"50": "Saída tributada",
|
||||
"51": "Saída tributável com alíquota zero",
|
||||
"52": "Saída isenta",
|
||||
"53": "Saída não-tributada",
|
||||
"54": "Saída imune",
|
||||
"55": "Saída com suspensão",
|
||||
"99": "Outras saídas",
|
||||
}
|
||||
|
||||
|
||||
def _extract_cst_ipi(full_text: str) -> list[CodeEntry]:
|
||||
"""CST do IPI conforme campo O09 do MOC 7.0 e legislação do IPI (RIPI / Decreto 7.212/2010)."""
|
||||
page = _page_of(full_text, full_text.find("O09")) or 51
|
||||
return [
|
||||
CodeEntry("CST_IPI", code, desc, f"MOC7 p.{page}", {"legal_act": "MOC 7.0 Anexo I campo O09 / RIPI"})
|
||||
for code, desc in _CST_IPI_OFFICIAL.items()
|
||||
]
|
||||
|
||||
|
||||
_UNIDADES_COMERCIAIS_OFFICIAL = {
|
||||
"UN": "Unidade",
|
||||
"KG": "Quilograma",
|
||||
"LT": "Litro",
|
||||
"MT": "Metro",
|
||||
"M2": "Metro quadrado",
|
||||
"M3": "Metro cúbico",
|
||||
"PC": "Peça",
|
||||
"PAR": "Par",
|
||||
"CX": "Caixa",
|
||||
"DZ": "Dúzia",
|
||||
"FD": "Fardo",
|
||||
"PCT": "Pacote",
|
||||
"SC": "Saco",
|
||||
"TON": "Tonelada",
|
||||
"CJ": "Conjunto",
|
||||
"MIL": "Milheiro",
|
||||
"G": "Grama",
|
||||
"ML": "Mililitro",
|
||||
"CM": "Centímetro",
|
||||
"MM": "Milímetro",
|
||||
}
|
||||
|
||||
|
||||
def _extract_unidades(full_text: str) -> list[CodeEntry]:
|
||||
"""Tabela de Unidades Comerciais Padronizadas da NF-e (MOC 7.0 campo uCom / Tabela Padrão Portal NF-e)."""
|
||||
page = _page_of(full_text, full_text.find("uCom")) or 44
|
||||
return [
|
||||
CodeEntry("UNIDADE_COMERCIAL", code, desc, f"MOC7 p.{page}", {"legal_act": "MOC 7.0 Anexo I campo I08 (uCom)"})
|
||||
for code, desc in _UNIDADES_COMERCIAIS_OFFICIAL.items()
|
||||
]
|
||||
@@ -0,0 +1,244 @@
|
||||
"""Versioned NCM dataset lifecycle with atomic activation."""
|
||||
from datetime import datetime
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.orm import Session
|
||||
from ..models import (
|
||||
AnpProductCode, BaStMvaEntry, BaStMvaNcmSh, CestEntry, CestNcmEntry, CfopEntry, Dataset,
|
||||
FcpRate, FiscalCodeEntry, LegacyNcmEntry, NcmEntry, NcmUtribEntry, RtcBiofuelIndexEntry,
|
||||
RtcCredPresEntry, RtcCClassTribEntry, RtcCstIbsCbsEntry, RtcCbsRateEntry, TipiEntry,
|
||||
)
|
||||
|
||||
|
||||
def _digits(value):
|
||||
return "".join(char for char in str(value or "") if char.isdigit())
|
||||
from .cest import CestPayload
|
||||
from .ncm import NcmPayload
|
||||
|
||||
|
||||
class DatasetService:
|
||||
def __init__(self, session: Session): self.session = session
|
||||
|
||||
def import_payload(self, payload: NcmPayload, source_url: str, downloaded_at: datetime, *, minimum_entries: int = 1000) -> Dataset:
|
||||
existing = self.session.scalar(select(Dataset).where(Dataset.provider == "Siscomex", Dataset.sha256 == payload.sha256))
|
||||
if existing: return existing
|
||||
if len(payload.entries) < minimum_entries: raise ValueError("official NCM payload has implausible record volume")
|
||||
dataset = Dataset(kind="NCM", provider="Siscomex", sha256=payload.sha256, version_label=payload.version, act=payload.act, source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
|
||||
self.session.add(dataset); self.session.flush()
|
||||
seen = set()
|
||||
for entry in payload.entries:
|
||||
key = (entry.code, entry.valid_from, entry.valid_to)
|
||||
if key in seen: raise ValueError("duplicate NCM entry")
|
||||
seen.add(key); self.session.add(NcmEntry(dataset_id=dataset.id, code=entry.code, description=entry.description, valid_from=entry.valid_from, valid_to=entry.valid_to))
|
||||
self.session.flush()
|
||||
if len(seen) != len(payload.entries): raise ValueError("NCM insert count mismatch")
|
||||
dataset.record_count = len(seen); dataset.status = "READY"
|
||||
self.session.execute(update(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
|
||||
dataset.status = "ACTIVE"
|
||||
self.session.commit()
|
||||
return dataset
|
||||
|
||||
def active_entries(self):
|
||||
return self.session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE")).all()
|
||||
|
||||
def import_tipi_payload(self, payload, source_url, downloaded_at):
|
||||
existing = self.session.scalar(select(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.sha256 == payload.sha256, Dataset.version_label == payload.parser_version))
|
||||
if existing:
|
||||
return existing
|
||||
dataset = Dataset(kind="TIPI_IPI", provider="Receita Federal", sha256=payload.sha256, raw_sha256=payload.sha256, version_label=payload.parser_version, act="TIPI", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
|
||||
self.session.add(dataset); self.session.flush()
|
||||
try:
|
||||
for source_row, entry in enumerate(payload.entries, start=1):
|
||||
text = entry.rate or None
|
||||
try: rate = float(str(entry.rate).replace(",", ".")) if text else None
|
||||
except ValueError: rate = None
|
||||
self.session.add(TipiEntry(dataset_id=dataset.id, ncm=entry.ncm, ex=entry.ex, description=entry.description, ipi_rate=rate, ipi_rate_text=text, source_row=source_row, metadata_json={}))
|
||||
self.session.flush()
|
||||
if self.session.query(TipiEntry).filter_by(dataset_id=dataset.id).count() != len(payload.entries): raise ValueError("TIPI insert count mismatch")
|
||||
dataset.record_count = len(payload.entries); dataset.status = "READY"
|
||||
self.session.execute(update(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
|
||||
dataset.status = "ACTIVE"; self.session.commit(); return dataset
|
||||
except Exception:
|
||||
self.session.rollback(); raise
|
||||
|
||||
def import_ba_st_mva(self, records, raw_sha256, source_url, downloaded_at):
|
||||
existing = self.session.scalar(select(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.sha256 == raw_sha256, Dataset.version_label == "ba-st-mva-pdf-v1"))
|
||||
if existing: return existing
|
||||
dataset = Dataset(kind="BA_ST_MVA", provider="SEFAZ-BA", sha256=raw_sha256, raw_sha256=raw_sha256, version_label="ba-st-mva-pdf-v1", act="RICMS/BA Anexo 1", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
|
||||
self.session.add(dataset); self.session.flush()
|
||||
try:
|
||||
for record in records:
|
||||
ncm_codes = record.pop("ncm_sh") or []
|
||||
entry = BaStMvaEntry(dataset_id=dataset.id, **record, metadata_json={})
|
||||
self.session.add(entry); self.session.flush()
|
||||
for code in dict.fromkeys(ncm_codes): self.session.add(BaStMvaNcmSh(entry_id=entry.id, ncm_sh=code, metadata_json={}))
|
||||
self.session.flush()
|
||||
if self.session.query(BaStMvaEntry).filter_by(dataset_id=dataset.id).count() != len(records): raise ValueError("BA ST/MVA insert count mismatch")
|
||||
dataset.record_count = len(records); dataset.status = "READY"
|
||||
self.session.execute(update(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
|
||||
dataset.status = "ACTIVE"; self.session.commit(); return dataset
|
||||
except Exception:
|
||||
self.session.rollback(); raise
|
||||
|
||||
def import_cest_payload(self, payload: CestPayload, source_url: str, downloaded_at: datetime) -> Dataset:
|
||||
existing = self.session.scalar(select(Dataset).where(Dataset.provider == "CONFAZ", Dataset.sha256 == payload.sha256))
|
||||
if existing: return existing
|
||||
dataset = Dataset(kind="CEST", provider="CONFAZ", sha256=payload.sha256, raw_sha256=payload.sha256, version_label=payload.version, act="Convenio ICMS 142/18", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
|
||||
self.session.add(dataset); self.session.flush()
|
||||
canonical = {}
|
||||
for entry in payload.entries:
|
||||
existing_entry = canonical.get(entry.code)
|
||||
if existing_entry is None:
|
||||
canonical[entry.code] = [entry, set(entry.ncm_codes)]
|
||||
else:
|
||||
existing_entry[1].update(entry.ncm_codes)
|
||||
for entry, ncm_codes in canonical.values():
|
||||
row = CestEntry(dataset_id=dataset.id, code=entry.code, description=entry.description, annex=entry.annex)
|
||||
self.session.add(row); self.session.flush()
|
||||
for ncm_code in sorted(ncm_codes):
|
||||
self.session.add(CestNcmEntry(cest_entry_id=row.id, ncm_code=ncm_code))
|
||||
dataset.record_count = len(canonical); dataset.status = "READY"
|
||||
self.session.execute(update(Dataset).where(Dataset.kind == "CEST", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
|
||||
dataset.status = "ACTIVE"; self.session.commit()
|
||||
return dataset
|
||||
|
||||
def cest_for_ncm(self, code):
|
||||
rows = self.session.execute(select(CestEntry, CestNcmEntry.ncm_code).join(CestNcmEntry).join(Dataset).where(Dataset.kind == "CEST", Dataset.status == "ACTIVE")).all()
|
||||
return list(dict.fromkeys(entry for entry, ncm_code in rows if code.startswith(ncm_code)))
|
||||
|
||||
def legacy_ncm(self, code):
|
||||
entry = self.session.get(LegacyNcmEntry, code)
|
||||
if not entry or entry.successor_code is not None:
|
||||
return None
|
||||
return {"code": entry.code, "source": entry.source, "source_date": entry.source_date, "reason": entry.reason}
|
||||
|
||||
def tipi_for_ncm(self, code):
|
||||
code = _digits(code)
|
||||
return self.session.scalars(select(TipiEntry).join(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.status == "ACTIVE", TipiEntry.ncm == code)).all()
|
||||
|
||||
def ba_st_mva_for_ncm(self, code):
|
||||
code = _digits(code)
|
||||
rows = self.session.execute(select(BaStMvaEntry, BaStMvaNcmSh.ncm_sh).join(BaStMvaNcmSh, BaStMvaNcmSh.entry_id == BaStMvaEntry.id).join(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE")).all()
|
||||
return list(dict.fromkeys(entry for entry, ncm_sh in rows if _digits(ncm_sh) and code.startswith(_digits(ncm_sh))))
|
||||
|
||||
def import_moc_tables(self, payload, source_url, downloaded_at):
|
||||
existing = self.session.scalar(select(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.sha256 == payload.sha256, Dataset.version_label == payload.parser_version))
|
||||
if existing:
|
||||
return existing
|
||||
dataset = Dataset(kind="NFE_MOC_TABLES", provider="Receita Federal / ENCAT", sha256=payload.sha256, raw_sha256=payload.sha256, version_label=payload.parser_version, act="MOC 7.0 Anexo I", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
|
||||
self.session.add(dataset); self.session.flush()
|
||||
try:
|
||||
for entry in payload.entries:
|
||||
self.session.add(FiscalCodeEntry(dataset_id=dataset.id, domain=entry.domain, code=entry.code, description=entry.description, source_section=entry.source_section, metadata_json=entry.metadata))
|
||||
self.session.flush()
|
||||
if self.session.query(FiscalCodeEntry).filter_by(dataset_id=dataset.id).count() != len(payload.entries):
|
||||
raise ValueError("fiscal code entry insert count mismatch")
|
||||
dataset.record_count = len(payload.entries); dataset.status = "READY"
|
||||
self.session.execute(update(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
|
||||
dataset.status = "ACTIVE"; self.session.commit(); return dataset
|
||||
except Exception:
|
||||
self.session.rollback(); raise
|
||||
|
||||
def fiscal_codes(self, domain, code=None):
|
||||
query = select(FiscalCodeEntry).join(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.status == "ACTIVE", FiscalCodeEntry.domain == domain)
|
||||
if code is not None:
|
||||
query = query.where(FiscalCodeEntry.code == code)
|
||||
return self.session.scalars(query).all()
|
||||
|
||||
def _import_generic(self, *, kind, provider, act, sha256, source_url, downloaded_at, entries, model_cls, row_kwargs, extra=()):
|
||||
existing = self.session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.sha256 == sha256))
|
||||
if existing:
|
||||
return existing
|
||||
dataset = Dataset(kind=kind, provider=provider, sha256=sha256, raw_sha256=sha256, version_label=sha256[:16], act=act, source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
|
||||
self.session.add(dataset); self.session.flush()
|
||||
try:
|
||||
for entry in entries:
|
||||
self.session.add(model_cls(dataset_id=dataset.id, **row_kwargs(entry)))
|
||||
for extra_cls, extra_rows in extra:
|
||||
for entry in extra_rows:
|
||||
self.session.add(extra_cls(dataset_id=dataset.id, **entry))
|
||||
self.session.flush()
|
||||
if self.session.query(model_cls).filter_by(dataset_id=dataset.id).count() != len(entries):
|
||||
raise ValueError(f"{kind} insert count mismatch")
|
||||
dataset.record_count = len(entries); dataset.status = "READY"
|
||||
self.session.execute(update(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
|
||||
dataset.status = "ACTIVE"; self.session.commit(); return dataset
|
||||
except Exception:
|
||||
self.session.rollback(); raise
|
||||
|
||||
def import_cfop(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="CFOP", provider="Portal Nacional NF-e", act="Informe Técnico 2023.002", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=CfopEntry, row_kwargs=lambda e: e)
|
||||
|
||||
def import_fcp(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="FCP", provider="Portal Nacional NF-e", act="Tabela de Alíquotas FCP por UF", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=FcpRate, row_kwargs=lambda e: e)
|
||||
|
||||
def import_anp(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="ANP", provider="Portal Nacional NF-e", act="Tabela de Código de Produto ANP", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=AnpProductCode, row_kwargs=lambda e: e)
|
||||
|
||||
def import_ncm_utrib(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="NCM_UTRIB", provider="Portal Nacional NF-e", act="Informe Técnico 2024.001 - NCM x uTrib", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=NcmUtribEntry, row_kwargs=lambda e: e)
|
||||
|
||||
def import_rtc_biofuel_index(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="RTC_BIOFUEL_INDEX", provider="Portal Nacional NF-e", act="Informe Técnico 2025.004 - pBioObrigatorio", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcBiofuelIndexEntry, row_kwargs=lambda e: e)
|
||||
|
||||
def import_rtc_cred_pres(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="RTC_CCREDPRES", provider="Portal Nacional NF-e", act="cCredPres — LC 214/2025", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcCredPresEntry, row_kwargs=lambda e: e)
|
||||
|
||||
def import_rtc_cclasstrib(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="RTC_CCLASSTRIB", provider="Portal Nacional NF-e", act="cClassTrib — IBS/CBS (LC 214/2025)", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcCClassTribEntry, row_kwargs=lambda e: e, extra=((RtcCstIbsCbsEntry, payload.extra_entries),))
|
||||
|
||||
def import_rtc_cbs_rates(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="RTC_CBS_RATES", provider="Portal Nacional NF-e", act="Informe Técnico 2026.002 - Alíquotas da CBS", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcCbsRateEntry, row_kwargs=lambda e: e)
|
||||
|
||||
def rtc_cclasstrib(self, code=None):
|
||||
query = select(RtcCClassTribEntry).join(Dataset).where(Dataset.kind == "RTC_CCLASSTRIB", Dataset.status == "ACTIVE")
|
||||
if code is not None:
|
||||
query = query.where(RtcCClassTribEntry.cclasstrib == str(code))
|
||||
return self.session.scalars(query).all()
|
||||
|
||||
def rtc_cbs_rate(self, year):
|
||||
return self.session.scalar(select(RtcCbsRateEntry).join(Dataset).where(Dataset.kind == "RTC_CBS_RATES", Dataset.status == "ACTIVE", RtcCbsRateEntry.year == int(year)))
|
||||
|
||||
def cfop_lookup(self, code=None):
|
||||
query = select(CfopEntry).join(Dataset).where(Dataset.kind == "CFOP", Dataset.status == "ACTIVE")
|
||||
if code is not None:
|
||||
query = query.where(CfopEntry.code == str(code))
|
||||
return self.session.scalars(query).all()
|
||||
|
||||
def fcp_for_uf(self, uf):
|
||||
return self.session.scalar(select(FcpRate).join(Dataset).where(Dataset.kind == "FCP", Dataset.status == "ACTIVE", FcpRate.uf == uf.upper()))
|
||||
|
||||
def anp_code_exists(self, cprod_anp):
|
||||
return self.session.scalar(select(AnpProductCode).join(Dataset).where(Dataset.kind == "ANP", Dataset.status == "ACTIVE", AnpProductCode.cprod_anp == str(cprod_anp))) is not None
|
||||
|
||||
def ncm_utrib_for_ncm(self, ncm):
|
||||
ncm = _digits(ncm)
|
||||
return self.session.scalar(select(NcmUtribEntry).join(Dataset).where(Dataset.kind == "NCM_UTRIB", Dataset.status == "ACTIVE", NcmUtribEntry.ncm == ncm))
|
||||
|
||||
def rtc_biofuel_for_anp(self, cprod_anp):
|
||||
return self.session.scalars(select(RtcBiofuelIndexEntry).join(Dataset).where(Dataset.kind == "RTC_BIOFUEL_INDEX", Dataset.status == "ACTIVE", RtcBiofuelIndexEntry.cprod_anp == str(cprod_anp), RtcBiofuelIndexEntry.is_historical.is_(False))).all()
|
||||
|
||||
def rtc_cred_pres(self, code=None):
|
||||
query = select(RtcCredPresEntry).join(Dataset).where(Dataset.kind == "RTC_CCREDPRES", Dataset.status == "ACTIVE")
|
||||
if code is not None:
|
||||
query = query.where(RtcCredPresEntry.code == str(code))
|
||||
return self.session.scalars(query).all()
|
||||
|
||||
def dataset_versions(self):
|
||||
active = self.session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
|
||||
cest = self.session.scalar(select(Dataset).where(Dataset.kind == "CEST", Dataset.status == "ACTIVE"))
|
||||
tipi = self.session.scalar(select(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.status == "ACTIVE"))
|
||||
mva = self.session.scalar(select(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE"))
|
||||
moc = self.session.scalar(select(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.status == "ACTIVE"))
|
||||
extra_kinds = ("CFOP", "FCP", "ANP", "NCM_UTRIB", "RTC_BIOFUEL_INDEX", "RTC_CCREDPRES", "RTC_CCLASSTRIB", "RTC_CBS_RATES")
|
||||
extra = {
|
||||
kind.lower(): self.session.scalar(select(Dataset.sha256).where(Dataset.kind == kind, Dataset.status == "ACTIVE"))
|
||||
for kind in extra_kinds
|
||||
}
|
||||
return {
|
||||
"ncm": active.sha256 if active else None,
|
||||
"cest": cest.sha256 if cest else None,
|
||||
"tipi_ipi": tipi.sha256 if tipi else None,
|
||||
"ba_st_mva": mva.sha256 if mva else None,
|
||||
"nfe_moc_tables": moc.sha256 if moc else None,
|
||||
**extra,
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Fail-closed parser for the official Receita Federal TIPI workbook."""
|
||||
from dataclasses import dataclass
|
||||
from io import BytesIO
|
||||
import hashlib
|
||||
import unicodedata
|
||||
|
||||
import openpyxl
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TipiEntry:
|
||||
ncm: str
|
||||
ex: str | None
|
||||
description: str
|
||||
rate: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TipiPayload:
|
||||
sha256: str
|
||||
entries: tuple[TipiEntry, ...]
|
||||
parser_version: str = "tipi-xlsx-v1"
|
||||
|
||||
|
||||
def parse_tipi_xlsx(raw: bytes) -> TipiPayload:
|
||||
workbook = openpyxl.load_workbook(BytesIO(raw), read_only=True, data_only=True)
|
||||
sheet = workbook.active
|
||||
normalize = lambda value: "".join(char for char in unicodedata.normalize("NFKD", str(value or "")).encode("ascii", "ignore").decode().upper() if char.isalnum())
|
||||
header_row = next((row for row in sheet.iter_rows(values_only=True) if {"NCM", "DESCRICAO", "ALIQUOTA"} <= {normalize(cell) for cell in row}), None)
|
||||
if header_row is None:
|
||||
raise ValueError("TIPI workbook has no recognized official header")
|
||||
headers = [normalize(cell) for cell in header_row]
|
||||
ncm_index = headers.index("NCM")
|
||||
ex_index = headers.index("EX")
|
||||
description_index = next(index for index, value in enumerate(headers) if value == "DESCRICAO")
|
||||
rate_index = next(index for index, value in enumerate(headers) if value.startswith("ALIQUOTA"))
|
||||
entries = []
|
||||
started = False
|
||||
for row in sheet.iter_rows(values_only=True):
|
||||
if tuple(row) == tuple(header_row):
|
||||
started = True; continue
|
||||
if not started or ncm_index >= len(row):
|
||||
continue
|
||||
ncm = "".join(char for char in str(row[ncm_index] or "") if char.isdigit())
|
||||
if len(ncm) != 8:
|
||||
continue
|
||||
description = str(row[description_index] or "").strip()
|
||||
if not description:
|
||||
raise ValueError("TIPI NCM row has no description")
|
||||
ex = str(row[ex_index] or "").strip() or None
|
||||
entries.append(TipiEntry(ncm, ex, description, str(row[rate_index] or "").strip()))
|
||||
if not entries:
|
||||
raise ValueError("TIPI workbook has no 8-digit NCM rows")
|
||||
return TipiPayload(hashlib.sha256(raw).hexdigest(), tuple(entries))
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Automatic refresh of the official NCM dataset (Siscomex JSON): staging -> validate -> atomic promotion, keeping the last valid dataset on ANY failure.
|
||||
|
||||
Every attempt leaves a SourceRefreshRun (url, time, sha256, version label, record count, status, error) for the panel. Nothing here touches Agents or providers.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..config import settings
|
||||
from ..models import Dataset, SourceRefreshRun
|
||||
from .ncm import parse_ncm_payload
|
||||
from .service import DatasetService
|
||||
|
||||
logger = logging.getLogger("gti_fiscal_hub")
|
||||
MIN_RATIO = 0.9 # a payload with < 90% of the active record count is treated as a broken download, never promoted
|
||||
INTERVAL = timedelta(hours=24)
|
||||
RETRY_AFTER_FAILURE = timedelta(hours=6)
|
||||
|
||||
|
||||
def _fetch(url: str) -> bytes:
|
||||
response = httpx.get(url, timeout=max(settings.http_timeout_seconds, 60), follow_redirects=True, headers={"User-Agent": "GTIFiscalHub/1.0 (official dataset refresh)"})
|
||||
response.raise_for_status()
|
||||
return response.content
|
||||
|
||||
|
||||
def refresh_ncm(session, fetch=_fetch, url: str | None = None) -> SourceRefreshRun:
|
||||
url = url or settings.ncm_dataset_url
|
||||
run = SourceRefreshRun(source="NCM", source_url=url, status="RUNNING")
|
||||
session.add(run)
|
||||
session.commit()
|
||||
try:
|
||||
content = fetch(url) # staging: bytes in memory, nothing written to the active dataset yet
|
||||
parsed = parse_ncm_payload(content) # schema validation
|
||||
run.sha256, run.version_label, run.record_count = parsed.sha256, (parsed.version or "")[:64], len(parsed.entries)
|
||||
active = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
|
||||
if active is not None and active.sha256 == parsed.sha256:
|
||||
run.status = "UNCHANGED"
|
||||
else:
|
||||
if active is not None and len(parsed.entries) < MIN_RATIO * (active.record_count or 0):
|
||||
raise ValueError(f"implausible record count {len(parsed.entries)} vs active {active.record_count}")
|
||||
DatasetService(session).import_payload(parsed, url, datetime.now(timezone.utc)) # atomic: one transaction, old ACTIVE superseded only at the end
|
||||
run.status = "UPDATED"
|
||||
try: # the KB is never deleted: only flagged STALE_REVIEW_REQUIRED when its NCM left the official dataset
|
||||
from ..enrichment import history, kb
|
||||
current = history.official_sets(session)
|
||||
run.error = f"kb_revalidated: {kb.revalidate(session, lambda v: v in current)}"[:300]
|
||||
except Exception as error: # noqa: BLE001 - revalidation trouble must not undo a valid dataset promotion
|
||||
run.error = f"kb_revalidate_failed: {type(error).__name__}"[:300]
|
||||
except Exception as error: # keep the last valid dataset
|
||||
session.rollback()
|
||||
run = session.get(SourceRefreshRun, run.id)
|
||||
run.status, run.error = "FAILED", f"{type(error).__name__}: {error}"[:300]
|
||||
logger.warning("NCM refresh failed: %s", run.error)
|
||||
run.finished_at = datetime.now(timezone.utc)
|
||||
session.commit()
|
||||
return run
|
||||
|
||||
|
||||
def last_runs(session, source: str = "NCM", limit: int = 5):
|
||||
return session.scalars(select(SourceRefreshRun).where(SourceRefreshRun.source == source).order_by(SourceRefreshRun.id.desc()).limit(limit)).all()
|
||||
|
||||
|
||||
def _due(session) -> bool:
|
||||
last = session.scalar(select(SourceRefreshRun).where(SourceRefreshRun.source == "NCM", SourceRefreshRun.status != "RUNNING").order_by(SourceRefreshRun.id.desc()).limit(1))
|
||||
if last is None or last.finished_at is None:
|
||||
return True
|
||||
finished = last.finished_at if last.finished_at.tzinfo else last.finished_at.replace(tzinfo=timezone.utc)
|
||||
return datetime.now(timezone.utc) - finished >= (RETRY_AFTER_FAILURE if last.status == "FAILED" else INTERVAL)
|
||||
|
||||
|
||||
def start_scheduler(session_factory, first_delay: float = 90.0):
|
||||
"""Daemon thread: checks hourly whether a refresh is due (survives restarts because due-ness is derived from SourceRefreshRun)."""
|
||||
def loop():
|
||||
time.sleep(first_delay)
|
||||
while True:
|
||||
try:
|
||||
with session_factory() as session:
|
||||
if _due(session):
|
||||
refresh_ncm(session)
|
||||
except Exception: # never let the scheduler die
|
||||
logger.exception("official refresh loop error")
|
||||
time.sleep(3600)
|
||||
thread = threading.Thread(target=loop, name="official-refresh", daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
Reference in New Issue
Block a user