GTI_GERENCIADOR_FISCAL/scripts/build_rtc_reference_data.py

"""Build central/app/rtc_reference_data.py: the OFFICIAL cClassTrib catalog + the LC 214/2025 goods annexes as structured, hash-pinned reference data.

    python scripts/build_rtc_reference_data.py

Inputs (all official, already downloaded/hashed):
  sources/raw/bootstrap_v1/cClassTrib 2026-06-22.xlsx   Portal Nacional NF-e cClassTrib table (parsed with the Hub's own parser)
  sources_cache/wb_lc214.bin                            LC 214/2025 compiled (Planalto, through LC 227/2026), extracted to text

Nothing here decides anything: it only turns the published texts into data with provenance. Item texts that carry a condition/exception are
marked `clean=False` so the resolver never treats them as a deterministic match.
"""
import hashlib
import html
import json
import re
import sys
from datetime import date
from pathlib import Path
from pprint import pformat

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from central.app.datasets.bootstrap_v1_extra import parse_rtc_cclasstrib  # noqa: E402

XLSX = ROOT / "sources" / "raw" / "bootstrap_v1" / "cClassTrib 2026-06-22.xlsx"
LC214 = ROOT / "sources_cache" / "wb_lc214.bin"
ROMAN = {"I": 1, "II": 2, "III": 3, "IV": 4, "V": 5, "VI": 6, "VII": 7, "VIII": 8, "IX": 9, "X": 10, "XI": 11, "XII": 12, "XIII": 13, "XIV": 14, "XV": 15}
RESTRICTION = re.compile(r"exceto|ressalv|desde que|em conformidade|requisitos|somente|exclusiv|quando|observad|salvo|apenas|conforme|vedad|\ba\)|\bb\)|\bex\b|não", re.I)
CODE = re.compile(r"\b(\d{2}\.\d{2}(?:\.\d{1,2}){0,2}|\d{4}(?:\.\d{1,2}){1,3})\b")


def lc214_text():
    raw = LC214.read_bytes()
    text = raw.decode("latin-1", errors="replace")
    text = re.sub(r"<script.*?</script>|<style.*?</style>", "", text, flags=re.S)
    text = html.unescape(re.sub(r"<[^>]+>", " ", text))
    return raw, re.sub(r"[ \t\r\f\v]+", " ", text)


def parse_annexes(text):
    heads = [(m.start(), m.group(1)) for m in re.finditer(r"\n[ \xa0]*ANEXO ([IVXL]+)[ \xa0]*\n", text)]
    blocks, seen = {}, set()
    for i, (pos, roman) in enumerate(heads):
        if roman in seen or roman not in ROMAN:
            continue
        # the goods annexes I..XV are the first occurrence of each numeral, before the Simples annexes (which repeat I..VII)
        seen.add(roman)
        end = heads[i + 1][0] if i + 1 < len(heads) else len(text)
        blocks[roman] = text[pos:end]
    out = {}
    for roman, block in blocks.items():
        head = " ".join(block[:600].split())
        items, current, expected = [], None, 1
        lines = [ln.strip() for ln in block.split("\n")]
        for idx, ln in enumerate(lines):
            if ln == str(expected):
                nxt = next((x for x in lines[idx + 1:idx + 8] if x), "")
                if nxt and not nxt.isdigit():
                    current = {"item": expected, "text": []}
                    items.append(current)
                    expected += 1
                    continue
            if current is not None and ln:
                current["text"].append(ln)
        parsed = []
        for it in items:
            desc = " ".join(" ".join(it["text"]).split())
            prefixes = sorted({re.sub(r"\D", "", c) for c in CODE.findall(desc)})
            clean = not RESTRICTION.search(desc) and bool(prefixes)
            parsed.append({"item": it["item"], "description": desc[:400], "prefixes": prefixes, "clean": clean})
        out[roman] = {"number": ROMAN[roman], "title": head[:200], "revoked": "Revogado" in head, "items": parsed}
    return out


def main():
    payload = parse_rtc_cclasstrib(XLSX.read_bytes())
    official = []
    for e in payload.entries:
        ind = e["indicators"]
        official.append({"code": e["cclasstrib"], "cst": e["cst"], "name": (e["name"] or "")[:160], "tipo_aliquota": e["tipo_aliquota"], "p_red_ibs": e["p_red_ibs"], "p_red_cbs": e["p_red_cbs"],
                         "start": (e["d_ini_vig"] or "")[:10] or None, "end": (e["d_fim_vig"] or "")[:10] or None, "anexo": e.get("anexo"), "ind_nfe": ind.get("indNFe") == "1", "ind_nfce": ind.get("indNFCe") == "1"})
    raw, text = lc214_text()
    annexes = parse_annexes(text)
    counts = {k: (len(v["items"]), sum(1 for i in v["items"] if i["clean"])) for k, v in annexes.items()}
    module = ROOT / "central" / "app" / "rtc_reference_data.py"
    catalog_meta = {"file": XLSX.name, "sha256": hashlib.sha256(XLSX.read_bytes()).hexdigest(), "publication": "2026-06-22", "source": "Portal Nacional da NF-e (cClassTrib)", "parser": payload.parser_version}
    lc_meta = {"document": "Lei Complementar 214/2025 compiled (through LC 227/2026)", "url": "https://www.planalto.gov.br/ccivil_03/leis/lcp/lcp214.htm", "sha256": hashlib.sha256(raw).hexdigest(),
               "accessed_at": date.today().isoformat(), "fetched_via": "Internet Archive copy of the official URL"}
    header = '"""GENERATED by scripts/build_rtc_reference_data.py - do not edit. Official cClassTrib catalog and LC 214/2025 goods annexes with provenance."""' + "\n\n"
    body = "\n\n".join([
        "OFFICIAL_CATALOG = " + pformat(catalog_meta, width=160),
        "OFFICIAL_CCLASSTRIB = " + pformat(official, width=220),
        "LC214_SOURCE = " + pformat(lc_meta, width=160),
        "LC214_ANNEXES = " + pformat(annexes, width=220),
    ]) + "\n"
    module.write_text(header + body, encoding="utf-8")
    (ROOT / "docs" / "sources" / "RTC_REFERENCE_MANIFEST_V1.json").write_text(json.dumps({"official_catalog": catalog_meta, "official_classes": len(official), "lc214": lc_meta, "annex_items_total_and_clean": counts}, ensure_ascii=False, indent=1), encoding="utf-8")
    print(len(official), "official classes;", "annex items (total, clean):", counts)


if __name__ == "__main__":
    main()