GTI_GERENCIADOR_FISCAL/central/app/enrichment/history.py

"""Historical ERP evidence (product master + NFC-e/NF-e items) -> central KB evidence. Local data only: NO provider is contacted here.

Evidence model
- One 'lineage' per client per (GTIN, value): master-data flag, fiscal-document count and documents per year. Master data and the fiscal documents of the SAME ERP are one family (CLIENT_DATA):
  the documents were issued from that very master, so they are never counted as two independent sources.
- CLIENT_DATA alone can reach CORROBORATED (>= 20 documents over >= 2 years, or >= 2 distinct clients) but never VERIFIED_KB: that needs an independent NCM-bearing source (kb.evaluate).
- NCM validity is judged against the ACTIVE official dataset today: CURRENT_VALID / HISTORICAL_EXPIRED / HISTORICAL_INVALID; a different NCM at least as recent -> HISTORICAL_CHANGED; two current values -> CONFLICT.
- Company-specific tax data (CRT, CFOP, CSOSN, ...) is never read nor stored; only GTIN, descriptions/aliases, NCM, CEST and the source tenant label.
Idempotent: a client's lineage for a (GTIN, value) is REPLACED on every import, never accumulated.
"""
from __future__ import annotations

import re
from collections import defaultdict

from sqlalchemy import select

from ..models import KbKnowledge, KbProduct
from . import kb, textnorm
from .gtin import normalize_gtin


def _digits(v) -> str:
    return re.sub(r"\D", "", str(v or ""))


def build_observations(stagings: list[dict]) -> dict:
    """stagings: miner output per client. Returns {"gtins": {key: {...}}, "stats": {...}}."""
    gtins: dict[str, dict] = {}
    stats = defaultdict(int)
    for st in stagings:
        client = st["client"]
        product_gtin: dict[str, set] = defaultdict(set)
        master_gtin: dict[str, str] = {}
        master_desc: dict[str, str] = {}
        for m in st["master"]:
            stats["master_rows"] += 1
            master_desc[m["code"]] = m["desc"]
            for raw in (m["gtin"], m["codbarra"]):
                g = normalize_gtin(raw)
                if g["status"] == "VALID_GTIN":
                    master_gtin.setdefault(m["code"], g["key"])
                    break
        # product<->EAN links seen in documents/movements are used ONLY when the master has none, and only when unambiguous
        seen: dict[str, set] = defaultdict(set)
        for row in st.get("nfe", []):
            g = normalize_gtin(row["codbarra"])
            if g["status"] == "VALID_GTIN":
                seen[row["code"]].add(g["key"])
        for row in st.get("barcodes", []):
            g = normalize_gtin(row["codbarra"])
            if g["status"] == "VALID_GTIN":
                seen[row["code"]].add(g["key"])
        link: dict[str, str] = dict(master_gtin)
        for code, keys in seen.items():
            if code in link:
                continue
            if len(keys) == 1:
                link[code] = next(iter(keys))
                stats["links_from_documents"] += 1
            else:
                stats["ambiguous_product_links_skipped"] += 1

        def slot(key):
            return gtins.setdefault(key, {"aliases": set(), "ncm": defaultdict(lambda: defaultdict(lambda: {"master": 0, "docs": 0, "years": {}})), "cest": defaultdict(lambda: defaultdict(lambda: {"master": 0, "docs": 0, "years": {}})), "clients": set()})
        for m in st["master"]:
            key = link.get(m["code"])
            if not key:
                continue
            s = slot(key)
            s["clients"].add(client)
            if m["desc"]:
                s["aliases"].add(textnorm.normalized(m["desc"]))
            n = _digits(m["ncm"])
            if len(n) == 8:
                s["ncm"][n][client]["master"] = 1
            c = _digits(m["cest"])
            if len(c) == 7:
                s["cest"][c][client]["master"] = 1
        for row in st.get("nfce", []):
            key = link.get(row["code"])
            stats["nfce_groups"] += 1
            if not key:
                stats["nfce_groups_without_gtin"] += 1
                continue
            s = slot(key)
            n = _digits(row["ncm"])
            if len(n) == 8:
                lin = s["ncm"][n][client]
                lin["docs"] += row["n"]
                if row["year"]:
                    lin["years"][str(row["year"])] = lin["years"].get(str(row["year"]), 0) + row["n"]
                stats["nfce_documents"] += row["n"]
            c = _digits(row["cest"])
            if len(c) == 7:
                lin = s["cest"][c][client]
                lin["docs"] += row["n"]
                if row["year"]:
                    lin["years"][str(row["year"])] = lin["years"].get(str(row["year"]), 0) + row["n"]
        for row in st.get("nfe", []):
            key = link.get(row["code"])
            if not key:
                continue
            s = slot(key)
            if row["desc"]:
                s["aliases"].add(textnorm.normalized(row["desc"]))
            n = _digits(row["ncm"])
            if len(n) == 8:
                s["ncm"][n][client]["docs"] += row["n"]
                stats["nfe_documents"] += row["n"]
    stats["gtins"] = len(gtins)
    return {"gtins": gtins, "stats": dict(stats)}


def apply(session, observations: dict, is_official, *, dry_run: bool = True, chunk: int = 500) -> dict:
    """Bulk, idempotent import into kb_products / kb_knowledge. dry_run=True computes every status and rolls back."""
    keys = sorted(observations["gtins"])
    report = {"GTINS_FOUND": len(keys), "OBSERVATIONS": 0, "by_status": defaultdict(int), "CONFLICTS": 0, "WOULD_VERIFY": 0, "with_current_valid_ncm": 0, "stable_across_years": 0, "changed_over_time": 0,
              "with_historical_ncm": 0, "multi_client": 0}
    nested = session.begin_nested()
    try:
        for start in range(0, len(keys), chunk):
            part = keys[start:start + chunk]
            existing = {p.gtin: p for p in session.scalars(select(KbProduct).where(KbProduct.gtin.in_(part)))}
            for key in part:
                obs = observations["gtins"][key]
                product = existing.get(key)
                if product is None:
                    alias = next(iter(sorted(obs["aliases"])), "")
                    product = kb.upsert_product(session, key, alias)
                    existing[key] = product
                for a in sorted(obs["aliases"])[:20]:
                    kb.add_alias(session, product, a)
                    fp = textnorm.fingerprint(a)
                    if fp:
                        kb.add_evidence(session, product, "name_fingerprint", fp, "IDENTITY", family="IDENTITY", status=kb.OBSERVED)
                for field, values in (("ncm", obs["ncm"]), ("cest", obs["cest"])):
                    for value, per_client in values.items():
                        for client, lin in per_client.items():
                            kb.add_evidence(session, product, field, value, "client:history", family=kb.CLIENT_FAMILY, client_label=client, lineage=None)
                            row = session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == field, KbKnowledge.value == value))
                            ev = dict(row.evidence or {})
                            lineages = dict(ev.get("lineages") or {})
                            lineages[client] = {"master": lin["master"], "docs": lin["docs"], "years": dict(lin["years"])}      # REPLACE: idempotent
                            row.evidence = {**ev, "lineages": lineages}
                            report["OBSERVATIONS"] += 1
            session.flush()
            for key in part:
                product = existing[key]
                ev = kb.evaluate(session, product, "ncm", is_official)
                report["by_status"][ev["status"]] += 1
                if ev["status"] == kb.CONFLICT:
                    report["CONFLICTS"] += 1
                if ev["status"] == kb.VERIFIED_KB:
                    report["WOULD_VERIFY"] += 1
                if ev["value"]:
                    report["with_current_valid_ncm"] += 1
                if observations["gtins"][key]["ncm"]:
                    report["with_historical_ncm"] += 1
                if len(observations["gtins"][key]["clients"]) >= 2:
                    report["multi_client"] += 1
                rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm")))
                states = {(r.evidence.get("info") or {}).get("historical_state") for r in rows}
                if "HISTORICAL_STABLE_CURRENT" in states and ev["value"]:
                    years = set()
                    for r in rows:
                        for lin in (r.evidence.get("lineages") or {}).values():
                            years |= set(lin.get("years") or {})
                    if len(years) >= 2:
                        report["stable_across_years"] += 1
                if "HISTORICAL_CHANGED" in states or ("HISTORICAL_EXPIRED" in states and ev["value"]):
                    report["changed_over_time"] += 1
        if dry_run:
            nested.rollback()
        else:
            nested.commit()
            session.commit()
    except Exception:
        nested.rollback()
        raise
    report["by_status"] = dict(report["by_status"])
    return report


def official_sets(session):
    """Precomputed 'vigente today' membership: one query instead of one per value."""
    from datetime import date
    from ..models import Dataset, NcmEntry
    from .resolver import _parse_br
    today = date.today()
    current = set()
    for row in session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE")):
        start, end = _parse_br(row.valid_from), _parse_br(row.valid_to)
        if (start is None or start <= today) and (end is None or end >= today):
            current.add(row.code)
    return current