GTI_GERENCIADOR_FISCAL/central/app/enrichment/resolver.py

"""Product resolver: GTIN/name -> KB -> providers -> official NCM validation -> existing fiscal engine, decided FIELD BY FIELD.

Deterministic. No AI, no scores: discrete states only.
Resolution states: EXACT_GTIN_VERIFIED, EXACT_KB_NAME_VERIFIED, MULTI_SOURCE_MATCH, SINGLE_WEAK_SOURCE, AMBIGUOUS, CONFLICT, NOT_FOUND
(+ operational: INVALID_GTIN, PENDING_PROVIDER_QUOTA, PENDING_PROVIDERS_OFFLINE).
Field decisions: AUTO_FILL_SAFE | SUGGESTION_ONLY | CONFLICT | INSUFFICIENT_EVIDENCE | ALREADY_SET | NOT_APPLICABLE.
"""
from __future__ import annotations

import time
from datetime import date

from sqlalchemy import select

from ..classification import Classifier, normalize_code
from ..datasets.service import DatasetService
from ..models import Dataset, NcmEntry
from . import fiscal_completion, kb, textnorm
from .gtin import normalize_gtin
from .placeholders import is_placeholder, is_writable_empty
from .quota import ProviderGateway

AUTO, SUGGEST, CONFLICT, INSUFFICIENT, ALREADY, NA = "AUTO_FILL_SAFE", "SUGGESTION_ONLY", "CONFLICT", "INSUFFICIENT_EVIDENCE", "ALREADY_SET", "NOT_APPLICABLE"
FIELDS = ("CLASSIFICACAO_FISCAL", "CEST", "IPI")


def _parse_br(text: str | None) -> date | None:
    try:
        d, m, y = str(text).split("/")
        return date(int(y), int(m), int(d))
    except (ValueError, TypeError):
        return None


def ncm_is_official(session, code: str, today: date | None = None) -> bool:
    """8-digit code present in the ACTIVE official NCM dataset and vigente today (dd/mm/yyyy, 31/12/9999 = open)."""
    code = normalize_code(code)
    if len(code) != 8:
        return False
    today = today or date.today()
    rows = session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == code)).all()
    for row in rows:
        start, end = _parse_br(row.valid_from), _parse_br(row.valid_to)
        if (start is None or start <= today) and (end is None or end >= today):
            return True
    return False


class _ScopedDatasets(DatasetService):
    """The Classifier only needs the ONE exact NCM row here; loading the whole 15k-row catalogue on every product would cost ~300 ms of the 5 s budget."""

    def __init__(self, session, code: str):
        super().__init__(session)
        self.code = code

    def active_entries(self):
        return self.session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == self.code)).all()


def ncm_exists_any(session, code: str, *, current: bool, today: date | None = None) -> bool:
    """current=True: vigente today; current=False: present in the ACTIVE dataset with any validity window (i.e. known but not vigente)."""
    code = normalize_code(code)
    if len(code) != 8:
        return False
    if current:
        return ncm_is_official(session, code, today)
    return session.scalar(select(NcmEntry.id).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == code).limit(1)) is not None


def dataset_versions(session) -> dict:
    return {d.kind: {"sha256": (d.sha256 or "")[:12], "version": d.version_label} for d in session.scalars(select(Dataset).where(Dataset.status == "ACTIVE"))}


def _field(status, value=None, rule=None, kb_status=None, evidence=None, reason=None):
    return {"status": status, "value": value, "rule": rule, "kb_status": kb_status, "evidence": evidence or [], "reason": reason}


def _empty(current: dict, name: str) -> bool:
    return is_writable_empty(name, current.get(name))


def _merge_fiscal_completion(session, out: dict, *, ncm_official: bool, ncm_for_engine: str | None, current: dict, description: str,
                             product_code: str | None, installation, tenant_id: int | None, decision: dict | None, product_evidence: dict | None = None) -> None:
    """Merges field_contract.resolve_product's own field-by-field decisions (CFOP/CST/CSOSN/PIS-COFINS/...) into `out`, in the SAME
    AUTO_FILL_SAFE/ALREADY_SET/NOT_APPLICABLE/INSUFFICIENT_EVIDENCE vocabulary CEST/IPI already use, plus NEEDS_REVIEW for anything that needs an
    operator's context answer. No-op when the caller (a test, or a code path with no installation on hand) did not pass installation/tenant_id/product_code."""
    if installation is None or tenant_id is None or product_code is None:
        return
    fields = fiscal_completion.resolve_fiscal_fields(session, product_code=product_code, description=description, current=current, ncm=ncm_for_engine or "",
                                                      ncm_official=ncm_official, installation=installation, tenant_id=tenant_id, decision=decision, extra_columns=current,
                                                      ev=product_evidence)
    for field, r in fields.items():
        out[field] = _field(r["status"], r.get("value"), r.get("rule"), reason=r.get("reason"))
        if r.get("missing_key"):
            out[field]["missing_key"], out[field]["missing_scope"] = r["missing_key"], r.get("missing_scope")


def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current: dict, description: str, *, name_verified=False,
                      product_code: str | None = None, installation=None, tenant_id: int | None = None, product_evidence: dict | None = None) -> dict:
    out = {}
    if not _empty(current, "CLASSIFICACAO_FISCAL"):
        out["CLASSIFICACAO_FISCAL"] = _field(ALREADY, reason="campo já preenchido")
    elif ncm_eval["status"] == kb.CONFLICT:
        out["CLASSIFICACAO_FISCAL"] = _field(CONFLICT, kb_status=kb.CONFLICT, evidence=ncm_eval["candidates"], reason="fontes divergem no NCM")
    elif ncm_eval["value"] and ncm_eval["status"] == kb.VERIFIED_KB:
        out["CLASSIFICACAO_FISCAL"] = _field(AUTO, ncm_eval["value"], "NCM_KB_VERIFIED" if gtin_kb_verified or name_verified else "NCM_TWO_INDEPENDENT_SOURCES", kb.VERIFIED_KB, ncm_eval["families"])
    elif ncm_eval["value"]:
        rule = {"CORROBORATED": "NCM_CORROBORATED_LOCAL_HISTORY", "EXTERNAL_CORROBORATED": "NCM_EXTERNAL_CORROBORATED"}.get(ncm_eval["status"], "NCM_SINGLE_SOURCE")
        out["CLASSIFICACAO_FISCAL"] = _field(SUGGEST, ncm_eval["value"], rule, ncm_eval["status"], ncm_eval["families"], "evidência ainda insuficiente para preencher automaticamente")
    else:
        out["CLASSIFICACAO_FISCAL"] = _field(INSUFFICIENT, reason="nenhuma fonte com NCM válido no dataset oficial")
    ncm = out["CLASSIFICACAO_FISCAL"]
    own = current.get("CLASSIFICACAO_FISCAL")
    ncm_for_engine = ncm["value"] if ncm["status"] == AUTO else (None if is_placeholder("CLASSIFICACAO_FISCAL", own) else (normalize_code(own) or None))
    if not ncm_for_engine or (ncm["status"] == ALREADY and not ncm_is_official(session, ncm_for_engine)):
        for name in ("CEST", "IPI"):
            out[name] = _field(ALREADY if not _empty(current, name) else INSUFFICIENT, reason="depende de um NCM oficial válido")
        _merge_fiscal_completion(session, out, ncm_official=False, ncm_for_engine=ncm_for_engine, current=current, description=description,
                                 product_code=product_code, installation=installation, tenant_id=tenant_id, decision=None, product_evidence=product_evidence)
        return out
    # CEST / IPI are deterministic functions of an OFFICIAL NCM: the NCM we are about to write (AUTO) or one already in the product (validated above against the official dataset)
    ncm_solid = ncm["status"] == AUTO or (ncm["status"] == ALREADY and ncm_is_official(session, ncm_for_engine))
    decision = Classifier(_ScopedDatasets(session, ncm_for_engine)).classify({"current_ncm": ncm_for_engine, "description": description})
    cest = (decision["classification"].get("cest") or [])
    if not _empty(current, "CEST"):
        out["CEST"] = _field(ALREADY, reason="campo já preenchido")
    elif decision["cest_status"] == "CEST_CONFIRMED" and len(cest) == 1 and ncm_solid:
        out["CEST"] = _field(AUTO, normalize_code(cest[0]), "CEST_OFFICIAL_UNIQUE_MAPPING", ncm["kb_status"], [f"NCM {ncm_for_engine}"])
    elif decision["cest_status"] == "NO_CEST_MAPPING_FOUND":
        out["CEST"] = _field(NA, reason="NCM sem mapeamento de CEST no dataset oficial")
    else:
        out["CEST"] = _field(INSUFFICIENT, reason=f"CEST não determinístico ({decision['cest_status']})")
    ipi = decision["classification"].get("ipi") or {}
    if not _empty(current, "IPI"):
        out["IPI"] = _field(ALREADY, reason="campo já preenchido")
    elif decision["ipi_status"] == "IPI_RESOLVED" and ipi.get("rate_kind") == "NUMERIC" and ncm_solid:
        out["IPI"] = _field(AUTO, str(ipi["rate_numeric"]), "IPI_TIPI_NUMERIC", ncm["kb_status"], [f"TIPI NCM {ncm_for_engine}"])
    elif decision["ipi_status"] in {"IPI_RESOLVED"}:
        out["IPI"] = _field(NA, reason="TIPI sem alíquota numérica (NT/branca)")
    else:
        out["IPI"] = _field(INSUFFICIENT, reason=f"IPI não determinístico ({decision['ipi_status']})")
    _merge_fiscal_completion(session, out, ncm_official=ncm_solid, ncm_for_engine=ncm_for_engine, current=current, description=description,
                             product_code=product_code, installation=installation, tenant_id=tenant_id, decision=decision, product_evidence=product_evidence)
    return out


def resolve(session, gateway: ProviderGateway, *, gtin_raw, description, current: dict | None = None, client_label: str | None = None, allow_external: bool = True, budget_s: float | None = None,
           product_code: str | None = None, installation=None, tenant_id: int | None = None, product_evidence: dict | None = None) -> dict:
    started = time.monotonic()
    """Returns {state, gtin_status, gtin, fields:{...}, providers:[...], retry_at, dataset_versions}. Never raises for provider trouble.

    `product_code`/`installation`/`tenant_id`, when given, additionally resolve the post-NCM fiscal-completion fields (CFOP/CST/CSOSN/PIS-COFINS/...)
    via the existing field_contract engine (see `_merge_fiscal_completion`) and merge them into `fields` alongside CLASSIFICACAO_FISCAL/CEST/IPI.
    `product_evidence` is the Agent's targeted, product_code-scoped recipes/composed/purchases/stock evidence (firebird_reader.read_product_evidence) -
    never a full-table scan. Named separately from the local `ev` variable below (NCM knowledge-base evidence), a different, pre-existing concept."""
    current = {k: v for k, v in (current or {}).items()}
    g = normalize_gtin(gtin_raw)
    result = {"state": "NOT_FOUND", "gtin_status": g["status"], "gtin": g.get("key"), "fields": {}, "providers": [], "retry_at": None, "dataset_versions": dataset_versions(session)}
    session.info["is_official"] = lambda v: ncm_is_official(session, v)
    is_official = session.info["is_official"]
    product, name_verified, key = None, False, g.get("key")

    if g["status"] == "VALID_GTIN":                                                # GTIN has absolute priority over the description
        product = kb.find_by_gtin(session, key)
        ev = kb.evaluate(session, product, "ncm", is_official) if product else None
        if ev and ev["status"] == kb.VERIFIED_KB:
            gateway.quota.record_kb_hit()
            result["state"] = "EXACT_GTIN_VERIFIED"
            result["fields"] = _fiscal_decisions(session, ev, True, current, description or "", product_code=product_code, installation=installation, tenant_id=tenant_id, product_evidence=product_evidence)
            session.commit()
            return result
        if ev and ev["status"] == kb.CONFLICT:                                     # local evidence already conflicts: an external answer cannot settle it, so spend no quota
            result["state"] = "CONFLICT"
            result["fields"] = _fiscal_decisions(session, ev, False, current, description or "", product_code=product_code, installation=installation, tenant_id=tenant_id, product_evidence=product_evidence)
            gateway.quota.record_kb_hit()
            session.commit()
            return result
        locally_known = bool(ev and ev["status"] in (kb.CORROBORATED, kb.EXTERNAL_CORROBORATED))      # identity + NCM candidate already local: only an NCM-bearing source can still add value
        query = g["gtin"] if len(g["gtin"]) in (8, 14) else key[1:]      # providers/caches use the canonical EAN-13 (UPC-12 is zero-padded to it)
        found, pending = [], []
        if allow_external:
            for provider in gateway.providers:
                if locally_known and not getattr(provider, "provides_ncm", False):
                    result["providers"].append({"provider": provider.name, "state": "SKIPPED", "reason": "LOCAL_IDENTITY_KNOWN", "from_cache": False})
                    continue
                if budget_s is not None and time.monotonic() - started > budget_s:
                    result["providers"].append({"provider": provider.name, "state": "SKIPPED", "reason": "TIME_BUDGET", "from_cache": False})
                    pending.append(type("P", (), {"reason": "TIME_BUDGET", "retry_at": None})())
                    continue
                out = gateway.lookup(provider, query)
                result["providers"].append({"provider": out.provider, "state": out.state, "reason": out.reason, "from_cache": out.from_cache})
                if out.state in ("FOUND", "CACHE_HIT") and out.product:
                    found.append(out.product)
                    product = kb.upsert_product(session, key, out.product.title or description, brand=out.product.brand, quantity=out.product.quantity)
                    if out.product.ncm:
                        kb.add_evidence(session, product, "ncm", out.product.ncm, f"provider:{out.provider}"[:32], family=f"PROVIDER:{out.provider}", info={"raw_hash": out.product.raw_hash})
                    ev = kb.evaluate(session, product, "ncm", is_official)
                    if ev["status"] == kb.VERIFIED_KB:
                        break                                                      # enough evidence: spare the remaining quota
                elif out.state in ("SKIPPED", "ERROR"):
                    pending.append(out)
        if product is None and description:
            product = kb.upsert_product(session, key, description)
        ev = kb.evaluate(session, product, "ncm", is_official) if product else {"value": None, "status": kb.UNVERIFIED, "families": [], "candidates": [], "rejected": []}
        result["fields"] = _fiscal_decisions(session, ev, False, current, description or "", product_code=product_code, installation=installation, tenant_id=tenant_id, product_evidence=product_evidence)
        if ev["status"] == kb.CONFLICT:
            result["state"] = "CONFLICT"
        elif ev["status"] == kb.VERIFIED_KB:
            result["state"] = "MULTI_SOURCE_MATCH" if found else "EXACT_GTIN_VERIFIED"
        elif len(found) >= 2:
            result["state"] = "MULTI_SOURCE_MATCH"
        elif found:
            result["state"] = "SINGLE_WEAK_SOURCE"
        elif pending and not found:
            quota = any(p.reason in ("RATE_LIMITED", "RATE_WINDOW") for p in pending)
            offline = [p for p in pending if p.reason not in ("NOT_CONFIGURED",)]
            if offline:
                result["state"] = "PENDING_PROVIDER_QUOTA" if quota else "PENDING_PROVIDERS_OFFLINE"
                result["retry_at"] = min((p.retry_at for p in offline if p.retry_at), default=None)
        session.commit()
        return result

    if g["status"] == "INVALID_GTIN":
        result["state"] = "INVALID_GTIN"                                           # never spends provider quota on an invalid GTIN

    # ---- name only: only a UNIQUE, specific description with a VERIFIED_KB fingerprint may fill anything
    if not textnorm.is_specific(description):
        if result["state"] != "INVALID_GTIN":
            result["state"] = "AMBIGUOUS"
        result["fields"] = {f: _field(INSUFFICIENT, reason="descrição genérica demais e sem GTIN válido") if _empty(current, f) else _field(ALREADY) for f in FIELDS}
        return result
    candidates = kb.name_candidates(session, description)
    verified = []
    for cand in candidates:
        ev = kb.evaluate(session, cand, "ncm", is_official)
        if ev["status"] == kb.VERIFIED_KB:
            verified.append((cand, ev))
    if candidates and len(verified) == len(candidates) and len({ev["value"] for _, ev in verified}) == 1:
        _, ev = verified[0]
        result["state"] = "EXACT_KB_NAME_VERIFIED"
        result["fields"] = _fiscal_decisions(session, ev, True, current, description or "", name_verified=True, product_code=product_code, installation=installation, tenant_id=tenant_id, product_evidence=product_evidence)
    elif len({ev["value"] for _, ev in verified}) > 1:
        result["state"] = "CONFLICT"
        result["fields"] = {f: _field(CONFLICT, reason="descrição corresponde a produtos com NCM divergentes") if _empty(current, f) else _field(ALREADY) for f in FIELDS}
    else:
        if result["state"] != "INVALID_GTIN":
            result["state"] = "AMBIGUOUS" if len(candidates) > 1 else "NOT_FOUND"
        result["fields"] = {f: _field(INSUFFICIENT, reason="sem correspondência verificada na base de conhecimento") if _empty(current, f) else _field(ALREADY) for f in FIELDS}
    session.commit()
    return result