"""Product resolver: GTIN/name -> KB -> providers -> official NCM validation -> existing fiscal engine, decided FIELD BY FIELD. Deterministic. No AI, no scores: discrete states only. Resolution states: EXACT_GTIN_VERIFIED, EXACT_KB_NAME_VERIFIED, MULTI_SOURCE_MATCH, SINGLE_WEAK_SOURCE, AMBIGUOUS, CONFLICT, NOT_FOUND (+ operational: INVALID_GTIN, PENDING_PROVIDER_QUOTA, PENDING_PROVIDERS_OFFLINE). Field decisions: AUTO_FILL_SAFE | SUGGESTION_ONLY | CONFLICT | INSUFFICIENT_EVIDENCE | ALREADY_SET | NOT_APPLICABLE. """ from __future__ import annotations import time from datetime import date from sqlalchemy import select from ..classification import Classifier, normalize_code from ..datasets.service import DatasetService from ..models import Dataset, NcmEntry from . import kb, textnorm from .gtin import normalize_gtin from .placeholders import is_placeholder, is_writable_empty from .quota import ProviderGateway AUTO, SUGGEST, CONFLICT, INSUFFICIENT, ALREADY, NA = "AUTO_FILL_SAFE", "SUGGESTION_ONLY", "CONFLICT", "INSUFFICIENT_EVIDENCE", "ALREADY_SET", "NOT_APPLICABLE" FIELDS = ("CLASSIFICACAO_FISCAL", "CEST", "IPI") def _parse_br(text: str | None) -> date | None: try: d, m, y = str(text).split("/") return date(int(y), int(m), int(d)) except (ValueError, TypeError): return None def ncm_is_official(session, code: str, today: date | None = None) -> bool: """8-digit code present in the ACTIVE official NCM dataset and vigente today (dd/mm/yyyy, 31/12/9999 = open).""" code = normalize_code(code) if len(code) != 8: return False today = today or date.today() rows = session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == code)).all() for row in rows: start, end = _parse_br(row.valid_from), _parse_br(row.valid_to) if (start is None or start <= today) and (end is None or end >= today): return True return False class _ScopedDatasets(DatasetService): """The Classifier only needs the ONE exact NCM row here; loading the whole 15k-row catalogue on every product would cost ~300 ms of the 5 s budget.""" def __init__(self, session, code: str): super().__init__(session) self.code = code def active_entries(self): return self.session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == self.code)).all() def ncm_exists_any(session, code: str, *, current: bool, today: date | None = None) -> bool: """current=True: vigente today; current=False: present in the ACTIVE dataset with any validity window (i.e. known but not vigente).""" code = normalize_code(code) if len(code) != 8: return False if current: return ncm_is_official(session, code, today) return session.scalar(select(NcmEntry.id).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == code).limit(1)) is not None def dataset_versions(session) -> dict: return {d.kind: {"sha256": (d.sha256 or "")[:12], "version": d.version_label} for d in session.scalars(select(Dataset).where(Dataset.status == "ACTIVE"))} def _field(status, value=None, rule=None, kb_status=None, evidence=None, reason=None): return {"status": status, "value": value, "rule": rule, "kb_status": kb_status, "evidence": evidence or [], "reason": reason} def _empty(current: dict, name: str) -> bool: return is_writable_empty(name, current.get(name)) def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current: dict, description: str, *, name_verified=False) -> dict: out = {} if not _empty(current, "CLASSIFICACAO_FISCAL"): out["CLASSIFICACAO_FISCAL"] = _field(ALREADY, reason="campo já preenchido") elif ncm_eval["status"] == kb.CONFLICT: out["CLASSIFICACAO_FISCAL"] = _field(CONFLICT, kb_status=kb.CONFLICT, evidence=ncm_eval["candidates"], reason="fontes divergem no NCM") elif ncm_eval["value"] and ncm_eval["status"] == kb.VERIFIED_KB: out["CLASSIFICACAO_FISCAL"] = _field(AUTO, ncm_eval["value"], "NCM_KB_VERIFIED" if gtin_kb_verified or name_verified else "NCM_TWO_INDEPENDENT_SOURCES", kb.VERIFIED_KB, ncm_eval["families"]) elif ncm_eval["value"]: rule = {"CORROBORATED": "NCM_CORROBORATED_LOCAL_HISTORY", "EXTERNAL_CORROBORATED": "NCM_EXTERNAL_CORROBORATED"}.get(ncm_eval["status"], "NCM_SINGLE_SOURCE") out["CLASSIFICACAO_FISCAL"] = _field(SUGGEST, ncm_eval["value"], rule, ncm_eval["status"], ncm_eval["families"], "evidência ainda insuficiente para preencher automaticamente") else: out["CLASSIFICACAO_FISCAL"] = _field(INSUFFICIENT, reason="nenhuma fonte com NCM válido no dataset oficial") ncm = out["CLASSIFICACAO_FISCAL"] own = current.get("CLASSIFICACAO_FISCAL") ncm_for_engine = ncm["value"] if ncm["status"] == AUTO else (None if is_placeholder("CLASSIFICACAO_FISCAL", own) else (normalize_code(own) or None)) if not ncm_for_engine or (ncm["status"] == ALREADY and not ncm_is_official(session, ncm_for_engine)): for name in ("CEST", "IPI"): out[name] = _field(ALREADY if not _empty(current, name) else INSUFFICIENT, reason="depende de um NCM oficial válido") return out # CEST / IPI are deterministic functions of an OFFICIAL NCM: the NCM we are about to write (AUTO) or one already in the product (validated above against the official dataset) ncm_solid = ncm["status"] == AUTO or (ncm["status"] == ALREADY and ncm_is_official(session, ncm_for_engine)) decision = Classifier(_ScopedDatasets(session, ncm_for_engine)).classify({"current_ncm": ncm_for_engine, "description": description}) cest = (decision["classification"].get("cest") or []) if not _empty(current, "CEST"): out["CEST"] = _field(ALREADY, reason="campo já preenchido") elif decision["cest_status"] == "CEST_CONFIRMED" and len(cest) == 1 and ncm_solid: out["CEST"] = _field(AUTO, normalize_code(cest[0]), "CEST_OFFICIAL_UNIQUE_MAPPING", ncm["kb_status"], [f"NCM {ncm_for_engine}"]) elif decision["cest_status"] == "NO_CEST_MAPPING_FOUND": out["CEST"] = _field(NA, reason="NCM sem mapeamento de CEST no dataset oficial") else: out["CEST"] = _field(INSUFFICIENT, reason=f"CEST não determinístico ({decision['cest_status']})") ipi = decision["classification"].get("ipi") or {} if not _empty(current, "IPI"): out["IPI"] = _field(ALREADY, reason="campo já preenchido") elif decision["ipi_status"] == "IPI_RESOLVED" and ipi.get("rate_kind") == "NUMERIC" and ncm_solid: out["IPI"] = _field(AUTO, str(ipi["rate_numeric"]), "IPI_TIPI_NUMERIC", ncm["kb_status"], [f"TIPI NCM {ncm_for_engine}"]) elif decision["ipi_status"] in {"IPI_RESOLVED"}: out["IPI"] = _field(NA, reason="TIPI sem alíquota numérica (NT/branca)") else: out["IPI"] = _field(INSUFFICIENT, reason=f"IPI não determinístico ({decision['ipi_status']})") return out def resolve(session, gateway: ProviderGateway, *, gtin_raw, description, current: dict | None = None, client_label: str | None = None, allow_external: bool = True, budget_s: float | None = None) -> dict: started = time.monotonic() """Returns {state, gtin_status, gtin, fields:{...}, providers:[...], retry_at, dataset_versions}. Never raises for provider trouble.""" current = {k: v for k, v in (current or {}).items()} g = normalize_gtin(gtin_raw) result = {"state": "NOT_FOUND", "gtin_status": g["status"], "gtin": g.get("key"), "fields": {}, "providers": [], "retry_at": None, "dataset_versions": dataset_versions(session)} session.info["is_official"] = lambda v: ncm_is_official(session, v) is_official = session.info["is_official"] product, name_verified, key = None, False, g.get("key") if g["status"] == "VALID_GTIN": # GTIN has absolute priority over the description product = kb.find_by_gtin(session, key) ev = kb.evaluate(session, product, "ncm", is_official) if product else None if ev and ev["status"] == kb.VERIFIED_KB: gateway.quota.record_kb_hit() result["state"] = "EXACT_GTIN_VERIFIED" result["fields"] = _fiscal_decisions(session, ev, True, current, description or "") session.commit() return result if ev and ev["status"] == kb.CONFLICT: # local evidence already conflicts: an external answer cannot settle it, so spend no quota result["state"] = "CONFLICT" result["fields"] = _fiscal_decisions(session, ev, False, current, description or "") gateway.quota.record_kb_hit() session.commit() return result locally_known = bool(ev and ev["status"] in (kb.CORROBORATED, kb.EXTERNAL_CORROBORATED)) # identity + NCM candidate already local: only an NCM-bearing source can still add value query = g["gtin"] if len(g["gtin"]) in (8, 14) else key[1:] # providers/caches use the canonical EAN-13 (UPC-12 is zero-padded to it) found, pending = [], [] if allow_external: for provider in gateway.providers: if locally_known and not getattr(provider, "provides_ncm", False): result["providers"].append({"provider": provider.name, "state": "SKIPPED", "reason": "LOCAL_IDENTITY_KNOWN", "from_cache": False}) continue if budget_s is not None and time.monotonic() - started > budget_s: result["providers"].append({"provider": provider.name, "state": "SKIPPED", "reason": "TIME_BUDGET", "from_cache": False}) pending.append(type("P", (), {"reason": "TIME_BUDGET", "retry_at": None})()) continue out = gateway.lookup(provider, query) result["providers"].append({"provider": out.provider, "state": out.state, "reason": out.reason, "from_cache": out.from_cache}) if out.state in ("FOUND", "CACHE_HIT") and out.product: found.append(out.product) product = kb.upsert_product(session, key, out.product.title or description, brand=out.product.brand, quantity=out.product.quantity) if out.product.ncm: kb.add_evidence(session, product, "ncm", out.product.ncm, f"provider:{out.provider}"[:32], family=f"PROVIDER:{out.provider}", info={"raw_hash": out.product.raw_hash}) ev = kb.evaluate(session, product, "ncm", is_official) if ev["status"] == kb.VERIFIED_KB: break # enough evidence: spare the remaining quota elif out.state in ("SKIPPED", "ERROR"): pending.append(out) if product is None and description: product = kb.upsert_product(session, key, description) ev = kb.evaluate(session, product, "ncm", is_official) if product else {"value": None, "status": kb.UNVERIFIED, "families": [], "candidates": [], "rejected": []} result["fields"] = _fiscal_decisions(session, ev, False, current, description or "") if ev["status"] == kb.CONFLICT: result["state"] = "CONFLICT" elif ev["status"] == kb.VERIFIED_KB: result["state"] = "MULTI_SOURCE_MATCH" if found else "EXACT_GTIN_VERIFIED" elif len(found) >= 2: result["state"] = "MULTI_SOURCE_MATCH" elif found: result["state"] = "SINGLE_WEAK_SOURCE" elif pending and not found: quota = any(p.reason in ("RATE_LIMITED", "RATE_WINDOW") for p in pending) offline = [p for p in pending if p.reason not in ("NOT_CONFIGURED",)] if offline: result["state"] = "PENDING_PROVIDER_QUOTA" if quota else "PENDING_PROVIDERS_OFFLINE" result["retry_at"] = min((p.retry_at for p in offline if p.retry_at), default=None) session.commit() return result if g["status"] == "INVALID_GTIN": result["state"] = "INVALID_GTIN" # never spends provider quota on an invalid GTIN # ---- name only: only a UNIQUE, specific description with a VERIFIED_KB fingerprint may fill anything if not textnorm.is_specific(description): if result["state"] != "INVALID_GTIN": result["state"] = "AMBIGUOUS" result["fields"] = {f: _field(INSUFFICIENT, reason="descrição genérica demais e sem GTIN válido") if _empty(current, f) else _field(ALREADY) for f in FIELDS} return result candidates = kb.name_candidates(session, description) verified = [] for cand in candidates: ev = kb.evaluate(session, cand, "ncm", is_official) if ev["status"] == kb.VERIFIED_KB: verified.append((cand, ev)) if candidates and len(verified) == len(candidates) and len({ev["value"] for _, ev in verified}) == 1: _, ev = verified[0] result["state"] = "EXACT_KB_NAME_VERIFIED" result["fields"] = _fiscal_decisions(session, ev, True, current, description or "", name_verified=True) elif len({ev["value"] for _, ev in verified}) > 1: result["state"] = "CONFLICT" result["fields"] = {f: _field(CONFLICT, reason="descrição corresponde a produtos com NCM divergentes") if _empty(current, f) else _field(ALREADY) for f in FIELDS} else: if result["state"] != "INVALID_GTIN": result["state"] = "AMBIGUOUS" if len(candidates) > 1 else "NOT_FOUND" result["fields"] = {f: _field(INSUFFICIENT, reason="sem correspondência verificada na base de conhecimento") if _empty(current, f) else _field(ALREADY) for f in FIELDS} session.commit() return result