222 lines
14 KiB
Python
222 lines
14 KiB
Python
"""Product resolver: GTIN/name -> KB -> providers -> official NCM validation -> existing fiscal engine, decided FIELD BY FIELD.
|
|
|
|
Deterministic. No AI, no scores: discrete states only.
|
|
Resolution states: EXACT_GTIN_VERIFIED, EXACT_KB_NAME_VERIFIED, MULTI_SOURCE_MATCH, SINGLE_WEAK_SOURCE, AMBIGUOUS, CONFLICT, NOT_FOUND
|
|
(+ operational: INVALID_GTIN, PENDING_PROVIDER_QUOTA, PENDING_PROVIDERS_OFFLINE).
|
|
Field decisions: AUTO_FILL_SAFE | SUGGESTION_ONLY | CONFLICT | INSUFFICIENT_EVIDENCE | ALREADY_SET | NOT_APPLICABLE.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from datetime import date
|
|
|
|
from sqlalchemy import select
|
|
|
|
from ..classification import Classifier, normalize_code
|
|
from ..datasets.service import DatasetService
|
|
from ..models import Dataset, NcmEntry
|
|
from . import kb, textnorm
|
|
from .gtin import normalize_gtin
|
|
from .quota import ProviderGateway
|
|
|
|
AUTO, SUGGEST, CONFLICT, INSUFFICIENT, ALREADY, NA = "AUTO_FILL_SAFE", "SUGGESTION_ONLY", "CONFLICT", "INSUFFICIENT_EVIDENCE", "ALREADY_SET", "NOT_APPLICABLE"
|
|
FIELDS = ("CLASSIFICACAO_FISCAL", "CEST", "IPI")
|
|
|
|
|
|
def _parse_br(text: str | None) -> date | None:
|
|
try:
|
|
d, m, y = str(text).split("/")
|
|
return date(int(y), int(m), int(d))
|
|
except (ValueError, TypeError):
|
|
return None
|
|
|
|
|
|
def ncm_is_official(session, code: str, today: date | None = None) -> bool:
|
|
"""8-digit code present in the ACTIVE official NCM dataset and vigente today (dd/mm/yyyy, 31/12/9999 = open)."""
|
|
code = normalize_code(code)
|
|
if len(code) != 8:
|
|
return False
|
|
today = today or date.today()
|
|
rows = session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == code)).all()
|
|
for row in rows:
|
|
start, end = _parse_br(row.valid_from), _parse_br(row.valid_to)
|
|
if (start is None or start <= today) and (end is None or end >= today):
|
|
return True
|
|
return False
|
|
|
|
|
|
class _ScopedDatasets(DatasetService):
|
|
"""The Classifier only needs the ONE exact NCM row here; loading the whole 15k-row catalogue on every product would cost ~300 ms of the 5 s budget."""
|
|
|
|
def __init__(self, session, code: str):
|
|
super().__init__(session)
|
|
self.code = code
|
|
|
|
def active_entries(self):
|
|
return self.session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == self.code)).all()
|
|
|
|
|
|
def ncm_exists_any(session, code: str, *, current: bool, today: date | None = None) -> bool:
|
|
"""current=True: vigente today; current=False: present in the ACTIVE dataset with any validity window (i.e. known but not vigente)."""
|
|
code = normalize_code(code)
|
|
if len(code) != 8:
|
|
return False
|
|
if current:
|
|
return ncm_is_official(session, code, today)
|
|
return session.scalar(select(NcmEntry.id).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == code).limit(1)) is not None
|
|
|
|
|
|
def dataset_versions(session) -> dict:
|
|
return {d.kind: {"sha256": (d.sha256 or "")[:12], "version": d.version_label} for d in session.scalars(select(Dataset).where(Dataset.status == "ACTIVE"))}
|
|
|
|
|
|
def _field(status, value=None, rule=None, kb_status=None, evidence=None, reason=None):
|
|
return {"status": status, "value": value, "rule": rule, "kb_status": kb_status, "evidence": evidence or [], "reason": reason}
|
|
|
|
|
|
def _empty(current: dict, name: str) -> bool:
|
|
return not str(current.get(name) or "").strip()
|
|
|
|
|
|
def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current: dict, description: str, *, name_verified=False) -> dict:
|
|
out = {}
|
|
if not _empty(current, "CLASSIFICACAO_FISCAL"):
|
|
out["CLASSIFICACAO_FISCAL"] = _field(ALREADY, reason="campo já preenchido")
|
|
elif ncm_eval["status"] == kb.CONFLICT:
|
|
out["CLASSIFICACAO_FISCAL"] = _field(CONFLICT, kb_status=kb.CONFLICT, evidence=ncm_eval["candidates"], reason="fontes divergem no NCM")
|
|
elif ncm_eval["value"] and ncm_eval["status"] == kb.VERIFIED_KB:
|
|
out["CLASSIFICACAO_FISCAL"] = _field(AUTO, ncm_eval["value"], "NCM_KB_VERIFIED" if gtin_kb_verified or name_verified else "NCM_TWO_INDEPENDENT_SOURCES", kb.VERIFIED_KB, ncm_eval["families"])
|
|
elif ncm_eval["value"]:
|
|
rule = {"CORROBORATED": "NCM_CORROBORATED_LOCAL_HISTORY", "EXTERNAL_CORROBORATED": "NCM_EXTERNAL_CORROBORATED"}.get(ncm_eval["status"], "NCM_SINGLE_SOURCE")
|
|
out["CLASSIFICACAO_FISCAL"] = _field(SUGGEST, ncm_eval["value"], rule, ncm_eval["status"], ncm_eval["families"], "evidência ainda insuficiente para preencher automaticamente")
|
|
else:
|
|
out["CLASSIFICACAO_FISCAL"] = _field(INSUFFICIENT, reason="nenhuma fonte com NCM válido no dataset oficial")
|
|
ncm = out["CLASSIFICACAO_FISCAL"]
|
|
ncm_for_engine = ncm["value"] if ncm["status"] == AUTO else (normalize_code(current.get("CLASSIFICACAO_FISCAL")) or None)
|
|
if not ncm_for_engine or (ncm["status"] == ALREADY and not ncm_is_official(session, ncm_for_engine)):
|
|
for name in ("CEST", "IPI"):
|
|
out[name] = _field(ALREADY if not _empty(current, name) else INSUFFICIENT, reason="depende de um NCM oficial válido")
|
|
return out
|
|
# CEST / IPI are deterministic functions of an OFFICIAL NCM: the NCM we are about to write (AUTO) or one already in the product (validated above against the official dataset)
|
|
ncm_solid = ncm["status"] == AUTO or (ncm["status"] == ALREADY and ncm_is_official(session, ncm_for_engine))
|
|
decision = Classifier(_ScopedDatasets(session, ncm_for_engine)).classify({"current_ncm": ncm_for_engine, "description": description})
|
|
cest = (decision["classification"].get("cest") or [])
|
|
if not _empty(current, "CEST"):
|
|
out["CEST"] = _field(ALREADY, reason="campo já preenchido")
|
|
elif decision["cest_status"] == "CEST_CONFIRMED" and len(cest) == 1 and ncm_solid:
|
|
out["CEST"] = _field(AUTO, normalize_code(cest[0]), "CEST_OFFICIAL_UNIQUE_MAPPING", ncm["kb_status"], [f"NCM {ncm_for_engine}"])
|
|
elif decision["cest_status"] == "NO_CEST_MAPPING_FOUND":
|
|
out["CEST"] = _field(NA, reason="NCM sem mapeamento de CEST no dataset oficial")
|
|
else:
|
|
out["CEST"] = _field(INSUFFICIENT, reason=f"CEST não determinístico ({decision['cest_status']})")
|
|
ipi = decision["classification"].get("ipi") or {}
|
|
if not _empty(current, "IPI"):
|
|
out["IPI"] = _field(ALREADY, reason="campo já preenchido")
|
|
elif decision["ipi_status"] == "IPI_RESOLVED" and ipi.get("rate_kind") == "NUMERIC" and ncm_solid:
|
|
out["IPI"] = _field(AUTO, str(ipi["rate_numeric"]), "IPI_TIPI_NUMERIC", ncm["kb_status"], [f"TIPI NCM {ncm_for_engine}"])
|
|
elif decision["ipi_status"] in {"IPI_RESOLVED"}:
|
|
out["IPI"] = _field(NA, reason="TIPI sem alíquota numérica (NT/branca)")
|
|
else:
|
|
out["IPI"] = _field(INSUFFICIENT, reason=f"IPI não determinístico ({decision['ipi_status']})")
|
|
return out
|
|
|
|
|
|
def resolve(session, gateway: ProviderGateway, *, gtin_raw, description, current: dict | None = None, client_label: str | None = None, allow_external: bool = True, budget_s: float | None = None) -> dict:
|
|
started = time.monotonic()
|
|
"""Returns {state, gtin_status, gtin, fields:{...}, providers:[...], retry_at, dataset_versions}. Never raises for provider trouble."""
|
|
current = {k: v for k, v in (current or {}).items()}
|
|
g = normalize_gtin(gtin_raw)
|
|
result = {"state": "NOT_FOUND", "gtin_status": g["status"], "gtin": g.get("key"), "fields": {}, "providers": [], "retry_at": None, "dataset_versions": dataset_versions(session)}
|
|
session.info["is_official"] = lambda v: ncm_is_official(session, v)
|
|
is_official = session.info["is_official"]
|
|
product, name_verified, key = None, False, g.get("key")
|
|
|
|
if g["status"] == "VALID_GTIN": # GTIN has absolute priority over the description
|
|
product = kb.find_by_gtin(session, key)
|
|
ev = kb.evaluate(session, product, "ncm", is_official) if product else None
|
|
if ev and ev["status"] == kb.VERIFIED_KB:
|
|
gateway.quota.record_kb_hit()
|
|
result["state"] = "EXACT_GTIN_VERIFIED"
|
|
result["fields"] = _fiscal_decisions(session, ev, True, current, description or "")
|
|
session.commit()
|
|
return result
|
|
if ev and ev["status"] == kb.CONFLICT: # local evidence already conflicts: an external answer cannot settle it, so spend no quota
|
|
result["state"] = "CONFLICT"
|
|
result["fields"] = _fiscal_decisions(session, ev, False, current, description or "")
|
|
gateway.quota.record_kb_hit()
|
|
session.commit()
|
|
return result
|
|
locally_known = bool(ev and ev["status"] in (kb.CORROBORATED, kb.EXTERNAL_CORROBORATED)) # identity + NCM candidate already local: only an NCM-bearing source can still add value
|
|
query = g["gtin"] if len(g["gtin"]) in (8, 14) else key[1:] # providers/caches use the canonical EAN-13 (UPC-12 is zero-padded to it)
|
|
found, pending = [], []
|
|
if allow_external:
|
|
for provider in gateway.providers:
|
|
if locally_known and not getattr(provider, "provides_ncm", False):
|
|
result["providers"].append({"provider": provider.name, "state": "SKIPPED", "reason": "LOCAL_IDENTITY_KNOWN", "from_cache": False})
|
|
continue
|
|
if budget_s is not None and time.monotonic() - started > budget_s:
|
|
result["providers"].append({"provider": provider.name, "state": "SKIPPED", "reason": "TIME_BUDGET", "from_cache": False})
|
|
pending.append(type("P", (), {"reason": "TIME_BUDGET", "retry_at": None})())
|
|
continue
|
|
out = gateway.lookup(provider, query)
|
|
result["providers"].append({"provider": out.provider, "state": out.state, "reason": out.reason, "from_cache": out.from_cache})
|
|
if out.state in ("FOUND", "CACHE_HIT") and out.product:
|
|
found.append(out.product)
|
|
product = kb.upsert_product(session, key, out.product.title or description, brand=out.product.brand, quantity=out.product.quantity)
|
|
if out.product.ncm:
|
|
kb.add_evidence(session, product, "ncm", out.product.ncm, f"provider:{out.provider}"[:32], family=f"PROVIDER:{out.provider}", info={"raw_hash": out.product.raw_hash})
|
|
ev = kb.evaluate(session, product, "ncm", is_official)
|
|
if ev["status"] == kb.VERIFIED_KB:
|
|
break # enough evidence: spare the remaining quota
|
|
elif out.state in ("SKIPPED", "ERROR"):
|
|
pending.append(out)
|
|
if product is None and description:
|
|
product = kb.upsert_product(session, key, description)
|
|
ev = kb.evaluate(session, product, "ncm", is_official) if product else {"value": None, "status": kb.UNVERIFIED, "families": [], "candidates": [], "rejected": []}
|
|
result["fields"] = _fiscal_decisions(session, ev, False, current, description or "")
|
|
if ev["status"] == kb.CONFLICT:
|
|
result["state"] = "CONFLICT"
|
|
elif ev["status"] == kb.VERIFIED_KB:
|
|
result["state"] = "MULTI_SOURCE_MATCH" if found else "EXACT_GTIN_VERIFIED"
|
|
elif len(found) >= 2:
|
|
result["state"] = "MULTI_SOURCE_MATCH"
|
|
elif found:
|
|
result["state"] = "SINGLE_WEAK_SOURCE"
|
|
elif pending and not found:
|
|
quota = any(p.reason in ("RATE_LIMITED", "RATE_WINDOW") for p in pending)
|
|
offline = [p for p in pending if p.reason not in ("NOT_CONFIGURED",)]
|
|
if offline:
|
|
result["state"] = "PENDING_PROVIDER_QUOTA" if quota else "PENDING_PROVIDERS_OFFLINE"
|
|
result["retry_at"] = min((p.retry_at for p in offline if p.retry_at), default=None)
|
|
session.commit()
|
|
return result
|
|
|
|
if g["status"] == "INVALID_GTIN":
|
|
result["state"] = "INVALID_GTIN" # never spends provider quota on an invalid GTIN
|
|
|
|
# ---- name only: only a UNIQUE, specific description with a VERIFIED_KB fingerprint may fill anything
|
|
if not textnorm.is_specific(description):
|
|
if result["state"] != "INVALID_GTIN":
|
|
result["state"] = "AMBIGUOUS"
|
|
result["fields"] = {f: _field(INSUFFICIENT, reason="descrição genérica demais e sem GTIN válido") if _empty(current, f) else _field(ALREADY) for f in FIELDS}
|
|
return result
|
|
candidates = kb.name_candidates(session, description)
|
|
verified = []
|
|
for cand in candidates:
|
|
ev = kb.evaluate(session, cand, "ncm", is_official)
|
|
if ev["status"] == kb.VERIFIED_KB:
|
|
verified.append((cand, ev))
|
|
if candidates and len(verified) == len(candidates) and len({ev["value"] for _, ev in verified}) == 1:
|
|
_, ev = verified[0]
|
|
result["state"] = "EXACT_KB_NAME_VERIFIED"
|
|
result["fields"] = _fiscal_decisions(session, ev, True, current, description or "", name_verified=True)
|
|
elif len({ev["value"] for _, ev in verified}) > 1:
|
|
result["state"] = "CONFLICT"
|
|
result["fields"] = {f: _field(CONFLICT, reason="descrição corresponde a produtos com NCM divergentes") if _empty(current, f) else _field(ALREADY) for f in FIELDS}
|
|
else:
|
|
if result["state"] != "INVALID_GTIN":
|
|
result["state"] = "AMBIGUOUS" if len(candidates) > 1 else "NOT_FOUND"
|
|
result["fields"] = {f: _field(INSUFFICIENT, reason="sem correspondência verificada na base de conhecimento") if _empty(current, f) else _field(ALREADY) for f in FIELDS}
|
|
session.commit()
|
|
return result
|