gustavosantos/GTI_GERENCIADOR_FISCAL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: coverage V2 - historical evidence miner, KB evidence model + promotion audit + revalidation, Kodebar/OSCBR limits, KB/provider panel, seed import; 1.3.2 (Hub only)

Browse Source 
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
gti
2026-09-26 08:14:57 -03:00
co-authored by Claude Sonnet 5
parent 19e145ea47
commit ac1ecccc50
41 changed files with 1418 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
central/app/enrichment/kb.py
+130 -31
View File
@@ -12,11 +12,16 @@ from datetime import datetime, timezone
from sqlalchemy import select
from ..models import KbKnowledge, KbProduct
from sqlalchemy import func
from ..models import KbKnowledge, KbProduct, KbPromotionAudit
from . import textnorm
UNVERIFIED, EVIDENCE_ONLY, VERIFIED_KB, CONFLICT, STALE = "UNVERIFIED", "EVIDENCE_ONLY", "VERIFIED_KB", "CONFLICT", "STALE"
CLIENT_FAMILY = "CLIENT_OBSERVATION"
UNVERIFIED, OBSERVED, CORROBORATED, EXTERNAL_CORROBORATED, VERIFIED_KB, CONFLICT, STALE = "UNVERIFIED", "OBSERVED", "CORROBORATED", "EXTERNAL_CORROBORATED", "VERIFIED_KB", "CONFLICT", "STALE"
EVIDENCE_ONLY = OBSERVED # legacy name
CLIENT_FAMILY = "CLIENT_DATA" # every client-derived lineage (master data + fiscal documents of the same ERP are NOT independent of each other)
RULE_VERSION = "kb-promotion-v2"
STRONG_DOCS, STRONG_YEARS = 20, 2
def _now():
@@ -56,15 +61,29 @@ def upsert_product(session, key: str | None, description, *, brand=None, quantit
if quantity and not product.package_quantity:
product.package_quantity = str(quantity)[:32]
fp = textnorm.fingerprint(description)
norm = textnorm.normalized(description)
if norm:
add_alias(session, product, norm)
if fp:
add_evidence(session, product, "name_fingerprint", fp, "IDENTITY", family="IDENTITY", status=EVIDENCE_ONLY)
add_evidence(session, product, "name_fingerprint", fp, "IDENTITY", family="IDENTITY", status=OBSERVED)
return product
def add_evidence(session, product: KbProduct, key: str, value: str, source: str, *, family: str, client_label: str | None = None, status: str = EVIDENCE_ONLY, info: dict | None = None) -> KbKnowledge:
def add_alias(session, product: KbProduct, normalized: str, cap: int = 20):
"""Non-fiscal aliases of the same GTIN (descriptions seen over the years). Helps name matching; never authority for a fiscal write."""
if not normalized or session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "alias", KbKnowledge.value == normalized[:200])):
return
if (session.scalar(select(func.count()).select_from(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "alias")) or 0) >= cap:
return
session.add(KbKnowledge(kb_product_id=product.id, knowledge_key="alias", value=normalized[:200], source="ALIAS", status=OBSERVED, evidence={"families": {}, "clients": [], "info": {}}))
def add_evidence(session, product: KbProduct, key: str, value: str, source: str, *, family: str, client_label: str | None = None, status: str = OBSERVED, info: dict | None = None,
lineage: dict | None = None) -> KbKnowledge:
"""lineage (client data only): {"master": 0/1, "docs": n, "years": {"2024": n}} of ONE client for this (GTIN, value); merged, never double counted."""
row = session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == key, KbKnowledge.value == value))
if row is None:
row = KbKnowledge(kb_product_id=product.id, knowledge_key=key, value=value, source=source[:32], status=status, evidence={"families": {}, "clients": [], "info": info or {}})
row = KbKnowledge(kb_product_id=product.id, knowledge_key=key, value=value, source=source[:32], status=status, evidence={"families": {}, "clients": [], "lineages": {}, "info": info or {}})
session.add(row)
session.flush()
evidence = dict(row.evidence or {})
@@ -73,48 +92,128 @@ def add_evidence(session, product: KbProduct, key: str, value: str, source: str,
clients = list(evidence.get("clients") or [])
if client_label and client_label not in clients:
clients.append(client_label)
row.evidence = {**evidence, "families": families, "clients": clients, "info": {**(evidence.get("info") or {}), **(info or {})}}
lineages = dict(evidence.get("lineages") or {})
if client_label and lineage:
cur = dict(lineages.get(client_label) or {"master": 0, "docs": 0, "years": {}})
cur["master"] = max(int(cur.get("master", 0)), int(lineage.get("master", 0)))
cur["docs"] = int(cur.get("docs", 0)) + int(lineage.get("docs", 0))
years = dict(cur.get("years") or {})
for y, n in (lineage.get("years") or {}).items():
years[str(y)] = int(years.get(str(y), 0)) + int(n)
cur["years"] = years
lineages[client_label] = cur
row.evidence = {**evidence, "families": families, "clients": clients, "lineages": lineages, "info": {**(evidence.get("info") or {}), **(info or {})}}
return row
def _lineage_summary(rows):
clients, docs, years = set(), 0, {}
for r in rows:
ev = r.evidence or {}
for c, lin in (ev.get("lineages") or {}).items():
clients.add(c)
docs += int(lin.get("docs", 0))
for y, n in (lin.get("years") or {}).items():
years[y] = years.get(y, 0) + int(n)
for c in ev.get("clients") or []:
clients.add(c)
return {"clients": sorted(clients), "docs": docs, "years": sorted(years), "last_year": max(int(y) for y in years) if years else None}
def independent_count(rows) -> tuple[int, list[str]]:
families = {}
clients: set[str] = set()
for row in rows:
ev = row.evidence or {}
for fam in (ev.get("families") or {}):
if fam.startswith("PROVIDER:"):
families[fam] = True
elif fam == CLIENT_FAMILY:
clients.update(ev.get("clients") or [])
names = sorted(families)
if len(clients) >= 2:
names.append(f"{CLIENT_FAMILY}(x{len(clients)})")
"""Independent families: each NCM-bearing provider = 1; ALL client-derived data = 1 and only when STRONG (>= STRONG_DOCS documents over >= STRONG_YEARS years, or >= 2 distinct clients)."""
names = sorted({f for r in rows for f in ((r.evidence or {}).get("families") or {}) if f.startswith("PROVIDER:")})
summary = _lineage_summary(rows)
has_client = any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in rows)
if has_client and (len(summary["clients"]) >= 2 or (summary["docs"] >= STRONG_DOCS and len(summary["years"]) >= STRONG_YEARS)):
names.append(CLIENT_FAMILY)
return len(names), names
def _official_period(session, value):
"""CURRENT_VALID / HISTORICAL_EXPIRED (exists in the dataset but not vigente today) / HISTORICAL_INVALID (never in the dataset)."""
from .resolver import ncm_exists_any
return "CURRENT_VALID" if ncm_exists_any(session, value, current=True) else ("HISTORICAL_EXPIRED" if ncm_exists_any(session, value, current=False) else "HISTORICAL_INVALID")
def _mark(row, state):
row.evidence = {**(row.evidence or {}), "info": {**((row.evidence or {}).get("info") or {}), "historical_state": state}}
def evaluate(session, product: KbProduct, key: str, is_official) -> dict:
"""Recomputes and PERSISTS the status of one field. Returns {value, status, families, candidates, rejected}."""
"""Recomputes and PERSISTS the status of one field. Never 'majority wins': two current-valid values => CONFLICT."""
rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == key)))
if not rows:
return {"value": None, "status": UNVERIFIED, "families": [], "candidates": [], "rejected": []}
return {"value": None, "status": UNVERIFIED, "families": [], "candidates": [], "rejected": [], "detail": {}}
valid = [r for r in rows if is_official(r.value)]
rejected = [r.value for r in rows if not is_official(r.value)]
for r in rows:
if r not in valid:
state = _official_period(session, r.value) if key == "ncm" else "HISTORICAL_INVALID"
r.status = STALE if state == "HISTORICAL_EXPIRED" else OBSERVED
_mark(r, state)
values = sorted({r.value for r in valid})
if len(values) > 1:
for r in rows:
for r in valid:
r.status = CONFLICT
return {"value": None, "status": CONFLICT, "families": [], "candidates": values, "rejected": rejected}
_mark(r, "HISTORICAL_CONFLICT")
return {"value": None, "status": CONFLICT, "families": [], "candidates": values, "rejected": rejected, "detail": {"clients": _lineage_summary(valid)["clients"]}}
if not values:
for r in rows:
r.status = EVIDENCE_ONLY
return {"value": None, "status": EVIDENCE_ONLY, "families": [], "candidates": [], "rejected": rejected}
return {"value": None, "status": OBSERVED, "families": [], "candidates": [], "rejected": rejected, "detail": {}}
chosen = [r for r in valid if r.value == values[0]]
old = chosen[0].status
count, names = independent_count(chosen)
summary = _lineage_summary(chosen)
other_last = _lineage_summary([r for r in rows if r not in chosen])["last_year"]
superseded = other_last is not None and summary["last_year"] is not None and other_last >= summary["last_year"] # a different (expired) NCM is at least as recent: not stable
if superseded and CLIENT_FAMILY in names:
names.remove(CLIENT_FAMILY)
count -= 1
providers = [n for n in names if n.startswith("PROVIDER:")]
already = any(r.status == VERIFIED_KB and r.validated_at for r in chosen)
status = VERIFIED_KB if (count >= 2 or already) else EVIDENCE_ONLY
for r in rows:
r.status = status if r.value == values[0] else EVIDENCE_ONLY
if status == VERIFIED_KB and r.value == values[0] and not r.validated_at:
has_client_data = any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in chosen)
if count >= 2 or already:
status = VERIFIED_KB
elif providers and has_client_data:
status = EXTERNAL_CORROBORATED
elif CLIENT_FAMILY in names:
status = CORROBORATED
else:
status = OBSERVED
for r in chosen:
r.status = status
_mark(r, "HISTORICAL_CHANGED" if superseded else "HISTORICAL_STABLE_CURRENT")
if status == VERIFIED_KB and not r.validated_at:
r.validated_at = _now()
return {"value": values[0], "status": status, "families": names, "candidates": values, "rejected": rejected}
if status == VERIFIED_KB and old != VERIFIED_KB and key == "ncm":
from .resolver import dataset_versions
session.add(KbPromotionAudit(kb_product_id=product.id, gtin=product.gtin, field=key, value=values[0], old_status=old, new_status=status, families=names,
sources={"clients": summary["clients"], "documents": summary["docs"], "years": summary["years"], "providers": providers},
official_dataset=dataset_versions(session).get("NCM", {}), rule_version=RULE_VERSION))
return {"value": values[0], "status": status, "families": names, "candidates": values, "rejected": rejected, "detail": summary}
def revalidate(session, is_official) -> dict:
"""After the OFFICIAL NCM dataset changed: never delete knowledge. Rows whose NCM is no longer vigente become STALE (STALE_REVIEW_REQUIRED, audited when they were VERIFIED_KB);
rows still vigente keep their status (VERIFIED_KB stays valid while the NCM is vigente)."""
from .resolver import dataset_versions
stale = still = 0
versions = dataset_versions(session).get("NCM", {})
for row in session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm")):
if is_official(row.value):
still += 1
if row.status == STALE:
row.status = OBSERVED # became vigente again: needs a fresh evaluate() to regain its level
continue
if row.status != STALE:
was = row.status
row.status = STALE
_mark(row, "HISTORICAL_EXPIRED")
row.evidence = {**(row.evidence or {}), "info": {**((row.evidence or {}).get("info") or {}), "stale_reason": "STALE_REVIEW_REQUIRED"}}
stale += 1
if was == VERIFIED_KB:
product = session.get(KbProduct, row.kb_product_id)
session.add(KbPromotionAudit(kb_product_id=row.kb_product_id, gtin=product.gtin if product else None, field="ncm", value=row.value, old_status=was, new_status=STALE,
families=sorted((row.evidence or {}).get("families") or {}), sources={"reason": "NCM_NO_LONGER_VIGENTE"}, official_dataset=versions, rule_version=RULE_VERSION))
session.flush()
return {"stale": stale, "still_vigente": still}