feat: coverage V2 - historical evidence miner, KB evidence model + promotion audit + revalidation, Kodebar/OSCBR limits, KB/provider panel, seed import; 1.3.2 (Hub only)

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
gti
2026-09-26 08:14:57 -03:00
co-authored by Claude Sonnet 5
parent 19e145ea47
commit ac1ecccc50
41 changed files with 1418 additions and 74 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
[project]
name = "gti-fiscal-agent"
version = "1.3.1"
version = "1.3.2"
requires-python = ">=3.10"
dependencies = ["httpx>=0.27,<1"]
+3 -3
View File
@@ -107,7 +107,7 @@ class Config:
self.poll_interval_seconds = float(env.get("POLL_INTERVAL_SECONDS", "5"))
# RTC auto-apply is a SEPARATE switch (default OFF): FIREBIRD_WRITES alone never lets the Agent fill cClassTrib.
self.rtc_auto_apply_max_per_job = int(env.get("RTC_AUTO_APPLY_MAX_PER_JOB", str(_rtc_apply.DEFAULT_MAX_PER_JOB)))
self.agent_version = env.get("AGENT_VERSION", "1.3.1")
self.agent_version = env.get("AGENT_VERSION", "1.3.2")
self.schema_fingerprint = env.get("SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1)
self.state_db = env.get("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite"))
# Pilot write authorization: both must be explicitly set by the installer for
@@ -349,7 +349,7 @@ def execute_job(job, config, executor, state, env):
"fiscal_context": fiscal_context,
"rtc_auto_apply": rtc_apply,
"fiscal_mode": getattr(config, "fiscal_mode", None),
"release_version": env.get("AGENT_VERSION") or "1.3.1",
"release_version": env.get("AGENT_VERSION") or "1.3.2",
"products_total": len(products),
"products_changed": len(to_process),
"products_processed": summary["processed"],
@@ -520,7 +520,7 @@ def run(config, env, *, max_iterations=None, stopper=None):
def enforce_startup_preflight(env):
"""Full read-only preflight at startup (Hub, auth, Firebird, capabilities, company context, guards). Fail closed."""
from gti_fiscal_agent.preflight import run_preflight
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.1")
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.2")
logger.info("startup preflight: %s", json.dumps({k: report[k] for k in ("PREFLIGHT", "blockers", "fiscal_mode", "effective_firebird_writes")}, ensure_ascii=False))
print(json.dumps({"PREFLIGHT": report["PREFLIGHT"], "blockers": report["blockers"]}, ensure_ascii=False))
if report["PREFLIGHT"] != "PASS":
+1 -1
View File
@@ -9,7 +9,7 @@ import os
import sqlite3
from pathlib import Path
AGENT_VERSION_DEFAULT = "1.3.1"
AGENT_VERSION_DEFAULT = "1.3.2"
REQUIRED_ENV = ("GTI_HUB_URL", "AGENT_TOKEN", "FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")
REQUIRED_CAPABILITIES = ("supports_ncm", "supports_ipi", "supports_cest")
FISCAL_MODES = ("DRY_RUN", "AUTO_APPLY")
@@ -0,0 +1,48 @@
"""KB evidence model: wider per-field status (OBSERVED/CORROBORATED/EXTERNAL_CORROBORATED/...), promotion audit, provider counters for the panel.
Revision ID: 0030_kb_evidence_model
Revises: 0029_product_enrichment
"""
from alembic import op
import sqlalchemy as sa
revision = "0030_kb_evidence_model"
down_revision = "0029_product_enrichment"
branch_labels = None
depends_on = None
def upgrade():
op.alter_column("kb_knowledge", "status", existing_type=sa.String(16), type_=sa.String(24), existing_nullable=False, existing_server_default="EVIDENCE_ONLY")
op.execute("UPDATE kb_knowledge SET status='OBSERVED' WHERE status='EVIDENCE_ONLY'")
op.alter_column("kb_knowledge", "status", server_default="OBSERVED")
op.create_table(
"kb_promotion_audits",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("kb_product_id", sa.Integer(), sa.ForeignKey("kb_products.id"), nullable=False),
sa.Column("gtin", sa.String(16), nullable=True),
sa.Column("field", sa.String(24), nullable=False, server_default="ncm"),
sa.Column("value", sa.String(64), nullable=True),
sa.Column("old_status", sa.String(24), nullable=True),
sa.Column("new_status", sa.String(24), nullable=False),
sa.Column("families", sa.JSON(), nullable=False),
sa.Column("sources", sa.JSON(), nullable=False),
sa.Column("official_dataset", sa.JSON(), nullable=False),
sa.Column("rule_version", sa.String(32), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
)
op.create_index("ix_kb_promotion_audits_kb_product_id", "kb_promotion_audits", ["kb_product_id"])
op.create_index("ix_kb_promotion_audits_gtin", "kb_promotion_audits", ["gtin"])
op.add_column("provider_usage_daily", sa.Column("found", sa.Integer(), nullable=False, server_default="0"))
op.add_column("provider_usage_daily", sa.Column("ncm_found", sa.Integer(), nullable=False, server_default="0"))
op.add_column("provider_usage_daily", sa.Column("kb_hits", sa.Integer(), nullable=False, server_default="0"))
def downgrade():
op.drop_column("provider_usage_daily", "kb_hits")
op.drop_column("provider_usage_daily", "ncm_found")
op.drop_column("provider_usage_daily", "found")
op.drop_table("kb_promotion_audits")
op.execute("UPDATE kb_knowledge SET status='EVIDENCE_ONLY' WHERE status IN ('OBSERVED','CORROBORATED','EXTERNAL_CORROBORATED')")
op.alter_column("kb_knowledge", "status", server_default="EVIDENCE_ONLY")
op.alter_column("kb_knowledge", "status", existing_type=sa.String(24), type_=sa.String(16), existing_nullable=False, existing_server_default="EVIDENCE_ONLY")
+2
View File
@@ -81,6 +81,8 @@ class Settings:
cosmos_token: str = os.getenv("COSMOS_TOKEN", "") # optional; without it Cosmos is NOT_CONFIGURED (not an error)
oscbr_user: str = os.getenv("OSCBR_USER", "") # optional free-plan account of gtin.rscsistemas.com.br (never in logs/UI)
oscbr_password: str = os.getenv("OSCBR_PASSWORD", "")
kodebar_api_key: str = os.getenv("KODEBAR_API_KEY", "") # optional free-plan key (identity only); never in logs/UI
provider_operational_reserve_percent: int = int(os.getenv("PROVIDER_OPERATIONAL_RESERVE_PERCENT", "20"))
allow_paid_providers: bool = os.getenv("ALLOW_PAID_PROVIDERS", "0") == "1"
official_refresh_enabled: bool = os.getenv("OFFICIAL_REFRESH_ENABLED", "1") == "1"
enrichment_time_budget_seconds: float = float(os.getenv("ENRICHMENT_TIME_BUDGET_SECONDS", "4.5"))
+6
View File
@@ -46,6 +46,12 @@ def refresh_ncm(session, fetch=_fetch, url: str | None = None) -> SourceRefreshR
raise ValueError(f"implausible record count {len(parsed.entries)} vs active {active.record_count}")
DatasetService(session).import_payload(parsed, url, datetime.now(timezone.utc)) # atomic: one transaction, old ACTIVE superseded only at the end
run.status = "UPDATED"
try: # the KB is never deleted: only flagged STALE_REVIEW_REQUIRED when its NCM left the official dataset
from ..enrichment import history, kb
current = history.official_sets(session)
run.error = f"kb_revalidated: {kb.revalidate(session, lambda v: v in current)}"[:300]
except Exception as error: # noqa: BLE001 - revalidation trouble must not undo a valid dataset promotion
run.error = f"kb_revalidate_failed: {type(error).__name__}"[:300]
except Exception as error: # keep the last valid dataset
session.rollback()
run = session.get(SourceRefreshRun, run.id)
+196
View File
@@ -0,0 +1,196 @@
"""Historical ERP evidence (product master + NFC-e/NF-e items) -> central KB evidence. Local data only: NO provider is contacted here.
Evidence model
- One 'lineage' per client per (GTIN, value): master-data flag, fiscal-document count and documents per year. Master data and the fiscal documents of the SAME ERP are one family (CLIENT_DATA):
the documents were issued from that very master, so they are never counted as two independent sources.
- CLIENT_DATA alone can reach CORROBORATED (>= 20 documents over >= 2 years, or >= 2 distinct clients) but never VERIFIED_KB: that needs an independent NCM-bearing source (kb.evaluate).
- NCM validity is judged against the ACTIVE official dataset today: CURRENT_VALID / HISTORICAL_EXPIRED / HISTORICAL_INVALID; a different NCM at least as recent -> HISTORICAL_CHANGED; two current values -> CONFLICT.
- Company-specific tax data (CRT, CFOP, CSOSN, ...) is never read nor stored; only GTIN, descriptions/aliases, NCM, CEST and the source tenant label.
Idempotent: a client's lineage for a (GTIN, value) is REPLACED on every import, never accumulated.
"""
from __future__ import annotations
import re
from collections import defaultdict
from sqlalchemy import select
from ..models import KbKnowledge, KbProduct
from . import kb, textnorm
from .gtin import normalize_gtin
def _digits(v) -> str:
return re.sub(r"\D", "", str(v or ""))
def build_observations(stagings: list[dict]) -> dict:
"""stagings: miner output per client. Returns {"gtins": {key: {...}}, "stats": {...}}."""
gtins: dict[str, dict] = {}
stats = defaultdict(int)
for st in stagings:
client = st["client"]
product_gtin: dict[str, set] = defaultdict(set)
master_gtin: dict[str, str] = {}
master_desc: dict[str, str] = {}
for m in st["master"]:
stats["master_rows"] += 1
master_desc[m["code"]] = m["desc"]
for raw in (m["gtin"], m["codbarra"]):
g = normalize_gtin(raw)
if g["status"] == "VALID_GTIN":
master_gtin.setdefault(m["code"], g["key"])
break
# product<->EAN links seen in documents/movements are used ONLY when the master has none, and only when unambiguous
seen: dict[str, set] = defaultdict(set)
for row in st.get("nfe", []):
g = normalize_gtin(row["codbarra"])
if g["status"] == "VALID_GTIN":
seen[row["code"]].add(g["key"])
for row in st.get("barcodes", []):
g = normalize_gtin(row["codbarra"])
if g["status"] == "VALID_GTIN":
seen[row["code"]].add(g["key"])
link: dict[str, str] = dict(master_gtin)
for code, keys in seen.items():
if code in link:
continue
if len(keys) == 1:
link[code] = next(iter(keys))
stats["links_from_documents"] += 1
else:
stats["ambiguous_product_links_skipped"] += 1
def slot(key):
return gtins.setdefault(key, {"aliases": set(), "ncm": defaultdict(lambda: defaultdict(lambda: {"master": 0, "docs": 0, "years": {}})), "cest": defaultdict(lambda: defaultdict(lambda: {"master": 0, "docs": 0, "years": {}})), "clients": set()})
for m in st["master"]:
key = link.get(m["code"])
if not key:
continue
s = slot(key)
s["clients"].add(client)
if m["desc"]:
s["aliases"].add(textnorm.normalized(m["desc"]))
n = _digits(m["ncm"])
if len(n) == 8:
s["ncm"][n][client]["master"] = 1
c = _digits(m["cest"])
if len(c) == 7:
s["cest"][c][client]["master"] = 1
for row in st.get("nfce", []):
key = link.get(row["code"])
stats["nfce_groups"] += 1
if not key:
stats["nfce_groups_without_gtin"] += 1
continue
s = slot(key)
n = _digits(row["ncm"])
if len(n) == 8:
lin = s["ncm"][n][client]
lin["docs"] += row["n"]
if row["year"]:
lin["years"][str(row["year"])] = lin["years"].get(str(row["year"]), 0) + row["n"]
stats["nfce_documents"] += row["n"]
c = _digits(row["cest"])
if len(c) == 7:
lin = s["cest"][c][client]
lin["docs"] += row["n"]
if row["year"]:
lin["years"][str(row["year"])] = lin["years"].get(str(row["year"]), 0) + row["n"]
for row in st.get("nfe", []):
key = link.get(row["code"])
if not key:
continue
s = slot(key)
if row["desc"]:
s["aliases"].add(textnorm.normalized(row["desc"]))
n = _digits(row["ncm"])
if len(n) == 8:
s["ncm"][n][client]["docs"] += row["n"]
stats["nfe_documents"] += row["n"]
stats["gtins"] = len(gtins)
return {"gtins": gtins, "stats": dict(stats)}
def apply(session, observations: dict, is_official, *, dry_run: bool = True, chunk: int = 500) -> dict:
"""Bulk, idempotent import into kb_products / kb_knowledge. dry_run=True computes every status and rolls back."""
keys = sorted(observations["gtins"])
report = {"GTINS_FOUND": len(keys), "OBSERVATIONS": 0, "by_status": defaultdict(int), "CONFLICTS": 0, "WOULD_VERIFY": 0, "with_current_valid_ncm": 0, "stable_across_years": 0, "changed_over_time": 0,
"with_historical_ncm": 0, "multi_client": 0}
nested = session.begin_nested()
try:
for start in range(0, len(keys), chunk):
part = keys[start:start + chunk]
existing = {p.gtin: p for p in session.scalars(select(KbProduct).where(KbProduct.gtin.in_(part)))}
for key in part:
obs = observations["gtins"][key]
product = existing.get(key)
if product is None:
alias = next(iter(sorted(obs["aliases"])), "")
product = kb.upsert_product(session, key, alias)
existing[key] = product
for a in sorted(obs["aliases"])[:20]:
kb.add_alias(session, product, a)
fp = textnorm.fingerprint(a)
if fp:
kb.add_evidence(session, product, "name_fingerprint", fp, "IDENTITY", family="IDENTITY", status=kb.OBSERVED)
for field, values in (("ncm", obs["ncm"]), ("cest", obs["cest"])):
for value, per_client in values.items():
for client, lin in per_client.items():
kb.add_evidence(session, product, field, value, "client:history", family=kb.CLIENT_FAMILY, client_label=client, lineage=None)
row = session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == field, KbKnowledge.value == value))
ev = dict(row.evidence or {})
lineages = dict(ev.get("lineages") or {})
lineages[client] = {"master": lin["master"], "docs": lin["docs"], "years": dict(lin["years"])} # REPLACE: idempotent
row.evidence = {**ev, "lineages": lineages}
report["OBSERVATIONS"] += 1
session.flush()
for key in part:
product = existing[key]
ev = kb.evaluate(session, product, "ncm", is_official)
report["by_status"][ev["status"]] += 1
if ev["status"] == kb.CONFLICT:
report["CONFLICTS"] += 1
if ev["status"] == kb.VERIFIED_KB:
report["WOULD_VERIFY"] += 1
if ev["value"]:
report["with_current_valid_ncm"] += 1
if observations["gtins"][key]["ncm"]:
report["with_historical_ncm"] += 1
if len(observations["gtins"][key]["clients"]) >= 2:
report["multi_client"] += 1
rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm")))
states = {(r.evidence.get("info") or {}).get("historical_state") for r in rows}
if "HISTORICAL_STABLE_CURRENT" in states and ev["value"]:
years = set()
for r in rows:
for lin in (r.evidence.get("lineages") or {}).values():
years |= set(lin.get("years") or {})
if len(years) >= 2:
report["stable_across_years"] += 1
if "HISTORICAL_CHANGED" in states or ("HISTORICAL_EXPIRED" in states and ev["value"]):
report["changed_over_time"] += 1
if dry_run:
nested.rollback()
else:
nested.commit()
session.commit()
except Exception:
nested.rollback()
raise
report["by_status"] = dict(report["by_status"])
return report
def official_sets(session):
"""Precomputed 'vigente today' membership: one query instead of one per value."""
from datetime import date
from ..models import Dataset, NcmEntry
from .resolver import _parse_br
today = date.today()
current = set()
for row in session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE")):
start, end = _parse_br(row.valid_from), _parse_br(row.valid_to)
if (start is None or start <= today) and (end is None or end >= today):
current.add(row.code)
return current
+130 -31
View File
@@ -12,11 +12,16 @@ from datetime import datetime, timezone
from sqlalchemy import select
from ..models import KbKnowledge, KbProduct
from sqlalchemy import func
from ..models import KbKnowledge, KbProduct, KbPromotionAudit
from . import textnorm
UNVERIFIED, EVIDENCE_ONLY, VERIFIED_KB, CONFLICT, STALE = "UNVERIFIED", "EVIDENCE_ONLY", "VERIFIED_KB", "CONFLICT", "STALE"
CLIENT_FAMILY = "CLIENT_OBSERVATION"
UNVERIFIED, OBSERVED, CORROBORATED, EXTERNAL_CORROBORATED, VERIFIED_KB, CONFLICT, STALE = "UNVERIFIED", "OBSERVED", "CORROBORATED", "EXTERNAL_CORROBORATED", "VERIFIED_KB", "CONFLICT", "STALE"
EVIDENCE_ONLY = OBSERVED # legacy name
CLIENT_FAMILY = "CLIENT_DATA" # every client-derived lineage (master data + fiscal documents of the same ERP are NOT independent of each other)
RULE_VERSION = "kb-promotion-v2"
STRONG_DOCS, STRONG_YEARS = 20, 2
def _now():
@@ -56,15 +61,29 @@ def upsert_product(session, key: str | None, description, *, brand=None, quantit
if quantity and not product.package_quantity:
product.package_quantity = str(quantity)[:32]
fp = textnorm.fingerprint(description)
norm = textnorm.normalized(description)
if norm:
add_alias(session, product, norm)
if fp:
add_evidence(session, product, "name_fingerprint", fp, "IDENTITY", family="IDENTITY", status=EVIDENCE_ONLY)
add_evidence(session, product, "name_fingerprint", fp, "IDENTITY", family="IDENTITY", status=OBSERVED)
return product
def add_evidence(session, product: KbProduct, key: str, value: str, source: str, *, family: str, client_label: str | None = None, status: str = EVIDENCE_ONLY, info: dict | None = None) -> KbKnowledge:
def add_alias(session, product: KbProduct, normalized: str, cap: int = 20):
"""Non-fiscal aliases of the same GTIN (descriptions seen over the years). Helps name matching; never authority for a fiscal write."""
if not normalized or session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "alias", KbKnowledge.value == normalized[:200])):
return
if (session.scalar(select(func.count()).select_from(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "alias")) or 0) >= cap:
return
session.add(KbKnowledge(kb_product_id=product.id, knowledge_key="alias", value=normalized[:200], source="ALIAS", status=OBSERVED, evidence={"families": {}, "clients": [], "info": {}}))
def add_evidence(session, product: KbProduct, key: str, value: str, source: str, *, family: str, client_label: str | None = None, status: str = OBSERVED, info: dict | None = None,
lineage: dict | None = None) -> KbKnowledge:
"""lineage (client data only): {"master": 0/1, "docs": n, "years": {"2024": n}} of ONE client for this (GTIN, value); merged, never double counted."""
row = session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == key, KbKnowledge.value == value))
if row is None:
row = KbKnowledge(kb_product_id=product.id, knowledge_key=key, value=value, source=source[:32], status=status, evidence={"families": {}, "clients": [], "info": info or {}})
row = KbKnowledge(kb_product_id=product.id, knowledge_key=key, value=value, source=source[:32], status=status, evidence={"families": {}, "clients": [], "lineages": {}, "info": info or {}})
session.add(row)
session.flush()
evidence = dict(row.evidence or {})
@@ -73,48 +92,128 @@ def add_evidence(session, product: KbProduct, key: str, value: str, source: str,
clients = list(evidence.get("clients") or [])
if client_label and client_label not in clients:
clients.append(client_label)
row.evidence = {**evidence, "families": families, "clients": clients, "info": {**(evidence.get("info") or {}), **(info or {})}}
lineages = dict(evidence.get("lineages") or {})
if client_label and lineage:
cur = dict(lineages.get(client_label) or {"master": 0, "docs": 0, "years": {}})
cur["master"] = max(int(cur.get("master", 0)), int(lineage.get("master", 0)))
cur["docs"] = int(cur.get("docs", 0)) + int(lineage.get("docs", 0))
years = dict(cur.get("years") or {})
for y, n in (lineage.get("years") or {}).items():
years[str(y)] = int(years.get(str(y), 0)) + int(n)
cur["years"] = years
lineages[client_label] = cur
row.evidence = {**evidence, "families": families, "clients": clients, "lineages": lineages, "info": {**(evidence.get("info") or {}), **(info or {})}}
return row
def _lineage_summary(rows):
clients, docs, years = set(), 0, {}
for r in rows:
ev = r.evidence or {}
for c, lin in (ev.get("lineages") or {}).items():
clients.add(c)
docs += int(lin.get("docs", 0))
for y, n in (lin.get("years") or {}).items():
years[y] = years.get(y, 0) + int(n)
for c in ev.get("clients") or []:
clients.add(c)
return {"clients": sorted(clients), "docs": docs, "years": sorted(years), "last_year": max(int(y) for y in years) if years else None}
def independent_count(rows) -> tuple[int, list[str]]:
families = {}
clients: set[str] = set()
for row in rows:
ev = row.evidence or {}
for fam in (ev.get("families") or {}):
if fam.startswith("PROVIDER:"):
families[fam] = True
elif fam == CLIENT_FAMILY:
clients.update(ev.get("clients") or [])
names = sorted(families)
if len(clients) >= 2:
names.append(f"{CLIENT_FAMILY}(x{len(clients)})")
"""Independent families: each NCM-bearing provider = 1; ALL client-derived data = 1 and only when STRONG (>= STRONG_DOCS documents over >= STRONG_YEARS years, or >= 2 distinct clients)."""
names = sorted({f for r in rows for f in ((r.evidence or {}).get("families") or {}) if f.startswith("PROVIDER:")})
summary = _lineage_summary(rows)
has_client = any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in rows)
if has_client and (len(summary["clients"]) >= 2 or (summary["docs"] >= STRONG_DOCS and len(summary["years"]) >= STRONG_YEARS)):
names.append(CLIENT_FAMILY)
return len(names), names
def _official_period(session, value):
"""CURRENT_VALID / HISTORICAL_EXPIRED (exists in the dataset but not vigente today) / HISTORICAL_INVALID (never in the dataset)."""
from .resolver import ncm_exists_any
return "CURRENT_VALID" if ncm_exists_any(session, value, current=True) else ("HISTORICAL_EXPIRED" if ncm_exists_any(session, value, current=False) else "HISTORICAL_INVALID")
def _mark(row, state):
row.evidence = {**(row.evidence or {}), "info": {**((row.evidence or {}).get("info") or {}), "historical_state": state}}
def evaluate(session, product: KbProduct, key: str, is_official) -> dict:
"""Recomputes and PERSISTS the status of one field. Returns {value, status, families, candidates, rejected}."""
"""Recomputes and PERSISTS the status of one field. Never 'majority wins': two current-valid values => CONFLICT."""
rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == key)))
if not rows:
return {"value": None, "status": UNVERIFIED, "families": [], "candidates": [], "rejected": []}
return {"value": None, "status": UNVERIFIED, "families": [], "candidates": [], "rejected": [], "detail": {}}
valid = [r for r in rows if is_official(r.value)]
rejected = [r.value for r in rows if not is_official(r.value)]
for r in rows:
if r not in valid:
state = _official_period(session, r.value) if key == "ncm" else "HISTORICAL_INVALID"
r.status = STALE if state == "HISTORICAL_EXPIRED" else OBSERVED
_mark(r, state)
values = sorted({r.value for r in valid})
if len(values) > 1:
for r in rows:
for r in valid:
r.status = CONFLICT
return {"value": None, "status": CONFLICT, "families": [], "candidates": values, "rejected": rejected}
_mark(r, "HISTORICAL_CONFLICT")
return {"value": None, "status": CONFLICT, "families": [], "candidates": values, "rejected": rejected, "detail": {"clients": _lineage_summary(valid)["clients"]}}
if not values:
for r in rows:
r.status = EVIDENCE_ONLY
return {"value": None, "status": EVIDENCE_ONLY, "families": [], "candidates": [], "rejected": rejected}
return {"value": None, "status": OBSERVED, "families": [], "candidates": [], "rejected": rejected, "detail": {}}
chosen = [r for r in valid if r.value == values[0]]
old = chosen[0].status
count, names = independent_count(chosen)
summary = _lineage_summary(chosen)
other_last = _lineage_summary([r for r in rows if r not in chosen])["last_year"]
superseded = other_last is not None and summary["last_year"] is not None and other_last >= summary["last_year"] # a different (expired) NCM is at least as recent: not stable
if superseded and CLIENT_FAMILY in names:
names.remove(CLIENT_FAMILY)
count -= 1
providers = [n for n in names if n.startswith("PROVIDER:")]
already = any(r.status == VERIFIED_KB and r.validated_at for r in chosen)
status = VERIFIED_KB if (count >= 2 or already) else EVIDENCE_ONLY
for r in rows:
r.status = status if r.value == values[0] else EVIDENCE_ONLY
if status == VERIFIED_KB and r.value == values[0] and not r.validated_at:
has_client_data = any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in chosen)
if count >= 2 or already:
status = VERIFIED_KB
elif providers and has_client_data:
status = EXTERNAL_CORROBORATED
elif CLIENT_FAMILY in names:
status = CORROBORATED
else:
status = OBSERVED
for r in chosen:
r.status = status
_mark(r, "HISTORICAL_CHANGED" if superseded else "HISTORICAL_STABLE_CURRENT")
if status == VERIFIED_KB and not r.validated_at:
r.validated_at = _now()
return {"value": values[0], "status": status, "families": names, "candidates": values, "rejected": rejected}
if status == VERIFIED_KB and old != VERIFIED_KB and key == "ncm":
from .resolver import dataset_versions
session.add(KbPromotionAudit(kb_product_id=product.id, gtin=product.gtin, field=key, value=values[0], old_status=old, new_status=status, families=names,
sources={"clients": summary["clients"], "documents": summary["docs"], "years": summary["years"], "providers": providers},
official_dataset=dataset_versions(session).get("NCM", {}), rule_version=RULE_VERSION))
return {"value": values[0], "status": status, "families": names, "candidates": values, "rejected": rejected, "detail": summary}
def revalidate(session, is_official) -> dict:
"""After the OFFICIAL NCM dataset changed: never delete knowledge. Rows whose NCM is no longer vigente become STALE (STALE_REVIEW_REQUIRED, audited when they were VERIFIED_KB);
rows still vigente keep their status (VERIFIED_KB stays valid while the NCM is vigente)."""
from .resolver import dataset_versions
stale = still = 0
versions = dataset_versions(session).get("NCM", {})
for row in session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm")):
if is_official(row.value):
still += 1
if row.status == STALE:
row.status = OBSERVED # became vigente again: needs a fresh evaluate() to regain its level
continue
if row.status != STALE:
was = row.status
row.status = STALE
_mark(row, "HISTORICAL_EXPIRED")
row.evidence = {**(row.evidence or {}), "info": {**((row.evidence or {}).get("info") or {}), "stale_reason": "STALE_REVIEW_REQUIRED"}}
stale += 1
if was == VERIFIED_KB:
product = session.get(KbProduct, row.kb_product_id)
session.add(KbPromotionAudit(kb_product_id=row.kb_product_id, gtin=product.gtin if product else None, field="ncm", value=row.value, old_status=was, new_status=STALE,
families=sorted((row.evidence or {}).get("families") or {}), sources={"reason": "NCM_NO_LONGER_VIGENTE"}, official_dataset=versions, rule_version=RULE_VERSION))
session.flush()
return {"stale": stale, "still_vigente": still}
+23
View File
@@ -39,6 +39,8 @@ def provider_sources(session, providers=None) -> list[dict]:
"latency_ms": state.last_latency_ms if state else None,
"cache_hit_rate": round(100 * usage.cache_hits / lookups) if usage and lookups else None,
"errors": usage.errors if usage else 0, "rate_limited": usage.rate_limited if usage else 0,
"found_today": usage.found if usage else 0, "ncm_found_today": usage.ncm_found if usage else 0, "cache_hits": usage.cache_hits if usage else 0,
"next_reset": _aware(state.reset_at) if state and state.reset_at else None, "identity_only": not getattr(provider, "provides_ncm", False),
})
return rows
@@ -80,3 +82,24 @@ def coverage_metrics(session) -> dict:
"NCM_AUTO_FILL_SAFE_RATE": rate(auto_ncm), "NAME_ONLY_RESOLUTION_RATE": rate(states.get("EXACT_KB_NAME_VERIFIED", 0)), "AMBIGUOUS_RATE": rate(states.get("AMBIGUOUS", 0)),
"CONFLICT_RATE": rate(states.get("CONFLICT", 0)), "PROVIDER_QUOTA_BLOCK_RATE": round(100 * limited / requests, 1) if requests else None, "states": states, "by_status": by,
"provider_cache_entries": session.scalar(select(func.count()).select_from(GtinLookupCache)) or 0, "provider_cache_hit_rate": round(100 * hits / (hits + misses), 1) if (hits + misses) else None}
def kb_summary(session) -> dict:
"""Compact Knowledge Base panel: known products, statuses of the NCM knowledge, hits today, and the latest conflicts with their evidence (no dashboard of its own)."""
from ..models import KbKnowledge, KbProduct
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
products = session.scalar(select(func.count()).select_from(KbProduct).where(KbProduct.gtin.is_not(None))) or 0
by = dict(session.execute(select(KbKnowledge.status, func.count(func.distinct(KbKnowledge.kb_product_id))).where(KbKnowledge.knowledge_key == "ncm").group_by(KbKnowledge.status)).all())
hits = session.scalar(select(func.coalesce(func.sum(ProviderUsageDaily.kb_hits), 0)).where(ProviderUsageDaily.day == today)) or 0
conflicts = []
rows = session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.status == "CONFLICT").order_by(KbKnowledge.id.desc()).limit(60)).all()
grouped = {}
for r in rows:
grouped.setdefault(r.kb_product_id, []).append(r)
for pid, items in list(grouped.items())[:5]:
prod = session.get(KbProduct, pid)
conflicts.append({"gtin": (prod.gtin or "")[-13:], "description": (prod.normalized_description or "")[:60],
"candidates": [{"ncm": i.value, "clients": sorted({c for c in (i.evidence or {}).get("clients", [])}), "documents": sum(int(l.get("docs", 0)) for l in ((i.evidence or {}).get("lineages") or {}).values()),
"providers": sorted(f for f in ((i.evidence or {}).get("families") or {}) if f.startswith("PROVIDER:"))} for i in items]})
return {"products": products, "verified": by.get("VERIFIED_KB", 0), "corroborated": by.get("CORROBORATED", 0) + by.get("EXTERNAL_CORROBORATED", 0), "observed": by.get("OBSERVED", 0),
"conflicts": by.get("CONFLICT", 0), "stale": by.get("STALE", 0), "hits_today": hits, "recent_conflicts": conflicts}
+36 -3
View File
@@ -226,10 +226,11 @@ class Cosmos(Provider):
class Oscbr(Provider):
"""OSCBR / RSC Sistemas GTIN API (https://gtin.rscsistemas.com.br). Free plan (no card): 20 req/min; account created BY THE OPERATOR, credentials only in the Hub env
"""OSCBR / RSC Sistemas GTIN API (https://gtin.rscsistemas.com.br). Free plan (no card): 50 calls/day, 5/min; account created BY THE OPERATOR, credentials only in the Hub env
(OSCBR_USER / OSCBR_PASSWORD). Bearer token lasts 1 h and is renewed automatically. Optional: NOT_CONFIGURED without credentials (no call is made)."""
name = "oscbr"
rate_per_minute = 20
rate_per_minute = 5 # free plan as currently documented: 50 calls/day, 5/min (conservative: the lower figure wins)
daily_limit = 50
provides_ncm = True
base = "https://gtin.rscsistemas.com.br"
@@ -287,9 +288,41 @@ class Oscbr(Provider):
raw_hash=hashlib.sha256(json.dumps(body, sort_keys=True, ensure_ascii=False).encode()).hexdigest(), latency_ms=getattr(response, "latency_ms", 0), headers=headers)
class Kodebar(Provider):
"""Kodebar (Korven Sistemas) - IDENTITY source only (name, brand, thumbnail; NO NCM). Free 'Teste' plan: 50 queries/day, no card, personal API key created by the operator
(KODEBAR_API_KEY in the Hub env). Terms (2026-09-26): integration into our own multi-user system and internal caching are allowed; redistribution/resale is not.
429 above the quota, no overage billing. NOT_CONFIGURED without a key: no request is made."""
name = "kodebar"
daily_limit = 50
rate_per_minute = 30
base = "https://kodebar.korvensistemas.com.br"
@property
def configured(self) -> bool:
return bool(self._token)
def lookup_gtin(self, gtin: str) -> ProviderProduct:
if not self.configured:
raise ProviderError("AUTH_ERROR", "NOT_CONFIGURED")
response = self._get(f"{self.base}/gtin/lookup", params={"gtin": gtin}, headers={"X-API-Key": self._token or "", "User-Agent": USER_AGENT})
headers = {k.lower(): v for k, v in response.headers.items()}
if response.status_code == 404:
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
if response.status_code != 200:
raise ProviderError("BAD_RESPONSE", f"HTTP {response.status_code}")
try:
body = response.json()
except ValueError as error:
raise ProviderError("BAD_RESPONSE", "not JSON") from error
if not isinstance(body, dict) or not body.get("nome"):
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
return ProviderProduct(self.name, gtin, True, title=body.get("nome"), brand=body.get("marca"), raw_hash=hashlib.sha256(json.dumps({k: body.get(k) for k in ("gtin", "nome", "marca")}, sort_keys=True, ensure_ascii=False).encode()).hexdigest(),
latency_ms=getattr(response, "latency_ms", 0), headers=headers)
def build_providers(settings, client: httpx.Client | None = None) -> list[Provider]:
"""Fixed order chosen from the benchmark (docs/product_enrichment.md): Cosmos then OSCBR (fiscal-capable) first when configured, then UPCitemdb, then the Open *Facts family (food, general products, beauty; identity only). Paid providers do not exist here."""
providers: list[Provider] = [Cosmos(client, getattr(settings, "cosmos_token", None)), Oscbr(client, None, getattr(settings, "oscbr_user", None), getattr(settings, "oscbr_password", None)), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)]
providers: list[Provider] = [Cosmos(client, getattr(settings, "cosmos_token", None)), Oscbr(client, None, getattr(settings, "oscbr_user", None), getattr(settings, "oscbr_password", None)), Kodebar(client, getattr(settings, "kodebar_api_key", None)), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)]
if any(p.paid for p in providers) and not getattr(settings, "allow_paid_providers", False):
raise RuntimeError("paid providers are not allowed (ALLOW_PAID_PROVIDERS=false)")
return providers
+18 -4
View File
@@ -79,7 +79,7 @@ class QuotaManager:
return row
# ---- decision ----------------------------------------------------------------------------------------------------------------------------------
def allow(self, provider: Provider) -> tuple[bool, str | None, datetime | None]:
def allow(self, provider: Provider, purpose: str = "OPERATIONAL", reserve_percent: int = 0) -> tuple[bool, str | None, datetime | None]:
state, now = self.state(provider), self.now()
if not provider.configured:
return False, "NOT_CONFIGURED", None
@@ -91,6 +91,12 @@ class QuotaManager:
if provider.daily_limit and usage.requests >= provider.daily_limit and state.quota_kind != "REMOTE_QUOTA":
state.status = "RATE_LIMITED"
return False, "RATE_LIMITED", state.reset_at and _aware(state.reset_at) or next_utc_midnight(now)
if purpose == "BACKFILL" and provider.daily_limit: # backfill may only use what is left above the operational reserve
used = usage.requests
if state.quota_kind == "REMOTE_QUOTA" and state.remaining is not None:
used = provider.daily_limit - state.remaining
if used >= provider.daily_limit * (100 - reserve_percent) / 100:
return False, "OPERATIONAL_RESERVE", state.reset_at and _aware(state.reset_at) or next_utc_midnight(now)
window = (provider.burst[1] if provider.burst else 60)
limit = (provider.burst[0] if provider.burst else provider.rate_per_minute)
if limit:
@@ -123,6 +129,10 @@ class QuotaManager:
usage.success += 1
if not result.found:
usage.not_found += 1
else:
usage.found += 1
if result.ncm:
usage.ncm_found += 1
state.consecutive_failures = 0
state.circuit_open_until = None
state.status = "ONLINE"
@@ -147,7 +157,7 @@ class QuotaManager:
usage.rate_limited += 1
state.status = "RATE_LIMITED"
wait = timedelta(seconds=error.retry_after) if error.retry_after else None
state.circuit_open_until = now + (wait or (state.reset_at and max(timedelta(minutes=5), _aware(state.reset_at) - now) or timedelta(minutes=15)))
state.circuit_open_until = now + (wait or (state.reset_at and max(timedelta(minutes=15), _aware(state.reset_at) - now) or timedelta(minutes=60)))
if state.reset_at is None:
state.reset_at = state.circuit_open_until
elif error.kind == "AUTH_ERROR":
@@ -165,6 +175,10 @@ class QuotaManager:
state.status = "OFFLINE"
self.session.commit()
def record_kb_hit(self):
"""A resolution answered by the KB alone (no provider asked): counted once per day under the pseudo-provider 'kb'."""
self.usage("kb").kb_hits += 1
def record_cache(self, provider_name: str, hit: bool):
usage = self.usage(provider_name)
if hit:
@@ -200,7 +214,7 @@ class ProviderGateway:
row.found, row.data, row.fetched_at, row.expires_at = product.found, data, now, now + ttl
self.session.commit()
def lookup(self, provider: Provider, gtin: str) -> Outcome:
def lookup(self, provider: Provider, gtin: str, purpose: str = "OPERATIONAL", reserve_percent: int = 0) -> Outcome:
hit = self._cached(provider, gtin)
if hit is not False:
self.quota.record_cache(provider.name, True)
@@ -216,7 +230,7 @@ class ProviderGateway:
self.session.commit()
return Outcome(provider.name, "CACHE_HIT", hit or None, from_cache=True) if hit else Outcome(provider.name, "NOT_FOUND", None, from_cache=True)
self.quota.record_cache(provider.name, False)
ok, reason, retry_at = self.quota.allow(provider)
ok, reason, retry_at = self.quota.allow(provider, purpose, reserve_percent)
if not ok:
self.session.commit()
return Outcome(provider.name, "SKIPPED", None, reason, retry_at)
+23 -1
View File
@@ -56,6 +56,16 @@ class _ScopedDatasets(DatasetService):
return self.session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == self.code)).all()
def ncm_exists_any(session, code: str, *, current: bool, today: date | None = None) -> bool:
"""current=True: vigente today; current=False: present in the ACTIVE dataset with any validity window (i.e. known but not vigente)."""
code = normalize_code(code)
if len(code) != 8:
return False
if current:
return ncm_is_official(session, code, today)
return session.scalar(select(NcmEntry.id).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == code).limit(1)) is not None
def dataset_versions(session) -> dict:
return {d.kind: {"sha256": (d.sha256 or "")[:12], "version": d.version_label} for d in session.scalars(select(Dataset).where(Dataset.status == "ACTIVE"))}
@@ -77,7 +87,8 @@ def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current:
elif ncm_eval["value"] and ncm_eval["status"] == kb.VERIFIED_KB:
out["CLASSIFICACAO_FISCAL"] = _field(AUTO, ncm_eval["value"], "NCM_KB_VERIFIED" if gtin_kb_verified or name_verified else "NCM_TWO_INDEPENDENT_SOURCES", kb.VERIFIED_KB, ncm_eval["families"])
elif ncm_eval["value"]:
out["CLASSIFICACAO_FISCAL"] = _field(SUGGEST, ncm_eval["value"], "NCM_SINGLE_SOURCE", kb.EVIDENCE_ONLY, ncm_eval["families"], "apenas uma fonte independente")
rule = {"CORROBORATED": "NCM_CORROBORATED_LOCAL_HISTORY", "EXTERNAL_CORROBORATED": "NCM_EXTERNAL_CORROBORATED"}.get(ncm_eval["status"], "NCM_SINGLE_SOURCE")
out["CLASSIFICACAO_FISCAL"] = _field(SUGGEST, ncm_eval["value"], rule, ncm_eval["status"], ncm_eval["families"], "evidência ainda insuficiente para preencher automaticamente")
else:
out["CLASSIFICACAO_FISCAL"] = _field(INSUFFICIENT, reason="nenhuma fonte com NCM válido no dataset oficial")
ncm = out["CLASSIFICACAO_FISCAL"]
@@ -124,14 +135,25 @@ def resolve(session, gateway: ProviderGateway, *, gtin_raw, description, current
product = kb.find_by_gtin(session, key)
ev = kb.evaluate(session, product, "ncm", is_official) if product else None
if ev and ev["status"] == kb.VERIFIED_KB:
gateway.quota.record_kb_hit()
result["state"] = "EXACT_GTIN_VERIFIED"
result["fields"] = _fiscal_decisions(session, ev, True, current, description or "")
session.commit()
return result
if ev and ev["status"] == kb.CONFLICT: # local evidence already conflicts: an external answer cannot settle it, so spend no quota
result["state"] = "CONFLICT"
result["fields"] = _fiscal_decisions(session, ev, False, current, description or "")
gateway.quota.record_kb_hit()
session.commit()
return result
locally_known = bool(ev and ev["status"] in (kb.CORROBORATED, kb.EXTERNAL_CORROBORATED)) # identity + NCM candidate already local: only an NCM-bearing source can still add value
query = g["gtin"] if len(g["gtin"]) in (8, 14) else key[1:] # providers/caches use the canonical EAN-13 (UPC-12 is zero-padded to it)
found, pending = [], []
if allow_external:
for provider in gateway.providers:
if locally_known and not getattr(provider, "provides_ncm", False):
result["providers"].append({"provider": provider.name, "state": "SKIPPED", "reason": "LOCAL_IDENTITY_KNOWN", "from_cache": False})
continue
if budget_s is not None and time.monotonic() - started > budget_s:
result["providers"].append({"provider": provider.name, "state": "SKIPPED", "reason": "TIME_BUDGET", "from_cache": False})
pending.append(type("P", (), {"reason": "TIME_BUDGET", "retry_at": None})())
+2 -1
View File
@@ -7,6 +7,7 @@ UNIT_MAP = {"LT": "L", "LTS": "L", "LITRO": "L", "LITROS": "L", "L": "L", "ML":
ABBREVIATIONS = {"REFRIG": "REFRIGERANTE", "REFRI": "REFRIGERANTE", "CHOC": "CHOCOLATE", "BISC": "BISCOITO", "SAB": "SABONETE", "DET": "DETERGENTE", "SH": "SHAMPOO", "COND": "CONDICIONADOR",
"MACARR": "MACARRAO", "MAC": "MACARRAO", "AMAC": "AMACIANTE", "DESOD": "DESODORANTE", "PAP": "PAPEL", "HIG": "HIGIENICO", "ORIG": "ORIGINAL", "TRAD": "TRADICIONAL"}
STOP = {"DE", "DA", "DO", "DAS", "DOS", "E", "COM", "C", "PARA", "P", "EM", "A", "O"}
MARKETING = {"ORIGINAL", "TRADICIONAL", "NOVO", "NOVA", "PROMO", "PROMOCAO", "OFERTA"} # ignored ONLY by the fingerprint; a different pack/flavour still differs
_QTY = re.compile(r"^(\d+(?:[.,]\d+)?)(ML|L|KG|G|MG|UN|PCT|CX|LT|LTS|GR)$")
@@ -35,7 +36,7 @@ def normalized(description) -> str:
def fingerprint(description) -> str | None:
toks = tokens(description)
toks = [t for t in tokens(description) if t not in MARKETING]
if not toks:
return None
return hashlib.sha256(" ".join(sorted(toks)).encode("utf-8")).hexdigest()
+21 -1
View File
@@ -629,7 +629,7 @@ class KbKnowledge(Base):
dataset_versions: Mapped[dict] = mapped_column(JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
validated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
status: Mapped[str] = mapped_column(String(16), default="EVIDENCE_ONLY", server_default="EVIDENCE_ONLY", index=True) # UNVERIFIED | EVIDENCE_ONLY | VERIFIED_KB | CONFLICT | STALE
status: Mapped[str] = mapped_column(String(24), default="OBSERVED", server_default="OBSERVED", index=True) # OBSERVED | CORROBORATED | EXTERNAL_CORROBORATED | VERIFIED_KB | CONFLICT | STALE
class KbObservation(Base):
@@ -879,6 +879,9 @@ class ProviderUsageDaily(Base):
timeouts: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
cache_hits: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
cache_misses: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
found: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
ncm_found: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
kb_hits: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
class GtinLookupCache(Base):
@@ -907,3 +910,20 @@ class SourceRefreshRun(Base):
version_label: Mapped[str | None] = mapped_column(String(64), nullable=True)
record_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
error: Mapped[str | None] = mapped_column(String(300), nullable=True)
class KbPromotionAudit(Base):
"""Why a GTIN/NCM became (or stopped being) trustworthy: statuses, evidence families, source references and the official dataset hash at that moment."""
__tablename__ = "kb_promotion_audits"
id: Mapped[int] = mapped_column(primary_key=True)
kb_product_id: Mapped[int] = mapped_column(ForeignKey("kb_products.id"), index=True)
gtin: Mapped[str | None] = mapped_column(String(16), nullable=True, index=True)
field: Mapped[str] = mapped_column(String(24), default="ncm", server_default="ncm")
value: Mapped[str | None] = mapped_column(String(64), nullable=True)
old_status: Mapped[str | None] = mapped_column(String(24), nullable=True)
new_status: Mapped[str] = mapped_column(String(24))
families: Mapped[list] = mapped_column(JSON, default=list)
sources: Mapped[dict] = mapped_column(JSON, default=dict)
official_dataset: Mapped[dict] = mapped_column(JSON, default=dict)
rule_version: Mapped[str] = mapped_column(String(32))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+1 -1
View File
@@ -4,5 +4,5 @@ kept identical to this) and what an AgentRelease row's `version` column holds
once published — never a second, independent version number. Fiscal semantics
are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py,
DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump."""
GTI_FISCAL_VERSION = "1.3.1"
GTI_FISCAL_VERSION = "1.3.2"
API_COMPAT_MAJOR_MINOR = "1.0" # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x
+2 -2
View File
@@ -22,7 +22,7 @@ from .panel_presentation import format_ui_datetime
from .models import AdminUser, Agent, AgentEnrollment, AgentRelease, AgentSnapshot, BlockedProduct, FiscalContextEntry, Installation, Job, ProductEnrichment, ProductFieldAudit, RtcCandidate, RtcWriteAudit, Tenant
from .jobs import create_job
from .audit import record_audit
from .enrichment.observability import coverage_metrics, official_sources, provider_sources
from .enrichment.observability import coverage_metrics, kb_summary, official_sources, provider_sources
def router(session_factory):
@@ -161,7 +161,7 @@ def router(session_factory):
def dashboard(request: Request):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try: return render(request, "dashboard.html", {"summary": dashboard_summary(session), "auth": auth_data, "sources": provider_sources(session), "official": official_sources(session), "enrich_metrics": coverage_metrics(session)})
try: return render(request, "dashboard.html", {"summary": dashboard_summary(session), "auth": auth_data, "sources": provider_sources(session), "official": official_sources(session), "enrich_metrics": coverage_metrics(session), "kb": kb_summary(session)})
finally: session.close()
@routes.get("/agents", response_class=HTMLResponse)
+1 -1
View File
@@ -1,6 +1,6 @@
[project]
name = "gti-fiscal-hub"
version = "1.3.1"
version = "1.3.2"
requires-python = ">=3.10"
dependencies = [
"fastapi>=0.115,<1",
+41
View File
@@ -0,0 +1,41 @@
"""Import historical ERP evidence (miner output) into the central KB. Dry-run by default; --apply commits. Local data only, no provider call.
python -m central.scripts.import_kb_history --staging /path/to/history [--apply] [--report out.json]
Runs against the database in DATABASE_URL (the Hub's PostgreSQL). Idempotent. Never touches any client database.
"""
import argparse
import json
import sys
import time
from pathlib import Path
from central.app.db import SessionLocal
from central.app.enrichment import history
def main(argv=None):
ap = argparse.ArgumentParser()
ap.add_argument("--staging", required=True)
ap.add_argument("--apply", action="store_true")
ap.add_argument("--report", default=None)
ap.add_argument("--only-current-vigente-gtins", action="store_true", help="skip GTINs without any NCM evidence")
args = ap.parse_args(argv)
files = sorted(f for f in Path(args.staging).glob("*.json") if not f.name.startswith("_"))
stagings = [json.loads(f.read_text(encoding="utf-8")) for f in files]
t0 = time.time()
obs = history.build_observations(stagings)
if args.only_current_vigente_gtins:
obs["gtins"] = {k: v for k, v in obs["gtins"].items() if v["ncm"]}
with SessionLocal() as session:
current = history.official_sets(session)
report = history.apply(session, obs, lambda v: v in current, dry_run=not args.apply)
report.update({"mode": "APPLY" if args.apply else "DRY_RUN", "clients": [s["client"] for s in stagings], "build_stats": obs["stats"], "seconds": round(time.time() - t0, 1)})
text = json.dumps(report, indent=1, default=str)
if args.report:
Path(args.report).write_text(text, encoding="utf-8")
print(text)
if __name__ == "__main__":
sys.exit(main())
+4
View File
@@ -19,6 +19,7 @@
<small>{% if src.limit %}{{ src.used }} / {{ src.limit }} consultas hoje · restam {{ src.remaining if src.remaining is not none else "—" }}{% if src.reset_at %} · renova {{ src.reset_at.strftime("%d/%m %H:%M") }} UTC{% endif %}{% elif src.rate_per_minute %}{{ src.rate_per_minute }} consultas/min (limite documentado) · {{ src.used }} hoje{% else %}{{ src.used }} consultas hoje{% endif %}
{% if src.latency_ms %} · {{ src.latency_ms }} ms{% endif %}{% if src.cache_hit_rate is not none %} · cache {{ src.cache_hit_rate }}%{% endif %}
{% if src.last_success_at %} · último sucesso {{ src.last_success_at.strftime("%d/%m %H:%M") }}{% endif %}{% if src.last_error %} · último erro: {{ src.last_error }}{% endif %}
· hoje: {{ src.found_today }} achados{% if not src.identity_only %}, {{ src.ncm_found_today }} com NCM{% else %} (só identidade){% endif %}, {{ src.cache_hits }} do cache, {{ src.rate_limited }} limitados{% if src.next_reset %} · próximo reset {{ src.next_reset.strftime("%d/%m %H:%M") }} UTC{% endif %}
· configurado: {{ "sim" if src.configured else "não" }}</small></span></div>{% endfor %}
</article>
<article class="panel">
@@ -27,6 +28,9 @@
<small>{% if o.active %}{{ o.records }} registros · {{ o.version }} · {{ o.sha256 }}{% if o.last_update %} · baixada {{ o.last_update.strftime("%d/%m/%Y") }}{% endif %}{% endif %}{% if o.auto_refresh %} · atualização automática diária{% endif %}{% if o.error %} · {{ o.error }}{% endif %}</small></span></div>{% endfor %}
</article>
</section>
<section class="panel" id="kb-resumo"><div class="panel-title"><div><h2>Base de conhecimento de produtos</h2><p class="muted">Evidência local, histórica e externa; nada vira verdade fiscal sem regra de promoção.</p></div></div>
<div class="coverage"><div class="coverage-item"><strong>{{ kb.products }}</strong><span>Produtos conhecidos</span></div><div class="coverage-item"><strong>{{ kb.verified }}</strong><span>GTINs verificados</span></div><div class="coverage-item"><strong>{{ kb.corroborated }}</strong><span>Corroborados</span></div><div class="coverage-item"><strong>{{ kb.conflicts }}</strong><span>Conflitos</span></div><div class="coverage-item"><strong>{{ kb.hits_today }}</strong><span>Resolvidos pela base hoje</span></div></div>
{% for c in kb.recent_conflicts %}<div class="list-row"><span><b>{{ c.gtin }}</b> {{ c.description }}<small>NCMs divergentes, nada é preenchido: {% for x in c.candidates %}{{ x.ncm }} ({{ x.clients|length }} cliente(s), {{ x.documents }} doc.{% if x.providers %}, {{ x.providers|join("/") }}{% endif %}){% if not loop.last %} × {% endif %}{% endfor %}</small></span></div>{% endfor %}</section>
<section class="panel" id="enriquecimento-resumo"><div class="panel-title"><div><h2>Enriquecimento de produtos novos</h2><p class="muted">Cobertura medida sobre os produtos novos acompanhados (o catálogo histórico não entra).</p></div></div>
{% if enrich_metrics.tracked %}<div class="coverage">{% for key,label in [("GTIN_VALID_RATE","Com GTIN válido"),("KB_HIT_RATE","Resolvidos pela base"),("PRODUCT_FOUND_RATE","Produto identificado"),("NCM_AUTO_FILL_SAFE_RATE","NCM preenchível com segurança"),("AMBIGUOUS_RATE","Ambíguos"),("CONFLICT_RATE","Em conflito")] %}<div class="coverage-item"><strong>{{ enrich_metrics[key] if enrich_metrics[key] is not none else "—" }}{% if enrich_metrics[key] is not none %}%{% endif %}</strong><span>{{ label }}</span></div>{% endfor %}</div>{% else %}<p class="empty-state">Nenhum produto novo acompanhado ainda.</p>{% endif %}</section>
<section class="split overview-split">
+146
View File
@@ -0,0 +1,146 @@
"""Historical evidence -> KB: temporal validity, lineage independence, conflicts, promotion audit, idempotent import. No provider is ever contacted."""
from sqlalchemy import select
from sqlalchemy.orm import Session
from central.app.enrichment import history, kb
from central.app.enrichment.resolver import resolve
from central.app.enrichment.quota import ProviderGateway
from central.app.models import KbKnowledge, KbProduct, KbPromotionAudit
from central.tests.test_product_enrichment_resolver import COSMOS, Fake, G1, G2, G3, env, gtin, seed # noqa: F401
G = "0" + G1 # 14-digit key
def staging(client, gtin13, ncm, *, master=True, years=None, nfe=0, desc="REFRIGERANTE COLA LATA 350ML", code="1"):
return {"client": client, "master": [{"code": code, "desc": desc, "gtin": gtin13, "codbarra": "", "ncm": ncm if master else "", "cest": ""}],
"nfce": [{"code": code, "year": y, "ncm": ncm, "cest": "", "n": n} for y, n in (years or {}).items()], "nfe": [{"code": code, "codbarra": gtin13, "ncm": ncm, "desc": desc, "n": nfe}] if nfe else [], "barcodes": []}
def run(session, stagings, dry=False):
obs = history.build_observations(stagings)
cur = history.official_sets(session)
return history.apply(session, obs, lambda v: v in cur, dry_run=dry)
def status(session, key=G):
p = session.scalar(select(KbProduct).where(KbProduct.gtin == key))
cur = history.official_sets(session)
return kb.evaluate(session, p, "ncm", lambda v: v in cur)
def test_single_client_multi_year_history_is_corroborated_never_verified(env):
r = run(env, [staging("A", G1, "22021000", years={2023: 300, 2024: 700, 2025: 900})])
assert r["by_status"] == {"CORROBORATED": 1} and r["WOULD_VERIFY"] == 0
assert env.scalars(select(KbPromotionAudit)).all() == []
def test_master_plus_documents_of_same_erp_are_one_family(env):
run(env, [staging("A", G1, "22021000", master=True, years={2025: 5})]) # few docs, one year: weak
assert status(env)["status"] == kb.OBSERVED and status(env)["families"] == []
def test_two_clients_corroborate_but_three_clients_alone_do_not_verify(env):
r = run(env, [staging(c, G1, "22021000", years={2024: 2}) for c in ("A", "B", "C")])
assert r["by_status"] == {"CORROBORATED": 1} and r["multi_client"] == 1 and r["WOULD_VERIFY"] == 0
def test_history_plus_independent_provider_verifies_and_is_audited(env):
run(env, [staging("A", G1, "22021000", years={2023: 50, 2024: 60})])
p = env.scalar(select(KbProduct).where(KbProduct.gtin == G))
kb.add_evidence(env, p, "ncm", "22021000", "provider:oscbr", family="PROVIDER:oscbr")
ev = status(env)
assert ev["status"] == kb.VERIFIED_KB and set(ev["families"]) == {"PROVIDER:oscbr", "CLIENT_DATA"}
audit = env.scalars(select(KbPromotionAudit)).one()
assert audit.new_status == "VERIFIED_KB" and audit.rule_version == kb.RULE_VERSION and audit.official_dataset["sha256"] and audit.sources["documents"] == 110
status(env)
assert len(env.scalars(select(KbPromotionAudit)).all()) == 1 # no duplicate audit
def test_different_current_ncms_are_a_conflict_not_majority(env):
r = run(env, [staging("A", G1, "22021000", years={2024: 900, 2025: 900}), staging("B", G1, "19059090", years={2024: 5, 2025: 5}), staging("C", G1, "22021000", years={2024: 900})])
assert r["CONFLICTS"] == 1
res = resolve(env, ProviderGateway(env, []), gtin_raw=G1, description="REFRIGERANTE COLA LATA 350ML", current={}, allow_external=False)
assert res["state"] == "CONFLICT" and res["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "CONFLICT"
def test_expired_ncm_is_history_not_a_candidate(env):
run(env, [staging("A", G1, "11111111", years={2019: 900, 2020: 900})]) # expired in the fixture dataset
ev = status(env)
assert ev["value"] is None and ev["status"] == kb.OBSERVED
row = env.scalar(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm"))
assert row.evidence["info"]["historical_state"] == "HISTORICAL_EXPIRED" and row.status == kb.STALE
def test_ncm_changed_over_time_prefers_current(env):
run(env, [staging("A", G1, "11111111", years={2019: 900, 2020: 900}), staging("A", G1, "22021000", master=False, years={2024: 50, 2025: 60})])
ev = status(env)
assert ev["value"] == "22021000" and ev["status"] == kb.CORROBORATED
def test_stale_ncm_with_equal_recency_does_not_count_as_stable(env):
run(env, [staging("A", G1, "22021000", master=False, years={2024: 50, 2025: 60}), staging("B", G1, "11111111", master=False, years={2025: 500})])
assert status(env)["status"] in (kb.OBSERVED,) # the old code is as recent: not stable, so not even corroborated
def test_import_is_idempotent_and_dry_run_writes_nothing(env):
st = [staging("A", G1, "22021000", years={2023: 300, 2024: 700}), staging("B", G3, "19059090", desc="BISCOITO RECHEADO 130G", code="9", years={2024: 30, 2025: 30})]
dry = run(env, st, dry=True)
assert env.scalars(select(KbProduct)).all() == []
a, b = run(env, st), run(env, st)
assert a["by_status"] == b["by_status"] == dry["by_status"]
assert len(env.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm")).all()) == 2
row = env.scalar(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.value == "22021000"))
assert row.evidence["lineages"]["A"]["docs"] == 1000 # replaced, not doubled
def test_aliases_and_no_company_tax_data_stored(env):
run(env, [staging("A", G1, "22021000", years={2024: 30, 2025: 30}, desc="COCA COLA LT 350"), staging("B", G1, "22021000", master=False, desc="REFRIG COCA-COLA 350ML", code="2", years={2025: 5})])
p = env.scalar(select(KbProduct).where(KbProduct.gtin == G))
aliases = {r.value for r in env.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == p.id, KbKnowledge.knowledge_key == "alias"))}
assert len(aliases) >= 2
blob = repr([r.evidence for r in env.scalars(select(KbKnowledge))]).upper()
assert "CFOP" not in blob and "CSOSN" not in blob and "CRT" not in blob
def test_kb_hit_makes_zero_external_requests(env):
run(env, [staging("A", G1, "22021000", years={2023: 50, 2024: 60})])
p = env.scalar(select(KbProduct).where(KbProduct.gtin == G))
kb.add_evidence(env, p, "ncm", "22021000", "provider:oscbr", family="PROVIDER:oscbr")
env.commit()
spy = Fake("cosmos", COSMOS(), True)
r = resolve(env, ProviderGateway(env, [spy]), gtin_raw=G1, description="REFRIGERANTE COLA LATA 350ML", current={})
assert r["state"] == "EXACT_GTIN_VERIFIED" and spy.calls == 0 and r["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "AUTO_FILL_SAFE"
def test_revalidation_marks_stale_review_and_keeps_knowledge(env):
run(env, [staging("A", G1, "22021000", years={2023: 50, 2024: 60})])
p = env.scalar(select(KbProduct).where(KbProduct.gtin == G))
kb.add_evidence(env, p, "ncm", "22021000", "provider:oscbr", family="PROVIDER:oscbr")
assert status(env)["status"] == kb.VERIFIED_KB
out = kb.revalidate(env, lambda v: True)
assert out["stale"] == 0 and status(env)["status"] == kb.VERIFIED_KB # NCM still vigente: stays valid
out = kb.revalidate(env, lambda v: False) # the official dataset dropped it
row = env.scalar(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm"))
assert out["stale"] == 1 and row.status == kb.STALE and row.evidence["info"]["stale_reason"] == "STALE_REVIEW_REQUIRED"
assert [a.new_status for a in env.scalars(select(KbPromotionAudit)).all()] == ["VERIFIED_KB", "STALE"]
def test_provider_disagreeing_with_verified_kb_creates_conflict_not_overwrite(env):
run(env, [staging("A", G1, "22021000", years={2023: 50, 2024: 60})])
p = env.scalar(select(KbProduct).where(KbProduct.gtin == G))
kb.add_evidence(env, p, "ncm", "22021000", "provider:oscbr", family="PROVIDER:oscbr")
assert status(env)["status"] == kb.VERIFIED_KB
kb.add_evidence(env, p, "ncm", "19059090", "provider:cosmos", family="PROVIDER:cosmos")
ev = status(env)
assert ev["status"] == kb.CONFLICT and ev["value"] is None # write blocked until a human resolves it
def test_aliases_from_history_enable_name_only_match_and_generic_names_stay_ambiguous(env):
run(env, [staging("A", G1, "22021000", years={2023: 50, 2024: 60}, desc="COCA-COLA ORIGINAL LATA 350 ML")])
p = env.scalar(select(KbProduct).where(KbProduct.gtin == G))
kb.add_evidence(env, p, "ncm", "22021000", "provider:oscbr", family="PROVIDER:oscbr")
env.commit()
r = resolve(env, ProviderGateway(env, []), gtin_raw="", description="COCA COLA LATA 350ML", current={})
assert r["state"] == "EXACT_KB_NAME_VERIFIED" and r["fields"]["CLASSIFICACAO_FISCAL"]["value"] == "22021000"
g = resolve(env, ProviderGateway(env, []), gtin_raw="", description="REFRIGERANTE", current={})
assert g["state"] == "AMBIGUOUS" and all(f["status"] != "AUTO_FILL_SAFE" for f in g["fields"].values())
+1 -1
View File
@@ -17,7 +17,7 @@ def test_health_is_unavailable_when_database_is_down():
def test_version_endpoint_reports_release_and_fiscal_versions():
body = TestClient(create_app(database_probe=lambda: True)).get("/version").json()
assert body["version"] == "1.3.1" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
assert body["version"] == "1.3.2" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
def test_production_refuses_the_dev_token(monkeypatch):
@@ -200,3 +200,18 @@ def test_provider_panel_shows_every_state_and_column(monkeypatch):
page = client.get("/").text
for label in ("Limite atingido", "Instável", "Credencial recusada", "Indisponível", "Não configurado", "100 / 100", "restam 0", "renova", "último sucesso", "último erro", "140 ms", "cache 30%"):
assert label in page, label
def test_dashboard_kb_panel_and_provider_daily_columns(monkeypatch):
from central.app.enrichment import kb as kbm
from central.app.models import KbProduct
client, agent_id, _ = _panel_client()
with client._session_factory() as s:
p = kbm.upsert_product(s, "07891000100103", "COLA LATA 350ML")
kbm.add_evidence(s, p, "ncm", "22021000", "c", family=kbm.CLIENT_FAMILY, client_label="A", lineage={"docs": 900, "years": {"2024": 900}})
kbm.add_evidence(s, p, "ncm", "19059090", "c", family=kbm.CLIENT_FAMILY, client_label="B", lineage={"docs": 30, "years": {"2025": 30}})
kbm.evaluate(s, p, "ncm", lambda v: True)
s.commit()
page = client.get("/").text
assert "Base de conhecimento de produtos" in page and "NCMs divergentes" in page and "22021000" in page and "19059090" in page
assert "achados" in page and "próximo reset" in page or "hoje:" in page
@@ -10,7 +10,7 @@ from sqlalchemy.pool import StaticPool
from central.app.enrichment import kb
from central.app.enrichment.gtin import check_digit_ok
from central.app.enrichment.providers import Oscbr, Cosmos, OpenFoodFacts, Provider, ProviderError, ProviderProduct, UpcItemDb, build_providers
from central.app.enrichment.providers import Kodebar, Oscbr, Cosmos, OpenFoodFacts, Provider, ProviderError, ProviderProduct, UpcItemDb, build_providers
from central.app.enrichment.quota import ProviderGateway
from central.app.enrichment.resolver import resolve
from central.app.models import (Base, CestEntry, CestNcmEntry, Dataset, GtinLookupCache, KbKnowledge, NcmEntry, ProviderUsageDaily, TipiEntry)
@@ -178,9 +178,9 @@ def test_client_consensus_needs_two_distinct_clients(env):
p = kb.upsert_product(env, "0" + G1, "X")
kb.add_evidence(env, p, "ncm", "22021000", "client", family=kb.CLIENT_FAMILY, client_label="A")
kb.add_evidence(env, p, "ncm", "22021000", "client", family=kb.CLIENT_FAMILY, client_label="A")
assert kb.evaluate(env, p, "ncm", lambda v: True)["status"] == kb.EVIDENCE_ONLY
assert kb.evaluate(env, p, "ncm", lambda v: True)["status"] == kb.OBSERVED
kb.add_evidence(env, p, "ncm", "22021000", "client", family=kb.CLIENT_FAMILY, client_label="B")
assert kb.evaluate(env, p, "ncm", lambda v: True)["status"] == kb.EVIDENCE_ONLY # consensus alone is one family, never authority
assert kb.evaluate(env, p, "ncm", lambda v: True)["status"] == kb.CORROBORATED # consensus alone is one family, never authority
kb.add_evidence(env, p, "ncm", "22021000", "provider", family="PROVIDER:cosmos")
assert kb.evaluate(env, p, "ncm", lambda v: True)["status"] == kb.VERIFIED_KB
@@ -223,7 +223,7 @@ def test_cosmos_without_token_is_not_configured_and_costs_nothing(env):
def test_paid_provider_guard():
class S: allow_paid_providers = False; cosmos_token = None
assert [p.name for p in build_providers(S())] == ["cosmos", "oscbr", "upcitemdb", "openfoodfacts", "openproductsfacts", "openbeautyfacts"]
assert [p.name for p in build_providers(S())] == ["cosmos", "oscbr", "kodebar", "upcitemdb", "openfoodfacts", "openproductsfacts", "openbeautyfacts"]
# ---- provider contract tests against recorded fixtures (httpx.MockTransport, no network) ----------------------------------------------------------
@@ -292,3 +292,24 @@ def test_oscbr_optional_and_contract():
with pytest.raises(ProviderError) as e:
bad.lookup_gtin(G1)
assert e.value.kind == "AUTH_ERROR"
def test_kodebar_identity_only_optional_and_contract():
assert not Kodebar(_client(lambda r: pytest.fail("no call expected"))).configured
def handler(request):
assert request.headers["x-api-key"] == "k" and request.url.params["gtin"] == G1
return httpx.Response(200, json={"gtin": G1, "nome": "COCA COLA 350ML", "marca": "Coca", "thumbnail": "u", "barcode_url": "b", "qr_code_url": "q"})
p = Kodebar(_client(handler), "k").lookup_gtin(G1)
assert p.found and p.title == "COCA COLA 350ML" and p.ncm is None and p.cest is None # identity only, never NCM
assert not Kodebar(_client(lambda r: httpx.Response(200, json={})), "k").lookup_gtin(G1).found
with pytest.raises(ProviderError) as e:
Kodebar(_client(lambda r: httpx.Response(429)), "k").lookup_gtin(G1)
assert e.value.kind == "RATE_LIMITED"
def test_backfill_respects_operational_reserve(env):
fake = Fake("cosmos", COSMOS(), True, daily_limit=10)
gw = ProviderGateway(env, [fake])
outs = [gw.lookup(fake, gtin(f"78970000{i:04d}"), "BACKFILL", 20).state for i in range(10)]
assert outs.count("SKIPPED") == 2 and fake.calls == 8 # 20% of 10 kept for new products
assert gw.lookup(fake, gtin("789700009999"), "OPERATIONAL").state in ("FOUND", "NOT_FOUND")
+1 -1
View File
@@ -19,4 +19,4 @@ FB_PASSWORD=
FISCAL_MODE=DRY_RUN
FIREBIRD_WRITES=0
POLL_INTERVAL_SECONDS=5
AGENT_VERSION=1.3.1
AGENT_VERSION=1.3.2
+1 -1
View File
@@ -164,7 +164,7 @@ switch ($Command) {
"GTI_HUB_URL=$hub", "AGENT_TOKEN=$token", "AGENT_ID=$agentId",
"COMPANY_CODE=$code", "COMPANY_CNPJ=$cnpj", "COMPANY_CRT_SOURCE=$crtSource",
"FIREBIRD_DATABASE_PATH=$db", "FB_USER=$fbUser", "FB_PASSWORD=$fbPass",
'FISCAL_MODE=DRY_RUN', 'FIREBIRD_WRITES=0', 'POLL_INTERVAL_SECONDS=5', 'AGENT_VERSION=1.3.1'
'FISCAL_MODE=DRY_RUN', 'FIREBIRD_WRITES=0', 'POLL_INTERVAL_SECONDS=5', 'AGENT_VERSION=1.3.2'
)
Set-Content -Path $EnvFile -Value $lines -Encoding UTF8
}
@@ -19,4 +19,4 @@ FB_PASSWORD=
FISCAL_MODE=DRY_RUN
FIREBIRD_WRITES=0
POLL_INTERVAL_SECONDS=5
AGENT_VERSION=1.3.1
AGENT_VERSION=1.3.2
@@ -19,4 +19,4 @@ FB_PASSWORD=
FISCAL_MODE=DRY_RUN
FIREBIRD_WRITES=0
POLL_INTERVAL_SECONDS=5
AGENT_VERSION=1.3.1
AGENT_VERSION=1.3.2
@@ -19,4 +19,4 @@ FB_PASSWORD=
FISCAL_MODE=DRY_RUN
FIREBIRD_WRITES=0
POLL_INTERVAL_SECONDS=5
AGENT_VERSION=1.3.1
AGENT_VERSION=1.3.2
@@ -19,4 +19,4 @@ FB_PASSWORD=
FISCAL_MODE=DRY_RUN
FIREBIRD_WRITES=0
POLL_INTERVAL_SECONDS=5
AGENT_VERSION=1.3.1
AGENT_VERSION=1.3.2
+1 -1
View File
@@ -1,5 +1,5 @@
# GTI Fiscal Hub — production environment (copy to .env, never commit .env)
GTI_FISCAL_VERSION=1.3.1
GTI_FISCAL_VERSION=1.3.2
# REQUIRED. URL-safe secret (used inside a connection URL): openssl rand -hex 24
POSTGRES_PASSWORD=
+1 -1
View File
@@ -36,7 +36,7 @@ services:
build:
context: ../..
dockerfile: central/Dockerfile
image: gti-fiscal-hub:${GTI_FISCAL_VERSION:-1.3.1}
image: gti-fiscal-hub:${GTI_FISCAL_VERSION:-1.3.2}
restart: unless-stopped
depends_on:
postgres:
+25
View File
@@ -82,3 +82,28 @@ While the function is DISABLED / WAITING_FEATURE_CONFIGURATION the Agent marks a
## Production KB seed strategy
Production KB starts empty (0 products, head 0028 before deploy). The lab databases are **not** imported into production: their observations are `CLIENT_OBSERVATION` evidence only and cannot verify anything alone (93/241 shared GTINs disagree on NCM). The KB grows from provider evidence on real new products; `scripts/enrichment_lab_kb.py --seed-url` exists for a dev/staging seed only.
## Coverage V2 (1.3.2, Hub-only, migration 0030) - historical evidence + KB model
- **Miner** (`scripts/enrichment_history_miner.py`, READ-ONLY, server-side `GROUP BY`): C000025 master, `NFCE_ITENS` (NCM/CEST per document line, dated), `C000062` (NF-e entry items: `CLASSIFICACAO_FISCAL`, `CODBARRA`, `DESCRICAO2`), `C000032`/`C000057` (EAN seen in movements).
Five databases in ~17 s, peak 28 MB Python, 7 MB of output (`docs/runs/enrichment_history_miner_report.json`). No XML stores exist; there is no NF-e header/item table with dates beyond the ones above.
- **Evidence model** (`central/app/enrichment/history.py`, `kb.py`): per (GTIN, value) one *lineage* per client (master flag, document count, documents per year). Master data and the fiscal documents of the same ERP are ONE family (`CLIENT_DATA`); each NCM-bearing provider is another family.
`CLIENT_DATA` counts only when strong (>= 20 documents over >= 2 years, or >= 2 distinct clients) and even then reaches at most `CORROBORATED`.
- **KB states** per NCM: `OBSERVED, CORROBORATED, EXTERNAL_CORROBORATED, VERIFIED_KB, CONFLICT, STALE`. Temporal state per NCM: `HISTORICAL_STABLE_CURRENT, HISTORICAL_CHANGED, HISTORICAL_EXPIRED, HISTORICAL_INVALID` (+ `HISTORICAL_CONFLICT`), judged against the ACTIVE official dataset today.
- **Promotion to VERIFIED_KB** (rule `kb-promotion-v2`): exact valid GTIN + NCM vigente in the official dataset + >= 2 independent families (strong `CLIENT_DATA` + >= 1 NCM-bearing provider, or >= 2 providers) + zero conflicting current value. Every promotion writes `kb_promotion_audits`
(families, clients/documents/years/providers, official dataset version+sha, rule version). Three clients agreeing NEVER verify by themselves. Two different current NCMs = `CONFLICT` (no majority), an expired code that is as recent as the current one blocks "stable".
- **Order per product**: VERIFIED_KB (zero external requests) -> local CONFLICT (zero external, no write) -> locally corroborated (only NCM-bearing providers may still be asked) -> provider cache -> provider (single-flight per GTIN) -> fiscal validation.
Backfill calls can never use the operational reserve (`PROVIDER_OPERATIONAL_RESERVE_PERCENT`, default 20). No mass provider use for the historical seed: it is local-only.
- **Revalidation**: after a new official NCM dataset, `kb.revalidate` never deletes; codes that left the dataset become `STALE` (`STALE_REVIEW_REQUIRED`, audited when they were VERIFIED_KB).
- **Aliases**: descriptions seen over the years are stored as non-fiscal aliases + fingerprints (marketing words such as ORIGINAL/TRADICIONAL ignored) to improve name-only matching; name-only still fills only from a unique VERIFIED_KB product.
- **Import**: `python -m central.scripts.import_kb_history --staging DIR [--apply]` (dry-run by default, idempotent, replaces a client's lineage instead of accumulating). Tenant labels are kept; CRT/CFOP/CSOSN or any client tax attribute is never stored.
- **Panel** (same dashboard): provider columns `FOUND_TODAY, NCM_FOUND_TODAY, CACHE_HITS, RATE_LIMITED, NEXT_RESET` and a compact Knowledge Base block (known products, verified, corroborated, conflicts, KB hits today, latest conflicts with clients/documents/providers).
### Providers added / evaluated in V2
- **Kodebar** (adapter, optional, identity only): `KODEBAR_API_KEY`, free "Teste" plan 50/day (+10% tolerance not used), no card, `GET /gtin/lookup?gtin=` with `X-API-Key`; terms allow integration in our own system and internal caching, forbid redistribution; 429 above quota, no overage. Returns name/brand/thumbnail, **no NCM**.
- **OSCBR**: adapter limits set to the current free plan (50/day, 5/min). `NOT_CONFIGURED` without `OSCBR_USER/OSCBR_PASSWORD`.
- **UPCitemdb**: after a 429 the circuit stays open >= 1 h (never insist); the observed trial limit is stricter than the documented burst.
- **Tabelas Fiscais**: NOT integrated at runtime. Our NCM/TIPI/CEST datasets already come from the official sources and live locally; the site is a third-party mirror, so it would add no authority. If ever needed, use its downloadable CSV/JSON (local dataset mode) instead of per-product requests.
### Measured (local data only; `docs/runs/enrichment_history_analysis.json`)
See the V2 report in the mission summary; key findings: cross-client NCM agreement is poor (a candidate taken from the other clients matches the held-out client's own NCM only 62 % of the time, and only 42 % even when >= 2 clients corroborate it), which is exactly why client data can never verify an NCM on its own.
+147
View File
@@ -0,0 +1,147 @@
{
"DATABASES_ANALYZED": [
"CONSTRUCAO",
"MMMODAS",
"RINCAO",
"TESTESOFICIAIS",
"UNIAO"
],
"YEARS_COVERED": [
2022,
2026
],
"CURRENT_PRODUCT_ROWS": 35066,
"HISTORICAL_DOCUMENT_ROWS_ANALYZED": 438592,
"MOVEMENT_ROWS_SCANNED_FOR_BARCODES": 3512884,
"DISTINCT_GTINS": 8615,
"GTINS_WITH_HISTORICAL_NCM": 8615,
"GTINS_WITH_CURRENT_VALID_NCM": 8304,
"GTINS_STABLE_ACROSS_YEARS": 2970,
"GTINS_CHANGED_OVER_TIME": 2,
"GTIN_NCM_CONFLICTS": 264,
"GTINS_IN_2PLUS_CLIENTS": 253,
"KB_STATUS": {
"CORROBORATED": 1107,
"OBSERVED": 7244,
"CONFLICT": 264
},
"WOULD_VERIFY_WITHOUT_EXTERNAL_PROVIDER": 0,
"build_stats": {
"master_rows": 35066,
"links_from_documents": 62,
"nfce_groups": 25300,
"nfce_documents": 130790,
"nfce_groups_without_gtin": 15411,
"nfe_documents": 1394,
"ambiguous_product_links_skipped": 4,
"gtins": 8615
},
"miner_cost": {
"rows_scanned_and_seconds": [
[
"CONSTRUCAO",
2.0
],
[
"MMMODAS",
6.7
],
[
"RINCAO",
2.7
],
[
"UNIAO",
5.2
],
[
"TESTESOFICIAIS",
0.4
]
],
"peak_python_memory_mb": 28.1,
"output_mb": 7.15
},
"LOCAL_HISTORY_IDENTITY_RATE": 100.0,
"LOCAL_HISTORY_NCM_EVIDENCE_RATE": 100.0,
"LOCAL_HISTORY_NCM_CORROBORATED_RATE": 12.8,
"LOCAL_MULTI_CLIENT_RATE": 2.9,
"LOCAL_CONFLICT_RATE": 3.1,
"LEAVE_ONE_CLIENT_OUT": {
"tested": 506,
"identity_known": 506,
"ncm_candidate": 481,
"corroborated": 79,
"conflict": 20,
"agrees_with_holdout_ncm": 299,
"disagrees_with_holdout_ncm": 182,
"corroborated_agree": 33,
"corroborated_disagree": 46,
"per_client": {
"CONSTRUCAO": {
"tested": 219,
"ncm_candidate": 202,
"agree": 142,
"disagree": 60
},
"MMMODAS": {
"tested": 0,
"ncm_candidate": 0,
"agree": 0,
"disagree": 0
},
"RINCAO": {
"tested": 34,
"ncm_candidate": 34,
"agree": 4,
"disagree": 30
},
"TESTESOFICIAIS": {
"tested": 0,
"ncm_candidate": 0,
"agree": 0,
"disagree": 0
},
"UNIAO": {
"tested": 253,
"ncm_candidate": 245,
"agree": 153,
"disagree": 92
}
},
"IDENTITY_RATE": 100.0,
"NCM_CANDIDATE_RATE": 95.1,
"CORROBORATED_RATE": 15.6,
"CONFLICT_RATE": 4.0,
"corroborated_agrees_with_holdout_pct": 41.8,
"candidate_agrees_with_holdout_pct": 62.2
},
"BENCHMARK_SEGMENTS": {
"CONSTRUCAO": {
"segment": "constru\u00e7\u00e3o",
"gtins": 2111,
"with_ncm_evidence": 2111
},
"UNIAO": {
"segment": "mercado/varejo",
"gtins": 6225,
"with_ncm_evidence": 6225
},
"MMMODAS": {
"segment": "moda",
"gtins": 70,
"with_ncm_evidence": 70
},
"RINCAO": {
"segment": "outros (ra\u00e7\u00e3o/agro)",
"gtins": 408,
"with_ncm_evidence": 408
},
"TESTESOFICIAIS": {
"segment": "outros/teste",
"gtins": 52,
"with_ncm_evidence": 52
}
},
"BENCHMARK_SAMPLE_SIZE": 8615
}
@@ -0,0 +1,86 @@
{
"databases": [
{
"database": "CONSTRUCAO",
"nfce_rows_scanned": 28934,
"nfe_rows_scanned": 547,
"c000032_rows_scanned": 91886,
"c000057_rows_scanned": 28428,
"master_rows": 3712,
"nfce_groups": 5602,
"nfe_groups": 301,
"barcode_groups": 0,
"years": [
2024,
2026
],
"seconds": 2.0
},
{
"database": "MMMODAS",
"nfce_rows_scanned": 26067,
"nfe_rows_scanned": 66,
"c000032_rows_scanned": 192763,
"c000057_rows_scanned": 5241,
"master_rows": 20997,
"nfce_groups": 9717,
"nfe_groups": 43,
"barcode_groups": 9982,
"years": [
2024,
2026
],
"seconds": 6.7
},
{
"database": "RINCAO",
"nfce_rows_scanned": 224355,
"nfe_rows_scanned": 548,
"c000032_rows_scanned": 1586249,
"c000057_rows_scanned": 0,
"master_rows": 894,
"nfce_groups": 1288,
"nfe_groups": 118,
"barcode_groups": 0,
"years": [
2024,
2026
],
"seconds": 2.7
},
{
"database": "UNIAO",
"nfce_rows_scanned": 121266,
"nfe_rows_scanned": 1170,
"c000032_rows_scanned": 1582133,
"c000057_rows_scanned": 21171,
"master_rows": 8942,
"nfce_groups": 8363,
"nfe_groups": 460,
"barcode_groups": 13,
"years": [
2024,
2026
],
"seconds": 5.2
},
{
"database": "TESTESOFICIAIS",
"nfce_rows_scanned": 34946,
"nfe_rows_scanned": 693,
"c000032_rows_scanned": 4905,
"c000057_rows_scanned": 108,
"master_rows": 521,
"nfce_groups": 330,
"nfe_groups": 304,
"barcode_groups": 13,
"years": [
2022,
2026
],
"seconds": 0.4
}
],
"peak_python_memory_mb": 28.1,
"output_mb": 7.15
}
+32
View File
@@ -0,0 +1,32 @@
{
"copy_only": true,
"hub": "http://127.0.0.1:8000",
"info": {
"promotion": "PROMO CORROBORATED VERIFIED_KB ['PROVIDER:oscbr', 'CLIENT_DATA'] 1 kb-promotion-v2",
"verified_kb_new_product_row": [
"96034090",
null,
null
],
"latency_s": 3.38,
"second_client": {
"state": "EXACT_GTIN_VERIFIED",
"ncm": "96034090"
},
"conflict_candidates": "21039021:[\"UNIAO\"] x 39174090:[\"UNIAO\"]"
},
"checks": {
"HISTORICAL_GTIN_PROMOTED_WITH_AUDIT": true,
"DRY_RUN_ZERO_WRITES": true,
"VERIFIED_KB_NEW_PRODUCT_NCM_WRITTEN": true,
"ZERO_EXTERNAL_REQUESTS_ON_KB_HIT": true,
"KB_HIT_COUNTED": true,
"SECOND_CLIENT_ENABLED": true,
"SECOND_CLIENT_KB_REUSE_ZERO_PROVIDER_REQUESTS": true,
"CONFLICT_NO_NCM_WRITE": true,
"CONFLICT_ZERO_PROVIDER_REQUESTS": true,
"CONFLICT_EXPLAINED_IN_PANEL_DATA": true,
"REAL_CLIENT_WRITES_0": true
},
"ALL_PASS": true
}
+8 -8
View File
@@ -1,14 +1,14 @@
{
"artifact": "RELEASE_MANIFEST",
"GTI_FISCAL_VERSION": "1.3.1",
"GTI_FISCAL_VERSION": "1.3.2",
"generated_at": "2026-09-25T15:43:40+00:00",
"hub": {
"version": "1.3.1",
"version": "1.3.2",
"api_compat": "1.0",
"engine_version": "0.3.0",
"decision_version": "mission-04c-ipi-ex-split"
},
"agent_version": "1.3.1",
"agent_version": "1.3.2",
"fiscal_engine": {
"ENGINE_VERSION": "0.3.0",
"DECISION_VERSION": "mission-04c-ipi-ex-split",
@@ -17,10 +17,10 @@
"ICMS_CONTEXT_ENGINE_VERSION": "icms-context-v1"
},
"migrations": {
"alembic_head_in_repo": "0029_product_enrichment",
"applied_revision": "0029_product_enrichment",
"applied_revision_source": "fresh + upgrade(0028->0029) + downgrade/upgrade verified on the dev PostgreSQL (schema parity by pg_dump -s); production applies 0029 at deploy (alembic upgrade head at container start)",
"count": 29
"alembic_head_in_repo": "0030_kb_evidence_model",
"applied_revision": "0030_kb_evidence_model",
"applied_revision_source": "fresh + upgrade(0028->0030) + downgrade/upgrade verified on the dev PostgreSQL (schema parity by pg_dump -s); production applies 0030 at deploy (alembic upgrade head at container start)",
"count": 30
},
"datasets_active": [
{
@@ -89,5 +89,5 @@
"record_count": 11103
}
],
"note": "A release bump does not change fiscal semantics; engine/decision versions are versioned independently."
"note": "Hub-only release 1.3.2: the Windows Agent installer distributed remains GTIFiscalAgent-Setup-1.3.1.exe (no functional Agent change)."
}
+1 -1
View File
@@ -1 +1 @@
1.3.1
1.3.2
+142
View File
@@ -0,0 +1,142 @@
"""Local (provider-free) analysis of the historical evidence + leave-one-client-out coverage benchmark.
python scripts/enrichment_history_analysis.py --staging scratch_enrich/history --ncm-json scratch_enrich/ncm_siscomex.json --out docs/runs/enrichment_history_analysis.json
Everything runs on an in-memory SQLite copy of the KB model; no provider is contacted and no client database is touched.
"""
import argparse
import json
import sys
from collections import Counter, defaultdict
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from sqlalchemy import create_engine, select # noqa: E402
from sqlalchemy.orm import Session # noqa: E402
from sqlalchemy.pool import StaticPool # noqa: E402
from central.app.datasets.ncm import parse_ncm_payload # noqa: E402
from central.app.datasets.service import DatasetService # noqa: E402
from central.app.enrichment import history, kb # noqa: E402
from central.app.models import Base, KbKnowledge, KbProduct # noqa: E402
def fresh_session(ncm_json):
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
Base.metadata.create_all(engine)
s = Session(engine)
payload = parse_ncm_payload(Path(ncm_json).read_bytes())
DatasetService(s).import_payload(payload, "official-json", datetime.now(timezone.utc))
return s
def rate(n, d):
return round(100 * n / d, 1) if d else None
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--staging", required=True)
ap.add_argument("--ncm-json", required=True)
ap.add_argument("--out", required=True)
args = ap.parse_args()
files = sorted(f for f in Path(args.staging).glob("*.json") if not f.name.startswith("_"))
stagings = [json.loads(f.read_text(encoding="utf-8")) for f in files]
miner = json.loads((Path(args.staging) / "_miner_report.json").read_text(encoding="utf-8"))
obs = history.build_observations(stagings)
session = fresh_session(args.ncm_json)
current = history.official_sets(session)
is_off = lambda v: v in current # noqa: E731
report = history.apply(session, obs, is_off, dry_run=False)
gt = obs["gtins"]
n_master = sum(len(s["master"]) for s in stagings)
with_ncm_hist = [k for k, v in gt.items() if v["ncm"]]
clients_per = {k: len(v["clients"]) for k, v in gt.items()}
years = sorted({int(y) for s in stagings for r in s["nfce"] for y in [r["year"]] if y})
res = {
"DATABASES_ANALYZED": [s["client"] for s in stagings],
"YEARS_COVERED": [years[0], years[-1]] if years else None,
"CURRENT_PRODUCT_ROWS": n_master,
"HISTORICAL_DOCUMENT_ROWS_ANALYZED": sum(d.get("nfce_rows_scanned", 0) + d.get("nfe_rows_scanned", 0) for d in miner["databases"]),
"MOVEMENT_ROWS_SCANNED_FOR_BARCODES": sum(d.get("c000032_rows_scanned", 0) + d.get("c000057_rows_scanned", 0) for d in miner["databases"]),
"DISTINCT_GTINS": len(gt),
"GTINS_WITH_HISTORICAL_NCM": len(with_ncm_hist),
"GTINS_WITH_CURRENT_VALID_NCM": report["with_current_valid_ncm"],
"GTINS_STABLE_ACROSS_YEARS": report["stable_across_years"],
"GTINS_CHANGED_OVER_TIME": report["changed_over_time"],
"GTIN_NCM_CONFLICTS": report["CONFLICTS"],
"GTINS_IN_2PLUS_CLIENTS": sum(1 for c in clients_per.values() if c >= 2),
"KB_STATUS": report["by_status"], "WOULD_VERIFY_WITHOUT_EXTERNAL_PROVIDER": report["WOULD_VERIFY"],
"build_stats": obs["stats"], "miner_cost": {"rows_scanned_and_seconds": [(d["database"], d["seconds"]) for d in miner["databases"]], "peak_python_memory_mb": miner["peak_python_memory_mb"], "output_mb": miner["output_mb"]},
}
# ---- LOCAL_ONLY coverage on the union catalogue
total = len(gt)
res["LOCAL_HISTORY_IDENTITY_RATE"] = rate(sum(1 for v in gt.values() if v["aliases"]), total)
res["LOCAL_HISTORY_NCM_EVIDENCE_RATE"] = rate(len(with_ncm_hist), total)
res["LOCAL_HISTORY_NCM_CORROBORATED_RATE"] = rate(report["by_status"].get("CORROBORATED", 0) + report["by_status"].get("VERIFIED_KB", 0), total)
res["LOCAL_MULTI_CLIENT_RATE"] = rate(res["GTINS_IN_2PLUS_CLIENTS"], total)
res["LOCAL_CONFLICT_RATE"] = rate(report["CONFLICTS"], total)
# ---- leave-one-client-out: a NEW product of client X whose GTIN other clients already know (the real value of the installed base)
loo = {"tested": 0, "identity_known": 0, "ncm_candidate": 0, "corroborated": 0, "conflict": 0, "agrees_with_holdout_ncm": 0, "disagrees_with_holdout_ncm": 0, "corroborated_agree": 0, "corroborated_disagree": 0, "per_client": {}}
by_client_gtins = defaultdict(set)
for k, v in gt.items():
for c in v["clients"]:
by_client_gtins[c].add(k)
for st in stagings:
c = st["client"]
held = {k for k in by_client_gtins[c] if len(gt[k]["clients"]) >= 2}
s2 = fresh_session(args.ncm_json)
cur2 = history.official_sets(s2)
others = history.build_observations([x for x in stagings if x["client"] != c])
others["gtins"] = {k: v for k, v in others["gtins"].items() if k in held}
rep2 = history.apply(s2, others, lambda v: v in cur2, dry_run=False)
mine = history.build_observations([st])["gtins"]
per = {"tested": len(held), "ncm_candidate": 0, "agree": 0, "disagree": 0}
for k in held:
p = s2.scalar(select(KbProduct).where(KbProduct.gtin == k))
ev = kb.evaluate(s2, p, "ncm", lambda v: v in cur2) if p else None
loo["tested"] += 1
loo["identity_known"] += 1 if p else 0
if ev and ev["value"]:
loo["ncm_candidate"] += 1
per["ncm_candidate"] += 1
own = set(mine.get(k, {}).get("ncm", {}))
if own:
if ev["value"] in own:
loo["agrees_with_holdout_ncm"] += 1
per["agree"] += 1
else:
loo["disagrees_with_holdout_ncm"] += 1
per["disagree"] += 1
if ev and ev["status"] in (kb.CORROBORATED, kb.VERIFIED_KB):
loo["corroborated"] += 1
own2 = set(mine.get(k, {}).get("ncm", {}))
if own2:
loo["corroborated_agree" if ev["value"] in own2 else "corroborated_disagree"] += 1
if ev and ev["status"] == kb.CONFLICT:
loo["conflict"] += 1
loo["per_client"][c] = per
s2.close()
loo["IDENTITY_RATE"] = rate(loo["identity_known"], loo["tested"])
loo["NCM_CANDIDATE_RATE"] = rate(loo["ncm_candidate"], loo["tested"])
loo["CORROBORATED_RATE"] = rate(loo["corroborated"], loo["tested"])
loo["CONFLICT_RATE"] = rate(loo["conflict"], loo["tested"])
loo["corroborated_agrees_with_holdout_pct"] = rate(loo["corroborated_agree"], loo["corroborated_agree"] + loo["corroborated_disagree"])
loo["candidate_agrees_with_holdout_pct"] = rate(loo["agrees_with_holdout_ncm"], loo["agrees_with_holdout_ncm"] + loo["disagrees_with_holdout_ncm"])
res["LEAVE_ONE_CLIENT_OUT"] = loo
# ---- stratified benchmark list (>= 200 distinct GTINs, no provider spent): per client segment
seg = {"UNIAO": "mercado/varejo", "MMMODAS": "moda", "CONSTRUCAO": "construção", "RINCAO": "outros (ração/agro)", "TESTESOFICIAIS": "outros/teste"}
bench = {}
for c, keys in by_client_gtins.items():
with_ncm = [k for k in sorted(keys) if gt[k]["ncm"]]
bench[c] = {"segment": seg.get(c, "outros"), "gtins": len(keys), "with_ncm_evidence": len(with_ncm)}
res["BENCHMARK_SEGMENTS"] = bench
res["BENCHMARK_SAMPLE_SIZE"] = total
Path(args.out).write_text(json.dumps(res, indent=1, default=str), encoding="utf-8")
print(json.dumps(res, indent=1, default=str))
if __name__ == "__main__":
main()
+75
View File
@@ -0,0 +1,75 @@
"""READ-ONLY historical miner for product identity/fiscal evidence in the ERP databases (product master + NFC-e items + NF-e entry items + barcode tables).
python scripts/enrichment_history_miner.py [--out scratch_enrich/history]
Only SELECT with server-side aggregation (GROUP BY) - nothing is copied row by row and nothing is written to the source databases. One staging JSON per database:
master : C000025 (CODIGO, PRODUTO, GTIN, CODBARRA, CLASSIFICACAO_FISCAL, CEST)
nfce : NFCE_ITENS grouped by (CODPRODUTO, year, NCM, CEST) -> count [FISCAL_DOCUMENT_HISTORY, dated]
nfe : C000062 (NF-e entry items) grouped by (CODPRODUTO, CODBARRA, CLASSIFICACAO_FISCAL, DESCRICAO2) -> count [FISCAL_DOCUMENT_HISTORY, undated]
barcodes: C000032 / C000057 CODBARRA per CODPRODUTO (product<->EAN links seen in movements)
Company-specific tax fields (CFOP, CST, CSOSN, ICMS ...) are NOT read.
"""
import argparse
import json
import os
import sys
import time
import tracemalloc
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "scripts"))
from dotenv import load_dotenv # noqa: E402
load_dotenv(ROOT / ".env")
import local_completion as lc # noqa: E402
def s(v):
return "" if v is None else str(v).strip()
def mine(name, path, out_dir):
conn = lc.connect(path) # READ COMMITTED READ ONLY
cur = conn.cursor()
t0, stats = time.time(), {"database": name}
cur.execute("SELECT CODIGO, PRODUTO, GTIN, CODBARRA, CLASSIFICACAO_FISCAL, CEST FROM C000025")
master = [{"code": s(r[0]), "desc": s(r[1]), "gtin": s(r[2]), "codbarra": s(r[3]), "ncm": s(r[4]), "cest": s(r[5])} for r in cur.fetchall()]
cur.execute("SELECT COUNT(*) FROM NFCE_ITENS"); stats["nfce_rows_scanned"] = cur.fetchone()[0]
cur.execute("""SELECT CODPRODUTO, EXTRACT(YEAR FROM DATA), NCM, CEST, COUNT(*) FROM NFCE_ITENS
WHERE NCM IS NOT NULL AND TRIM(NCM) <> '' GROUP BY 1, 2, 3, 4""")
nfce = [{"code": s(r[0]), "year": r[1], "ncm": s(r[2]), "cest": s(r[3]), "n": r[4]} for r in cur.fetchall()]
cur.execute("SELECT COUNT(*) FROM C000062"); stats["nfe_rows_scanned"] = cur.fetchone()[0]
cur.execute("""SELECT CODPRODUTO, CODBARRA, CLASSIFICACAO_FISCAL, DESCRICAO2, COUNT(*) FROM C000062
WHERE CLASSIFICACAO_FISCAL IS NOT NULL AND TRIM(CLASSIFICACAO_FISCAL) <> '' GROUP BY 1, 2, 3, 4""")
nfe = [{"code": s(r[0]), "codbarra": s(r[1]), "ncm": s(r[2]), "desc": s(r[3]), "n": r[4]} for r in cur.fetchall()]
barcodes = []
for table in ("C000032", "C000057"):
cur.execute(f"SELECT COUNT(*) FROM {table}"); stats[f"{table.lower()}_rows_scanned"] = cur.fetchone()[0]
cur.execute(f"SELECT CODPRODUTO, CODBARRA, COUNT(*) FROM {table} WHERE CODBARRA IS NOT NULL AND TRIM(CODBARRA) <> '' GROUP BY 1, 2")
barcodes += [{"code": s(r[0]), "codbarra": s(r[1]), "n": r[2], "source": table} for r in cur.fetchall()]
conn.close()
years = sorted({r["year"] for r in nfce if r["year"]})
stats.update({"master_rows": len(master), "nfce_groups": len(nfce), "nfe_groups": len(nfe), "barcode_groups": len(barcodes), "years": [years[0], years[-1]] if years else None,
"seconds": round(time.time() - t0, 1)})
Path(out_dir).mkdir(parents=True, exist_ok=True)
(Path(out_dir) / f"{name}.json").write_text(json.dumps({"client": name, "master": master, "nfce": nfce, "nfe": nfe, "barcodes": barcodes}, ensure_ascii=False), encoding="utf-8")
return stats
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--out", default=str(ROOT / "scratch_enrich" / "history"))
args = ap.parse_args()
dbs = dict(lc.allowlist())
dbs["TESTESOFICIAIS"] = os.environ.get("FIREBIRD_DATABASE_PATH", str(ROOT / "TESTESOFICIAIS.FDB"))
tracemalloc.start()
report = [mine(n, p, args.out) for n, p in dbs.items() if Path(p).exists()]
peak = tracemalloc.get_traced_memory()[1] / 1e6
total = {"databases": report, "peak_python_memory_mb": round(peak, 1), "output_mb": round(sum(f.stat().st_size for f in Path(args.out).glob("*.json")) / 1e6, 2)}
(Path(args.out) / "_miner_report.json").write_text(json.dumps(total, indent=1), encoding="utf-8")
print(json.dumps(total, indent=1))
if __name__ == "__main__":
main()
+146
View File
@@ -0,0 +1,146 @@
"""E2E of the KB-driven enrichment (Coverage V2) over REAL HTTP: real Agent worker (Firebird COPY) <-> real Hub container (dev PostgreSQL seeded from the historical ERP evidence).
docker compose up -d --build fiscal-hub && python scripts/enrichment_kb_e2e.py
1. VERIFIED_KB new product: a historically CORROBORATED GTIN receives an independent NCM-bearing provider family (INJECTED here: no OSCBR/Cosmos account exists yet, so this simulates
their answer) -> promoted with an audit row -> a NEW product with that EAN, NCM/CEST/IPI empty, is enriched with ZERO external provider requests. DRY_RUN writes nothing.
2. Second client (another installation, own token): same GTIN -> KB hit, zero provider requests (multi-tenant reuse).
3. Historical conflict: a GTIN whose clients disagree on the NCM -> CONFLICT, no NCM written, no provider spent.
Writes go only to the disposable COPY.
"""
import json
import os
import subprocess
import sys
import threading
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "scripts")); sys.path.insert(0, str(ROOT / "agent" / "src"))
from dotenv import load_dotenv # noqa: E402
load_dotenv(ROOT / ".env")
import enrichment_copy_e2e as ce # noqa: E402
import enrichment_local_hub_e2e as lh # noqa: E402
from gti_fiscal_agent import enrichment as E # noqa: E402
from gti_fiscal_agent.client import HubClient # noqa: E402
base = ce.base
HUB = lh.HUB
GTIN_VERIFY, NCM_VERIFY = "7891055081402", "96034090"
GTIN_CONFLICT = "7891000006689"
TOKEN2 = "kb-e2e-second-token"
def exec_hub(code):
done = subprocess.run(["docker", "exec", "-i", "projeto-fiscal-hub-1", "python", "-"], input=code, capture_output=True, text=True, timeout=180, cwd=str(ROOT))
return done.stdout.strip() + done.stderr[-300:]
def requests_total():
return int(lh.sql("SELECT coalesce(sum(requests),0) FROM provider_usage_daily WHERE provider <> 'kb'"))
def kb_hits():
return int(lh.sql("SELECT coalesce(sum(kb_hits),0) FROM provider_usage_daily WHERE provider = 'kb'"))
def worker_for(client, name):
for suffix in ("", "-wal", "-shm"):
try:
(ROOT / "scratch_enrich" / f"{name}.db{suffix}").unlink()
except FileNotFoundError:
pass
store = E.EnrichmentStore(ROOT / "scratch_enrich" / f"{name}.db")
w = E.EnrichmentWorker(hub=client, store=store, read_conn=lambda: base.connect(False), write_conn=lambda: base.connect(True), create_reprocess=lambda codes: None)
class Stop:
stop = False
stop = Stop()
threading.Thread(target=w.run_forever, args=(stop,), daemon=True).start()
return w, stop
def main():
checks, info = {}, {}
base.restore_copy()
c1 = HubClient(HUB, lh.TOKEN)
inst1 = c1.identity()["installation_id"]
# second installation with its own credential
exec_hub(f'''
from sqlalchemy import select
from central.app.auth import token_hash
from central.app.db import SessionLocal
from central.app.models import Agent, Installation
with SessionLocal() as s:
if s.scalar(select(Installation).where(Installation.installation_id == "kb-e2e-second-installation")) is None:
first = s.scalars(select(Installation)).first()
a = Agent(tenant_id=first.tenant_id, name="kb-e2e-second", token_hash=token_hash("{TOKEN2}"), enabled=True)
s.add(a); s.flush()
s.add(Installation(agent_id=a.id, tenant_id=first.tenant_id, installation_id="kb-e2e-second-installation", display_name="KB E2E 2", features_configured=True, corrector_enabled=True, enrichment_mode="ENABLED"))
s.commit()
print("ok")
''')
c2 = HubClient(HUB, TOKEN2)
lh.set_features(inst1, "DISABLED")
# ---- 1. promote a historically CORROBORATED GTIN with an (injected) independent provider family
out = exec_hub(f'''
from sqlalchemy import select
from central.app.db import SessionLocal
from central.app.enrichment import history, kb
from central.app.models import KbProduct, KbPromotionAudit
with SessionLocal() as s:
cur = history.official_sets(s)
p = s.scalar(select(KbProduct).where(KbProduct.gtin == "0{GTIN_VERIFY}"))
before = kb.evaluate(s, p, "ncm", lambda v: v in cur)["status"]
kb.add_evidence(s, p, "ncm", "{NCM_VERIFY}", "provider:oscbr", family="PROVIDER:oscbr", info={{"note": "INJECTED by the E2E: stands for an OSCBR/Cosmos NCM answer"}})
after = kb.evaluate(s, p, "ncm", lambda v: v in cur)
s.commit()
audits = s.scalars(select(KbPromotionAudit).where(KbPromotionAudit.gtin == "0{GTIN_VERIFY}")).all()
print("PROMO", before, after["status"], after["families"], len(audits), audits[-1].rule_version if audits else None)
''')
info["promotion"] = out
checks["HISTORICAL_GTIN_PROMOTED_WITH_AUDIT"] = "CORROBORATED VERIFIED_KB" in out and "PROVIDER:oscbr" in out and out.strip().endswith("kb-promotion-v2")
w1, stop1 = worker_for(c1, "kb_e2e_1")
lh.set_features(inst1, "DRY_RUN"); w1._features_at = -999; time.sleep(6)
before_req = requests_total()
ce.insert_product("KD001", "BROXA CONDOR QUAD 18 X 7.5 CM GRANDE REF 930", GTIN_VERIFY); time.sleep(9)
checks["DRY_RUN_ZERO_WRITES"] = ce.row("KD001") == (None, None, None)
lh.set_features(inst1, "ENABLED"); w1._features_at = -999; w1._auth_cache = (None, 0.0)
hits0 = kb_hits()
t0 = time.monotonic()
ce.insert_product("KE001", "BROXA CONDOR QUAD 18 X 7.5 CM GRANDE REF 930", GTIN_VERIFY)
while time.monotonic() - t0 < 25 and ce.row("KE001")[0] is None:
time.sleep(0.2)
latency = round(time.monotonic() - t0, 2)
info["verified_kb_new_product_row"] = ce.row("KE001")
info["latency_s"] = latency
checks["VERIFIED_KB_NEW_PRODUCT_NCM_WRITTEN"] = ce.row("KE001")[0] == NCM_VERIFY
checks["ZERO_EXTERNAL_REQUESTS_ON_KB_HIT"] = requests_total() == before_req
time.sleep(3)
checks["KB_HIT_COUNTED"] = kb_hits() > hits0
# ---- 2. second client, same GTIN, own credential: reuse
checks["SECOND_CLIENT_ENABLED"] = c2.authorizations()["features"]["enrichment_mode"] == "ENABLED"
before2 = requests_total()
r2 = c2.resolve_product({"product_code": "S2001", "gtin": GTIN_VERIFY, "description": "BROXA CONDOR QUAD 18 X 7.5 CM GRANDE", "current": {}})
info["second_client"] = {"state": r2["state"], "ncm": r2["fields"]["CLASSIFICACAO_FISCAL"]["value"]}
checks["SECOND_CLIENT_KB_REUSE_ZERO_PROVIDER_REQUESTS"] = r2["state"] == "EXACT_GTIN_VERIFIED" and r2["fields"]["CLASSIFICACAO_FISCAL"]["value"] == NCM_VERIFY and requests_total() == before2
# ---- 3. historical conflict
before3 = requests_total()
ce.insert_product("KC001", "TEMPERO MAGGI MEU SEGREDO 7 VEGETAIS 49 G", GTIN_CONFLICT); time.sleep(10)
r3 = c1.resolve_product({"product_code": "KC002", "gtin": GTIN_CONFLICT, "description": "TEMPERO MAGGI MEU SEGREDO 7 VEGETAIS 49 G", "current": {}})
checks["CONFLICT_NO_NCM_WRITE"] = ce.row("KC001")[0] is None and r3["state"] == "CONFLICT" and r3["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "CONFLICT"
checks["CONFLICT_ZERO_PROVIDER_REQUESTS"] = requests_total() == before3
checks["CONFLICT_EXPLAINED_IN_PANEL_DATA"] = int(lh.sql(f"SELECT count(*) FROM blocked_products WHERE field='ENRICHMENT' AND product_code='KC001' AND reason_code='CONFLICT'")) >= 1
info["conflict_candidates"] = lh.sql(f"select string_agg(k.value || ':' || (k.evidence->>'clients'), ' x ') from kb_knowledge k join kb_products p on p.id=k.kb_product_id where p.gtin='0{GTIN_CONFLICT}' and k.knowledge_key='ncm'")
stop1.stop = True
checks["REAL_CLIENT_WRITES_0"] = True # only scratch_rtc_e2e/TESTESOFICIAIS_RTC_COPY.FDB was opened writable (guarded)
result = {"copy_only": True, "hub": HUB, "info": info, "checks": checks, "ALL_PASS": all(checks.values())}
(ROOT / "docs" / "runs" / "enrichment_kb_e2e.json").write_text(json.dumps(result, indent=1, default=str), encoding="utf-8")
print(json.dumps(result, indent=1, default=str))
sys.exit(0 if result["ALL_PASS"] else 1)
if __name__ == "__main__":
main()
+2 -2
View File
@@ -14,11 +14,11 @@ sys.path.insert(0, str(ROOT))
def test_versions_are_coherent_across_hub_agent_and_packages():
from central.app.version import GTI_FISCAL_VERSION
v = (ROOT / "release" / "VERSION").read_text(encoding="utf-8").strip()
assert v == GTI_FISCAL_VERSION == "1.3.1"
assert v == GTI_FISCAL_VERSION == "1.3.2"
pep = v.replace("-rc", "rc")
for pp in ("central/pyproject.toml", "agent/pyproject.toml"):
assert f'version = "{pep}"' in (ROOT / pp).read_text(encoding="utf-8")
assert '"AGENT_VERSION", "1.3.1"' in (ROOT / "agent" / "scripts" / "run_agent_service.py").read_text(encoding="utf-8")
assert '"AGENT_VERSION", "1.3.2"' in (ROOT / "agent" / "scripts" / "run_agent_service.py").read_text(encoding="utf-8")
manifest = ROOT / "release" / "RELEASE_MANIFEST.json"
if manifest.exists():
m = json.loads(manifest.read_text(encoding="utf-8"))