fix: KB dashboard search was applying the strict fiscal name-match rule (used to authorize automatic writes) to human browsing, so a natural query (e.g. missing the unit word) found nothing; now a separate, lenient, global, paginated server-side search — filter over the whole KB, then count/order/paginate. 1.3.6 (Hub only)
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
gti
Claude Sonnet 5
@@ -100,23 +100,50 @@ def _candidate_cards(session, products) -> list[dict]:
|
||||
return out
|
||||
|
||||
|
||||
def kb_search(session, query: str, limit: int = 6) -> list[dict]:
|
||||
from ..models import KbProduct
|
||||
from . import kb as kbm
|
||||
def kb_search_ids(session, query: str) -> list[int]:
|
||||
"""GLOBAL, server-side match over the whole KB (never limited to a rendered page): GTIN exact, or every normalized query word present somewhere among the
|
||||
product's known names (any alias row OR its own normalized_description - words can come from different aliases of the same product). Deliberately lenient
|
||||
(a human exploring the KB, e.g. "COCA COLA LATA 350" without the unit, must still find "COCA-COLA ORIGINAL LATA 350 ML"); this is DIFFERENT from
|
||||
`kb.alias_match`, the strict rule that authorizes an automatic fiscal write, which stays untouched."""
|
||||
import re
|
||||
|
||||
from ..models import KbKnowledge, KbProduct
|
||||
from . import kb as kbm, textnorm
|
||||
q = (query or "").strip()
|
||||
if not q:
|
||||
return []
|
||||
digits = "".join(ch for ch in q if ch.isdigit())
|
||||
products = []
|
||||
if len(digits) >= 8 and len(digits) == len(q.replace(" ", "")):
|
||||
if len(digits) >= 8 and re.fullmatch(r"[\d\s.\-]+", q):
|
||||
p = kbm.find_by_gtin(session, digits.zfill(14))
|
||||
products = [p] if p else []
|
||||
else:
|
||||
products = kbm.name_candidates(session, q)[:limit]
|
||||
return _candidate_cards(session, products)
|
||||
return [p.id] if p else []
|
||||
tokens = [t for t in textnorm.tokens(q) if t not in textnorm.MARKETING]
|
||||
if not tokens:
|
||||
return []
|
||||
ids: set[int] | None = None
|
||||
for token in tokens: # SQL ILIKE narrows the candidates (portable: Postgres in production, SQLite in tests); the exact whole-word
|
||||
like = f"%{token}%" # check happens in Python so "ML" never matches inside "SIMPLES", etc.
|
||||
matched = {pid for pid, val in session.execute(select(KbKnowledge.kb_product_id, KbKnowledge.value).where(KbKnowledge.knowledge_key == "alias", KbKnowledge.value.ilike(like))) if token in (val or "").split()}
|
||||
matched |= {pid for pid, val in session.execute(select(KbProduct.id, KbProduct.normalized_description).where(KbProduct.normalized_description.ilike(like))) if token in (val or "").split()}
|
||||
ids = matched if ids is None else (ids & matched)
|
||||
if not ids:
|
||||
return []
|
||||
return sorted(ids)
|
||||
|
||||
|
||||
def kb_summary(session, query: str | None = None) -> dict:
|
||||
def kb_search(session, query: str, *, page: int = 1, page_size: int = 10) -> dict:
|
||||
"""query/filter GLOBAL over the whole KB -> count -> order -> pagination (never the other way round)."""
|
||||
from ..models import KbProduct
|
||||
ids = kb_search_ids(session, query)
|
||||
total = len(ids)
|
||||
page = max(1, page)
|
||||
pages = max(1, -(-total // page_size))
|
||||
page = min(page, pages)
|
||||
page_ids = ids[(page - 1) * page_size: page * page_size]
|
||||
products = [session.get(KbProduct, i) for i in page_ids]
|
||||
return {"results": _candidate_cards(session, products), "total": total, "page": page, "pages": pages, "page_size": page_size}
|
||||
|
||||
|
||||
def kb_summary(session, query: str | None = None, page: int = 1) -> dict:
|
||||
"""Compact Knowledge Base panel: known products, statuses of the NCM knowledge, hits today, and the latest conflicts with their evidence (no dashboard of its own)."""
|
||||
from ..models import KbKnowledge, KbProduct
|
||||
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
@@ -134,4 +161,5 @@ def kb_summary(session, query: str | None = None) -> dict:
|
||||
"candidates": [{"ncm": i.value, "clients": sorted({c for c in (i.evidence or {}).get("clients", [])}), "documents": sum(int(l.get("docs", 0)) for l in ((i.evidence or {}).get("lineages") or {}).values()),
|
||||
"providers": sorted(f for f in ((i.evidence or {}).get("families") or {}) if f.startswith("PROVIDER:"))} for i in items]})
|
||||
return {"products": products, "verified": by.get("VERIFIED_KB", 0), "corroborated": by.get("CORROBORATED", 0) + by.get("EXTERNAL_CORROBORATED", 0), "observed": by.get("OBSERVED", 0),
|
||||
"conflicts": by.get("CONFLICT", 0), "stale": by.get("STALE", 0), "hits_today": hits, "recent_conflicts": conflicts, "search": kb_search(session, query) if query else []}
|
||||
"conflicts": by.get("CONFLICT", 0), "stale": by.get("STALE", 0), "hits_today": hits, "recent_conflicts": conflicts,
|
||||
"search": kb_search(session, query, page=page) if query else {"results": [], "total": 0, "page": 1, "pages": 1, "page_size": 10}}
|
||||
|
||||
@@ -4,5 +4,5 @@ kept identical to this) and what an AgentRelease row's `version` column holds
|
||||
once published — never a second, independent version number. Fiscal semantics
|
||||
are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py,
|
||||
DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump."""
|
||||
GTI_FISCAL_VERSION = "1.3.5"
|
||||
GTI_FISCAL_VERSION = "1.3.6"
|
||||
API_COMPAT_MAJOR_MINOR = "1.0" # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x
|
||||
|
||||
+2
-2
@@ -159,10 +159,10 @@ def router(session_factory):
|
||||
return {key: items[0] for key, items in parse_qs((await request.body()).decode()).items() if items}
|
||||
|
||||
@routes.get("/", response_class=HTMLResponse)
|
||||
def dashboard(request: Request, kb_q: str | None = None):
|
||||
def dashboard(request: Request, kb_q: str | None = None, kb_page: int = 1):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try: return render(request, "dashboard.html", {"summary": dashboard_summary(session), "auth": auth_data, "kb_q": kb_q or "", "is_admin": _is_privileged(auth_data[1]), "csrf_token": request.cookies.get("gti_csrf", ""), "sources": provider_sources(session), "official": official_sources(session), "enrich_metrics": coverage_metrics(session), "kb": kb_summary(session, kb_q)})
|
||||
try: return render(request, "dashboard.html", {"summary": dashboard_summary(session), "auth": auth_data, "kb_q": kb_q or "", "is_admin": _is_privileged(auth_data[1]), "csrf_token": request.cookies.get("gti_csrf", ""), "sources": provider_sources(session), "official": official_sources(session), "enrich_metrics": coverage_metrics(session), "kb": kb_summary(session, kb_q, kb_page)})
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/agents", response_class=HTMLResponse)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "gti-fiscal-hub"
|
||||
version = "1.3.5"
|
||||
version = "1.3.6"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"fastapi>=0.115,<1",
|
||||
|
||||
@@ -30,10 +30,12 @@
|
||||
</section>
|
||||
<section class="panel" id="kb-resumo"><div class="panel-title"><div><h2>Base de conhecimento de produtos</h2><p class="muted">Evidência local, histórica e externa; nada vira verdade fiscal sem regra de promoção.</p></div></div>
|
||||
<div class="coverage"><div class="coverage-item"><strong>{{ kb.products }}</strong><span>Produtos conhecidos</span></div><div class="coverage-item"><strong>{{ kb.verified }}</strong><span>GTINs verificados</span></div><div class="coverage-item"><strong>{{ kb.corroborated }}</strong><span>Corroborados</span></div><div class="coverage-item"><strong>{{ kb.conflicts }}</strong><span>Conflitos</span></div><div class="coverage-item"><strong>{{ kb.hits_today }}</strong><span>Resolvidos pela base hoje</span></div></div>
|
||||
<form method="get" action="/" class="filters compact-filters"><label>Buscar produto na base (EAN ou nome)<input name="kb_q" value="{{ kb_q or '' }}" placeholder="ex.: COCA COLA LATA 350"></label><button class="secondary" type="submit">Buscar</button></form>
|
||||
{% for c in kb.search %}<div class="list-row"><span><b>{{ c.gtin }}</b> {{ c.description }} · <em class="badge">{{ c.status }}</em>
|
||||
<form method="get" action="/" class="filters compact-filters"><label>Buscar produto na base (EAN ou nome)<input name="kb_q" value="{{ kb_q or '' }}" placeholder="ex.: COCA COLA LATA 350"></label><button class="secondary" type="submit">Buscar</button>{% if kb_q %}<small class="muted">{{ kb.search.total }} resultado(s) em toda a base{% if kb.search.pages > 1 %} · página {{ kb.search.page }}/{{ kb.search.pages }}{% endif %}</small>{% endif %}</form>
|
||||
{% for c in kb.search.results %}<div class="list-row"><span><b>{{ c.gtin }}</b> {{ c.description }} · <em class="badge">{{ c.status }}</em>
|
||||
<small>{% for x in c.candidates %}NCM {{ x.ncm }} — <i>{{ x.official_text or "fora do dataset oficial vigente" }}</i> ({{ x.clients|length }} cliente(s), {{ x.documents }} doc.{% if x.providers %}, {{ x.providers|join("/") }}{% endif %}{% if x.attested %}, decidido por {{ x.attested }}{% endif %}){% if is_admin and x.official_text and c.status != "VERIFIED_KB" %}
|
||||
<form method="post" action="/kb/attest" style="display:inline"><input type="hidden" name="csrf_token" value="{{ csrf_token }}"><input type="hidden" name="gtin" value="{{ c.gtin_key }}"><input type="hidden" name="ncm" value="{{ x.ncm }}"><button class="secondary" type="submit" title="Decisão fiscal de um administrador: este é o NCM correto deste EAN">Confirmar este NCM</button></form>{% endif %}<br>{% endfor %}</small></span></div>{% endfor %}
|
||||
{% if kb_q and kb.search.total == 0 %}<p class="empty-state">Nenhum produto encontrado para "{{ kb_q }}".</p>{% endif %}
|
||||
{% if kb.search.pages > 1 %}<nav class="pagination" aria-label="Paginação da busca na base">{% if kb.search.page > 1 %}<a href="/?kb_q={{ kb_q|urlencode }}&kb_page={{ kb.search.page - 1 }}#kb-resumo">Anterior</a>{% endif %}<span>Página {{ kb.search.page }} de {{ kb.search.pages }}</span>{% if kb.search.page < kb.search.pages %}<a href="/?kb_q={{ kb_q|urlencode }}&kb_page={{ kb.search.page + 1 }}#kb-resumo">Próxima</a>{% endif %}</nav>{% endif %}
|
||||
{% for c in kb.recent_conflicts %}<div class="list-row"><span><b>{{ c.gtin }}</b> {{ c.description }}<small>NCMs divergentes, nada é preenchido: {% for x in c.candidates %}{{ x.ncm }} ({{ x.clients|length }} cliente(s), {{ x.documents }} doc.{% if x.providers %}, {{ x.providers|join("/") }}{% endif %}){% if not loop.last %} × {% endif %}{% endfor %}</small></span></div>{% endfor %}</section>
|
||||
<section class="panel" id="enriquecimento-resumo"><div class="panel-title"><div><h2>Enriquecimento de produtos novos</h2><p class="muted">Cobertura medida sobre os produtos novos acompanhados (o catálogo histórico não entra).</p></div></div>
|
||||
{% if enrich_metrics.tracked %}<div class="coverage">{% for key,label in [("GTIN_VALID_RATE","Com GTIN válido"),("KB_HIT_RATE","Resolvidos pela base"),("PRODUCT_FOUND_RATE","Produto identificado"),("NCM_AUTO_FILL_SAFE_RATE","NCM preenchível com segurança"),("AMBIGUOUS_RATE","Ambíguos"),("CONFLICT_RATE","Em conflito")] %}<div class="coverage-item"><strong>{{ enrich_metrics[key] if enrich_metrics[key] is not none else "—" }}{% if enrich_metrics[key] is not none %}%{% endif %}</strong><span>{{ label }}</span></div>{% endfor %}</div>{% else %}<p class="empty-state">Nenhum produto novo acompanhado ainda.</p>{% endif %}</section>
|
||||
|
||||
@@ -17,7 +17,7 @@ def test_health_is_unavailable_when_database_is_down():
|
||||
|
||||
def test_version_endpoint_reports_release_and_fiscal_versions():
|
||||
body = TestClient(create_app(database_probe=lambda: True)).get("/version").json()
|
||||
assert body["version"] == "1.3.5" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
|
||||
assert body["version"] == "1.3.6" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
|
||||
|
||||
|
||||
def test_production_refuses_the_dev_token(monkeypatch):
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
"""KB search bug fix: the search must be GLOBAL server-side (filter -> count -> order -> paginate), lenient for humans (distinct from the strict
|
||||
fiscal `kb.alias_match`), and find a product regardless of which page it would fall on. Never touches the fiscal auto-fill rule."""
|
||||
from sqlalchemy import select
|
||||
|
||||
from central.app.enrichment import kb
|
||||
from central.app.enrichment.observability import kb_search, kb_search_ids
|
||||
from central.tests.test_control_panel_pages import _panel_client
|
||||
from central.tests.test_product_enrichment_resolver import env # noqa: F401
|
||||
from central.tests.test_rtc_security import _become, _post
|
||||
|
||||
|
||||
def seed_product(session, gtin, aliases, ncm=None, client="A"):
|
||||
p = kb.upsert_product(session, gtin, aliases[0])
|
||||
for a in aliases[1:]:
|
||||
kb.add_alias(session, p, a)
|
||||
if ncm:
|
||||
kb.add_evidence(session, p, "ncm", ncm, "x", family=kb.CLIENT_FAMILY, client_label=client, lineage={"docs": 20, "years": {"2024": 20}})
|
||||
session.commit()
|
||||
return p
|
||||
|
||||
|
||||
def gtin(body12):
|
||||
total = sum(int(d) * (3 if i % 2 == 0 else 1) for i, d in enumerate(reversed(body12)))
|
||||
return "0" + body12 + str((10 - total % 10) % 10)
|
||||
|
||||
|
||||
def test_search_is_global_not_limited_to_a_rendered_page(env):
|
||||
target = seed_product(env, gtin("789100050001"), ["COCA-COLA ORIGINAL LATA 350 ML"], ncm="22021000")
|
||||
for i in range(30): # unrelated noise: if search were page-limited, the target could fall off the visible slice
|
||||
seed_product(env, gtin(f"78920005{i:04d}"), [f"PRODUTO IRRELEVANTE {i} CAIXA 1 UN"])
|
||||
ids = kb_search_ids(env, "COCA COLA LATA 350") # missing the unit "ML": would fail the strict fiscal alias_match
|
||||
assert target.id in ids
|
||||
|
||||
|
||||
def test_search_finds_product_on_first_middle_and_last_page(env):
|
||||
target = seed_product(env, gtin("789100050002"), ["COCA-COLA ORIGINAL LATA 350 ML"])
|
||||
for i in range(25):
|
||||
seed_product(env, gtin(f"78920006{i:04d}"), [f"COCA-COLA ORIGINAL LATA 350 ML VARIANTE {i}"])
|
||||
for page in (1, 2, 3):
|
||||
res = kb_search(env, "COCA COLA LATA 350", page=page, page_size=10)
|
||||
assert res["total"] == 26 and res["pages"] == 3
|
||||
found_on_last_page = kb_search(env, "COCA COLA LATA 350", page=3, page_size=10)
|
||||
assert len(found_on_last_page["results"]) >= 1
|
||||
|
||||
|
||||
def test_search_by_gtin_exact(env):
|
||||
key = gtin("789100050003")
|
||||
seed_product(env, key, ["COCA-COLA ORIGINAL LATA 350 ML"])
|
||||
body = key[1:]
|
||||
assert kb_search_ids(env, body) and kb_search_ids(env, body)[0] == kb_search_ids(env, key)[0]
|
||||
assert kb_search_ids(env, "0000000000000") == []
|
||||
|
||||
|
||||
def test_search_by_name_with_and_without_hyphen_and_accents(env):
|
||||
p = seed_product(env, gtin("789100050004"), ["ACAI ORIGINAL POLPA 100 G"])
|
||||
for query in ("ACAI POLPA 100", "AÇAÍ POLPA 100", "acai polpa 100"):
|
||||
assert p.id in kb_search_ids(env, query), query
|
||||
|
||||
|
||||
def test_search_zero_results_for_unknown_product(env):
|
||||
seed_product(env, gtin("789100050005"), ["FEIJAO CARIOCA 1 KG"])
|
||||
assert kb_search_ids(env, "PRODUTO QUE NAO EXISTE NA BASE") == []
|
||||
|
||||
|
||||
def test_search_before_pagination_semantics(env):
|
||||
"""Filter runs over the WHOLE table, THEN the result is paginated - not the other way round."""
|
||||
matches = [seed_product(env, gtin(f"78910006{i:04d}"), [f"SABONETE LIQUIDO ERVA DOCE {i} 200 ML"]) for i in range(15)]
|
||||
for i in range(40):
|
||||
seed_product(env, gtin(f"78910007{i:04d}"), [f"OUTRO PRODUTO {i} QUALQUER"])
|
||||
res = kb_search(env, "SABONETE LIQUIDO ERVA DOCE", page=2, page_size=10)
|
||||
assert res["total"] == 15 and res["page"] == 2 and len(res["results"]) == 5
|
||||
found_ids = {r["gtin_key"] for r in kb_search(env, "SABONETE LIQUIDO ERVA DOCE", page=1, page_size=10)["results"]} | {r["gtin_key"] for r in res["results"]}
|
||||
assert found_ids == {p.gtin for p in matches}
|
||||
|
||||
|
||||
def test_conflict_candidates_all_visible_and_none_auto_confirmed(env):
|
||||
seed_product(env, gtin("789100050006"), ["COCA COLA LATA 350 ML"], ncm="22030000", client="RINCAO")
|
||||
p2 = seed_product(env, gtin("789100050006"), ["REFRIGERANTE COCA COLA LATA 350 ML"], ncm="22021000", client="UNIAO")
|
||||
# both aliases resolve to the SAME kb product (same GTIN key), so seeding into it twice just adds evidence
|
||||
ids = kb_search_ids(env, "COCA COLA LATA 350")
|
||||
assert p2.id in ids
|
||||
|
||||
|
||||
def test_admin_attest_button_authorization_end_to_end():
|
||||
client, agent_id, _ = _panel_client()
|
||||
with client._session_factory() as s:
|
||||
from central.tests.test_enrichment_history import staging
|
||||
from central.tests.test_enrichment_history import run as seed_history
|
||||
from central.tests.test_product_enrichment_resolver import seed as seed_ncm_dataset
|
||||
seed_ncm_dataset(s)
|
||||
seed_history(s, [staging("RINCAO", "7891000500071", "19059090", years={2024: 900, 2025: 900}, desc="COCA LATA 350 ML"),
|
||||
staging("UNIAO", "7891000500071", "22021000", years={2024: 20, 2025: 13}, desc="REFRIGERANTE COCA COLA LATA 350 ML")])
|
||||
_become(client, "PADRAO")
|
||||
page = client.get("/?kb_q=COCA COLA LATA 350").text
|
||||
assert "Confirmar este NCM" not in page # button hidden for non-admin
|
||||
_become(client, "ADMIN")
|
||||
page = client.get("/?kb_q=COCA COLA LATA 350").text
|
||||
assert "Confirmar este NCM" in page and "22021000" in page and "19059090" in page
|
||||
Reference in New Issue
Block a user