feat: coverage V2 - historical evidence miner, KB evidence model + promotion audit + revalidation, Kodebar/OSCBR limits, KB/provider panel, seed import; 1.3.2 (Hub only)
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
gti
Claude Sonnet 5
@@ -0,0 +1,146 @@
|
||||
"""Historical evidence -> KB: temporal validity, lineage independence, conflicts, promotion audit, idempotent import. No provider is ever contacted."""
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from central.app.enrichment import history, kb
|
||||
from central.app.enrichment.resolver import resolve
|
||||
from central.app.enrichment.quota import ProviderGateway
|
||||
from central.app.models import KbKnowledge, KbProduct, KbPromotionAudit
|
||||
from central.tests.test_product_enrichment_resolver import COSMOS, Fake, G1, G2, G3, env, gtin, seed # noqa: F401
|
||||
|
||||
G = "0" + G1 # 14-digit key
|
||||
|
||||
|
||||
def staging(client, gtin13, ncm, *, master=True, years=None, nfe=0, desc="REFRIGERANTE COLA LATA 350ML", code="1"):
|
||||
return {"client": client, "master": [{"code": code, "desc": desc, "gtin": gtin13, "codbarra": "", "ncm": ncm if master else "", "cest": ""}],
|
||||
"nfce": [{"code": code, "year": y, "ncm": ncm, "cest": "", "n": n} for y, n in (years or {}).items()], "nfe": [{"code": code, "codbarra": gtin13, "ncm": ncm, "desc": desc, "n": nfe}] if nfe else [], "barcodes": []}
|
||||
|
||||
|
||||
def run(session, stagings, dry=False):
|
||||
obs = history.build_observations(stagings)
|
||||
cur = history.official_sets(session)
|
||||
return history.apply(session, obs, lambda v: v in cur, dry_run=dry)
|
||||
|
||||
|
||||
def status(session, key=G):
|
||||
p = session.scalar(select(KbProduct).where(KbProduct.gtin == key))
|
||||
cur = history.official_sets(session)
|
||||
return kb.evaluate(session, p, "ncm", lambda v: v in cur)
|
||||
|
||||
|
||||
def test_single_client_multi_year_history_is_corroborated_never_verified(env):
|
||||
r = run(env, [staging("A", G1, "22021000", years={2023: 300, 2024: 700, 2025: 900})])
|
||||
assert r["by_status"] == {"CORROBORATED": 1} and r["WOULD_VERIFY"] == 0
|
||||
assert env.scalars(select(KbPromotionAudit)).all() == []
|
||||
|
||||
|
||||
def test_master_plus_documents_of_same_erp_are_one_family(env):
|
||||
run(env, [staging("A", G1, "22021000", master=True, years={2025: 5})]) # few docs, one year: weak
|
||||
assert status(env)["status"] == kb.OBSERVED and status(env)["families"] == []
|
||||
|
||||
|
||||
def test_two_clients_corroborate_but_three_clients_alone_do_not_verify(env):
|
||||
r = run(env, [staging(c, G1, "22021000", years={2024: 2}) for c in ("A", "B", "C")])
|
||||
assert r["by_status"] == {"CORROBORATED": 1} and r["multi_client"] == 1 and r["WOULD_VERIFY"] == 0
|
||||
|
||||
|
||||
def test_history_plus_independent_provider_verifies_and_is_audited(env):
|
||||
run(env, [staging("A", G1, "22021000", years={2023: 50, 2024: 60})])
|
||||
p = env.scalar(select(KbProduct).where(KbProduct.gtin == G))
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "provider:oscbr", family="PROVIDER:oscbr")
|
||||
ev = status(env)
|
||||
assert ev["status"] == kb.VERIFIED_KB and set(ev["families"]) == {"PROVIDER:oscbr", "CLIENT_DATA"}
|
||||
audit = env.scalars(select(KbPromotionAudit)).one()
|
||||
assert audit.new_status == "VERIFIED_KB" and audit.rule_version == kb.RULE_VERSION and audit.official_dataset["sha256"] and audit.sources["documents"] == 110
|
||||
status(env)
|
||||
assert len(env.scalars(select(KbPromotionAudit)).all()) == 1 # no duplicate audit
|
||||
|
||||
|
||||
def test_different_current_ncms_are_a_conflict_not_majority(env):
|
||||
r = run(env, [staging("A", G1, "22021000", years={2024: 900, 2025: 900}), staging("B", G1, "19059090", years={2024: 5, 2025: 5}), staging("C", G1, "22021000", years={2024: 900})])
|
||||
assert r["CONFLICTS"] == 1
|
||||
res = resolve(env, ProviderGateway(env, []), gtin_raw=G1, description="REFRIGERANTE COLA LATA 350ML", current={}, allow_external=False)
|
||||
assert res["state"] == "CONFLICT" and res["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "CONFLICT"
|
||||
|
||||
|
||||
def test_expired_ncm_is_history_not_a_candidate(env):
|
||||
run(env, [staging("A", G1, "11111111", years={2019: 900, 2020: 900})]) # expired in the fixture dataset
|
||||
ev = status(env)
|
||||
assert ev["value"] is None and ev["status"] == kb.OBSERVED
|
||||
row = env.scalar(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm"))
|
||||
assert row.evidence["info"]["historical_state"] == "HISTORICAL_EXPIRED" and row.status == kb.STALE
|
||||
|
||||
|
||||
def test_ncm_changed_over_time_prefers_current(env):
|
||||
run(env, [staging("A", G1, "11111111", years={2019: 900, 2020: 900}), staging("A", G1, "22021000", master=False, years={2024: 50, 2025: 60})])
|
||||
ev = status(env)
|
||||
assert ev["value"] == "22021000" and ev["status"] == kb.CORROBORATED
|
||||
|
||||
|
||||
def test_stale_ncm_with_equal_recency_does_not_count_as_stable(env):
|
||||
run(env, [staging("A", G1, "22021000", master=False, years={2024: 50, 2025: 60}), staging("B", G1, "11111111", master=False, years={2025: 500})])
|
||||
assert status(env)["status"] in (kb.OBSERVED,) # the old code is as recent: not stable, so not even corroborated
|
||||
|
||||
|
||||
def test_import_is_idempotent_and_dry_run_writes_nothing(env):
|
||||
st = [staging("A", G1, "22021000", years={2023: 300, 2024: 700}), staging("B", G3, "19059090", desc="BISCOITO RECHEADO 130G", code="9", years={2024: 30, 2025: 30})]
|
||||
dry = run(env, st, dry=True)
|
||||
assert env.scalars(select(KbProduct)).all() == []
|
||||
a, b = run(env, st), run(env, st)
|
||||
assert a["by_status"] == b["by_status"] == dry["by_status"]
|
||||
assert len(env.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm")).all()) == 2
|
||||
row = env.scalar(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.value == "22021000"))
|
||||
assert row.evidence["lineages"]["A"]["docs"] == 1000 # replaced, not doubled
|
||||
|
||||
|
||||
def test_aliases_and_no_company_tax_data_stored(env):
|
||||
run(env, [staging("A", G1, "22021000", years={2024: 30, 2025: 30}, desc="COCA COLA LT 350"), staging("B", G1, "22021000", master=False, desc="REFRIG COCA-COLA 350ML", code="2", years={2025: 5})])
|
||||
p = env.scalar(select(KbProduct).where(KbProduct.gtin == G))
|
||||
aliases = {r.value for r in env.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == p.id, KbKnowledge.knowledge_key == "alias"))}
|
||||
assert len(aliases) >= 2
|
||||
blob = repr([r.evidence for r in env.scalars(select(KbKnowledge))]).upper()
|
||||
assert "CFOP" not in blob and "CSOSN" not in blob and "CRT" not in blob
|
||||
|
||||
|
||||
def test_kb_hit_makes_zero_external_requests(env):
|
||||
run(env, [staging("A", G1, "22021000", years={2023: 50, 2024: 60})])
|
||||
p = env.scalar(select(KbProduct).where(KbProduct.gtin == G))
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "provider:oscbr", family="PROVIDER:oscbr")
|
||||
env.commit()
|
||||
spy = Fake("cosmos", COSMOS(), True)
|
||||
r = resolve(env, ProviderGateway(env, [spy]), gtin_raw=G1, description="REFRIGERANTE COLA LATA 350ML", current={})
|
||||
assert r["state"] == "EXACT_GTIN_VERIFIED" and spy.calls == 0 and r["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "AUTO_FILL_SAFE"
|
||||
|
||||
|
||||
def test_revalidation_marks_stale_review_and_keeps_knowledge(env):
|
||||
run(env, [staging("A", G1, "22021000", years={2023: 50, 2024: 60})])
|
||||
p = env.scalar(select(KbProduct).where(KbProduct.gtin == G))
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "provider:oscbr", family="PROVIDER:oscbr")
|
||||
assert status(env)["status"] == kb.VERIFIED_KB
|
||||
out = kb.revalidate(env, lambda v: True)
|
||||
assert out["stale"] == 0 and status(env)["status"] == kb.VERIFIED_KB # NCM still vigente: stays valid
|
||||
out = kb.revalidate(env, lambda v: False) # the official dataset dropped it
|
||||
row = env.scalar(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm"))
|
||||
assert out["stale"] == 1 and row.status == kb.STALE and row.evidence["info"]["stale_reason"] == "STALE_REVIEW_REQUIRED"
|
||||
assert [a.new_status for a in env.scalars(select(KbPromotionAudit)).all()] == ["VERIFIED_KB", "STALE"]
|
||||
|
||||
|
||||
def test_provider_disagreeing_with_verified_kb_creates_conflict_not_overwrite(env):
|
||||
run(env, [staging("A", G1, "22021000", years={2023: 50, 2024: 60})])
|
||||
p = env.scalar(select(KbProduct).where(KbProduct.gtin == G))
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "provider:oscbr", family="PROVIDER:oscbr")
|
||||
assert status(env)["status"] == kb.VERIFIED_KB
|
||||
kb.add_evidence(env, p, "ncm", "19059090", "provider:cosmos", family="PROVIDER:cosmos")
|
||||
ev = status(env)
|
||||
assert ev["status"] == kb.CONFLICT and ev["value"] is None # write blocked until a human resolves it
|
||||
|
||||
|
||||
def test_aliases_from_history_enable_name_only_match_and_generic_names_stay_ambiguous(env):
|
||||
run(env, [staging("A", G1, "22021000", years={2023: 50, 2024: 60}, desc="COCA-COLA ORIGINAL LATA 350 ML")])
|
||||
p = env.scalar(select(KbProduct).where(KbProduct.gtin == G))
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "provider:oscbr", family="PROVIDER:oscbr")
|
||||
env.commit()
|
||||
r = resolve(env, ProviderGateway(env, []), gtin_raw="", description="COCA COLA LATA 350ML", current={})
|
||||
assert r["state"] == "EXACT_KB_NAME_VERIFIED" and r["fields"]["CLASSIFICACAO_FISCAL"]["value"] == "22021000"
|
||||
g = resolve(env, ProviderGateway(env, []), gtin_raw="", description="REFRIGERANTE", current={})
|
||||
assert g["state"] == "AMBIGUOUS" and all(f["status"] != "AUTO_FILL_SAFE" for f in g["fields"].values())
|
||||
@@ -17,7 +17,7 @@ def test_health_is_unavailable_when_database_is_down():
|
||||
|
||||
def test_version_endpoint_reports_release_and_fiscal_versions():
|
||||
body = TestClient(create_app(database_probe=lambda: True)).get("/version").json()
|
||||
assert body["version"] == "1.3.1" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
|
||||
assert body["version"] == "1.3.2" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
|
||||
|
||||
|
||||
def test_production_refuses_the_dev_token(monkeypatch):
|
||||
|
||||
@@ -200,3 +200,18 @@ def test_provider_panel_shows_every_state_and_column(monkeypatch):
|
||||
page = client.get("/").text
|
||||
for label in ("Limite atingido", "Instável", "Credencial recusada", "Indisponível", "Não configurado", "100 / 100", "restam 0", "renova", "último sucesso", "último erro", "140 ms", "cache 30%"):
|
||||
assert label in page, label
|
||||
|
||||
|
||||
def test_dashboard_kb_panel_and_provider_daily_columns(monkeypatch):
|
||||
from central.app.enrichment import kb as kbm
|
||||
from central.app.models import KbProduct
|
||||
client, agent_id, _ = _panel_client()
|
||||
with client._session_factory() as s:
|
||||
p = kbm.upsert_product(s, "07891000100103", "COLA LATA 350ML")
|
||||
kbm.add_evidence(s, p, "ncm", "22021000", "c", family=kbm.CLIENT_FAMILY, client_label="A", lineage={"docs": 900, "years": {"2024": 900}})
|
||||
kbm.add_evidence(s, p, "ncm", "19059090", "c", family=kbm.CLIENT_FAMILY, client_label="B", lineage={"docs": 30, "years": {"2025": 30}})
|
||||
kbm.evaluate(s, p, "ncm", lambda v: True)
|
||||
s.commit()
|
||||
page = client.get("/").text
|
||||
assert "Base de conhecimento de produtos" in page and "NCMs divergentes" in page and "22021000" in page and "19059090" in page
|
||||
assert "achados" in page and "próximo reset" in page or "hoje:" in page
|
||||
|
||||
@@ -10,7 +10,7 @@ from sqlalchemy.pool import StaticPool
|
||||
|
||||
from central.app.enrichment import kb
|
||||
from central.app.enrichment.gtin import check_digit_ok
|
||||
from central.app.enrichment.providers import Oscbr, Cosmos, OpenFoodFacts, Provider, ProviderError, ProviderProduct, UpcItemDb, build_providers
|
||||
from central.app.enrichment.providers import Kodebar, Oscbr, Cosmos, OpenFoodFacts, Provider, ProviderError, ProviderProduct, UpcItemDb, build_providers
|
||||
from central.app.enrichment.quota import ProviderGateway
|
||||
from central.app.enrichment.resolver import resolve
|
||||
from central.app.models import (Base, CestEntry, CestNcmEntry, Dataset, GtinLookupCache, KbKnowledge, NcmEntry, ProviderUsageDaily, TipiEntry)
|
||||
@@ -178,9 +178,9 @@ def test_client_consensus_needs_two_distinct_clients(env):
|
||||
p = kb.upsert_product(env, "0" + G1, "X")
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "client", family=kb.CLIENT_FAMILY, client_label="A")
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "client", family=kb.CLIENT_FAMILY, client_label="A")
|
||||
assert kb.evaluate(env, p, "ncm", lambda v: True)["status"] == kb.EVIDENCE_ONLY
|
||||
assert kb.evaluate(env, p, "ncm", lambda v: True)["status"] == kb.OBSERVED
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "client", family=kb.CLIENT_FAMILY, client_label="B")
|
||||
assert kb.evaluate(env, p, "ncm", lambda v: True)["status"] == kb.EVIDENCE_ONLY # consensus alone is one family, never authority
|
||||
assert kb.evaluate(env, p, "ncm", lambda v: True)["status"] == kb.CORROBORATED # consensus alone is one family, never authority
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "provider", family="PROVIDER:cosmos")
|
||||
assert kb.evaluate(env, p, "ncm", lambda v: True)["status"] == kb.VERIFIED_KB
|
||||
|
||||
@@ -223,7 +223,7 @@ def test_cosmos_without_token_is_not_configured_and_costs_nothing(env):
|
||||
|
||||
def test_paid_provider_guard():
|
||||
class S: allow_paid_providers = False; cosmos_token = None
|
||||
assert [p.name for p in build_providers(S())] == ["cosmos", "oscbr", "upcitemdb", "openfoodfacts", "openproductsfacts", "openbeautyfacts"]
|
||||
assert [p.name for p in build_providers(S())] == ["cosmos", "oscbr", "kodebar", "upcitemdb", "openfoodfacts", "openproductsfacts", "openbeautyfacts"]
|
||||
|
||||
|
||||
# ---- provider contract tests against recorded fixtures (httpx.MockTransport, no network) ----------------------------------------------------------
|
||||
@@ -292,3 +292,24 @@ def test_oscbr_optional_and_contract():
|
||||
with pytest.raises(ProviderError) as e:
|
||||
bad.lookup_gtin(G1)
|
||||
assert e.value.kind == "AUTH_ERROR"
|
||||
|
||||
|
||||
def test_kodebar_identity_only_optional_and_contract():
|
||||
assert not Kodebar(_client(lambda r: pytest.fail("no call expected"))).configured
|
||||
def handler(request):
|
||||
assert request.headers["x-api-key"] == "k" and request.url.params["gtin"] == G1
|
||||
return httpx.Response(200, json={"gtin": G1, "nome": "COCA COLA 350ML", "marca": "Coca", "thumbnail": "u", "barcode_url": "b", "qr_code_url": "q"})
|
||||
p = Kodebar(_client(handler), "k").lookup_gtin(G1)
|
||||
assert p.found and p.title == "COCA COLA 350ML" and p.ncm is None and p.cest is None # identity only, never NCM
|
||||
assert not Kodebar(_client(lambda r: httpx.Response(200, json={})), "k").lookup_gtin(G1).found
|
||||
with pytest.raises(ProviderError) as e:
|
||||
Kodebar(_client(lambda r: httpx.Response(429)), "k").lookup_gtin(G1)
|
||||
assert e.value.kind == "RATE_LIMITED"
|
||||
|
||||
|
||||
def test_backfill_respects_operational_reserve(env):
|
||||
fake = Fake("cosmos", COSMOS(), True, daily_limit=10)
|
||||
gw = ProviderGateway(env, [fake])
|
||||
outs = [gw.lookup(fake, gtin(f"78970000{i:04d}"), "BACKFILL", 20).state for i in range(10)]
|
||||
assert outs.count("SKIPPED") == 2 and fake.calls == 8 # 20% of 10 kept for new products
|
||||
assert gw.lookup(fake, gtin("789700009999"), "OPERATIONAL").state in ("FOUND", "NOT_FOUND")
|
||||
|
||||
Reference in New Issue
Block a user