feat: product enrichment foundation (GTIN, KB evidence, providers, quota gateway, resolver, migration 0029)
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
gti
Claude Sonnet 5
@@ -0,0 +1,267 @@
|
||||
"""Product enrichment resolver: 12 resolution scenarios, quota/cache/circuit, provider fixtures. No internet."""
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from central.app.enrichment import kb
|
||||
from central.app.enrichment.gtin import check_digit_ok
|
||||
from central.app.enrichment.providers import Cosmos, OpenFoodFacts, Provider, ProviderError, ProviderProduct, UpcItemDb, build_providers
|
||||
from central.app.enrichment.quota import ProviderGateway
|
||||
from central.app.enrichment.resolver import resolve
|
||||
from central.app.models import (Base, CestEntry, CestNcmEntry, Dataset, GtinLookupCache, KbKnowledge, NcmEntry, ProviderUsageDaily, TipiEntry)
|
||||
|
||||
|
||||
def gtin(body12: str) -> str:
|
||||
total = sum(int(d) * (3 if i % 2 == 0 else 1) for i, d in enumerate(reversed(body12)))
|
||||
return body12 + str((10 - total % 10) % 10)
|
||||
|
||||
|
||||
G1, G2, G3, G4 = gtin("789100010010"), gtin("789100010020"), gtin("789100010030"), gtin("789100010040")
|
||||
|
||||
|
||||
class Fake(Provider):
|
||||
def __init__(self, name, data=None, ncm_provider=False, error=None, daily_limit=None):
|
||||
super().__init__(client=httpx.Client())
|
||||
self.name, self.data, self.provides_ncm, self.error, self.calls = name, data or {}, ncm_provider, error, 0
|
||||
self.daily_limit = daily_limit
|
||||
|
||||
def lookup_gtin(self, g):
|
||||
self.calls += 1
|
||||
if self.error:
|
||||
raise self.error
|
||||
d = self.data.get(g) or self.data.get(g.lstrip('0'))
|
||||
return ProviderProduct(self.name, g, True, **d) if d else ProviderProduct(self.name, g, False)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def env():
|
||||
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||
Base.metadata.create_all(engine)
|
||||
session = Session(engine)
|
||||
ncm = Dataset(kind="NCM", provider="Siscomex", sha256="a" * 64, version_label="v", act="a", source_url="u", status="ACTIVE", record_count=3)
|
||||
session.add(ncm)
|
||||
session.flush()
|
||||
for code in ("22021000", "19059090", "21069090"):
|
||||
session.add(NcmEntry(dataset_id=ncm.id, code=code, description="x", valid_from="01/04/2022", valid_to="31/12/9999"))
|
||||
session.add(NcmEntry(dataset_id=ncm.id, code="11111111", description="expired", valid_from="01/01/2000", valid_to="31/12/2001"))
|
||||
tipi = Dataset(kind="TIPI_IPI", provider="RFB", sha256="b" * 64, version_label="v", act="a", source_url="u", status="ACTIVE", record_count=1)
|
||||
cest = Dataset(kind="CEST", provider="CONFAZ", sha256="c" * 64, version_label="v", act="a", source_url="u", status="ACTIVE", record_count=1)
|
||||
session.add_all([tipi, cest])
|
||||
session.flush()
|
||||
session.add(TipiEntry(dataset_id=tipi.id, ncm="22021000", ex=None, description="x", ipi_rate=0.0, ipi_rate_text="0", source_row=1))
|
||||
entry = CestEntry(dataset_id=cest.id, code="03.007.00", description="REFRIGERANTE COLA LATA 350ML refrigerantes", annex="I")
|
||||
session.add(entry)
|
||||
session.flush()
|
||||
session.add(CestNcmEntry(cest_entry_id=entry.id, ncm_code="22021000"))
|
||||
session.commit()
|
||||
yield session
|
||||
session.close()
|
||||
|
||||
|
||||
def run(session, providers, g, desc, current=None, **kw):
|
||||
return resolve(session, ProviderGateway(session, providers), gtin_raw=g, description=desc, current=current or {}, **kw)
|
||||
|
||||
|
||||
COSMOS = lambda ncm="22021000": {G1: dict(title="REFRIGERANTE COLA LATA 350ML", brand="X", ncm=ncm)}
|
||||
UPC = {G1: dict(title="Refrigerante Cola Lata 350ml", brand="X")}
|
||||
|
||||
|
||||
def test_1_gtin_verified_kb_no_provider_call(env):
|
||||
p = kb.upsert_product(env, kb.gtin_fingerprint("0" + G1), "COLA", brand=None) if False else kb.upsert_product(env, "0" + G1, "REFRIGERANTE COLA LATA 350ML")
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "provider:cosmos", family="PROVIDER:cosmos")
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "provider:other", family="PROVIDER:other")
|
||||
env.commit()
|
||||
fake = Fake("cosmos", COSMOS(), True)
|
||||
r = run(env, [fake], G1, "REFRIGERANTE COLA LATA 350ML")
|
||||
assert r["state"] == "EXACT_GTIN_VERIFIED" and fake.calls == 0
|
||||
f = r["fields"]
|
||||
assert f["CLASSIFICACAO_FISCAL"]["status"] == "AUTO_FILL_SAFE" and f["CLASSIFICACAO_FISCAL"]["value"] == "22021000"
|
||||
assert f["CEST"]["value"] == "03.007.00" and f["IPI"]["value"] == "0.0"
|
||||
|
||||
|
||||
def test_2_two_independent_providers_agree_multi_source(env):
|
||||
a, b = Fake("cosmos", COSMOS(), True), Fake("other", {G1: dict(title="REFRIGERANTE COLA LATA 350ML", ncm="22021000")}, True)
|
||||
r = run(env, [a, b], G1, "REFRIGERANTE COLA LATA 350ML")
|
||||
assert r["state"] == "MULTI_SOURCE_MATCH"
|
||||
assert r["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "AUTO_FILL_SAFE"
|
||||
# promoted: a second installation is served from the KB alone
|
||||
a2 = Fake("cosmos", COSMOS(), True)
|
||||
r2 = run(env, [a2], G1, "REFRIGERANTE COLA LATA 350ML")
|
||||
assert r2["state"] == "EXACT_GTIN_VERIFIED" and a2.calls == 0
|
||||
|
||||
|
||||
def test_3_single_source_ncm_is_suggestion_only(env):
|
||||
r = run(env, [Fake("cosmos", COSMOS(), True)], G1, "REFRIGERANTE COLA LATA 350ML")
|
||||
assert r["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "SUGGESTION_ONLY"
|
||||
assert r["fields"]["CEST"]["status"] == "INSUFFICIENT_EVIDENCE" and r["fields"]["IPI"]["status"] == "INSUFFICIENT_EVIDENCE"
|
||||
|
||||
|
||||
def test_4_identity_only_sources_never_give_ncm(env):
|
||||
r = run(env, [Fake("upcitemdb", UPC), Fake("openfoodfacts", UPC)], G1, "REFRIGERANTE COLA LATA 350ML")
|
||||
assert r["state"] == "MULTI_SOURCE_MATCH"
|
||||
assert r["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "INSUFFICIENT_EVIDENCE"
|
||||
|
||||
|
||||
def test_5_conflicting_ncm_zero_write(env):
|
||||
a, b = Fake("cosmos", COSMOS("22021000"), True), Fake("other", COSMOS("19059090"), True)
|
||||
r = run(env, [a, b], G1, "REFRIGERANTE COLA LATA 350ML")
|
||||
assert r["state"] == "CONFLICT"
|
||||
assert all(v["status"] != "AUTO_FILL_SAFE" for v in r["fields"].values())
|
||||
|
||||
|
||||
def test_6_ncm_not_in_official_dataset_rejected(env):
|
||||
a, b = Fake("cosmos", COSMOS("99999999"), True), Fake("other", COSMOS("99999999"), True)
|
||||
r = run(env, [a, b], G1, "REFRIGERANTE COLA LATA 350ML")
|
||||
assert r["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "INSUFFICIENT_EVIDENCE"
|
||||
a, b = Fake("cosmos", COSMOS("11111111"), True), Fake("other", COSMOS("11111111"), True) # expired NCM is not vigente
|
||||
assert run(env, [a, b], gtin("789100010050"), "X")["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "INSUFFICIENT_EVIDENCE"
|
||||
|
||||
|
||||
def test_7_invalid_gtin_never_calls_providers(env):
|
||||
fake = Fake("cosmos", COSMOS(), True)
|
||||
r = run(env, [fake], G1[:-1] + str((int(G1[-1]) + 1) % 10), "PIPOCA")
|
||||
assert r["gtin_status"] == "INVALID_GTIN" and r["state"] == "INVALID_GTIN" and fake.calls == 0
|
||||
|
||||
|
||||
def test_8_gtin_not_found_anywhere(env):
|
||||
r = run(env, [Fake("cosmos", {}, True), Fake("upcitemdb")], G2, "PRODUTO QUALQUER")
|
||||
assert r["state"] == "NOT_FOUND" and r["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "INSUFFICIENT_EVIDENCE"
|
||||
|
||||
|
||||
def test_9_name_only_exact_verified_fingerprint(env):
|
||||
p = kb.upsert_product(env, "0" + G1, "REFRIGERANTE COLA ORIGINAL LATA 350 ML")
|
||||
for fam in ("PROVIDER:a", "PROVIDER:b"):
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "x", family=fam)
|
||||
env.commit()
|
||||
r = run(env, [Fake("cosmos")], "", "REFRIGERANTE COLA ORIGINAL LATA 350ML")
|
||||
assert r["state"] == "EXACT_KB_NAME_VERIFIED" and r["fields"]["CLASSIFICACAO_FISCAL"]["value"] == "22021000"
|
||||
|
||||
|
||||
def test_10_generic_name_never_fills(env):
|
||||
p = kb.upsert_product(env, "0" + G1, "REFRIGERANTE")
|
||||
for fam in ("PROVIDER:a", "PROVIDER:b"):
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "x", family=fam)
|
||||
env.commit()
|
||||
r = run(env, [], "", "REFRIGERANTE")
|
||||
assert r["state"] == "AMBIGUOUS" and all(v["status"] != "AUTO_FILL_SAFE" for v in r["fields"].values())
|
||||
|
||||
|
||||
def test_11_name_matching_different_ncms_is_conflict(env):
|
||||
for g, ncm in ((G1, "22021000"), (G3, "21069090")):
|
||||
p = kb.upsert_product(env, "0" + g, "SUCO NATURAL LARANJA GARRAFA 1L")
|
||||
for fam in ("PROVIDER:a", "PROVIDER:b"):
|
||||
kb.add_evidence(env, p, "ncm", ncm, "x", family=fam)
|
||||
env.commit()
|
||||
r = run(env, [], "", "SUCO NATURAL LARANJA GARRAFA 1L")
|
||||
assert r["state"] == "CONFLICT" and all(v["status"] == "CONFLICT" for v in r["fields"].values())
|
||||
|
||||
|
||||
def test_12_already_filled_fields_are_never_touched(env):
|
||||
p = kb.upsert_product(env, "0" + G1, "REFRIGERANTE COLA LATA 350ML")
|
||||
for fam in ("PROVIDER:a", "PROVIDER:b"):
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "x", family=fam)
|
||||
env.commit()
|
||||
r = run(env, [], G1, "REFRIGERANTE COLA LATA 350ML", current={"CLASSIFICACAO_FISCAL": "19059090", "CEST": "", "IPI": None})
|
||||
assert r["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "ALREADY_SET"
|
||||
|
||||
|
||||
def test_client_consensus_needs_two_distinct_clients(env):
|
||||
p = kb.upsert_product(env, "0" + G1, "X")
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "client", family=kb.CLIENT_FAMILY, client_label="A")
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "client", family=kb.CLIENT_FAMILY, client_label="A")
|
||||
assert kb.evaluate(env, p, "ncm", lambda v: True)["status"] == kb.EVIDENCE_ONLY
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "client", family=kb.CLIENT_FAMILY, client_label="B")
|
||||
assert kb.evaluate(env, p, "ncm", lambda v: True)["status"] == kb.EVIDENCE_ONLY # consensus alone is one family, never authority
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "provider", family="PROVIDER:cosmos")
|
||||
assert kb.evaluate(env, p, "ncm", lambda v: True)["status"] == kb.VERIFIED_KB
|
||||
|
||||
|
||||
def test_cache_spares_quota_across_installations(env):
|
||||
fake = Fake("upcitemdb", UPC)
|
||||
for _ in range(5):
|
||||
run(env, [fake], G1, "REFRIGERANTE COLA LATA 350ML")
|
||||
assert fake.calls == 1
|
||||
usage = env.scalar(select(ProviderUsageDaily).where(ProviderUsageDaily.provider == "upcitemdb"))
|
||||
assert usage.requests == 1 and usage.cache_hits == 4
|
||||
|
||||
|
||||
def test_negative_cache(env):
|
||||
fake = Fake("upcitemdb")
|
||||
for _ in range(3):
|
||||
run(env, [fake], G2, "PRODUTO X")
|
||||
assert fake.calls == 1
|
||||
|
||||
|
||||
def test_quota_exhausted_pending_with_retry(env):
|
||||
fake = Fake("cosmos", COSMOS(), True, daily_limit=1)
|
||||
run(env, [fake], G1, "REFRIGERANTE COLA LATA 350ML")
|
||||
r = run(env, [fake], G3, "OUTRO PRODUTO")
|
||||
assert r["state"] == "PENDING_PROVIDER_QUOTA" and r["retry_at"] is not None and fake.calls == 1
|
||||
|
||||
|
||||
def test_circuit_breaker_and_timeout_is_not_a_crash(env):
|
||||
fake = Fake("upcitemdb", error=ProviderError("TIMEOUT", "x"))
|
||||
states = [run(env, [fake], gtin(f"78910001{i:04d}"), "P")["state"] for i in range(6)]
|
||||
assert fake.calls == 3 # circuit opened after 3 failures, no more calls
|
||||
assert set(states) == {"PENDING_PROVIDERS_OFFLINE"}
|
||||
|
||||
|
||||
def test_cosmos_without_token_is_not_configured_and_costs_nothing(env):
|
||||
cosmos = Cosmos(httpx.Client(transport=httpx.MockTransport(lambda r: pytest.fail("no call expected"))), None)
|
||||
r = run(env, [cosmos], G1, "REFRIGERANTE COLA LATA 350ML")
|
||||
assert r["providers"][0]["reason"] == "NOT_CONFIGURED" and r["state"] == "NOT_FOUND"
|
||||
|
||||
|
||||
def test_paid_provider_guard():
|
||||
class S: allow_paid_providers = False; cosmos_token = None
|
||||
assert [p.name for p in build_providers(S())] == ["cosmos", "upcitemdb", "openfoodfacts"]
|
||||
|
||||
|
||||
# ---- provider contract tests against recorded fixtures (httpx.MockTransport, no network) ----------------------------------------------------------
|
||||
def _client(handler):
|
||||
return httpx.Client(transport=httpx.MockTransport(handler))
|
||||
|
||||
|
||||
def test_upcitemdb_contract():
|
||||
seen = {}
|
||||
|
||||
def handler(request):
|
||||
seen["url"] = str(request.url)
|
||||
return httpx.Response(200, json={"code": "OK", "total": 1, "items": [{"ean": G1, "title": "Cola 350ml", "brand": "X", "category": "Beverages", "size": "350ml"}]},
|
||||
headers={"X-RateLimit-Limit": "100", "X-RateLimit-Remaining": "99", "X-RateLimit-Reset": "1900000000"})
|
||||
p = UpcItemDb(_client(handler)).lookup_gtin(G1)
|
||||
assert p.found and p.title == "Cola 350ml" and p.ncm is None and "trial/lookup" in seen["url"]
|
||||
assert p.headers["x-ratelimit-remaining"] == "99"
|
||||
assert not UpcItemDb(_client(lambda r: httpx.Response(200, json={"code": "OK", "total": 0, "items": []}))).lookup_gtin(G1).found
|
||||
|
||||
|
||||
def test_openfoodfacts_contract_and_user_agent():
|
||||
seen = {}
|
||||
|
||||
def handler(request):
|
||||
seen["ua"] = request.headers.get("user-agent")
|
||||
return httpx.Response(200, json={"status": 1, "product": {"product_name": "Biscoito", "brands": "Marca, Outra", "quantity": "130 g"}})
|
||||
off = OpenFoodFacts(httpx.Client(transport=httpx.MockTransport(handler), headers={"User-Agent": "GTIFiscalHub/1.0 (+contact)"}))
|
||||
p = off.lookup_gtin(G1)
|
||||
assert p.found and p.brand == "Marca" and "GTIFiscalHub" in seen["ua"]
|
||||
assert not OpenFoodFacts(_client(lambda r: httpx.Response(200, json={"status": 0, "status_verbose": "product not found"}))).lookup_gtin(G1).found
|
||||
|
||||
|
||||
def test_cosmos_contract_and_errors():
|
||||
body = {"description": "COCA COLA 350ML", "brand": {"name": "Coca"}, "ncm": {"code": "2202.10.00"}, "cest": {"code": "03.007.00"}}
|
||||
p = Cosmos(_client(lambda r: httpx.Response(200, json=body)), "tok").lookup_gtin(G1)
|
||||
assert p.ncm == "22021000" and p.cest == "0300700" and p.brand == "Coca"
|
||||
assert not Cosmos(_client(lambda r: httpx.Response(404, json={})), "tok").lookup_gtin(G1).found
|
||||
for status, kind in ((429, "RATE_LIMITED"), (401, "AUTH_ERROR"), (503, "SERVER_ERROR")):
|
||||
with pytest.raises(ProviderError) as e:
|
||||
Cosmos(_client(lambda r, s=status: httpx.Response(s)), "tok").lookup_gtin(G1)
|
||||
assert e.value.kind == kind
|
||||
def boom(request): raise httpx.ReadTimeout("t")
|
||||
with pytest.raises(ProviderError) as e:
|
||||
Cosmos(_client(boom), "tok").lookup_gtin(G1)
|
||||
assert e.value.kind == "TIMEOUT"
|
||||
Reference in New Issue
Block a user