feat: close autonomous operations for blocked products

This commit is contained in:
gti
2026-09-22 14:51:54 -03:00
parent dca4847c21
commit 3a36ac630b
16 changed files with 666 additions and 3 deletions
@@ -0,0 +1,43 @@
"""Add deduplicated blocked product registry."""
from alembic import op
import sqlalchemy as sa
revision = "0016_blocked_product_registry"
down_revision = "0015_ai_budget_and_product_cache"
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
"blocked_products",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("tenant_id", sa.Integer(), sa.ForeignKey("tenants.id"), nullable=False),
sa.Column("company", sa.String(length=128), nullable=False),
sa.Column("product_code", sa.String(length=64), nullable=False),
sa.Column("gtin", sa.String(length=16)),
sa.Column("description", sa.Text()),
sa.Column("blocker_type", sa.String(length=48), nullable=False),
sa.Column("field", sa.String(length=64), nullable=False),
sa.Column("reason_code", sa.String(length=96), nullable=False),
sa.Column("evidence_summary", sa.JSON(), nullable=False),
sa.Column("input_fingerprint", sa.String(length=64), nullable=False),
sa.Column("blocker_fingerprint", sa.String(length=64), nullable=False),
sa.Column("first_seen_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.Column("last_seen_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.Column("retry_count", sa.Integer(), server_default="0", nullable=False),
sa.Column("next_retry_at", sa.DateTime(timezone=True)),
sa.Column("resolved_at", sa.DateTime(timezone=True)),
sa.Column("resolution_source", sa.String(length=48)),
sa.UniqueConstraint("blocker_fingerprint", name="uq_blocked_product_fingerprint"),
)
for name, column in (("tenant", "tenant_id"), ("company", "company"), ("product", "product_code"), ("gtin", "gtin"), ("blocker_type", "blocker_type"), ("field", "field"), ("input", "input_fingerprint")):
op.create_index(f"ix_blocked_products_{name}", "blocked_products", [column])
def downgrade():
for name in ("input", "field", "blocker_type", "gtin", "product", "company", "tenant"):
op.drop_index(f"ix_blocked_products_{name}", table_name="blocked_products")
op.drop_table("blocked_products")
+68 -3
View File
@@ -1,15 +1,18 @@
"""Authenticated version-one Hub routes."""
import hashlib, json, logging
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException, Request
from fastapi import APIRouter, Depends, HTTPException, Request, Response
import csv
import io
from sqlalchemy import select
from .auth import authenticate, bootstrap
from .classification import ENGINE_VERSION, Classifier
from .config import settings
from .datasets.service import DatasetService
from .jobs import claim_next_job, complete_job, create_job, fail_job
from .blocked_products import list_blockers, resolve_blocker, resolve_product_blockers, upsert_blocker
from .context_keys import validate_entry
from .models import FiscalContextEntry, ClassificationCache, ClassificationRequest, ClassificationResult, Dataset, Job
from .models import BlockedProduct, FiscalContextEntry, ClassificationCache, ClassificationRequest, ClassificationResult, Dataset, Job
logger = logging.getLogger("gti_fiscal_hub")
@@ -182,6 +185,63 @@ def router(session_factory, bootstrap_token):
}
finally: session.close()
def _blocked_view(row):
return {
"id": row.id, "tenant_id": row.tenant_id, "company": row.company, "product_code": row.product_code,
"gtin": row.gtin, "description": row.description, "blocker_type": row.blocker_type, "field": row.field,
"reason_code": row.reason_code, "evidence_summary": row.evidence_summary, "first_seen_at": row.first_seen_at.isoformat() if row.first_seen_at else None,
"last_seen_at": row.last_seen_at.isoformat() if row.last_seen_at else None, "retry_count": row.retry_count,
"next_retry_at": row.next_retry_at.isoformat() if row.next_retry_at else None, "resolved_at": row.resolved_at.isoformat() if row.resolved_at else None,
"resolution_source": row.resolution_source,
}
@routes.get("/blockers")
def blockers_list(blocker_type: str | None = None, field: str | None = None, identity=Depends(principal)):
session, agent = identity
try:
return {"blockers": [_blocked_view(row) for row in list_blockers(session, agent.tenant_id, blocker_type=blocker_type, field=field)]}
finally:
session.close()
@routes.get("/blockers/export.csv")
def blockers_export(identity=Depends(principal)):
session, agent = identity
try:
output = io.StringIO()
writer = csv.writer(output)
writer.writerow(["id", "company", "product_code", "gtin", "blocker_type", "field", "reason_code", "retry_count", "first_seen_at", "last_seen_at"])
for row in list_blockers(session, agent.tenant_id):
writer.writerow([row.id, row.company, row.product_code, row.gtin or "", row.blocker_type, row.field, row.reason_code, row.retry_count, row.first_seen_at.isoformat() if row.first_seen_at else "", row.last_seen_at.isoformat() if row.last_seen_at else ""])
return Response(output.getvalue(), media_type="text/csv", headers={"Content-Disposition": "attachment; filename=blocked-products.csv"})
finally:
session.close()
@routes.post("/blockers/reprocess")
def blockers_reprocess(payload: dict, identity=Depends(principal)):
session, agent = identity
try:
rows = list_blockers(session, agent.tenant_id, blocker_type=payload.get("blocker_type"), field=payload.get("field"))
codes = [row.product_code for row in rows if row.product_code]
if not codes:
return {"created": False, "reason": "NO_DUE_BLOCKERS"}
job = create_job(session, agent, "REPROCESS_PRODUCTS", {"codigos": codes, "reason": "BLOCKER_REPROCESS"})
return {"created": True, "job_id": job.id, "products": len(codes)}
finally:
session.close()
@routes.post("/blockers/{blocker_id}/resolve")
def blockers_resolve(blocker_id: int, payload: dict, identity=Depends(principal)):
session, agent = identity
try:
row = session.get(BlockedProduct, blocker_id)
if row is None or row.tenant_id != agent.tenant_id:
raise HTTPException(404, "blocker not found")
if not resolve_blocker(session, blocker_id, payload.get("resolution_source", "OPERATOR")):
raise HTTPException(404, "blocker not found")
return {"resolved": True, "blocker_id": blocker_id}
finally:
session.close()
def _job_view(job):
return {
"job_id": job.id, "agent_id": job.agent_id, "kind": job.kind, "status": job.status,
@@ -231,7 +291,12 @@ def router(session_factory, bootstrap_token):
try:
job = session.get(Job, job_id)
if job is None or job.agent_id != agent.id: raise HTTPException(404, "job not found")
if not complete_job(session, job, payload.get("result_summary", {})): raise HTTPException(409, "job is not RUNNING")
result_summary = payload.get("result_summary", {})
for blocker in result_summary.get("blocked_products", []):
upsert_blocker(session, agent.tenant_id, blocker)
for product_code in result_summary.get("resolved_products", []):
resolve_product_blockers(session, agent.tenant_id, product_code)
if not complete_job(session, job, result_summary): raise HTTPException(409, "job is not RUNNING")
logger.info("job completed job_id=%s agent_id=%s result=%s", job_id, agent.id, payload.get("result_summary", {}))
return _job_view(session.get(Job, job_id))
finally: session.close()
+62
View File
@@ -0,0 +1,62 @@
"""Central blocked-product registry with fingerprint deduplication."""
from datetime import datetime, timezone
from sqlalchemy import select
from .models import BlockedProduct
from .operational_state import blocker_fingerprint
def upsert_blocker(session, tenant_id: int, payload: dict) -> BlockedProduct:
fingerprint = payload.get("blocker_fingerprint") or blocker_fingerprint(payload.get("tenant", str(tenant_id)), payload.get("company", ""), payload.get("product_code", ""), payload.get("field", ""), payload.get("reason_code", ""), payload.get("input_fingerprint", ""))
row = session.scalar(select(BlockedProduct).where(BlockedProduct.blocker_fingerprint == fingerprint))
now = datetime.now(timezone.utc)
if row is None:
row = BlockedProduct(tenant_id=tenant_id, company=payload.get("company", ""), product_code=str(payload.get("product_code", "")), gtin=payload.get("gtin"), description=payload.get("description"), blocker_type=payload.get("blocker_type", "BLOCKED_MISSING_EVIDENCE"), field=payload.get("field", ""), reason_code=payload.get("reason_code", ""), evidence_summary=payload.get("evidence_summary") or {}, input_fingerprint=payload.get("input_fingerprint", ""), blocker_fingerprint=fingerprint)
session.add(row)
else:
row.last_seen_at = now
row.evidence_summary = payload.get("evidence_summary") or row.evidence_summary
row.input_fingerprint = payload.get("input_fingerprint", row.input_fingerprint)
row.resolved_at = None
session.commit()
return row
def list_blockers(session, tenant_id: int, *, input_fingerprint: str | None = None, blocker_type: str | None = None, field: str | None = None):
query = select(BlockedProduct).where(BlockedProduct.tenant_id == tenant_id, BlockedProduct.resolved_at.is_(None))
if input_fingerprint is not None:
query = query.where(BlockedProduct.input_fingerprint != input_fingerprint)
if blocker_type:
query = query.where(BlockedProduct.blocker_type == blocker_type)
if field:
query = query.where(BlockedProduct.field == field)
return list(session.scalars(query).all())
def retry_due(row: BlockedProduct, current_input_fingerprint: str, now: datetime | None = None) -> bool:
now = now or datetime.now(timezone.utc)
if row.input_fingerprint != current_input_fingerprint:
return True
return row.next_retry_at is not None and row.next_retry_at <= now
def resolve_blocker(session, blocker_id: int, source: str) -> bool:
row = session.get(BlockedProduct, blocker_id)
if row is None:
return False
row.resolved_at = datetime.now(timezone.utc)
row.resolution_source = source
session.commit()
return True
def resolve_product_blockers(session, tenant_id: int, product_code: str, source: str = "AUTO_REEVALUATION") -> int:
rows = session.scalars(select(BlockedProduct).where(BlockedProduct.tenant_id == tenant_id, BlockedProduct.product_code == str(product_code), BlockedProduct.resolved_at.is_(None))).all()
now = datetime.now(timezone.utc)
for row in rows:
row.resolved_at = now
row.resolution_source = source
if rows:
session.commit()
return len(rows)
+24
View File
@@ -250,6 +250,30 @@ class Job(Base):
error: Mapped[str | None] = mapped_column(Text, nullable=True)
class BlockedProduct(Base):
"""Deduplicated safe exception for one tenant/product/field/input."""
__tablename__ = "blocked_products"
__table_args__ = (UniqueConstraint("blocker_fingerprint", name="uq_blocked_product_fingerprint"),)
id: Mapped[int] = mapped_column(primary_key=True)
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
company: Mapped[str] = mapped_column(String(128), index=True)
product_code: Mapped[str] = mapped_column(String(64), index=True)
gtin: Mapped[str | None] = mapped_column(String(16), nullable=True, index=True)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
blocker_type: Mapped[str] = mapped_column(String(48), index=True)
field: Mapped[str] = mapped_column(String(64), index=True)
reason_code: Mapped[str] = mapped_column(String(96))
evidence_summary: Mapped[dict] = mapped_column(JSON, default=dict)
input_fingerprint: Mapped[str] = mapped_column(String(64), index=True)
blocker_fingerprint: Mapped[str] = mapped_column(String(64))
first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
retry_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
next_retry_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
resolution_source: Mapped[str | None] = mapped_column(String(48), nullable=True)
class ClassificationRequest(Base):
__tablename__ = "classification_requests"
id: Mapped[int] = mapped_column(primary_key=True)
+48
View File
@@ -0,0 +1,48 @@
"""Per-product operational state and safe partial-processing metrics."""
from collections import Counter
import hashlib
import json
STATES = {
"COMPLETE", "CORRECTED", "BLOCKED_MISSING_EVIDENCE", "BLOCKED_CONFLICT",
"BLOCKED_COMPANY_CONTEXT", "BLOCKED_SCHEMA", "FAILED_TECHNICAL",
}
def blocker_fingerprint(tenant: str, company: str, product_code: str, field: str, reason_code: str, input_fingerprint: str) -> str:
raw = {"tenant": tenant, "company": company, "product_code": product_code, "field": field, "reason_code": reason_code, "input_fingerprint": input_fingerprint}
return hashlib.sha256(json.dumps(raw, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
def classify_product_state(product: dict, decision: dict, *, write_status: str | None = None) -> dict:
status = str(decision.get("status") or "").upper()
if write_status == "FAILED":
state = "FAILED_TECHNICAL"
elif status in {"AMBIGUOUS", "CONFLICT", "NCM_CONFLICT"}:
state = "BLOCKED_CONFLICT"
elif status in {"INVALID_NCM", "LEGACY_NCM", "NOT_FOUND"}:
state = "BLOCKED_MISSING_EVIDENCE"
elif status in {"COMPANY_CONTEXT_REQUIRED", "COMPANY_CONTEXT_FAIL"}:
state = "BLOCKED_COMPANY_CONTEXT"
elif status in {"ERP_SCHEMA_LIMITATION", "UNSUPPORTED_SCHEMA"}:
state = "BLOCKED_SCHEMA"
elif write_status == "SUCCEEDED":
state = "CORRECTED"
else:
state = "COMPLETE"
return {"product_code": str(product.get("CODIGO") or ""), "state": state, "decision_status": status}
def should_retry_blocker(previous_input_fingerprint: str | None, current_input_fingerprint: str) -> bool:
return previous_input_fingerprint != current_input_fingerprint
def summarize_operational_states(states: list[dict]) -> dict:
counts = Counter(item.get("state") for item in states)
result = {state: counts.get(state, 0) for state in sorted(STATES)}
result["TOTAL_PRODUCTS"] = len(states)
result["SAFE_BLOCKED"] = sum(result[state] for state in STATES if state.startswith("BLOCKED_"))
result["AUTONOMOUSLY_PROCESSABLE_PRODUCTS"] = result["TOTAL_PRODUCTS"] - result["FAILED_TECHNICAL"]
result["AGENT_HEALTH"] = "HEALTHY" if result["FAILED_TECHNICAL"] == 0 else "DEGRADED"
return result
@@ -0,0 +1,18 @@
from central.app.operational_state import classify_product_state, summarize_operational_states
def test_twenty_thousand_products_keep_agent_healthy_with_safe_blocks():
states = [{"state": "COMPLETE"}] * 19900 + [{"state": "BLOCKED_MISSING_EVIDENCE"}] * 100
summary = summarize_operational_states(states)
assert summary["AUTONOMOUSLY_PROCESSABLE_PRODUCTS"] == 20000
assert summary["SAFE_BLOCKED"] == 100
assert summary["AGENT_HEALTH"] == "HEALTHY"
def test_new_evidence_changes_product_state_without_global_failure():
blocked = classify_product_state({"CODIGO": "P1"}, {"status": "INVALID_NCM"})
resolved = classify_product_state({"CODIGO": "P1"}, {"status": "EXACT_NCM"})
assert blocked["state"] == "BLOCKED_MISSING_EVIDENCE"
assert resolved["state"] == "COMPLETE"
+50
View File
@@ -0,0 +1,50 @@
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session
from central.app.blocked_products import list_blockers, resolve_product_blockers, retry_due, upsert_blocker
from central.app.models import Agent, Base, BlockedProduct, Tenant
def _session():
engine = create_engine("sqlite://")
Base.metadata.create_all(engine)
session = Session(engine)
tenant = Tenant(name="tenant-blocked")
session.add(tenant)
session.flush()
return session, tenant
def test_same_blocker_is_upserted_not_duplicated():
session, tenant = _session()
payload = {"tenant": "t", "company": "c", "product_code": "P1", "field": "CFOP", "blocker_type": "BLOCKED_MISSING_EVIDENCE", "reason_code": "MISSING_EVIDENCE", "input_fingerprint": "v1", "evidence_summary": {"source": "ERP"}}
first = upsert_blocker(session, tenant.id, payload)
second = upsert_blocker(session, tenant.id, payload)
assert first.id == second.id
assert session.query(BlockedProduct).count() == 1
def test_new_input_fingerprint_is_due_for_retry():
session, tenant = _session()
payload = {"tenant": "t", "company": "c", "product_code": "P1", "field": "CFOP", "blocker_type": "BLOCKED_MISSING_EVIDENCE", "reason_code": "MISSING_EVIDENCE", "input_fingerprint": "v1", "evidence_summary": {}}
row = upsert_blocker(session, tenant.id, payload)
assert list_blockers(session, tenant.id, input_fingerprint="v2") == [row]
def test_product_re_evaluation_closes_previous_blocker():
session, tenant = _session()
row = upsert_blocker(session, tenant.id, {"company": "c", "product_code": "P1", "field": "CFOP", "reason_code": "MISSING", "input_fingerprint": "v1"})
assert resolve_product_blockers(session, tenant.id, "P1") == 1
assert session.get(BlockedProduct, row.id).resolved_at is not None
def test_same_input_without_retry_schedule_is_cache_hit():
session, tenant = _session()
row = upsert_blocker(session, tenant.id, {"company": "c", "product_code": "P1", "field": "CFOP", "reason_code": "MISSING", "input_fingerprint": "v1"})
assert retry_due(row, "v1") is False
assert retry_due(row, "v2") is True
+17
View File
@@ -130,3 +130,20 @@ def test_reprocess_products_requires_codigos():
headers = {"Authorization": "Bearer test-token"}
response = client.post("/v1/jobs", json={"kind": "REPROCESS_PRODUCTS", "payload": {}}, headers=headers)
assert response.status_code == 400
def test_blocker_registry_api_deduplicates_and_resolves():
client, _ = _client()
headers = {"Authorization": "Bearer test-token"}
summary = {"blocked_products": [{"company": "C", "product_code": "P1", "blocker_type": "BLOCKED_MISSING_EVIDENCE", "field": "CFOP", "reason_code": "MISSING", "input_fingerprint": "v1", "evidence_summary": {"source": "ERP"}}]}
created = client.post("/v1/jobs", json={"kind": "REPROCESS_PRODUCTS", "payload": {"codigos": ["P1"]}}, headers=headers)
job_id = created.json()["job_id"]
client.get("/v1/jobs/next", headers=headers)
client.post(f"/v1/jobs/{job_id}/complete", json={"result_summary": summary}, headers=headers)
listed = client.get("/v1/blockers?field=CFOP", headers=headers)
assert listed.status_code == 200
assert len(listed.json()["blockers"]) == 1
blocker_id = listed.json()["blockers"][0]["id"]
resolved = client.post(f"/v1/blockers/{blocker_id}/resolve", json={"resolution_source": "TEST"}, headers=headers)
assert resolved.json()["resolved"] is True
assert client.get("/v1/blockers", headers=headers).json()["blockers"] == []
+33
View File
@@ -0,0 +1,33 @@
from central.app.operational_state import blocker_fingerprint, classify_product_state, should_retry_blocker, summarize_operational_states
def test_blocked_product_does_not_make_other_products_fail():
states = [
classify_product_state({"CODIGO": "1"}, {"status": "EXACT_NCM"}),
classify_product_state({"CODIGO": "2"}, {"status": "AMBIGUOUS"}),
]
summary = summarize_operational_states(states)
assert summary["COMPLETE"] == 1
assert summary["BLOCKED_CONFLICT"] == 1
assert summary["AGENT_HEALTH"] == "HEALTHY"
def test_blocker_fingerprint_deduplicates_same_input():
first = blocker_fingerprint("tenant", "company", "P1", "CFOP", "MISSING_EVIDENCE", "engine-v1")
second = blocker_fingerprint("tenant", "company", "P1", "CFOP", "MISSING_EVIDENCE", "engine-v1")
assert first == second
def test_retry_only_when_input_fingerprint_changes():
assert should_retry_blocker("old", "new") is True
assert should_retry_blocker("old", "old") is False
def test_technical_failure_is_not_healthy_when_present():
summary = summarize_operational_states([{"state": "FAILED_TECHNICAL"}])
assert summary["FAILED_TECHNICAL"] == 1
assert summary["AGENT_HEALTH"] == "DEGRADED"