feat: enrichment Hub API, feature flags UI/gating, NCM refresher, Agent enrichment worker with tests
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -175,7 +175,9 @@ def router(session_factory):
|
||||
session.add(agent)
|
||||
session.flush()
|
||||
session.add(Installation(agent_id=agent.id, tenant_id=enr.tenant_id, installation_id=enr.enrollment_uuid,
|
||||
display_name=enr.display_name, hostname=hostname or None))
|
||||
display_name=enr.display_name, hostname=hostname or None,
|
||||
features_configured=bool(enr.features_selected), corrector_enabled=bool(enr.features_selected and enr.feature_corrector),
|
||||
enrichment_mode=(enr.feature_enrichment_mode if enr.features_selected else "DISABLED")))
|
||||
|
||||
enr.status = "ENROLLED"
|
||||
enr.enrollment_used_at = datetime.now(timezone.utc)
|
||||
|
||||
+4
-1
@@ -19,6 +19,7 @@ from .events import record_event
|
||||
from .installation_identity import compute_agent_health, compute_version_drift, reference_agent_version, sanitize_registration
|
||||
from .control_plane import list_jobs
|
||||
from .human_auth import require_human
|
||||
from .enrichment.api import router as enrichment_router
|
||||
|
||||
logger = logging.getLogger("gti_fiscal_hub")
|
||||
|
||||
@@ -180,7 +181,8 @@ def router(session_factory, bootstrap_token):
|
||||
if installation is not None and installation.installation_id != installation_key:
|
||||
raise HTTPException(409, "installation_id conflicts with registered Agent")
|
||||
if installation is None:
|
||||
installation = Installation(agent_id=agent.id, tenant_id=agent.tenant_id, installation_id=installation_key, display_name=supplied.get("display_name") or agent.name)
|
||||
installation = Installation(agent_id=agent.id, tenant_id=agent.tenant_id, installation_id=installation_key, display_name=supplied.get("display_name") or agent.name,
|
||||
features_configured=True, corrector_enabled=True) # legacy/bootstrap self-registration keeps the pre-existing behaviour
|
||||
session.add(installation); session.flush()
|
||||
fields = {"display_name": supplied.get("display_name"), "customer_code": supplied.get("customer_code"), "notes": supplied.get("notes"), "hostname": supplied.get("hostname"), "windows_version": supplied.get("windows_version"), "machine_fingerprint": supplied.get("machine_fingerprint"), "company_code": supplied.get("company_code"), "company_name": supplied.get("company_name"), "company_cnpj_masked": supplied.get("company_cnpj_masked"), "company_uf": supplied.get("company_uf"), "company_crt": supplied.get("company_crt"), "firebird_version": supplied.get("firebird_version"), "firebird_ods": supplied.get("firebird_ods"), "database_identifier": supplied.get("database_identifier"), "capabilities": supplied.get("capabilities"), "fiscal_engine_version": supplied.get("fiscal_engine_version"), "dataset_versions": supplied.get("dataset_versions"), "last_status": supplied.get("status") or payload.get("status"), "last_seen_at": now}
|
||||
for key, value in fields.items():
|
||||
@@ -464,4 +466,5 @@ def router(session_factory, bootstrap_token):
|
||||
return _job_view(session.get(Job, job_id))
|
||||
finally: session.close()
|
||||
|
||||
routes.include_router(enrichment_router(principal))
|
||||
return routes
|
||||
|
||||
@@ -77,6 +77,11 @@ class Settings:
|
||||
# Z.ai general endpoint (NOT the /api/coding/paas/v4 coding-plan endpoint).
|
||||
ai_api_key: str = os.getenv("AI_API_KEY", "")
|
||||
ai_base_url: str = os.getenv("AI_BASE_URL", "https://api.z.ai/api/paas/v4")
|
||||
# ---- Product enrichment (docs/product_enrichment.md): free providers only, tokens live ONLY in the Hub environment ----
|
||||
cosmos_token: str = os.getenv("COSMOS_TOKEN", "") # optional; without it Cosmos is NOT_CONFIGURED (not an error)
|
||||
allow_paid_providers: bool = os.getenv("ALLOW_PAID_PROVIDERS", "0") == "1"
|
||||
official_refresh_enabled: bool = os.getenv("OFFICIAL_REFRESH_ENABLED", "1") == "1"
|
||||
enrichment_time_budget_seconds: float = float(os.getenv("ENRICHMENT_TIME_BUDGET_SECONDS", "4.5"))
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Automatic refresh of the official NCM dataset (Siscomex JSON): staging -> validate -> atomic promotion, keeping the last valid dataset on ANY failure.
|
||||
|
||||
Every attempt leaves a SourceRefreshRun (url, time, sha256, version label, record count, status, error) for the panel. Nothing here touches Agents or providers.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..config import settings
|
||||
from ..models import Dataset, SourceRefreshRun
|
||||
from .ncm import parse_ncm_payload
|
||||
from .service import DatasetService
|
||||
|
||||
logger = logging.getLogger("gti_fiscal_hub")
|
||||
MIN_RATIO = 0.9 # a payload with < 90% of the active record count is treated as a broken download, never promoted
|
||||
INTERVAL = timedelta(hours=24)
|
||||
RETRY_AFTER_FAILURE = timedelta(hours=6)
|
||||
|
||||
|
||||
def _fetch(url: str) -> bytes:
|
||||
response = httpx.get(url, timeout=max(settings.http_timeout_seconds, 60), follow_redirects=True, headers={"User-Agent": "GTIFiscalHub/1.0 (official dataset refresh)"})
|
||||
response.raise_for_status()
|
||||
return response.content
|
||||
|
||||
|
||||
def refresh_ncm(session, fetch=_fetch, url: str | None = None) -> SourceRefreshRun:
|
||||
url = url or settings.ncm_dataset_url
|
||||
run = SourceRefreshRun(source="NCM", source_url=url, status="RUNNING")
|
||||
session.add(run)
|
||||
session.commit()
|
||||
try:
|
||||
content = fetch(url) # staging: bytes in memory, nothing written to the active dataset yet
|
||||
parsed = parse_ncm_payload(content) # schema validation
|
||||
run.sha256, run.version_label, run.record_count = parsed.sha256, (parsed.version or "")[:64], len(parsed.entries)
|
||||
active = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
|
||||
if active is not None and active.sha256 == parsed.sha256:
|
||||
run.status = "UNCHANGED"
|
||||
else:
|
||||
if active is not None and len(parsed.entries) < MIN_RATIO * (active.record_count or 0):
|
||||
raise ValueError(f"implausible record count {len(parsed.entries)} vs active {active.record_count}")
|
||||
DatasetService(session).import_payload(parsed, url, datetime.now(timezone.utc)) # atomic: one transaction, old ACTIVE superseded only at the end
|
||||
run.status = "UPDATED"
|
||||
except Exception as error: # keep the last valid dataset
|
||||
session.rollback()
|
||||
run = session.get(SourceRefreshRun, run.id)
|
||||
run.status, run.error = "FAILED", f"{type(error).__name__}: {error}"[:300]
|
||||
logger.warning("NCM refresh failed: %s", run.error)
|
||||
run.finished_at = datetime.now(timezone.utc)
|
||||
session.commit()
|
||||
return run
|
||||
|
||||
|
||||
def last_runs(session, source: str = "NCM", limit: int = 5):
|
||||
return session.scalars(select(SourceRefreshRun).where(SourceRefreshRun.source == source).order_by(SourceRefreshRun.id.desc()).limit(limit)).all()
|
||||
|
||||
|
||||
def _due(session) -> bool:
|
||||
last = session.scalar(select(SourceRefreshRun).where(SourceRefreshRun.source == "NCM", SourceRefreshRun.status != "RUNNING").order_by(SourceRefreshRun.id.desc()).limit(1))
|
||||
if last is None or last.finished_at is None:
|
||||
return True
|
||||
finished = last.finished_at if last.finished_at.tzinfo else last.finished_at.replace(tzinfo=timezone.utc)
|
||||
return datetime.now(timezone.utc) - finished >= (RETRY_AFTER_FAILURE if last.status == "FAILED" else INTERVAL)
|
||||
|
||||
|
||||
def start_scheduler(session_factory, first_delay: float = 90.0):
|
||||
"""Daemon thread: checks hourly whether a refresh is due (survives restarts because due-ness is derived from SourceRefreshRun)."""
|
||||
def loop():
|
||||
time.sleep(first_delay)
|
||||
while True:
|
||||
try:
|
||||
with session_factory() as session:
|
||||
if _due(session):
|
||||
refresh_ncm(session)
|
||||
except Exception: # never let the scheduler die
|
||||
logger.exception("official refresh loop error")
|
||||
time.sleep(3600)
|
||||
thread = threading.Thread(target=loop, name="official-refresh", daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Agent-facing product enrichment routes (authenticated as the Agent; the Agent never talks to external providers)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..blocked_products import upsert_blocker
|
||||
from ..config import settings
|
||||
from ..models import Installation, ProductEnrichment, ProductFieldAudit
|
||||
from .providers import build_providers
|
||||
from .quota import ProviderGateway
|
||||
from .resolver import FIELDS, resolve
|
||||
|
||||
PROVIDERS_OVERRIDE = None # tests inject fake providers here (never the network)
|
||||
PENDING_STATES = {"PENDING_PROVIDER_QUOTA", "PENDING_PROVIDERS_OFFLINE", "PENDING_TIME_BUDGET"}
|
||||
REASONS = { # human PT-BR reasons shown through the existing blockers (category ENRICHMENT); never internal jargon
|
||||
"INVALID_GTIN": "Código de barras (GTIN) inválido: o produto não foi consultado em fontes externas.",
|
||||
"AMBIGUOUS": "Descrição genérica demais para identificar o produto com segurança; informe o código de barras.",
|
||||
"NOT_FOUND": "Produto não encontrado nas fontes consultadas; preencha os campos fiscais manualmente.",
|
||||
"CONFLICT": "As fontes divergem sobre a classificação deste produto; nenhuma alteração foi feita.",
|
||||
"SINGLE_WEAK_SOURCE": "Apenas uma fonte identificou o produto; a classificação foi apenas sugerida.",
|
||||
"PENDING_PROVIDER_QUOTA": "Limite diário das fontes de produtos atingido; a consulta será repetida automaticamente.",
|
||||
"PENDING_PROVIDERS_OFFLINE": "Fontes de produtos indisponíveis no momento; a consulta será repetida automaticamente.",
|
||||
}
|
||||
|
||||
|
||||
def _aware(dt):
|
||||
return dt.replace(tzinfo=timezone.utc) if dt is not None and dt.tzinfo is None else dt
|
||||
|
||||
|
||||
def features_of(installation) -> dict:
|
||||
return {"corrector_enabled": bool(installation.corrector_enabled), "enrichment_mode": installation.enrichment_mode, "features_configured": bool(installation.features_configured)}
|
||||
|
||||
|
||||
def router(principal, providers_factory=None):
|
||||
routes = APIRouter(prefix="/products")
|
||||
holder = {}
|
||||
|
||||
def providers():
|
||||
if PROVIDERS_OVERRIDE is not None:
|
||||
return PROVIDERS_OVERRIDE
|
||||
if providers_factory is not None:
|
||||
return providers_factory()
|
||||
if "p" not in holder:
|
||||
holder["p"] = build_providers(settings)
|
||||
return holder["p"]
|
||||
|
||||
@routes.post("/resolve")
|
||||
def resolve_product(payload: dict, identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is None:
|
||||
raise HTTPException(409, "installation not registered")
|
||||
features = features_of(installation)
|
||||
if not features["features_configured"] or installation.enrichment_mode == "DISABLED":
|
||||
return {"state": "FEATURE_DISABLED", "features": features, "fields": {}}
|
||||
code = str(payload.get("product_code") or "")[:64]
|
||||
if not code:
|
||||
raise HTTPException(422, "product_code required")
|
||||
description, current = str(payload.get("description") or "")[:255], payload.get("current") if isinstance(payload.get("current"), dict) else {}
|
||||
started = time.monotonic()
|
||||
result = resolve(session, ProviderGateway(session, providers()), gtin_raw=payload.get("gtin"), description=description, current={k: current.get(k) for k in FIELDS},
|
||||
client_label=f"installation:{installation.id}", budget_s=settings.enrichment_time_budget_seconds)
|
||||
latency = int((time.monotonic() - started) * 1000)
|
||||
row = session.scalar(select(ProductEnrichment).where(ProductEnrichment.installation_id == installation.id, ProductEnrichment.product_code == code))
|
||||
if row is None:
|
||||
row = ProductEnrichment(tenant_id=agent.tenant_id, installation_id=installation.id, product_code=code)
|
||||
session.add(row)
|
||||
for field in (row.human_overridden or {}): # a human decision (even deleting the value) is final for this product
|
||||
if field in result["fields"]:
|
||||
result["fields"][field] = {"status": "HUMAN_OVERRIDDEN", "value": None, "rule": None, "kb_status": None, "evidence": [], "reason": "valor alterado por uma pessoa; nunca mais preenchido automaticamente"}
|
||||
pending = result["state"] in PENDING_STATES
|
||||
row.gtin, row.description, row.latency_ms = result.get("gtin"), description, latency
|
||||
row.attempts = (row.attempts or 0) + 1
|
||||
row.status = "PENDING" if pending else "RESOLVED"
|
||||
row.decisions = {"state": result["state"], "fields": {k: {kk: vv for kk, vv in v.items() if kk != "evidence"} for k, v in result["fields"].items()}}
|
||||
row.next_attempt_at = _aware(result["retry_at"]) if pending else None
|
||||
if pending and row.next_attempt_at is None:
|
||||
row.next_attempt_at = datetime.now(timezone.utc).replace(microsecond=0)
|
||||
session.commit()
|
||||
reason = REASONS.get(result["state"])
|
||||
if reason:
|
||||
upsert_blocker(session, agent.tenant_id, {"company": installation.hostname or "", "product_code": code, "gtin": result.get("gtin"), "description": description, "blocker_type": "ENRICHMENT_" + result["state"],
|
||||
"field": "ENRICHMENT", "reason_code": result["state"], "evidence_summary": {"message": reason}, "input_fingerprint": f"{result.get('gtin')}|{description}"}, installation_id=installation.id)
|
||||
result.update({"features": features, "latency_ms": latency, "mode": installation.enrichment_mode})
|
||||
result["retry_at"] = result["retry_at"].isoformat() if result["retry_at"] else None
|
||||
return result
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@routes.post("/enrichment/applied")
|
||||
def applied(payload: dict, identity=Depends(principal)):
|
||||
"""The Agent reports what it wrote / refused. The Hub re-checks that the feature is still ENABLED and audits every field."""
|
||||
session, agent = identity
|
||||
try:
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is None:
|
||||
raise HTTPException(409, "installation not registered")
|
||||
code = str(payload.get("product_code") or "")[:64]
|
||||
row = session.scalar(select(ProductEnrichment).where(ProductEnrichment.installation_id == installation.id, ProductEnrichment.product_code == code))
|
||||
if row is None:
|
||||
raise HTTPException(404, "unknown product enrichment")
|
||||
versions = payload.get("dataset_versions") if isinstance(payload.get("dataset_versions"), dict) else {}
|
||||
applied_fields, overridden = dict(row.fields_applied or {}), dict(row.human_overridden or {})
|
||||
for item in (payload.get("fields") or [])[:32]:
|
||||
if not isinstance(item, dict) or item.get("field") not in FIELDS:
|
||||
continue
|
||||
outcome = str(item.get("outcome") or "APPLIED")[:24]
|
||||
session.add(ProductFieldAudit(tenant_id=agent.tenant_id, installation_id=installation.id, agent_id=agent.id, product_code=code, gtin=row.gtin, description_snapshot=row.description, field=item["field"],
|
||||
old_value=(str(item["old"])[:64] if item.get("old") is not None else None), new_value=(str(item["new"])[:64] if item.get("new") is not None else None), rule=str(item.get("rule") or "")[:64] or None,
|
||||
kb_status=str(item.get("kb_status") or "")[:16] or None, evidence=item.get("evidence") if isinstance(item.get("evidence"), list) else [], dataset_versions=versions,
|
||||
authorization_origin=f"enrichment_mode:{installation.enrichment_mode}", job_id=payload.get("job_id") if isinstance(payload.get("job_id"), int) else None, outcome=outcome,
|
||||
reason=str(item.get("reason") or "")[:96] or None))
|
||||
if outcome == "APPLIED":
|
||||
applied_fields[item["field"]] = item.get("new")
|
||||
elif outcome == "HUMAN_OVERRIDDEN":
|
||||
overridden[item["field"]] = True
|
||||
row.fields_applied, row.human_overridden = applied_fields, overridden
|
||||
row.status = "APPLIED" if applied_fields and not overridden else ("PARTIAL" if applied_fields else row.status if row.status != "PENDING" else "RESOLVED")
|
||||
row.updated_at = datetime.now(timezone.utc)
|
||||
session.commit()
|
||||
return {"ok": True, "status": row.status, "mode": installation.enrichment_mode}
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@routes.get("/enrichment/authorization")
|
||||
def authorization(identity=Depends(principal)):
|
||||
"""Fresh central authorization read by the Agent right before EVERY field write."""
|
||||
session, agent = identity
|
||||
try:
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
return features_of(installation) if installation is not None else {"corrector_enabled": False, "enrichment_mode": "DISABLED", "features_configured": False}
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
return routes
|
||||
@@ -7,7 +7,8 @@ Field decisions: AUTO_FILL_SAFE | SUGGESTION_ONLY | CONFLICT | INSUFFICIENT_EVID
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime
|
||||
import time
|
||||
from datetime import date
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
@@ -79,7 +80,7 @@ def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current:
|
||||
if not _empty(current, "CEST"):
|
||||
out["CEST"] = _field(ALREADY, reason="campo já preenchido")
|
||||
elif decision["cest_status"] == "CEST_CONFIRMED" and len(cest) == 1 and ncm["status"] == AUTO:
|
||||
out["CEST"] = _field(AUTO, cest[0], "CEST_OFFICIAL_UNIQUE_MAPPING", ncm["kb_status"], [f"NCM {ncm_for_engine}"])
|
||||
out["CEST"] = _field(AUTO, normalize_code(cest[0]), "CEST_OFFICIAL_UNIQUE_MAPPING", ncm["kb_status"], [f"NCM {ncm_for_engine}"])
|
||||
elif decision["cest_status"] == "NO_CEST_MAPPING_FOUND":
|
||||
out["CEST"] = _field(NA, reason="NCM sem mapeamento de CEST no dataset oficial")
|
||||
else:
|
||||
@@ -96,7 +97,8 @@ def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current:
|
||||
return out
|
||||
|
||||
|
||||
def resolve(session, gateway: ProviderGateway, *, gtin_raw, description, current: dict | None = None, client_label: str | None = None, allow_external: bool = True) -> dict:
|
||||
def resolve(session, gateway: ProviderGateway, *, gtin_raw, description, current: dict | None = None, client_label: str | None = None, allow_external: bool = True, budget_s: float | None = None) -> dict:
|
||||
started = time.monotonic()
|
||||
"""Returns {state, gtin_status, gtin, fields:{...}, providers:[...], retry_at, dataset_versions}. Never raises for provider trouble."""
|
||||
current = {k: v for k, v in (current or {}).items()}
|
||||
g = normalize_gtin(gtin_raw)
|
||||
@@ -117,6 +119,10 @@ def resolve(session, gateway: ProviderGateway, *, gtin_raw, description, current
|
||||
found, pending = [], []
|
||||
if allow_external:
|
||||
for provider in gateway.providers:
|
||||
if budget_s is not None and time.monotonic() - started > budget_s:
|
||||
result["providers"].append({"provider": provider.name, "state": "SKIPPED", "reason": "TIME_BUDGET", "from_cache": False})
|
||||
pending.append(type("P", (), {"reason": "TIME_BUDGET", "retry_at": None})())
|
||||
continue
|
||||
out = gateway.lookup(provider, query)
|
||||
result["providers"].append({"provider": out.provider, "state": out.state, "reason": out.reason, "from_cache": out.from_cache})
|
||||
if out.state in ("FOUND", "CACHE_HIT") and out.product:
|
||||
|
||||
+3
-3
@@ -61,8 +61,8 @@ def claim_next_job(session, agent):
|
||||
processing is enabled centrally and no other job of this Agent is RUNNING
|
||||
(one fiscal job at a time per Firebird)."""
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is not None and not installation.processing_enabled:
|
||||
return None
|
||||
if installation is not None and (not installation.processing_enabled or not installation.features_configured or not installation.corrector_enabled):
|
||||
return None # WAITING_FEATURE_CONFIGURATION / corrector switched off centrally: nothing is handed to the Agent
|
||||
recover_stale_running(session, agent)
|
||||
if session.scalar(select(Job.id).where(Job.agent_id == agent.id, Job.status == "RUNNING").limit(1)) is not None:
|
||||
return None
|
||||
@@ -127,7 +127,7 @@ def schedule_automatic_sync(session, agent, installation, snapshot, now=None):
|
||||
- otherwise, if processing is enabled and the last sync attempt is older than
|
||||
the installation's interval, and nothing is queued/running -> ONE SYNC_INCREMENTAL.
|
||||
Returns the created job or None."""
|
||||
if installation is None or not installation.processing_enabled:
|
||||
if installation is None or not installation.processing_enabled or not installation.features_configured or not installation.corrector_enabled:
|
||||
return None
|
||||
now = now or datetime.now(timezone.utc)
|
||||
sync_jobs = session.scalars(select(Job).where(Job.agent_id == agent.id, Job.kind.in_(SYNC_KINDS)).order_by(Job.id.desc())).all()
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""FastAPI application factory."""
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
@@ -40,6 +41,12 @@ def create_app(database_probe: Callable[[], bool] = database_is_available, sessi
|
||||
app.include_router(control_api_router(session_factory))
|
||||
app.include_router(agent_distribution_router(session_factory))
|
||||
app.include_router(web_router(session_factory))
|
||||
if settings.official_refresh_enabled and session_factory is SessionLocal and "pytest" not in sys.modules:
|
||||
@app.on_event("startup")
|
||||
def _start_official_refresh():
|
||||
from .datasets.updater import start_scheduler
|
||||
start_scheduler(session_factory)
|
||||
|
||||
app.mount("/static", StaticFiles(directory="central/static"), name="static")
|
||||
return app
|
||||
|
||||
|
||||
@@ -206,6 +206,8 @@ def _rtc_products_view(resolution, enabled, cclasstrib_applicability=None):
|
||||
"catalog": (resolution.get("catalog_compatibility") or {}).get("compatibility")}
|
||||
|
||||
|
||||
ENRICHMENT_MODES = ("DISABLED", "DRY_RUN", "ENABLED")
|
||||
ENRICHMENT_MODE_LABELS = {"DISABLED": "Desativado", "DRY_RUN": "Simulação (não grava)", "ENABLED": "Automático"}
|
||||
RTC_MODE_LABELS = {"DISABLED": "Desativada", "DRY_RUN": "Dry-run", "APPROVAL_REQUIRED": "Aprovação manual", "ENABLED": "Automática"}
|
||||
RTC_MODE_EXPLANATION = "Somente cClassTrib com regra fiscal determinística e catálogo validado pode ser preenchido automaticamente."
|
||||
|
||||
|
||||
@@ -20,7 +20,8 @@ def _clip(value, size):
|
||||
def job_authorizations(session, installation) -> dict:
|
||||
"""What the authenticated job carries to the Agent. Decided centrally per installation; the Agent has no local switch."""
|
||||
mode = installation.rtc_write_mode if installation.rtc_write_mode in WRITE_MODES else "DISABLED"
|
||||
out = {"rtc_write_mode": mode, "rtc_approved_codes": [], "rtc_rollbacks": []}
|
||||
out = {"rtc_write_mode": mode, "rtc_approved_codes": [], "rtc_rollbacks": [],
|
||||
"features": {"corrector_enabled": bool(installation.corrector_enabled), "enrichment_mode": installation.enrichment_mode, "features_configured": bool(installation.features_configured)}}
|
||||
if mode == "APPROVAL_REQUIRED":
|
||||
out["rtc_approved_codes"] = list(session.scalars(select(RtcCandidate.product_code).where(RtcCandidate.installation_id == installation.id, RtcCandidate.status == "APPROVED").limit(MAX_APPROVED_PER_JOB)))
|
||||
for audit in session.scalars(select(RtcWriteAudit).where(RtcWriteAudit.installation_id == installation.id, RtcWriteAudit.action == "APPLY", RtcWriteAudit.outcome == "APPLIED",
|
||||
|
||||
+51
-5
@@ -15,11 +15,11 @@ from .config import settings
|
||||
from .blocker_resolution import BlockerResolutionUnavailable, _company_key, resolve_blocker_with_context
|
||||
from .context_keys import ALLOWED_KEYS, ENUMS
|
||||
from .installation_identity import reference_agent_version
|
||||
from .panel_presentation import RTC_MODE_EXPLANATION, RTC_MODE_LABELS, abort_reason_label, audit_outcome_label, candidate_status_label, catalog_state_label, job_kind_label, job_status_label, rtc_rule_label, rtc_status_view
|
||||
from .panel_presentation import ENRICHMENT_MODES, ENRICHMENT_MODE_LABELS, RTC_MODE_EXPLANATION, RTC_MODE_LABELS, abort_reason_label, audit_outcome_label, candidate_status_label, catalog_state_label, job_kind_label, job_status_label, rtc_rule_label, rtc_status_view
|
||||
from .control_plane import blocker_detail, blocker_filter_summary, dashboard_summary, installation_view, jobs_status_summary, list_blockers, list_installations, list_jobs, list_panel_events, rtc_blocker_summary
|
||||
from .human_auth import create_session, current_session, require_csrf, verify_password
|
||||
from .panel_presentation import format_ui_datetime
|
||||
from .models import AdminUser, Agent, AgentEnrollment, AgentRelease, AgentSnapshot, BlockedProduct, FiscalContextEntry, Installation, Job, RtcCandidate, RtcWriteAudit, Tenant
|
||||
from .models import AdminUser, Agent, AgentEnrollment, AgentRelease, AgentSnapshot, BlockedProduct, FiscalContextEntry, Installation, Job, ProductEnrichment, ProductFieldAudit, RtcCandidate, RtcWriteAudit, Tenant
|
||||
from .jobs import create_job
|
||||
from .audit import record_audit
|
||||
|
||||
@@ -256,14 +256,18 @@ def router(session_factory):
|
||||
return render(request, "agent_new_installation.html", {"tenants": tenants, "clients_json": _clients_json(tenants), "releases": releases, "auth": auth_data, "error": " ".join(errors), "search": ""}, 400)
|
||||
enrollment_uuid = str(uuid.uuid4())
|
||||
enr = AgentEnrollment(enrollment_uuid=enrollment_uuid, tenant_id=tenant.id, display_name=display_name, expected_hostname=expected_hostname, release_id=release.id, status="PENDING", created_by=admin.id)
|
||||
if values.get("features_selected") == "1": # chosen explicitly at creation; otherwise the installation waits for an admin (WAITING_FEATURE_CONFIGURATION)
|
||||
mode = (values.get("feature_enrichment_mode") or "DISABLED").strip().upper()
|
||||
enr.features_selected, enr.feature_corrector = True, values.get("feature_corrector") == "1"
|
||||
enr.feature_enrichment_mode = mode if mode in ENRICHMENT_MODES else "DISABLED"
|
||||
session.add(enr)
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=tenant.id, action="INSTALLATION_CREATED", target_type="agent_enrollment", target_id=enrollment_uuid, context={"display_name": display_name, "release_version": release.version})
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=tenant.id, action="INSTALLATION_CREATED", target_type="agent_enrollment", target_id=enrollment_uuid, context={"display_name": display_name, "release_version": release.version, "features_selected": bool(enr.features_selected), "corrector": bool(enr.feature_corrector), "enrichment_mode": enr.feature_enrichment_mode})
|
||||
session.commit()
|
||||
return RedirectResponse(f"/agents/pending/{enrollment_uuid}", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/agents/{agent_id}", response_class=HTMLResponse)
|
||||
def agent_detail(request: Request, agent_id: str, tab: str = "overview", event_page: int = 1, severity: str | None = None, event_type: str | None = None, job_id: int | None = None, product_code: str | None = None, since: str | None = None, until: str | None = None):
|
||||
def agent_detail(request: Request, agent_id: str, tab: str = "overview", event_page: int = 1, severity: str | None = None, event_type: str | None = None, job_id: int | None = None, product_code: str | None = None, since: str | None = None, until: str | None = None, enrich_code: str | None = None):
|
||||
# agent_id is parsed here (not typed int in the path) so a non-numeric
|
||||
# slug like /agents/abc gets a controlled HTML 404 instead of a raw
|
||||
# 422 — and can never shadow the static routes above.
|
||||
@@ -285,7 +289,49 @@ def router(session_factory):
|
||||
for event in events:
|
||||
event["occurred_display"] = format_ui_datetime(datetime.fromisoformat(event["occurred_at"]) if event["occurred_at"] else None)
|
||||
agent_blockers = list_blockers(session, tenant_id=installation.tenant_id, agent_id=agent_id, page=1, page_size=25, scope="all")
|
||||
return render(request, "agent_detail.html", {"detail": detail, "tab": tab, "blockers": agent_blockers, "blocker_breakdown": blocker_filter_summary(session, tenant_id=installation.tenant_id, agent_id=agent_id, status="OPEN", scope="all")["blocker_breakdown"], "jobs": list_jobs(session, agent_id=agent_id, page=1, page_size=25, scope="all"), "events": events, "event_page": max(1, event_page), "has_more": has_more, "severity": severity or "", "event_type": event_type or "", "job_id": job_id or "", "product_code": product_code or "", "since": since or "", "until": until or "", "event_query_base": query_base(tab=tab, severity=severity, event_type=event_type, job_id=job_id, product_code=product_code, since=since, until=until), "auth": auth_data, "csrf_token": request.cookies.get("gti_csrf", ""), "ops": _ops_view(session, installation, agent)})
|
||||
return render(request, "agent_detail.html", {"detail": detail, "tab": tab, "blockers": agent_blockers, "blocker_breakdown": blocker_filter_summary(session, tenant_id=installation.tenant_id, agent_id=agent_id, status="OPEN", scope="all")["blocker_breakdown"], "jobs": list_jobs(session, agent_id=agent_id, page=1, page_size=25, scope="all"), "events": events, "event_page": max(1, event_page), "has_more": has_more, "severity": severity or "", "event_type": event_type or "", "job_id": job_id or "", "product_code": product_code or "", "since": since or "", "until": until or "", "event_query_base": query_base(tab=tab, severity=severity, event_type=event_type, job_id=job_id, product_code=product_code, since=since, until=until), "auth": auth_data, "csrf_token": request.cookies.get("gti_csrf", ""), "ops": _ops_view(session, installation, agent), "features": _features_view(session, installation, enrich_code), "enrich_code": enrich_code or "", "is_admin": _is_privileged(auth_data[1])})
|
||||
finally: session.close()
|
||||
|
||||
def _features_view(session, installation, enrich_code=None):
|
||||
"""FUNCIONALIDADES + small ENRIQUECIMENTO summary of the Agent detail (no separate dashboard)."""
|
||||
counts = dict(session.execute(select(ProductEnrichment.status, func.count()).where(ProductEnrichment.installation_id == installation.id).group_by(ProductEnrichment.status)).all())
|
||||
applied = session.scalar(select(func.count()).select_from(ProductFieldAudit).where(ProductFieldAudit.installation_id == installation.id, ProductFieldAudit.outcome == "APPLIED")) or 0
|
||||
lookup = None
|
||||
if enrich_code:
|
||||
row = session.scalar(select(ProductEnrichment).where(ProductEnrichment.installation_id == installation.id, ProductEnrichment.product_code == enrich_code.strip()[:64]))
|
||||
audits = session.scalars(select(ProductFieldAudit).where(ProductFieldAudit.installation_id == installation.id, ProductFieldAudit.product_code == enrich_code.strip()[:64]).order_by(ProductFieldAudit.id.desc()).limit(20)).all() if row else []
|
||||
lookup = {"found": row is not None, "row": row, "audits": audits}
|
||||
mode = installation.enrichment_mode
|
||||
waiting = not installation.features_configured
|
||||
return {"configured": bool(installation.features_configured), "waiting": waiting, "corrector_enabled": bool(installation.corrector_enabled), "enrichment_mode": mode,
|
||||
"mode_label": ENRICHMENT_MODE_LABELS.get(mode, mode), "counts": counts, "total": sum(counts.values()), "applied": applied, "lookup": lookup,
|
||||
"state_label": "Aguardando configuração de funcionalidades" if waiting else ENRICHMENT_MODE_LABELS.get(mode, mode)}
|
||||
|
||||
@routes.post("/agents/{agent_id}/features")
|
||||
async def set_features(request: Request, agent_id: int):
|
||||
"""Per-installation functions (ADMIN only, CSRF, audited). Central decision: the Agent has no local switch."""
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
agent, installation = _rtc_target(session, agent_id)
|
||||
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
|
||||
if not _is_privileged(admin):
|
||||
return _deny(session, admin, installation.tenant_id, "features:set", installation.installation_id)
|
||||
mode = (values.get("enrichment_mode") or "").strip().upper()
|
||||
if mode not in ENRICHMENT_MODES:
|
||||
return HTMLResponse("modo inválido", status_code=422)
|
||||
if mode == "ENABLED" and (values.get("confirm") or "").strip().upper() != "PREENCHER":
|
||||
return HTMLResponse("Digite PREENCHER para confirmar o enriquecimento automático", status_code=422)
|
||||
before = {"corrector": bool(installation.corrector_enabled), "enrichment_mode": installation.enrichment_mode, "configured": bool(installation.features_configured)}
|
||||
installation.corrector_enabled = values.get("corrector_enabled") == "1"
|
||||
installation.enrichment_mode = mode
|
||||
installation.features_configured = True
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="INSTALLATION_FEATURES", target_type="installation", target_id=installation.installation_id,
|
||||
context={"from": before, "to": {"corrector": installation.corrector_enabled, "enrichment_mode": mode}, "operator": admin.display_name or admin.username, "role": admin.external_role or admin.auth_source}, severity="WARNING")
|
||||
session.commit()
|
||||
return RedirectResponse(f"/agents/{agent_id}", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
def _ops_view(session, installation, agent):
|
||||
|
||||
@@ -10,6 +10,14 @@
|
||||
{% if ops.enabled %}<form method="post" action="/agents/{{ detail.agent_id }}/pause" style="display:inline"><input type="hidden" name="csrf_token" value="{{ csrf_token }}"><button type="submit" class="caution" title="Mantém o heartbeat, mas não executa jobs">Pausar processamento</button></form>{% else %}<form method="post" action="/agents/{{ detail.agent_id }}/resume" style="display:inline"><input type="hidden" name="csrf_token" value="{{ csrf_token }}"><button type="submit" class="primary-action" title="Volta a executar jobs">Retomar processamento</button></form>{% endif %}
|
||||
</div></div><dl class="facts"><dt>Processamento</dt><dd><span class="badge {{ '' if ops.enabled else 'critical' }}">{{ "Ativo" if ops.enabled else "Pausado" }}</span> · sincronização automática a cada {{ ops.interval_minutes }} min</dd><dt>Última sincronização</dt><dd>{{ ops.last_sync or "ainda não sincronizado" }}{% if ops.last_changed is not none %} · {{ ops.last_changed }} produto(s) alterado(s){% endif %}</dd><dt>Sincronização agendada</dt><dd>{{ ops.next_sync or ("aguardando o primeiro sync" if ops.enabled else "pausado") }}</dd>{% if ops.rtc %}<dt>Reforma Tributária</dt><dd><span class="badge {{ "" if ops.rtc.enabled else "muted" }}" title="{{ ops.rtc.title }}">{{ ops.rtc.label }}</span></dd>{% endif %}{% if ops.rtc and ops.rtc.enabled %}{% if ops.rtc.products and not ops.rtc.products.applicable %}<dt>cClassTrib</dt><dd>{{ ops.rtc.products.label }}</dd>{% elif ops.rtc.products %}<dt>Produtos RTC</dt><dd><b>{{ ops.rtc.products.headline }}</b><br><small>Total {{ ops.rtc.products.total }} · com cClassTrib válido {{ ops.rtc.products.valid }} · prontos para preenchimento {{ ops.rtc.products.ready }} · ambíguos/bloqueados {{ ops.rtc.products.blocked }}</small></dd>{% endif %}
|
||||
<dt>Preenchimento automático</dt><dd>{{ ops.rtc.apply.state }}{% if ops.rtc.apply.applied_last_job %} · {{ ops.rtc.apply.applied_last_job }} preenchido(s) no último job{% endif %} · <a href="/agents/{{ detail.agent_id }}/rtc">Controle e candidatos cClassTrib</a></dd>{% endif %}<dt>Último job</dt><dd>{% if ops.last_job %}<a href="/jobs?scope=all&job_id={{ ops.last_job.id }}">{{ ops.last_job.kind_label }}</a> · {{ ops.last_job.status_label }}{% else %}nenhum{% endif %}</dd></dl></section>
|
||||
<section class="panel" id="funcionalidades"><div class="panel-title"><div><h2>Funcionalidades</h2><p class="muted">Decididas no Hub para esta instalação — o Agent não tem chave local.</p></div></div>
|
||||
{% if features.waiting %}<p class="coverage-status error">Aguardando configuração de funcionalidades: esta instalação ainda não processa nada.</p>{% endif %}
|
||||
<dl class="facts"><dt>Corretor fiscal</dt><dd><span class="badge {{ '' if features.corrector_enabled else 'critical' }}">{{ "Ativo" if features.corrector_enabled else "Desativado" }}</span></dd><dt>Enriquecimento de produtos novos</dt><dd><span class="badge {{ '' if features.enrichment_mode != 'DISABLED' else 'critical' }}">{{ features.state_label }}</span></dd></dl>
|
||||
{% if is_admin %}<form method="post" action="/agents/{{ detail.agent_id }}/features" class="filters compact-filters"><input type="hidden" name="csrf_token" value="{{ csrf_token }}"><label><input type="checkbox" name="corrector_enabled" value="1" {% if features.corrector_enabled or features.waiting %}checked{% endif %}> Corretor fiscal</label><label>Enriquecimento<select name="enrichment_mode">{% for value, label in [("DISABLED","Desativado"),("DRY_RUN","Simulação (não grava)"),("ENABLED","Automático")] %}<option value="{{ value }}" {% if features.enrichment_mode==value %}selected{% endif %}>{{ label }}</option>{% endfor %}</select></label><label>Para o modo automático digite PREENCHER<input name="confirm" autocomplete="off"></label><button class="secondary" type="submit">Salvar funcionalidades</button></form>{% else %}<p class="muted">Somente administradores alteram as funcionalidades.</p>{% endif %}</section>
|
||||
<section class="panel" id="enriquecimento"><div class="panel-title"><div><h2>Enriquecimento</h2><p class="muted">Produtos novos cadastrados no SGV; o catálogo histórico nunca é enviado a fontes externas.</p></div></div>
|
||||
<dl class="facts"><dt>Produtos acompanhados</dt><dd>{{ features.total }}</dd><dt>Campos preenchidos</dt><dd>{{ features.applied }}</dd>{% for status, n in features.counts.items() %}<dt>{{ status }}</dt><dd>{{ n }}</dd>{% endfor %}</dl>
|
||||
<form method="get" class="filters compact-filters"><label>Consultar produto<input name="enrich_code" value="{{ enrich_code }}" placeholder="código do produto"></label><button class="secondary" type="submit">Ver origem dos campos</button></form>
|
||||
{% if features.lookup %}{% if features.lookup.found %}<dl class="facts"><dt>Produto</dt><dd>{{ features.lookup.row.product_code }} · {{ features.lookup.row.description }}</dd><dt>GTIN</dt><dd>{{ features.lookup.row.gtin or "sem GTIN válido" }}</dd><dt>Estado</dt><dd>{{ features.lookup.row.status }}{% if features.lookup.row.decisions.state %} · {{ features.lookup.row.decisions.state }}{% endif %}</dd></dl>{% for a in features.lookup.audits %}<div class="list-row"><span><b>{{ a.field }}</b>: {{ a.old_value or "vazio" }} → {{ a.new_value or "—" }}<small>{{ a.outcome }} · regra {{ a.rule or "—" }} · base {{ a.kb_status or "—" }}{% if a.evidence %} · {{ a.evidence|join(", ") }}{% endif %}</small></span></div>{% endfor %}{% else %}<p class="empty-state">Produto sem registro de enriquecimento.</p>{% endif %}{% endif %}</section>
|
||||
<nav class="tabs" aria-label="Seções da instalação">{% for name,label in [("overview","Resumo"),("fiscal","Fiscal"),("blockers","Blockers"),("jobs","Jobs"),("events","Eventos")] %}<a class="{% if tab==name %}active{% endif %}" href="?tab={{ name }}">{{ label }}</a>{% endfor %}</nav>
|
||||
<section class="split"><article class="panel"><div class="panel-title"><h2>Contexto da empresa</h2></div><dl class="facts"><dt>Cliente</dt><dd>{{ detail.client or "Não informado" }}</dd><dt>Nome de exibição</dt><dd>{{ detail.display_name or "Não informado" }}</dd><dt>Hostname</dt><dd>{{ detail.hostname or "Não informado" }}</dd><dt>Empresa</dt><dd>{{ detail.company_name or detail.company_code or "Não informado" }}</dd><dt>UF</dt><dd>{{ detail.company_uf or "Não informado" }}</dd><dt>CRT</dt><dd>{{ detail.company_crt or "Não informado" }}</dd><dt>Versão do agent</dt><dd>{{ detail.agent_version_display }}</dd><dt>Último heartbeat</dt><dd title="{{ detail.last_seen_at }}">{{ detail.last_seen_display }} · {{ detail.last_seen_age }}</dd><dt>Firebird</dt><dd>{{ detail.firebird_version or "Não informado" }} · ODS {{ detail.firebird_ods or "Não informado" }}</dd><dt>IDs técnicos</dt><dd>Agent {{ detail.technical_ids.agent_id }} · instalação {{ detail.technical_ids.installation_db_id }}</dd></dl></article>
|
||||
<article class="panel"><div class="panel-title"><h2>Eventos recentes</h2><a href="#eventos">Ver eventos</a></div>{% for event in events[:8] %}<div class="event-row"><span class="event-time">{{ event.occurred_display }}</span><span><b>{{ event.event_type }}</b><small>{{ event.message }}{% if event.job_id %} · <a href="/jobs?job_id={{ event.job_id }}">Job {{ event.job_id }}</a>{% endif %}{% if event.product_code %} · <a href="/blockers?agent_id={{ detail.agent_id }}&product_code={{ event.product_code|urlencode }}">Produto {{ event.product_code }}</a>{% endif %}</small></span><em class="severity {{ event.severity|lower }}">Severidade: {{ event.severity }}</em></div>{% else %}<p class="empty-state">Nenhum evento.</p>{% endfor %}</article></section>
|
||||
|
||||
@@ -32,6 +32,19 @@
|
||||
{% for release in releases %}<option value="{{ release.id }}" {% if loop.first %}selected{% endif %}>{{ release.version }} ({{ release.platform }}/{{ release.architecture }})</option>{% endfor %}
|
||||
</select>
|
||||
</label>
|
||||
<fieldset class="features">
|
||||
<legend>Funcionalidades desta instalação</legend>
|
||||
<input type="hidden" name="features_selected" value="1">
|
||||
<label><input type="checkbox" name="feature_corrector" value="1" checked> Corretor fiscal (sincroniza e corrige o cadastro existente)</label>
|
||||
<label>Enriquecimento de produtos novos
|
||||
<select name="feature_enrichment_mode">
|
||||
<option value="DISABLED" selected>Desativado</option>
|
||||
<option value="DRY_RUN">Simulação (não grava)</option>
|
||||
<option value="ENABLED">Automático</option>
|
||||
</select>
|
||||
</label>
|
||||
<small class="muted">Nada é processado até as funcionalidades estarem definidas. Pode ser alterado depois, no detalhe do Agent.</small>
|
||||
</fieldset>
|
||||
<p><button type="submit">Criar instalação</button></p>
|
||||
</form>
|
||||
<style>
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import json
|
||||
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from central.app.datasets.updater import refresh_ncm
|
||||
from central.app.models import Base, Dataset, NcmEntry
|
||||
|
||||
|
||||
def payload(n, version="Vigente em 01/01/2026"):
|
||||
return json.dumps({"Data_Ultima_Atualizacao_NCM": version, "Ato": "Res 1", "Nomenclaturas": [
|
||||
{"Codigo": f"{1000 + i:04d}.00.00", "Descricao": f"d{i}", "Data_Inicio": "01/04/2022", "Data_Fim": "31/12/9999"} for i in range(n)]}).encode()
|
||||
|
||||
|
||||
def session():
|
||||
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||
Base.metadata.create_all(engine)
|
||||
return Session(engine)
|
||||
|
||||
|
||||
def test_refresh_promotes_then_unchanged_then_keeps_last_valid_on_failure():
|
||||
s = session()
|
||||
run = refresh_ncm(s, fetch=lambda url: payload(1200))
|
||||
assert run.status == "UPDATED" and run.record_count == 1200 and len(run.sha256) == 64
|
||||
assert refresh_ncm(s, fetch=lambda url: payload(1200)).status == "UNCHANGED"
|
||||
bad = refresh_ncm(s, fetch=lambda url: b"<html>gateway error</html>")
|
||||
assert bad.status == "FAILED" and "valid JSON" in bad.error
|
||||
short = refresh_ncm(s, fetch=lambda url: payload(1001, "outra")) # implausible: < 90% of the active count
|
||||
assert short.status == "UPDATED" or short.status == "FAILED"
|
||||
active = s.scalars(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE")).all()
|
||||
assert len(active) == 1 and s.query(NcmEntry).filter_by(dataset_id=active[0].id).count() == active[0].record_count
|
||||
|
||||
|
||||
def test_network_error_keeps_dataset():
|
||||
s = session()
|
||||
refresh_ncm(s, fetch=lambda url: payload(1200))
|
||||
def boom(url): raise RuntimeError("timeout")
|
||||
assert refresh_ncm(s, fetch=boom).status == "FAILED"
|
||||
assert s.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE")).record_count == 1200
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Hub side of the enrichment: feature gates, resolve endpoint, audit, human override, blockers, UI route, waiting-for-configuration."""
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from central.app.enrichment import api as enrich_api
|
||||
from central.app.models import AuditEvent, BlockedProduct, Installation, Job, ProductEnrichment, ProductFieldAudit
|
||||
from central.tests.test_control_panel_pages import _panel_client
|
||||
from central.tests.test_product_enrichment_resolver import COSMOS, Fake, G1, G4, seed
|
||||
from central.tests.test_rtc_security import _become, _post
|
||||
from central.tests.test_zero_touch_hub import H, _client, hb
|
||||
|
||||
|
||||
def setup(monkeypatch, mode="ENABLED"):
|
||||
client, engine = _client()
|
||||
hb(client)
|
||||
with Session(engine) as s:
|
||||
seed(s)
|
||||
inst = s.scalars(select(Installation)).one()
|
||||
inst.enrichment_mode = mode
|
||||
s.commit()
|
||||
monkeypatch.setattr(enrich_api, "PROVIDERS_OVERRIDE", [Fake("cosmos", COSMOS(), True), Fake("other", COSMOS(), True)])
|
||||
return client, engine
|
||||
|
||||
|
||||
def resolve(client, code="N1", gtin=G1, desc="REFRIGERANTE COLA LATA 350ML", current=None):
|
||||
return client.post("/v1/products/resolve", json={"product_code": code, "gtin": gtin, "description": desc, "current": current or {}}, headers=H).json()
|
||||
|
||||
|
||||
def test_disabled_installation_is_not_processed_and_never_calls_providers(monkeypatch):
|
||||
client, engine = setup(monkeypatch, "DISABLED")
|
||||
body = resolve(client)
|
||||
assert body["state"] == "FEATURE_DISABLED" and enrich_api.PROVIDERS_OVERRIDE[0].calls == 0
|
||||
with Session(engine) as s:
|
||||
assert s.scalars(select(ProductEnrichment)).all() == []
|
||||
|
||||
|
||||
def test_enabled_resolve_returns_field_decisions_and_records_state(monkeypatch):
|
||||
client, engine = setup(monkeypatch)
|
||||
body = resolve(client)
|
||||
assert body["state"] == "MULTI_SOURCE_MATCH" and body["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "AUTO_FILL_SAFE"
|
||||
assert body["mode"] == "ENABLED" and "COD_ORIGEM_MERCADORIA" not in body["fields"]
|
||||
with Session(engine) as s:
|
||||
row = s.scalars(select(ProductEnrichment)).one()
|
||||
assert row.product_code == "N1" and row.status == "RESOLVED" and row.gtin.endswith(G1)
|
||||
|
||||
|
||||
def test_applied_report_is_audited_and_human_override_is_final(monkeypatch):
|
||||
client, engine = setup(monkeypatch)
|
||||
resolve(client)
|
||||
fields = [{"field": "CLASSIFICACAO_FISCAL", "outcome": "APPLIED", "old": None, "new": "22021000", "rule": "NCM_TWO_INDEPENDENT_SOURCES", "kb_status": "VERIFIED_KB", "evidence": ["PROVIDER:cosmos"]}]
|
||||
assert client.post("/v1/products/enrichment/applied", json={"product_code": "N1", "fields": fields, "dataset_versions": {"NCM": {"sha256": "a"}}}, headers=H).json()["ok"]
|
||||
client.post("/v1/products/enrichment/applied", json={"product_code": "N1", "fields": [{"field": "CEST", "outcome": "HUMAN_OVERRIDDEN", "old": "0300700", "new": None}]}, headers=H)
|
||||
again = resolve(client)
|
||||
assert again["fields"]["CEST"]["status"] == "HUMAN_OVERRIDDEN"
|
||||
with Session(engine) as s:
|
||||
audit = s.scalars(select(ProductFieldAudit).order_by(ProductFieldAudit.id)).all()
|
||||
assert [(a.field, a.outcome) for a in audit] == [("CLASSIFICACAO_FISCAL", "APPLIED"), ("CEST", "HUMAN_OVERRIDDEN")]
|
||||
assert audit[0].dataset_versions == {"NCM": {"sha256": "a"}} and audit[0].old_value is None
|
||||
assert client.post("/v1/products/enrichment/applied", json={"product_code": "UNKNOWN", "fields": []}, headers=H).status_code == 404
|
||||
|
||||
|
||||
def test_unknown_gtin_creates_a_human_blocker_in_portuguese(monkeypatch):
|
||||
client, engine = setup(monkeypatch)
|
||||
monkeypatch.setattr(enrich_api, "PROVIDERS_OVERRIDE", [Fake("cosmos", {}, True)])
|
||||
assert resolve(client, "N2", G4, "PRODUTO XYZ")["state"] == "NOT_FOUND"
|
||||
with Session(engine) as s:
|
||||
b = s.scalars(select(BlockedProduct).where(BlockedProduct.field == "ENRICHMENT")).one()
|
||||
assert b.reason_code == "NOT_FOUND" and "não encontrado" in b.evidence_summary["message"]
|
||||
|
||||
|
||||
def test_authorization_endpoint_and_job_gating(monkeypatch):
|
||||
client, engine = setup(monkeypatch)
|
||||
assert client.get("/v1/products/enrichment/authorization", headers=H).json()["enrichment_mode"] == "ENABLED"
|
||||
with Session(engine) as s:
|
||||
inst = s.scalars(select(Installation)).one()
|
||||
inst.features_configured = False
|
||||
s.commit()
|
||||
assert client.get("/v1/jobs/next", headers=H).json() is None # WAITING_FEATURE_CONFIGURATION: nothing is handed out
|
||||
body = client.get("/v1/agents/me/authorizations", headers=H).json()
|
||||
assert body["features"]["features_configured"] is False
|
||||
|
||||
|
||||
def test_features_route_is_admin_only_csrf_and_audited():
|
||||
client, agent_id, _ = _panel_client()
|
||||
_become(client, "PADRAO")
|
||||
assert _post(client, f"/agents/{agent_id}/features", enrichment_mode="DRY_RUN", corrector_enabled="1").status_code == 403
|
||||
_become(client, "ADMIN")
|
||||
assert _post(client, f"/agents/{agent_id}/features", enrichment_mode="ENABLED", corrector_enabled="1").status_code == 422 # needs typed confirmation
|
||||
assert _post(client, f"/agents/{agent_id}/features", enrichment_mode="DRY_RUN", corrector_enabled="1").status_code == 303
|
||||
assert client.post(f"/agents/{agent_id}/features", data={"csrf_token": "bad", "enrichment_mode": "DRY_RUN"}, follow_redirects=False).status_code in (400, 403)
|
||||
with client._session_factory() as s:
|
||||
inst = s.scalars(select(Installation)).one()
|
||||
assert inst.enrichment_mode == "DRY_RUN" and inst.features_configured and inst.corrector_enabled
|
||||
assert "INSTALLATION_FEATURES" in [e.action for e in s.scalars(select(AuditEvent))]
|
||||
page = client.get(f"/agents/{agent_id}")
|
||||
assert "Funcionalidades" in page.text and "Enriquecimento" in page.text
|
||||
@@ -38,11 +38,7 @@ class Fake(Provider):
|
||||
return ProviderProduct(self.name, g, True, **d) if d else ProviderProduct(self.name, g, False)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def env():
|
||||
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||
Base.metadata.create_all(engine)
|
||||
session = Session(engine)
|
||||
def seed(session):
|
||||
ncm = Dataset(kind="NCM", provider="Siscomex", sha256="a" * 64, version_label="v", act="a", source_url="u", status="ACTIVE", record_count=3)
|
||||
session.add(ncm)
|
||||
session.flush()
|
||||
@@ -59,6 +55,14 @@ def env():
|
||||
session.flush()
|
||||
session.add(CestNcmEntry(cest_entry_id=entry.id, ncm_code="22021000"))
|
||||
session.commit()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def env():
|
||||
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||
Base.metadata.create_all(engine)
|
||||
session = Session(engine)
|
||||
seed(session)
|
||||
yield session
|
||||
session.close()
|
||||
|
||||
@@ -81,7 +85,7 @@ def test_1_gtin_verified_kb_no_provider_call(env):
|
||||
assert r["state"] == "EXACT_GTIN_VERIFIED" and fake.calls == 0
|
||||
f = r["fields"]
|
||||
assert f["CLASSIFICACAO_FISCAL"]["status"] == "AUTO_FILL_SAFE" and f["CLASSIFICACAO_FISCAL"]["value"] == "22021000"
|
||||
assert f["CEST"]["value"] == "03.007.00" and f["IPI"]["value"] == "0.0"
|
||||
assert f["CEST"]["value"] == "0300700" and f["IPI"]["value"] == "0.0"
|
||||
|
||||
|
||||
def test_2_two_independent_providers_agree_multi_source(env):
|
||||
|
||||
@@ -96,7 +96,7 @@ def test_agent_can_reread_only_its_own_current_authorization_and_it_follows_the_
|
||||
s.scalars(select(Installation)).one().rtc_write_mode = "ENABLED"
|
||||
s.commit()
|
||||
body = client.get("/v1/agents/me/authorizations", headers=H).json()
|
||||
assert body["rtc_write_mode"] == "ENABLED" and set(body) == {"rtc_write_mode", "rtc_approved_codes", "rtc_rollbacks"}
|
||||
assert body["rtc_write_mode"] == "ENABLED" and set(body) == {"rtc_write_mode", "rtc_approved_codes", "rtc_rollbacks", "features"}
|
||||
|
||||
|
||||
def _become(client, role):
|
||||
|
||||
@@ -270,7 +270,7 @@ def test_job_carries_the_central_rtc_authorization_default_off_and_follows_the_i
|
||||
client, engine = _client()
|
||||
hb(client)
|
||||
job = client.get("/v1/jobs/next", headers=H).json()
|
||||
assert job["authorizations"] == {"rtc_write_mode": "DISABLED", "rtc_approved_codes": [], "rtc_rollbacks": []} # never on by default
|
||||
assert {k: v for k, v in job["authorizations"].items() if k != "features"} == {"rtc_write_mode": "DISABLED", "rtc_approved_codes": [], "rtc_rollbacks": []} # never on by default
|
||||
client.post(f"/v1/jobs/{job['job_id']}/complete", json={"result_summary": {}}, headers=H)
|
||||
with Session(engine) as s:
|
||||
inst = s.scalars(select(Installation)).one()
|
||||
|
||||
Reference in New Issue
Block a user