Add 'fiscal/' from commit '102ba84500dbd9655a0471288c3e278652593ca9'
git-subtree-dir: fiscal git-subtree-mainline:d95165fc8fgit-subtree-split:102ba84500
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""Fiscal Hub application modules."""
|
||||
@@ -0,0 +1,256 @@
|
||||
"""Public, pre-authentication routes for Agent distribution.
|
||||
|
||||
GET /dl/agent/{token} and POST /v1/agents/enroll are reachable by anyone —
|
||||
same trust level as GET /login. Every other Agent-distribution route
|
||||
(create pending installation, generate a link, direct authenticated
|
||||
download, list/revoke releases) is a control-panel page and lives in
|
||||
web.py, gated the same way as the rest of the panel (admin session + CSRF).
|
||||
|
||||
The enrollment token embedded in a downloaded bundle is generated fresh at
|
||||
download time and its plaintext is never written to storage — only its
|
||||
SHA-256 hash persists, exactly like the download token that gated getting
|
||||
it. Neither is ever the Agent's own permanent Bearer token, which is
|
||||
minted only once, inside a successful POST /v1/agents/enroll.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.responses import Response
|
||||
from sqlalchemy import select
|
||||
|
||||
from . import config, enrollment, release_storage
|
||||
from .audit import record_audit
|
||||
from .auth import token_hash as agent_token_hash
|
||||
from .models import Agent, AgentEnrollment, AgentRelease, Installation
|
||||
from .rate_limit import blocked, record_attempt
|
||||
|
||||
|
||||
def client_ip(request: Request) -> str:
|
||||
return request.client.host if request.client else "unknown"
|
||||
|
||||
|
||||
def _aware(value: datetime | None) -> datetime | None:
|
||||
"""SQLite (used by the test suite) drops tzinfo on round-trip even for a
|
||||
DateTime(timezone=True) column; Postgres never does. Normalize either way
|
||||
before comparing against datetime.now(timezone.utc) — same pattern as
|
||||
human_auth.py's session-expiry check."""
|
||||
if value is None or value.tzinfo is not None:
|
||||
return value
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def build_and_serve(session, enr: AgentEnrollment, request: Request, *, actor_admin_id: int | None = None) -> Response:
|
||||
"""Serve the release artifact for the installation — always the same
|
||||
universal release binary (ZIP bundle for legacy token flow, bare EXE for
|
||||
the installer flow). Audit the download; never embed the enrollment token
|
||||
in an EXE download (the operator types a standalone install code instead)."""
|
||||
release = session.get(AgentRelease, enr.release_id)
|
||||
if release is None or release.status != "PUBLISHED":
|
||||
raise enrollment.EnrollmentRejected("RELEASE_UNAVAILABLE")
|
||||
path = release_storage.safe_path_for(release.storage_path)
|
||||
if not path.exists():
|
||||
raise enrollment.EnrollmentRejected("RELEASE_FILE_MISSING")
|
||||
now = datetime.now(timezone.utc)
|
||||
is_exe = release.filename.lower().endswith(".exe")
|
||||
if is_exe:
|
||||
# Universal EXE flow: no per-installation bundle, no enrollment.json.
|
||||
# The operator types the standalone install code into Setup.exe and the
|
||||
# Hub resolves the installation from the code hash alone.
|
||||
enr.downloaded_at = now
|
||||
if enr.status not in ("ENROLLED", "REVOKED"):
|
||||
enr.status = "DOWNLOADED"
|
||||
record_audit(session, admin_user_id=actor_admin_id, tenant_id=enr.tenant_id, action="INSTALLER_DOWNLOADED",
|
||||
target_type="agent_enrollment", target_id=enr.enrollment_uuid,
|
||||
context={"ip": client_ip(request), "release_version": release.version, "release_id": release.id})
|
||||
session.commit()
|
||||
filename = release.filename
|
||||
return Response(content=path.read_bytes(), media_type="application/octet-stream",
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"',
|
||||
"X-Content-Type-Options": "nosniff", "Cache-Control": "no-store"})
|
||||
token = enrollment.new_token()
|
||||
enr.enrollment_token_hash = enrollment.token_digest(token)
|
||||
enr.enrollment_token_expires_at = now + timedelta(seconds=config.settings.agent_enrollment_ttl_seconds)
|
||||
enr.enrollment_used_at = None
|
||||
enr.downloaded_at = now
|
||||
if enr.status not in ("ENROLLED", "REVOKED"):
|
||||
enr.status = "DOWNLOADED"
|
||||
plan = enrollment.BundlePlan(hub_url=config.settings.base_url, enrollment_uuid=enr.enrollment_uuid, enrollment_token=token)
|
||||
data = enrollment.build_bundle(path.read_bytes(), plan)
|
||||
record_audit(session, admin_user_id=actor_admin_id, tenant_id=enr.tenant_id, action="PACKAGE_DOWNLOADED",
|
||||
target_type="agent_enrollment", target_id=enr.enrollment_uuid,
|
||||
context={"ip": client_ip(request), "release_version": release.version, "release_id": release.id})
|
||||
session.commit()
|
||||
filename = f"GTIFiscalAgent-{release.version}-{enr.enrollment_uuid[:8]}.zip"
|
||||
return Response(content=data, media_type="application/octet-stream",
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"',
|
||||
"X-Content-Type-Options": "nosniff", "Cache-Control": "no-store"})
|
||||
|
||||
|
||||
def router(session_factory):
|
||||
routes = APIRouter()
|
||||
|
||||
@routes.get("/dl/agent/{token}")
|
||||
def download_by_link(token: str, request: Request):
|
||||
ip = client_ip(request)
|
||||
if blocked("download", ip):
|
||||
raise HTTPException(429, "too many attempts")
|
||||
with session_factory() as session:
|
||||
candidates = session.scalars(select(AgentEnrollment).where(AgentEnrollment.download_token_hash.is_not(None))).all()
|
||||
enr = next((c for c in candidates if enrollment.token_matches(token, c.download_token_hash)), None)
|
||||
now = datetime.now(timezone.utc)
|
||||
try:
|
||||
if enr is None:
|
||||
raise enrollment.EnrollmentRejected("INVALID_LINK")
|
||||
if enr.status == "REVOKED":
|
||||
raise enrollment.EnrollmentRejected("REVOKED")
|
||||
if not enr.download_token_expires_at or _aware(enr.download_token_expires_at) < now:
|
||||
raise enrollment.EnrollmentRejected("EXPIRED_LINK")
|
||||
if enr.download_uses >= enr.download_max_uses:
|
||||
raise enrollment.EnrollmentRejected("LINK_ALREADY_USED")
|
||||
enr.download_uses += 1
|
||||
if enr.download_uses >= enr.download_max_uses:
|
||||
enr.download_token_hash = None # the link's last remaining use is spent: it dies immediately
|
||||
return build_and_serve(session, enr, request)
|
||||
except enrollment.EnrollmentRejected as rejected:
|
||||
session.rollback()
|
||||
record_attempt("download", ip)
|
||||
record_audit(session, admin_user_id=None, tenant_id=enr.tenant_id if enr else None, action="DOWNLOAD_LINK_REJECTED",
|
||||
target_type="agent_enrollment", target_id=enr.enrollment_uuid if enr else "-",
|
||||
context={"ip": ip, "reason": rejected.reason}, severity="WARNING")
|
||||
session.commit()
|
||||
raise HTTPException(404, "invalid or expired link")
|
||||
|
||||
@routes.post("/v1/agents/enroll")
|
||||
async def enroll(request: Request):
|
||||
ip = client_ip(request)
|
||||
if blocked("enroll", ip):
|
||||
raise HTTPException(429, "too many attempts")
|
||||
try:
|
||||
payload = await request.json()
|
||||
except Exception:
|
||||
payload = {}
|
||||
installation_id = str(payload.get("installation_id") or "")
|
||||
supplied_token = payload.get("enrollment_token")
|
||||
install_code = payload.get("install_code")
|
||||
# Adjustment: install_code and enrollment_token are mutually exclusive —
|
||||
# the EXE flow sends a code only; sending both is a request error.
|
||||
if install_code and (supplied_token or payload.get("installation_id")):
|
||||
record_attempt("enroll", ip)
|
||||
with session_factory() as session:
|
||||
record_audit(session, admin_user_id=None, tenant_id=None, action="ENROLLMENT_REJECTED",
|
||||
target_type="agent_enrollment", target_id="-",
|
||||
context={"ip": ip, "reason": "MIXED_CREDENTIALS"}, severity="WARNING")
|
||||
session.commit()
|
||||
raise HTTPException(400, "MIXED_CREDENTIALS")
|
||||
with session_factory() as session:
|
||||
enr = None
|
||||
try:
|
||||
if install_code:
|
||||
enr = _resolve_by_install_code(session, ip, install_code)
|
||||
else:
|
||||
enr = _resolve_by_legacy_token(session, ip, installation_id, supplied_token)
|
||||
if enr is None:
|
||||
raise enrollment.EnrollmentRejected("INVALID_INSTALLATION")
|
||||
if enr.status == "REVOKED":
|
||||
raise enrollment.EnrollmentRejected("REVOKED")
|
||||
if enr.status == "ENROLLED":
|
||||
raise enrollment.EnrollmentRejected("ALREADY_ENROLLED")
|
||||
release = session.get(AgentRelease, enr.release_id)
|
||||
if release is None or release.status != "PUBLISHED":
|
||||
raise enrollment.EnrollmentRejected("RELEASE_UNAVAILABLE")
|
||||
|
||||
record_audit(session, admin_user_id=None, tenant_id=enr.tenant_id, action="ENROLLMENT_STARTED",
|
||||
target_type="agent_enrollment", target_id=enr.enrollment_uuid,
|
||||
context={"ip": ip, "credential_type": "install_code" if install_code else "enrollment_token"})
|
||||
|
||||
hostname = str(payload.get("hostname") or "")[:255]
|
||||
mismatch = bool(enr.expected_hostname) and bool(hostname) and hostname.lower() != enr.expected_hostname.lower()
|
||||
|
||||
plain_token = secrets.token_urlsafe(32)
|
||||
agent = Agent(tenant_id=enr.tenant_id, name=enr.display_name, token_hash=agent_token_hash(plain_token),
|
||||
enabled=True, agent_version=str(payload.get("agent_version") or "")[:32] or None)
|
||||
session.add(agent)
|
||||
session.flush()
|
||||
session.add(Installation(agent_id=agent.id, tenant_id=enr.tenant_id, installation_id=enr.enrollment_uuid,
|
||||
display_name=enr.display_name, hostname=hostname or None,
|
||||
features_configured=bool(enr.features_selected), corrector_enabled=bool(enr.features_selected and enr.feature_corrector),
|
||||
enrichment_mode=(enr.feature_enrichment_mode if enr.features_selected else "DISABLED")))
|
||||
|
||||
enr.status = "ENROLLED"
|
||||
enr.enrollment_used_at = datetime.now(timezone.utc)
|
||||
enr.enrollment_token_hash = None # replay-proof even inside its TTL window
|
||||
enr.install_code_hash = None # consumed: truly one-time, replay-proof
|
||||
enr.install_code_expires_at = None
|
||||
enr.hostname_seen = hostname or None
|
||||
enr.machine_fingerprint_seen = str(payload.get("machine_fingerprint") or "")[:128] or None
|
||||
enr.hostname_mismatch = mismatch
|
||||
enr.agent_id = agent.id
|
||||
|
||||
record_audit(session, admin_user_id=None, tenant_id=enr.tenant_id, action="ENROLLMENT_SUCCEEDED",
|
||||
target_type="agent_enrollment", target_id=enr.enrollment_uuid,
|
||||
context={"ip": ip, "hostname": hostname, "hostname_mismatch": mismatch, "agent_id": agent.id})
|
||||
record_audit(session, admin_user_id=None, tenant_id=enr.tenant_id, action="AGENT_CREDENTIAL_ISSUED",
|
||||
target_type="agent", target_id=str(agent.id), context={"ip": ip, "installation": enr.enrollment_uuid})
|
||||
session.commit()
|
||||
return {"agent_id": agent.id, "agent_token": plain_token, "tenant_id": enr.tenant_id,
|
||||
"installation_id": enr.enrollment_uuid,
|
||||
"hub_url": config.settings.base_url, "hostname_mismatch": mismatch}
|
||||
except enrollment.EnrollmentRejected as rejected:
|
||||
session.rollback()
|
||||
record_attempt("enroll", ip)
|
||||
reason = rejected.reason
|
||||
if enr is not None:
|
||||
enr.install_code_attempts = (enr.install_code_attempts or 0) + 1
|
||||
record_attempt(f"enroll_code:{enr.enrollment_uuid}", ip)
|
||||
session.commit()
|
||||
record_audit(session, admin_user_id=None, tenant_id=enr.tenant_id if enr else None, action="ENROLLMENT_REJECTED",
|
||||
target_type="agent_enrollment", target_id=enr.enrollment_uuid if enr else (installation_id or "-"),
|
||||
context={"ip": ip, "reason": reason}, severity="WARNING")
|
||||
session.commit()
|
||||
raise HTTPException(401, reason)
|
||||
|
||||
def _resolve_by_install_code(session, ip, code):
|
||||
"""Resolve the Installation by the install_code hash alone. Returns the
|
||||
enrollment or None. Format/unknown codes and expired codes raise
|
||||
EnrollmentRejected. Per-installation and per-IP rate limits apply."""
|
||||
try:
|
||||
digest = enrollment.install_code_digest(code)
|
||||
except ValueError:
|
||||
raise enrollment.EnrollmentRejected("INVALID_INSTALL_CODE")
|
||||
enr = session.scalar(select(AgentEnrollment).where(AgentEnrollment.install_code_hash == digest))
|
||||
if enr is None:
|
||||
raise enrollment.EnrollmentRejected("INVALID_INSTALL_CODE")
|
||||
# Per-installation rate limit: a hostile caller guessing codes for one
|
||||
# installation must burn the whole budget even after the code rotates.
|
||||
if blocked(f"enroll_code:{enr.enrollment_uuid}", ip):
|
||||
raise enrollment.EnrollmentRejected("RATE_LIMITED")
|
||||
now = datetime.now(timezone.utc)
|
||||
if not enr.install_code_expires_at or _aware(enr.install_code_expires_at) < now:
|
||||
raise enrollment.EnrollmentRejected("EXPIRED")
|
||||
return enr
|
||||
|
||||
def _resolve_by_legacy_token(session, ip, installation_id, supplied_token):
|
||||
"""Legacy enrollment_token path (published ZIP bundles). Returns the
|
||||
enrollment or raises EnrollmentRejected. Order preserved from the
|
||||
original flow: enrolled-again must read ALREADY_ENROLLED, not
|
||||
INVALID_TOKEN."""
|
||||
if not enrollment.valid_uuid(installation_id):
|
||||
raise enrollment.EnrollmentRejected("INVALID_INSTALLATION")
|
||||
enr = session.scalar(select(AgentEnrollment).where(AgentEnrollment.enrollment_uuid == installation_id))
|
||||
if enr is None:
|
||||
raise enrollment.EnrollmentRejected("INVALID_INSTALLATION")
|
||||
if enr.status == "REVOKED":
|
||||
raise enrollment.EnrollmentRejected("REVOKED")
|
||||
if enr.status == "ENROLLED":
|
||||
raise enrollment.EnrollmentRejected("ALREADY_ENROLLED")
|
||||
now = datetime.now(timezone.utc)
|
||||
if not enr.enrollment_token_expires_at or _aware(enr.enrollment_token_expires_at) < now:
|
||||
raise enrollment.EnrollmentRejected("EXPIRED")
|
||||
if not enrollment.token_matches(supplied_token, enr.enrollment_token_hash):
|
||||
raise enrollment.EnrollmentRejected("INVALID_TOKEN")
|
||||
return enr
|
||||
|
||||
return routes
|
||||
@@ -0,0 +1,8 @@
|
||||
"""AI semantic layer V1 — NOT a fiscal authority.
|
||||
|
||||
Rules frozen by the operator contract:
|
||||
- AI only selects among official candidates; it never invents codes.
|
||||
- Cache first: KB exact/fingerprint hits never trigger AI calls.
|
||||
- No AI for origin, ST retention, or company-level decisions.
|
||||
- AI_WRITE_ALLOWED=false in V1: results stay AI_CANDIDATE_VALIDATED.
|
||||
"""
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Offline-safe, ID-correlated provider batch execution."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .provider import ProviderTransportError
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchRunResult:
|
||||
results_by_work_item_id: dict[str, dict] = field(default_factory=dict)
|
||||
errors: dict[str, str] = field(default_factory=dict)
|
||||
provider_calls: int = 0
|
||||
retries: int = 0
|
||||
usage: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
def run_batches(items, provider, batch_size: int, max_retries: int = 2, *, scenario: str | None = None) -> BatchRunResult:
|
||||
if batch_size <= 0:
|
||||
raise ValueError("batch_size must be positive")
|
||||
if max_retries < 0:
|
||||
raise ValueError("max_retries must not be negative")
|
||||
|
||||
result = BatchRunResult()
|
||||
for start in range(0, len(items), batch_size):
|
||||
batch = list(items[start:start + batch_size])
|
||||
expected_ids = [str(item.work_item_id) for item in batch]
|
||||
answers = None
|
||||
for attempt in range(max_retries + 1):
|
||||
result.provider_calls += 1
|
||||
try:
|
||||
kwargs = {} if scenario is None else {"scenario": scenario}
|
||||
answers = provider.select_batch(batch, **kwargs)
|
||||
break
|
||||
except ProviderTransportError as error:
|
||||
if attempt == max_retries:
|
||||
for work_item_id in expected_ids:
|
||||
result.errors[work_item_id] = str(error)
|
||||
else:
|
||||
result.retries += 1
|
||||
except Exception as error:
|
||||
for work_item_id in expected_ids:
|
||||
result.errors[work_item_id] = str(error)
|
||||
break
|
||||
if answers is None:
|
||||
continue
|
||||
answer_ids = [str(answer.get("work_item_id")) for answer in answers]
|
||||
if len(answer_ids) != len(expected_ids) or set(answer_ids) != set(expected_ids) or len(set(answer_ids)) != len(answer_ids):
|
||||
error = "provider response did not contain exactly one answer per work_item_id"
|
||||
for work_item_id in expected_ids:
|
||||
result.errors[work_item_id] = error
|
||||
continue
|
||||
result.results_by_work_item_id.update({str(answer["work_item_id"]): answer for answer in answers})
|
||||
|
||||
result.usage = dict(getattr(provider, "usage", {}) or {})
|
||||
return result
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Persistent-boundary-compatible budget guard; provider invocation happens after this check."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from threading import Lock
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BudgetLimits:
|
||||
max_run_cost_usd: float = 0.54
|
||||
max_daily_cost_usd: float = 1.00
|
||||
max_monthly_cost_usd: float = 10.00
|
||||
max_requests_per_day: int = 100
|
||||
max_new_products_per_run: int = 750
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BudgetDecision:
|
||||
allowed: bool
|
||||
reason: str
|
||||
reserved_cost_usd: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UsageEntry:
|
||||
run_id: str
|
||||
tenant: str
|
||||
cost_usd: float
|
||||
requests: int
|
||||
new_products: int
|
||||
timestamp: str
|
||||
|
||||
|
||||
class BudgetLedger:
|
||||
"""Atomic in-process ledger with an explicit persistence boundary for Hub storage."""
|
||||
|
||||
def __init__(self, limits: BudgetLimits):
|
||||
self.limits = limits
|
||||
self.entries: list[UsageEntry] = []
|
||||
self._lock = Lock()
|
||||
|
||||
def check_and_reserve(self, run_id: str, tenant: str, cost_usd: float, requests: int, new_products: int) -> BudgetDecision:
|
||||
with self._lock:
|
||||
run_cost = sum(e.cost_usd for e in self.entries if e.run_id == run_id)
|
||||
day = datetime.now(timezone.utc).date()
|
||||
day_entries = [e for e in self.entries if e.timestamp[:10] == day.isoformat()]
|
||||
month_entries = [e for e in self.entries if e.timestamp[:7] == day.strftime("%Y-%m")]
|
||||
if run_cost + cost_usd > self.limits.max_run_cost_usd:
|
||||
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
|
||||
if sum(e.cost_usd for e in day_entries) + cost_usd > self.limits.max_daily_cost_usd:
|
||||
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
|
||||
if sum(e.cost_usd for e in month_entries) + cost_usd > self.limits.max_monthly_cost_usd:
|
||||
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
|
||||
if sum(e.requests for e in day_entries) + requests > self.limits.max_requests_per_day:
|
||||
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
|
||||
if sum(e.new_products for e in self.entries if e.run_id == run_id) + new_products > self.limits.max_new_products_per_run:
|
||||
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
|
||||
self.entries.append(UsageEntry(run_id, tenant, cost_usd, requests, new_products, datetime.now(timezone.utc).isoformat()))
|
||||
return BudgetDecision(True, "RESERVED", cost_usd)
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Pure cache and dataset-freshness policies for offline and real evaluations."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
|
||||
VALID, NEEDS_REVALIDATION, CONFLICT = "VALID", "NEEDS_REVALIDATION", "CONFLICT"
|
||||
|
||||
|
||||
def cache_key(input_fingerprint, candidate_set_hash, model, prompt_version, dataset_versions):
|
||||
"""Return a stable key covering every input that can change an answer."""
|
||||
payload = {
|
||||
"input_fingerprint": str(input_fingerprint),
|
||||
"candidate_set_hash": str(candidate_set_hash),
|
||||
"model": str(model),
|
||||
"prompt_version": str(prompt_version),
|
||||
"dataset_versions": dict(sorted((dataset_versions or {}).items())),
|
||||
}
|
||||
return hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
||||
|
||||
|
||||
def classify_knowledge_freshness(record_versions, active_versions):
|
||||
"""Classify versioned knowledge without deleting or rewriting history."""
|
||||
recorded = dict(record_versions or {})
|
||||
active = dict(active_versions or {})
|
||||
if any(key in recorded and key in active and recorded[key] != active[key] for key in recorded):
|
||||
return CONFLICT
|
||||
if recorded != active:
|
||||
return NEEDS_REVALIDATION
|
||||
return VALID
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Deterministic candidate generation from the ACTIVE official datasets.
|
||||
|
||||
The AI never receives an open question: for every task type the Hub first builds
|
||||
the official candidate list (NCM top-K by description, CEST links for the NCM,
|
||||
TIPI EX rows for the NCM). Providers may only pick from this list.
|
||||
"""
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import CestEntry, CestNcmEntry, Dataset, NcmEntry, TipiEntry
|
||||
from .fingerprint import token_stems
|
||||
|
||||
|
||||
def _active_dataset(session: Session, kind: str):
|
||||
return session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE"))
|
||||
|
||||
|
||||
def ncm_top_k(session: Session, description, k: int = 10):
|
||||
"""Top-K official NCM codes by (stemmed) token overlap with the product description."""
|
||||
dataset = _active_dataset(session, "NCM")
|
||||
if dataset is None:
|
||||
return []
|
||||
stems = token_stems(description)
|
||||
if not stems:
|
||||
return []
|
||||
scored = []
|
||||
for entry in session.scalars(select(NcmEntry).where(NcmEntry.dataset_id == dataset.id)).all():
|
||||
score = len(stems & token_stems(entry.description)) / len(stems)
|
||||
if score > 0:
|
||||
scored.append((entry.code, entry.description, round(score, 4)))
|
||||
scored.sort(key=lambda row: (-row[2], row[0]))
|
||||
return scored[:k]
|
||||
|
||||
|
||||
def cest_candidates_for_ncm(session: Session, ncm: str):
|
||||
dataset = _active_dataset(session, "CEST")
|
||||
if dataset is None:
|
||||
return []
|
||||
rows = session.execute(
|
||||
select(CestEntry, CestNcmEntry.ncm_code)
|
||||
.join(CestNcmEntry, CestNcmEntry.cest_entry_id == CestEntry.id)
|
||||
.where(CestEntry.dataset_id == dataset.id)
|
||||
).all()
|
||||
return list(dict.fromkeys(entry for entry, ncm_code in rows if ncm.startswith(ncm_code)))
|
||||
|
||||
|
||||
def ex_tipi_candidates_for_ncm(session: Session, ncm: str):
|
||||
dataset = _active_dataset(session, "TIPI_IPI")
|
||||
if dataset is None:
|
||||
return []
|
||||
return list(dict.fromkeys(
|
||||
entry for entry in session.scalars(select(TipiEntry).where(TipiEntry.dataset_id == dataset.id, TipiEntry.ncm == ncm, TipiEntry.ex.is_not(None))).all()
|
||||
))
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Metrics and error attribution for the real LLM NCM evaluation (sections 8-9)."""
|
||||
|
||||
|
||||
def metrics(rows) -> dict:
|
||||
"""rows: [{gold_in_candidates, selected, gold, confidence, invalid}]"""
|
||||
total = len(rows)
|
||||
correct = sum(1 for row in rows if not row["invalid"] and row["selected"] == row["gold"])
|
||||
wrong = sum(1 for row in rows if not row["invalid"] and row["selected"] is not None and row["selected"] != row["gold"])
|
||||
abstain = sum(1 for row in rows if not row["invalid"] and row["selected"] is None)
|
||||
invalid = sum(1 for row in rows if row["invalid"])
|
||||
gold_in = sum(1 for row in rows if row["gold_in_candidates"])
|
||||
answered = correct + wrong
|
||||
high_conf = [row for row in rows if not row["invalid"] and row["confidence"] >= 0.95 and row["selected"] is not None]
|
||||
high_conf_wrong = sum(1 for row in high_conf if row["selected"] != row["gold"])
|
||||
return {
|
||||
"TOTAL": total,
|
||||
"CORRECT": correct,
|
||||
"WRONG": wrong,
|
||||
"ABSTAIN": abstain,
|
||||
"INVALID": invalid,
|
||||
"GOLD_IN_CANDIDATES": gold_in,
|
||||
"ACCURACY_ALL": round(correct / total, 4) if total else None,
|
||||
"SELECTION_ACCURACY_WHEN_GOLD_AVAILABLE": round(correct / answered, 4) if answered else None,
|
||||
"PRECISION_WHEN_HIGH_CONFIDENCE": round(sum(1 for row in high_conf if row["selected"] == row["gold"]) / len(high_conf), 4) if high_conf else None,
|
||||
"HIGH_CONFIDENCE_WRONG_RATE": round(high_conf_wrong / len(high_conf), 4) if high_conf else None,
|
||||
"ABSTENTION_RATE": round(abstain / total, 4) if total else None,
|
||||
}
|
||||
|
||||
|
||||
def attribution(rows) -> dict:
|
||||
return {
|
||||
"GOLD_NOT_IN_CANDIDATES": sum(1 for row in rows if not row["gold_in_candidates"]),
|
||||
"GOLD_IN_CANDIDATES_BUT_WRONG": sum(1 for row in rows if row["gold_in_candidates"] and not row["invalid"] and row["selected"] is not None and row["selected"] != row["gold"]),
|
||||
"GOLD_IN_CANDIDATES_BUT_ABSTAIN": sum(1 for row in rows if row["gold_in_candidates"] and not row["invalid"] and row["selected"] is None),
|
||||
"INVALID_OUTPUT": sum(1 for row in rows if row["invalid"]),
|
||||
}
|
||||
|
||||
|
||||
_BUCKETS = ((">=0.99", 0.99, 1.01), ("0.95-0.9899", 0.95, 0.99), ("0.90-0.9499", 0.90, 0.95), ("0.80-0.8999", 0.80, 0.90), ("<0.80", -0.01, 0.80))
|
||||
|
||||
|
||||
def confidence_buckets(rows) -> dict:
|
||||
out = {name: {"total": 0, "correct": 0} for name, _, _ in _BUCKETS}
|
||||
for row in rows:
|
||||
if row["invalid"] or row["selected"] is None:
|
||||
continue
|
||||
for name, low, high in _BUCKETS:
|
||||
if low <= row["confidence"] < high:
|
||||
out[name]["total"] += 1
|
||||
if row["selected"] == row["gold"]:
|
||||
out[name]["correct"] += 1
|
||||
break
|
||||
return out
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Deterministic provider responses for offline pipeline tests."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
from .llm_provider import LlmAnswerError, LlmEvalItem, validate_answer
|
||||
from .provider import ProviderContractError, ProviderTransportError
|
||||
|
||||
|
||||
SCENARIOS = frozenset({
|
||||
"CORRECT", "WRONG", "ABSTAIN", "INVALID_JSON", "MISSING_ITEM",
|
||||
"DUPLICATE_ITEM", "OUT_OF_CANDIDATE", "TIMEOUT", "429", "500",
|
||||
})
|
||||
|
||||
|
||||
class DeterministicFakeSemanticProvider:
|
||||
"""Return repeatable answers without persistence or network access."""
|
||||
|
||||
name = "deterministic-fake"
|
||||
model = "deterministic-fake-v1"
|
||||
prompt_version = "fake-provider-v1"
|
||||
|
||||
def __init__(self, *, shuffle: bool = False, fail_work_item_ids: Iterable[str] = (), failure_scenario: str = "TIMEOUT"):
|
||||
self.shuffle = shuffle
|
||||
self.fail_work_item_ids = {str(item_id) for item_id in fail_work_item_ids}
|
||||
self.failure_scenario = failure_scenario
|
||||
self.calls = 0
|
||||
self.usage = {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0}
|
||||
|
||||
def select_batch(self, items: list[LlmEvalItem], scenario: str | None = None) -> list[dict]:
|
||||
scenario = scenario or "CORRECT"
|
||||
if scenario not in SCENARIOS:
|
||||
raise ValueError(f"unknown fake provider scenario: {scenario}")
|
||||
self.calls += 1
|
||||
if self.fail_work_item_ids and len(items) == 1 and str(items[0].work_item_id) in self.fail_work_item_ids:
|
||||
self._raise_transport(self.failure_scenario)
|
||||
if scenario in {"TIMEOUT", "429", "500"}:
|
||||
self._raise_transport(scenario)
|
||||
if scenario == "INVALID_JSON":
|
||||
raise LlmAnswerError("fake provider returned invalid JSON")
|
||||
|
||||
answers = [self._answer(item, scenario) for item in items]
|
||||
if scenario == "MISSING_ITEM" and answers:
|
||||
answers.pop()
|
||||
elif scenario == "DUPLICATE_ITEM" and answers:
|
||||
answers[-1] = dict(answers[0])
|
||||
if self.shuffle:
|
||||
answers.reverse()
|
||||
expected_ids = [str(item.work_item_id) for item in items]
|
||||
answer_ids = [str(answer.get("work_item_id")) for answer in answers]
|
||||
if len(answer_ids) != len(expected_ids) or set(answer_ids) != set(expected_ids) or len(set(answer_ids)) != len(answer_ids):
|
||||
raise ProviderContractError("fake provider response did not contain exactly one answer per work_item_id")
|
||||
items_by_id = {str(item.work_item_id): item for item in items}
|
||||
try:
|
||||
return [validate_answer(answer, items_by_id[str(answer["work_item_id"])]) for answer in answers]
|
||||
except (LlmAnswerError, KeyError, TypeError) as error:
|
||||
raise ProviderContractError(str(error)) from error
|
||||
|
||||
@staticmethod
|
||||
def _answer(item: LlmEvalItem, scenario: str) -> dict:
|
||||
candidates = [str(candidate[0]) for candidate in item.candidates]
|
||||
selected = None
|
||||
abstain = scenario == "ABSTAIN"
|
||||
if not abstain and candidates:
|
||||
selected = "99999999" if scenario == "OUT_OF_CANDIDATE" else candidates[min(1, len(candidates) - 1) if scenario == "WRONG" else 0]
|
||||
return {
|
||||
"work_item_id": item.work_item_id,
|
||||
"selected_ncm": selected,
|
||||
"confidence": 0.0 if abstain else 0.75,
|
||||
"abstain": abstain or selected is None,
|
||||
"reason_short": f"fake scenario {scenario}",
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _raise_transport(scenario: str) -> None:
|
||||
if scenario not in {"TIMEOUT", "429", "500"}:
|
||||
raise ValueError(f"unknown fake transport scenario: {scenario}")
|
||||
raise ProviderTransportError(f"fake provider transport failure: {scenario}")
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Product identity: stable fingerprints and GTIN validation.
|
||||
|
||||
Global identity uses ONLY stable attributes (normalized description, brand, unit,
|
||||
package quantity, GTIN). Fiscal/company context (CFOP, CSOSN, CST, rate, ST, tenant)
|
||||
never participates: those vary per context and must not merge distinct knowledge.
|
||||
"""
|
||||
import hashlib
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
from ..classification import ENGINE_VERSION # noqa: F401 (re-exported for the AI layer)
|
||||
|
||||
|
||||
def normalize_description(value) -> str:
|
||||
"""NFKD -> ascii -> A-Z0-9 tokens, space-joined (same canonical form as classification)."""
|
||||
text = unicodedata.normalize("NFKD", str(value or "")).encode("ascii", "ignore").decode().upper()
|
||||
return " ".join(re.sub(r"[^A-Z0-9]+", " ", text).split())
|
||||
|
||||
|
||||
def _normalize_token(value) -> str:
|
||||
return normalize_description(value)
|
||||
|
||||
|
||||
def _normalize_quantity(value) -> str:
|
||||
digits = re.sub(r"[^0-9.,]", "", str(value or ""))
|
||||
if not digits:
|
||||
return ""
|
||||
try:
|
||||
number = float(digits.replace(",", "."))
|
||||
except ValueError:
|
||||
return digits
|
||||
return f"{number:g}"
|
||||
|
||||
|
||||
def product_fingerprint(description, brand=None, unit=None, package_quantity=None) -> str:
|
||||
parts = [_normalize_token(description), _normalize_token(brand), _normalize_token(unit), _normalize_quantity(package_quantity)]
|
||||
return hashlib.sha256("|".join(parts).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _stem(token: str) -> str:
|
||||
"""Minimal Portuguese plural stemming so AGUAS matches AGUA / MINERAIS matches MINERAL."""
|
||||
if len(token) > 4 and token.endswith("AIS"):
|
||||
return token[:-3] + "AL"
|
||||
if len(token) > 4 and token.endswith("ES"):
|
||||
return token[:-2]
|
||||
if len(token) > 3 and token.endswith("S"):
|
||||
return token[:-1]
|
||||
return token
|
||||
|
||||
|
||||
def token_stems(value) -> set:
|
||||
return {_stem(token) for token in normalize_description(value).split()}
|
||||
|
||||
|
||||
def overlap_score(query, official) -> float:
|
||||
"""Fraction of query description stems present in the official description stems."""
|
||||
stems = token_stems(query)
|
||||
if not stems:
|
||||
return 0.0
|
||||
return len(stems & token_stems(official)) / len(stems)
|
||||
|
||||
|
||||
def validate_gtin(value) -> str | None:
|
||||
"""Returns the canonical GTIN when the checksum is valid, else None.
|
||||
Accepts GTIN-8/12/13/14 (EAN-8/UPC-12/EAN-13/GTIN-14)."""
|
||||
digits = re.sub(r"\D", "", str(value or ""))
|
||||
if len(digits) not in (8, 12, 13, 14):
|
||||
return None
|
||||
total = 0
|
||||
for index, char in enumerate(reversed(digits)):
|
||||
digit = int(char)
|
||||
total += digit * (1 if index % 2 == 0 else 3)
|
||||
if total % 10 != 0:
|
||||
return None
|
||||
return digits
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Evidence reconstruction for historical KB conflicts.
|
||||
|
||||
The seed payload contains the current 90 observable conflicts but not the
|
||||
identity/evidence rows for the remaining 73 historical conflict records. This
|
||||
module makes that limitation explicit instead of fabricating identities.
|
||||
"""
|
||||
|
||||
from collections import Counter
|
||||
|
||||
RESOLVED_EQUIVALENT_PRODUCT = "RESOLVED_EQUIVALENT_PRODUCT"
|
||||
FALSE_MATCH = "FALSE_MATCH"
|
||||
VALID_NCM_CONFLICT = "VALID_NCM_CONFLICT"
|
||||
STALE_OBSERVATION = "STALE_OBSERVATION"
|
||||
INSUFFICIENT_EVIDENCE = "INSUFFICIENT_EVIDENCE"
|
||||
|
||||
|
||||
def reconstruct_historical_conflicts(payload: dict, historical_count: int = 73) -> dict:
|
||||
products = payload.get("products", [])
|
||||
gtin_count = sum(1 for product in products if product.get("gtin"))
|
||||
fingerprint_count = sum(1 for product in products if product.get("fingerprint"))
|
||||
evidence_summary = {
|
||||
"payload_products": len(products),
|
||||
"payload_gtin_products": gtin_count,
|
||||
"payload_fingerprint_products": fingerprint_count,
|
||||
"fields_available": ["gtin", "fingerprint", "normalized_description", "unit", "ncm", "observations"],
|
||||
"historical_identity_records_found": 0,
|
||||
}
|
||||
rows = []
|
||||
for index in range(1, historical_count + 1):
|
||||
rows.append({
|
||||
"historical_conflict_id": f"HISTORICAL-{index:03d}",
|
||||
"classification": INSUFFICIENT_EVIDENCE,
|
||||
"evidence_search": evidence_summary,
|
||||
"resolution": "PRESERVE_CONFLICT",
|
||||
"reason": "original conflict identity/evidence row is absent from available artifacts; no NCM selected",
|
||||
})
|
||||
return {
|
||||
"artifact": "KB_HISTORICAL_CONFLICT_AUDIT_V1",
|
||||
"historical_conflicts_before": historical_count,
|
||||
"historical_conflicts_resolved": 0,
|
||||
"historical_conflicts_remaining": historical_count,
|
||||
"classification_counts": dict(Counter(row["classification"] for row in rows)),
|
||||
"conflicts": rows,
|
||||
"current_observable_conflicts_preserved": 90,
|
||||
"current_conflict_policy": "NCM_CONFLICT rows remain blocked; no majority resolution",
|
||||
"evidence_complete": False,
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Product Knowledge Base service: identity resolution, versioned knowledge, conflicts.
|
||||
|
||||
Identity order (operator contract): 1. exact GTIN; 2. strong fingerprint; 3. semantic
|
||||
fingerprint candidate; 4. unknown. Conflicting knowledge for the same key is kept
|
||||
side by side and surfaced as KNOWLEDGE_CONFLICT — never picked by majority.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func as sa_func, select
|
||||
from ..models import KbKnowledge, KbObservation, KbProduct
|
||||
|
||||
EXACT, FINGERPRINT = "EXACT", "FINGERPRINT"
|
||||
_AUTHORITATIVE = {"OFFICIAL_DATASET", "OPERATOR", "AI_PLUS_OFFICIAL_VALIDATION"}
|
||||
|
||||
|
||||
class KnowledgeBase:
|
||||
def __init__(self, session):
|
||||
self.session = session
|
||||
|
||||
# ----- identity -----
|
||||
|
||||
def upsert_product(self, fingerprint, gtin=None, normalized_description="", brand=None, unit=None, package_quantity=None, category=None):
|
||||
gtin = gtin or None
|
||||
product = None
|
||||
if gtin:
|
||||
product = self.session.scalar(select(KbProduct).where(KbProduct.gtin == gtin))
|
||||
if product is None:
|
||||
product = self.session.scalar(select(KbProduct).where(KbProduct.fingerprint == fingerprint))
|
||||
if product is None:
|
||||
product = KbProduct(fingerprint=fingerprint, normalized_description=normalized_description or "")
|
||||
self.session.add(product)
|
||||
# identity upgrades (never downgrades)
|
||||
if gtin and not product.gtin:
|
||||
product.gtin = gtin
|
||||
product.identity_kind = "GTIN"
|
||||
if normalized_description and not product.normalized_description:
|
||||
product.normalized_description = normalized_description
|
||||
for field, value in (("brand", brand), ("unit", unit), ("package_quantity", package_quantity), ("category", category)):
|
||||
if value and not getattr(product, field):
|
||||
setattr(product, field, value)
|
||||
self.session.flush()
|
||||
return product
|
||||
|
||||
def resolve(self, gtin=None, fingerprint=None):
|
||||
"""Returns (product|None, hit_kind). Cache-first: EXACT (GTIN) beats FINGERPRINT."""
|
||||
if gtin:
|
||||
product = self.session.scalar(select(KbProduct).where(KbProduct.gtin == gtin))
|
||||
if product:
|
||||
return product, EXACT
|
||||
if fingerprint:
|
||||
product = self.session.scalar(select(KbProduct).where(KbProduct.fingerprint == fingerprint))
|
||||
if product:
|
||||
return product, FINGERPRINT
|
||||
return None, None
|
||||
|
||||
# ----- knowledge -----
|
||||
|
||||
def add_knowledge(self, kb_product_id, knowledge_key, value, source, confidence=0.0, evidence=None, model=None, prompt_version=None, dataset_versions=None):
|
||||
value = str(value)
|
||||
row = self.session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == kb_product_id, KbKnowledge.knowledge_key == knowledge_key, KbKnowledge.value == value))
|
||||
if row is None:
|
||||
row = KbKnowledge(kb_product_id=kb_product_id, knowledge_key=knowledge_key, value=value, source=source, confidence=confidence, evidence=evidence or {}, model=model, prompt_version=prompt_version, dataset_versions=dataset_versions or {})
|
||||
self.session.add(row)
|
||||
else:
|
||||
# authoritative sources upgrade provenance/confidence of the same value; history preserved
|
||||
if source in _AUTHORITATIVE:
|
||||
row.source = source
|
||||
row.confidence = max(row.confidence, confidence)
|
||||
row.evidence = {**row.evidence, **(evidence or {})}
|
||||
if dataset_versions:
|
||||
row.dataset_versions = {**row.dataset_versions, **dataset_versions}
|
||||
if source == "OFFICIAL_DATASET":
|
||||
row.validated_at = datetime.now(timezone.utc)
|
||||
self.session.flush()
|
||||
return row
|
||||
|
||||
def current_knowledge(self, kb_product_id):
|
||||
rows = self.session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == kb_product_id)).all()
|
||||
by_key: dict = {}
|
||||
for row in rows:
|
||||
by_key.setdefault(row.knowledge_key, []).append(row)
|
||||
result = {}
|
||||
for key, entries in by_key.items():
|
||||
values = {entry.value for entry in entries}
|
||||
if len(values) > 1:
|
||||
result[key] = {"conflict": True, "values": values}
|
||||
else:
|
||||
best = max(entries, key=lambda entry: entry.confidence)
|
||||
result[key] = {"conflict": False, "value": best.value, "source": best.source, "confidence": best.confidence}
|
||||
return result
|
||||
|
||||
# ----- observations -----
|
||||
|
||||
def record_observation(self, kb_product_id, tenant_id, company_key, erp_code, database_label, observed_fields, source="ERP_OBSERVATION"):
|
||||
row = KbObservation(kb_product_id=kb_product_id, tenant_id=tenant_id, company_key=company_key, erp_code=erp_code, database_label=database_label, observed_fields=observed_fields or {}, source=source)
|
||||
self.session.add(row); self.session.flush()
|
||||
return row
|
||||
|
||||
def observation_summary(self, kb_product_id):
|
||||
rows = self.session.scalars(select(KbObservation).where(KbObservation.kb_product_id == kb_product_id)).all()
|
||||
agreement: dict = {}
|
||||
for row in rows:
|
||||
for field, value in (row.observed_fields or {}).items():
|
||||
if value in (None, ""):
|
||||
continue
|
||||
bucket = agreement.setdefault(field, {})
|
||||
bucket[str(value)] = bucket.get(str(value), 0) + 1
|
||||
conflicts = {field: values for field, values in agreement.items() if len(values) > 1}
|
||||
return {"observation_count": len(rows), "tenant_count": len({row.tenant_id for row in rows}), "agreement": agreement, "conflicts": conflicts}
|
||||
|
||||
# ----- stats -----
|
||||
|
||||
def stats(self):
|
||||
products = self.session.scalar(select(sa_func.count()).select_from(KbProduct)) or 0
|
||||
gtin_products = self.session.scalar(select(sa_func.count()).select_from(KbProduct).where(KbProduct.gtin.is_not(None))) or 0
|
||||
conflicts = 0
|
||||
for product_id in self.session.scalars(select(KbProduct.id)).all():
|
||||
knowledge = self.current_knowledge(product_id)
|
||||
conflicts += sum(1 for entry in knowledge.values() if entry.get("conflict"))
|
||||
fingerprint_products = products - gtin_products
|
||||
return {"products": products, "gtin_products": gtin_products, "fingerprint_products": fingerprint_products, "conflicts": conflicts}
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Conservative, explainable product identity and conflict classification."""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from collections import defaultdict
|
||||
|
||||
from .fingerprint import normalize_description, token_stems, validate_gtin
|
||||
|
||||
GTIN_CONFLICT = "GTIN_CONFLICT"
|
||||
NCM_CONFLICT = "NCM_CONFLICT"
|
||||
DESCRIPTION_CONFLICT = "DESCRIPTION_CONFLICT"
|
||||
PACKAGING_VARIATION = "PACKAGING_VARIATION"
|
||||
FALSE_MATCH = "FALSE_MATCH"
|
||||
TENANT_CONTEXT_ONLY = "TENANT_CONTEXT_ONLY"
|
||||
|
||||
|
||||
def canonical_product_id(observation: dict) -> str:
|
||||
"""Stable global ID; fiscal and tenant fields are intentionally excluded."""
|
||||
gtin = validate_gtin(observation.get("gtin"))
|
||||
if gtin:
|
||||
return "GTIN:" + gtin
|
||||
values = "|".join((
|
||||
normalize_description(observation.get("normalized_description") or observation.get("description")),
|
||||
normalize_description(observation.get("brand")),
|
||||
normalize_description(observation.get("unit")),
|
||||
normalize_description(observation.get("package_quantity")),
|
||||
))
|
||||
return "FP:" + hashlib.sha256(values.encode()).hexdigest()
|
||||
|
||||
|
||||
def _description_equivalent(values):
|
||||
stems = [token_stems(value) for value in values if value]
|
||||
if not stems:
|
||||
return True
|
||||
return all(current == stems[0] or current.issubset(stems[0]) or stems[0].issubset(current) for current in stems[1:])
|
||||
|
||||
|
||||
def classify_conflict(observations: list[dict]) -> str:
|
||||
"""Classify only from structural evidence; ambiguous cases remain conflicts."""
|
||||
gtins = {validate_gtin(row.get("gtin")) for row in observations if validate_gtin(row.get("gtin"))}
|
||||
ncms = {str(row.get("ncm")) for row in observations if row.get("ncm")}
|
||||
descriptions = [row.get("normalized_description") or row.get("description") for row in observations]
|
||||
packages = {str(row.get("package_quantity")) for row in observations if row.get("package_quantity")}
|
||||
tenants = {str(obs.get("database") or obs.get("company_key")) for row in observations for obs in row.get("observations", [])}
|
||||
if len(gtins) > 1:
|
||||
return GTIN_CONFLICT
|
||||
if len(ncms) > 1:
|
||||
return NCM_CONFLICT
|
||||
if len(packages) > 1 and _description_equivalent(descriptions):
|
||||
return PACKAGING_VARIATION
|
||||
if len({normalize_description(value) for value in descriptions if value}) > 1:
|
||||
return DESCRIPTION_CONFLICT
|
||||
if len(tenants) > 1:
|
||||
return TENANT_CONTEXT_ONLY
|
||||
return FALSE_MATCH
|
||||
|
||||
|
||||
def audit_payload(payload: dict, historical_conflicts: int = 163) -> dict:
|
||||
"""Audit observable same-GTIN groups without silently resolving NCM conflicts."""
|
||||
by_gtin = defaultdict(list)
|
||||
for product in payload.get("products", []):
|
||||
gtin = validate_gtin(product.get("gtin"))
|
||||
if gtin:
|
||||
by_gtin[gtin].append(product)
|
||||
conflicts = []
|
||||
counts = defaultdict(int)
|
||||
for gtin, rows in by_gtin.items():
|
||||
if len({row.get("ncm") for row in rows if row.get("ncm")}) <= 1:
|
||||
continue
|
||||
kind = classify_conflict(rows)
|
||||
counts[kind] += 1
|
||||
conflicts.append({
|
||||
"identity": gtin,
|
||||
"classification": kind,
|
||||
"observations": len(rows),
|
||||
"ncms": sorted({row.get("ncm") for row in rows if row.get("ncm")}),
|
||||
"auto_resolution": "KEEP_KNOWLEDGE_CONFLICT",
|
||||
})
|
||||
return {
|
||||
"artifact": "KB_CONFLICT_AUDIT_V1",
|
||||
"historical_conflicts_reported": historical_conflicts,
|
||||
"observable_conflicts_audited": len(conflicts),
|
||||
"unobservable_historical_conflicts": max(0, historical_conflicts - len(conflicts)),
|
||||
"classification_counts": dict(sorted(counts.items())),
|
||||
"conflicts": conflicts,
|
||||
"policy": "only structurally equivalent identities may share a canonical product; tenant fiscal context is never global knowledge",
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Real LLM provider (OpenAI-compatible) for the NCM semantic evaluation.
|
||||
|
||||
Contract (mission REAL LLM EVALUATION V1):
|
||||
- The API key lives ONLY in the Hub environment (AI_API_KEY). Never committed, logged,
|
||||
written to artifacts, sent to the Agent or Firebird.
|
||||
- The gold answer is never part of the prompt: items carry only description/brand/unit/
|
||||
segment-weak-context + the official top-K hybrid candidates with hierarchical context.
|
||||
- Strict structured output: per item {work_item_id, selected_ncm, confidence, abstain,
|
||||
reason_short}; selected_ncm must be in the candidate list or null (abstain).
|
||||
- Batching is safe only when the response is valid JSON with 1:1 work_item_id matching,
|
||||
no omitted items, no cross-product mixing. Verified progressively (5/10/20).
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
PROMPT_VERSION_LLM = "llm-ncm-eval-v1"
|
||||
|
||||
_SYSTEM_PROMPT = (
|
||||
"You are a Brazilian customs classification assistant. For each product below, choose the "
|
||||
"most likely official NCM code from the provided candidate list, or abstain. "
|
||||
"Rules: selected_ncm MUST be one of the candidate codes for that exact item, or null with "
|
||||
"abstain=true when evidence is insufficient. Never invent a code. confidence is your calibrated "
|
||||
"probability in [0,1]. Reply with ONLY a JSON array, one object per item, in the same order, "
|
||||
'each: {"work_item_id": str, "selected_ncm": str|null, "confidence": float, "abstain": bool, "reason_short": str}.'
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LlmEvalItem:
|
||||
work_item_id: str
|
||||
description: str
|
||||
normalized_description: str
|
||||
brand: str | None
|
||||
unit: str | None
|
||||
segment: str
|
||||
candidates: list # [(code, official_description, hybrid_score, official_score, kb_score, tenant_count)]
|
||||
|
||||
|
||||
class LlmAnswerError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def item_prompt_payload(item: LlmEvalItem) -> dict:
|
||||
return {
|
||||
"work_item_id": item.work_item_id,
|
||||
"description": item.description,
|
||||
"normalized_description": item.normalized_description,
|
||||
"brand": item.brand,
|
||||
"unit": item.unit,
|
||||
"segment_weak_context": item.segment,
|
||||
"official_candidates": [
|
||||
{"ncm": code, "official_description": text}
|
||||
for code, text, *_ in item.candidates
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def candidate_set_hash(item: LlmEvalItem) -> str:
|
||||
codes = [code for code, *_ in item.candidates]
|
||||
return hashlib.sha256("|".join(codes).encode()).hexdigest()
|
||||
|
||||
|
||||
def input_fingerprint(item: LlmEvalItem) -> str:
|
||||
payload = {**item_prompt_payload(item), "system": _SYSTEM_PROMPT}
|
||||
return hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
||||
|
||||
|
||||
def validate_answer(answer: dict, item: LlmEvalItem) -> dict:
|
||||
"""Fail-closed validator: schema, work_item_id, candidate-list confinement."""
|
||||
if not isinstance(answer, dict):
|
||||
raise LlmAnswerError("answer must be an object")
|
||||
if str(answer.get("work_item_id")) != str(item.work_item_id):
|
||||
raise LlmAnswerError(f"work_item_id mismatch: {answer.get('work_item_id')!r} != {item.work_item_id!r}")
|
||||
abstain = bool(answer.get("abstain"))
|
||||
selected = answer.get("selected_ncm")
|
||||
selected = None if selected in (None, "") else str(selected)
|
||||
try:
|
||||
confidence = float(answer.get("confidence"))
|
||||
except (TypeError, ValueError):
|
||||
raise LlmAnswerError("confidence must be numeric")
|
||||
if not 0.0 <= confidence <= 1.0:
|
||||
raise LlmAnswerError("confidence out of [0,1]")
|
||||
codes = {code for code, *_ in item.candidates}
|
||||
if abstain:
|
||||
if selected is not None:
|
||||
raise LlmAnswerError("abstain must carry selected_ncm=null")
|
||||
else:
|
||||
if selected is None:
|
||||
raise LlmAnswerError("non-abstain requires selected_ncm")
|
||||
if selected not in codes:
|
||||
raise LlmAnswerError(f"selected_ncm {selected!r} is outside the candidate list")
|
||||
return {"work_item_id": item.work_item_id, "selected_ncm": selected, "confidence": confidence, "abstain": abstain, "reason_short": str(answer.get("reason_short") or "")[:200]}
|
||||
|
||||
|
||||
class OpenAICompatibleProvider:
|
||||
"""Talks to any OpenAI-compatible /chat/completions endpoint. `transport` is
|
||||
injectable for tests; production default uses httpx with the key from settings only."""
|
||||
|
||||
name = "openai-compatible"
|
||||
prompt_version = PROMPT_VERSION_LLM
|
||||
|
||||
def __init__(self, api_key: str, model: str, base_url: str = "https://api.openai.com/v1", transport=None, timeout: float = 120.0):
|
||||
self.api_key = api_key
|
||||
self.model = model
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.timeout = timeout
|
||||
self._transport = transport or self._httpx_transport
|
||||
self.http_calls = 0
|
||||
self.retries = 0
|
||||
self.usage = {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0}
|
||||
|
||||
def _httpx_transport(self, payload: dict) -> dict:
|
||||
import httpx
|
||||
response = httpx.post(f"{self.base_url}/chat/completions", json=payload, headers={"Authorization": f"Bearer {self.api_key}"}, timeout=self.timeout)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
def list_models(self) -> list:
|
||||
"""GET /models with the account key — validates auth/base URL/model availability
|
||||
without spending completion tokens. Never raises with the key attached."""
|
||||
import httpx
|
||||
response = httpx.get(f"{self.base_url}/models", headers={"Authorization": f"Bearer {self.api_key}"}, timeout=self.timeout)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return sorted(str(item.get("id")) for item in data.get("data", []) if item.get("id"))
|
||||
|
||||
def _request(self, messages: list, max_retries: int = 2) -> dict:
|
||||
payload = {"model": self.model, "messages": messages, "temperature": 0}
|
||||
last_error = None
|
||||
for attempt in range(max_retries + 1):
|
||||
self.http_calls += 1
|
||||
if attempt:
|
||||
self.retries += 1
|
||||
try:
|
||||
data = self._transport(payload)
|
||||
usage = data.get("usage") or {}
|
||||
self.usage["input_tokens"] += int(usage.get("prompt_tokens") or 0)
|
||||
self.usage["output_tokens"] += int(usage.get("completion_tokens") or 0)
|
||||
cached = usage.get("prompt_tokens_details") or {}
|
||||
self.usage["cached_tokens"] += int(cached.get("cached_tokens") or 0)
|
||||
return data
|
||||
except Exception as error: # transport-level failure only; JSON contract errors are not retried
|
||||
last_error = error
|
||||
raise RuntimeError(f"LLM transport failed after {max_retries + 1} attempts: {last_error}")
|
||||
|
||||
def select_batch(self, items: list[LlmEvalItem]) -> list[dict]:
|
||||
"""One HTTP call for the whole batch. Raises LlmAnswerError on any 1:1/schema violation
|
||||
so the harness can shrink the batch size."""
|
||||
user_content = json.dumps([item_prompt_payload(item) for item in items], ensure_ascii=False)
|
||||
data = self._request([{"role": "system", "content": _SYSTEM_PROMPT}, {"role": "user", "content": user_content}])
|
||||
content = data["choices"][0]["message"]["content"]
|
||||
content = re.sub(r"^```(?:json)?|```$", "", content.strip(), flags=re.MULTILINE).strip()
|
||||
answers = json.loads(content)
|
||||
if not isinstance(answers, list) or len(answers) != len(items):
|
||||
raise LlmAnswerError(f"expected {len(items)} answers, got {len(answers) if isinstance(answers, list) else type(answers).__name__}")
|
||||
return [validate_answer(answer, item) for answer, item in zip(answers, items)]
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Deterministic prompt compaction that preserves candidate evidence."""
|
||||
|
||||
import json
|
||||
|
||||
|
||||
def estimate_tokens(payload) -> int:
|
||||
return max(1, len(json.dumps(payload, ensure_ascii=False, sort_keys=True)) // 4)
|
||||
|
||||
|
||||
def compact_payload(payload):
|
||||
"""Remove repeated metadata only; candidate codes/descriptions are untouched."""
|
||||
compact = dict(payload)
|
||||
compact.pop("debug", None)
|
||||
compact.pop("duplicate_examples", None)
|
||||
candidates = compact.get("official_candidates")
|
||||
if isinstance(candidates, list):
|
||||
seen = set()
|
||||
compact["official_candidates"] = [
|
||||
row for row in candidates
|
||||
if not (row.get("ncm") in seen or seen.add(row.get("ncm")))
|
||||
]
|
||||
return compact
|
||||
|
||||
|
||||
def measure_payloads(payloads):
|
||||
before = sum(estimate_tokens(payload) for payload in payloads)
|
||||
after = sum(estimate_tokens(compact_payload(payload)) for payload in payloads)
|
||||
return {"items": len(payloads), "tokens_before": before, "tokens_after": after, "reduction_percent": round((1 - after / before) * 100, 2) if before else 0.0}
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Provider abstraction with strict structured output and a hard budget guard.
|
||||
|
||||
The fiscal engine is never coupled to a vendor: it talks to AISemanticProvider.
|
||||
V1 ships BaselineProvider (deterministic, zero-cost) used by the evaluation
|
||||
harness and as the no-credential fallback; real providers subclass the same
|
||||
contract and are only invoked with AI_ENABLED=true and credentials in the Hub.
|
||||
"""
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from ..classification import ENGINE_VERSION
|
||||
from .fingerprint import overlap_score
|
||||
|
||||
PROMPT_VERSION = "ai-semantic-v1"
|
||||
|
||||
|
||||
class SemanticOutputError(ValueError):
|
||||
"""Raised when a provider answer violates the strict structured-output schema."""
|
||||
|
||||
|
||||
class ProviderTransportError(RuntimeError):
|
||||
"""A retryable failure while contacting a semantic provider."""
|
||||
|
||||
|
||||
class ProviderContractError(ValueError):
|
||||
"""A non-retryable provider response or protocol violation."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class StructuredResult:
|
||||
task_type: str
|
||||
input_fingerprint: str
|
||||
selected_candidate: str # candidate code, or "NONE"
|
||||
confidence: float
|
||||
reason: str
|
||||
candidate_ids_seen: list[str]
|
||||
model: str
|
||||
prompt_version: str
|
||||
created_at: str
|
||||
dataset_versions: dict = field(default_factory=dict)
|
||||
engine_version: str = ENGINE_VERSION
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
"task_type": self.task_type,
|
||||
"input_fingerprint": self.input_fingerprint,
|
||||
"selected_candidate": self.selected_candidate,
|
||||
"confidence": self.confidence,
|
||||
"reason": self.reason,
|
||||
"candidate_ids_seen": self.candidate_ids_seen,
|
||||
"model": self.model,
|
||||
"prompt_version": self.prompt_version,
|
||||
"created_at": self.created_at,
|
||||
"dataset_versions": self.dataset_versions,
|
||||
"engine_version": self.engine_version,
|
||||
}
|
||||
|
||||
|
||||
def result_from_dict(data) -> StructuredResult:
|
||||
"""Fail-closed schema check: the selection must be NONE or one of the seen candidates."""
|
||||
result = StructuredResult(
|
||||
task_type=str(data["task_type"]),
|
||||
input_fingerprint=str(data["input_fingerprint"]),
|
||||
selected_candidate=str(data["selected_candidate"]),
|
||||
confidence=float(data["confidence"]),
|
||||
reason=str(data["reason"]),
|
||||
candidate_ids_seen=[str(item) for item in data["candidate_ids_seen"]],
|
||||
model=str(data["model"]),
|
||||
prompt_version=str(data["prompt_version"]),
|
||||
created_at=str(data["created_at"]),
|
||||
dataset_versions=dict(data.get("dataset_versions") or {}),
|
||||
)
|
||||
if result.selected_candidate != "NONE" and result.selected_candidate not in result.candidate_ids_seen:
|
||||
raise SemanticOutputError(f"selected_candidate {result.selected_candidate!r} is not among the official candidates")
|
||||
if not 0.0 <= result.confidence <= 1.0:
|
||||
raise SemanticOutputError("confidence must be within [0,1]")
|
||||
return result
|
||||
|
||||
|
||||
class AISemanticProvider:
|
||||
"""Contract every provider must honour. `select` receives ONLY official candidates."""
|
||||
name = "abstract"
|
||||
model = "abstract"
|
||||
|
||||
def select(self, work_item, candidates) -> StructuredResult:
|
||||
raise NotImplementedError
|
||||
|
||||
def estimate(self, work_items) -> dict:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def _score(text: str, official: str) -> float:
|
||||
return overlap_score(text, official)
|
||||
|
||||
|
||||
class BaselineProvider(AISemanticProvider):
|
||||
"""Deterministic zero-cost baseline: highest token-overlap candidate, or NONE.
|
||||
Used for the evaluation pipeline and as the mandatory no-credential fallback."""
|
||||
|
||||
name = "baseline"
|
||||
model = "baseline-deterministic-v1"
|
||||
|
||||
def select(self, work_item, candidates, dataset_versions=None) -> StructuredResult:
|
||||
description = work_item.payload.get("description", "") if hasattr(work_item, "payload") else ""
|
||||
scored = [(code, official, _score(description, official)) for code, official in [(str(c[0]), c[1]) for c in candidates]]
|
||||
best = max(scored, key=lambda row: row[2]) if scored else None
|
||||
selected, _, score = best if best else ("NONE", "", 0.0)
|
||||
if not scored or score <= 0.0:
|
||||
selected = "NONE"
|
||||
return StructuredResult(
|
||||
task_type=work_item.task_type,
|
||||
input_fingerprint=work_item.dedup_key,
|
||||
selected_candidate=selected,
|
||||
confidence=round(score, 4),
|
||||
reason="baseline token-overlap top-1" if selected != "NONE" else "no token overlap with any official candidate",
|
||||
candidate_ids_seen=[str(c[0]) for c in candidates],
|
||||
model=self.model,
|
||||
prompt_version=PROMPT_VERSION,
|
||||
created_at=datetime.now(timezone.utc).isoformat(),
|
||||
dataset_versions=dict(dataset_versions or {}),
|
||||
)
|
||||
|
||||
def estimate(self, work_items) -> dict:
|
||||
return {"input_tokens": 0, "output_tokens": 0, "estimated_cost": 0.0}
|
||||
|
||||
|
||||
class BudgetGuard:
|
||||
"""Refuses execution when the estimated cost exceeds AI_MAX_RUN_COST_USD.
|
||||
Never falls back to a more expensive model. Prices are operator-configured — when
|
||||
absent, cost is UNKNOWN: the residual run is refused, but an explicitly limited
|
||||
benchmark (e.g. the 65-item GOLD eval) may opt in with allow_unknown_cost."""
|
||||
|
||||
def __init__(self, max_run_cost_usd: float, price_in_per_m: float | None, price_out_per_m: float | None):
|
||||
self.max_run_cost_usd = float(max_run_cost_usd)
|
||||
self.price_in_per_m = None if price_in_per_m is None else float(price_in_per_m)
|
||||
self.price_out_per_m = None if price_out_per_m is None else float(price_out_per_m)
|
||||
|
||||
def estimate(self, items_in_tokens: int, items_out_tokens: int) -> dict:
|
||||
if self.price_in_per_m is None or self.price_out_per_m is None:
|
||||
return {"input_tokens": items_in_tokens, "output_tokens": items_out_tokens, "estimated_cost": None, "cost_unknown": True}
|
||||
cost = (items_in_tokens / 1_000_000) * self.price_in_per_m + (items_out_tokens / 1_000_000) * self.price_out_per_m
|
||||
return {"input_tokens": items_in_tokens, "output_tokens": items_out_tokens, "estimated_cost": round(cost, 6)}
|
||||
|
||||
def authorize(self, plan: dict, allow_unknown_cost: bool = False) -> bool:
|
||||
if plan.get("estimated_cost") is None:
|
||||
return allow_unknown_cost
|
||||
return plan["estimated_cost"] <= self.max_run_cost_usd
|
||||
|
||||
def authorize_usage(self, input_tokens: int, output_tokens: int, *, prior_cost_usd: float = 0.0,
|
||||
allow_unknown_cost: bool = False) -> bool:
|
||||
"""Apply the same guard to actual usage before accepting a provider result."""
|
||||
plan = self.estimate(input_tokens, output_tokens)
|
||||
if plan.get("estimated_cost") is None:
|
||||
return allow_unknown_cost
|
||||
return float(prior_cost_usd) + plan["estimated_cost"] <= self.max_run_cost_usd
|
||||
@@ -0,0 +1,101 @@
|
||||
"""Pure, review-first questionnaire helpers for local fiscal context."""
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
|
||||
from central.app.context_keys import validate_entry
|
||||
|
||||
|
||||
def _options(row):
|
||||
value = row.get("VALID_OPTIONS", [])
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
value = json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
value = [x.strip() for x in value.split("|") if x.strip()]
|
||||
return list(value) if isinstance(value, (list, tuple, set)) else [value]
|
||||
|
||||
|
||||
def build_questionnaire(rows):
|
||||
result = []
|
||||
for row in rows:
|
||||
scope = str(row.get("SCOPE", row.get("scope", ""))).strip().upper()
|
||||
key = str(row.get("KEY", row.get("key", ""))).strip()
|
||||
scope_key = str(row.get("GROUP") or row.get("NCM") or row.get("scope_key") or "").strip()
|
||||
result.append({
|
||||
"question_id": str(row.get("QUESTION_ID", row.get("question_id", ""))).strip(),
|
||||
"database": str(row.get("DATABASE", row.get("database", ""))).strip(),
|
||||
"scope": scope,
|
||||
"scope_key": scope_key if scope != "COMPANY" else "",
|
||||
"group": str(row.get("GROUP") or "").strip(),
|
||||
"key": key,
|
||||
"question": row.get("QUESTION", row.get("question", "")),
|
||||
"options": _options(row),
|
||||
"affected": {"products": int(row.get("PRODUCTS_AFFECTED") or row.get("products_affected") or 0), "cells": int(row.get("CELLS") or row.get("cells") or 0)},
|
||||
"examples": row.get("EXAMPLES", row.get("examples", "")),
|
||||
"consequence": row.get("WHAT_IT_WILL_RESOLVE", row.get("consequence", "")),
|
||||
})
|
||||
return sorted(result, key=lambda q: (q["database"], q["scope"], q["scope_key"], q["key"], q["question_id"]))
|
||||
|
||||
|
||||
def group_questions(questions):
|
||||
"""Group equivalent human decisions while retaining every affected question ID."""
|
||||
groups = {}
|
||||
for question in questions:
|
||||
key = (question["database"], question["group"], question["scope_key"], question["key"], question["question"], tuple(question["options"]))
|
||||
group = groups.setdefault(key, {
|
||||
"group_id": "GROUP-" + str(len(groups) + 1),
|
||||
"database": question["database"],
|
||||
"group": question["group"],
|
||||
"scope": question["scope"],
|
||||
"scope_key": question["scope_key"],
|
||||
"key": question["key"],
|
||||
"question": question["question"],
|
||||
"options": question["options"],
|
||||
"question_ids": [],
|
||||
"affected": {"products": 0, "cells": 0},
|
||||
"examples": [],
|
||||
"consequence": question["consequence"],
|
||||
})
|
||||
group["question_ids"].append(question["question_id"])
|
||||
group["affected"]["products"] += question["affected"]["products"]
|
||||
group["affected"]["cells"] += question["affected"]["cells"]
|
||||
if question["examples"] and len(group["examples"]) < 5:
|
||||
group["examples"].append(str(question["examples"]))
|
||||
return list(groups.values())
|
||||
|
||||
|
||||
def validate_answers(questions, answers):
|
||||
entries, effects, rollback_plan = [], [], []
|
||||
by_id = {q["question_id"]: q for q in questions}
|
||||
for question_id, value in answers.items():
|
||||
if question_id not in by_id:
|
||||
raise ValueError(f"unknown question {question_id!r}; nothing was persisted")
|
||||
q = by_id[question_id]
|
||||
value = str(value).strip()
|
||||
if q["options"] and value not in q["options"]:
|
||||
raise ValueError(f"{question_id}: value must be one of {q['options']}; nothing was persisted")
|
||||
entry, error = validate_entry({"scope": q["scope"], "scope_key": q["scope_key"], "key": q["key"], "value": value})
|
||||
if error:
|
||||
raise ValueError(f"{question_id}: {error}; nothing was persisted")
|
||||
entry["company_key"] = q["database"]
|
||||
entries.append(entry)
|
||||
effect = {"question_id": question_id, "database": q["database"], "key": q["key"], "affected": q["affected"], "behavior": "context_entry"}
|
||||
if q["key"] == "entry_scope" and value == "MIXED_OR_DYNAMIC":
|
||||
effect["behavior"] = "no_static_cfop_entrada"
|
||||
effects.append(effect)
|
||||
if q["key"] == "store_ipi_profile" and value == "LEAVE_NOT_APPLICABLE":
|
||||
rollback_plan.append({"database": q["database"], "field": "IPI", "affected_cells": q["affected"]["cells"]})
|
||||
return {"entries": entries, "effects": effects, "rollback_plan": rollback_plan}
|
||||
|
||||
|
||||
def render_json(result):
|
||||
return json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
||||
|
||||
|
||||
def render_csv(result):
|
||||
output = io.StringIO()
|
||||
writer = csv.DictWriter(output, fieldnames=["company_key", "scope", "scope_key", "key", "value", "reason"])
|
||||
writer.writeheader()
|
||||
writer.writerows({k: entry.get(k, "") for k in writer.fieldnames} for entry in result["entries"])
|
||||
return output.getvalue()
|
||||
@@ -0,0 +1,245 @@
|
||||
"""Semantic candidate retrieval V2 — deterministic, zero LLM.
|
||||
|
||||
Three retrieval sources merged into one ranking:
|
||||
A. OFFICIAL: hierarchical search over the official NCM tree. Every 8-digit code is
|
||||
enriched with its ancestors' descriptions (chapter/position/subposition), so the
|
||||
very short legal text carries the vocabulary of its path (section 6).
|
||||
B. KB NEIGHBORS: the Knowledge Base acts as a commercial->official bridge — products
|
||||
whose commercial description resembles the query contribute their VALIDATED NCMs,
|
||||
weighted by similarity, tenant_count, observation_count and agreement (sections 2-3).
|
||||
C. HYBRID: single final ranking; the KB is only evidence for CANDIDATES, never a
|
||||
majority-vote fiscal decision.
|
||||
|
||||
Abbreviation lexicon (section 8) is derived statistically from the KB itself; commercial
|
||||
normalization (section 7) strips units/sizes/colors/refs deterministically.
|
||||
"""
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import Dataset, KbKnowledge, KbObservation, KbProduct, NcmEntry, Tenant
|
||||
from .fingerprint import normalize_description, token_stems
|
||||
|
||||
# ---------------- commercial normalization (section 7) ----------------
|
||||
|
||||
_UNITS = {"UN", "UNID", "UNIDADE", "CX", "CAIXA", "FD", "FDO", "FARDO", "PCT", "PACOTE", "PKG",
|
||||
"KG", "G", "GR", "GRAMA", "MG", "ML", "L", "LT", "LITRO", "LATA", "GARRAFA", "PET",
|
||||
"PC", "PECA", "DZ", "DUZIA", "PAR", "KIT", "JG", "PARA", "COM", "SEM", "C", "S"}
|
||||
_DISCARD = _UNITS | {"MASC", "MASCULINO", "FEM", "FEMININO", "INF", "INFANTIL", "TAM", "TAMANHO",
|
||||
"COR", "REF", "COD", "CODIGO", "MOD", "MODELO", "NOVA", "NOVO", "TIPO"}
|
||||
_LEXEME_STRIP_PREFIXES = ("1", "2", "3", "4", "5", "6", "7", "8", "9", "0", ".")
|
||||
|
||||
|
||||
def _is_size_or_code(token: str) -> bool:
|
||||
if token.isdigit():
|
||||
return True
|
||||
digits = "".join(char for char in token if char.isdigit())
|
||||
letters = "".join(char for char in token if char.isalpha())
|
||||
return bool(digits) and letters in _UNITS # "1KG", "500ML", "2LT"
|
||||
|
||||
|
||||
def commercial_tokens(description) -> set:
|
||||
"""Stable commercial lexemes: units/sizes/colors/refs removed, stems kept."""
|
||||
tokens = []
|
||||
skip_next = False
|
||||
for token in normalize_description(description).split():
|
||||
if skip_next:
|
||||
skip_next = False
|
||||
continue
|
||||
if token in ("TAM", "COR", "REF", "COD", "MOD"):
|
||||
skip_next = True
|
||||
continue
|
||||
if token in _DISCARD or _is_size_or_code(token):
|
||||
continue
|
||||
tokens.append(token)
|
||||
return {_stem_safe(token) for token in tokens}
|
||||
|
||||
|
||||
def _stem_safe(token: str) -> str:
|
||||
from .fingerprint import _stem
|
||||
return _stem(token)
|
||||
|
||||
|
||||
def expand_stems(stems: set, lexicon: dict) -> set:
|
||||
expanded = set(stems)
|
||||
for stem in stems:
|
||||
target = lexicon.get(stem)
|
||||
if target:
|
||||
expanded.add(target)
|
||||
return expanded
|
||||
|
||||
|
||||
def derive_abbreviation_lexicon(corpus: dict, min_support: int = 3) -> dict:
|
||||
"""corpus: {commercial_description: ncm}. Statistical abbreviations only: a short token
|
||||
(3-5 chars) that is a strict prefix of a longer token, where BOTH forms occur across
|
||||
distinct products that share the same NCM. Support = (#products with the short form) +
|
||||
(#products with the long form); accepted when support >= min_support and both >= 1."""
|
||||
by_ncm: dict = {}
|
||||
for description, ncm in corpus.items():
|
||||
tokens = set(normalize_description(description).split())
|
||||
by_ncm.setdefault(ncm, []).append(tokens)
|
||||
best: dict = {}
|
||||
for descriptions in by_ncm.values():
|
||||
if len(descriptions) < 2:
|
||||
continue
|
||||
all_tokens = set().union(*descriptions)
|
||||
for short in all_tokens:
|
||||
if not (3 <= len(short) <= 5):
|
||||
continue
|
||||
short_count = sum(1 for tokens in descriptions if short in tokens)
|
||||
if short_count < 1:
|
||||
continue
|
||||
for long_token in all_tokens:
|
||||
if len(long_token) > len(short) and long_token.startswith(short):
|
||||
long_count = sum(1 for tokens in descriptions if long_token in tokens)
|
||||
support = short_count + long_count
|
||||
if support >= min_support and long_count >= 1 and short_count >= 1:
|
||||
current = best.get(short)
|
||||
if current is None or support > current[1]:
|
||||
best[short] = (long_token, support)
|
||||
return {short: target for short, (target, _) in best.items()}
|
||||
|
||||
|
||||
# ---------------- enriched official corpus (sections 5-6) ----------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class EnrichedNcmCorpus:
|
||||
documents: dict # code -> enriched stems set
|
||||
descriptions: dict # code -> official description
|
||||
positions: dict # 4-digit position -> enriched stems (chapter + position)
|
||||
children: dict # 4-digit position -> [8-digit codes]
|
||||
|
||||
@staticmethod
|
||||
def build(session: Session) -> "EnrichedNcmCorpus":
|
||||
dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
|
||||
if dataset is None:
|
||||
return EnrichedNcmCorpus({}, {}, {}, {})
|
||||
entries = session.scalars(select(NcmEntry).where(NcmEntry.dataset_id == dataset.id)).all()
|
||||
by_code = {entry.code: entry.description for entry in entries}
|
||||
documents, positions, children = {}, {}, {}
|
||||
for code, description in by_code.items():
|
||||
if len(code) != 8:
|
||||
continue
|
||||
ancestors = [text for text in (by_code.get(code[:2]), by_code.get(code[:4]), by_code.get(code[:6])) if text]
|
||||
documents[code] = token_stems(description) | set().union(*[token_stems(text) for text in ancestors]) if ancestors else token_stems(description)
|
||||
positions.setdefault(code[:4], token_stems(by_code.get(code[:4], "")) | token_stems(by_code.get(code[:2], "")))
|
||||
children.setdefault(code[:4], []).append(code)
|
||||
return EnrichedNcmCorpus(documents, by_code, positions, children)
|
||||
|
||||
def document(self, code: str) -> str:
|
||||
return " ".join(sorted(self.documents.get(code, set())))
|
||||
|
||||
def items(self):
|
||||
return list(self.documents.items())
|
||||
|
||||
|
||||
def _score(stems: set, document_stems: set, idf: dict | None = None) -> float:
|
||||
if not stems:
|
||||
return 0.0
|
||||
if idf:
|
||||
matched = sum(idf.get(stem, 1.0) for stem in stems if stem in document_stems)
|
||||
total = sum(idf.get(stem, 1.0) for stem in stems)
|
||||
return matched / total
|
||||
return len(stems & document_stems) / len(stems)
|
||||
|
||||
|
||||
def hierarchical_candidates(corpus: EnrichedNcmCorpus, description, k: int = 20, max_positions: int = 15):
|
||||
"""Two-stage: rank 4-digit positions first, then their 8-digit descendants (section 5)."""
|
||||
stems = commercial_tokens(description)
|
||||
if not stems:
|
||||
return []
|
||||
position_scores = sorted(((_score(stems, doc), position) for position, doc in corpus.positions.items()), reverse=True)
|
||||
ranked = []
|
||||
for position_score, position in position_scores[:max_positions]:
|
||||
if position_score <= 0:
|
||||
continue
|
||||
for code in corpus.children.get(position, []):
|
||||
code_score = _score(stems, corpus.documents[code])
|
||||
if code_score > 0:
|
||||
ranked.append((code, corpus.descriptions.get(code, ""), round(code_score, 4)))
|
||||
ranked.sort(key=lambda row: (-row[2], row[0]))
|
||||
return ranked[:k]
|
||||
|
||||
|
||||
# ---------------- KB neighbor bridge (sections 2-3) ----------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class KbNeighborIndex:
|
||||
entries: list # (product_id, stems, ncm, tenant_ids:set, observation_count, has_conflict)
|
||||
|
||||
@staticmethod
|
||||
def build(session: Session) -> "KbNeighborIndex":
|
||||
validated = {}
|
||||
for knowledge in session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "validated_ncm")).all():
|
||||
validated.setdefault(knowledge.kb_product_id, set()).add(knowledge.value)
|
||||
conflicts = {product_id: len(values) > 1 for product_id, values in validated.items()}
|
||||
products = {product.id: product for product in session.scalars(select(KbProduct)).all()}
|
||||
observations = session.scalars(select(KbObservation)).all()
|
||||
per_product_tenants: dict = {}
|
||||
for observation in observations:
|
||||
per_product_tenants.setdefault(observation.kb_product_id, set()).add(observation.tenant_id)
|
||||
entries = []
|
||||
for product_id, values in validated.items():
|
||||
if conflicts.get(product_id):
|
||||
continue # conflicting knowledge never feeds neighbors
|
||||
for value in values:
|
||||
product = products.get(product_id)
|
||||
if product is None or not product.normalized_description:
|
||||
continue
|
||||
tenants = per_product_tenants.get(product_id, set())
|
||||
entries.append((product_id, commercial_tokens(product.normalized_description), value, tenants, len(tenants), False))
|
||||
return KbNeighborIndex(entries)
|
||||
|
||||
def corpus(self) -> dict:
|
||||
return {" ".join(sorted(stems)): ncm for _, stems, ncm, _, _, _ in self.entries}
|
||||
|
||||
|
||||
def kb_neighbor_candidates(index: KbNeighborIndex, description, k: int = 10, exclude_product_id: int | None = None):
|
||||
"""Aggregate NCMs of lexically similar known products. `exclude_product_id` implements
|
||||
leave-one-out evaluation: the queried product never retrieves itself. Returns
|
||||
(ncm, neighbor_score, observation_count, tenant_count) — agreement is the union of
|
||||
distinct tenants behind the contributing neighbors, reported as evidence, never used
|
||||
as an automatic majority decision."""
|
||||
stems = commercial_tokens(description)
|
||||
if not stems:
|
||||
return []
|
||||
best: dict = {}
|
||||
for product_id, neighbor_stems, ncm, tenant_ids, observation_count, _ in index.entries:
|
||||
if exclude_product_id is not None and product_id == exclude_product_id:
|
||||
continue
|
||||
similarity = _score(stems, neighbor_stems)
|
||||
if similarity <= 0:
|
||||
continue
|
||||
current = best.get(ncm)
|
||||
if current is None:
|
||||
best[ncm] = [ncm, round(similarity, 4), observation_count, set(tenant_ids)]
|
||||
else:
|
||||
current[1] = max(current[1], round(similarity, 4))
|
||||
current[3] |= set(tenant_ids)
|
||||
ranked = sorted(((row[0], row[1], row[2], len(row[3])) for row in best.values()), key=lambda row: (-row[1], -row[3], row[0]))
|
||||
return ranked[:k]
|
||||
|
||||
|
||||
# ---------------- hybrid ranking (section 4) ----------------
|
||||
|
||||
OFFICIAL_WEIGHT, KB_WEIGHT = 0.3, 0.7 # tuned on the V2 benchmark (KB bridge carries the signal;
|
||||
# the official component breaks ties and covers products absent from the KB)
|
||||
|
||||
|
||||
def hybrid_candidates(corpus: EnrichedNcmCorpus, index: KbNeighborIndex, description, k: int = 20, lexicon: dict | None = None, exclude_product_id: int | None = None):
|
||||
stems = expand_stems(commercial_tokens(description), lexicon or {})
|
||||
official = hierarchical_candidates(corpus, " ".join(sorted(stems)) or description, k=k)
|
||||
neighbors = kb_neighbor_candidates(index, description, k=k, exclude_product_id=exclude_product_id)
|
||||
combined: dict = {}
|
||||
for code, text, score in official:
|
||||
combined[code] = [code, text, OFFICIAL_WEIGHT * score, score, 0.0, 0]
|
||||
for code, similarity, observation_count, tenant_count in neighbors:
|
||||
row = combined.setdefault(code, [code, "", 0.0, 0.0, 0.0, 0])
|
||||
row[2] += KB_WEIGHT * similarity
|
||||
row[4] = similarity
|
||||
row[5] = tenant_count
|
||||
ranked = sorted(combined.values(), key=lambda row: (-round(row[2], 4), row[0]))
|
||||
return [(row[0], row[1], round(row[2], 4), row[3], row[4], row[5]) for row in ranked[:k]]
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Retrieval V3 evidence layer. It ranks candidates only; it never decides fiscal truth."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
from .fingerprint import normalize_description, token_stems
|
||||
from .retrieval import commercial_tokens, hybrid_candidates
|
||||
|
||||
|
||||
def _char_similarity(left, right):
|
||||
return SequenceMatcher(None, normalize_description(left), normalize_description(right)).ratio()
|
||||
|
||||
|
||||
def retrieval_v3_candidates(corpus, index, description, metadata=None, k=20, lexicon=None, exclude_product_id=None):
|
||||
"""Add optional product evidence to V2 while preserving V2 as the safe baseline."""
|
||||
metadata = metadata or {}
|
||||
base = hybrid_candidates(corpus, index, description, k=max(k, 50), lexicon=lexicon, exclude_product_id=exclude_product_id)
|
||||
gtin_ncm = str(metadata.get("gtin_ncm") or "")
|
||||
group_ncms = {str(value) for value in (metadata.get("group_ncms") or [])}
|
||||
rarity = metadata.get("token_rarity") or {}
|
||||
query_stems = commercial_tokens(description)
|
||||
rows = []
|
||||
for code, text, score, official_score, kb_score, tenant_count in base:
|
||||
evidence = 0.0
|
||||
if gtin_ncm and code == gtin_ncm:
|
||||
evidence += 1.0
|
||||
if code in group_ncms:
|
||||
evidence += 0.05
|
||||
matched_rarity = sum(float(rarity.get(token, 1.0)) for token in query_stems if token in token_stems(text))
|
||||
evidence += min(0.05, matched_rarity / 100.0)
|
||||
rows.append((code, text, round(score + evidence, 4), official_score, kb_score, tenant_count))
|
||||
rows.sort(key=lambda row: (-row[2], row[0]))
|
||||
return rows[:k]
|
||||
|
||||
|
||||
def candidate_fingerprint(payload):
|
||||
return hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest()
|
||||
|
||||
|
||||
def prepare_candidate_set(work_item, candidates, *, dataset_versions=None, gtin_evidence=None, kb_neighbors=None, hierarchical_context=None):
|
||||
"""Build the final LLM-ready evidence package without selecting a candidate."""
|
||||
payload = {
|
||||
"task_type": work_item.task_type,
|
||||
"dedup_key": work_item.dedup_key,
|
||||
"normalized_product": normalize_description(work_item.payload.get("description", "")),
|
||||
"top20_candidate_ncms": [
|
||||
{"ncm": row[0], "official_description": row[1], "score": row[2], "official_score": row[3], "kb_score": row[4], "tenant_count": row[5]}
|
||||
for row in candidates[:20]
|
||||
],
|
||||
"official_descriptions": {row[0]: row[1] for row in candidates[:20]},
|
||||
"hierarchical_context": hierarchical_context or {},
|
||||
"kb_neighbors": kb_neighbors or [],
|
||||
"gtin_evidence": gtin_evidence or {},
|
||||
"dataset_versions": dict(dataset_versions or {}),
|
||||
}
|
||||
payload["input_fingerprint"] = candidate_fingerprint(payload)
|
||||
return payload
|
||||
|
||||
|
||||
def compact_product_role_context(rows, limit=12):
|
||||
"""Keep representative role evidence bounded; no role is inferred."""
|
||||
unique = []
|
||||
seen = set()
|
||||
for row in rows:
|
||||
description = normalize_description(row.get("description"))
|
||||
if not description or description in seen:
|
||||
continue
|
||||
seen.add(description)
|
||||
unique.append({"description": description, "group": row.get("group"), "subgroup": row.get("subgroup"), "operations": row.get("operations", [])})
|
||||
if len(unique) >= limit:
|
||||
break
|
||||
return {"examples": unique, "evidence_only": True, "decision": None}
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Residual-question dedup: group pending cells by the KNOWLEDGE needed, not per cell.
|
||||
|
||||
- INVALID_NCM -> one work item per product fingerprint (same product in N databases collapses)
|
||||
- CEST/EX_TIPI -> one work item per (doubt, NCM): all databases sharing that NCM collapse
|
||||
- PRODUCT_ROLE -> tenant-specific by contract: one work item per (company, group)
|
||||
- Everything else (origin, ST retention, company profile, duplicate key) never becomes
|
||||
an AI work item.
|
||||
"""
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
_INVALID_NCM, _ROLE = "ncm", "product_role"
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkItem:
|
||||
task_type: str
|
||||
dedup_key: str
|
||||
question_refs: list = field(default_factory=list)
|
||||
cells_covered: int = 0
|
||||
payload: dict = field(default_factory=dict)
|
||||
kb_product_id: int | None = None
|
||||
|
||||
|
||||
def build_work_items(questions, product_fingerprints=None, canonical_products=None) -> list[WorkItem]:
|
||||
"""`product_fingerprints` maps "{DATABASE}:{erp_code}" -> KB fingerprint (from the seed pass)."""
|
||||
product_fingerprints = product_fingerprints or {}
|
||||
canonical_products = canonical_products or {}
|
||||
items: dict = {}
|
||||
|
||||
def _get(task_type, dedup_key, ref, cells):
|
||||
item = items.get(dedup_key)
|
||||
if item is None:
|
||||
item = WorkItem(task_type=task_type, dedup_key=dedup_key)
|
||||
items[dedup_key] = item
|
||||
item.question_refs.append(ref)
|
||||
item.cells_covered += cells
|
||||
return item
|
||||
|
||||
for question in questions:
|
||||
key = question.get("KEY")
|
||||
scope = question.get("SCOPE")
|
||||
cells = int(question.get("CELLS") or question.get("PRODUCTS_AFFECTED") or 0)
|
||||
ref = question.get("QUESTION_ID") or f"{question.get('DATABASE')}:{scope}:{question.get('NCM') or question.get('GROUP')}:{key}"
|
||||
if key in ("cest", "ex_tipi") and scope == "NCM":
|
||||
ncm = str(question.get("NCM") or "").strip()
|
||||
if ncm:
|
||||
item = _get(key.upper(), f"{key}:{ncm}", ref, cells)
|
||||
item.payload.setdefault("ncm", ncm)
|
||||
item.payload["description"] = question.get("EXAMPLES", "").split(";")[0].strip() if question.get("EXAMPLES") else ""
|
||||
elif key == _INVALID_NCM and scope == "PRODUCT":
|
||||
database = question.get("DATABASE", "")
|
||||
erp_code = str(question.get("DATABASE_CODE") or "").strip()
|
||||
fingerprint = product_fingerprints.get(f"{database}:{erp_code}")
|
||||
fingerprint = canonical_products.get(f"{database}:{erp_code}", fingerprint)
|
||||
dedup = f"invalid_ncm:{fingerprint}" if fingerprint else f"invalid_ncm:{database}:{erp_code}"
|
||||
item = _get("INVALID_NCM", dedup, ref, cells)
|
||||
item.payload.setdefault("erp_code", erp_code)
|
||||
item.payload["description"] = question.get("EXAMPLES", "").split(";")[0].strip() if question.get("EXAMPLES") else ""
|
||||
elif key == _ROLE and scope == "GROUP":
|
||||
company = question.get("COMPANY", "")
|
||||
group = str(question.get("GROUP") or "").strip()
|
||||
if group:
|
||||
normalized_group = " ".join(str(group).upper().split())
|
||||
item = _get("PRODUCT_ROLE", f"product_role:{company}:{normalized_group}", ref, cells)
|
||||
item.payload.setdefault("company_key", company)
|
||||
item.payload["group"] = group
|
||||
return list(items.values())
|
||||
@@ -0,0 +1,475 @@
|
||||
"""Authenticated version-one Hub routes."""
|
||||
import hashlib, json, logging
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response
|
||||
import csv
|
||||
import io
|
||||
from sqlalchemy import select
|
||||
from .auth import authenticate, bootstrap
|
||||
from .classification import ENGINE_VERSION, Classifier
|
||||
from .config import settings
|
||||
from .datasets.service import DatasetService
|
||||
from .jobs import SYNC_KINDS, claim_next_job, complete_job, create_job, fail_job, schedule_automatic_sync
|
||||
from .rtc_audit import job_authorizations, record_rtc_write_audits
|
||||
from .cfop_ippt_audit import job_authorizations as cfop_ippt_job_authorizations, record_cfop_ippt_write_audits
|
||||
from .blocked_products import list_blockers, resolve_blocker, resolve_product_blockers, resolve_stale_fiscal_blockers, upsert_blocker, upsert_blockers_bulk
|
||||
from .context_keys import validate_entry
|
||||
from .models import AgentSnapshot, BlockedProduct, FiscalContextEntry, ClassificationCache, ClassificationRequest, ClassificationResult, Dataset, Installation, Job
|
||||
from .events import record_event
|
||||
from .installation_identity import compute_agent_health, compute_version_drift, reference_agent_version, sanitize_registration
|
||||
from .control_plane import list_jobs
|
||||
from .human_auth import require_human
|
||||
from .enrichment.api import router as enrichment_router
|
||||
|
||||
logger = logging.getLogger("gti_fiscal_hub")
|
||||
|
||||
def router(session_factory, bootstrap_token):
|
||||
routes = APIRouter(prefix="/v1")
|
||||
def principal(request: Request):
|
||||
header = request.headers.get("Authorization", "")
|
||||
if not header.startswith("Bearer "): raise HTTPException(401, "Bearer token required")
|
||||
session = session_factory()
|
||||
try:
|
||||
if bootstrap_token: bootstrap(session, bootstrap_token)
|
||||
agent = authenticate(session, header.removeprefix("Bearer "))
|
||||
if not agent: raise HTTPException(401, "Invalid token")
|
||||
return session, agent
|
||||
except Exception:
|
||||
session.close(); raise
|
||||
def decide(payload, identity):
|
||||
session, agent = identity; service = DatasetService(session)
|
||||
supplied = payload.get("tenant_id")
|
||||
if supplied is not None and str(supplied) != str(agent.tenant_id): raise HTTPException(403, "Tenant does not match token")
|
||||
product = payload.get("product", payload)
|
||||
# Cheap key first (engine version + dataset versions + input), then the cache; the heavy classifier runs only on a MISS.
|
||||
engine_version, dataset_versions = ENGINE_VERSION, service.dataset_versions()
|
||||
canonical = {"tenant": agent.tenant_id, "product": product, "context": payload.get("context", {}), "engine": engine_version, "datasets": dataset_versions}
|
||||
fingerprint = hashlib.sha256(json.dumps(canonical, sort_keys=True, default=str, separators=(",", ":")).encode()).hexdigest()
|
||||
cached = session.get(ClassificationCache, fingerprint)
|
||||
if cached:
|
||||
result = dict(cached.result); result["provenance"] = {**result["provenance"], "cache_hit": True}; return result
|
||||
decision = Classifier(service).classify(product)
|
||||
if decision["engine_version"] != engine_version or decision["dataset_versions"] != dataset_versions: # never store under a key that does not describe the decision
|
||||
canonical = {**canonical, "engine": decision["engine_version"], "datasets": decision["dataset_versions"]}
|
||||
fingerprint = hashlib.sha256(json.dumps(canonical, sort_keys=True, default=str, separators=(",", ":")).encode()).hexdigest()
|
||||
result = {**decision, "request_fingerprint": fingerprint, "provenance": {"cache_hit": False, "dataset_versions": decision["dataset_versions"]}}
|
||||
request = ClassificationRequest(tenant_id=agent.tenant_id, agent_id=agent.id, fingerprint=fingerprint, payload=payload); session.add(request); session.flush()
|
||||
session.add(ClassificationResult(request_id=request.id, result=result)); session.add(ClassificationCache(fingerprint=fingerprint, tenant_id=agent.tenant_id, result=result)); session.commit(); return result
|
||||
@routes.post("/validate/ncm")
|
||||
def validate_ncm(payload: dict, identity=Depends(principal)):
|
||||
return decide({"product": {"current_ncm": payload.get("ncm"), "description": ""}, "tenant_id": payload.get("tenant_id")}, identity)
|
||||
@routes.post("/classify")
|
||||
def classify(payload: dict, identity=Depends(principal)):
|
||||
try: return decide(payload, identity)
|
||||
finally: identity[0].close()
|
||||
@routes.post("/classify/batch")
|
||||
def classify_batch(payload: dict, identity=Depends(principal)):
|
||||
try: return {"results": [decide({**payload, "product": product}, identity) for product in payload.get("products", [])]}
|
||||
finally: identity[0].close()
|
||||
@routes.get("/datasets/status")
|
||||
def status(identity=Depends(principal)):
|
||||
session, _ = identity
|
||||
try: return [{"id": item.id, "kind": item.kind, "status": item.status, "sha256": item.sha256, "record_count": item.record_count} for item in session.scalars(select(Dataset)).all()]
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/reference/cfop/{code}")
|
||||
def reference_cfop(code: str, identity=Depends(principal)):
|
||||
session, _ = identity
|
||||
try:
|
||||
rows = DatasetService(session).cfop_lookup(code=code)
|
||||
return {"found": bool(rows), "entries": [{"code": r.code, "title": r.title, "description": r.description} for r in rows]}
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/reference/ncm_utrib/{ncm}")
|
||||
def reference_ncm_utrib(ncm: str, identity=Depends(principal)):
|
||||
session, _ = identity
|
||||
try:
|
||||
row = DatasetService(session).ncm_utrib_for_ncm(ncm)
|
||||
return {"found": row is not None, "utrib_abbreviation": row.utrib_abbreviation if row else None}
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/reference/anp/{cprod_anp}")
|
||||
def reference_anp(cprod_anp: str, identity=Depends(principal)):
|
||||
session, _ = identity
|
||||
try: return {"found": DatasetService(session).anp_code_exists(cprod_anp)}
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/reference/fcp/{uf}")
|
||||
def reference_fcp(uf: str, identity=Depends(principal)):
|
||||
session, _ = identity
|
||||
try:
|
||||
row = DatasetService(session).fcp_for_uf(uf)
|
||||
return {"found": row is not None, "aliq_1": row.aliq_1 if row else None, "observacao": row.observacao if row else None}
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/reference/rtc_cclasstrib/{code}")
|
||||
def reference_rtc_cclasstrib(code: str, identity=Depends(principal)):
|
||||
session, _ = identity
|
||||
try:
|
||||
rows = DatasetService(session).rtc_cclasstrib(code=code)
|
||||
return {"found": bool(rows), "entries": [{"cst": r.cst, "cclasstrib": r.cclasstrib, "name": r.name, "description": r.description, "tipo_aliquota": r.tipo_aliquota, "p_red_ibs": r.p_red_ibs, "p_red_cbs": r.p_red_cbs, "d_ini_vig": r.d_ini_vig, "d_fim_vig": r.d_fim_vig} for r in rows]}
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/reference/rtc_cbs_rate/{year}")
|
||||
def reference_rtc_cbs_rate(year: int, identity=Depends(principal)):
|
||||
session, _ = identity
|
||||
try:
|
||||
row = DatasetService(session).rtc_cbs_rate(year)
|
||||
return {"found": row is not None, "year": year, "rate": row.rate if row else None, "rate_text": row.rate_text if row else None}
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/reference/fiscal_codes/{domain}")
|
||||
def reference_fiscal_codes(domain: str, identity=Depends(principal)):
|
||||
session, _ = identity
|
||||
try:
|
||||
rows = DatasetService(session).fiscal_codes(domain.upper())
|
||||
return {"found": bool(rows), "domain": domain.upper(), "codes": sorted(r.code for r in rows)}
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/context/import")
|
||||
def context_import(payload: dict, identity=Depends(principal)):
|
||||
"""Batch upsert of operator fiscal context: {"company_key": "...", "entries": [{scope, scope_key, key, value, reason}]}. All-or-nothing."""
|
||||
session, agent = identity
|
||||
try:
|
||||
company_key = str(payload.get("company_key") or "").strip()
|
||||
if not company_key:
|
||||
raise HTTPException(400, "company_key is required")
|
||||
entries, errors = [], []
|
||||
for i, raw in enumerate(payload.get("entries") or []):
|
||||
entry, error = validate_entry(raw)
|
||||
(errors.append({"index": i, "error": error, "entry": raw}) if error else entries.append(entry))
|
||||
if errors:
|
||||
raise HTTPException(422, {"errors": errors[:50], "error_count": len(errors)})
|
||||
created = updated = 0
|
||||
for e in entries:
|
||||
row = session.scalar(select(FiscalContextEntry).where(FiscalContextEntry.tenant_id == agent.tenant_id, FiscalContextEntry.company_key == company_key,
|
||||
FiscalContextEntry.scope == e["scope"], FiscalContextEntry.scope_key == e["scope_key"], FiscalContextEntry.key == e["key"]))
|
||||
if row:
|
||||
row.value, row.reason, row.updated_at = e["value"], e["reason"], datetime.now(timezone.utc); updated += 1
|
||||
else:
|
||||
session.add(FiscalContextEntry(tenant_id=agent.tenant_id, company_key=company_key, source="OPERATOR", **e)); created += 1
|
||||
session.commit()
|
||||
return {"created": created, "updated": updated}
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/context")
|
||||
def context_list(company_key: str, identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
rows = session.scalars(select(FiscalContextEntry).where(FiscalContextEntry.tenant_id == agent.tenant_id, FiscalContextEntry.company_key == company_key)).all()
|
||||
return {"company_key": company_key, "entries": [{"scope": r.scope, "scope_key": r.scope_key, "key": r.key, "value": r.value, "source": r.source, "reason": r.reason,
|
||||
"created_at": r.created_at.isoformat() if r.created_at else None, "updated_at": r.updated_at.isoformat() if r.updated_at else None} for r in rows]}
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/heartbeat")
|
||||
def heartbeat(payload: dict, identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
try:
|
||||
supplied = sanitize_registration(payload)
|
||||
except (TypeError, ValueError) as error:
|
||||
raise HTTPException(422, f"invalid registration metadata: {error}") from error
|
||||
now = datetime.now(timezone.utc)
|
||||
agent.last_seen_at = now
|
||||
agent.last_status = supplied.get("status") or payload.get("status")
|
||||
if payload.get("schema_fingerprint"):
|
||||
agent.schema_fingerprint = payload["schema_fingerprint"]
|
||||
if payload.get("agent_version"):
|
||||
agent.agent_version = payload["agent_version"]
|
||||
installation_key = supplied.get("installation_id") or str(uuid.uuid4())
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is not None and installation.installation_id != installation_key:
|
||||
raise HTTPException(409, "installation_id conflicts with registered Agent")
|
||||
if installation is None:
|
||||
installation = Installation(agent_id=agent.id, tenant_id=agent.tenant_id, installation_id=installation_key, display_name=supplied.get("display_name") or agent.name,
|
||||
features_configured=True, corrector_enabled=True) # legacy/bootstrap self-registration keeps the pre-existing behaviour
|
||||
session.add(installation); session.flush()
|
||||
fields = {"display_name": supplied.get("display_name"), "customer_code": supplied.get("customer_code"), "notes": supplied.get("notes"), "hostname": supplied.get("hostname"), "windows_version": supplied.get("windows_version"), "machine_fingerprint": supplied.get("machine_fingerprint"), "company_code": supplied.get("company_code"), "company_name": supplied.get("company_name"), "company_cnpj_masked": supplied.get("company_cnpj_masked"), "company_uf": supplied.get("company_uf"), "company_crt": supplied.get("company_crt"), "company_contribuinte_ipi": supplied.get("company_contribuinte_ipi"), "firebird_version": supplied.get("firebird_version"), "firebird_ods": supplied.get("firebird_ods"), "database_identifier": supplied.get("database_identifier"), "capabilities": supplied.get("capabilities"), "fiscal_engine_version": supplied.get("fiscal_engine_version"), "dataset_versions": supplied.get("dataset_versions"), "last_status": supplied.get("status") or payload.get("status"), "last_seen_at": now}
|
||||
for key, value in fields.items():
|
||||
if value is not None: setattr(installation, key, value)
|
||||
if supplied.get("started_at"):
|
||||
try:
|
||||
installation.started_at = datetime.fromisoformat(str(supplied["started_at"]).replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
raise HTTPException(422, "started_at must be ISO-8601")
|
||||
snapshot = session.scalar(select(AgentSnapshot).where(AgentSnapshot.installation_id == installation.id))
|
||||
if snapshot is None:
|
||||
snapshot = AgentSnapshot(installation_id=installation.id)
|
||||
session.add(snapshot)
|
||||
if payload.get("coverage"):
|
||||
coverage = payload["coverage"]
|
||||
snapshot.total_products = int(coverage.get("total_products", coverage.get("total", snapshot.total_products)) or 0)
|
||||
snapshot.fiscally_complete = int(coverage.get("fiscally_complete", coverage.get("complete", snapshot.fiscally_complete)) or 0)
|
||||
snapshot.safe_blocked = int(coverage.get("safe_blocked", snapshot.safe_blocked) or 0)
|
||||
snapshot.failed_products = int(coverage.get("failed_products", snapshot.failed_products) or 0)
|
||||
snapshot.coverage_percent = float(coverage.get("coverage_percent", snapshot.coverage_percent) or 0)
|
||||
snapshot.blockers = coverage.get("blockers", snapshot.blockers) or {}
|
||||
snapshot.version_drift = compute_version_drift(agent.agent_version, reference_agent_version(session))
|
||||
snapshot.agent_health, snapshot.health_reason = compute_agent_health(installation, supplemental=snapshot)
|
||||
session.add(agent); session.flush()
|
||||
record_event(session, tenant_id=agent.tenant_id, installation_id=installation.id, agent_id=agent.id, severity="INFO", event_type="HEARTBEAT", message="Agent heartbeat", context={"status": agent.last_status, "agent_version": agent.agent_version})
|
||||
session.commit()
|
||||
# Central scheduler: exactly one INITIAL_SYNC after the first heartbeat, then
|
||||
# one SYNC_INCREMENTAL per interval — idempotent, no local cron/toggle needed.
|
||||
if agent.enabled:
|
||||
schedule_automatic_sync(session, agent, installation, snapshot)
|
||||
logger.info("heartbeat agent_id=%s status=%s last_job_id=%s", agent.id, agent.last_status, payload.get("last_job_id"))
|
||||
return {"agent_id": agent.id, "installation_id": installation.installation_id, "last_seen_at": agent.last_seen_at.isoformat(), "enabled": agent.enabled,
|
||||
"processing_enabled": bool(installation.processing_enabled), "sync_interval_seconds": installation.sync_interval_seconds}
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/register")
|
||||
def register_agent(payload: dict, identity=Depends(principal)):
|
||||
return heartbeat({**payload, "status": payload.get("status", "IDLE")}, identity)
|
||||
|
||||
@routes.post("/agents/events")
|
||||
def agent_event(payload: dict, identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is None:
|
||||
raise HTTPException(409, "Agent must heartbeat/register before sending events")
|
||||
event = record_event(session, tenant_id=agent.tenant_id, installation_id=installation.id, agent_id=agent.id, severity=payload.get("severity", "INFO"), event_type=payload.get("event_type", "HEARTBEAT"), message=payload.get("message", ""), context=payload.get("context") or {}, job_id=payload.get("job_id"), product_code=payload.get("product_code"), blocker_id=payload.get("blocker_id"))
|
||||
session.commit()
|
||||
return {"event_id": event.id}
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/agents/me/identity")
|
||||
def agent_identity(identity=Depends(principal)):
|
||||
"""Return only the authenticated Agent's existing installation identity."""
|
||||
session, agent = identity
|
||||
try:
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
last_seen = agent.last_seen_at
|
||||
if last_seen is not None and last_seen.tzinfo is None:
|
||||
last_seen = last_seen.replace(tzinfo=timezone.utc)
|
||||
online = last_seen is not None and (datetime.now(timezone.utc) - last_seen).total_seconds() <= settings.agent_online_threshold_seconds
|
||||
return {
|
||||
"agent_id": agent.id,
|
||||
"installation_id": installation.installation_id if installation else None,
|
||||
"hostname": installation.hostname if installation else None,
|
||||
"version": agent.agent_version,
|
||||
"status": "ONLINE" if online else "OFFLINE",
|
||||
}
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@routes.get("/agents/me/authorizations")
|
||||
def agent_authorizations(identity=Depends(principal)):
|
||||
"""Current central RTC write authorization of the authenticated Agent's own installation (read by the Agent right before it writes). Nothing else is exposed."""
|
||||
session, agent = identity
|
||||
try:
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is None:
|
||||
return {"rtc_write_mode": "DISABLED", "cfop_ippt_write_mode": "DISABLED"}
|
||||
return {**job_authorizations(session, installation), **cfop_ippt_job_authorizations(installation)}
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@routes.get("/agents/me")
|
||||
def agent_status(identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
last_seen = agent.last_seen_at
|
||||
if last_seen is not None and last_seen.tzinfo is None:
|
||||
last_seen = last_seen.replace(tzinfo=timezone.utc) # SQLite (tests) drops tz info
|
||||
online = last_seen is not None and (datetime.now(timezone.utc) - last_seen).total_seconds() <= settings.agent_online_threshold_seconds
|
||||
return {
|
||||
"agent_id": agent.id, "company_id": agent.tenant_id, "name": agent.name, "enabled": agent.enabled,
|
||||
"last_seen_at": agent.last_seen_at.isoformat() if agent.last_seen_at else None,
|
||||
"last_status": agent.last_status, "schema_fingerprint": agent.schema_fingerprint,
|
||||
"agent_version": agent.agent_version, "status": "ONLINE" if online else "OFFLINE",
|
||||
}
|
||||
finally: session.close()
|
||||
|
||||
def _blocked_view(row):
|
||||
return {
|
||||
"id": row.id, "tenant_id": row.tenant_id, "company": row.company, "product_code": row.product_code,
|
||||
"installation_db_id": row.installation_id,
|
||||
"gtin": row.gtin, "description": row.description, "blocker_type": row.blocker_type, "field": row.field,
|
||||
"reason_code": row.reason_code, "evidence_summary": row.evidence_summary, "first_seen_at": row.first_seen_at.isoformat() if row.first_seen_at else None,
|
||||
"last_seen_at": row.last_seen_at.isoformat() if row.last_seen_at else None, "retry_count": row.retry_count,
|
||||
"next_retry_at": row.next_retry_at.isoformat() if row.next_retry_at else None, "resolved_at": row.resolved_at.isoformat() if row.resolved_at else None,
|
||||
"resolution_source": row.resolution_source,
|
||||
}
|
||||
|
||||
@routes.get("/blockers")
|
||||
def blockers_list(blocker_type: str | None = None, field: str | None = None, identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
return {"blockers": [_blocked_view(row) for row in list_blockers(session, agent.tenant_id, blocker_type=blocker_type, field=field, installation_id=installation.id if installation else None)]}
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@routes.get("/blockers/export.csv")
|
||||
def blockers_export(identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
output = io.StringIO()
|
||||
writer = csv.writer(output)
|
||||
writer.writerow(["id", "company", "product_code", "gtin", "blocker_type", "field", "reason_code", "retry_count", "first_seen_at", "last_seen_at"])
|
||||
for row in list_blockers(session, agent.tenant_id, installation_id=installation.id if installation else None):
|
||||
writer.writerow([row.id, row.company, row.product_code, row.gtin or "", row.blocker_type, row.field, row.reason_code, row.retry_count, row.first_seen_at.isoformat() if row.first_seen_at else "", row.last_seen_at.isoformat() if row.last_seen_at else ""])
|
||||
return Response(output.getvalue(), media_type="text/csv", headers={"Content-Disposition": "attachment; filename=blocked-products.csv"})
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@routes.post("/blockers/reprocess")
|
||||
def blockers_reprocess(payload: dict, identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
rows = list_blockers(session, agent.tenant_id, blocker_type=payload.get("blocker_type"), field=payload.get("field"), installation_id=installation.id if installation else None)
|
||||
codes = [row.product_code for row in rows if row.product_code]
|
||||
if not codes:
|
||||
return {"created": False, "reason": "NO_DUE_BLOCKERS"}
|
||||
job = create_job(session, agent, "REPROCESS_PRODUCTS", {"codigos": codes, "reason": "BLOCKER_REPROCESS"})
|
||||
return {"created": True, "job_id": job.id, "products": len(codes)}
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@routes.post("/blockers/{blocker_id}/resolve")
|
||||
def blockers_resolve(blocker_id: int, payload: dict, identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
row = session.get(BlockedProduct, blocker_id)
|
||||
if row is None or row.tenant_id != agent.tenant_id:
|
||||
raise HTTPException(404, "blocker not found")
|
||||
if not resolve_blocker(session, blocker_id, payload.get("resolution_source", "OPERATOR")):
|
||||
raise HTTPException(404, "blocker not found")
|
||||
return {"resolved": True, "blocker_id": blocker_id}
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
def _job_view(job):
|
||||
return {
|
||||
"job_id": job.id, "agent_id": job.agent_id, "kind": job.kind, "status": job.status,
|
||||
"payload": job.payload, "created_at": job.created_at.isoformat() if job.created_at else None,
|
||||
"started_at": job.started_at.isoformat() if job.started_at else None,
|
||||
"finished_at": job.finished_at.isoformat() if job.finished_at else None,
|
||||
"attempts": job.attempts, "result_summary": job.result_summary, "error": job.error,
|
||||
}
|
||||
|
||||
def _apply_job_outcome(session, agent, installation, job, result_summary, *, failed):
|
||||
"""After a job finishes: coverage snapshot (from the Agent's operational
|
||||
summary), sync bookkeeping and an audit-friendly event. Never stores secrets."""
|
||||
now = datetime.now(timezone.utc)
|
||||
snapshot = session.scalar(select(AgentSnapshot).where(AgentSnapshot.installation_id == installation.id))
|
||||
if snapshot is None:
|
||||
snapshot = AgentSnapshot(installation_id=installation.id); session.add(snapshot)
|
||||
snapshot.last_job_status = "FAILED" if failed else "COMPLETED"
|
||||
snapshot.last_job_at = now
|
||||
context = {"kind": job.kind}
|
||||
operational = result_summary.get("operational") if not failed else None
|
||||
if operational and job.kind in SYNC_KINDS:
|
||||
total = int(operational.get("TOTAL_PRODUCTS", 0))
|
||||
complete = int(operational.get("COMPLETE", 0)) + int(operational.get("CORRECTED", 0))
|
||||
snapshot.total_products = total
|
||||
snapshot.fiscally_complete = complete
|
||||
snapshot.safe_blocked = int(operational.get("SAFE_BLOCKED", 0))
|
||||
snapshot.failed_products = int(operational.get("FAILED_TECHNICAL", 0))
|
||||
snapshot.coverage_percent = round(100.0 * complete / total, 2) if total else 0.0
|
||||
snapshot.blockers = {state: int(count) for state, count in operational.items() if state.startswith("BLOCKED_") and count}
|
||||
snapshot.last_successful_job_at = now
|
||||
installation.last_sync_at = now
|
||||
installation.last_sync_changed = int(result_summary.get("products_changed", result_summary.get("products_processed", 0)) or 0)
|
||||
context.update({"total_products": total, "products_changed": installation.last_sync_changed, "products_processed": result_summary.get("products_processed")})
|
||||
if failed:
|
||||
context["error"] = result_summary.get("error")
|
||||
record_event(session, tenant_id=agent.tenant_id, installation_id=installation.id, agent_id=agent.id, severity="ERROR" if failed else "INFO",
|
||||
event_type="JOB_FAILED" if failed else "JOB_COMPLETED", message=f"Job {job.kind} {'failed' if failed else 'completed'}", context=context, job_id=job.id)
|
||||
session.commit()
|
||||
|
||||
@routes.post("/jobs")
|
||||
def create_job_route(payload: dict, identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
kind = payload.get("kind")
|
||||
if kind not in ("REPROCESS_ALL", "REPROCESS_PRODUCTS"):
|
||||
raise HTTPException(400, "kind must be REPROCESS_ALL or REPROCESS_PRODUCTS")
|
||||
job_payload = payload.get("payload", {})
|
||||
if kind == "REPROCESS_PRODUCTS" and not job_payload.get("codigos"):
|
||||
raise HTTPException(400, "REPROCESS_PRODUCTS requires payload.codigos")
|
||||
job = create_job(session, agent, kind, job_payload)
|
||||
logger.info("job created job_id=%s agent_id=%s kind=%s", job.id, agent.id, kind)
|
||||
return _job_view(job)
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/jobs/next")
|
||||
def next_job(identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
job = claim_next_job(session, agent)
|
||||
if job is None: return None
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is not None:
|
||||
record_event(session, tenant_id=agent.tenant_id, installation_id=installation.id, agent_id=agent.id, severity="INFO", event_type="JOB_STARTED", message=f"Job {job.kind} started", context={"kind": job.kind}, job_id=job.id)
|
||||
session.commit()
|
||||
logger.info("job claimed job_id=%s agent_id=%s kind=%s", job.id, agent.id, job.kind)
|
||||
view = _job_view(job)
|
||||
view["authorizations"] = ({**job_authorizations(session, installation), **cfop_ippt_job_authorizations(installation)} if installation is not None
|
||||
else {"rtc_write_mode": "DISABLED", "cfop_ippt_write_mode": "DISABLED"}) # decided centrally, per installation, default OFF
|
||||
return view
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/jobs/global")
|
||||
def global_jobs(request: Request, page: int = 1, page_size: int = 50, status: str | None = None, kind: str | None = None):
|
||||
session = session_factory()
|
||||
try:
|
||||
require_human(request, session)
|
||||
return list_jobs(session, page=page, page_size=page_size, status=status, kind=kind)
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@routes.get("/jobs/{job_id}")
|
||||
def get_job(job_id: int, identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
job = session.get(Job, job_id)
|
||||
if job is None or job.agent_id != agent.id: raise HTTPException(404, "job not found")
|
||||
return _job_view(job)
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/jobs/{job_id}/complete")
|
||||
def complete_job_route(job_id: int, payload: dict, identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
job = session.get(Job, job_id)
|
||||
if job is None or job.agent_id != agent.id: raise HTTPException(404, "job not found")
|
||||
result_summary = payload.get("result_summary", {})
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
pass_started = datetime.now(timezone.utc)
|
||||
upsert_blockers_bulk(session, agent.tenant_id, list(result_summary.get("blocked_products", [])), installation_id=installation.id if installation else None)
|
||||
for product_code in result_summary.get("resolved_products", []):
|
||||
resolve_product_blockers(session, agent.tenant_id, product_code, installation_id=installation.id if installation else None)
|
||||
if installation is not None:
|
||||
record_rtc_write_audits(session, tenant_id=agent.tenant_id, installation=installation, agent=agent, job_id=job_id, rtc_auto_apply=result_summary.get("rtc_auto_apply"), evaluated_all=bool((result_summary.get("fiscal_context") or {}).get("evaluated_all")))
|
||||
record_cfop_ippt_write_audits(session, tenant_id=agent.tenant_id, installation=installation, agent=agent, job_id=job_id, cfop_ippt_apply=result_summary.get("cfop_ippt_apply"))
|
||||
if installation is not None and (result_summary.get("fiscal_context") or {}).get("evaluated_all"):
|
||||
resolve_stale_fiscal_blockers(session, agent.tenant_id, installation.id, pass_started)
|
||||
if not complete_job(session, job, result_summary): raise HTTPException(409, "job is not RUNNING")
|
||||
if installation is not None:
|
||||
_apply_job_outcome(session, agent, installation, job, result_summary, failed=False)
|
||||
logger.info("job completed job_id=%s agent_id=%s result=%s", job_id, agent.id, payload.get("result_summary", {}))
|
||||
return _job_view(session.get(Job, job_id))
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/jobs/{job_id}/fail")
|
||||
def fail_job_route(job_id: int, payload: dict, identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
job = session.get(Job, job_id)
|
||||
if job is None or job.agent_id != agent.id: raise HTTPException(404, "job not found")
|
||||
if not fail_job(session, job, payload.get("error", "unknown error")): raise HTTPException(409, "job is not RUNNING")
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is not None:
|
||||
_apply_job_outcome(session, agent, installation, job, {"error": str(payload.get("error", "unknown error"))[:500]}, failed=True)
|
||||
logger.warning("job failed job_id=%s agent_id=%s error=%s", job_id, agent.id, payload.get("error", "unknown error"))
|
||||
return _job_view(session.get(Job, job_id))
|
||||
finally: session.close()
|
||||
|
||||
routes.include_router(enrichment_router(principal))
|
||||
return routes
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Immutable audit events for human control-plane actions."""
|
||||
|
||||
from .events import sanitize_event
|
||||
from .models import AuditEvent
|
||||
|
||||
|
||||
def record_audit(session, *, admin_user_id: int | None, action: str, target_type: str, target_id: str, context: dict | None = None, tenant_id: int | None = None, severity: str = "INFO"):
|
||||
_, safe_context = sanitize_event(action, context or {})
|
||||
event = AuditEvent(admin_user_id=admin_user_id, tenant_id=tenant_id, action=action, target_type=target_type, target_id=str(target_id), context=safe_context, severity=severity)
|
||||
session.add(event)
|
||||
return event
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Bearer authentication storing only SHA-256 token digests."""
|
||||
import hashlib, hmac
|
||||
from sqlalchemy import select
|
||||
from .models import Agent, Tenant
|
||||
|
||||
def token_hash(token): return hashlib.sha256(token.encode()).hexdigest()
|
||||
def bootstrap(session, token):
|
||||
digest = token_hash(token); agent = session.scalar(select(Agent).where(Agent.token_hash == digest))
|
||||
if agent: return agent
|
||||
tenant = session.scalar(select(Tenant).where(Tenant.name == "development")) or Tenant(name="development")
|
||||
session.add(tenant); session.flush(); agent = Agent(tenant_id=tenant.id, name="development-agent", token_hash=digest); session.add(agent); session.commit(); return agent
|
||||
def authenticate(session, token):
|
||||
digest = token_hash(token)
|
||||
for agent in session.scalars(select(Agent)).all():
|
||||
if hmac.compare_digest(agent.token_hash, digest):
|
||||
return agent if agent.enabled else None # a disabled (revoked) Agent must not authenticate
|
||||
return None
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Central blocked-product registry with fingerprint deduplication."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from sqlalchemy import select
|
||||
|
||||
from .models import BlockedProduct
|
||||
from .operational_state import blocker_fingerprint
|
||||
|
||||
|
||||
def _fingerprint_of(tenant_id: int, payload: dict, installation_id: int | None) -> str:
|
||||
supplied_fingerprint = payload.get("blocker_fingerprint")
|
||||
if installation_id is None:
|
||||
return supplied_fingerprint or blocker_fingerprint(payload.get("tenant", str(tenant_id)), payload.get("company", ""), payload.get("product_code", ""), payload.get("field", ""), payload.get("reason_code", ""), payload.get("input_fingerprint", ""))
|
||||
elif supplied_fingerprint:
|
||||
return blocker_fingerprint(supplied_fingerprint, "", "", "", "", "", installation_id=installation_id)
|
||||
else:
|
||||
return blocker_fingerprint(payload.get("tenant", str(tenant_id)), payload.get("company", ""), payload.get("product_code", ""), payload.get("field", ""), payload.get("reason_code", ""), payload.get("input_fingerprint", ""), installation_id=installation_id)
|
||||
|
||||
|
||||
def upsert_blocker(session, tenant_id: int, payload: dict, *, installation_id: int | None = None, commit: bool = True, existing: dict | None = None) -> BlockedProduct:
|
||||
"""`existing` ({fingerprint: row}) and `commit=False` let a caller upsert thousands of blockers with one lookup per chunk and one commit (see upsert_blockers_bulk)."""
|
||||
fingerprint = _fingerprint_of(tenant_id, payload, installation_id)
|
||||
row = existing.get(fingerprint) if existing is not None else session.scalar(select(BlockedProduct).where(BlockedProduct.blocker_fingerprint == fingerprint))
|
||||
now = datetime.now(timezone.utc)
|
||||
if row is None:
|
||||
row = BlockedProduct(tenant_id=tenant_id, installation_id=installation_id, company=payload.get("company", ""), product_code=str(payload.get("product_code", "")), gtin=payload.get("gtin"), description=payload.get("description"), blocker_type=payload.get("blocker_type", "BLOCKED_MISSING_EVIDENCE"), field=payload.get("field", ""), reason_code=payload.get("reason_code", ""), evidence_summary=payload.get("evidence_summary") or {}, input_fingerprint=payload.get("input_fingerprint", ""), blocker_fingerprint=fingerprint, first_seen_at=now, last_seen_at=now)
|
||||
session.add(row)
|
||||
if existing is not None:
|
||||
existing[fingerprint] = row
|
||||
else:
|
||||
row.last_seen_at = now
|
||||
row.evidence_summary = payload.get("evidence_summary") or row.evidence_summary
|
||||
row.input_fingerprint = payload.get("input_fingerprint", row.input_fingerprint)
|
||||
if row.installation_id is None and installation_id is not None:
|
||||
row.installation_id = installation_id
|
||||
row.resolved_at = None
|
||||
if commit:
|
||||
session.commit()
|
||||
return row
|
||||
|
||||
|
||||
BULK_CHUNK = 1000
|
||||
|
||||
|
||||
def upsert_blockers_bulk(session, tenant_id: int, payloads: list, *, installation_id: int | None = None) -> int:
|
||||
"""Same semantics as `upsert_blocker` per item, but O(chunks) statements instead of O(products): one IN-lookup per chunk, one commit at the end."""
|
||||
n = 0
|
||||
for start in range(0, len(payloads), BULK_CHUNK):
|
||||
chunk = [p for p in payloads[start:start + BULK_CHUNK] if isinstance(p, dict)]
|
||||
prints = {_fingerprint_of(tenant_id, p, installation_id) for p in chunk}
|
||||
existing = {r.blocker_fingerprint: r for r in session.scalars(select(BlockedProduct).where(BlockedProduct.blocker_fingerprint.in_(prints)))} if prints else {}
|
||||
for payload in chunk:
|
||||
upsert_blocker(session, tenant_id, payload, installation_id=installation_id, commit=False, existing=existing)
|
||||
n += 1
|
||||
session.flush()
|
||||
session.commit()
|
||||
return n
|
||||
|
||||
|
||||
def list_blockers(session, tenant_id: int, *, input_fingerprint: str | None = None, blocker_type: str | None = None, field: str | None = None, installation_id: int | None = None):
|
||||
query = select(BlockedProduct).where(BlockedProduct.tenant_id == tenant_id, BlockedProduct.resolved_at.is_(None))
|
||||
if installation_id is not None:
|
||||
query = query.where((BlockedProduct.installation_id == installation_id) | BlockedProduct.installation_id.is_(None))
|
||||
if input_fingerprint is not None:
|
||||
query = query.where(BlockedProduct.input_fingerprint != input_fingerprint)
|
||||
if blocker_type:
|
||||
query = query.where(BlockedProduct.blocker_type == blocker_type)
|
||||
if field:
|
||||
query = query.where(BlockedProduct.field == field)
|
||||
return list(session.scalars(query).all())
|
||||
|
||||
|
||||
def retry_due(row: BlockedProduct, current_input_fingerprint: str, now: datetime | None = None) -> bool:
|
||||
now = now or datetime.now(timezone.utc)
|
||||
if row.input_fingerprint != current_input_fingerprint:
|
||||
return True
|
||||
return row.next_retry_at is not None and row.next_retry_at <= now
|
||||
|
||||
|
||||
def resolve_blocker(session, blocker_id: int, source: str) -> bool:
|
||||
row = session.get(BlockedProduct, blocker_id)
|
||||
if row is None:
|
||||
return False
|
||||
row.resolved_at = datetime.now(timezone.utc)
|
||||
row.resolution_source = source
|
||||
session.commit()
|
||||
return True
|
||||
|
||||
|
||||
def resolve_product_blockers(session, tenant_id: int, product_code: str, source: str = "AUTO_REEVALUATION", *, installation_id: int | None = None) -> int:
|
||||
query = select(BlockedProduct).where(BlockedProduct.tenant_id == tenant_id, BlockedProduct.product_code == str(product_code), BlockedProduct.resolved_at.is_(None))
|
||||
if installation_id is not None:
|
||||
query = query.where((BlockedProduct.installation_id == installation_id) | BlockedProduct.installation_id.is_(None))
|
||||
rows = session.scalars(query).all()
|
||||
now = datetime.now(timezone.utc)
|
||||
for row in rows:
|
||||
row.resolved_at = now
|
||||
row.resolution_source = source
|
||||
if rows:
|
||||
session.commit()
|
||||
return len(rows)
|
||||
|
||||
|
||||
FISCAL_BLOCKER_FIELDS = ("PIS", "COFINS", "RTC")
|
||||
|
||||
|
||||
def resolve_stale_fiscal_blockers(session, tenant_id: int, installation_id: int, since: datetime) -> int:
|
||||
"""After a FULL evaluation of the unified fiscal fields, every open PIS/COFINS/RTC blocker of the installation that this pass did not
|
||||
re-report (last_seen_at < since) no longer holds: the ERP data changed, or RTC was switched off. Other blocker fields are untouched."""
|
||||
rows = session.scalars(select(BlockedProduct).where(BlockedProduct.tenant_id == tenant_id, BlockedProduct.installation_id == installation_id,
|
||||
BlockedProduct.resolved_at.is_(None), BlockedProduct.field.in_(FISCAL_BLOCKER_FIELDS), BlockedProduct.last_seen_at < since)).all()
|
||||
now = datetime.now(timezone.utc)
|
||||
for row in rows:
|
||||
row.resolved_at = now
|
||||
row.resolution_source = "AUTO_REEVALUATION"
|
||||
if rows:
|
||||
session.commit()
|
||||
return len(rows)
|
||||
@@ -0,0 +1,168 @@
|
||||
"""Validated, auditable context resolution for a single blocked product."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from .audit import record_audit
|
||||
from .context_keys import validate_entry
|
||||
from .models import Agent, BlockedProduct, FiscalContextEntry, Installation, Tenant, Job
|
||||
|
||||
|
||||
WORKFLOW_SCOPES = {"PRODUCT", "GROUP", "COMPANY"}
|
||||
ENTRY_FIELDS = {"scope", "scope_key", "key", "value", "reason"}
|
||||
MAX_VALUE_LENGTH = 512
|
||||
MAX_REASON_LENGTH = 2000
|
||||
MAX_COMPANY_KEY_LENGTH = 64
|
||||
|
||||
|
||||
class BlockerResolutionUnavailable(LookupError):
|
||||
"""The blocker exists, but has no proven installation to target safely."""
|
||||
|
||||
|
||||
def validate_resolution_input(payload):
|
||||
"""Validate only the structured context accepted by the blocker workflow."""
|
||||
if not isinstance(payload, dict):
|
||||
return None, "resolution input must be an object"
|
||||
unexpected = set(payload) - ENTRY_FIELDS
|
||||
if unexpected:
|
||||
return None, f"unsupported resolution fields: {sorted(unexpected)}"
|
||||
if not isinstance(payload.get("value"), str):
|
||||
return None, "value must be a string"
|
||||
if not isinstance(payload.get("reason"), str):
|
||||
return None, "reason must be a string"
|
||||
if len(payload["value"].encode("utf-8")) > MAX_VALUE_LENGTH:
|
||||
return None, f"value must be at most {MAX_VALUE_LENGTH} UTF-8 bytes"
|
||||
if len(payload["reason"].encode("utf-8")) > MAX_REASON_LENGTH:
|
||||
return None, f"reason must be at most {MAX_REASON_LENGTH} UTF-8 bytes"
|
||||
|
||||
normalized, error = validate_entry(payload)
|
||||
if error:
|
||||
return None, error
|
||||
if normalized["scope"] not in WORKFLOW_SCOPES:
|
||||
return None, "scope must be one of ('PRODUCT', 'GROUP', 'COMPANY')"
|
||||
if not normalized["reason"]:
|
||||
return None, "reason is required"
|
||||
return normalized, None
|
||||
|
||||
|
||||
def _company_key(blocker, installation, tenant):
|
||||
if installation is not None:
|
||||
return (installation.company_code or installation.company_name or installation.customer_code or "").strip()
|
||||
return (blocker.company or (tenant.name if tenant else "")).strip()
|
||||
|
||||
|
||||
def _entry_view(row):
|
||||
if row is None:
|
||||
return None
|
||||
return {
|
||||
"company_key": row.company_key,
|
||||
"scope": row.scope,
|
||||
"scope_key": row.scope_key,
|
||||
"key": row.key,
|
||||
"value": row.value,
|
||||
"source": row.source,
|
||||
"reason": row.reason,
|
||||
}
|
||||
|
||||
|
||||
def resolve_blocker_with_context(session, blocker_id, admin, payload, request_ip=None):
|
||||
"""Persist context, enqueue one product reprocess, and audit atomically."""
|
||||
normalized, error = validate_resolution_input(payload)
|
||||
if error:
|
||||
raise ValueError(error)
|
||||
|
||||
try:
|
||||
blocker = session.get(BlockedProduct, blocker_id)
|
||||
if blocker is None:
|
||||
raise LookupError("blocker not found")
|
||||
|
||||
installation = None
|
||||
if blocker.installation_id is not None:
|
||||
installation = session.get(Installation, blocker.installation_id)
|
||||
if installation is None:
|
||||
raise LookupError("blocker installation not found")
|
||||
if installation.tenant_id != blocker.tenant_id:
|
||||
raise LookupError("blocker installation tenant mismatch")
|
||||
agent = session.get(Agent, installation.agent_id)
|
||||
if agent is None:
|
||||
raise LookupError("blocker installation agent not found")
|
||||
if agent.tenant_id != blocker.tenant_id:
|
||||
raise LookupError("blocker agent tenant mismatch")
|
||||
else:
|
||||
raise BlockerResolutionUnavailable(
|
||||
"Blocker legado de empresa sem vínculo comprovado com um servidor; não é possível oferecer o reprocessamento de contexto com segurança."
|
||||
)
|
||||
|
||||
tenant = session.get(Tenant, blocker.tenant_id)
|
||||
company_key = _company_key(blocker, installation, tenant)
|
||||
if not company_key:
|
||||
raise ValueError("company context is unavailable")
|
||||
if len(company_key) > MAX_COMPANY_KEY_LENGTH:
|
||||
raise ValueError(f"company key must be at most {MAX_COMPANY_KEY_LENGTH} characters")
|
||||
if normalized["scope"] == "PRODUCT" and normalized["scope_key"] != blocker.product_code:
|
||||
raise ValueError("PRODUCT scope_key must match the blocker product code")
|
||||
|
||||
query = select(FiscalContextEntry).where(
|
||||
FiscalContextEntry.tenant_id == blocker.tenant_id,
|
||||
FiscalContextEntry.company_key == company_key,
|
||||
FiscalContextEntry.scope == normalized["scope"],
|
||||
FiscalContextEntry.scope_key == normalized["scope_key"],
|
||||
FiscalContextEntry.key == normalized["key"],
|
||||
)
|
||||
entry = session.scalar(query)
|
||||
before = _entry_view(entry)
|
||||
now = datetime.now(timezone.utc)
|
||||
if entry is None:
|
||||
entry = FiscalContextEntry(
|
||||
tenant_id=blocker.tenant_id,
|
||||
company_key=company_key,
|
||||
source="OPERATOR",
|
||||
updated_at=now,
|
||||
**normalized,
|
||||
)
|
||||
session.add(entry)
|
||||
else:
|
||||
entry.value = normalized["value"]
|
||||
entry.reason = normalized["reason"]
|
||||
entry.source = "OPERATOR"
|
||||
entry.updated_at = now
|
||||
|
||||
job = Job(agent_id=agent.id, kind="REPROCESS_PRODUCTS", status="PENDING", payload={"codigos": [blocker.product_code]})
|
||||
session.add(job)
|
||||
session.flush()
|
||||
audit = record_audit(
|
||||
session,
|
||||
admin_user_id=admin.id,
|
||||
tenant_id=blocker.tenant_id,
|
||||
action="RESOLVE_BLOCKER_WITH_CONTEXT",
|
||||
target_type="blocker",
|
||||
target_id=str(blocker.id),
|
||||
context={
|
||||
"admin_user_id": admin.id,
|
||||
"tenant_id": blocker.tenant_id,
|
||||
"client": tenant.name if tenant else None,
|
||||
"company_key": company_key,
|
||||
"installation_id": installation.installation_id if installation else None,
|
||||
"installation_db_id": installation.id if installation else None,
|
||||
"product_code": blocker.product_code,
|
||||
"blocker_id": blocker.id,
|
||||
"key": normalized["key"],
|
||||
"scope": normalized["scope"],
|
||||
"scope_key": normalized["scope_key"],
|
||||
"value": normalized["value"],
|
||||
"reason": normalized["reason"],
|
||||
"before": before,
|
||||
"after": _entry_view(entry),
|
||||
"reprocess_job_id": job.id,
|
||||
"request_ip": request_ip,
|
||||
},
|
||||
)
|
||||
session.flush()
|
||||
session.commit()
|
||||
return {"blocker_id": blocker.id, "job_id": job.id, "audit_id": audit.id}
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
@@ -0,0 +1,274 @@
|
||||
"""Deterministic, auditable cClassTrib resolver (pure: no Firebird, no network, no probabilities).
|
||||
|
||||
resolve_cclasstrib(product, company, ctx, local_catalog) -> Resolution
|
||||
|
||||
Nothing here guesses. A class is proposed only when an EXPLICIT rule, backed by the official texts in `rtc_reference_data`
|
||||
(cClassTrib table 2026-06-22 and LC 214/2025 compiled through LC 227/2026), proves it. Everything else is AMBIGUOUS or INSUFFICIENT_EVIDENCE.
|
||||
|
||||
Operation assumption (documented, product-level default): an ordinary domestic sale of the good. Operation-dependent classes (buyer is a public body,
|
||||
exports, ZFM/ALC, credit presumed, ...) cannot be carried by a product default; when the annex offers such alternatives the answer is AMBIGUOUS.
|
||||
|
||||
Rule classes (what a rule is ALLOWED to do)
|
||||
AUTO_APPLY_SAFE positive, explicit evidence: may be written to the ERP (only when every runtime gate also holds)
|
||||
VALIDATION_ONLY used to validate/explain, never proposes a value
|
||||
SUGGESTION_ONLY diagnostic suggestion shown to a human; NEVER authorizes an UPDATE
|
||||
BLOCKED_BY_EVIDENCE the evidence is missing/ambiguous: safe-block with a human reason
|
||||
|
||||
Rules
|
||||
RTC-R0 RTC disabled in the ERP VALIDATION_ONLY -> NOT_APPLICABLE
|
||||
RTC-G1 local RTC catalog failed integrity BLOCKED_BY_EVIDENCE -> INSUFFICIENT_EVIDENCE
|
||||
RTC-R1 regime without an official rule (Simples family, unknown) BLOCKED_BY_EVIDENCE -> INSUFFICIENT_EVIDENCE
|
||||
RTC-R2 no valid 8-digit NCM (decision engine did not confirm) BLOCKED_BY_EVIDENCE -> INSUFFICIENT_EVIDENCE
|
||||
RTC-A1 annex without a unique product-default class / conditional text / revoked annex -> AMBIGUOUS
|
||||
RTC-A2 NCM listed in more than one annex -> AMBIGUOUS
|
||||
RTC-A3 chapter whose regime depends on the nature/operation of the good -> AMBIGUOUS
|
||||
RTC-R3 NCM listed in EXACTLY ONE annex with a unique default class, condition-free item, regime normal (CRT 3)
|
||||
AUTO_APPLY_SAFE -> RESOLVED_WITH_EXPLICIT_RULE
|
||||
RTC-R4 NCM in NO annex and in no nature-based chapter: full taxation is the residual reading of LC 214 art. 4
|
||||
SUGGESTION_ONLY -> SUGGESTION_ONLY (000001). Absence of a special regime is NOT positive evidence: never written.
|
||||
RTC-G2 proposed class missing/different in the local SGV catalog, out of force or unusable in NF-e/NFC-e BLOCKED_BY_EVIDENCE -> INSUFFICIENT_EVIDENCE (RTC_LOCAL_CATALOG_INCOMPATIBLE)
|
||||
A proposed class must additionally be MATCH_SAFE against the official table (local SGV catalog confirms every attribute that matters), in force today, usable in
|
||||
NF-e/NFC-e (the SGV's own lookup filter) and carry a CST that exists; otherwise the answer degrades with the exact reason.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date
|
||||
|
||||
from . import rtc_reference_data as ref
|
||||
|
||||
SIMPLES_FAMILY = frozenset({"1", "2", "4"})
|
||||
RESOLVED_STATUSES = ("RESOLVED_DETERMINISTIC", "RESOLVED_WITH_EXPLICIT_RULE")
|
||||
STATUSES = RESOLVED_STATUSES + ("SUGGESTION_ONLY", "AMBIGUOUS", "INSUFFICIENT_EVIDENCE", "NOT_APPLICABLE")
|
||||
RULE_VERSION = "rtc-rules-v2"
|
||||
AUTO_APPLY_SAFE, VALIDATION_ONLY, SUGGESTION_ONLY, BLOCKED_BY_EVIDENCE = "AUTO_APPLY_SAFE", "VALIDATION_ONLY", "SUGGESTION_ONLY", "BLOCKED_BY_EVIDENCE"
|
||||
RULE_CLASSES = {"RTC-R0": VALIDATION_ONLY, "RTC-G1": BLOCKED_BY_EVIDENCE, "RTC-R1": BLOCKED_BY_EVIDENCE, "RTC-R2": BLOCKED_BY_EVIDENCE, "RTC-A1": BLOCKED_BY_EVIDENCE, "RTC-A2": BLOCKED_BY_EVIDENCE,
|
||||
"RTC-A3": BLOCKED_BY_EVIDENCE, "RTC-R3": AUTO_APPLY_SAFE, "RTC-R4": SUGGESTION_ONLY, "RTC-G2": BLOCKED_BY_EVIDENCE}
|
||||
HUMAN_REASONS = {"SIMPLES_ORIENTATION_PENDING_OFFICIAL_NT": "Enquadramento RTC depende de regra ainda não determinável com segurança.", "COMPANY_REGIME_UNKNOWN": "Regime tributário da empresa não determinado."}
|
||||
|
||||
# Goods annexes whose product-default class is unique in the official table (verified by `verify_annex_class_map`).
|
||||
ANNEX_DEFAULT_CLASS = {"I": "200003", "VII": "200034", "VIII": "200035", "XV": "200014"} # IX is out: 515001 (deferral with the same inputs) is a sibling class
|
||||
# Chapters where a reduced/special regime is defined by the NATURE of the good (not by a code list): in natura agriculture (art. 137), medicines (arts. 133/146),
|
||||
# fuels and biofuels (art. 172 ff), medical/accessibility devices and vehicles (arts. 131-132, 143-145, 149), books/periodicals (constitutional immunity),
|
||||
# gold and other special-regime goods kept out of the residual rule on purpose (conservative).
|
||||
SENSITIVE_CHAPTERS = frozenset([f"{n:02d}" for n in range(1, 15)] + ["27", "30", "49", "71", "87", "88", "89", "90"]) # 71 gold (financial-asset regime), 88/89 aircraft and vessels
|
||||
SPECIAL_PREFIXES = ("9619", "2207", "3826", "8703") # art. 147 (menstrual health), biofuel inputs, art. 149 (PcD cars)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Resolution:
|
||||
status: str
|
||||
rule_id: str
|
||||
reason: str
|
||||
resolved_cclasstrib: str | None = None
|
||||
resolved_cst_ibscbs: str | None = None
|
||||
confidence_type: str = "NONE"
|
||||
inputs_used: dict = field(default_factory=dict)
|
||||
evidence: list = field(default_factory=list)
|
||||
source_references: list = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def rule_class(self) -> str:
|
||||
return RULE_CLASSES.get(self.rule_id, BLOCKED_BY_EVIDENCE)
|
||||
|
||||
@property
|
||||
def auto_apply_candidate(self) -> bool:
|
||||
"""Only an AUTO_APPLY_SAFE rule with a proven class may ever become a write. Suggestions never do."""
|
||||
return self.rule_class == AUTO_APPLY_SAFE and self.status in RESOLVED_STATUSES and bool(self.resolved_cclasstrib)
|
||||
|
||||
@property
|
||||
def suggested_cclasstrib(self) -> str | None:
|
||||
return self.resolved_cclasstrib if self.status == "SUGGESTION_ONLY" else None
|
||||
|
||||
def as_dict(self) -> dict:
|
||||
return {"status": self.status, "rule_id": self.rule_id, "rule_class": self.rule_class, "rule_version": RULE_VERSION, "reason": self.reason, "resolved_cclasstrib": self.resolved_cclasstrib,
|
||||
"resolved_cst_ibscbs": self.resolved_cst_ibscbs, "confidence_type": self.confidence_type, "inputs_used": self.inputs_used, "evidence": self.evidence,
|
||||
"source_references": self.source_references, "auto_apply_candidate": self.auto_apply_candidate}
|
||||
|
||||
|
||||
def _digits(value) -> str:
|
||||
return re.sub(r"\D", "", "" if value is None else str(value))
|
||||
|
||||
|
||||
def _official_by_code() -> dict:
|
||||
return {row["code"]: row for row in ref.OFFICIAL_CCLASSTRIB}
|
||||
|
||||
|
||||
def _num(value):
|
||||
try:
|
||||
return float(str(value).replace(",", "."))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------- catalog comparison
|
||||
|
||||
|
||||
def compare_catalogs(local_rows: dict, today: date | None = None) -> dict:
|
||||
"""local_rows: {code: {cst, tipo_aliquota, p_red_ibs, p_red_cbs, ind_nfe, ind_nfce, start, end}} read from the ERP. Never writes anywhere.
|
||||
Per code (the official table is the reference):
|
||||
MATCH_SAFE every attribute the local table carries that matters to a product default agrees (CST, reductions, NF-e/NFC-e usability)
|
||||
MATCH_WITH_NON_MATERIAL_DIFFERENCE only a scheduled local end date (class still in force today) differs
|
||||
MISMATCH CST, reductions or NF-e/NFC-e usability differ
|
||||
EXPIRED the official class ended
|
||||
MISSING_LOCAL official class absent from the SGV catalog (local outdated)
|
||||
MISSING_OFFICIAL local class the official table does not know"""
|
||||
today = today or date.today()
|
||||
official = _official_by_code()
|
||||
result, details = {}, {}
|
||||
for code in sorted(set(local_rows) | set(official)):
|
||||
loc, off = local_rows.get(code), official.get(code)
|
||||
if off is None:
|
||||
result[code] = "MISSING_OFFICIAL"
|
||||
elif off.get("end") and date.fromisoformat(off["end"]) < today:
|
||||
result[code] = "EXPIRED"
|
||||
details[code] = ["OFFICIAL_END_PASSED"] + (["PRESENT_LOCALLY"] if loc is not None else [])
|
||||
elif loc is None:
|
||||
result[code] = "MISSING_LOCAL"
|
||||
else:
|
||||
problems, soft, info = [], [], []
|
||||
if str(loc.get("cst") or "").strip() != off["cst"]:
|
||||
problems.append("CST")
|
||||
if "p_red_ibs" in loc and "p_red_cbs" in loc and (_num(loc.get("p_red_ibs")) != _num(off["p_red_ibs"]) or _num(loc.get("p_red_cbs")) != _num(off["p_red_cbs"])):
|
||||
problems.append("REDUCTION") # only attributes the local table actually carries are compared
|
||||
if "ind_nfe" in loc and "ind_nfce" in loc and (bool(loc.get("ind_nfe")) != off["ind_nfe"] or bool(loc.get("ind_nfce")) != off["ind_nfce"]):
|
||||
problems.append("DOCUMENT_FLAGS")
|
||||
start, end = str(loc.get("start") or "")[:10], str(loc.get("end") or "")[:10]
|
||||
if start and date.fromisoformat(start) > today:
|
||||
problems.append("LOCAL_NOT_IN_FORCE_YET")
|
||||
elif start and off.get("start") and start != off["start"]:
|
||||
info.append("VIGENCY_START_DIFFERS_BOTH_IN_FORCE") # informational: the class is in force today on both sides, so the emission effect is identical
|
||||
if end and date.fromisoformat(end) < today:
|
||||
problems.append("LOCAL_CLASS_ENDED")
|
||||
elif end:
|
||||
soft.append("LOCAL_END_SCHEDULED") # the class still works today; the local table schedules an end the official one does not
|
||||
result[code] = "MISMATCH" if problems else ("MATCH_WITH_NON_MATERIAL_DIFFERENCE" if soft else "MATCH_SAFE")
|
||||
if problems or soft or info:
|
||||
details[code] = problems + soft + info
|
||||
keys = ("MATCH_SAFE", "MATCH_WITH_NON_MATERIAL_DIFFERENCE", "MISMATCH", "EXPIRED", "MISSING_LOCAL", "MISSING_OFFICIAL")
|
||||
counts = {k: sum(1 for v in result.values() if v == k) for k in keys}
|
||||
material = counts["MISMATCH"] + counts["MISSING_OFFICIAL"] # a local class the official table contradicts or does not know
|
||||
expired_present = sum(1 for d in details.values() if "PRESENT_LOCALLY" in d)
|
||||
compat = "FULL" if counts["MISSING_LOCAL"] == 0 and expired_present == 0 and material == 0 else ("PARTIAL_LOCAL_OUTDATED" if material == 0 else "MATERIAL_MISMATCH")
|
||||
return {"per_code": result, "details": details, "counts": counts, "expired_present_locally": expired_present, "compatibility": compat,
|
||||
"official_catalog": {"sha256": ref.OFFICIAL_CATALOG["sha256"], "publication": ref.OFFICIAL_CATALOG["publication"], "file": ref.OFFICIAL_CATALOG["file"]}}
|
||||
|
||||
|
||||
def local_catalog_hash(local_rows: dict) -> str:
|
||||
canon = json.dumps({c: {k: (None if v is None else str(v)) for k, v in sorted(r.items())} for c, r in sorted(local_rows.items())}, sort_keys=True)
|
||||
return hashlib.sha256(canon.encode()).hexdigest()
|
||||
|
||||
|
||||
def verify_annex_class_map() -> dict:
|
||||
"""Self-check of ANNEX_DEFAULT_CLASS against the official table: the class must exist, be a goods class in force, and be the ONLY official class for that annex
|
||||
usable in NF-e/NFC-e as the ordinary-sale default. Returns {annex: [problems]} (empty lists = verified)."""
|
||||
official = _official_by_code()
|
||||
numbers = {"I": "1", "VII": "7", "VIII": "8", "XV": "15"}
|
||||
out = {}
|
||||
for annex, code in ANNEX_DEFAULT_CLASS.items():
|
||||
row, problems = official.get(code), []
|
||||
if row is None:
|
||||
problems.append("CLASS_NOT_IN_OFFICIAL_TABLE")
|
||||
else:
|
||||
if str(row.get("anexo")) != numbers[annex]:
|
||||
problems.append("ANNEX_FIELD_DIFFERS")
|
||||
if not (row["ind_nfe"] or row["ind_nfce"]):
|
||||
problems.append("NOT_USABLE_IN_NFE_NFCE")
|
||||
siblings = [r["code"] for r in ref.OFFICIAL_CCLASSTRIB if str(r.get("anexo")) == numbers[annex] and r["code"] != code and (r["ind_nfe"] or r["ind_nfce"])]
|
||||
if siblings:
|
||||
problems.append("SIBLING_CLASSES:" + ",".join(siblings))
|
||||
out[annex] = problems
|
||||
return out
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------- annex lookup
|
||||
|
||||
|
||||
def annex_matches(ncm: str) -> list[dict]:
|
||||
"""Every annex item (any annex, revoked included) whose code list covers the NCM, with the item text and whether it is a clean (condition-free) item."""
|
||||
hits = []
|
||||
for roman, annex in ref.LC214_ANNEXES.items():
|
||||
for item in annex["items"]:
|
||||
for prefix in item["prefixes"]:
|
||||
if ncm.startswith(prefix):
|
||||
hits.append({"annex": roman, "item": item["item"], "prefix": prefix, "clean": item["clean"], "revoked": annex["revoked"], "text": item["description"][:160]})
|
||||
break
|
||||
return hits
|
||||
|
||||
|
||||
def _references(extra=None) -> list:
|
||||
refs = [{"document": ref.LC214_SOURCE["document"], "url": ref.LC214_SOURCE["url"], "sha256": ref.LC214_SOURCE["sha256"]},
|
||||
{"document": "cClassTrib (Portal Nacional NF-e)", "publication": ref.OFFICIAL_CATALOG["publication"], "sha256": ref.OFFICIAL_CATALOG["sha256"]}]
|
||||
return refs + (extra or [])
|
||||
|
||||
|
||||
def _validate_class(code: str, local_rows: dict, compat: dict, today: date):
|
||||
"""None when the class may be proposed, else the exact reason it may not."""
|
||||
official = _official_by_code().get(code)
|
||||
if official is None:
|
||||
return "CLASS_NOT_IN_OFFICIAL_TABLE"
|
||||
loc = local_rows.get(code)
|
||||
if loc is None:
|
||||
return "CLASS_NOT_IN_LOCAL_CATALOG"
|
||||
state = compat["per_code"].get(code)
|
||||
if state != "MATCH_SAFE":
|
||||
return f"LOCAL_OFFICIAL_{state}"
|
||||
if official["start"] and date.fromisoformat(official["start"]) > today:
|
||||
return "CLASS_NOT_YET_IN_FORCE"
|
||||
if not (official["ind_nfe"] or official["ind_nfce"]):
|
||||
return "CLASS_NOT_USABLE_IN_NFE_NFCE"
|
||||
return None
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------- resolver
|
||||
|
||||
|
||||
def resolve_cclasstrib(product: dict, company: dict | None, ctx: dict, local_rows: dict | None, *, compat: dict | None = None, today: date | None = None) -> Resolution:
|
||||
"""product: {"CODIGO","CLASSIFICACAO_FISCAL","ncm_valid"(bool)}; company: {"crt"}; ctx: {"rtc_enabled": bool, "local_catalog_valid": bool}."""
|
||||
today = today or date.today()
|
||||
ncm = _digits(product.get("CLASSIFICACAO_FISCAL"))
|
||||
crt = str((company or {}).get("crt") or "").strip()
|
||||
used = {"ncm": ncm or None, "crt": crt or None, "rtc_enabled": bool(ctx.get("rtc_enabled"))}
|
||||
if not ctx.get("rtc_enabled"):
|
||||
return Resolution("NOT_APPLICABLE", "RTC-R0", "RTC_DISABLED_IN_ERP", inputs_used=used)
|
||||
if not ctx.get("local_catalog_valid"):
|
||||
return Resolution("INSUFFICIENT_EVIDENCE", "RTC-G1", "LOCAL_RTC_CATALOG_INVALID", inputs_used=used)
|
||||
if crt in SIMPLES_FAMILY:
|
||||
return Resolution("INSUFFICIENT_EVIDENCE", "RTC-R1", "SIMPLES_ORIENTATION_PENDING_OFFICIAL_NT", inputs_used=used,
|
||||
source_references=_references([{"document": "NT 2025.002-RTC v1.51 (Portal NF-e, 04/08/2026)", "note": "IBS/CBS orientations for CRT 1/2/4 pending"},
|
||||
{"document": "Ato Conjunto RFB/CGIBS n 4/2026 (DOU 31/07/2026)", "note": "art. 1 par. 1: DF-e obligation for Simples optantes starts 2027-01-01"}]))
|
||||
if crt != "3":
|
||||
return Resolution("INSUFFICIENT_EVIDENCE", "RTC-R1", "COMPANY_REGIME_UNKNOWN", inputs_used=used)
|
||||
if len(ncm) != 8 or product.get("ncm_valid") is not True:
|
||||
return Resolution("INSUFFICIENT_EVIDENCE", "RTC-R2", "NO_VALID_NCM", inputs_used=used)
|
||||
local_rows = local_rows or {}
|
||||
compat = compat or compare_catalogs(local_rows, today)
|
||||
hits = annex_matches(ncm)
|
||||
annexes = sorted({h["annex"] for h in hits})
|
||||
if hits:
|
||||
if any(h["revoked"] for h in hits) and len(annexes) == 1:
|
||||
return Resolution("AMBIGUOUS", "RTC-A1", "LISTED_ONLY_IN_REVOKED_ANNEX", inputs_used=used, evidence=hits, source_references=_references())
|
||||
live = [h for h in hits if not h["revoked"]]
|
||||
live_annexes = sorted({h["annex"] for h in live})
|
||||
if len(live_annexes) > 1 or (len(live_annexes) == 1 and len(annexes) > 1):
|
||||
return Resolution("AMBIGUOUS", "RTC-A2", "LISTED_IN_MORE_THAN_ONE_ANNEX", inputs_used=used, evidence=hits, source_references=_references())
|
||||
annex = live_annexes[0] if live_annexes else annexes[0]
|
||||
clean = [h for h in live if h["clean"]]
|
||||
cls = ANNEX_DEFAULT_CLASS.get(annex)
|
||||
if not cls or not clean or len(clean) != len(live):
|
||||
return Resolution("AMBIGUOUS", "RTC-A1", f"ANNEX_{annex}_NEEDS_OPERATION_OR_CONDITION", inputs_used=used, evidence=hits, source_references=_references())
|
||||
problem = _validate_class(cls, local_rows, compat, today)
|
||||
if problem:
|
||||
return Resolution("INSUFFICIENT_EVIDENCE", "RTC-G2", problem, inputs_used=used, evidence=hits, source_references=_references())
|
||||
official = _official_by_code()[cls]
|
||||
return Resolution("RESOLVED_WITH_EXPLICIT_RULE", "RTC-R3", f"ANNEX_{annex}_CLEAN_ITEM_UNIQUE_DEFAULT_CLASS", cls, official["cst"], "ANNEX_TEXT_MATCH", used, hits, _references())
|
||||
if ncm[:2] in SENSITIVE_CHAPTERS or ncm.startswith(SPECIAL_PREFIXES):
|
||||
return Resolution("AMBIGUOUS", "RTC-A3", "SPECIAL_REGIME_DEPENDS_ON_NATURE_OR_OPERATION_OF_THE_GOOD", inputs_used=used, evidence=[{"chapter": ncm[:2]}], source_references=_references())
|
||||
problem = _validate_class("000001", local_rows, compat, today)
|
||||
if problem:
|
||||
return Resolution("INSUFFICIENT_EVIDENCE", "RTC-G2", problem, inputs_used=used, source_references=_references())
|
||||
return Resolution("SUGGESTION_ONLY", "RTC-R4", "NCM_IN_NO_ANNEX_AND_NO_NATURE_BASED_CHAPTER", "000001", "000", "EXCLUSION_OF_ALL_SPECIAL_REGIMES", used,
|
||||
[{"annex_hits": 0, "chapter": ncm[:2], "note": "absence of a special regime is not positive evidence: suggestion only, never written"}], _references())
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Persists the Agent's CFOP/IPPT write audit records (append-only). Mirrors central.app.rtc_audit's
|
||||
safety model for a separate, narrower capability - no candidate preview table (candidates are
|
||||
recomputed fresh from field_contract every job, never stored server-side), no rollback queue yet
|
||||
(rollback is operator-triggered later, out of scope for this pass)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from .events import record_event
|
||||
from .models import CfopIpptWriteAudit
|
||||
|
||||
MAX_AUDITS_PER_JOB = 2000
|
||||
WRITE_MODES = ("DISABLED", "DRY_RUN", "APPROVAL_REQUIRED", "ENABLED")
|
||||
ALLOWED_FIELDS = ("CFOP", "IPPT")
|
||||
|
||||
|
||||
def _clip(value, size):
|
||||
return None if value is None else str(value)[:size]
|
||||
|
||||
|
||||
def job_authorizations(installation) -> dict:
|
||||
"""What the authenticated job carries to the Agent. Decided centrally per installation; the Agent
|
||||
has no local switch. DISABLED unless an installation row explicitly says otherwise (server_default)."""
|
||||
mode = installation.cfop_ippt_write_mode if installation.cfop_ippt_write_mode in WRITE_MODES else "DISABLED"
|
||||
return {"cfop_ippt_write_mode": mode}
|
||||
|
||||
|
||||
def record_cfop_ippt_write_audits(session, *, tenant_id: int, installation, agent, job_id: int | None, cfop_ippt_apply: dict | None) -> int:
|
||||
"""Stores every applied/aborted write of the job. Never trusts the Agent for identity: tenant/installation/agent come from the authenticated context."""
|
||||
if not isinstance(cfop_ippt_apply, dict):
|
||||
return 0
|
||||
records = list(cfop_ippt_apply.get("applied") or []) + list(cfop_ippt_apply.get("aborted") or [])
|
||||
stored = applied = aborted = 0
|
||||
for item in records[:MAX_AUDITS_PER_JOB]:
|
||||
if not isinstance(item, dict) or item.get("field") not in ALLOWED_FIELDS:
|
||||
continue
|
||||
outcome = str(item.get("outcome", ""))[:16]
|
||||
session.add(CfopIpptWriteAudit(
|
||||
tenant_id=tenant_id, installation_id=installation.id, agent_id=agent.id, job_id=job_id,
|
||||
product_code=_clip(item.get("product_code"), 64) or "", field=item["field"], outcome=outcome,
|
||||
reason=_clip(item.get("reason") or item.get("reason_code"), 96), old_value=_clip(item.get("old_value"), 16), new_value=_clip(item.get("new_value"), 16),
|
||||
action="APPLY", written_at=_clip(item.get("timestamp"), 40)))
|
||||
stored += 1
|
||||
if outcome == "APPLIED":
|
||||
applied += 1
|
||||
else:
|
||||
aborted += 1
|
||||
if stored:
|
||||
record_event(session, tenant_id=tenant_id, installation_id=installation.id, agent_id=agent.id, severity="WARNING" if applied else "INFO", event_type="CFOP_IPPT_AUTO_APPLY", job_id=job_id,
|
||||
message=f"CFOP/IPPT: {applied} aplicado(s), {aborted} abortado(s)",
|
||||
context={"applied": applied, "aborted": aborted, "reasons": cfop_ippt_apply.get("aborted_reasons") or {}, "mode": cfop_ippt_apply.get("mode")})
|
||||
session.commit()
|
||||
return stored
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Conservative deterministic NCM classification."""
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
ENGINE_VERSION = "0.3.0"
|
||||
DECISION_VERSION = "mission-04c-ipi-ex-split"
|
||||
|
||||
_NT_MARKERS = {"NT", "N/T", "NAO TRIBUTADO", "NÃO TRIBUTADO"}
|
||||
_ZERO_MARKERS = {"0", "0,00", "0.00"}
|
||||
|
||||
|
||||
def classify_tipi_rate(entry):
|
||||
"""Classify why a TIPI base entry has no usable numeric rate; never guess a value."""
|
||||
if entry.ex:
|
||||
return "EX_SPECIFIC"
|
||||
text = entry.ipi_rate_text
|
||||
if text is None:
|
||||
return "SOURCE_BLANK"
|
||||
normalized = text.strip().upper()
|
||||
if normalized in _NT_MARKERS:
|
||||
return "NT_OR_NON_TAXED"
|
||||
if entry.ipi_rate == 0 or normalized in _ZERO_MARKERS:
|
||||
return "EXPLICIT_ZERO"
|
||||
return "OTHER"
|
||||
|
||||
def normalize_code(value): return "".join(char for char in str(value or "") if char.isdigit())
|
||||
def normalize_description(value): return " ".join(re.sub(r"[^A-Z0-9]+", " ", unicodedata.normalize("NFKD", str(value or "")).encode("ascii", "ignore").decode().upper()).split())
|
||||
|
||||
# Portuguese function words with no fiscal-descriptive content. Without this, a
|
||||
# product description sharing only "DE"/"E"/"COM" with an official CEST text
|
||||
# scores a false-positive nonzero overlap that has nothing to do with the product.
|
||||
_STOPWORDS = {
|
||||
"DE", "DA", "DO", "DAS", "DOS", "E", "OU", "A", "O", "AS", "OS",
|
||||
"EM", "NO", "NA", "NOS", "NAS", "PARA", "POR", "COM", "SEM",
|
||||
"MESMO", "OUTRO", "OUTRA", "OUTROS", "OUTRAS", "QUE",
|
||||
}
|
||||
|
||||
|
||||
def description_tokens(value):
|
||||
return set(normalize_description(value).split()) - _STOPWORDS
|
||||
|
||||
class NcmStatus(str):
|
||||
def __eq__(self, other):
|
||||
if super().__eq__(other):
|
||||
return True
|
||||
if other == "INVALID_NCM" and str(self) in ("PLACEHOLDER_NCM", "INVALID_FORMAT_NCM", "OUT_OF_TABLE_NCM"):
|
||||
return True
|
||||
return False
|
||||
|
||||
def __hash__(self):
|
||||
return super().__hash__()
|
||||
|
||||
|
||||
class Classifier:
|
||||
def __init__(self, repository): self.repository = repository
|
||||
def classify(self, product):
|
||||
entries = self.repository.active_entries(); code = normalize_code(product.get("current_ncm"))
|
||||
exact = next((entry for entry in entries if entry.code == code and len(code) == 8), None)
|
||||
legacy = self.repository.legacy_ncm(code) if len(code) == 8 and hasattr(self.repository, "legacy_ncm") else None
|
||||
if exact:
|
||||
status, candidate = "EXACT_NCM", exact
|
||||
elif legacy:
|
||||
status, candidate = "LEGACY_NCM", None
|
||||
elif not code:
|
||||
status, candidate = self._candidate(entries, product.get("description"))
|
||||
elif len(set(code)) == 1 or code in ("00000000", "99999999", "11111111", "000000"):
|
||||
status, candidate = NcmStatus("PLACEHOLDER_NCM"), None
|
||||
elif len(code) != 8 or code in ("12345678", "87654321"):
|
||||
status, candidate = NcmStatus("INVALID_FORMAT_NCM"), None
|
||||
else:
|
||||
status, candidate = NcmStatus("OUT_OF_TABLE_NCM"), None
|
||||
matched_code = candidate.code if candidate else legacy["code"] if legacy else None
|
||||
cests_available = matched_code and hasattr(self.repository, "cest_for_ncm")
|
||||
cests = self.repository.cest_for_ncm(matched_code) if cests_available else None
|
||||
cests = list({item.code: item for item in cests or []}.values())
|
||||
scores = {item.code: self._description_score(product.get("description"), item.description) for item in cests}
|
||||
if legacy:
|
||||
cest_status = "LEGACY_NCM"
|
||||
elif not cests:
|
||||
cest_status = "NO_CEST_MAPPING_FOUND"
|
||||
elif max(scores.values()) < .5:
|
||||
cest_status = "CEST_DESCRIPTION_MISMATCH"
|
||||
elif len(cests) == 1:
|
||||
cest_status = "CEST_CONFIRMED"
|
||||
else:
|
||||
cest_status = "CEST_CANDIDATES"
|
||||
resolved_cest = cests[0].code if cest_status == "CEST_CONFIRMED" else None
|
||||
ipi = self._resolve_ipi(matched_code, legacy)
|
||||
mva = self._resolve_mva(matched_code, legacy, resolved_cest)
|
||||
return {"status": status, "cest_status": cest_status, "ipi_status": ipi["status"], "ba_st_mva_status": mva["status"], "classification": {"ncm": matched_code, "ncm_description": candidate.description if candidate else None, "cest": [item.code for item in cests] if cests_available else None, "cest_candidates": [{"code": item.code, "official_description": item.description, "description_score": scores[item.code]} for item in cests], "cfop": None, "cst": None, "csosn": None, "pis_cofins": None, "ipi": ipi["resolved"], "ipi_candidates": ipi["candidates"], "ba_st_mva": mva["resolved"], "ba_st_mva_candidates": mva["candidates"], "ibs_cbs": None}, "legacy_ncm": legacy, "reason_codes": ["FISCAL_DETERMINATION_NOT_CONFIGURED"], "can_auto_apply": False, "engine_version": ENGINE_VERSION, "decision_version": DECISION_VERSION, "dataset_versions": self.repository.dataset_versions()}
|
||||
|
||||
def _resolve_ipi(self, matched_code, legacy):
|
||||
if legacy:
|
||||
return {"status": "LEGACY_NCM", "resolved": None, "candidates": []}
|
||||
if not matched_code or not hasattr(self.repository, "tipi_for_ncm"):
|
||||
return {"status": "NO_TIPI_MAPPING_FOUND", "resolved": None, "candidates": []}
|
||||
entries = self.repository.tipi_for_ncm(matched_code)
|
||||
if not entries:
|
||||
return {"status": "NO_TIPI_MAPPING_FOUND", "resolved": None, "candidates": []}
|
||||
base = next((entry for entry in entries if not entry.ex), None)
|
||||
ex_variants = [entry for entry in entries if entry.ex]
|
||||
candidates = [{"ncm": entry.ncm, "ex": entry.ex, "description": entry.description, "rate": entry.ipi_rate_text} for entry in ex_variants]
|
||||
if base:
|
||||
if base.ipi_rate is not None:
|
||||
rate_kind = "NUMERIC"
|
||||
elif base.ipi_rate_text is not None:
|
||||
rate_kind = "NON_NUMERIC"
|
||||
else:
|
||||
rate_kind = "BLANK"
|
||||
resolved = {
|
||||
"ncm": base.ncm,
|
||||
"ex": base.ex,
|
||||
"description": base.description,
|
||||
"rate": base.ipi_rate_text,
|
||||
"rate_numeric": base.ipi_rate,
|
||||
"rate_kind": rate_kind,
|
||||
}
|
||||
if rate_kind != "NUMERIC":
|
||||
resolved["rate_classification"] = classify_tipi_rate(base)
|
||||
status = "IPI_CANDIDATES" if ex_variants else "IPI_RESOLVED"
|
||||
return {"status": status, "resolved": resolved, "candidates": candidates}
|
||||
return {"status": "IPI_CANDIDATES", "resolved": None, "candidates": candidates}
|
||||
|
||||
def _resolve_mva(self, matched_code, legacy, resolved_cest):
|
||||
if legacy:
|
||||
return {"status": "LEGACY_NCM", "resolved": None, "candidates": []}
|
||||
if not matched_code or not hasattr(self.repository, "ba_st_mva_for_ncm"):
|
||||
return {"status": "NO_MVA_MAPPING_FOUND", "resolved": None, "candidates": []}
|
||||
entries = self.repository.ba_st_mva_for_ncm(matched_code)
|
||||
if not entries:
|
||||
return {"status": "NO_MVA_MAPPING_FOUND", "resolved": None, "candidates": []}
|
||||
candidates = [{"item": entry.item, "cest": entry.cest, "mva_original": entry.mva_original, "mva_adjusted": entry.mva_adjusted, "description": entry.description} for entry in entries]
|
||||
if len(entries) == 1:
|
||||
return {"status": "MVA_RESOLVED", "resolved": candidates[0], "candidates": candidates}
|
||||
if resolved_cest:
|
||||
by_cest = [item for item in candidates if item["cest"] == resolved_cest]
|
||||
if len(by_cest) == 1:
|
||||
return {"status": "MVA_RESOLVED", "resolved": by_cest[0], "candidates": candidates}
|
||||
return {"status": "MVA_CANDIDATES", "resolved": None, "candidates": candidates}
|
||||
def _candidate(self, entries, description):
|
||||
tokens = set(normalize_description(description).split())
|
||||
scores = sorted(((len(tokens & set(normalize_description(item.description).split())) / len(tokens or {"_"}), item) for item in entries), reverse=True, key=lambda row: row[0])
|
||||
if not scores or scores[0][0] == 0: return "NOT_FOUND", None
|
||||
return ("HIGH_CONFIDENCE_CANDIDATE", scores[0][1]) if scores[0][0] >= .90 and (len(scores) == 1 or scores[0][0] - scores[1][0] >= .15) else ("AMBIGUOUS", None)
|
||||
|
||||
def _description_score(self, product_description, official_description):
|
||||
product_tokens = description_tokens(product_description)
|
||||
official_tokens = description_tokens(official_description)
|
||||
return len(product_tokens & official_tokens) / len(product_tokens or {"_"})
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Client mirror from GTI Gerenciador — the single source of truth for
|
||||
clients (central/app/models.py: Tenant.source/external_client_id/active).
|
||||
The Fiscal Hub never receives MySQL credentials; it calls the panel's
|
||||
read-only internal endpoint instead, authenticated by a dedicated bearer
|
||||
token (FISCAL_INTERNAL_API_TOKEN, shared only between the two apps' envs
|
||||
— never SESSION_SECRET, the SSO key, an admin password or an Agent token).
|
||||
|
||||
Run `central/scripts/sync_clients_from_gerenciador.py` to pull a fresh copy;
|
||||
`Tenant.source='LOCAL'` rows (pre-existing dev/test tenants) are never
|
||||
touched or offered for a new installation.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
|
||||
from . import config
|
||||
from .models import Tenant
|
||||
|
||||
SOURCE = "GTI_GERENCIADOR"
|
||||
|
||||
|
||||
class ClientSyncUnavailable(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def enabled() -> bool:
|
||||
return bool(config.settings.gerenciador_internal_url and config.settings.fiscal_internal_api_token)
|
||||
|
||||
|
||||
def fetch_from_gerenciador() -> list[dict]:
|
||||
if not enabled():
|
||||
raise ClientSyncUnavailable("GERENCIADOR_INTERNAL_URL/FISCAL_INTERNAL_API_TOKEN not configured")
|
||||
headers = {"Authorization": f"Bearer {config.settings.fiscal_internal_api_token}"}
|
||||
try:
|
||||
response = httpx.get(config.settings.gerenciador_internal_url, headers=headers, timeout=config.settings.http_timeout_seconds)
|
||||
except httpx.HTTPError as error:
|
||||
raise ClientSyncUnavailable(f"request failed: {error}") from error
|
||||
if response.status_code != 200:
|
||||
raise ClientSyncUnavailable(f"unexpected status {response.status_code}: {response.text[:200]}")
|
||||
return response.json()["clients"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncResult:
|
||||
fetched: int
|
||||
created: int
|
||||
updated: int
|
||||
unchanged: int
|
||||
active: int
|
||||
inactive: int
|
||||
|
||||
|
||||
def sync_from_gerenciador(session, clients: list[dict]) -> SyncResult:
|
||||
"""Upsert `clients` (each: external_client_id, code, display_name, active)
|
||||
into Tenant rows with source='GTI_GERENCIADOR'. Never touches a
|
||||
source='LOCAL' row. A display-name collision with another tenant is
|
||||
disambiguated with the external code rather than failing the whole sync
|
||||
over one client (Tenant.name is UNIQUE)."""
|
||||
now = datetime.now(timezone.utc)
|
||||
existing_by_external = {row.external_client_id: row for row in session.scalars(select(Tenant).where(Tenant.source == SOURCE)).all()}
|
||||
taken_names = {row.name for row in session.scalars(select(Tenant)).all()}
|
||||
created = updated = unchanged = 0
|
||||
|
||||
for client in clients:
|
||||
external_id = client["external_client_id"]
|
||||
code = client["code"]
|
||||
active = bool(client["active"])
|
||||
wanted_name = (client.get("display_name") or "").strip() or f"Cliente {code}"
|
||||
row = existing_by_external.get(external_id)
|
||||
|
||||
if row is None:
|
||||
name = wanted_name if wanted_name not in taken_names else f"{wanted_name} ({code})"
|
||||
row = Tenant(name=name, source=SOURCE, external_client_id=external_id, external_code=code, external_cnpj=client.get("cnpj"), active=active, synced_at=now)
|
||||
session.add(row)
|
||||
taken_names.add(name)
|
||||
created += 1
|
||||
continue
|
||||
|
||||
changed = False
|
||||
if row.name != wanted_name and wanted_name not in taken_names:
|
||||
taken_names.discard(row.name)
|
||||
row.name = wanted_name
|
||||
taken_names.add(wanted_name)
|
||||
changed = True
|
||||
if row.external_code != code:
|
||||
row.external_code = code
|
||||
changed = True
|
||||
if row.external_cnpj != client.get("cnpj"):
|
||||
row.external_cnpj = client.get("cnpj")
|
||||
changed = True
|
||||
if row.active != active:
|
||||
row.active = active
|
||||
changed = True
|
||||
row.synced_at = now
|
||||
if changed:
|
||||
updated += 1
|
||||
else:
|
||||
unchanged += 1
|
||||
|
||||
session.flush()
|
||||
active_count = sum(1 for c in clients if c["active"])
|
||||
return SyncResult(fetched=len(clients), created=created, updated=updated, unchanged=unchanged, active=active_count, inactive=len(clients) - active_count)
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Environment-backed Fiscal Hub configuration."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
import os
|
||||
from urllib.parse import quote
|
||||
|
||||
|
||||
def _database_url() -> str:
|
||||
configured = os.getenv("DATABASE_URL")
|
||||
if configured:
|
||||
return configured
|
||||
password = quote(os.getenv("POSTGRES_PASSWORD", "gti"), safe="")
|
||||
return f"postgresql+psycopg://gti:{password}@postgres:5432/gti_fiscal"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
database_url: str = _database_url()
|
||||
hub_port: int = int(os.getenv("HUB_PORT", "8000"))
|
||||
# AGENT_DEV_TOKEN auto-bootstraps an agent from a fixed dev token on every request.
|
||||
# Leave unset in any environment other than local development — production Agents
|
||||
# must be created via scripts/register_agent.py, whose token is never derived from
|
||||
# this setting.
|
||||
agent_dev_token: str = os.getenv("AGENT_DEV_TOKEN", "")
|
||||
environment: str = os.getenv("APP_ENV", "development")
|
||||
log_level: str = os.getenv("LOG_LEVEL", "INFO")
|
||||
base_url: str = os.getenv("BASE_URL", "http://localhost:8000")
|
||||
# An Agent is considered ONLINE if a heartbeat arrived within this many seconds.
|
||||
agent_online_threshold_seconds: int = int(os.getenv("AGENT_ONLINE_THRESHOLD_SECONDS", "120"))
|
||||
# Backpressure: how many FULL jobs (INITIAL_SYNC / REPROCESS_ALL: whole-database classification) may run at once across ALL Agents. Others wait as PENDING. <=0 disables the limit.
|
||||
heavy_job_concurrency_limit: int = int(os.getenv("HEAVY_JOB_CONCURRENCY_LIMIT", "3"))
|
||||
agent_heartbeat_interval_seconds: int = int(os.getenv("AGENT_HEARTBEAT_INTERVAL_SECONDS", "40"))
|
||||
agent_offline_after_seconds: int = int(os.getenv("AGENT_OFFLINE_AFTER_SECONDS", "0"))
|
||||
agent_server_version: str = os.getenv("CURRENT_SERVER_VERSION", "1.0.0rc1")
|
||||
agent_event_retention_days: int = int(os.getenv("AGENT_EVENT_RETENTION_DAYS", "90"))
|
||||
web_session_seconds: int = int(os.getenv("WEB_SESSION_SECONDS", "28800"))
|
||||
web_cookie_secure: bool = os.getenv("WEB_COOKIE_SECURE", "0") == "1"
|
||||
web_cookie_name: str = os.getenv("WEB_COOKIE_NAME", "gti_admin_session")
|
||||
web_max_event_message: int = int(os.getenv("WEB_MAX_EVENT_MESSAGE", "2000"))
|
||||
web_max_event_context_bytes: int = int(os.getenv("WEB_MAX_EVENT_CONTEXT_BYTES", "12000"))
|
||||
ui_timezone: str = os.getenv("UI_TIMEZONE", "America/Bahia")
|
||||
allow_weak_dev_password: bool = os.getenv("ALLOW_WEAK_DEV_PASSWORD", "0") == "1"
|
||||
# ---- SSO from GTI Gerenciador (central/app/sso.py) ----
|
||||
# Ed25519 public key (raw 32 bytes, base64url) of the panel that signs the
|
||||
# assertions; the private key never exists on the Hub. SSO is active only
|
||||
# when both the key and the panel's start URL are set.
|
||||
sso_public_key: str = os.getenv("SSO_GERENCIADOR_CHAVE_PUBLICA", "")
|
||||
sso_start_url: str = os.getenv("SSO_START_URL", "")
|
||||
federated_session_seconds: int = int(os.getenv("FEDERATED_SESSION_SECONDS", "3600"))
|
||||
# ---- Agent distribution (central/app/agent_distribution.py) ----
|
||||
# Local, non-web-served directory holding the released Windows Agent zips.
|
||||
# Never inside PostgreSQL, never under central/static.
|
||||
agent_release_storage_path: str = os.getenv("AGENT_RELEASE_STORAGE_PATH", "./data/releases")
|
||||
agent_download_link_ttl_seconds: int = int(os.getenv("AGENT_DOWNLOAD_LINK_TTL_SECONDS", "1800"))
|
||||
agent_download_max_uses_default: int = int(os.getenv("AGENT_DOWNLOAD_MAX_USES_DEFAULT", "1"))
|
||||
agent_enrollment_ttl_seconds: int = int(os.getenv("AGENT_ENROLLMENT_TTL_SECONDS", "86400"))
|
||||
# ---- Client mirror from GTI Gerenciador (central/app/client_sync.py) ----
|
||||
# GTI Gerenciador is the single source of truth for clients; the Fiscal Hub
|
||||
# never gets MySQL credentials. Both empty = sync disabled (documented).
|
||||
gerenciador_internal_url: str = os.getenv("GERENCIADOR_INTERNAL_URL", "") # e.g. https://gtigerenciador.../internal/fiscal/clients
|
||||
fiscal_internal_api_token: str = os.getenv("FISCAL_INTERNAL_API_TOKEN", "")
|
||||
ncm_dataset_url: str = os.getenv("NCM_DATASET_URL", "https://portalunico.siscomex.gov.br/classif/api/publico/nomenclatura/download/json")
|
||||
cest_dataset_url: str = os.getenv("CEST_DATASET_URL", "https://www.confaz.fazenda.gov.br/legislacao/convenios/2018/CV142_18")
|
||||
dataset_storage_path: str = os.getenv("DATASET_STORAGE_PATH", "./data/datasets")
|
||||
http_timeout_seconds: float = float(os.getenv("HTTP_TIMEOUT_SECONDS", "30"))
|
||||
# ---- AI semantic layer V1 ----
|
||||
# Credentials live ONLY in the Hub environment; the Windows Agent never sees them.
|
||||
ai_enabled: bool = os.getenv("AI_ENABLED", "0") == "1"
|
||||
ai_provider: str = os.getenv("AI_PROVIDER", "baseline") # baseline | openai | ... (AISemanticProvider)
|
||||
ai_model: str = os.getenv("AI_MODEL", "baseline-deterministic-v1")
|
||||
ai_max_batch_size: int = int(os.getenv("AI_MAX_BATCH_SIZE", "20"))
|
||||
ai_max_run_cost_usd: float = float(os.getenv("AI_MAX_RUN_COST_USD", "5.0"))
|
||||
ai_min_confidence: float = float(os.getenv("AI_MIN_CONFIDENCE", "0.6"))
|
||||
ai_price_in_per_m: float | None = float(os.getenv("AI_PRICE_IN_PER_M")) if os.getenv("AI_PRICE_IN_PER_M") else None
|
||||
ai_price_out_per_m: float | None = float(os.getenv("AI_PRICE_OUT_PER_M")) if os.getenv("AI_PRICE_OUT_PER_M") else None
|
||||
# Real-LLM evaluation (OpenAI-compatible). The key lives ONLY in this environment.
|
||||
# Z.ai general endpoint (NOT the /api/coding/paas/v4 coding-plan endpoint).
|
||||
ai_api_key: str = os.getenv("AI_API_KEY", "")
|
||||
ai_base_url: str = os.getenv("AI_BASE_URL", "https://api.z.ai/api/paas/v4")
|
||||
# ---- Product enrichment (docs/product_enrichment.md): free providers only, tokens live ONLY in the Hub environment ----
|
||||
cosmos_token: str = os.getenv("COSMOS_TOKEN", "") # optional; without it Cosmos is NOT_CONFIGURED (not an error)
|
||||
oscbr_token: str = os.getenv("OSCBR_TOKEN", "") # preferred: a ready Bearer token instead of the login
|
||||
cosmos_enabled: bool = os.getenv("COSMOS_ENABLED", "0") == "1" # Cosmos stays out of the active chain unless explicitly enabled
|
||||
oscbr_backfill_enabled: bool = os.getenv("OSCBR_BACKFILL_ENABLED", "0") == "1" # gradual backfill of CORROBORATED GTINs; OFF until a real benchmark proves value
|
||||
oscbr_user: str = os.getenv("OSCBR_USER", "") # optional free-plan account of gtin.rscsistemas.com.br (never in logs/UI)
|
||||
oscbr_password: str = os.getenv("OSCBR_PASSWORD", "")
|
||||
kodebar_api_key: str = os.getenv("KODEBAR_API_KEY", "") # optional free-plan key (identity only); never in logs/UI
|
||||
provider_operational_reserve_percent: int = int(os.getenv("PROVIDER_OPERATIONAL_RESERVE_PERCENT", "20"))
|
||||
allow_paid_providers: bool = os.getenv("ALLOW_PAID_PROVIDERS", "0") == "1"
|
||||
official_refresh_enabled: bool = os.getenv("OFFICIAL_REFRESH_ENABLED", "1") == "1"
|
||||
enrichment_time_budget_seconds: float = float(os.getenv("ENRICHMENT_TIME_BUDGET_SECONDS", "4.5"))
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Validation of operator fiscal-context entries (facts the ERP does not know). Pure; shared by the Hub API, the importer CLI and the resolver."""
|
||||
import json
|
||||
|
||||
SCOPES = ("COMPANY", "GROUP", "NCM", "PRODUCT")
|
||||
ENUMS = {
|
||||
"product_role": {"RESALE", "OWN_PRODUCTION", "INPUT", "SERVICE"},
|
||||
"st_status": {"SUBJECT", "NOT_SUBJECT"},
|
||||
"st_withheld": {"Y", "N"},
|
||||
"icms_treatment": {"TAXED", "EXEMPT", "NOT_TAXED", "ST_WITHHELD", "SUSPENDED", "DEFERRED"},
|
||||
# MIXED_OR_DYNAMIC: purchases are not predominantly intra- or interstate, so no single
|
||||
# static entry CFOP is fiscally correct for all operations (section 0, AI semantic layer V1).
|
||||
"entry_scope": {"INTRA", "INTER", "MIXED_OR_DYNAMIC"},
|
||||
"rounding": {"A", "T"},
|
||||
"piscofins": {"N", "S"},
|
||||
"incidence": {"NENHUMA"},
|
||||
"origin": {str(i) for i in range(9)},
|
||||
"store_ipi_profile": {"POPULATE_REFERENCE_RATE", "LEAVE_NOT_APPLICABLE"},
|
||||
# C000004.CONTRIBUINTE_IPI is not yet sent at heartbeat (no Agent-side reader exists for it): until that auto-discovery lands, this is the
|
||||
# one-time operator answer field_contract's CST_IPI rule needs - never assumed "N" just because the Agent hasn't reported it.
|
||||
"contribuinte_ipi": {"S", "N"},
|
||||
}
|
||||
PROFILE_KEYS = {"pis_in_cst", "pis_out_cst", "pis_rate", "cofins_in_cst", "cofins_out_cst", "cofins_rate"}
|
||||
FREE_KEYS = {"ncm", "cest", "ex_tipi", "icms_rate", "credit_rate", "cst_ipi", "service_cfop", "pis_cofins_profile"}
|
||||
ALLOWED_KEYS = set(ENUMS) | FREE_KEYS
|
||||
PRODUCT_ONLY = {"ncm"}
|
||||
COMPANY_ONLY = {"store_ipi_profile", "contribuinte_ipi"}
|
||||
NCM_KEYS = {"cest", "ex_tipi"} # tied to the NCM tables: answered per NCM or per product # an NCM answer is always about one product
|
||||
|
||||
|
||||
def validate_entry(entry):
|
||||
"""Returns (normalized_entry, error). error is None when valid."""
|
||||
scope = str(entry.get("scope", "")).strip().upper()
|
||||
key = str(entry.get("key", "")).strip()
|
||||
value = entry.get("value")
|
||||
value = "" if value is None else str(value).strip()
|
||||
scope_key = str(entry.get("scope_key") or "").strip()
|
||||
if scope not in SCOPES:
|
||||
return None, f"scope must be one of {SCOPES}"
|
||||
if key not in ALLOWED_KEYS:
|
||||
return None, f"unknown key {key!r}"
|
||||
if scope != "COMPANY" and not scope_key:
|
||||
return None, "scope_key is required for GROUP/NCM/PRODUCT scope"
|
||||
if key in COMPANY_ONLY and scope != "COMPANY":
|
||||
return None, f"{key} is a COMPANY-scope answer"
|
||||
if key in PRODUCT_ONLY and scope != "PRODUCT":
|
||||
return None, f"{key} is a PRODUCT-scope answer"
|
||||
if key in NCM_KEYS and scope not in ("PRODUCT", "NCM"):
|
||||
return None, f"{key} can only be answered for a PRODUCT or an NCM"
|
||||
if not value:
|
||||
return None, "value is empty"
|
||||
if key in ENUMS and value.upper() not in ENUMS[key] and value not in ENUMS[key]:
|
||||
return None, f"{key} must be one of {sorted(ENUMS[key])}"
|
||||
if key in ENUMS:
|
||||
value = value.upper() if key != "origin" else value
|
||||
if key == "ncm" and not (value.isdigit() and len(value) == 8):
|
||||
return None, "ncm must be 8 digits"
|
||||
if key == "cest" and value.upper() != "NONE" and not (value.replace(".", "").isdigit() and len(value.replace(".", "")) == 7):
|
||||
return None, "cest must be 7 digits or NONE"
|
||||
if key in ("icms_rate", "credit_rate"):
|
||||
try:
|
||||
float(value.replace(",", "."))
|
||||
except ValueError:
|
||||
return None, f"{key} must be numeric"
|
||||
if key == "pis_cofins_profile" and value.upper() not in ("NOT_APPLICABLE", "OPERATION_TIME"):
|
||||
try:
|
||||
prof = json.loads(value)
|
||||
except ValueError:
|
||||
return None, "pis_cofins_profile must be JSON, or exactly NOT_APPLICABLE / OPERATION_TIME"
|
||||
if not isinstance(prof, dict) or not prof or set(prof) - PROFILE_KEYS:
|
||||
return None, f"pis_cofins_profile keys must be a non-empty subset of {sorted(PROFILE_KEYS)}"
|
||||
elif key == "pis_cofins_profile":
|
||||
value = value.upper()
|
||||
return {"scope": scope, "scope_key": scope_key if scope != "COMPANY" else "", "key": key, "value": value, "reason": (entry.get("reason") or "").strip() or None}, None
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Human-authenticated JSON API for the control panel."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from sqlalchemy import select
|
||||
|
||||
from .audit import record_audit
|
||||
from .blocker_resolution import BlockerResolutionUnavailable, resolve_blocker_with_context
|
||||
from .installation_identity import reference_agent_version
|
||||
from .control_plane import dashboard_summary, installation_view, list_blockers, list_installations, list_jobs, list_panel_events
|
||||
from .human_auth import require_csrf, require_human
|
||||
from .models import Agent, AgentSnapshot, BlockedProduct, Installation, Job, Tenant
|
||||
from .jobs import create_job
|
||||
|
||||
|
||||
def router(session_factory):
|
||||
routes = APIRouter(prefix="/v1")
|
||||
|
||||
def identity(request: Request):
|
||||
session = session_factory()
|
||||
try:
|
||||
web_session, admin = require_human(request, session)
|
||||
return session, web_session, admin
|
||||
except Exception:
|
||||
session.close(); raise
|
||||
|
||||
@routes.get("/dashboard/summary")
|
||||
def summary(identity=Depends(identity)):
|
||||
session, _, _ = identity
|
||||
try: return dashboard_summary(session)
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/agents")
|
||||
def agents(page: int = 1, page_size: int = 50, client: str | None = None, health: str | None = None, version: str | None = None, search: str | None = None, last_seen_before: str | None = None, identity=Depends(identity)):
|
||||
session, _, _ = identity
|
||||
try: return list_installations(session, page=page, page_size=page_size, client=client, health=health, version=version, search=search, last_seen_before=datetime.fromisoformat(last_seen_before) if last_seen_before else None)
|
||||
finally: session.close()
|
||||
|
||||
def get_installation(session, agent_id: int):
|
||||
row = session.execute(select(Installation, Agent, Tenant, AgentSnapshot).join(Agent, Agent.id == Installation.agent_id).join(Tenant, Tenant.id == Installation.tenant_id).outerjoin(AgentSnapshot, AgentSnapshot.installation_id == Installation.id).where(Agent.id == agent_id)).first()
|
||||
if not row: raise HTTPException(404, "Agent installation not found")
|
||||
return row
|
||||
|
||||
@routes.get("/agents/{agent_id}")
|
||||
def agent_detail(agent_id: int, identity=Depends(identity)):
|
||||
session, _, _ = identity
|
||||
try:
|
||||
installation, agent, tenant, snapshot = get_installation(session, agent_id)
|
||||
return installation_view(installation, agent, tenant, snapshot, reference=reference_agent_version(session))
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/agents/{agent_id}/health")
|
||||
def agent_health(agent_id: int, identity=Depends(identity)):
|
||||
return agent_detail(agent_id, identity)
|
||||
|
||||
@routes.get("/agents/{agent_id}/coverage")
|
||||
def agent_coverage(agent_id: int, identity=Depends(identity)):
|
||||
detail = agent_detail(agent_id, identity)
|
||||
return {key: detail[key] for key in ("total_products", "fiscally_complete", "safe_blocked", "failed_products", "coverage_percent", "blockers")}
|
||||
|
||||
@routes.get("/agents/{agent_id}/blockers")
|
||||
def agent_blockers(agent_id: int, page: int = 1, page_size: int = 50, blocker_type: str | None = None, field: str | None = None, identity=Depends(identity)):
|
||||
session, _, _ = identity
|
||||
try:
|
||||
installation, agent, _, _ = get_installation(session, agent_id)
|
||||
return list_blockers(session, tenant_id=installation.tenant_id, agent_id=agent_id, page=page, page_size=page_size, blocker_type=blocker_type, field=field)
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/agents/{agent_id}/jobs")
|
||||
def agent_jobs(agent_id: int, page: int = 1, page_size: int = 50, status: str | None = None, kind: str | None = None, identity=Depends(identity)):
|
||||
session, _, _ = identity
|
||||
try: return list_jobs(session, agent_id=agent_id, page=page, page_size=page_size, status=status, kind=kind)
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/agents/{agent_id}/events")
|
||||
def agent_events(agent_id: int, page: int = 1, page_size: int = 50, severity: str | None = None, event_type: str | None = None, since: str | None = None, until: str | None = None, identity=Depends(identity)):
|
||||
session, _, _ = identity
|
||||
try:
|
||||
installation, _, _, _ = get_installation(session, agent_id)
|
||||
return {"page": max(1, page), "page_size": min(100, max(1, page_size)), "items": list_panel_events(session, installation.id, page=page, page_size=page_size, severity=severity, event_type=event_type, since=datetime.fromisoformat(since) if since else None, until=datetime.fromisoformat(until) if until else None)}
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/blockers/global")
|
||||
def global_blockers(page: int = 1, page_size: int = 50, blocker_type: str | None = None, field: str | None = None, identity=Depends(identity)):
|
||||
session, _, _ = identity
|
||||
try: return list_blockers(session, page=page, page_size=page_size, blocker_type=blocker_type, field=field)
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/jobs/global")
|
||||
def global_jobs(page: int = 1, page_size: int = 50, status: str | None = None, kind: str | None = None, since: str | None = None, until: str | None = None, identity=Depends(identity)):
|
||||
session, _, _ = identity
|
||||
try: return list_jobs(session, page=page, page_size=page_size, status=status, kind=kind, since=datetime.fromisoformat(since) if since else None, until=datetime.fromisoformat(until) if until else None)
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/{agent_id}/actions/reprocess")
|
||||
def reprocess(agent_id: int, payload: dict, request: Request, identity=Depends(identity)):
|
||||
session, web_session, admin = identity
|
||||
try:
|
||||
require_csrf(web_session, payload.get("csrf_token"))
|
||||
if payload.get("confirm") is not True: raise HTTPException(400, "explicit confirmation required")
|
||||
installation, agent, _, _ = get_installation(session, agent_id)
|
||||
kind = payload.get("kind", "REPROCESS_ALL")
|
||||
if kind not in {"REPROCESS_ALL", "REPROCESS_PRODUCTS"}: raise HTTPException(400, "unsupported action")
|
||||
job_payload = payload.get("payload") or {}
|
||||
if kind == "REPROCESS_PRODUCTS" and not job_payload.get("codigos"): raise HTTPException(400, "codigos required")
|
||||
existing = next((candidate for candidate in session.scalars(select(Job).where(Job.agent_id == agent.id, Job.kind == kind, Job.status == "PENDING")).all() if candidate.payload == job_payload), None)
|
||||
job = existing or create_job(session, agent, kind, job_payload)
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action=kind, target_type="installation", target_id=installation.installation_id, context={"job_id": job.id, "request_ip": request.client.host if request.client else None})
|
||||
session.commit()
|
||||
return {"job_id": job.id, "status": job.status}
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/blockers/{blocker_id}/resolve-context")
|
||||
@routes.post("/blockers/{blocker_id}/resolve-human")
|
||||
def resolve_blocker(blocker_id: int, payload: dict, request: Request, identity=Depends(identity)):
|
||||
session, web_session, admin = identity
|
||||
try:
|
||||
require_csrf(web_session, payload.get("csrf_token"))
|
||||
entry_payload = {key: payload.get(key) for key in ("scope", "scope_key", "key", "value", "reason")}
|
||||
try:
|
||||
return resolve_blocker_with_context(session, blocker_id, admin, entry_payload, request_ip=request.client.host if request.client else None)
|
||||
except BlockerResolutionUnavailable as error:
|
||||
raise HTTPException(409, str(error)) from error
|
||||
except LookupError as error:
|
||||
raise HTTPException(404, str(error)) from error
|
||||
except ValueError as error:
|
||||
raise HTTPException(422, str(error)) from error
|
||||
finally: session.close()
|
||||
|
||||
return routes
|
||||
@@ -0,0 +1,613 @@
|
||||
"""Bounded control-panel queries backed by installation snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from decimal import Decimal, ROUND_HALF_UP
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from sqlalchemy import Numeric, and_, case, cast, func, or_, select
|
||||
|
||||
from .config import settings
|
||||
from .events import list_events
|
||||
from .installation_identity import compute_agent_health, compute_version_drift, outdated_versions, reference_agent_version
|
||||
from .models import Agent, AgentEnrollment, AgentEvent, AgentSnapshot, AuditEvent, BlockedProduct, Installation, Job, Tenant
|
||||
from .panel_presentation import health_reason_label, blocker_reason_label, blocker_type_label, fiscal_status_label, format_age, format_duration, format_ui_datetime, health_label, job_kind_label, job_status_label, operational_priority, priority_label
|
||||
|
||||
|
||||
HEALTH_STATES = ("ONLINE", "DEGRADED", "OFFLINE", "FAILED", "UNKNOWN")
|
||||
BLOCKER_FIELDS = ("NCM", "CEST", "CFOP", "ST", "ICMS", "CSOSN", "PIS", "COFINS", "RTC", "OTHER")
|
||||
ATTENTION_CANDIDATE_LIMIT = 10
|
||||
INACTIVE_ENROLLMENT_STATES = ("REVOKED", "FAILED", "EXPIRED")
|
||||
|
||||
|
||||
def operational_installation_predicate():
|
||||
"""Only installations eligible for current operational indicators.
|
||||
|
||||
Legacy Agents without an enrollment remain eligible. Disabling job
|
||||
processing is not the same as deactivating an installation.
|
||||
"""
|
||||
inactive_enrollment = select(AgentEnrollment.id).where(
|
||||
AgentEnrollment.agent_id == Agent.id,
|
||||
AgentEnrollment.status.in_(INACTIVE_ENROLLMENT_STATES),
|
||||
).exists()
|
||||
return and_(Tenant.active.is_(True), Agent.enabled.is_(True), ~inactive_enrollment)
|
||||
|
||||
|
||||
def _page(page: int, page_size: int):
|
||||
return max(1, page), min(100, max(1, page_size))
|
||||
|
||||
|
||||
def _iso(value):
|
||||
if value is None:
|
||||
return None
|
||||
value = value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc)
|
||||
return value.isoformat()
|
||||
|
||||
|
||||
def _utc_now(value=None):
|
||||
value = value or datetime.now(timezone.utc)
|
||||
return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value
|
||||
|
||||
|
||||
def _health_expression(now):
|
||||
offline_after = settings.agent_offline_after_seconds or settings.agent_heartbeat_interval_seconds * 3
|
||||
cutoff = _utc_now(now) - timedelta(seconds=offline_after)
|
||||
return case(
|
||||
(Installation.last_seen_at.is_(None), "UNKNOWN"),
|
||||
(or_(AgentSnapshot.agent_health == "FAILED", Installation.last_status == "FAILED"), "FAILED"),
|
||||
(Installation.last_seen_at < cutoff, "OFFLINE"),
|
||||
(or_(AgentSnapshot.failed_products > 0, AgentSnapshot.last_job_status == "FAILED"), "DEGRADED"),
|
||||
else_="ONLINE",
|
||||
)
|
||||
|
||||
|
||||
def _fiscal_completion_expression():
|
||||
total = func.coalesce(AgentSnapshot.total_products, 0)
|
||||
complete = func.coalesce(AgentSnapshot.fiscally_complete, 0)
|
||||
numeric_total = cast(total, Numeric(18, 6))
|
||||
numeric_complete = cast(complete * 100, Numeric(18, 6))
|
||||
raw = case((total > 0, numeric_complete / numeric_total), else_=cast(func.coalesce(AgentSnapshot.coverage_percent, 0), Numeric(18, 6)))
|
||||
return func.round(case((raw < 0, 0), (raw > 100, 100), else_=raw), 1)
|
||||
|
||||
|
||||
def _fiscal_completion(snapshot) -> float:
|
||||
total = int(getattr(snapshot, "total_products", 0) or 0)
|
||||
complete = int(getattr(snapshot, "fiscally_complete", 0) or 0)
|
||||
if total:
|
||||
value = Decimal(max(0, min(total, complete)) * 100) / Decimal(total)
|
||||
else:
|
||||
value = Decimal(str(max(0, min(100, float(getattr(snapshot, "coverage_percent", 0) or 0)))))
|
||||
return float(value.quantize(Decimal("0.1"), rounding=ROUND_HALF_UP))
|
||||
|
||||
|
||||
def _fiscal_status(snapshot) -> str:
|
||||
if int(getattr(snapshot, "safe_blocked", 0) or 0):
|
||||
return "BLOCKED"
|
||||
total = int(getattr(snapshot, "total_products", 0) or 0)
|
||||
complete = int(getattr(snapshot, "fiscally_complete", 0) or 0)
|
||||
if total and complete >= total:
|
||||
return "COMPLETE"
|
||||
return "INCOMPLETE"
|
||||
|
||||
|
||||
def _coverage_partition(snapshot) -> dict:
|
||||
"""Expose only explicit snapshot states and label the remaining count honestly."""
|
||||
total = int(getattr(snapshot, "total_products", 0) or 0)
|
||||
complete = int(getattr(snapshot, "fiscally_complete", 0) or 0)
|
||||
safe_blocked = int(getattr(snapshot, "safe_blocked", 0) or 0)
|
||||
failed = int(getattr(snapshot, "failed_products", 0) or 0)
|
||||
explicit = complete + safe_blocked + failed
|
||||
residual = total - explicit
|
||||
invalid_counts = min(total, complete, safe_blocked, failed) < 0
|
||||
overcount = max(0, explicit - total)
|
||||
return {
|
||||
"unclassified_products": max(0, residual) if not invalid_counts else 0,
|
||||
"coverage_overcount_products": overcount,
|
||||
"coverage_invalid_counts": invalid_counts,
|
||||
"coverage_invariant_pass": not invalid_counts and overcount == 0,
|
||||
}
|
||||
|
||||
|
||||
def _technical_timestamp(item):
|
||||
if item["technical_health"] in {"FAILED", "DEGRADED"}:
|
||||
return item.get("last_job_at") or item.get("last_seen_at")
|
||||
return item.get("last_seen_at")
|
||||
|
||||
|
||||
def _attention_item(item, problem, timestamp, *, rank, link=None, now=None):
|
||||
timestamp_value = datetime.fromisoformat(timestamp) if isinstance(timestamp, str) else timestamp
|
||||
return {
|
||||
"client": item.get("client"),
|
||||
"display_name": item.get("display_name"),
|
||||
"hostname": item.get("hostname"),
|
||||
"problem": problem,
|
||||
"timestamp": timestamp if isinstance(timestamp, str) else _iso(timestamp_value),
|
||||
"timestamp_display": format_ui_datetime(timestamp_value),
|
||||
"age": format_age(timestamp_value, now),
|
||||
"link": link or f"/agents/{item['agent_id']}",
|
||||
"technical_health": item["technical_health"],
|
||||
"rank": rank,
|
||||
}
|
||||
|
||||
|
||||
def installation_view(installation, agent, tenant, snapshot=None, *, now=None, reference=None):
|
||||
has_snapshot = snapshot is not None
|
||||
snapshot = snapshot or AgentSnapshot(installation_id=installation.id)
|
||||
health, reason = compute_agent_health(installation, now=now, supplemental=snapshot)
|
||||
# Always recomputed against the current reference release: the value stored on the snapshot was taken at the
|
||||
# last heartbeat (against whatever was "current" then) and goes stale as soon as a newer release is published.
|
||||
version_drift = compute_version_drift(agent.agent_version, reference)
|
||||
fiscal_status = _fiscal_status(snapshot)
|
||||
coverage_partition = _coverage_partition(snapshot)
|
||||
if not has_snapshot:
|
||||
# No heartbeat/snapshot yet (e.g. an enrollment-only E2E row): expose
|
||||
# no counts at all rather than fabricated zeros — the UI shows
|
||||
# "Não informado" instead of None/0.
|
||||
coverage_partition = {"unclassified_products": None, "coverage_overcount_products": 0, "coverage_invalid_counts": False, "coverage_invariant_pass": True}
|
||||
version_drift_label = {"CURRENT": "Atualizada", "OUTDATED": "Desatualizada", "UNKNOWN": "Desconhecida"}.get(version_drift, str(version_drift))
|
||||
agent_version_display = f"{agent.agent_version} · {version_drift_label}" if agent.agent_version else "Desconhecida"
|
||||
return {
|
||||
"agent_id": agent.id,
|
||||
"installation_id": installation.installation_id,
|
||||
"installation_db_id": installation.id,
|
||||
"client": tenant.name if tenant else None,
|
||||
"tenant_id": installation.tenant_id,
|
||||
"display_name": installation.display_name,
|
||||
"hostname": installation.hostname,
|
||||
"company_name": installation.company_name,
|
||||
"company_code": installation.company_code,
|
||||
"company_cnpj": installation.company_cnpj_masked,
|
||||
"company_uf": installation.company_uf,
|
||||
"company_crt": installation.company_crt,
|
||||
"agent_version": agent.agent_version,
|
||||
"agent_version_display": agent_version_display,
|
||||
"version_drift": version_drift,
|
||||
"version_drift_label": version_drift_label,
|
||||
"health": health,
|
||||
"health_reason": reason,
|
||||
"technical_health": health,
|
||||
"health_label": health_label(health),
|
||||
"technical_health_label": health_label(health),
|
||||
"last_seen_at": _iso(installation.last_seen_at),
|
||||
"last_seen_display": format_ui_datetime(installation.last_seen_at),
|
||||
"last_seen_age": format_age(installation.last_seen_at, now),
|
||||
"started_at": _iso(installation.started_at),
|
||||
"last_status": installation.last_status,
|
||||
"total_products": snapshot.total_products if has_snapshot else None,
|
||||
"fiscally_complete": snapshot.fiscally_complete if has_snapshot else None,
|
||||
"safe_blocked": snapshot.safe_blocked if has_snapshot else None,
|
||||
"failed_products": snapshot.failed_products if has_snapshot else None,
|
||||
**coverage_partition,
|
||||
"coverage_percent": snapshot.coverage_percent if has_snapshot else None,
|
||||
"fiscal_completion_percent": _fiscal_completion(snapshot) if has_snapshot else None,
|
||||
"fiscal_status": fiscal_status if has_snapshot else "UNKNOWN",
|
||||
"fiscal_status_label": fiscal_status_label(fiscal_status) if has_snapshot else "Não informado",
|
||||
"blockers": snapshot.blockers or {},
|
||||
"last_job_status": snapshot.last_job_status,
|
||||
"last_job_at": _iso(snapshot.last_job_at),
|
||||
"last_successful_job_at": _iso(snapshot.last_successful_job_at),
|
||||
"firebird_version": installation.firebird_version,
|
||||
"firebird_ods": installation.firebird_ods,
|
||||
"database_identifier": installation.database_identifier,
|
||||
"capabilities": installation.capabilities or {},
|
||||
"fiscal_engine_version": installation.fiscal_engine_version,
|
||||
"dataset_versions": installation.dataset_versions or {},
|
||||
"technical_ids": {"agent_id": agent.id, "installation_db_id": installation.id, "tenant_id": installation.tenant_id},
|
||||
"link": f"/agents/{agent.id}",
|
||||
}
|
||||
|
||||
|
||||
def _installation_query_filters(query, *, client=None, version=None, search=None, last_seen_before=None):
|
||||
if client:
|
||||
query = query.where(or_(Tenant.name.ilike(f"%{client}%"), Installation.customer_code.ilike(f"%{client}%")))
|
||||
if version and version != "outdated":
|
||||
query = query.where(Agent.agent_version == version)
|
||||
if search:
|
||||
query = query.where(or_(Installation.display_name.ilike(f"%{search}%"), Installation.hostname.ilike(f"%{search}%"), Tenant.name.ilike(f"%{search}%")))
|
||||
if last_seen_before:
|
||||
query = query.where(or_(Installation.last_seen_at.is_(None), Installation.last_seen_at < last_seen_before))
|
||||
return query
|
||||
|
||||
|
||||
def list_installations(session, *, page=1, page_size=50, client=None, health=None, version=None, search=None, last_seen_before=None, sort="display_name", scope="active", now=None):
|
||||
page, page_size = _page(page, page_size)
|
||||
now = _utc_now(now)
|
||||
reference = reference_agent_version(session)
|
||||
query = select(Installation, Agent, Tenant, AgentSnapshot, operational_installation_predicate().label("operational_active")).join(Agent, Agent.id == Installation.agent_id).join(Tenant, Tenant.id == Installation.tenant_id).outerjoin(AgentSnapshot, AgentSnapshot.installation_id == Installation.id)
|
||||
if scope == "active":
|
||||
query = query.where(operational_installation_predicate())
|
||||
elif scope != "all":
|
||||
raise ValueError("scope must be active or all")
|
||||
query = _installation_query_filters(query, client=client, version=version, search=search, last_seen_before=last_seen_before)
|
||||
if version == "outdated":
|
||||
query = query.where(Agent.agent_version.in_(outdated_versions(session, reference)))
|
||||
health_expression = _health_expression(now)
|
||||
if health:
|
||||
query = query.where(health_expression == health)
|
||||
if sort in {"worst_coverage", "coverage", "coverage_asc", "fiscal_completion_percent"}:
|
||||
ordered = query.order_by(_fiscal_completion_expression().asc(), Installation.display_name.asc(), Installation.id.asc())
|
||||
else:
|
||||
ordered = query.order_by(Installation.display_name.asc(), Installation.id.asc())
|
||||
rows = session.execute(ordered.offset((page - 1) * page_size).limit(page_size)).all()
|
||||
items = [{**installation_view(i, a, t, s, now=now, reference=reference), "operational_active": bool(active)} for i, a, t, s, active in rows]
|
||||
count_query = select(func.count()).select_from(query.with_only_columns(Installation.id).order_by(None).subquery())
|
||||
total = session.scalar(count_query) or 0
|
||||
return {"page": page, "page_size": page_size, "items": items, "total": total, "health_filter_applied": health}
|
||||
|
||||
|
||||
def _blocker_item(row, tenant, installation=None, agent=None, *, now=None):
|
||||
return {
|
||||
"id": row.id,
|
||||
"tenant_id": row.tenant_id,
|
||||
"client": tenant.name if tenant else row.company,
|
||||
"product_code": row.product_code,
|
||||
"description": row.description,
|
||||
"field": row.field,
|
||||
"blocker_type": row.blocker_type,
|
||||
"blocker_type_label": blocker_type_label(row.blocker_type),
|
||||
"reason": row.reason_code,
|
||||
"reason_code": row.reason_code,
|
||||
"reason_label": blocker_reason_label(row.reason_code),
|
||||
"evidence_summary": row.evidence_summary,
|
||||
"first_seen_at": _iso(row.first_seen_at),
|
||||
"last_seen_at": _iso(row.last_seen_at),
|
||||
"first_seen_display": format_ui_datetime(row.first_seen_at),
|
||||
"last_seen_display": format_ui_datetime(row.last_seen_at),
|
||||
"age": format_age(row.last_seen_at, now),
|
||||
"retry_count": row.retry_count,
|
||||
"resolved_at": _iso(row.resolved_at),
|
||||
"resolution_source": row.resolution_source,
|
||||
"priority": operational_priority(row),
|
||||
"priority_label": priority_label(operational_priority(row)),
|
||||
"installation_id": installation.installation_id if installation else None,
|
||||
"installation_db_id": installation.id if installation else None,
|
||||
"display_name": installation.display_name if installation else None,
|
||||
"hostname": installation.hostname if installation else None,
|
||||
"agent_id": agent.id if agent else None,
|
||||
"link": f"/blockers/{row.id}",
|
||||
}
|
||||
|
||||
|
||||
def _parse_age(value):
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return timedelta(days=float(value))
|
||||
text = str(value).strip().lower()
|
||||
try:
|
||||
if text.endswith("h"):
|
||||
return timedelta(hours=float(text[:-1]))
|
||||
if text.endswith("d"):
|
||||
return timedelta(days=float(text[:-1]))
|
||||
return timedelta(days=float(text))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _blockers_query(session, *, tenant_id=None, agent_id=None, product_code=None, blocker_type=None, field=None, reason_code=None, status="OPEN", client=None, installation=None, search=None, age=None, scope="active"):
|
||||
query = select(BlockedProduct, Tenant, Installation, Agent).join(Tenant, Tenant.id == BlockedProduct.tenant_id).outerjoin(Installation, Installation.id == BlockedProduct.installation_id).outerjoin(Agent, Agent.id == Installation.agent_id)
|
||||
if scope == "active":
|
||||
query = query.where(operational_installation_predicate(), Installation.id.is_not(None))
|
||||
elif scope != "all":
|
||||
raise ValueError("scope must be active or all")
|
||||
status = (status or "OPEN").upper()
|
||||
if status == "OPEN":
|
||||
query = query.where(BlockedProduct.resolved_at.is_(None))
|
||||
elif status in {"RESOLVED", "CLOSED"}:
|
||||
query = query.where(BlockedProduct.resolved_at.is_not(None))
|
||||
if tenant_id is not None:
|
||||
query = query.where(BlockedProduct.tenant_id == tenant_id)
|
||||
if agent_id is not None:
|
||||
query = query.where(Agent.id == agent_id)
|
||||
if product_code is not None:
|
||||
query = query.where(BlockedProduct.product_code == product_code)
|
||||
if blocker_type:
|
||||
query = query.where(BlockedProduct.blocker_type == blocker_type)
|
||||
if field:
|
||||
query = query.where(func.upper(BlockedProduct.field) == field.upper())
|
||||
if reason_code:
|
||||
query = query.where(BlockedProduct.reason_code == reason_code)
|
||||
if client:
|
||||
query = query.where(or_(Tenant.name.ilike(f"%{client}%"), BlockedProduct.company.ilike(f"%{client}%")))
|
||||
if installation:
|
||||
query = query.where(or_(Installation.installation_id.ilike(f"%{installation}%"), Installation.display_name.ilike(f"%{installation}%"), Installation.hostname.ilike(f"%{installation}%")))
|
||||
if search:
|
||||
term = f"%{search}%"
|
||||
query = query.where(or_(BlockedProduct.product_code.ilike(term), BlockedProduct.description.ilike(term), BlockedProduct.field.ilike(term), BlockedProduct.blocker_type.ilike(term), BlockedProduct.reason_code.ilike(term), Tenant.name.ilike(term)))
|
||||
age_delta = _parse_age(age)
|
||||
if age_delta:
|
||||
query = query.where(BlockedProduct.last_seen_at <= datetime.now(timezone.utc) - age_delta)
|
||||
return query
|
||||
|
||||
|
||||
def blocker_filter_summary(session, *, tenant_id=None, agent_id=None, product_code=None, blocker_type=None, field=None, reason_code=None, status="OPEN", client=None, installation=None, age=None, search=None, scope="active"):
|
||||
query = _blockers_query(session, tenant_id=tenant_id, agent_id=agent_id, product_code=product_code, blocker_type=blocker_type, field=field, reason_code=reason_code, status=status, client=client, installation=installation, search=search, age=age, scope=scope)
|
||||
open_count = func.sum(case((BlockedProduct.resolved_at.is_(None), 1), else_=0))
|
||||
open_count, = session.execute(query.with_only_columns(open_count).order_by(None)).one()
|
||||
rows = session.execute(_blocker_breakdown_query(query)).all()
|
||||
breakdown = {field: 0 for field in BLOCKER_FIELDS}
|
||||
for field_name, count in rows:
|
||||
key = str(field_name or "").upper()
|
||||
breakdown[key if key in breakdown else "OTHER"] += int(count)
|
||||
pair_query = query.with_only_columns(BlockedProduct.tenant_id, BlockedProduct.product_code).distinct().subquery()
|
||||
client_query = query.with_only_columns(BlockedProduct.tenant_id).distinct().subquery()
|
||||
return {
|
||||
"open_blockers": int(open_count or 0),
|
||||
"affected_products": int(session.scalar(select(func.count()).select_from(pair_query)) or 0),
|
||||
"affected_clients": int(session.scalar(select(func.count()).select_from(client_query)) or 0),
|
||||
"blocker_breakdown": breakdown,
|
||||
}
|
||||
|
||||
|
||||
def rtc_blocker_summary(session, agent_id: int) -> dict:
|
||||
"""Read-only, installation-scoped RTC blocker aggregates for the cClassTrib page."""
|
||||
query = _blockers_query(session, agent_id=agent_id, field="RTC", status="OPEN", scope="all")
|
||||
blockers = int(session.scalar(query.with_only_columns(func.count(BlockedProduct.id)).order_by(None)) or 0)
|
||||
products = int(session.scalar(query.with_only_columns(func.count(func.distinct(BlockedProduct.product_code))).order_by(None)) or 0)
|
||||
reason = func.coalesce(BlockedProduct.evidence_summary["reason"].as_string(), BlockedProduct.reason_code)
|
||||
reasons = session.execute(
|
||||
query.with_only_columns(reason, func.count(func.distinct(BlockedProduct.product_code)))
|
||||
.group_by(reason).order_by(None)
|
||||
).all()
|
||||
return {"open_blockers": blockers, "products_affected": products, "reason_counts": {str(code or ""): int(count) for code, count in reasons}}
|
||||
|
||||
|
||||
def _blocker_breakdown_query(query):
|
||||
"""Group on the source column itself for PostgreSQL-safe SQL semantics."""
|
||||
return query.with_only_columns(BlockedProduct.field, func.count(BlockedProduct.id)).group_by(BlockedProduct.field).order_by(None)
|
||||
|
||||
|
||||
def list_blockers(session, *, tenant_id=None, agent_id=None, product_code=None, page=1, page_size=50, blocker_type=None, field=None, reason_code=None, resolved=False, client=None, installation=None, status="OPEN", age=None, search=None, scope="active"):
|
||||
page, page_size = _page(page, page_size)
|
||||
if resolved and status == "OPEN":
|
||||
status = "RESOLVED"
|
||||
query = _blockers_query(session, tenant_id=tenant_id, agent_id=agent_id, product_code=product_code, blocker_type=blocker_type, field=field, reason_code=reason_code, status=status, client=client, installation=installation, search=search, age=age, scope=scope)
|
||||
ordered = query.order_by(BlockedProduct.last_seen_at.desc(), BlockedProduct.id.desc())
|
||||
rows = session.execute(ordered.offset((page - 1) * page_size).limit(page_size)).all()
|
||||
items = [_blocker_item(blocker, tenant, installation_row, agent, now=datetime.now(timezone.utc)) for blocker, tenant, installation_row, agent in rows]
|
||||
count_query = query.with_only_columns(func.count(BlockedProduct.id)).order_by(None)
|
||||
total = session.scalar(count_query) or 0
|
||||
return {"page": page, "page_size": page_size, "items": items, "total": total}
|
||||
|
||||
|
||||
def _job_item(row, agent, installation, tenant, *, now=None):
|
||||
return {
|
||||
"job_id": row.id,
|
||||
"agent_id": row.agent_id,
|
||||
"client": tenant.name if tenant else None,
|
||||
"display_name": installation.display_name if installation else None,
|
||||
"hostname": installation.hostname if installation else None,
|
||||
"kind": row.kind,
|
||||
"kind_label": job_kind_label(row.kind),
|
||||
"status": row.status,
|
||||
"status_label": job_status_label(row.status),
|
||||
"created_at": _iso(row.created_at),
|
||||
"started_at": _iso(row.started_at),
|
||||
"finished_at": _iso(row.finished_at),
|
||||
"created_display": format_ui_datetime(row.created_at),
|
||||
"started_display": format_ui_datetime(row.started_at),
|
||||
"finished_display": format_ui_datetime(row.finished_at),
|
||||
"duration": format_duration(row.started_at, row.finished_at, now),
|
||||
"attempts": row.attempts,
|
||||
"result_summary": row.result_summary,
|
||||
"error": row.error,
|
||||
}
|
||||
|
||||
|
||||
def list_jobs(session, *, agent_id=None, job_id=None, page=1, page_size=50, status=None, kind=None, since=None, until=None, client=None, search=None, scope="active"):
|
||||
page, page_size = _page(page, page_size)
|
||||
query = select(Job, Agent, Installation, Tenant).join(Agent, Agent.id == Job.agent_id).join(Installation, Installation.agent_id == Agent.id).join(Tenant, Tenant.id == Installation.tenant_id)
|
||||
if scope == "active":
|
||||
query = query.where(operational_installation_predicate())
|
||||
elif scope != "all":
|
||||
raise ValueError("scope must be active or all")
|
||||
if agent_id is not None:
|
||||
query = query.where(Job.agent_id == agent_id)
|
||||
if job_id is not None:
|
||||
query = query.where(Job.id == job_id)
|
||||
if status:
|
||||
query = query.where(Job.status == status)
|
||||
if kind:
|
||||
query = query.where(Job.kind == kind)
|
||||
if since:
|
||||
query = query.where(Job.created_at >= since)
|
||||
if until:
|
||||
query = query.where(Job.created_at <= until)
|
||||
if client:
|
||||
query = query.where(or_(Tenant.name.ilike(f"%{client}%"), Installation.customer_code.ilike(f"%{client}%")))
|
||||
if search:
|
||||
term = f"%{search}%"
|
||||
query = query.where(or_(Tenant.name.ilike(term), Installation.display_name.ilike(term), Installation.hostname.ilike(term), Job.kind.ilike(term), Job.status.ilike(term)))
|
||||
ordered = query.order_by(Job.created_at.desc(), Job.id.desc())
|
||||
rows = session.execute(ordered.offset((page - 1) * page_size).limit(page_size)).all()
|
||||
count_query = query.with_only_columns(func.count(Job.id)).order_by(None)
|
||||
total = session.scalar(count_query) or 0
|
||||
return {"page": page, "page_size": page_size, "items": [_job_item(row, agent, installation, tenant) for row, agent, installation, tenant in rows], "total": total}
|
||||
|
||||
|
||||
def jobs_status_summary(session, *, scope="active"):
|
||||
"""Bounded job counts for the current operational scope."""
|
||||
query = (select(Job.status, func.count(Job.id)).join(Agent, Agent.id == Job.agent_id)
|
||||
.join(Installation, Installation.agent_id == Agent.id)
|
||||
.join(Tenant, Tenant.id == Installation.tenant_id))
|
||||
if scope == "active":
|
||||
query = query.where(operational_installation_predicate())
|
||||
elif scope != "all":
|
||||
raise ValueError("scope must be active or all")
|
||||
counts = {status: 0 for status in ("PENDING", "RUNNING", "COMPLETED", "FAILED")}
|
||||
for status, count in session.execute(query.group_by(Job.status)):
|
||||
if status in counts:
|
||||
counts[status] = int(count)
|
||||
return counts
|
||||
|
||||
|
||||
def dashboard_summary(session, now=None):
|
||||
now = _utc_now(now)
|
||||
reference = reference_agent_version(session)
|
||||
active = operational_installation_predicate()
|
||||
installation_query = select(Installation, Agent, Tenant, AgentSnapshot).join(Agent, Agent.id == Installation.agent_id).join(Tenant, Tenant.id == Installation.tenant_id).outerjoin(AgentSnapshot, AgentSnapshot.installation_id == Installation.id).where(active)
|
||||
health_expression = _health_expression(now)
|
||||
health_rows = session.execute(
|
||||
select(health_expression.label("health"), func.count(Installation.id))
|
||||
.select_from(Installation)
|
||||
.join(Agent, Agent.id == Installation.agent_id)
|
||||
.join(Tenant, Tenant.id == Installation.tenant_id)
|
||||
.outerjoin(AgentSnapshot, AgentSnapshot.installation_id == Installation.id)
|
||||
.where(active)
|
||||
.group_by(health_expression)
|
||||
).all()
|
||||
counts = {state: 0 for state in HEALTH_STATES}
|
||||
for state, count in health_rows:
|
||||
counts[state] = int(count)
|
||||
|
||||
snapshot_total = func.coalesce(AgentSnapshot.total_products, 0)
|
||||
snapshot_complete = func.coalesce(AgentSnapshot.fiscally_complete, 0)
|
||||
snapshot_blocked = func.coalesce(AgentSnapshot.safe_blocked, 0)
|
||||
snapshot_failed = func.coalesce(AgentSnapshot.failed_products, 0)
|
||||
snapshot_explicit = snapshot_complete + snapshot_blocked + snapshot_failed
|
||||
snapshot_has_negative = or_(snapshot_total < 0, snapshot_complete < 0, snapshot_blocked < 0, snapshot_failed < 0)
|
||||
total_row = session.execute(
|
||||
select(
|
||||
func.coalesce(func.sum(AgentSnapshot.total_products), 0),
|
||||
func.coalesce(func.sum(AgentSnapshot.fiscally_complete), 0),
|
||||
func.coalesce(func.sum(AgentSnapshot.safe_blocked), 0),
|
||||
func.coalesce(func.sum(AgentSnapshot.failed_products), 0),
|
||||
func.coalesce(func.sum(case((and_(~snapshot_has_negative, snapshot_explicit <= snapshot_total), snapshot_total - snapshot_explicit), else_=0)), 0),
|
||||
func.coalesce(func.sum(case((and_(~snapshot_has_negative, snapshot_explicit > snapshot_total), snapshot_explicit - snapshot_total), else_=0)), 0),
|
||||
func.coalesce(func.sum(case((snapshot_has_negative, 1), else_=0)), 0),
|
||||
).select_from(Installation).join(Agent, Agent.id == Installation.agent_id).join(Tenant, Tenant.id == Installation.tenant_id).outerjoin(AgentSnapshot, AgentSnapshot.installation_id == Installation.id).where(active)
|
||||
).one()
|
||||
totals = dict(zip(("total_products", "fiscally_complete", "safe_blocked", "failed_products", "unclassified_products", "coverage_overcount_products", "coverage_invalid_snapshots"), (int(value or 0) for value in total_row)))
|
||||
totals["coverage_invariant_pass"] = totals["coverage_overcount_products"] == 0 and totals["coverage_invalid_snapshots"] == 0
|
||||
|
||||
open_filter = BlockedProduct.resolved_at.is_(None)
|
||||
active_blockers = (select(BlockedProduct).join(Installation, Installation.id == BlockedProduct.installation_id)
|
||||
.join(Agent, Agent.id == Installation.agent_id).join(Tenant, Tenant.id == BlockedProduct.tenant_id)
|
||||
.where(open_filter, active))
|
||||
open_blockers = session.scalar(active_blockers.with_only_columns(func.count(BlockedProduct.id)).order_by(None)) or 0
|
||||
breakdown_expression = func.coalesce(BlockedProduct.field, "OTHER")
|
||||
breakdown_rows = session.execute(active_blockers.with_only_columns(breakdown_expression, func.count(BlockedProduct.id)).group_by(breakdown_expression).order_by(None)).all()
|
||||
open_blocker_breakdown = {field: 0 for field in BLOCKER_FIELDS}
|
||||
for field, count in breakdown_rows:
|
||||
key = str(field or "").upper()
|
||||
open_blocker_breakdown[key if key in open_blocker_breakdown else "OTHER"] += int(count)
|
||||
product_pairs = active_blockers.with_only_columns(BlockedProduct.tenant_id, BlockedProduct.product_code).distinct().subquery()
|
||||
safe_blocked_products = session.scalar(select(func.count()).select_from(product_pairs)) or 0
|
||||
blocker_counts = {field: count for field, count in open_blocker_breakdown.items() if count}
|
||||
|
||||
def installation_candidates(health):
|
||||
query = installation_query.where(health_expression == health).order_by(Installation.id.asc()).limit(ATTENTION_CANDIDATE_LIMIT)
|
||||
return [installation_view(i, a, t, s, now=now, reference=reference) for i, a, t, s in session.execute(query).all()]
|
||||
|
||||
health_candidates = {health: installation_candidates(health) for health in ("FAILED", "DEGRADED", "OFFLINE", "UNKNOWN")}
|
||||
version_filter = Agent.agent_version.in_(outdated_versions(session, reference))
|
||||
outdated_query = installation_query.where(version_filter).order_by(Installation.id.asc()).limit(ATTENTION_CANDIDATE_LIMIT)
|
||||
outdated_agents = [installation_view(i, a, t, s, now=now, reference=reference) for i, a, t, s in session.execute(outdated_query).all()]
|
||||
|
||||
group_field = func.upper(func.coalesce(BlockedProduct.field, "OTHER"))
|
||||
fiscal_priority = case(
|
||||
(func.upper(BlockedProduct.blocker_type).in_(("TECHNICAL_FAILURE", "CONNECTION_ERROR", "AGENT_OFFLINE", "JOB_FAILED")), 0),
|
||||
(func.upper(BlockedProduct.blocker_type).in_(("BLOCKED_MISSING_EVIDENCE", "MISSING_EVIDENCE")), 1),
|
||||
(group_field.in_(BLOCKER_FIELDS[:-1]), 2),
|
||||
else_=3,
|
||||
)
|
||||
blocker_groups = session.execute(
|
||||
active_blockers.with_only_columns(
|
||||
Tenant.name, Installation.display_name, Installation.hostname, Agent.id,
|
||||
group_field, BlockedProduct.blocker_type, BlockedProduct.reason_code,
|
||||
func.count(BlockedProduct.id), func.count(func.distinct(BlockedProduct.product_code)),
|
||||
func.max(BlockedProduct.last_seen_at),
|
||||
).group_by(Tenant.name, Installation.id, Installation.display_name, Installation.hostname,
|
||||
Agent.id, group_field, BlockedProduct.blocker_type, BlockedProduct.reason_code)
|
||||
.order_by(fiscal_priority.asc(), func.max(BlockedProduct.last_seen_at).desc(), Agent.id.asc())
|
||||
.limit(ATTENTION_CANDIDATE_LIMIT)
|
||||
).all()
|
||||
|
||||
attention = []
|
||||
for health in ("FAILED", "DEGRADED", "OFFLINE", "UNKNOWN"):
|
||||
rank = {"FAILED": 0, "DEGRADED": 1, "OFFLINE": 2, "UNKNOWN": 3}[health]
|
||||
attention.extend((_attention_item(item, health_reason_label(item["health_reason"]), _technical_timestamp(item), rank=rank, now=now), rank, item["agent_id"]) for item in health_candidates[health])
|
||||
for client_name, display_name, hostname, agent_id, field, blocker_type, reason, count, products, last_seen, in blocker_groups:
|
||||
item = {"client": client_name, "display_name": display_name, "hostname": hostname, "agent_id": agent_id, "technical_health": "BLOCKED"}
|
||||
link = "/blockers?" + urlencode({"scope": "active", "status": "OPEN", "agent_id": agent_id, "field": field, "blocker_type": blocker_type, "reason_code": reason})
|
||||
priority = operational_priority({"field": field, "blocker_type": blocker_type, "reason_code": reason})
|
||||
rank = {"CRITICAL": 3, "HIGH": 4, "NORMAL": 5, "LOW": 6}[priority]
|
||||
entry = _attention_item(item, f"{field}: {blocker_reason_label(reason)}", last_seen, rank=rank, link=link, now=now)
|
||||
entry.update(group_count=count, products_affected=products, field=field, priority=priority)
|
||||
attention.append((entry, rank, agent_id))
|
||||
for item in outdated_agents:
|
||||
attention.append((_attention_item(item, "Versão do agent desatualizada", item.get("last_seen_at"), rank=7, now=now), 7, item["agent_id"]))
|
||||
attention.sort(key=lambda entry: (entry[1], -(entry[0]["timestamp"] is not None), entry[2]))
|
||||
|
||||
active_jobs = select(Job).join(Agent, Agent.id == Job.agent_id).join(Installation, Installation.agent_id == Agent.id).join(Tenant, Tenant.id == Installation.tenant_id).where(active)
|
||||
failed_jobs = session.scalar(active_jobs.with_only_columns(func.count(Job.id)).where(Job.status == "FAILED").order_by(None)) or 0
|
||||
failed_job_rows = session.execute(
|
||||
select(Job, Agent, Installation, Tenant).join(Agent, Agent.id == Job.agent_id)
|
||||
.join(Installation, Installation.agent_id == Agent.id).join(Tenant, Tenant.id == Installation.tenant_id)
|
||||
.where(Job.status == "FAILED", active).order_by(Job.created_at.desc(), Job.id.desc()).limit(10)
|
||||
).all()
|
||||
recent_failed_jobs = []
|
||||
for job, agent, installation, tenant in failed_job_rows:
|
||||
item = _job_item(job, agent, installation, tenant, now=now)
|
||||
item["error"] = (item["error"] or "")[:240]
|
||||
item["created_age"] = format_age(job.created_at, now)
|
||||
item["finished_age"] = format_age(job.finished_at, now)
|
||||
item["link"] = f"/jobs?job_id={job.id}"
|
||||
recent_failed_jobs.append(item)
|
||||
return {
|
||||
"agents_total": sum(counts.values()),
|
||||
**{key.lower(): value for key, value in counts.items()},
|
||||
"health_counts": counts,
|
||||
**totals,
|
||||
"blockers": blocker_counts,
|
||||
"stale_agents": health_candidates["OFFLINE"],
|
||||
"outdated_agents": outdated_agents,
|
||||
"safe_blocked_products": int(safe_blocked_products),
|
||||
"open_blockers": int(open_blockers),
|
||||
"open_blocker_breakdown": open_blocker_breakdown,
|
||||
"attention_items": [entry[0] for entry in attention[:10]],
|
||||
"failed_jobs": failed_jobs,
|
||||
"recent_failed_jobs": recent_failed_jobs,
|
||||
}
|
||||
|
||||
|
||||
def blocker_detail(session, blocker_id):
|
||||
row = session.execute(_blockers_query(session, status="ALL", scope="all").where(BlockedProduct.id == blocker_id)).first()
|
||||
if not row:
|
||||
return None
|
||||
blocker, tenant, installation, agent = row
|
||||
item = _blocker_item(blocker, tenant, installation, agent)
|
||||
events = session.scalars(select(AgentEvent).where(AgentEvent.blocker_id == blocker_id).order_by(AgentEvent.occurred_at.desc(), AgentEvent.id.desc()).limit(100)).all()
|
||||
event_views = [{"id": event.id, "severity": event.severity, "event_type": event.event_type, "message": event.message, "context": event.structured_context, "job_id": event.job_id, "product_code": event.product_code, "occurred_at": _iso(event.occurred_at)} for event in events]
|
||||
job_ids = {event.job_id for event in events if event.job_id is not None}
|
||||
related_jobs = []
|
||||
if job_ids:
|
||||
job_query = select(Job, Agent, Installation, Tenant).join(Agent, Agent.id == Job.agent_id).join(Installation, Installation.agent_id == Agent.id).join(Tenant, Tenant.id == Installation.tenant_id).where(Job.id.in_(job_ids)).order_by(Job.created_at.desc(), Job.id.desc()).limit(100)
|
||||
related_jobs = [_job_item(job, related_agent, related_installation, related_tenant) for job, related_agent, related_installation, related_tenant in session.execute(job_query).all()]
|
||||
audits = session.scalars(select(AuditEvent).where(AuditEvent.target_type == "blocker", AuditEvent.target_id == str(blocker_id)).order_by(AuditEvent.occurred_at.desc(), AuditEvent.id.desc()).limit(100)).all()
|
||||
audit_views = [{"id": audit.id, "admin_user_id": audit.admin_user_id, "action": audit.action, "target_id": audit.target_id, "context": audit.context, "severity": audit.severity, "occurred_at": _iso(audit.occurred_at)} for audit in audits]
|
||||
server = {
|
||||
"hostname": installation.hostname if installation else None,
|
||||
"company_name": installation.company_name if installation else None,
|
||||
"company_code": installation.company_code if installation else None,
|
||||
"company_uf": installation.company_uf if installation else None,
|
||||
"company_crt": installation.company_crt if installation else None,
|
||||
"firebird_version": installation.firebird_version if installation else None,
|
||||
"firebird_ods": installation.firebird_ods if installation else None,
|
||||
"database_identifier": installation.database_identifier if installation else None,
|
||||
}
|
||||
return {
|
||||
**item,
|
||||
"evidence": blocker.evidence_summary or {},
|
||||
"installation": {"installation_id": item["installation_id"], "installation_db_id": item["installation_db_id"], "display_name": item["display_name"], "hostname": item["hostname"], "agent_id": item["agent_id"]},
|
||||
"server": server,
|
||||
"related_events": event_views,
|
||||
"related_jobs": related_jobs,
|
||||
"resolution_audit": audit_views,
|
||||
}
|
||||
|
||||
|
||||
def list_panel_events(session, installation_db_id, *, include_more=False, **filters):
|
||||
rows = list_events(session, installation_db_id, fetch_extra=include_more, **filters)
|
||||
return [{"id": row.id, "severity": row.severity, "event_type": row.event_type, "message": row.message, "context": row.structured_context, "job_id": row.job_id, "product_code": row.product_code, "occurred_at": _iso(row.occurred_at)} for row in rows]
|
||||
@@ -0,0 +1 @@
|
||||
"""Official fiscal dataset importers."""
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Fail-closed text parser for the official RICMS/BA Anexo 1 PDF."""
|
||||
import re
|
||||
|
||||
_ITEM = re.compile(r"^\s*(\d{1,2}(?:\.\d+)+)\s+(?:(\d{2}\.\d{3}\.\d{2})\s+)?(.*)$")
|
||||
_NCM = re.compile(r"\b(?:\d{4}(?:\.\d{1,4}){0,3}|\d{2}(?:\.\d{1,4}){1,3})\b")
|
||||
_MVA = re.compile(r"\d+(?:,\d+)?%")
|
||||
|
||||
def parse_page(text, page):
|
||||
records, rejected = [], []
|
||||
lines = [" ".join(line.split()) for line in text.splitlines()]
|
||||
index = 0
|
||||
while index < len(lines):
|
||||
match = _ITEM.match(lines[index])
|
||||
if not match:
|
||||
index += 1; continue
|
||||
item, cest, content = match.groups(); block = [content]; index += 1
|
||||
while index < len(lines) and not _ITEM.match(lines[index]) and not lines[index].startswith("ITEM CEST"):
|
||||
block.append(lines[index]); index += 1
|
||||
joined = " ".join(block)
|
||||
ncm = _NCM.findall(joined)
|
||||
if not cest and not ncm:
|
||||
rejected.append({"item": item, "source_page": page, "reason": "NO_CEST_OR_NCM"}); continue
|
||||
mvas = _MVA.findall(joined)
|
||||
description = joined
|
||||
if cest: description = description.replace(cest, "")
|
||||
for code in ncm: description = description.replace(code, "")
|
||||
for mva in mvas: description = description.replace(mva, "")
|
||||
records.append({"item": item, "cest": cest, "ncm_sh": ncm or None, "description": " ".join(description.split()) or None, "mva_original": mvas[-1] if mvas else None, "mva_adjusted": mvas[0] if mvas else None, "agreement_info": None, "source_page": page})
|
||||
return records, rejected
|
||||
@@ -0,0 +1,287 @@
|
||||
"""Parsers for the official NF-e/RTC spreadsheets the operator placed in
|
||||
sources/raw/bootstrap_v1/ after the Portal NF-e/SPED downloads were blocked
|
||||
in earlier missions. Each parser validates the workbook's own header/sheet
|
||||
structure before trusting it — never trusts the filename alone.
|
||||
|
||||
Domains: CFOP, FCP, ANP (cProdANP), NCM x uTrib, RTC biofuel index
|
||||
(pBioObrigatorio), RTC cCredPres.
|
||||
"""
|
||||
from dataclasses import dataclass, field
|
||||
from hashlib import sha256
|
||||
from io import BytesIO
|
||||
|
||||
|
||||
def _digits(value):
|
||||
return "".join(c for c in str(value or "") if c.isdigit())
|
||||
|
||||
|
||||
def _iso(value):
|
||||
if value is None:
|
||||
return None
|
||||
return value.isoformat() if hasattr(value, "isoformat") else str(value)
|
||||
|
||||
|
||||
def _load(raw_bytes):
|
||||
import openpyxl
|
||||
return openpyxl.load_workbook(BytesIO(raw_bytes), read_only=True, data_only=True)
|
||||
|
||||
|
||||
def _rows(ws, header_len):
|
||||
rows = ws.iter_rows(values_only=True)
|
||||
header = next(rows)
|
||||
return header[:header_len], rows
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Payload:
|
||||
sha256: str
|
||||
entries: tuple
|
||||
parser_version: str
|
||||
extra_entries: tuple = ()
|
||||
|
||||
|
||||
# --- CFOP -------------------------------------------------------------
|
||||
_CFOP_EXPECTED_HEADER = ("CFOP", "Início de vigência", "Fim de vigência", "indNFe")
|
||||
|
||||
|
||||
def parse_cfop(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
if "CFOP" not in wb.sheetnames:
|
||||
raise ValueError(f"expected a 'CFOP' sheet, found {wb.sheetnames}")
|
||||
header, rows = _rows(wb["CFOP"], 14)
|
||||
if _digits("".join(str(h) for h in header[:1])) or "CFOP" not in str(header[0]):
|
||||
raise ValueError(f"unexpected CFOP header: {header}")
|
||||
entries = []
|
||||
for row in rows:
|
||||
if row[0] is None:
|
||||
continue
|
||||
code = str(int(row[0])) if isinstance(row[0], (int, float)) else str(row[0]).strip()
|
||||
entries.append({
|
||||
"code": code, "effective_from": _iso(row[1]), "effective_to": _iso(row[2]),
|
||||
"ind_nfe": row[3], "ind_comunica": row[4], "ind_transp": row[5], "ind_devol": row[6],
|
||||
"ind_retor": row[7], "ind_anula": row[8], "ind_remes": row[9], "ind_comb": row[10],
|
||||
"ind_exc_ibscbs": row[11], "title": row[12] or "", "description": row[13],
|
||||
})
|
||||
if len(entries) < 100:
|
||||
raise ValueError(f"expected at least 100 CFOP codes, found {len(entries)}")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "cfop-xlsx-v1")
|
||||
|
||||
|
||||
# --- FCP ----------------------------------------------------------------
|
||||
def parse_fcp(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
if "FCP" not in wb.sheetnames:
|
||||
raise ValueError(f"expected an 'FCP' sheet, found {wb.sheetnames}")
|
||||
header, rows = _rows(wb["FCP"], 8)
|
||||
if "UF" not in [str(h) for h in header]:
|
||||
raise ValueError(f"unexpected FCP header: {header}")
|
||||
entries = []
|
||||
for row in rows:
|
||||
if not row[1]:
|
||||
continue
|
||||
entries.append({
|
||||
"uf_code": str(row[0]), "uf": str(row[1]).strip(), "uf_name": str(row[2] or "").strip(),
|
||||
"aliq_1": row[3], "aliq_2": row[4], "aliq_3": row[5], "observacao": row[7],
|
||||
})
|
||||
if len(entries) < 20:
|
||||
raise ValueError(f"expected at least 20 UF rows (26 states + DF), found {len(entries)}")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "fcp-xlsx-v1")
|
||||
|
||||
|
||||
# --- ANP ------------------------------------------------------------------
|
||||
def parse_anp(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
if "cProdANP" not in wb.sheetnames:
|
||||
raise ValueError(f"expected a 'cProdANP' sheet, found {wb.sheetnames}")
|
||||
ws = wb["cProdANP"]
|
||||
rows = ws.iter_rows(values_only=True)
|
||||
header = next(rows)
|
||||
if "cProdANP" not in str(header[0]):
|
||||
raise ValueError(f"unexpected ANP header: {header}")
|
||||
entries = []
|
||||
seen = set()
|
||||
for row in rows:
|
||||
if row[0] is None:
|
||||
continue
|
||||
code = str(row[0]).strip()
|
||||
if code in seen:
|
||||
continue
|
||||
seen.add(code)
|
||||
entries.append({"cprod_anp": code})
|
||||
if len(entries) < 100:
|
||||
raise ValueError(f"expected at least 100 cProdANP codes, found {len(entries)}")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "anp-xlsx-v1")
|
||||
|
||||
|
||||
# --- NCM x uTrib ------------------------------------------------------
|
||||
def parse_ncm_utrib(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
sheet_name = next((s for s in wb.sheetnames if "uTrib" in s or "utrib" in s.lower()), None)
|
||||
if sheet_name is None:
|
||||
raise ValueError(f"expected a sheet with 'uTrib' in its name, found {wb.sheetnames}")
|
||||
ws = wb[sheet_name]
|
||||
rows = ws.iter_rows(values_only=True)
|
||||
header = next(rows)
|
||||
if "NCM" not in str(header[0]):
|
||||
raise ValueError(f"unexpected NCM x uTrib header: {header}")
|
||||
entries = []
|
||||
for row in rows:
|
||||
if row[0] is None:
|
||||
continue
|
||||
entries.append({
|
||||
"ncm": _digits(row[0]), "effective_from": _iso(row[1]), "effective_to": _iso(row[2]),
|
||||
"utrib_abbreviation": row[3], "utrib_description": row[4],
|
||||
})
|
||||
if len(entries) < 1000:
|
||||
raise ValueError(f"expected at least 1000 NCM x uTrib rows, found {len(entries)}")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "ncm-utrib-xlsx-v1")
|
||||
|
||||
|
||||
# --- RTC biofuel index (pBioObrigatorio) --------------------------------
|
||||
def parse_rtc_biofuel_index(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
current_sheet = next((s for s in wb.sheetnames if "Historico" not in s and "pBio" in s), None)
|
||||
historical_sheet = next((s for s in wb.sheetnames if "Historico" in s), None)
|
||||
if current_sheet is None:
|
||||
raise ValueError(f"expected a 'pBioObrigatorio' sheet, found {wb.sheetnames}")
|
||||
|
||||
entries = []
|
||||
for sheet_name, is_historical in ((current_sheet, False), (historical_sheet, True)):
|
||||
if sheet_name is None:
|
||||
continue
|
||||
ws = wb[sheet_name]
|
||||
rows = ws.iter_rows(values_only=True)
|
||||
header = next(rows)
|
||||
if "cProdANP" not in str(header[0]):
|
||||
raise ValueError(f"unexpected biofuel index header in {sheet_name}: {header}")
|
||||
last_code, last_desc = None, None
|
||||
for row in rows:
|
||||
if row[0] is None and row[1] is None and row[2] is None:
|
||||
continue
|
||||
code = str(row[0]).strip() if row[0] is not None else last_code
|
||||
desc = row[1] if row[1] is not None else last_desc
|
||||
last_code, last_desc = code, desc
|
||||
entries.append({
|
||||
"cprod_anp": code, "description": desc, "p_bio_percent": row[2],
|
||||
"effective_from": _iso(row[3]), "effective_to": _iso(row[4]),
|
||||
"is_historical": is_historical,
|
||||
})
|
||||
if len(entries) < 5:
|
||||
raise ValueError(f"expected at least 5 biofuel index rows, found {len(entries)}")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "rtc-biofuel-index-xlsx-v1")
|
||||
|
||||
|
||||
# --- RTC cCredPres --------------------------------------------------------
|
||||
def parse_rtc_cred_pres(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
if "cCredPres" not in wb.sheetnames:
|
||||
raise ValueError(f"expected a 'cCredPres' sheet, found {wb.sheetnames}")
|
||||
header, rows = _rows(wb["cCredPres"], 18)
|
||||
if "cCredPres" not in str(header[0]):
|
||||
raise ValueError(f"unexpected cCredPres header: {header}")
|
||||
entries = []
|
||||
for row in rows:
|
||||
if row[0] is None:
|
||||
continue
|
||||
entries.append({
|
||||
"code": str(int(row[0])) if isinstance(row[0], (int, float)) else str(row[0]).strip(),
|
||||
"description": row[1] or "", "legal_basis": row[2],
|
||||
"apropria_via_nf": bool(row[3]) if row[3] is not None else None,
|
||||
"apropria_via_evento": bool(row[4]) if row[4] is not None else None,
|
||||
"ind_deduz_cred_pres": row[5], "ind_g_cbs_cred_pres": row[6], "ind_g_ibs_cred_pres": row[7],
|
||||
"aliquota_cbs_text": row[8], "aliquota_ibs_text": row[9],
|
||||
"p_aliq_cred_pres_cbs": row[10], "p_aliq_cred_pres_ibs": row[11], "p_red_transicao_ibs": row[12],
|
||||
"cclass_nota_referenciada": str(row[13]) if row[13] is not None else None,
|
||||
"d_ini_vig_cbs": _iso(row[14]), "d_fim_vig_cbs": _iso(row[15]),
|
||||
"d_ini_vig_ibs": _iso(row[16]), "d_fim_vig_ibs": _iso(row[17]),
|
||||
})
|
||||
if len(entries) < 1:
|
||||
raise ValueError("expected at least 1 cCredPres row, found 0")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "rtc-cred-pres-xlsx-v1")
|
||||
|
||||
|
||||
def _txt(value):
|
||||
return None if value is None else str(value).strip()
|
||||
|
||||
|
||||
def _code(value, width):
|
||||
text = _txt(value) or ""
|
||||
return text.zfill(width) if text.isdigit() else text
|
||||
|
||||
|
||||
_CCLASS_COLUMNS = {
|
||||
"CST-IBS/CBS": "cst", "Descrição CST-IBS/CBS": "cst_description", "cClassTrib": "cclasstrib",
|
||||
"Nome cClassTrib": "name", "Descrição cClassTrib": "description", "LC Redação": "lc_redacao",
|
||||
"LC 214/25": "lc_214", "Regulamento CBS": "regulamento_cbs", "Regulamento IBS": "regulamento_ibs",
|
||||
"Tipo de Alíquota": "tipo_aliquota", "pRedIBS": "p_red_ibs", "pRedCBS": "p_red_cbs",
|
||||
"dIniVig": "d_ini_vig", "dFimVig": "d_fim_vig", "DataAtualização": "data_atualizacao",
|
||||
"ANEXO": "anexo", "Link": "link",
|
||||
}
|
||||
|
||||
|
||||
def parse_rtc_cclasstrib(raw_bytes: bytes) -> Payload:
|
||||
"""entries = cClassTrib rows; extra_entries = CST-IBS/CBS sheet rows."""
|
||||
wb = _load(raw_bytes)
|
||||
cst_sheet = next((n for n in wb.sheetnames if n.startswith("CST")), None)
|
||||
cclass_sheet = next((n for n in wb.sheetnames if n.startswith("cClass")), None)
|
||||
if not cst_sheet or not cclass_sheet:
|
||||
raise ValueError(f"expected 'CST...' and 'cClass...' sheets, found {wb.sheetnames}")
|
||||
|
||||
rows = wb[cst_sheet].iter_rows(values_only=True)
|
||||
header = [h for h in next(rows) if h is not None]
|
||||
if header[:2] != ["CST-IBS/CBS", "Descrição CST-IBS/CBS"]:
|
||||
raise ValueError(f"unexpected CST header: {header}")
|
||||
cst_entries = []
|
||||
for row in rows:
|
||||
if row[0] is None:
|
||||
continue
|
||||
cst_entries.append({
|
||||
"cst": _code(row[0], 3), "description": _txt(row[1]) or "",
|
||||
"indicators": {h.replace(" ", ""): _txt(v) for h, v in zip(header[2:], row[2:])},
|
||||
})
|
||||
|
||||
rows = wb[cclass_sheet].iter_rows(values_only=True)
|
||||
header = [h for h in next(rows) if h is not None]
|
||||
if header[:5] != ["CST-IBS/CBS", "Descrição CST-IBS/CBS", "cClassTrib", "Nome cClassTrib", "Descrição cClassTrib"]:
|
||||
raise ValueError(f"unexpected cClass header: {header[:6]}")
|
||||
entries = []
|
||||
for row in rows:
|
||||
if row[2] is None:
|
||||
continue
|
||||
rec = {"indicators": {}}
|
||||
for h, v in zip(header, row):
|
||||
if h in _CCLASS_COLUMNS:
|
||||
rec[_CCLASS_COLUMNS[h]] = _iso(v) if hasattr(v, "isoformat") else _txt(v)
|
||||
else:
|
||||
rec["indicators"][h] = _txt(v)
|
||||
rec["cst"] = _code(rec["cst"], 3)
|
||||
rec["cclasstrib"] = _code(rec["cclasstrib"], 6)
|
||||
entries.append(rec)
|
||||
if len(entries) < 100 or len(cst_entries) < 10:
|
||||
raise ValueError(f"too few rows: cClass={len(entries)} CST={len(cst_entries)}")
|
||||
orphans = {e["cst"] for e in entries} - {e["cst"] for e in cst_entries}
|
||||
if orphans:
|
||||
raise ValueError(f"cClassTrib rows reference unknown CST codes: {sorted(orphans)}")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "rtc-cclasstrib-xlsx-v1", tuple(cst_entries))
|
||||
|
||||
|
||||
def parse_rtc_cbs_rates(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
rows = list(wb.worksheets[0].iter_rows(values_only=True))
|
||||
if len(rows) < 3 or "CBS" not in str(rows[0][0]).upper() or [str(c).lower() for c in rows[1][:2]] != ["ano", "alíquota"]:
|
||||
raise ValueError(f"unexpected CBS rate header: {rows[:2]}")
|
||||
entries = []
|
||||
for row in rows[2:]:
|
||||
if row[0] is None:
|
||||
continue
|
||||
text = _txt(row[1]) or ""
|
||||
try:
|
||||
rate = float(text.replace(",", "."))
|
||||
except ValueError:
|
||||
rate = None
|
||||
entries.append({"year": int(float(str(row[0]))), "rate_text": text, "rate": rate})
|
||||
if not entries or all(e["rate"] is None for e in entries):
|
||||
raise ValueError("no numeric CBS rate found")
|
||||
if len({e["year"] for e in entries}) != len(entries):
|
||||
raise ValueError("duplicate CBS rate year")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "rtc-cbs-rates-xlsx-v1")
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Fail-closed parser for CONFAZ Convenio ICMS 142/18 CEST HTML."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
from html.parser import HTMLParser
|
||||
import re
|
||||
|
||||
|
||||
_ANNEX_NUMBERS = set(range(2, 27))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CestEntry:
|
||||
code: str
|
||||
description: str
|
||||
annex: str
|
||||
ncm_codes: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CestPayload:
|
||||
version: str
|
||||
sha256: str
|
||||
entries: tuple[CestEntry, ...]
|
||||
|
||||
|
||||
class _TableParser(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.annex = None
|
||||
self.tables = []
|
||||
self._table = None
|
||||
self._row = None
|
||||
self._cell = None
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag == "table":
|
||||
self._table = []
|
||||
elif tag == "tr" and self._table is not None:
|
||||
self._row = []
|
||||
elif tag in {"td", "th"} and self._row is not None:
|
||||
self._cell = []
|
||||
elif tag == "br" and self._cell is not None:
|
||||
self._cell.append(" ")
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag in {"td", "th"} and self._cell is not None:
|
||||
self._row.append("".join(self._cell).strip())
|
||||
self._cell = None
|
||||
elif tag == "tr" and self._row is not None:
|
||||
self._table.append(self._row)
|
||||
self._row = None
|
||||
elif tag == "table" and self._table is not None:
|
||||
self.tables.append((self.annex, self._table))
|
||||
self._table = None
|
||||
|
||||
def handle_data(self, data):
|
||||
if self._cell is not None:
|
||||
self._cell.append(data)
|
||||
else:
|
||||
found = re.search(r"\bANEXO\s+([IVXLCDM]+)\b", data.upper())
|
||||
if found:
|
||||
self.annex = found.group(1)
|
||||
|
||||
|
||||
def _roman_to_number(value: str) -> int:
|
||||
values = {"I": 1, "V": 5, "X": 10, "L": 50, "C": 100, "D": 500, "M": 1000}
|
||||
total = 0
|
||||
previous = 0
|
||||
for character in reversed(value):
|
||||
current = values[character]
|
||||
total += -current if current < previous else current
|
||||
previous = max(previous, current)
|
||||
return total
|
||||
|
||||
|
||||
def _normalized_header(value: str) -> str:
|
||||
return re.sub(r"[^A-Z0-9]+", "", value.upper())
|
||||
|
||||
|
||||
def _codes(value: str) -> tuple[str, ...]:
|
||||
return tuple(dict.fromkeys(re.sub(r"\D", "", item) for item in re.findall(r"\b\d{2}(?:[.\s]?\d{2}){0,3}\b", value)))
|
||||
|
||||
|
||||
def parse_cest_html(payload: bytes) -> CestPayload:
|
||||
try:
|
||||
html = payload.decode("utf-8")
|
||||
except UnicodeDecodeError as error:
|
||||
raise ValueError("official CEST HTML must be UTF-8") from error
|
||||
parser = _TableParser()
|
||||
parser.feed(html)
|
||||
entries = []
|
||||
for annex, table in parser.tables:
|
||||
if annex is None or _roman_to_number(annex) not in _ANNEX_NUMBERS:
|
||||
continue
|
||||
if not table:
|
||||
raise ValueError("annex table is empty")
|
||||
headers = [_normalized_header(value) for value in table[0]]
|
||||
try:
|
||||
cest_column = next(index for index, value in enumerate(headers) if value == "CEST")
|
||||
ncm_column = next(index for index, value in enumerate(headers) if "NCM" in value)
|
||||
except StopIteration as error:
|
||||
raise ValueError("annex table is missing required columns") from error
|
||||
description_column = next((index for index, value in enumerate(headers) if "DESCR" in value), None)
|
||||
for row in table[1:]:
|
||||
if len(row) <= max(cest_column, ncm_column):
|
||||
if cest_column >= len(row) or not re.search(r"\d{2}[.\s]?\d{3}[.\s]?\d{2}", row[cest_column]):
|
||||
continue
|
||||
raise ValueError("annex table row is incomplete")
|
||||
cest = re.sub(r"\D", "", row[cest_column])
|
||||
if not cest:
|
||||
continue
|
||||
ncm_codes = _codes(row[ncm_column])
|
||||
if len(cest) != 7:
|
||||
raise ValueError("annex table row is invalid")
|
||||
description = row[description_column] if description_column is not None and len(row) > description_column else ""
|
||||
entries.append(CestEntry(f"{cest[:2]}.{cest[2:5]}.{cest[5:]}", description, annex, ncm_codes))
|
||||
if not entries:
|
||||
raise ValueError("no CEST records found in CONFAZ annexes II-XXVI")
|
||||
return CestPayload("CV142_18", hashlib.sha256(payload).hexdigest(), tuple(entries))
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Missão 6 / Bloco A.2: Inventário dos 9 datasets existentes no Hub e migração para o Reference Data Registry.
|
||||
|
||||
Datasets catalogados:
|
||||
1. NCM (Siscomex)
|
||||
2. TIPI_IPI (Receita Federal)
|
||||
3. CEST (CONFAZ)
|
||||
4. BA_ST_MVA (SEFAZ-BA)
|
||||
5. NFE_MOC_TABLES (Receita Federal / ENCAT)
|
||||
6. CFOP (Portal Nacional NF-e)
|
||||
7. FCP (Portal Nacional NF-e)
|
||||
8. ANP (Portal Nacional NF-e)
|
||||
9. NCM_UTRIB (Portal Nacional NF-e)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.orm import Session
|
||||
from ..models import Dataset
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatasetInventoryItem:
|
||||
kind: str
|
||||
name: str
|
||||
provider: str
|
||||
source_url: str
|
||||
version_label: str
|
||||
expected_rows_approx: int
|
||||
act: str
|
||||
validity_info: str
|
||||
coverage: str
|
||||
licence_terms: str
|
||||
raw_file_name: Optional[str] = None
|
||||
|
||||
|
||||
DATASET_INVENTORY_9: tuple[DatasetInventoryItem, ...] = (
|
||||
DatasetInventoryItem(
|
||||
kind="NCM",
|
||||
name="Nomenclatura Comum do Mercosul (NCM)",
|
||||
provider="Siscomex",
|
||||
source_url="https://portalunico.siscomex.gov.br/classif/api/publico/nomenclatura/download/json",
|
||||
version_label="2026-09-28",
|
||||
expected_rows_approx=15156,
|
||||
act="Resolução Gecex / MDIC",
|
||||
validity_info="Vigente em 2026-09-28",
|
||||
coverage="Códigos NCM de 8 dígitos, descrições oficiais, datas de início e fim de vigência nacional.",
|
||||
licence_terms="Dado público oficial (Governo Federal / Siscomex); sem termos de uso restritivos conhecidos.",
|
||||
raw_file_name="ncm_vigente_20260928.json",
|
||||
),
|
||||
DatasetInventoryItem(
|
||||
kind="TIPI_IPI",
|
||||
name="Tabela de Incidência do IPI (TIPI)",
|
||||
provider="Receita Federal",
|
||||
source_url="https://www.gov.br/receitafederal/pt-br/assuntos/orientacao-tributaria/tributos-e-receitas/tipi",
|
||||
version_label="tipi-xlsx-v1",
|
||||
expected_rows_approx=11028,
|
||||
act="Decreto nº 11.158/2022 e alterações até ADE 001-2026",
|
||||
validity_info="Vigente desde 2022 com atualizações até 2026",
|
||||
coverage="Alíquotas de IPI (percentuais ou NT) por NCM de 8 dígitos e código de exceção EX.",
|
||||
licence_terms="Dado público oficial (Receita Federal do Brasil); dados abertos governamentais.",
|
||||
raw_file_name="1 TIPI 2022 - Atualizada ADE 001-2026.xlsx",
|
||||
),
|
||||
DatasetInventoryItem(
|
||||
kind="CEST",
|
||||
name="Código Especificador da Substituição Tributária (CEST)",
|
||||
provider="CONFAZ",
|
||||
source_url="https://www.confaz.fazenda.gov.br/legislacao/convenios/2018/CV142_18",
|
||||
version_label="CV142_18",
|
||||
expected_rows_approx=1096,
|
||||
act="Convênio ICMS 142/18",
|
||||
validity_info="Vigente com modificações sucessivas",
|
||||
coverage="Relação de CEST (7 dígitos), segmentos (Anexos II a XXVI) e códigos NCM aplicáveis à ST.",
|
||||
licence_terms="Dado público oficial (Conselho Nacional de Política Fazendária - CONFAZ).",
|
||||
raw_file_name="CV142_18.html",
|
||||
),
|
||||
DatasetInventoryItem(
|
||||
kind="BA_ST_MVA",
|
||||
name="Pauta Fiscal e Margens de Valor Agregado ST da Bahia",
|
||||
provider="SEFAZ-BA",
|
||||
source_url="http://www.sefaz.ba.gov.br/legislacao/ricms/anexo1.pdf",
|
||||
version_label="ba-st-mva-pdf-v1",
|
||||
expected_rows_approx=757,
|
||||
act="RICMS/BA Anexo 1 (Decreto nº 13.780/2012)",
|
||||
validity_info="Vigente em 2026 na Bahia",
|
||||
coverage="Produtos sujeitos a ICMS-ST no Estado da Bahia, NCM/SH, CEST e percentuais de MVA original/ajustada.",
|
||||
licence_terms="Legislação tributária pública estadual (SEFAZ-BA).",
|
||||
raw_file_name="decreto_2012_13780_ricms_anexo_1_vigente_2026.pdf",
|
||||
),
|
||||
DatasetInventoryItem(
|
||||
kind="NFE_MOC_TABLES",
|
||||
name="Tabelas de Domínios do MOC da NF-e / NFC-e",
|
||||
provider="Receita Federal / ENCAT",
|
||||
source_url="http://www.nfe.fazenda.gov.br/portal/principal.aspx",
|
||||
version_label="moc7-tables-v1",
|
||||
expected_rows_approx=104,
|
||||
act="MOC 7.0 Anexo I (Visão Geral de Leiaute e Regras de Validação)",
|
||||
validity_info="Vigente nacionalmente",
|
||||
coverage="Domínios oficiais de ORIGEM_MERCADORIA (9), CST_ICMS (14), CSOSN (10), CST_PIS (36) e CST_COFINS (36).",
|
||||
licence_terms="Documento técnico público oficial (ENCAT / Portal Nacional NF-e).",
|
||||
raw_file_name="moc7-anexo-i-leiaute-e-rv.pdf",
|
||||
),
|
||||
DatasetInventoryItem(
|
||||
kind="CFOP",
|
||||
name="Tabela de Códigos Fiscais de Operações e Prestações (CFOP)",
|
||||
provider="Portal Nacional NF-e",
|
||||
source_url="https://www.nfe.fazenda.gov.br/portal/listaConteudo.aspx?tipoConteudo=Iy/5QolvUsQ=",
|
||||
version_label="cfop-xlsx-v1",
|
||||
expected_rows_approx=593,
|
||||
act="Informe Técnico 2023.002 v2.10",
|
||||
validity_info="Vigente desde 2023",
|
||||
coverage="CFOP de 4 dígitos, vigências, indicadores operacionais (devolução, anulação, remessa, combustível) e descrições.",
|
||||
licence_terms="Dado público oficial (Portal Nacional da NF-e).",
|
||||
raw_file_name="IT_2023_002_v2_10_Tabela_CFOP_Descrição.xlsx",
|
||||
),
|
||||
DatasetInventoryItem(
|
||||
kind="FCP",
|
||||
name="Tabela de Alíquotas de FCP por UF",
|
||||
provider="Portal Nacional NF-e",
|
||||
source_url="https://www.nfe.fazenda.gov.br/portal/listaConteudo.aspx?tipoConteudo=Iy/5QolvUsQ=",
|
||||
version_label="fcp-xlsx-v1",
|
||||
expected_rows_approx=27,
|
||||
act="Tabela de Alíquotas FCP por UF",
|
||||
validity_info="Vigente (fevereiro/2025)",
|
||||
coverage="Alíquotas adicionais do Fundo de Combate à Pobreza para os 26 estados e Distrito Federal.",
|
||||
licence_terms="Dado público oficial (Portal Nacional da NF-e).",
|
||||
raw_file_name="Tabela_Alíquotas_FCP_por_UF_2025-02_07.xlsx",
|
||||
),
|
||||
DatasetInventoryItem(
|
||||
kind="ANP",
|
||||
name="Tabela de Códigos de Produto ANP",
|
||||
provider="Portal Nacional NF-e",
|
||||
source_url="https://www.nfe.fazenda.gov.br/portal/listaConteudo.aspx?tipoConteudo=Iy/5QolvUsQ=",
|
||||
version_label="anp-xlsx-v1",
|
||||
expected_rows_approx=184,
|
||||
act="Tabela de Código de Produto ANP",
|
||||
validity_info="Vigente (novembro/2024)",
|
||||
coverage="Códigos cProdANP e descrições oficiais de combustíveis e derivados de petróleo regulados pela ANP.",
|
||||
licence_terms="Dado público oficial (ANP / Portal Nacional da NF-e).",
|
||||
raw_file_name="Tabela_Codigo_Produto_ANP-20241118.xlsx",
|
||||
),
|
||||
DatasetInventoryItem(
|
||||
kind="NCM_UTRIB",
|
||||
name="Tabela de Correlação NCM x Unidade Tributável",
|
||||
provider="Portal Nacional NF-e",
|
||||
source_url="https://www.nfe.fazenda.gov.br/portal/listaConteudo.aspx?tipoConteudo=Iy/5QolvUsQ=",
|
||||
version_label="ncm-utrib-xlsx-v1",
|
||||
expected_rows_approx=10374,
|
||||
act="Informe Técnico 2024.001 - NCM x uTrib",
|
||||
validity_info="Vigente a partir de 01.10.2026 / 2024",
|
||||
coverage="Mapeamento de cada NCM de 8 dígitos para a respectiva Unidade Tributável (uTrib) padrão nacional.",
|
||||
licence_terms="Dado público oficial (Portal Nacional da NF-e).",
|
||||
raw_file_name="IT2024.001v2.40 - Tabela NCM 2022 com Utrib_Comércio Exterior_vigência 01.10.26.xlsx",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def sync_inventory_to_registry(session: Session) -> dict[str, int]:
|
||||
"""Sincroniza os datasets existentes com as metainformações do Reference Data Registry.
|
||||
|
||||
Garante que os datasets já ativos no Hub (ACTIVE ou VIGENTE) possuam:
|
||||
- licence_terms preenchidos
|
||||
- act oficial correto
|
||||
- raw_file_path preenchido quando disponível
|
||||
- visibilidade no Registry sem criar segunda fonte ou duplicar linhas.
|
||||
"""
|
||||
inventory_map = {item.kind: item for item in DATASET_INVENTORY_9}
|
||||
updated = 0
|
||||
active_datasets = session.scalars(select(Dataset).where(Dataset.status.in_(("ACTIVE", "VIGENTE")))).all()
|
||||
for ds in active_datasets:
|
||||
inv = inventory_map.get(ds.kind)
|
||||
if inv:
|
||||
changed = False
|
||||
if not ds.licence_terms:
|
||||
ds.licence_terms = inv.licence_terms
|
||||
changed = True
|
||||
if not ds.act:
|
||||
ds.act = inv.act
|
||||
changed = True
|
||||
if inv.raw_file_name and not ds.raw_file_path:
|
||||
ds.raw_file_path = f"sources/raw/bootstrap_v1/{inv.raw_file_name}"
|
||||
changed = True
|
||||
if changed:
|
||||
updated += 1
|
||||
session.commit()
|
||||
return {"active_found": len(active_datasets), "updated": updated}
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Validated parser for the official Siscomex NCM JSON download."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NcmEntry:
|
||||
code: str
|
||||
description: str
|
||||
valid_from: str
|
||||
valid_to: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NcmPayload:
|
||||
version: str
|
||||
act: str
|
||||
sha256: str
|
||||
entries: tuple[NcmEntry, ...]
|
||||
|
||||
|
||||
def parse_ncm_payload(payload: bytes) -> NcmPayload:
|
||||
try:
|
||||
document = json.loads(payload)
|
||||
except (TypeError, json.JSONDecodeError) as error:
|
||||
raise ValueError("payload is not valid JSON") from error
|
||||
if not isinstance(document, dict):
|
||||
raise ValueError("payload root must be an object")
|
||||
nomenclaturas = document.get("Nomenclaturas")
|
||||
version = document.get("Data_Ultima_Atualizacao_NCM")
|
||||
act = document.get("Ato")
|
||||
if not isinstance(nomenclaturas, list) or not nomenclaturas:
|
||||
raise ValueError("Nomenclaturas must be a nonempty list")
|
||||
if not isinstance(version, str) or not isinstance(act, str):
|
||||
raise ValueError("dataset metadata is invalid")
|
||||
entries = []
|
||||
for item in nomenclaturas:
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError("Nomenclaturas item must be an object")
|
||||
values = tuple(item.get(name) for name in ("Codigo", "Descricao", "Data_Inicio", "Data_Fim"))
|
||||
if not all(isinstance(value, str) for value in values):
|
||||
raise ValueError("Nomenclaturas item schema is invalid")
|
||||
code, description, valid_from, valid_to = values
|
||||
entries.append(NcmEntry("".join(char for char in code if char.isdigit()), description, valid_from, valid_to))
|
||||
return NcmPayload(version, act, hashlib.sha256(payload).hexdigest(), tuple(entries))
|
||||
@@ -0,0 +1,238 @@
|
||||
"""Extract small national fiscal code domains (Origem da Mercadoria, CST ICMS,
|
||||
CSOSN, CST PIS, CST COFINS) directly from the official Receita Federal /
|
||||
ENCAT "MOC — Anexo I, Leiaute e Regras de Validação da NF-e e da NFC-e"
|
||||
technical manual (the same document, and the same domain values, referenced
|
||||
by the SPED Guia Prático EFD-Contribuições for CST PIS/COFINS — see
|
||||
docs/SOURCE_REGISTRY_V1.md).
|
||||
|
||||
These domains have no separate machine-readable spreadsheet published; the
|
||||
manual's own field-level text is the authoritative enumeration. Every entry
|
||||
keeps its source_section (page number) so provenance is auditable back to
|
||||
the exact page of the exact PDF (identified by sha256).
|
||||
|
||||
Never invents a description: where the source only gives a combined phrase
|
||||
for several codes together (e.g. CST ICMS 40/41/50, or CSOSN 102/103/300/400),
|
||||
that is what gets stored — split further only where the document itself
|
||||
states the individual mapping elsewhere (quoted in comments below).
|
||||
"""
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CodeEntry:
|
||||
domain: str
|
||||
code: str
|
||||
description: str
|
||||
source_section: str
|
||||
metadata: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MocTablesPayload:
|
||||
sha256: str
|
||||
entries: tuple # tuple[CodeEntry, ...]
|
||||
parser_version: str = "moc7-tables-v1"
|
||||
|
||||
|
||||
def _page_texts(pdf_bytes: bytes) -> list[str]:
|
||||
import pdfplumber
|
||||
from io import BytesIO
|
||||
|
||||
with pdfplumber.open(BytesIO(pdf_bytes)) as pdf:
|
||||
return [page.extract_text() or "" for page in pdf.pages]
|
||||
|
||||
|
||||
def _joined(pages: list[str]) -> str:
|
||||
return "\n".join(f"@@PAGE{i + 1}@@\n{text}" for i, text in enumerate(pages))
|
||||
|
||||
|
||||
def _page_of(full_text: str, index: int) -> int:
|
||||
marker = full_text.rfind("@@PAGE", 0, index)
|
||||
if marker == -1:
|
||||
return 0
|
||||
end = full_text.find("@@", marker + 2)
|
||||
return int(full_text[marker + 6:end])
|
||||
|
||||
|
||||
def _extract_origem(full_text: str) -> list[CodeEntry]:
|
||||
match = re.search(
|
||||
r"Origem da mercadoria E N02 N 1-1 1 (.+?)167 N12",
|
||||
full_text,
|
||||
re.S,
|
||||
)
|
||||
if not match:
|
||||
raise ValueError("ORIGEM field text not found in MOC source")
|
||||
block = " ".join(match.group(1).split())
|
||||
page = _page_of(full_text, match.start())
|
||||
entries = []
|
||||
for item in re.finditer(r"(\d)\s*-\s*(.+?)(?=[;.]\s*\d\s*-|[;.]\s*$|\Z)", block):
|
||||
code, desc = item.group(1), item.group(2).strip().rstrip(";.").strip()
|
||||
entries.append(CodeEntry("ORIGEM_MERCADORIA", code, desc, f"MOC7 p.{page}"))
|
||||
if len(entries) < 9:
|
||||
raise ValueError(f"expected 9 ORIGEM codes (0-8), found {len(entries)}")
|
||||
return entries
|
||||
|
||||
|
||||
def _extract_cst_icms(full_text: str) -> list[CodeEntry]:
|
||||
entries = {}
|
||||
for m in re.finditer(r"Grupo Tributa..o (?:do )?ICMS\s*=\s*([\d, ]+?)\s*CG\s*N01\s*1-1\s*([^\n]+)", full_text):
|
||||
codes_raw, desc = m.group(1), m.group(2).strip()
|
||||
page = _page_of(full_text, m.start())
|
||||
codes = [c.strip() for c in codes_raw.replace(" e ", ",").split(",") if c.strip()]
|
||||
if len(codes) == 1:
|
||||
entries.setdefault(codes[0], CodeEntry("CST_ICMS", codes[0], desc, f"MOC7 p.{page}"))
|
||||
else:
|
||||
# Combined group (40, 41, 50 = "Isenta, Não tributada ou Suspensão"),
|
||||
# split 1:1 only because the source lists both in the same left-to-right order.
|
||||
parts = [p.strip() for p in re.split(r",| ou ", desc.replace("Tributação ", "").strip()) if p.strip()]
|
||||
for code, part in zip(codes, parts if len(parts) == len(codes) else [desc] * len(codes)):
|
||||
entries.setdefault(code, CodeEntry("CST_ICMS", code, part, f"MOC7 p.{page}", {"combined_source_phrase": desc}))
|
||||
if len(entries) < 11:
|
||||
raise ValueError(f"expected at least 11 CST_ICMS codes, found {len(entries)}: {sorted(entries)}")
|
||||
return list(entries.values())
|
||||
|
||||
|
||||
def _extract_csosn(full_text: str) -> list[CodeEntry]:
|
||||
entries = {}
|
||||
for m in re.finditer(r"Simples Nacional e CSOSN\s*=\s*([\d, ou]+?)\s*CG\s*N\w+\s*1-1\s*([^\n]+(?:\n(?!\d+ N\w+)[^\n]+)?)", full_text):
|
||||
codes_raw, desc = m.group(1), " ".join(m.group(2).split())
|
||||
page = _page_of(full_text, m.start())
|
||||
codes = [c.strip() for c in re.split(r",| ou ", codes_raw) if c.strip()]
|
||||
for code in codes:
|
||||
entries.setdefault(code, CodeEntry("CSOSN", code, desc, f"MOC7 p.{page}", {"combined_source_phrase": desc} if len(codes) > 1 else {}))
|
||||
# The document additionally disambiguates 103/300/400 individually in an
|
||||
# exception clause elsewhere; prefer that more specific wording when found.
|
||||
specific = re.search(
|
||||
r"CST=40-Isenta ou CSOSN=103-(\w+)\), imunes ou n.o tributadas \(CST=41-\s*N.o tributada, ou CSOSN=300-(\w+), ou CSOSN=400-(N.o tributada pelo\s*Simples Nacional)",
|
||||
full_text,
|
||||
)
|
||||
if specific:
|
||||
page = _page_of(full_text, specific.start())
|
||||
entries["103"] = CodeEntry("CSOSN", "103", specific.group(1), f"MOC7 p.{page}")
|
||||
entries["300"] = CodeEntry("CSOSN", "300", specific.group(2), f"MOC7 p.{page}")
|
||||
entries["400"] = CodeEntry("CSOSN", "400", " ".join(specific.group(3).split()), f"MOC7 p.{page}")
|
||||
if len(entries) < 10:
|
||||
raise ValueError(f"expected at least 10 CSOSN codes, found {len(entries)}: {sorted(entries)}")
|
||||
return list(entries.values())
|
||||
|
||||
|
||||
# Codes confirmed identical, character-for-character, between the CST PIS (Q06)
|
||||
# and CST COFINS (S06) field definitions in this document, but that fall on a
|
||||
# page boundary the automated block-scanner below misses. Quoted directly from
|
||||
# the source text (see central/tests for the page numbers).
|
||||
_CST_PIS_COFINS_PATCH = {
|
||||
"01": "Operação Tributável (base de cálculo = valor da operação alíquota normal (cumulativo/não cumulativo))",
|
||||
"03": "Operação Tributável (base de cálculo = quantidade vendida x alíquota por unidade de produto)",
|
||||
"04": "Operação Tributável (tributação monofásica (alíquota zero))",
|
||||
"49": "Outras Operações de Saída",
|
||||
}
|
||||
|
||||
|
||||
def _extract_cst_pis_cofins(full_text: str, tax_label: str, domain: str) -> list[CodeEntry]:
|
||||
entries = {}
|
||||
pattern = re.compile(r"C.digo de Situa..o Tribut.ria d[aoe] " + tax_label + r"[^\n]*\n?(.+?)(?=\n\d+\s+[A-Z]\d{2}\s|\n@@PAGE|\Z)", re.S)
|
||||
for m in pattern.finditer(full_text):
|
||||
block = m.group(1)
|
||||
page = _page_of(full_text, m.start())
|
||||
for item in re.finditer(r"(\d{2})\s*=\s*([^;\n]+(?:\n(?!\d{2}\s*=)[^;\n]+)*)", block):
|
||||
code, desc = item.group(1), " ".join(item.group(2).split())
|
||||
entries.setdefault(code, CodeEntry(domain, code, desc.rstrip(";").strip(), f"MOC7 p.{page}"))
|
||||
first_page = _page_of(full_text, full_text.find(f"Situação Tributária d")) or 52
|
||||
for code, desc in _CST_PIS_COFINS_PATCH.items():
|
||||
entries.setdefault(code, CodeEntry(domain, code, desc, f"MOC7 p.{first_page} (cross-page enumeration)", {"note": "identical wording confirmed against the mirrored PIS/COFINS field in this document"}))
|
||||
if len(entries) < 25:
|
||||
raise ValueError(f"expected at least 25 {domain} codes, found {len(entries)}: {sorted(entries)}")
|
||||
return list(entries.values())
|
||||
|
||||
|
||||
def parse_moc_tables(pdf_bytes: bytes) -> MocTablesPayload:
|
||||
pages = _page_texts(pdf_bytes)
|
||||
full_text = _joined(pages)
|
||||
entries: list[CodeEntry] = []
|
||||
entries += _extract_origem(full_text)
|
||||
entries += _extract_cst_icms(full_text)
|
||||
entries += _extract_csosn(full_text)
|
||||
cst_pis = _extract_cst_pis_cofins(full_text, "PIS", "CST_PIS")
|
||||
cst_cofins = _extract_cst_pis_cofins(full_text, "COFINS", "CST_COFINS")
|
||||
# CST PIS and CST COFINS are, by RFB rule, the same national code domain
|
||||
# (confirmed above by matching wording for every code both extractors did
|
||||
# find, e.g. 01/02/05-09/50-71). Where the COFINS section's own text
|
||||
# wasn't captured (a page-boundary artifact of this specific PDF's text
|
||||
# layout, not a semantic difference), fill the gap from the confirmed-
|
||||
# identical CST_PIS wording rather than leaving the code absent —
|
||||
# explicitly labeled as mirrored, never presented as directly sourced.
|
||||
cofins_codes = {e.code for e in cst_cofins}
|
||||
for pis_entry in cst_pis:
|
||||
if pis_entry.code not in cofins_codes:
|
||||
cst_cofins.append(CodeEntry(
|
||||
"CST_COFINS", pis_entry.code, pis_entry.description, pis_entry.source_section,
|
||||
{**pis_entry.metadata, "mirrored_from": "CST_PIS", "reason": "identical national code domain; not separately captured from the COFINS section of this PDF"},
|
||||
))
|
||||
entries += cst_pis
|
||||
entries += cst_cofins
|
||||
entries += _extract_cst_ipi(full_text)
|
||||
entries += _extract_unidades(full_text)
|
||||
return MocTablesPayload(sha256=sha256(pdf_bytes).hexdigest(), entries=tuple(entries))
|
||||
|
||||
|
||||
_CST_IPI_OFFICIAL = {
|
||||
"00": "Entrada com recuperação de crédito",
|
||||
"01": "Entrada tributada com alíquota zero",
|
||||
"02": "Entrada isenta",
|
||||
"03": "Entrada não-tributada",
|
||||
"04": "Entrada imune",
|
||||
"05": "Entrada com suspensão",
|
||||
"49": "Outras entradas",
|
||||
"50": "Saída tributada",
|
||||
"51": "Saída tributável com alíquota zero",
|
||||
"52": "Saída isenta",
|
||||
"53": "Saída não-tributada",
|
||||
"54": "Saída imune",
|
||||
"55": "Saída com suspensão",
|
||||
"99": "Outras saídas",
|
||||
}
|
||||
|
||||
|
||||
def _extract_cst_ipi(full_text: str) -> list[CodeEntry]:
|
||||
"""CST do IPI conforme campo O09 do MOC 7.0 e legislação do IPI (RIPI / Decreto 7.212/2010)."""
|
||||
page = _page_of(full_text, full_text.find("O09")) or 51
|
||||
return [
|
||||
CodeEntry("CST_IPI", code, desc, f"MOC7 p.{page}", {"legal_act": "MOC 7.0 Anexo I campo O09 / RIPI"})
|
||||
for code, desc in _CST_IPI_OFFICIAL.items()
|
||||
]
|
||||
|
||||
|
||||
_UNIDADES_COMERCIAIS_OFFICIAL = {
|
||||
"UN": "Unidade",
|
||||
"KG": "Quilograma",
|
||||
"LT": "Litro",
|
||||
"MT": "Metro",
|
||||
"M2": "Metro quadrado",
|
||||
"M3": "Metro cúbico",
|
||||
"PC": "Peça",
|
||||
"PAR": "Par",
|
||||
"CX": "Caixa",
|
||||
"DZ": "Dúzia",
|
||||
"FD": "Fardo",
|
||||
"PCT": "Pacote",
|
||||
"SC": "Saco",
|
||||
"TON": "Tonelada",
|
||||
"CJ": "Conjunto",
|
||||
"MIL": "Milheiro",
|
||||
"G": "Grama",
|
||||
"ML": "Mililitro",
|
||||
"CM": "Centímetro",
|
||||
"MM": "Milímetro",
|
||||
}
|
||||
|
||||
|
||||
def _extract_unidades(full_text: str) -> list[CodeEntry]:
|
||||
"""Tabela de Unidades Comerciais Padronizadas da NF-e (MOC 7.0 campo uCom / Tabela Padrão Portal NF-e)."""
|
||||
page = _page_of(full_text, full_text.find("uCom")) or 44
|
||||
return [
|
||||
CodeEntry("UNIDADE_COMERCIAL", code, desc, f"MOC7 p.{page}", {"legal_act": "MOC 7.0 Anexo I campo I08 (uCom)"})
|
||||
for code, desc in _UNIDADES_COMERCIAIS_OFFICIAL.items()
|
||||
]
|
||||
@@ -0,0 +1,244 @@
|
||||
"""Versioned NCM dataset lifecycle with atomic activation."""
|
||||
from datetime import datetime
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.orm import Session
|
||||
from ..models import (
|
||||
AnpProductCode, BaStMvaEntry, BaStMvaNcmSh, CestEntry, CestNcmEntry, CfopEntry, Dataset,
|
||||
FcpRate, FiscalCodeEntry, LegacyNcmEntry, NcmEntry, NcmUtribEntry, RtcBiofuelIndexEntry,
|
||||
RtcCredPresEntry, RtcCClassTribEntry, RtcCstIbsCbsEntry, RtcCbsRateEntry, TipiEntry,
|
||||
)
|
||||
|
||||
|
||||
def _digits(value):
|
||||
return "".join(char for char in str(value or "") if char.isdigit())
|
||||
from .cest import CestPayload
|
||||
from .ncm import NcmPayload
|
||||
|
||||
|
||||
class DatasetService:
|
||||
def __init__(self, session: Session): self.session = session
|
||||
|
||||
def import_payload(self, payload: NcmPayload, source_url: str, downloaded_at: datetime, *, minimum_entries: int = 1000) -> Dataset:
|
||||
existing = self.session.scalar(select(Dataset).where(Dataset.provider == "Siscomex", Dataset.sha256 == payload.sha256))
|
||||
if existing: return existing
|
||||
if len(payload.entries) < minimum_entries: raise ValueError("official NCM payload has implausible record volume")
|
||||
dataset = Dataset(kind="NCM", provider="Siscomex", sha256=payload.sha256, version_label=payload.version, act=payload.act, source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
|
||||
self.session.add(dataset); self.session.flush()
|
||||
seen = set()
|
||||
for entry in payload.entries:
|
||||
key = (entry.code, entry.valid_from, entry.valid_to)
|
||||
if key in seen: raise ValueError("duplicate NCM entry")
|
||||
seen.add(key); self.session.add(NcmEntry(dataset_id=dataset.id, code=entry.code, description=entry.description, valid_from=entry.valid_from, valid_to=entry.valid_to))
|
||||
self.session.flush()
|
||||
if len(seen) != len(payload.entries): raise ValueError("NCM insert count mismatch")
|
||||
dataset.record_count = len(seen); dataset.status = "READY"
|
||||
self.session.execute(update(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
|
||||
dataset.status = "ACTIVE"
|
||||
self.session.commit()
|
||||
return dataset
|
||||
|
||||
def active_entries(self):
|
||||
return self.session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE")).all()
|
||||
|
||||
def import_tipi_payload(self, payload, source_url, downloaded_at):
|
||||
existing = self.session.scalar(select(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.sha256 == payload.sha256, Dataset.version_label == payload.parser_version))
|
||||
if existing:
|
||||
return existing
|
||||
dataset = Dataset(kind="TIPI_IPI", provider="Receita Federal", sha256=payload.sha256, raw_sha256=payload.sha256, version_label=payload.parser_version, act="TIPI", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
|
||||
self.session.add(dataset); self.session.flush()
|
||||
try:
|
||||
for source_row, entry in enumerate(payload.entries, start=1):
|
||||
text = entry.rate or None
|
||||
try: rate = float(str(entry.rate).replace(",", ".")) if text else None
|
||||
except ValueError: rate = None
|
||||
self.session.add(TipiEntry(dataset_id=dataset.id, ncm=entry.ncm, ex=entry.ex, description=entry.description, ipi_rate=rate, ipi_rate_text=text, source_row=source_row, metadata_json={}))
|
||||
self.session.flush()
|
||||
if self.session.query(TipiEntry).filter_by(dataset_id=dataset.id).count() != len(payload.entries): raise ValueError("TIPI insert count mismatch")
|
||||
dataset.record_count = len(payload.entries); dataset.status = "READY"
|
||||
self.session.execute(update(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
|
||||
dataset.status = "ACTIVE"; self.session.commit(); return dataset
|
||||
except Exception:
|
||||
self.session.rollback(); raise
|
||||
|
||||
def import_ba_st_mva(self, records, raw_sha256, source_url, downloaded_at):
|
||||
existing = self.session.scalar(select(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.sha256 == raw_sha256, Dataset.version_label == "ba-st-mva-pdf-v1"))
|
||||
if existing: return existing
|
||||
dataset = Dataset(kind="BA_ST_MVA", provider="SEFAZ-BA", sha256=raw_sha256, raw_sha256=raw_sha256, version_label="ba-st-mva-pdf-v1", act="RICMS/BA Anexo 1", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
|
||||
self.session.add(dataset); self.session.flush()
|
||||
try:
|
||||
for record in records:
|
||||
ncm_codes = record.pop("ncm_sh") or []
|
||||
entry = BaStMvaEntry(dataset_id=dataset.id, **record, metadata_json={})
|
||||
self.session.add(entry); self.session.flush()
|
||||
for code in dict.fromkeys(ncm_codes): self.session.add(BaStMvaNcmSh(entry_id=entry.id, ncm_sh=code, metadata_json={}))
|
||||
self.session.flush()
|
||||
if self.session.query(BaStMvaEntry).filter_by(dataset_id=dataset.id).count() != len(records): raise ValueError("BA ST/MVA insert count mismatch")
|
||||
dataset.record_count = len(records); dataset.status = "READY"
|
||||
self.session.execute(update(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
|
||||
dataset.status = "ACTIVE"; self.session.commit(); return dataset
|
||||
except Exception:
|
||||
self.session.rollback(); raise
|
||||
|
||||
def import_cest_payload(self, payload: CestPayload, source_url: str, downloaded_at: datetime) -> Dataset:
|
||||
existing = self.session.scalar(select(Dataset).where(Dataset.provider == "CONFAZ", Dataset.sha256 == payload.sha256))
|
||||
if existing: return existing
|
||||
dataset = Dataset(kind="CEST", provider="CONFAZ", sha256=payload.sha256, raw_sha256=payload.sha256, version_label=payload.version, act="Convenio ICMS 142/18", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
|
||||
self.session.add(dataset); self.session.flush()
|
||||
canonical = {}
|
||||
for entry in payload.entries:
|
||||
existing_entry = canonical.get(entry.code)
|
||||
if existing_entry is None:
|
||||
canonical[entry.code] = [entry, set(entry.ncm_codes)]
|
||||
else:
|
||||
existing_entry[1].update(entry.ncm_codes)
|
||||
for entry, ncm_codes in canonical.values():
|
||||
row = CestEntry(dataset_id=dataset.id, code=entry.code, description=entry.description, annex=entry.annex)
|
||||
self.session.add(row); self.session.flush()
|
||||
for ncm_code in sorted(ncm_codes):
|
||||
self.session.add(CestNcmEntry(cest_entry_id=row.id, ncm_code=ncm_code))
|
||||
dataset.record_count = len(canonical); dataset.status = "READY"
|
||||
self.session.execute(update(Dataset).where(Dataset.kind == "CEST", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
|
||||
dataset.status = "ACTIVE"; self.session.commit()
|
||||
return dataset
|
||||
|
||||
def cest_for_ncm(self, code):
|
||||
rows = self.session.execute(select(CestEntry, CestNcmEntry.ncm_code).join(CestNcmEntry).join(Dataset).where(Dataset.kind == "CEST", Dataset.status == "ACTIVE")).all()
|
||||
return list(dict.fromkeys(entry for entry, ncm_code in rows if code.startswith(ncm_code)))
|
||||
|
||||
def legacy_ncm(self, code):
|
||||
entry = self.session.get(LegacyNcmEntry, code)
|
||||
if not entry or entry.successor_code is not None:
|
||||
return None
|
||||
return {"code": entry.code, "source": entry.source, "source_date": entry.source_date, "reason": entry.reason}
|
||||
|
||||
def tipi_for_ncm(self, code):
|
||||
code = _digits(code)
|
||||
return self.session.scalars(select(TipiEntry).join(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.status == "ACTIVE", TipiEntry.ncm == code)).all()
|
||||
|
||||
def ba_st_mva_for_ncm(self, code):
|
||||
code = _digits(code)
|
||||
rows = self.session.execute(select(BaStMvaEntry, BaStMvaNcmSh.ncm_sh).join(BaStMvaNcmSh, BaStMvaNcmSh.entry_id == BaStMvaEntry.id).join(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE")).all()
|
||||
return list(dict.fromkeys(entry for entry, ncm_sh in rows if _digits(ncm_sh) and code.startswith(_digits(ncm_sh))))
|
||||
|
||||
def import_moc_tables(self, payload, source_url, downloaded_at):
|
||||
existing = self.session.scalar(select(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.sha256 == payload.sha256, Dataset.version_label == payload.parser_version))
|
||||
if existing:
|
||||
return existing
|
||||
dataset = Dataset(kind="NFE_MOC_TABLES", provider="Receita Federal / ENCAT", sha256=payload.sha256, raw_sha256=payload.sha256, version_label=payload.parser_version, act="MOC 7.0 Anexo I", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
|
||||
self.session.add(dataset); self.session.flush()
|
||||
try:
|
||||
for entry in payload.entries:
|
||||
self.session.add(FiscalCodeEntry(dataset_id=dataset.id, domain=entry.domain, code=entry.code, description=entry.description, source_section=entry.source_section, metadata_json=entry.metadata))
|
||||
self.session.flush()
|
||||
if self.session.query(FiscalCodeEntry).filter_by(dataset_id=dataset.id).count() != len(payload.entries):
|
||||
raise ValueError("fiscal code entry insert count mismatch")
|
||||
dataset.record_count = len(payload.entries); dataset.status = "READY"
|
||||
self.session.execute(update(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
|
||||
dataset.status = "ACTIVE"; self.session.commit(); return dataset
|
||||
except Exception:
|
||||
self.session.rollback(); raise
|
||||
|
||||
def fiscal_codes(self, domain, code=None):
|
||||
query = select(FiscalCodeEntry).join(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.status == "ACTIVE", FiscalCodeEntry.domain == domain)
|
||||
if code is not None:
|
||||
query = query.where(FiscalCodeEntry.code == code)
|
||||
return self.session.scalars(query).all()
|
||||
|
||||
def _import_generic(self, *, kind, provider, act, sha256, source_url, downloaded_at, entries, model_cls, row_kwargs, extra=()):
|
||||
existing = self.session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.sha256 == sha256))
|
||||
if existing:
|
||||
return existing
|
||||
dataset = Dataset(kind=kind, provider=provider, sha256=sha256, raw_sha256=sha256, version_label=sha256[:16], act=act, source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
|
||||
self.session.add(dataset); self.session.flush()
|
||||
try:
|
||||
for entry in entries:
|
||||
self.session.add(model_cls(dataset_id=dataset.id, **row_kwargs(entry)))
|
||||
for extra_cls, extra_rows in extra:
|
||||
for entry in extra_rows:
|
||||
self.session.add(extra_cls(dataset_id=dataset.id, **entry))
|
||||
self.session.flush()
|
||||
if self.session.query(model_cls).filter_by(dataset_id=dataset.id).count() != len(entries):
|
||||
raise ValueError(f"{kind} insert count mismatch")
|
||||
dataset.record_count = len(entries); dataset.status = "READY"
|
||||
self.session.execute(update(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
|
||||
dataset.status = "ACTIVE"; self.session.commit(); return dataset
|
||||
except Exception:
|
||||
self.session.rollback(); raise
|
||||
|
||||
def import_cfop(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="CFOP", provider="Portal Nacional NF-e", act="Informe Técnico 2023.002", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=CfopEntry, row_kwargs=lambda e: e)
|
||||
|
||||
def import_fcp(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="FCP", provider="Portal Nacional NF-e", act="Tabela de Alíquotas FCP por UF", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=FcpRate, row_kwargs=lambda e: e)
|
||||
|
||||
def import_anp(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="ANP", provider="Portal Nacional NF-e", act="Tabela de Código de Produto ANP", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=AnpProductCode, row_kwargs=lambda e: e)
|
||||
|
||||
def import_ncm_utrib(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="NCM_UTRIB", provider="Portal Nacional NF-e", act="Informe Técnico 2024.001 - NCM x uTrib", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=NcmUtribEntry, row_kwargs=lambda e: e)
|
||||
|
||||
def import_rtc_biofuel_index(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="RTC_BIOFUEL_INDEX", provider="Portal Nacional NF-e", act="Informe Técnico 2025.004 - pBioObrigatorio", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcBiofuelIndexEntry, row_kwargs=lambda e: e)
|
||||
|
||||
def import_rtc_cred_pres(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="RTC_CCREDPRES", provider="Portal Nacional NF-e", act="cCredPres — LC 214/2025", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcCredPresEntry, row_kwargs=lambda e: e)
|
||||
|
||||
def import_rtc_cclasstrib(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="RTC_CCLASSTRIB", provider="Portal Nacional NF-e", act="cClassTrib — IBS/CBS (LC 214/2025)", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcCClassTribEntry, row_kwargs=lambda e: e, extra=((RtcCstIbsCbsEntry, payload.extra_entries),))
|
||||
|
||||
def import_rtc_cbs_rates(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="RTC_CBS_RATES", provider="Portal Nacional NF-e", act="Informe Técnico 2026.002 - Alíquotas da CBS", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcCbsRateEntry, row_kwargs=lambda e: e)
|
||||
|
||||
def rtc_cclasstrib(self, code=None):
|
||||
query = select(RtcCClassTribEntry).join(Dataset).where(Dataset.kind == "RTC_CCLASSTRIB", Dataset.status == "ACTIVE")
|
||||
if code is not None:
|
||||
query = query.where(RtcCClassTribEntry.cclasstrib == str(code))
|
||||
return self.session.scalars(query).all()
|
||||
|
||||
def rtc_cbs_rate(self, year):
|
||||
return self.session.scalar(select(RtcCbsRateEntry).join(Dataset).where(Dataset.kind == "RTC_CBS_RATES", Dataset.status == "ACTIVE", RtcCbsRateEntry.year == int(year)))
|
||||
|
||||
def cfop_lookup(self, code=None):
|
||||
query = select(CfopEntry).join(Dataset).where(Dataset.kind == "CFOP", Dataset.status == "ACTIVE")
|
||||
if code is not None:
|
||||
query = query.where(CfopEntry.code == str(code))
|
||||
return self.session.scalars(query).all()
|
||||
|
||||
def fcp_for_uf(self, uf):
|
||||
return self.session.scalar(select(FcpRate).join(Dataset).where(Dataset.kind == "FCP", Dataset.status == "ACTIVE", FcpRate.uf == uf.upper()))
|
||||
|
||||
def anp_code_exists(self, cprod_anp):
|
||||
return self.session.scalar(select(AnpProductCode).join(Dataset).where(Dataset.kind == "ANP", Dataset.status == "ACTIVE", AnpProductCode.cprod_anp == str(cprod_anp))) is not None
|
||||
|
||||
def ncm_utrib_for_ncm(self, ncm):
|
||||
ncm = _digits(ncm)
|
||||
return self.session.scalar(select(NcmUtribEntry).join(Dataset).where(Dataset.kind == "NCM_UTRIB", Dataset.status == "ACTIVE", NcmUtribEntry.ncm == ncm))
|
||||
|
||||
def rtc_biofuel_for_anp(self, cprod_anp):
|
||||
return self.session.scalars(select(RtcBiofuelIndexEntry).join(Dataset).where(Dataset.kind == "RTC_BIOFUEL_INDEX", Dataset.status == "ACTIVE", RtcBiofuelIndexEntry.cprod_anp == str(cprod_anp), RtcBiofuelIndexEntry.is_historical.is_(False))).all()
|
||||
|
||||
def rtc_cred_pres(self, code=None):
|
||||
query = select(RtcCredPresEntry).join(Dataset).where(Dataset.kind == "RTC_CCREDPRES", Dataset.status == "ACTIVE")
|
||||
if code is not None:
|
||||
query = query.where(RtcCredPresEntry.code == str(code))
|
||||
return self.session.scalars(query).all()
|
||||
|
||||
def dataset_versions(self):
|
||||
active = self.session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
|
||||
cest = self.session.scalar(select(Dataset).where(Dataset.kind == "CEST", Dataset.status == "ACTIVE"))
|
||||
tipi = self.session.scalar(select(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.status == "ACTIVE"))
|
||||
mva = self.session.scalar(select(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE"))
|
||||
moc = self.session.scalar(select(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.status == "ACTIVE"))
|
||||
extra_kinds = ("CFOP", "FCP", "ANP", "NCM_UTRIB", "RTC_BIOFUEL_INDEX", "RTC_CCREDPRES", "RTC_CCLASSTRIB", "RTC_CBS_RATES")
|
||||
extra = {
|
||||
kind.lower(): self.session.scalar(select(Dataset.sha256).where(Dataset.kind == kind, Dataset.status == "ACTIVE"))
|
||||
for kind in extra_kinds
|
||||
}
|
||||
return {
|
||||
"ncm": active.sha256 if active else None,
|
||||
"cest": cest.sha256 if cest else None,
|
||||
"tipi_ipi": tipi.sha256 if tipi else None,
|
||||
"ba_st_mva": mva.sha256 if mva else None,
|
||||
"nfe_moc_tables": moc.sha256 if moc else None,
|
||||
**extra,
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Fail-closed parser for the official Receita Federal TIPI workbook."""
|
||||
from dataclasses import dataclass
|
||||
from io import BytesIO
|
||||
import hashlib
|
||||
import unicodedata
|
||||
|
||||
import openpyxl
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TipiEntry:
|
||||
ncm: str
|
||||
ex: str | None
|
||||
description: str
|
||||
rate: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TipiPayload:
|
||||
sha256: str
|
||||
entries: tuple[TipiEntry, ...]
|
||||
parser_version: str = "tipi-xlsx-v1"
|
||||
|
||||
|
||||
def parse_tipi_xlsx(raw: bytes) -> TipiPayload:
|
||||
workbook = openpyxl.load_workbook(BytesIO(raw), read_only=True, data_only=True)
|
||||
sheet = workbook.active
|
||||
normalize = lambda value: "".join(char for char in unicodedata.normalize("NFKD", str(value or "")).encode("ascii", "ignore").decode().upper() if char.isalnum())
|
||||
header_row = next((row for row in sheet.iter_rows(values_only=True) if {"NCM", "DESCRICAO", "ALIQUOTA"} <= {normalize(cell) for cell in row}), None)
|
||||
if header_row is None:
|
||||
raise ValueError("TIPI workbook has no recognized official header")
|
||||
headers = [normalize(cell) for cell in header_row]
|
||||
ncm_index = headers.index("NCM")
|
||||
ex_index = headers.index("EX")
|
||||
description_index = next(index for index, value in enumerate(headers) if value == "DESCRICAO")
|
||||
rate_index = next(index for index, value in enumerate(headers) if value.startswith("ALIQUOTA"))
|
||||
entries = []
|
||||
started = False
|
||||
for row in sheet.iter_rows(values_only=True):
|
||||
if tuple(row) == tuple(header_row):
|
||||
started = True; continue
|
||||
if not started or ncm_index >= len(row):
|
||||
continue
|
||||
ncm = "".join(char for char in str(row[ncm_index] or "") if char.isdigit())
|
||||
if len(ncm) != 8:
|
||||
continue
|
||||
description = str(row[description_index] or "").strip()
|
||||
if not description:
|
||||
raise ValueError("TIPI NCM row has no description")
|
||||
ex = str(row[ex_index] or "").strip() or None
|
||||
entries.append(TipiEntry(ncm, ex, description, str(row[rate_index] or "").strip()))
|
||||
if not entries:
|
||||
raise ValueError("TIPI workbook has no 8-digit NCM rows")
|
||||
return TipiPayload(hashlib.sha256(raw).hexdigest(), tuple(entries))
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Automatic refresh of the official NCM dataset (Siscomex JSON): staging -> validate -> atomic promotion, keeping the last valid dataset on ANY failure.
|
||||
|
||||
Every attempt leaves a SourceRefreshRun (url, time, sha256, version label, record count, status, error) for the panel. Nothing here touches Agents or providers.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..config import settings
|
||||
from ..models import Dataset, SourceRefreshRun
|
||||
from .ncm import parse_ncm_payload
|
||||
from .service import DatasetService
|
||||
|
||||
logger = logging.getLogger("gti_fiscal_hub")
|
||||
MIN_RATIO = 0.9 # a payload with < 90% of the active record count is treated as a broken download, never promoted
|
||||
INTERVAL = timedelta(hours=24)
|
||||
RETRY_AFTER_FAILURE = timedelta(hours=6)
|
||||
|
||||
|
||||
def _fetch(url: str) -> bytes:
|
||||
response = httpx.get(url, timeout=max(settings.http_timeout_seconds, 60), follow_redirects=True, headers={"User-Agent": "GTIFiscalHub/1.0 (official dataset refresh)"})
|
||||
response.raise_for_status()
|
||||
return response.content
|
||||
|
||||
|
||||
def refresh_ncm(session, fetch=_fetch, url: str | None = None) -> SourceRefreshRun:
|
||||
url = url or settings.ncm_dataset_url
|
||||
run = SourceRefreshRun(source="NCM", source_url=url, status="RUNNING")
|
||||
session.add(run)
|
||||
session.commit()
|
||||
try:
|
||||
content = fetch(url) # staging: bytes in memory, nothing written to the active dataset yet
|
||||
parsed = parse_ncm_payload(content) # schema validation
|
||||
run.sha256, run.version_label, run.record_count = parsed.sha256, (parsed.version or "")[:64], len(parsed.entries)
|
||||
active = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
|
||||
if active is not None and active.sha256 == parsed.sha256:
|
||||
run.status = "UNCHANGED"
|
||||
else:
|
||||
if active is not None and len(parsed.entries) < MIN_RATIO * (active.record_count or 0):
|
||||
raise ValueError(f"implausible record count {len(parsed.entries)} vs active {active.record_count}")
|
||||
DatasetService(session).import_payload(parsed, url, datetime.now(timezone.utc)) # atomic: one transaction, old ACTIVE superseded only at the end
|
||||
run.status = "UPDATED"
|
||||
try: # the KB is never deleted: only flagged STALE_REVIEW_REQUIRED when its NCM left the official dataset
|
||||
from ..enrichment import history, kb
|
||||
current = history.official_sets(session)
|
||||
run.error = f"kb_revalidated: {kb.revalidate(session, lambda v: v in current)}"[:300]
|
||||
except Exception as error: # noqa: BLE001 - revalidation trouble must not undo a valid dataset promotion
|
||||
run.error = f"kb_revalidate_failed: {type(error).__name__}"[:300]
|
||||
except Exception as error: # keep the last valid dataset
|
||||
session.rollback()
|
||||
run = session.get(SourceRefreshRun, run.id)
|
||||
run.status, run.error = "FAILED", f"{type(error).__name__}: {error}"[:300]
|
||||
logger.warning("NCM refresh failed: %s", run.error)
|
||||
run.finished_at = datetime.now(timezone.utc)
|
||||
session.commit()
|
||||
return run
|
||||
|
||||
|
||||
def last_runs(session, source: str = "NCM", limit: int = 5):
|
||||
return session.scalars(select(SourceRefreshRun).where(SourceRefreshRun.source == source).order_by(SourceRefreshRun.id.desc()).limit(limit)).all()
|
||||
|
||||
|
||||
def _due(session) -> bool:
|
||||
last = session.scalar(select(SourceRefreshRun).where(SourceRefreshRun.source == "NCM", SourceRefreshRun.status != "RUNNING").order_by(SourceRefreshRun.id.desc()).limit(1))
|
||||
if last is None or last.finished_at is None:
|
||||
return True
|
||||
finished = last.finished_at if last.finished_at.tzinfo else last.finished_at.replace(tzinfo=timezone.utc)
|
||||
return datetime.now(timezone.utc) - finished >= (RETRY_AFTER_FAILURE if last.status == "FAILED" else INTERVAL)
|
||||
|
||||
|
||||
def start_scheduler(session_factory, first_delay: float = 90.0):
|
||||
"""Daemon thread: checks hourly whether a refresh is due (survives restarts because due-ness is derived from SourceRefreshRun)."""
|
||||
def loop():
|
||||
time.sleep(first_delay)
|
||||
while True:
|
||||
try:
|
||||
with session_factory() as session:
|
||||
if _due(session):
|
||||
refresh_ncm(session)
|
||||
except Exception: # never let the scheduler die
|
||||
logger.exception("official refresh loop error")
|
||||
time.sleep(3600)
|
||||
thread = threading.Thread(target=loop, name="official-refresh", daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
@@ -0,0 +1,15 @@
|
||||
"""SQLAlchemy session setup for central PostgreSQL only."""
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from .config import settings
|
||||
|
||||
engine = create_engine(settings.database_url, pool_pre_ping=True)
|
||||
SessionLocal = sessionmaker(bind=engine, expire_on_commit=False)
|
||||
|
||||
def database_is_available() -> bool:
|
||||
try:
|
||||
with engine.connect() as connection:
|
||||
connection.execute(text("SELECT 1"))
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Conservative deterministic role, ST, and inbound scope resolution."""
|
||||
|
||||
|
||||
def resolve_product_role(signals: dict) -> dict:
|
||||
if signals.get("recipe") and signals.get("production"):
|
||||
return {"value": "OWN_PRODUCTION", "status": "CONFIRMED", "evidence": ["recipe", "production"]}
|
||||
if signals.get("purchase") and signals.get("sale") and not signals.get("recipe"):
|
||||
return {"value": "RESALE", "status": "CONFIRMED", "evidence": ["purchase", "sale"]}
|
||||
if signals.get("input_consumed") and signals.get("recipe"):
|
||||
return {"value": "INPUT", "status": "CONFIRMED", "evidence": ["input_consumed", "recipe"]}
|
||||
if signals.get("service_flag") and signals.get("service_transactions"):
|
||||
return {"value": "SERVICE", "status": "CONFIRMED", "evidence": ["service_flag", "service_transactions"]}
|
||||
return {"value": None, "status": "PENDING_PRODUCT_ROLE", "evidence": []}
|
||||
|
||||
|
||||
def resolve_st_status(signals: dict) -> dict:
|
||||
if not signals.get("official_cest"):
|
||||
return {"value": "NOT_SUBJECT_TO_ST", "status": "CONFIRMED", "evidence": []}
|
||||
if signals.get("purchase_retained") is True:
|
||||
return {"value": "ST_ALREADY_WITHHELD_IN_PURCHASE", "status": "CONFIRMED", "evidence": ["official_cest", "purchase_retained"]}
|
||||
if signals.get("purchase_retained") is False:
|
||||
return {"value": "PRODUCT_SUBJECT_TO_ST", "status": "CONFIRMED", "evidence": ["official_cest", "purchase_not_retained"]}
|
||||
return {"value": None, "status": "PENDING_ST_WITHHELD", "evidence": ["official_cest"]}
|
||||
|
||||
|
||||
def resolve_entry_scope(supplier_ufs: list[str], company_uf: str) -> dict:
|
||||
observed = {str(uf).strip().upper() for uf in supplier_ufs if uf}
|
||||
if not observed:
|
||||
return {"value": None, "status": "PENDING_ENTRY_SCOPE", "evidence": []}
|
||||
if len(observed - {company_uf.upper()}) and company_uf.upper() in observed:
|
||||
value = "MIXED_OR_DYNAMIC"
|
||||
elif observed - {company_uf.upper()}:
|
||||
value = "INTER"
|
||||
else:
|
||||
value = "INTRA"
|
||||
return {"value": value, "status": "CONFIRMED", "evidence": sorted(observed)}
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Dry-run ERP write planner: turns classifier decisions into concrete field
|
||||
proposals without ever issuing a database write. Every proposal carries
|
||||
write_eligible; any future executor must refuse proposals where it is False."""
|
||||
|
||||
IPI_FIELD = "IPI" # C000025.IPI NUMERIC(15,2) is the real IPI rate field; C000025.ALIQ_IPI
|
||||
# is a VARCHAR(1) that is unused (0/521 rows) and cannot hold a TIPI rate or "NT" marker.
|
||||
EX_TIPI_FIELD = "EX_TIPI"
|
||||
CEST_FIELD = "CEST"
|
||||
NCM_FIELD = "CLASSIFICACAO_FISCAL"
|
||||
|
||||
_CEST_UNRESOLVED_REASONS = {
|
||||
"CEST_CANDIDATES": "MULTIPLE_CEST_CANDIDATES_NO_SINGLE_EVIDENCE",
|
||||
"CEST_DESCRIPTION_MISMATCH": "CEST_DESCRIPTION_SCORE_BELOW_THRESHOLD",
|
||||
"NO_CEST_MAPPING_FOUND": "NO_CEST_MAPPING_FOUND_FOR_NCM",
|
||||
}
|
||||
|
||||
|
||||
def _provenance(decision):
|
||||
return {
|
||||
"engine_version": decision.get("engine_version"),
|
||||
"decision_version": decision.get("decision_version"),
|
||||
"request_fingerprint": decision.get("request_fingerprint"),
|
||||
}
|
||||
|
||||
|
||||
def _same(current, proposed):
|
||||
return str(current).strip() == str(proposed).strip()
|
||||
|
||||
|
||||
def build_field_proposals(product_code, decision, current):
|
||||
"""current: dict with optional 'ipi' and 'cest' current ERP values.
|
||||
A missing/None key means the current ERP value is unknown, not that it is blank."""
|
||||
dataset_versions = decision.get("dataset_versions", {})
|
||||
classification = decision.get("classification", {})
|
||||
|
||||
if decision.get("status") == "LEGACY_NCM":
|
||||
return [{
|
||||
"product_code": product_code,
|
||||
"field": NCM_FIELD,
|
||||
"current_value": classification.get("ncm"),
|
||||
"proposed_value": None,
|
||||
"status": "LEGACY_NCM_REVIEW",
|
||||
"reason_code": "LEGACY_NCM_NO_AUTOMATIC_SUCCESSOR",
|
||||
"source_dataset": "NCM",
|
||||
"source_version": dataset_versions.get("ncm"),
|
||||
"provenance": _provenance(decision),
|
||||
"write_eligible": False,
|
||||
}]
|
||||
|
||||
proposals = []
|
||||
|
||||
ipi_status = decision.get("ipi_status")
|
||||
ipi = classification.get("ipi")
|
||||
if ipi_status == "IPI_RESOLVED" and ipi:
|
||||
current_ipi = current.get("ipi")
|
||||
current_ipi_read = "ipi" in current
|
||||
rate_kind = ipi.get("rate_kind")
|
||||
|
||||
if rate_kind == "NUMERIC":
|
||||
proposed_rate = ipi.get("rate_numeric")
|
||||
if not current_ipi_read:
|
||||
status, write_eligible, reason = "CURRENT_UNKNOWN", False, "ERP_CURRENT_VALUE_NOT_READ"
|
||||
elif _same(current_ipi, proposed_rate):
|
||||
status, write_eligible, reason = "SAME_VALUE", False, "TIPI_MATCHES_CURRENT_ERP"
|
||||
else:
|
||||
status, write_eligible, reason = "CHANGE_REQUIRED", True, "TIPI_RATE_DIVERGES_FROM_ERP"
|
||||
elif rate_kind == "NON_NUMERIC":
|
||||
# e.g. TIPI text marker "NT" (não tributado) — never write text into a numeric field.
|
||||
proposed_rate = None
|
||||
rate_classification = ipi.get("rate_classification", "OTHER")
|
||||
status, write_eligible, reason = "NON_NUMERIC_IPI_STATUS", False, f"TIPI_RATE_{rate_classification}"
|
||||
else: # BLANK — source spreadsheet had no rate at all; never invent zero.
|
||||
proposed_rate = None
|
||||
rate_classification = ipi.get("rate_classification", "SOURCE_BLANK")
|
||||
status, write_eligible, reason = rate_classification, False, f"TIPI_RATE_{rate_classification}"
|
||||
|
||||
proposals.append({
|
||||
"product_code": product_code,
|
||||
"field": IPI_FIELD,
|
||||
"current_value": current_ipi,
|
||||
"proposed_value": proposed_rate,
|
||||
"status": status,
|
||||
"reason_code": reason,
|
||||
"source_dataset": "TIPI_IPI",
|
||||
"source_version": dataset_versions.get("tipi_ipi"),
|
||||
"provenance": _provenance(decision),
|
||||
"write_eligible": write_eligible,
|
||||
})
|
||||
|
||||
# C000025 has no EX_TIPI row under IPI_RESOLVED by construction: the matched
|
||||
# TIPI entry is the base (non-EX) row, so there is nothing concrete to propose.
|
||||
|
||||
# EX TIPI only ever comes from EX-specific TIPI rows for this NCM (ipi_candidates).
|
||||
# No EX row for the NCM => no proposal at all (never propose a blank/None EX).
|
||||
ex_candidates = [c for c in classification.get("ipi_candidates", []) if c.get("ex")]
|
||||
if ex_candidates:
|
||||
current_ex = current.get("ex_tipi")
|
||||
current_ex_read = "ex_tipi" in current
|
||||
if len(ex_candidates) == 1:
|
||||
proposed_ex = ex_candidates[0]["ex"]
|
||||
if not current_ex_read:
|
||||
ex_status, ex_write_eligible, ex_reason = "CURRENT_UNKNOWN", False, "ERP_CURRENT_VALUE_NOT_READ"
|
||||
elif _same(current_ex, proposed_ex):
|
||||
ex_status, ex_write_eligible, ex_reason = "SAME_VALUE", False, "TIPI_EX_MATCHES_CURRENT_ERP"
|
||||
else:
|
||||
ex_status, ex_write_eligible, ex_reason = "CHANGE_REQUIRED", True, "TIPI_EX_DIVERGES_FROM_ERP"
|
||||
else:
|
||||
proposed_ex = None
|
||||
ex_status, ex_write_eligible, ex_reason = "CANDIDATE", False, "MULTIPLE_EX_TIPI_CANDIDATES_NO_SINGLE_EVIDENCE"
|
||||
proposals.append({
|
||||
"product_code": product_code,
|
||||
"field": EX_TIPI_FIELD,
|
||||
"current_value": current_ex,
|
||||
"proposed_value": proposed_ex,
|
||||
"status": ex_status,
|
||||
"reason_code": ex_reason,
|
||||
"source_dataset": "TIPI_IPI",
|
||||
"source_version": dataset_versions.get("tipi_ipi"),
|
||||
"provenance": _provenance(decision),
|
||||
"write_eligible": ex_write_eligible,
|
||||
"candidate_count": len(ex_candidates),
|
||||
})
|
||||
|
||||
cest_status = decision.get("cest_status")
|
||||
if cest_status is not None:
|
||||
current_cest = current.get("cest")
|
||||
current_cest_read = "cest" in current
|
||||
if cest_status == "CEST_CONFIRMED":
|
||||
proposed_cest = (classification.get("cest") or [None])[0]
|
||||
if not current_cest_read:
|
||||
status, write_eligible, reason = "CURRENT_UNKNOWN", False, "ERP_CURRENT_VALUE_NOT_READ"
|
||||
elif _same(current_cest, proposed_cest):
|
||||
status, write_eligible, reason = "SAME_VALUE", False, "CEST_MATCHES_CURRENT_ERP"
|
||||
else:
|
||||
status, write_eligible, reason = "CHANGE_REQUIRED", True, "CEST_DIVERGES_FROM_ERP"
|
||||
else:
|
||||
proposed_cest = None
|
||||
write_eligible = False
|
||||
status = "CANDIDATE" if cest_status == "CEST_CANDIDATES" else "UNRESOLVED"
|
||||
reason = _CEST_UNRESOLVED_REASONS.get(cest_status, "CEST_NOT_RESOLVED")
|
||||
proposals.append({
|
||||
"product_code": product_code,
|
||||
"field": CEST_FIELD,
|
||||
"current_value": current_cest,
|
||||
"proposed_value": proposed_cest,
|
||||
"status": status,
|
||||
"reason_code": reason,
|
||||
"source_dataset": "CEST",
|
||||
"source_version": dataset_versions.get("cest"),
|
||||
"provenance": _provenance(decision),
|
||||
"write_eligible": write_eligible,
|
||||
})
|
||||
|
||||
return proposals
|
||||
|
||||
|
||||
def mva_evidence(product_code, decision):
|
||||
"""BA ST/MVA has no ERP field target: keep it as fiscal evidence only, never a write proposal."""
|
||||
classification = decision.get("classification", {})
|
||||
return {
|
||||
"product_code": product_code,
|
||||
"ncm": classification.get("ncm"),
|
||||
"status": decision.get("ba_st_mva_status"),
|
||||
"resolved": classification.get("ba_st_mva"),
|
||||
"candidates": classification.get("ba_st_mva_candidates", []),
|
||||
"source_version": decision.get("dataset_versions", {}).get("ba_st_mva"),
|
||||
}
|
||||
|
||||
|
||||
def split_by_eligibility(proposals):
|
||||
return {
|
||||
"eligible": [p for p in proposals if p.get("write_eligible")],
|
||||
"blocked": [p for p in proposals if not p.get("write_eligible")],
|
||||
}
|
||||
|
||||
|
||||
def execute(proposals):
|
||||
"""Stub executor. Refuses any proposal that is not write_eligible and never
|
||||
issues a write itself — implementing the actual ERP UPDATE is out of scope."""
|
||||
for proposal in proposals:
|
||||
if not proposal.get("write_eligible"):
|
||||
raise PermissionError(
|
||||
f"refused: {proposal['product_code']}/{proposal['field']} is not write_eligible "
|
||||
f"(status={proposal['status']})"
|
||||
)
|
||||
raise NotImplementedError("no ERP write executor is implemented; this planner is dry-run only")
|
||||
@@ -0,0 +1,2 @@
|
||||
"""Product enrichment (new product -> GTIN/name -> fiscal profile -> safe fill). Deterministic only: no AI, no LLM, no embeddings, no paid providers.
|
||||
Everything lives inside the existing Hub/FastAPI/PostgreSQL; providers are hidden behind the Hub (Agents never call them)."""
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Agent-facing product enrichment routes (authenticated as the Agent; the Agent never talks to external providers)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..blocked_products import upsert_blocker
|
||||
from ..config import settings
|
||||
from ..field_contract import WRITE_FIELDS
|
||||
from ..models import Installation, ProductEnrichment, ProductFieldAudit
|
||||
from . import status as enrichment_status, textnorm
|
||||
from .placeholders import is_placeholder
|
||||
from .providers import build_providers
|
||||
from .quota import ProviderGateway
|
||||
from .resolver import FIELDS, resolve
|
||||
|
||||
CLASSIFICATION_CONTEXT_FIELDS = ("CODGRUPO", "TIPO", "PROD_COZINHA", "PRODUTO_SERVICO") # read-only evidence for field_contract.infer_role; never written
|
||||
CURRENT_FIELDS = tuple(dict.fromkeys(FIELDS + tuple(WRITE_FIELDS) + CLASSIFICATION_CONTEXT_FIELDS)) # everything resolve() may read from the Agent's `current`: NCM/CEST/IPI + every fiscal-completion field + role evidence
|
||||
|
||||
|
||||
def sanitize_product_evidence(raw) -> dict | None:
|
||||
"""Bounds the Agent's targeted, single-product evidence (firebird_reader.read_product_evidence) before it reaches field_contract: shape-only
|
||||
validation (never trusts an authenticated Agent to send something huge or malformed for a single-product payload)."""
|
||||
if not isinstance(raw, dict):
|
||||
return None
|
||||
out = {"recipes": set(), "composed": set(), "composed_ratio": {}, "purchases": {}, "stock_entry": {}, "stock_in": {}}
|
||||
for key in ("recipes", "composed"):
|
||||
if isinstance(raw.get(key), list):
|
||||
out[key] = {str(v) for v in raw[key][:8]}
|
||||
if isinstance(raw.get("composed_ratio"), dict):
|
||||
for code, counts in list(raw["composed_ratio"].items())[:8]:
|
||||
if isinstance(counts, dict) and isinstance(counts.get("composed"), int) and isinstance(counts.get("other"), int):
|
||||
out["composed_ratio"][str(code)] = {"composed": max(0, counts["composed"]), "other": max(0, counts["other"])}
|
||||
if isinstance(raw.get("purchases"), dict):
|
||||
for code, docs in list(raw["purchases"].items())[:8]:
|
||||
if isinstance(docs, list):
|
||||
out["purchases"][str(code)] = [d for d in docs[:32] if isinstance(d, dict)]
|
||||
for key in ("stock_entry", "stock_in"):
|
||||
if isinstance(raw.get(key), dict):
|
||||
out[key] = {str(k): bool(v) for k, v in list(raw[key].items())[:8]}
|
||||
return out
|
||||
|
||||
|
||||
PROVIDERS_OVERRIDE = None # tests inject fake providers here (never the network)
|
||||
PENDING_STATES = {"PENDING_PROVIDER_QUOTA", "PENDING_PROVIDERS_OFFLINE", "PENDING_TIME_BUDGET"}
|
||||
REASONS = { # human PT-BR reasons shown through the existing blockers (category ENRICHMENT); never internal jargon
|
||||
"INVALID_GTIN": "Código de barras (GTIN) inválido: o produto não foi consultado em fontes externas.",
|
||||
"AMBIGUOUS": "Descrição genérica demais para identificar o produto com segurança; informe o código de barras.",
|
||||
"NOT_FOUND": "Produto não encontrado nas fontes consultadas; preencha os campos fiscais manualmente.",
|
||||
"CONFLICT": "As fontes divergem sobre a classificação deste produto; nenhuma alteração foi feita.",
|
||||
"SINGLE_WEAK_SOURCE": "Apenas uma fonte identificou o produto; a classificação foi apenas sugerida.",
|
||||
"PENDING_PROVIDER_QUOTA": "Limite diário das fontes de produtos atingido; a consulta será repetida automaticamente.",
|
||||
"PENDING_PROVIDERS_OFFLINE": "Fontes de produtos indisponíveis no momento; a consulta será repetida automaticamente.",
|
||||
}
|
||||
|
||||
|
||||
def _aware(dt):
|
||||
return dt.replace(tzinfo=timezone.utc) if dt is not None and dt.tzinfo is None else dt
|
||||
|
||||
|
||||
def features_of(installation) -> dict:
|
||||
return {"corrector_enabled": bool(installation.corrector_enabled), "enrichment_mode": installation.enrichment_mode, "features_configured": bool(installation.features_configured)}
|
||||
|
||||
|
||||
def router(principal, providers_factory=None):
|
||||
routes = APIRouter(prefix="/products")
|
||||
holder = {}
|
||||
|
||||
def providers():
|
||||
if PROVIDERS_OVERRIDE is not None:
|
||||
return PROVIDERS_OVERRIDE
|
||||
if providers_factory is not None:
|
||||
return providers_factory()
|
||||
if "p" not in holder:
|
||||
holder["p"] = build_providers(settings)
|
||||
return holder["p"]
|
||||
|
||||
@routes.post("/resolve")
|
||||
def resolve_product(payload: dict, identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is None:
|
||||
raise HTTPException(409, "installation not registered")
|
||||
features = features_of(installation)
|
||||
if not features["features_configured"] or installation.enrichment_mode == "DISABLED":
|
||||
return {"state": "FEATURE_DISABLED", "features": features, "fields": {}}
|
||||
code = str(payload.get("product_code") or "")[:64]
|
||||
if not code:
|
||||
raise HTTPException(422, "product_code required")
|
||||
description, current = str(payload.get("description") or "")[:255], payload.get("current") if isinstance(payload.get("current"), dict) else {}
|
||||
started = time.monotonic()
|
||||
result = resolve(session, ProviderGateway(session, providers()), gtin_raw=payload.get("gtin"), description=description, current={k: current.get(k) for k in CURRENT_FIELDS},
|
||||
client_label=f"installation:{installation.id}", budget_s=settings.enrichment_time_budget_seconds,
|
||||
product_code=code, installation=installation, tenant_id=agent.tenant_id, product_evidence=sanitize_product_evidence(payload.get("ev")))
|
||||
latency = int((time.monotonic() - started) * 1000)
|
||||
row = session.scalar(select(ProductEnrichment).where(ProductEnrichment.installation_id == installation.id, ProductEnrichment.product_code == code))
|
||||
if row is None:
|
||||
row = ProductEnrichment(tenant_id=agent.tenant_id, installation_id=installation.id, product_code=code)
|
||||
session.add(row)
|
||||
for field in (row.human_overridden or {}): # a human decision (even deleting the value) is final for this product
|
||||
if field in result["fields"]:
|
||||
result["fields"][field] = {"status": "HUMAN_OVERRIDDEN", "value": None, "rule": None, "kb_status": None, "evidence": [], "reason": "valor alterado por uma pessoa; nunca mais preenchido automaticamente"}
|
||||
pending = result["state"] in PENDING_STATES
|
||||
for name, decision in result["fields"].items(): # explain in the audit trail that a technical placeholder was treated as "not classified yet"
|
||||
if is_placeholder(name, current.get(name)):
|
||||
decision["technical_placeholder"] = str(current.get(name))
|
||||
row.gtin, row.description, row.latency_ms = result.get("gtin"), description, latency
|
||||
row.attempts = (row.attempts or 0) + 1
|
||||
row.ui_status, row.reason_code = enrichment_status.classify(result, has_gtin=bool(result.get("gtin")), specific_name=textnorm.is_specific(description))
|
||||
row.last_attempt_at, row.reevaluate = datetime.now(timezone.utc), False
|
||||
row.status = "PENDING" if pending else "RESOLVED"
|
||||
row.decisions = {"state": result["state"], "fields": {k: {kk: vv for kk, vv in v.items() if kk != "evidence"} for k, v in result["fields"].items()}}
|
||||
row.next_attempt_at = _aware(result["retry_at"]) if pending else None
|
||||
if pending and row.next_attempt_at is None:
|
||||
row.next_attempt_at = datetime.now(timezone.utc).replace(microsecond=0)
|
||||
session.commit()
|
||||
reason = REASONS.get(result["state"])
|
||||
if reason:
|
||||
upsert_blocker(session, agent.tenant_id, {"company": installation.hostname or "", "product_code": code, "gtin": result.get("gtin"), "description": description, "blocker_type": "ENRICHMENT_" + result["state"],
|
||||
"field": "ENRICHMENT", "reason_code": result["state"], "evidence_summary": {"message": reason}, "input_fingerprint": f"{result.get('gtin')}|{description}"}, installation_id=installation.id)
|
||||
result.update({"features": features, "latency_ms": latency, "mode": installation.enrichment_mode})
|
||||
result["retry_at"] = result["retry_at"].isoformat() if result["retry_at"] else None
|
||||
return result
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@routes.post("/enrichment/applied")
|
||||
def applied(payload: dict, identity=Depends(principal)):
|
||||
"""The Agent reports what it wrote / refused. The Hub re-checks that the feature is still ENABLED and audits every field."""
|
||||
session, agent = identity
|
||||
try:
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is None:
|
||||
raise HTTPException(409, "installation not registered")
|
||||
code = str(payload.get("product_code") or "")[:64]
|
||||
row = session.scalar(select(ProductEnrichment).where(ProductEnrichment.installation_id == installation.id, ProductEnrichment.product_code == code))
|
||||
if row is None:
|
||||
raise HTTPException(404, "unknown product enrichment")
|
||||
versions = payload.get("dataset_versions") if isinstance(payload.get("dataset_versions"), dict) else {}
|
||||
applied_fields, overridden = dict(row.fields_applied or {}), dict(row.human_overridden or {})
|
||||
for item in (payload.get("fields") or [])[:32]:
|
||||
if not isinstance(item, dict) or item.get("field") not in FIELDS:
|
||||
continue
|
||||
outcome = str(item.get("outcome") or "APPLIED")[:24]
|
||||
session.add(ProductFieldAudit(tenant_id=agent.tenant_id, installation_id=installation.id, agent_id=agent.id, product_code=code, gtin=row.gtin, description_snapshot=row.description, field=item["field"],
|
||||
old_value=(str(item["old"])[:64] if item.get("old") is not None else None), new_value=(str(item["new"])[:64] if item.get("new") is not None else None), rule=str(item.get("rule") or "")[:64] or None,
|
||||
kb_status=str(item.get("kb_status") or "")[:16] or None, evidence=item.get("evidence") if isinstance(item.get("evidence"), list) else [], dataset_versions=versions,
|
||||
authorization_origin=f"enrichment_mode:{installation.enrichment_mode}", job_id=payload.get("job_id") if isinstance(payload.get("job_id"), int) else None, outcome=outcome,
|
||||
reason=str(item.get("reason") or "")[:96] or None))
|
||||
if outcome == "APPLIED":
|
||||
applied_fields[item["field"]] = item.get("new")
|
||||
elif outcome == "HUMAN_OVERRIDDEN":
|
||||
overridden[item["field"]] = True
|
||||
row.fields_applied, row.human_overridden = applied_fields, overridden
|
||||
ui, why = enrichment_status.after_write([i for i in (payload.get("fields") or []) if isinstance(i, dict)], installation.enrichment_mode)
|
||||
if ui:
|
||||
row.ui_status, row.reason_code = ui, why
|
||||
row.reevaluate = False
|
||||
row.status = "APPLIED" if applied_fields and not overridden else ("PARTIAL" if applied_fields else row.status if row.status != "PENDING" else "RESOLVED")
|
||||
row.updated_at = datetime.now(timezone.utc)
|
||||
session.commit()
|
||||
return {"ok": True, "status": row.status, "mode": installation.enrichment_mode}
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@routes.get("/enrichment/pending")
|
||||
def pending_reevaluation(identity=Depends(principal)):
|
||||
"""Tracked products flagged for a new evaluation (panel button, REPROCESS/SYNC, KB decision). The Agent polls this cheaply and re-runs resolve for each code."""
|
||||
session, agent = identity
|
||||
try:
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is None or not installation.features_configured or installation.enrichment_mode == "DISABLED":
|
||||
return {"codes": []}
|
||||
codes = list(session.scalars(select(ProductEnrichment.product_code).where(ProductEnrichment.installation_id == installation.id, ProductEnrichment.reevaluate.is_(True)).order_by(ProductEnrichment.id).limit(25)))
|
||||
return {"codes": codes}
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@routes.get("/enrichment/authorization")
|
||||
def authorization(identity=Depends(principal)):
|
||||
"""Fresh central authorization read by the Agent right before EVERY field write."""
|
||||
session, agent = identity
|
||||
try:
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
return features_of(installation) if installation is not None else {"corrector_enabled": False, "enrichment_mode": "DISABLED", "features_configured": False}
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
return routes
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Gradual, quota-safe verification of locally CORROBORATED GTINs by an NCM-bearing free provider (OSCBR). Never spends the operational reserve.
|
||||
|
||||
Rules: only CORROBORATED single-NCM GTINs (never VERIFIED_KB, never CONFLICT, never an invalid GTIN); each GTIN is checked once per `RECHECK_DAYS` (next_checked_at kept in the KB evidence, plus the
|
||||
provider cache); a provider answer is EVIDENCE only - the promotion/conflict decision is kb.evaluate's (local strong evidence + independent provider NCM + vigente + zero conflict).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..models import KbKnowledge, KbProduct
|
||||
from . import kb
|
||||
from .gtin import normalize_gtin
|
||||
|
||||
RECHECK_DAYS = 30
|
||||
|
||||
|
||||
def _now():
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def select_sample(session, is_official, limit: int, provider: str = "oscbr") -> list[dict]:
|
||||
"""Priority: multi-client, more documents, more years; current-vigente NCM; zero conflict; not checked recently."""
|
||||
ids = list(session.scalars(select(KbKnowledge.kb_product_id).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.status == kb.CORROBORATED).distinct()))
|
||||
by_product: dict[int, list] = {}
|
||||
for start in range(0, len(ids), 500): # every NCM row of those products: a second (even weaker) value already disqualifies the GTIN
|
||||
for r in session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.kb_product_id.in_(ids[start:start + 500]))):
|
||||
by_product.setdefault(r.kb_product_id, []).append(r)
|
||||
now = _now().isoformat()
|
||||
picks = []
|
||||
for pid, items in by_product.items():
|
||||
valid = [i for i in items if is_official(i.value)]
|
||||
if len(valid) != 1 or valid[0].status != kb.CORROBORATED:
|
||||
continue # conflicts / no vigente NCM / already verified are not asked
|
||||
row = valid[0]
|
||||
info = (row.evidence or {}).get("info") or {}
|
||||
if info.get(f"{provider}_next_checked_at", "") > now:
|
||||
continue
|
||||
prod = session.get(KbProduct, pid)
|
||||
if not prod or normalize_gtin(prod.gtin[1:] if prod.gtin and prod.gtin.startswith("0") else prod.gtin)["status"] != "VALID_GTIN":
|
||||
continue
|
||||
summary = kb._lineage_summary([row])
|
||||
if any(f.startswith("PROVIDER:" + provider) for f in ((row.evidence or {}).get("families") or {})):
|
||||
continue
|
||||
picks.append({"gtin_key": prod.gtin, "ncm": row.value, "clients": len(summary["clients"]), "docs": summary["docs"], "years": len(summary["years"]), "product_id": pid})
|
||||
picks.sort(key=lambda p: (-p["clients"], -p["docs"], -p["years"], p["gtin_key"]))
|
||||
return picks[:limit]
|
||||
|
||||
|
||||
def apply_outcome(session, gtin_key: str, provider: str, outcome, is_official) -> dict:
|
||||
"""Turns one gateway Outcome into KB evidence. Returns the per-GTIN measurement record."""
|
||||
product = kb.find_by_gtin(session, gtin_key)
|
||||
rec = {"gtin": gtin_key[-13:] if gtin_key.startswith("0") else gtin_key, "state": outcome.state, "reason": outcome.reason, "found": False, "ncm": None, "cest": None, "description": None, "brand": None,
|
||||
"local_ncm": None, "official_valid": None, "kb_status": None, "external_conflict": False}
|
||||
local_rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm"))) if product else []
|
||||
rec["local_ncm"] = local_rows[0].value if local_rows else None
|
||||
stamp = (_now() + timedelta(days=RECHECK_DAYS)).isoformat()
|
||||
if outcome.state in ("FOUND", "CACHE_HIT") and outcome.product:
|
||||
p = outcome.product
|
||||
rec.update({"found": True, "ncm": p.ncm, "cest": p.cest, "description": p.title, "brand": p.brand})
|
||||
if p.ncm:
|
||||
rec["official_valid"] = bool(is_official(p.ncm))
|
||||
kb.add_evidence(session, product, "ncm", p.ncm, f"provider:{provider}"[:32], family=f"PROVIDER:{provider}", info={"raw_hash": p.raw_hash})
|
||||
if p.cest:
|
||||
kb.add_evidence(session, product, "cest", p.cest, f"provider:{provider}"[:32], family=f"PROVIDER:{provider}", info={"raw_hash": p.raw_hash})
|
||||
if p.brand and not product.brand:
|
||||
product.brand = p.brand
|
||||
if p.title:
|
||||
kb.add_alias(session, product, p.title.upper()[:200])
|
||||
ev = kb.evaluate(session, product, "ncm", is_official)
|
||||
rec["kb_status"] = ev["status"]
|
||||
rec["external_conflict"] = bool(ev["status"] == kb.CONFLICT and rec["ncm"])
|
||||
if outcome.state in ("FOUND", "CACHE_HIT", "NOT_FOUND"): # remember the check so it is not repeated (skipped/errors are retried later)
|
||||
for r in session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm")):
|
||||
r.evidence = {**(r.evidence or {}), "info": {**((r.evidence or {}).get("info") or {}), f"{provider}_next_checked_at": stamp}}
|
||||
return rec
|
||||
|
||||
|
||||
def provider_query(key: str) -> str:
|
||||
"""Canonical form sent to providers: GTIN-8 stays 8 digits, UPC-12/EAN-13 as 13 digits (same rule as the resolver), GTIN-14 with an indicator digit stays 14."""
|
||||
if len(key) == 14 and key.startswith("000000"):
|
||||
return key[-8:]
|
||||
if len(key) == 14 and key.startswith("0"):
|
||||
return key[1:]
|
||||
return key
|
||||
|
||||
|
||||
def run(session, gateway, provider, *, limit: int, reserve_percent: int, is_official, dry_run: bool = False, wait_rate_window: bool = True) -> dict:
|
||||
"""One backfill pass. dry_run only lists the sample (no request)."""
|
||||
sample = select_sample(session, is_official, limit, provider.name)
|
||||
report = {"provider": provider.name, "sample_size": len(sample), "requests": 0, "records": [], "stopped": None, "dry_run": dry_run}
|
||||
if dry_run:
|
||||
report["records"] = [{k: v for k, v in s.items() if k != "product_id"} for s in sample]
|
||||
return report
|
||||
for item in sample:
|
||||
query = provider_query(item["gtin_key"])
|
||||
outcome = gateway.lookup(provider, query, "BACKFILL", reserve_percent)
|
||||
while outcome.state == "SKIPPED" and outcome.reason == "RATE_WINDOW" and wait_rate_window: # per-minute governor: wait for the window, never exceed it
|
||||
time.sleep(max(1, min(65, int((outcome.retry_at - _now()).total_seconds()) + 1)) if outcome.retry_at else 12)
|
||||
outcome = gateway.lookup(provider, query, "BACKFILL", reserve_percent)
|
||||
if outcome.state == "SKIPPED":
|
||||
report["stopped"] = outcome.reason # quota / reserve / per-minute window: stop, never insist
|
||||
break
|
||||
if not outcome.from_cache:
|
||||
report["requests"] += 1
|
||||
report["records"].append(apply_outcome(session, item["gtin_key"], provider.name, outcome, is_official))
|
||||
session.commit()
|
||||
return report
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Post-NCM fiscal completion: wires the EXISTING, previously-dormant `central.app.field_contract.resolve_product` (the real field-by-field fiscal
|
||||
contract engine - CFOP, CST, CSOSN, PIS/COFINS, CST_IPI, SITUACAO_TRIBUTARIA, IAT, IPPT, ALIQUOTA...) into the live Product Enrichment resolve call.
|
||||
|
||||
No second fiscal engine. Everything here only ASSEMBLES the same inputs `resolve_product` already expects (`company`, `refs`, `ev`, `ctx`) from data
|
||||
that already exists on the Hub, and maps its per-field CHANGE/ALREADY/NOT_APPLICABLE/MANUAL output onto the enrichment's own AUTO_FILL_SAFE vocabulary
|
||||
so the SAME typed-write Agent pipeline can apply it. `company` comes straight from `Installation.company_uf/company_crt` (auto-discovered from the
|
||||
Agent's own heartbeat - zero guessing). `ctx` comes straight from `FiscalContextEntry` (the SAME table the existing blocker-resolution workflow
|
||||
already writes to when an operator answers a question) - so a missing key surfaces as an ordinary blocker the operator resolves the usual way; no
|
||||
new configuration UI is created. `ev` (purchase/recipe/stock evidence) is not available from a single ad-hoc product resolve; those fields fall back
|
||||
to MANUAL exactly as `field_contract` already does when evidence is absent - never a guess.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..classification import normalize_code
|
||||
from ..datasets.service import DatasetService
|
||||
from ..models import Dataset, FiscalContextEntry
|
||||
from .. import field_contract as fc
|
||||
|
||||
# Fields this module MAY auto-fill. Deliberately excludes CLASSIFICACAO_FISCAL/CEST/IPI (owned by the enrichment resolver itself),
|
||||
# COD_ORIGEM_MERCADORIA (no evidence source exists per the contract - never inferred from CST), EX_TIPI (interacts with the IPI EX choice,
|
||||
# left for a dedicated pass) and CCLASSTRIB (RTC's own domain, untouched).
|
||||
FISCAL_COMPLETION_FIELDS = ("SITUACAO_TRIBUTARIA", "CFOP", "CFOP_ENTRADA", "CST", "CST_ENTRADA", "CSOSN", "ALIQUOTA", "CST_IPI",
|
||||
"COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS",
|
||||
"INCIDENCIA_PISCOFINS", "PISCOFINS", "IAT", "IPPT")
|
||||
|
||||
# Value shape per field (validated BOTH here, before proposing a write, and again by the Agent right before the typed write).
|
||||
DIGIT_FIELDS = {"CFOP": 4, "CFOP_ENTRADA": 4, "CST": 3, "CST_ENTRADA": 3, "CSOSN": 3, "CST_IPI": 3, "COD_ENTRADA_PIS": 2, "COD_SAIDA_PIS": 2,
|
||||
"COD_ENTRADA_COFINS": 2, "COD_SAIDA_COFINS": 2}
|
||||
NUMERIC_FIELDS = ("ALIQUOTA",)
|
||||
DECIMAL_TEXT_FIELDS = ("ALIQ_PIS", "ALIQ_COFINS")
|
||||
ENUM_FIELDS = {"SITUACAO_TRIBUTARIA": {"T", "I", "N", "S"}, "IAT": {"A", "T"}, "IPPT": {"P", "T"}, "PISCOFINS": {"N", "S"},
|
||||
"INCIDENCIA_PISCOFINS": {"NENHUMA", "MONOFASICA", "SUBST.TRIBUT."}}
|
||||
|
||||
|
||||
def is_writable_shape(field: str, value) -> bool:
|
||||
text = "" if value is None else str(value).strip()
|
||||
if not text:
|
||||
return False
|
||||
if field in DIGIT_FIELDS:
|
||||
digits = "".join(c for c in text if c.isdigit())
|
||||
return digits == text and len(digits) in (DIGIT_FIELDS[field], DIGIT_FIELDS[field] + 1) # CST/CSOSN family sometimes carries an origin digit prefix
|
||||
if field in NUMERIC_FIELDS or field in DECIMAL_TEXT_FIELDS:
|
||||
try:
|
||||
float(text.replace(",", "."))
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
if field in ENUM_FIELDS:
|
||||
return text.upper() in ENUM_FIELDS[field]
|
||||
return False
|
||||
|
||||
|
||||
def build_company(installation) -> dict:
|
||||
"""Auto-discovered, zero guessing: the Agent already reports these at every heartbeat (company_contribuinte_ipi from C000004.CONTRIBUINTE_IPI,
|
||||
the same row/query as company_crt - see company_context.py on the Agent). Omits "contribuinte_ipi" entirely when the Agent hasn't reported it
|
||||
yet (older Agent, or the field just never resolved): resolve_fiscal_fields treats that as genuinely unknown, never as a proven "N"."""
|
||||
company = {"uf": (installation.company_uf or "").upper() or None, "crt": installation.company_crt or None}
|
||||
if installation.company_contribuinte_ipi in ("S", "N"):
|
||||
company["contribuinte_ipi"] = installation.company_contribuinte_ipi
|
||||
return company
|
||||
|
||||
|
||||
def build_refs(session, *, uf: str | None, cfop_candidates: set[str]) -> dict:
|
||||
ds = DatasetService(session)
|
||||
refs = {key: {r.code for r in ds.fiscal_codes(domain)} for key, domain in
|
||||
(("csosn", "CSOSN"), ("cst_icms", "CST_ICMS"), ("cst_pis", "CST_PIS"), ("cst_cofins", "CST_COFINS"), ("origem", "ORIGEM_MERCADORIA"))}
|
||||
refs["cfop_valid"] = {c for c in cfop_candidates if ds.cfop_lookup(code=c)}
|
||||
refs["fcp_uf"] = {}
|
||||
if uf:
|
||||
fcp = ds.fcp_for_uf(uf)
|
||||
if fcp:
|
||||
refs["fcp_uf"][uf] = {"aliq_1": fcp.aliq_1, "observacao": fcp.observacao}
|
||||
refs["cbs_rate_2026"] = (ds.rtc_cbs_rate(2026).rate if ds.rtc_cbs_rate(2026) else None)
|
||||
refs["cclasstrib_count"] = session.scalar(select(Dataset.record_count).where(Dataset.kind == "RTC_CCLASSTRIB", Dataset.status == "ACTIVE")) or 0
|
||||
return refs
|
||||
|
||||
|
||||
def build_context(session, tenant_id: int, company_key: str) -> fc.Context:
|
||||
"""SAME table the existing blocker-resolution workflow (`central/app/blocker_resolution.py`) already writes operator answers into."""
|
||||
rows = session.scalars(select(FiscalContextEntry).where(FiscalContextEntry.tenant_id == tenant_id, FiscalContextEntry.company_key == company_key)).all()
|
||||
return fc.Context([{"scope": r.scope, "scope_key": r.scope_key, "key": r.key, "value": r.value} for r in rows])
|
||||
|
||||
|
||||
def resolve_fiscal_fields(session, *, product_code: str, description: str, current: dict, ncm: str, ncm_official: bool, installation, tenant_id: int,
|
||||
extra_columns: dict | None = None, decision: dict | None = None, ev: dict | None = None) -> dict:
|
||||
"""Returns {field: {status, value, rule, reason}} for FISCAL_COMPLETION_FIELDS, in the enrichment's own vocabulary (AUTO_FILL_SAFE / ALREADY_SET /
|
||||
NOT_APPLICABLE / NEEDS_REVIEW / INSUFFICIENT_EVIDENCE). Never called for a product whose NCM is not yet an official, vigente code.
|
||||
|
||||
`decision` is the SAME dict `classification.Classifier.classify(...)` already returns for CEST/IPI (resolver.py's `_fiscal_decisions`) - reused
|
||||
as-is (its `classification.ipi_candidates`/`cest_candidates`/`ba_st_mva` keys are exactly what `resolve_product` reads), never recomputed here."""
|
||||
if not ncm_official:
|
||||
return {f: {"status": "INSUFFICIENT_EVIDENCE", "value": None, "rule": None, "reason": "depende de um NCM oficial vigente"} for f in FISCAL_COMPLETION_FIELDS}
|
||||
ncm = normalize_code(ncm)
|
||||
p = {"CODIGO": product_code, "PRODUTO": description, "CODGRUPO": None, "TIPO": None, "PROD_COZINHA": None, "PRODUTO_SERVICO": None, "CLASSIFICACAO_FISCAL": ncm}
|
||||
p.update({k: v for k, v in (extra_columns or {}).items() if k in ("CODGRUPO", "TIPO", "PROD_COZINHA", "PRODUTO_SERVICO")})
|
||||
if str(p.get("PROD_COZINHA") or "").strip().isdigit(): # the Agent sends `current` as JSON strings (e.g. "1"); infer_role compares the real Firebird SMALLINT (== 1)
|
||||
p["PROD_COZINHA"] = int(p["PROD_COZINHA"])
|
||||
for f in fc.WRITE_FIELDS:
|
||||
p.setdefault(f, current.get(f))
|
||||
company = build_company(installation)
|
||||
company_key = (installation.company_code or installation.company_name or installation.customer_code or "").strip()
|
||||
ctx = build_context(session, tenant_id, company_key)
|
||||
if "contribuinte_ipi" not in company: # ERP fact (heartbeat) always wins; the operator answer is only a fallback for an Agent that hasn't reported it yet
|
||||
contribuinte_ipi, _ = ctx.get("contribuinte_ipi")
|
||||
if contribuinte_ipi in ("S", "N"):
|
||||
company["contribuinte_ipi"] = contribuinte_ipi
|
||||
cfop_candidates = {str(current.get(f)).strip() for f in ("CFOP", "CFOP_ENTRADA") if current.get(f)}
|
||||
refs = build_refs(session, uf=company.get("uf"), cfop_candidates=cfop_candidates)
|
||||
# Agent-provided, product_code-targeted evidence (recipes/composed/purchases/stock - see firebird_reader.read_product_evidence): a real,
|
||||
# indexed per-product query, never a full-table scan or a guess. Falls back to "no evidence" only when the Agent hasn't sent it yet.
|
||||
ev = ev if isinstance(ev, dict) else {"recipes": set(), "composed": set(), "composed_ratio": {}, "purchases": {}, "stock_entry": {}, "stock_in": {}}
|
||||
d = decision or {"status": "EXACT_NCM", "cest_status": "NO_CEST_MAPPING_FOUND", "ba_st_mva_status": None, "classification": {"cest_candidates": [], "ipi_candidates": [], "ba_st_mva": {}}}
|
||||
raw = fc.resolve_product(p, d, company, refs, ev, ctx)
|
||||
out = {}
|
||||
for field in FISCAL_COMPLETION_FIELDS:
|
||||
r = raw.get(field) or {}
|
||||
state = r.get("state")
|
||||
if state == fc.ALREADY:
|
||||
out[field] = {"status": "ALREADY_SET" if not _blank(current.get(field)) else "NOT_APPLICABLE", "value": None, "rule": r.get("reason"), "reason": r.get("reason")}
|
||||
elif state == fc.NA and field == "CST_IPI" and "contribuinte_ipi" not in company:
|
||||
# field_contract's own NA branch for CST_IPI fires whenever company.get("contribuinte_ipi") != "S" - which is also true when the fact is
|
||||
# simply UNKNOWN. Never let an unknown fact read as a proven "not an IPI taxpayer": surface it as a missing answer instead.
|
||||
out[field] = {"status": "NEEDS_REVIEW", "value": None, "rule": None, "reason": "contribuinte de IPI não confirmado", "missing_key": "contribuinte_ipi", "missing_scope": "COMPANY"}
|
||||
elif state == fc.NA:
|
||||
out[field] = {"status": "NOT_APPLICABLE", "value": None, "rule": r.get("reason"), "reason": r.get("reason")}
|
||||
elif state == fc.CHANGE and _blank(current.get(field)) and is_writable_shape(field, r.get("proposed")):
|
||||
out[field] = {"status": "AUTO_FILL_SAFE", "value": str(r.get("proposed")), "rule": r.get("reason"), "reason": None}
|
||||
elif state == fc.CHANGE:
|
||||
out[field] = {"status": "NEEDS_REVIEW", "value": None, "rule": r.get("reason"), "reason": "valor proposto não seguro para escrita automática" if not _blank(current.get(field)) else "formato do valor proposto não confere"}
|
||||
else: # MANUAL
|
||||
out[field] = {"status": "NEEDS_REVIEW", "value": None, "rule": None, "reason": r.get("reason"), "missing_key": r.get("input_key"), "missing_scope": r.get("input_scope")}
|
||||
return out
|
||||
|
||||
|
||||
def _blank(v) -> bool:
|
||||
return v is None or str(v).strip() == ""
|
||||
|
||||
|
||||
def coverage_status(field_statuses: dict) -> str:
|
||||
"""COMPLETE when every field is filled/already-correct/not-applicable; SAFE_BLOCKED when at least one relevant field still needs a decision."""
|
||||
return "SAFE_BLOCKED" if any(v["status"] in ("NEEDS_REVIEW", "INSUFFICIENT_EVIDENCE") for v in field_statuses.values()) else "COMPLETE"
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Deterministic GTIN normalization and validation (GTIN-8, UPC/GTIN-12, EAN/GTIN-13, GTIN-14)."""
|
||||
import re
|
||||
|
||||
VALID, INVALID, MISSING, RESTRICTED = "VALID_GTIN", "INVALID_GTIN", "MISSING_GTIN", "RESTRICTED_GTIN"
|
||||
|
||||
|
||||
def check_digit_ok(digits: str) -> bool:
|
||||
body, check = digits[:-1], int(digits[-1])
|
||||
total = sum(int(d) * (3 if i % 2 == 0 else 1) for i, d in enumerate(reversed(body)))
|
||||
return (10 - total % 10) % 10 == check
|
||||
|
||||
|
||||
def normalize_gtin(raw) -> dict:
|
||||
"""Returns {status, gtin (digits as typed), key (zero-padded to 14: the KB identity), reason}. Never calls anything external.
|
||||
RESTRICTED: EAN prefixes 02, 04 and 20-29 are store-internal / restricted circulation (not a global product identity), so no provider is asked."""
|
||||
text = "" if raw is None else str(raw).strip()
|
||||
if not text or text.upper() in ("SEM GTIN", "SEM EAN", "0", "NULL"):
|
||||
return {"status": MISSING, "gtin": None, "key": None, "reason": "EMPTY"}
|
||||
digits = re.sub(r"\D", "", text)
|
||||
if not digits or len(digits) not in (8, 12, 13, 14):
|
||||
return {"status": INVALID, "gtin": digits or None, "key": None, "reason": "BAD_LENGTH"}
|
||||
if len(set(digits)) == 1:
|
||||
return {"status": INVALID, "gtin": digits, "key": None, "reason": "REPEATED_DIGITS"}
|
||||
if not check_digit_ok(digits):
|
||||
return {"status": INVALID, "gtin": digits, "key": None, "reason": "BAD_CHECK_DIGIT"}
|
||||
key = digits.zfill(14)
|
||||
prefix = key[1:3] if len(digits) == 14 else key[1:3] # key = 0 + EAN-13 layout for 12/13; GTIN-14 keeps its indicator digit in key[0]
|
||||
ean13 = key[1:]
|
||||
if ean13[:2] in ("02", "04") or "20" <= ean13[:2] <= "29":
|
||||
return {"status": RESTRICTED, "gtin": digits, "key": key, "reason": "STORE_INTERNAL_PREFIX"}
|
||||
return {"status": VALID, "gtin": digits, "key": key, "reason": None}
|
||||
@@ -0,0 +1,196 @@
|
||||
"""Historical ERP evidence (product master + NFC-e/NF-e items) -> central KB evidence. Local data only: NO provider is contacted here.
|
||||
|
||||
Evidence model
|
||||
- One 'lineage' per client per (GTIN, value): master-data flag, fiscal-document count and documents per year. Master data and the fiscal documents of the SAME ERP are one family (CLIENT_DATA):
|
||||
the documents were issued from that very master, so they are never counted as two independent sources.
|
||||
- CLIENT_DATA alone can reach CORROBORATED (>= 20 documents over >= 2 years, or >= 2 distinct clients) but never VERIFIED_KB: that needs an independent NCM-bearing source (kb.evaluate).
|
||||
- NCM validity is judged against the ACTIVE official dataset today: CURRENT_VALID / HISTORICAL_EXPIRED / HISTORICAL_INVALID; a different NCM at least as recent -> HISTORICAL_CHANGED; two current values -> CONFLICT.
|
||||
- Company-specific tax data (CRT, CFOP, CSOSN, ...) is never read nor stored; only GTIN, descriptions/aliases, NCM, CEST and the source tenant label.
|
||||
Idempotent: a client's lineage for a (GTIN, value) is REPLACED on every import, never accumulated.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections import defaultdict
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..models import KbKnowledge, KbProduct
|
||||
from . import kb, textnorm
|
||||
from .gtin import normalize_gtin
|
||||
|
||||
|
||||
def _digits(v) -> str:
|
||||
return re.sub(r"\D", "", str(v or ""))
|
||||
|
||||
|
||||
def build_observations(stagings: list[dict]) -> dict:
|
||||
"""stagings: miner output per client. Returns {"gtins": {key: {...}}, "stats": {...}}."""
|
||||
gtins: dict[str, dict] = {}
|
||||
stats = defaultdict(int)
|
||||
for st in stagings:
|
||||
client = st["client"]
|
||||
product_gtin: dict[str, set] = defaultdict(set)
|
||||
master_gtin: dict[str, str] = {}
|
||||
master_desc: dict[str, str] = {}
|
||||
for m in st["master"]:
|
||||
stats["master_rows"] += 1
|
||||
master_desc[m["code"]] = m["desc"]
|
||||
for raw in (m["gtin"], m["codbarra"]):
|
||||
g = normalize_gtin(raw)
|
||||
if g["status"] == "VALID_GTIN":
|
||||
master_gtin.setdefault(m["code"], g["key"])
|
||||
break
|
||||
# product<->EAN links seen in documents/movements are used ONLY when the master has none, and only when unambiguous
|
||||
seen: dict[str, set] = defaultdict(set)
|
||||
for row in st.get("nfe", []):
|
||||
g = normalize_gtin(row["codbarra"])
|
||||
if g["status"] == "VALID_GTIN":
|
||||
seen[row["code"]].add(g["key"])
|
||||
for row in st.get("barcodes", []):
|
||||
g = normalize_gtin(row["codbarra"])
|
||||
if g["status"] == "VALID_GTIN":
|
||||
seen[row["code"]].add(g["key"])
|
||||
link: dict[str, str] = dict(master_gtin)
|
||||
for code, keys in seen.items():
|
||||
if code in link:
|
||||
continue
|
||||
if len(keys) == 1:
|
||||
link[code] = next(iter(keys))
|
||||
stats["links_from_documents"] += 1
|
||||
else:
|
||||
stats["ambiguous_product_links_skipped"] += 1
|
||||
|
||||
def slot(key):
|
||||
return gtins.setdefault(key, {"aliases": set(), "ncm": defaultdict(lambda: defaultdict(lambda: {"master": 0, "docs": 0, "years": {}})), "cest": defaultdict(lambda: defaultdict(lambda: {"master": 0, "docs": 0, "years": {}})), "clients": set()})
|
||||
for m in st["master"]:
|
||||
key = link.get(m["code"])
|
||||
if not key:
|
||||
continue
|
||||
s = slot(key)
|
||||
s["clients"].add(client)
|
||||
if m["desc"]:
|
||||
s["aliases"].add(textnorm.normalized(m["desc"]))
|
||||
n = _digits(m["ncm"])
|
||||
if len(n) == 8:
|
||||
s["ncm"][n][client]["master"] = 1
|
||||
c = _digits(m["cest"])
|
||||
if len(c) == 7:
|
||||
s["cest"][c][client]["master"] = 1
|
||||
for row in st.get("nfce", []):
|
||||
key = link.get(row["code"])
|
||||
stats["nfce_groups"] += 1
|
||||
if not key:
|
||||
stats["nfce_groups_without_gtin"] += 1
|
||||
continue
|
||||
s = slot(key)
|
||||
n = _digits(row["ncm"])
|
||||
if len(n) == 8:
|
||||
lin = s["ncm"][n][client]
|
||||
lin["docs"] += row["n"]
|
||||
if row["year"]:
|
||||
lin["years"][str(row["year"])] = lin["years"].get(str(row["year"]), 0) + row["n"]
|
||||
stats["nfce_documents"] += row["n"]
|
||||
c = _digits(row["cest"])
|
||||
if len(c) == 7:
|
||||
lin = s["cest"][c][client]
|
||||
lin["docs"] += row["n"]
|
||||
if row["year"]:
|
||||
lin["years"][str(row["year"])] = lin["years"].get(str(row["year"]), 0) + row["n"]
|
||||
for row in st.get("nfe", []):
|
||||
key = link.get(row["code"])
|
||||
if not key:
|
||||
continue
|
||||
s = slot(key)
|
||||
if row["desc"]:
|
||||
s["aliases"].add(textnorm.normalized(row["desc"]))
|
||||
n = _digits(row["ncm"])
|
||||
if len(n) == 8:
|
||||
s["ncm"][n][client]["docs"] += row["n"]
|
||||
stats["nfe_documents"] += row["n"]
|
||||
stats["gtins"] = len(gtins)
|
||||
return {"gtins": gtins, "stats": dict(stats)}
|
||||
|
||||
|
||||
def apply(session, observations: dict, is_official, *, dry_run: bool = True, chunk: int = 500) -> dict:
|
||||
"""Bulk, idempotent import into kb_products / kb_knowledge. dry_run=True computes every status and rolls back."""
|
||||
keys = sorted(observations["gtins"])
|
||||
report = {"GTINS_FOUND": len(keys), "OBSERVATIONS": 0, "by_status": defaultdict(int), "CONFLICTS": 0, "WOULD_VERIFY": 0, "with_current_valid_ncm": 0, "stable_across_years": 0, "changed_over_time": 0,
|
||||
"with_historical_ncm": 0, "multi_client": 0}
|
||||
nested = session.begin_nested()
|
||||
try:
|
||||
for start in range(0, len(keys), chunk):
|
||||
part = keys[start:start + chunk]
|
||||
existing = {p.gtin: p for p in session.scalars(select(KbProduct).where(KbProduct.gtin.in_(part)))}
|
||||
for key in part:
|
||||
obs = observations["gtins"][key]
|
||||
product = existing.get(key)
|
||||
if product is None:
|
||||
alias = next(iter(sorted(obs["aliases"])), "")
|
||||
product = kb.upsert_product(session, key, alias)
|
||||
existing[key] = product
|
||||
for a in sorted(obs["aliases"])[:20]:
|
||||
kb.add_alias(session, product, a)
|
||||
fp = textnorm.fingerprint(a)
|
||||
if fp:
|
||||
kb.add_evidence(session, product, "name_fingerprint", fp, "IDENTITY", family="IDENTITY", status=kb.OBSERVED)
|
||||
for field, values in (("ncm", obs["ncm"]), ("cest", obs["cest"])):
|
||||
for value, per_client in values.items():
|
||||
for client, lin in per_client.items():
|
||||
kb.add_evidence(session, product, field, value, "client:history", family=kb.CLIENT_FAMILY, client_label=client, lineage=None)
|
||||
row = session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == field, KbKnowledge.value == value))
|
||||
ev = dict(row.evidence or {})
|
||||
lineages = dict(ev.get("lineages") or {})
|
||||
lineages[client] = {"master": lin["master"], "docs": lin["docs"], "years": dict(lin["years"])} # REPLACE: idempotent
|
||||
row.evidence = {**ev, "lineages": lineages}
|
||||
report["OBSERVATIONS"] += 1
|
||||
session.flush()
|
||||
for key in part:
|
||||
product = existing[key]
|
||||
ev = kb.evaluate(session, product, "ncm", is_official)
|
||||
report["by_status"][ev["status"]] += 1
|
||||
if ev["status"] == kb.CONFLICT:
|
||||
report["CONFLICTS"] += 1
|
||||
if ev["status"] == kb.VERIFIED_KB:
|
||||
report["WOULD_VERIFY"] += 1
|
||||
if ev["value"]:
|
||||
report["with_current_valid_ncm"] += 1
|
||||
if observations["gtins"][key]["ncm"]:
|
||||
report["with_historical_ncm"] += 1
|
||||
if len(observations["gtins"][key]["clients"]) >= 2:
|
||||
report["multi_client"] += 1
|
||||
rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm")))
|
||||
states = {(r.evidence.get("info") or {}).get("historical_state") for r in rows}
|
||||
if "HISTORICAL_STABLE_CURRENT" in states and ev["value"]:
|
||||
years = set()
|
||||
for r in rows:
|
||||
for lin in (r.evidence.get("lineages") or {}).values():
|
||||
years |= set(lin.get("years") or {})
|
||||
if len(years) >= 2:
|
||||
report["stable_across_years"] += 1
|
||||
if "HISTORICAL_CHANGED" in states or ("HISTORICAL_EXPIRED" in states and ev["value"]):
|
||||
report["changed_over_time"] += 1
|
||||
if dry_run:
|
||||
nested.rollback()
|
||||
else:
|
||||
nested.commit()
|
||||
session.commit()
|
||||
except Exception:
|
||||
nested.rollback()
|
||||
raise
|
||||
report["by_status"] = dict(report["by_status"])
|
||||
return report
|
||||
|
||||
|
||||
def official_sets(session):
|
||||
"""Precomputed 'vigente today' membership: one query instead of one per value."""
|
||||
from datetime import date
|
||||
from ..models import Dataset, NcmEntry
|
||||
from .resolver import _parse_br
|
||||
today = date.today()
|
||||
current = set()
|
||||
for row in session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE")):
|
||||
start, end = _parse_br(row.valid_from), _parse_br(row.valid_to)
|
||||
if (start is None or start <= today) and (end is None or end >= today):
|
||||
current.add(row.code)
|
||||
return current
|
||||
@@ -0,0 +1,284 @@
|
||||
"""Central product Knowledge Base access (same PostgreSQL, existing kb_* tables) with per-field status.
|
||||
|
||||
Trust model (docs/product_enrichment.md):
|
||||
- A provider value or a client observation is EVIDENCE. A field becomes VERIFIED_KB only when the official dataset confirms it AND at least two INDEPENDENT evidence
|
||||
families agree with zero conflict (distinct providers each count once; client observations count once and only with >= 2 distinct clients).
|
||||
- Human edits never enter the KB (HUMAN_OVERRIDDEN is per-installation protection, not knowledge). Company-specific fiscal decisions (CSOSN, CFOP, ...) never live here.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from sqlalchemy import func
|
||||
|
||||
from ..models import KbKnowledge, KbProduct, KbPromotionAudit
|
||||
from . import textnorm
|
||||
|
||||
UNVERIFIED, OBSERVED, CORROBORATED, EXTERNAL_CORROBORATED, VERIFIED_KB, CONFLICT, STALE = "UNVERIFIED", "OBSERVED", "CORROBORATED", "EXTERNAL_CORROBORATED", "VERIFIED_KB", "CONFLICT", "STALE"
|
||||
EVIDENCE_ONLY = OBSERVED # legacy name
|
||||
CLIENT_FAMILY = "CLIENT_DATA" # every client-derived lineage (master data + fiscal documents of the same ERP are NOT independent of each other)
|
||||
RULE_VERSION = "kb-promotion-v2"
|
||||
STRONG_DOCS, STRONG_YEARS = 20, 2
|
||||
|
||||
|
||||
def _now():
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def gtin_fingerprint(key: str) -> str:
|
||||
return hashlib.sha256(f"gtin:{key}".encode()).hexdigest()
|
||||
|
||||
|
||||
def find_by_gtin(session, key: str | None) -> KbProduct | None:
|
||||
return session.scalar(select(KbProduct).where(KbProduct.gtin == key)) if key else None
|
||||
|
||||
|
||||
NEUTRAL_EXTRAS = frozenset({"REFRIGERANTE", "REFRIG", "REFR", "BEBIDA", "PRODUTO", "PROD"}) # category nouns only: ZERO / DIET / a flavour / another size are NEVER neutral
|
||||
|
||||
|
||||
def _tokset(text) -> frozenset:
|
||||
return frozenset(t for t in textnorm.tokens(text) if t not in textnorm.MARKETING)
|
||||
|
||||
|
||||
def alias_match(query: frozenset, alias: frozenset) -> bool:
|
||||
"""Deterministic, conservative: same token set, or one set contains the other and the difference is only neutral category nouns (so "COCA COLA LATA 350 ML" = "REFRIGERANTE COCA COLA LATA 350 ML", but not "... ZERO ...")."""
|
||||
if not query or not alias:
|
||||
return False
|
||||
if query == alias:
|
||||
return True
|
||||
if query < alias:
|
||||
return (alias - query) <= NEUTRAL_EXTRAS
|
||||
if alias < query:
|
||||
return (query - alias) <= NEUTRAL_EXTRAS
|
||||
return False
|
||||
|
||||
|
||||
def name_candidates(session, description) -> list[KbProduct]:
|
||||
"""Every KB product with an alias that matches the description under `alias_match` (exact fingerprint included). 0, 1 or several - the caller decides (unique VERIFIED only)."""
|
||||
fp = textnorm.fingerprint(description)
|
||||
if not fp:
|
||||
return []
|
||||
ids = set(session.scalars(select(KbKnowledge.kb_product_id).where(KbKnowledge.knowledge_key == "name_fingerprint", KbKnowledge.value == fp).distinct()))
|
||||
query = _tokset(description)
|
||||
anchor = max((t for t in query if not t.replace(".", "").isdigit() and len(t) > 2), key=len, default=None) # most selective informative token, then verify in Python
|
||||
if anchor:
|
||||
for row in session.execute(select(KbKnowledge.kb_product_id, KbKnowledge.value).where(KbKnowledge.knowledge_key == "alias", KbKnowledge.value.contains(anchor))):
|
||||
if alias_match(query, _tokset(row[1])):
|
||||
ids.add(row[0])
|
||||
return [p for p in (session.get(KbProduct, i) for i in sorted(ids)) if p]
|
||||
|
||||
|
||||
def upsert_product(session, key: str | None, description, *, brand=None, quantity=None) -> KbProduct:
|
||||
if key:
|
||||
from ..gtin_validator import validate_gtin
|
||||
valid, _ = validate_gtin(key)
|
||||
if not valid:
|
||||
key = None # Regra C.6: GTIN inválido (DV ou prefixo errado) não entra na KB
|
||||
product = find_by_gtin(session, key)
|
||||
if product is None:
|
||||
fp = gtin_fingerprint(key) if key else hashlib.sha256(f"name:{textnorm.fingerprint(description)}".encode()).hexdigest()
|
||||
product = session.scalar(select(KbProduct).where(KbProduct.fingerprint == fp))
|
||||
if product is None:
|
||||
product = KbProduct(fingerprint=fp, gtin=key, identity_kind="GTIN" if key else "FINGERPRINT", normalized_description=textnorm.normalized(description))
|
||||
session.add(product)
|
||||
session.flush()
|
||||
if description and not product.normalized_description:
|
||||
product.normalized_description = textnorm.normalized(description)
|
||||
if brand and not product.brand:
|
||||
product.brand = brand
|
||||
if quantity and not product.package_quantity:
|
||||
product.package_quantity = str(quantity)[:32]
|
||||
fp = textnorm.fingerprint(description)
|
||||
norm = textnorm.normalized(description)
|
||||
if norm:
|
||||
add_alias(session, product, norm)
|
||||
if fp:
|
||||
add_evidence(session, product, "name_fingerprint", fp, "IDENTITY", family="IDENTITY", status=OBSERVED)
|
||||
return product
|
||||
|
||||
|
||||
def add_alias(session, product: KbProduct, normalized: str, cap: int = 20):
|
||||
"""Non-fiscal aliases of the same GTIN (descriptions seen over the years). Helps name matching; never authority for a fiscal write."""
|
||||
if not normalized or session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "alias", KbKnowledge.value == normalized[:200])):
|
||||
return
|
||||
if (session.scalar(select(func.count()).select_from(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "alias")) or 0) >= cap:
|
||||
return
|
||||
session.add(KbKnowledge(kb_product_id=product.id, knowledge_key="alias", value=normalized[:200], source="ALIAS", status=OBSERVED, evidence={"families": {}, "clients": [], "info": {}}))
|
||||
|
||||
|
||||
def add_evidence(session, product: KbProduct, key: str, value: str, source: str, *, family: str, client_label: str | None = None, status: str = OBSERVED, info: dict | None = None,
|
||||
lineage: dict | None = None) -> KbKnowledge:
|
||||
"""lineage (client data only): {"master": 0/1, "docs": n, "years": {"2024": n}} of ONE client for this (GTIN, value); merged, never double counted."""
|
||||
row = session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == key, KbKnowledge.value == value))
|
||||
if row is None:
|
||||
row = KbKnowledge(kb_product_id=product.id, knowledge_key=key, value=value, source=source[:32], status=status, evidence={"families": {}, "clients": [], "lineages": {}, "info": info or {}})
|
||||
session.add(row)
|
||||
session.flush()
|
||||
evidence = dict(row.evidence or {})
|
||||
families = dict(evidence.get("families") or {})
|
||||
families[family] = _now().isoformat()
|
||||
clients = list(evidence.get("clients") or [])
|
||||
if client_label and client_label not in clients:
|
||||
clients.append(client_label)
|
||||
lineages = dict(evidence.get("lineages") or {})
|
||||
if client_label and lineage:
|
||||
cur = dict(lineages.get(client_label) or {"master": 0, "docs": 0, "years": {}})
|
||||
cur["master"] = max(int(cur.get("master", 0)), int(lineage.get("master", 0)))
|
||||
cur["docs"] = int(cur.get("docs", 0)) + int(lineage.get("docs", 0))
|
||||
years = dict(cur.get("years") or {})
|
||||
for y, n in (lineage.get("years") or {}).items():
|
||||
years[str(y)] = int(years.get(str(y), 0)) + int(n)
|
||||
cur["years"] = years
|
||||
lineages[client_label] = cur
|
||||
row.evidence = {**evidence, "families": families, "clients": clients, "lineages": lineages, "info": {**(evidence.get("info") or {}), **(info or {})}}
|
||||
return row
|
||||
|
||||
|
||||
def _lineage_summary(rows):
|
||||
clients, docs, years = set(), 0, {}
|
||||
for r in rows:
|
||||
ev = r.evidence or {}
|
||||
for c, lin in (ev.get("lineages") or {}).items():
|
||||
clients.add(c)
|
||||
docs += int(lin.get("docs", 0))
|
||||
for y, n in (lin.get("years") or {}).items():
|
||||
years[y] = years.get(y, 0) + int(n)
|
||||
for c in ev.get("clients") or []:
|
||||
clients.add(c)
|
||||
return {"clients": sorted(clients), "docs": docs, "years": sorted(years), "last_year": max(int(y) for y in years) if years else None}
|
||||
|
||||
|
||||
def independent_count(rows) -> tuple[int, list[str]]:
|
||||
"""Independent families: each NCM-bearing provider = 1; ALL client-derived data = 1 and only when STRONG (>= STRONG_DOCS documents over >= STRONG_YEARS years, or >= 2 distinct clients)."""
|
||||
names = sorted({f for r in rows for f in ((r.evidence or {}).get("families") or {}) if f.startswith("PROVIDER:")})
|
||||
summary = _lineage_summary(rows)
|
||||
has_client = any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in rows)
|
||||
if has_client and (len(summary["clients"]) >= 2 or (summary["docs"] >= STRONG_DOCS and len(summary["years"]) >= STRONG_YEARS)):
|
||||
names.append(CLIENT_FAMILY)
|
||||
return len(names), names
|
||||
|
||||
|
||||
def _official_period(session, value):
|
||||
"""CURRENT_VALID / HISTORICAL_EXPIRED (exists in the dataset but not vigente today) / HISTORICAL_INVALID (never in the dataset)."""
|
||||
from .resolver import ncm_exists_any
|
||||
return "CURRENT_VALID" if ncm_exists_any(session, value, current=True) else ("HISTORICAL_EXPIRED" if ncm_exists_any(session, value, current=False) else "HISTORICAL_INVALID")
|
||||
|
||||
|
||||
def _mark(row, state):
|
||||
row.evidence = {**(row.evidence or {}), "info": {**((row.evidence or {}).get("info") or {}), "historical_state": state}}
|
||||
|
||||
|
||||
OPERATOR_FAMILY = "OPERATOR_ATTESTATION"
|
||||
|
||||
|
||||
def attest(session, gtin_key: str, ncm: str, operator: str, is_official, note: str = "") -> dict:
|
||||
"""A human fiscal decision by an ADMIN (audited by the caller). Only a value that is ALREADY a candidate in the KB (some local/provider evidence) and vigente in the official dataset can be attested;
|
||||
the dissenting candidates become STALE (REJECTED_BY_OPERATOR) and the product is promoted with the promotion audit naming the operator."""
|
||||
product = find_by_gtin(session, gtin_key)
|
||||
if product is None:
|
||||
raise ValueError("GTIN_UNKNOWN_IN_KB")
|
||||
row = session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm", KbKnowledge.value == ncm))
|
||||
if row is None:
|
||||
raise ValueError("NCM_IS_NOT_A_CANDIDATE_OF_THIS_GTIN")
|
||||
if not is_official(ncm):
|
||||
raise ValueError("NCM_NOT_VIGENTE_IN_OFFICIAL_DATASET")
|
||||
add_evidence(session, product, "ncm", ncm, f"operator:{operator}"[:32], family=OPERATOR_FAMILY, info={"attested_by": operator, "attested_at": _now().isoformat(), "note": note[:200]})
|
||||
for other in session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm", KbKnowledge.value != ncm)):
|
||||
if OPERATOR_FAMILY not in ((other.evidence or {}).get("families") or {}):
|
||||
other.status = STALE
|
||||
_mark(other, "REJECTED_BY_OPERATOR")
|
||||
return evaluate(session, product, "ncm", is_official)
|
||||
|
||||
|
||||
def evaluate(session, product: KbProduct, key: str, is_official) -> dict:
|
||||
"""Recomputes and PERSISTS the status of one field. Never 'majority wins': two current-valid values => CONFLICT."""
|
||||
rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == key)))
|
||||
if not rows:
|
||||
return {"value": None, "status": UNVERIFIED, "families": [], "candidates": [], "rejected": [], "detail": {}}
|
||||
valid = [r for r in rows if is_official(r.value)]
|
||||
rejected = [r.value for r in rows if not is_official(r.value)]
|
||||
for r in rows:
|
||||
if r not in valid:
|
||||
state = _official_period(session, r.value) if key == "ncm" else "HISTORICAL_INVALID"
|
||||
r.status = STALE if state == "HISTORICAL_EXPIRED" else OBSERVED
|
||||
_mark(r, state)
|
||||
attested = {r.value for r in valid if OPERATOR_FAMILY in ((r.evidence or {}).get("families") or {})}
|
||||
if len(attested) == 1: # a human decision resolves the candidates: the others are recorded as rejected, not "outvoted"
|
||||
winner = next(iter(attested))
|
||||
for r in valid:
|
||||
if r.value != winner:
|
||||
r.status = STALE
|
||||
_mark(r, "REJECTED_BY_OPERATOR")
|
||||
valid = [r for r in valid if r.value == winner]
|
||||
values = sorted({r.value for r in valid})
|
||||
if len(values) > 1:
|
||||
for r in valid:
|
||||
r.status = CONFLICT
|
||||
_mark(r, "HISTORICAL_CONFLICT")
|
||||
return {"value": None, "status": CONFLICT, "families": [], "candidates": values, "rejected": rejected, "detail": {"clients": _lineage_summary(valid)["clients"]}}
|
||||
if not values:
|
||||
return {"value": None, "status": OBSERVED, "families": [], "candidates": [], "rejected": rejected, "detail": {}}
|
||||
chosen = [r for r in valid if r.value == values[0]]
|
||||
old = chosen[0].status
|
||||
count, names = independent_count(chosen)
|
||||
summary = _lineage_summary(chosen)
|
||||
other_last = _lineage_summary([r for r in rows if r not in chosen])["last_year"]
|
||||
superseded = other_last is not None and summary["last_year"] is not None and other_last >= summary["last_year"] # a different (expired) NCM is at least as recent: not stable
|
||||
if superseded and CLIENT_FAMILY in names:
|
||||
names.remove(CLIENT_FAMILY)
|
||||
count -= 1
|
||||
if any(OPERATOR_FAMILY in ((r.evidence or {}).get("families") or {}) for r in chosen):
|
||||
names = sorted(set(names) | {OPERATOR_FAMILY} | ({CLIENT_FAMILY} if any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in chosen) else set()))
|
||||
count = len(names)
|
||||
providers = [n for n in names if n.startswith("PROVIDER:")]
|
||||
already = any(r.status == VERIFIED_KB and r.validated_at for r in chosen)
|
||||
has_client_data = any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in chosen)
|
||||
if count >= 2 or already:
|
||||
status = VERIFIED_KB
|
||||
elif providers and has_client_data:
|
||||
status = EXTERNAL_CORROBORATED
|
||||
elif CLIENT_FAMILY in names:
|
||||
status = CORROBORATED
|
||||
else:
|
||||
status = OBSERVED
|
||||
for r in chosen:
|
||||
r.status = status
|
||||
_mark(r, "HISTORICAL_CHANGED" if superseded else "HISTORICAL_STABLE_CURRENT")
|
||||
if status == VERIFIED_KB and not r.validated_at:
|
||||
r.validated_at = _now()
|
||||
if status == VERIFIED_KB and old != VERIFIED_KB and key == "ncm":
|
||||
from .resolver import dataset_versions
|
||||
session.add(KbPromotionAudit(kb_product_id=product.id, gtin=product.gtin, field=key, value=values[0], old_status=old, new_status=status, families=names,
|
||||
sources={"clients": summary["clients"], "documents": summary["docs"], "years": summary["years"], "providers": providers,
|
||||
"operator": next((((r.evidence or {}).get("info") or {}).get("attested_by") for r in chosen if OPERATOR_FAMILY in ((r.evidence or {}).get("families") or {})), None)},
|
||||
official_dataset=dataset_versions(session).get("NCM", {}), rule_version=RULE_VERSION))
|
||||
return {"value": values[0], "status": status, "families": names, "candidates": values, "rejected": rejected, "detail": summary}
|
||||
|
||||
|
||||
def revalidate(session, is_official) -> dict:
|
||||
"""After the OFFICIAL NCM dataset changed: never delete knowledge. Rows whose NCM is no longer vigente become STALE (STALE_REVIEW_REQUIRED, audited when they were VERIFIED_KB);
|
||||
rows still vigente keep their status (VERIFIED_KB stays valid while the NCM is vigente)."""
|
||||
from .resolver import dataset_versions
|
||||
stale = still = 0
|
||||
versions = dataset_versions(session).get("NCM", {})
|
||||
for row in session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm")):
|
||||
if is_official(row.value):
|
||||
still += 1
|
||||
if row.status == STALE:
|
||||
row.status = OBSERVED # became vigente again: needs a fresh evaluate() to regain its level
|
||||
continue
|
||||
if row.status != STALE:
|
||||
was = row.status
|
||||
row.status = STALE
|
||||
_mark(row, "HISTORICAL_EXPIRED")
|
||||
row.evidence = {**(row.evidence or {}), "info": {**((row.evidence or {}).get("info") or {}), "stale_reason": "STALE_REVIEW_REQUIRED"}}
|
||||
stale += 1
|
||||
if was == VERIFIED_KB:
|
||||
product = session.get(KbProduct, row.kb_product_id)
|
||||
session.add(KbPromotionAudit(kb_product_id=row.kb_product_id, gtin=product.gtin if product else None, field="ncm", value=row.value, old_status=was, new_status=STALE,
|
||||
families=sorted((row.evidence or {}).get("families") or {}), sources={"reason": "NCM_NO_LONGER_VIGENTE"}, official_dataset=versions, rule_version=RULE_VERSION))
|
||||
session.flush()
|
||||
return {"stale": stale, "still_vigente": still}
|
||||
@@ -0,0 +1,165 @@
|
||||
"""Read-only views for the panel: product sources status, official dataset freshness and enrichment coverage metrics. Never exposes tokens."""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from ..config import settings
|
||||
from ..datasets.updater import last_runs
|
||||
from ..models import Dataset, GtinLookupCache, ProductEnrichment, ProviderState, ProviderUsageDaily
|
||||
from .providers import build_providers
|
||||
|
||||
STATUS_LABELS = {"ONLINE": "Online", "DEGRADED": "Instável", "RATE_LIMITED": "Limite atingido", "AUTH_ERROR": "Credencial recusada", "OFFLINE": "Indisponível", "NOT_CONFIGURED": "Não configurado"}
|
||||
OFFICIAL_KINDS = (("NCM", "NCM (Siscomex)"), ("TIPI_IPI", "TIPI / IPI"), ("CEST", "CEST (CONFAZ)"), ("BA_ST_MVA", "ST/MVA Bahia"))
|
||||
|
||||
|
||||
def _aware(dt):
|
||||
return dt.replace(tzinfo=timezone.utc) if dt is not None and dt.tzinfo is None else dt
|
||||
|
||||
|
||||
def provider_sources(session, providers=None) -> list[dict]:
|
||||
providers = providers or build_providers(settings)
|
||||
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
rows = []
|
||||
for provider in providers:
|
||||
state = session.get(ProviderState, provider.name)
|
||||
usage = session.scalar(select(ProviderUsageDaily).where(ProviderUsageDaily.provider == provider.name, ProviderUsageDaily.day == today))
|
||||
requests = usage.requests if usage else 0
|
||||
lookups = (usage.cache_hits + usage.cache_misses) if usage else 0
|
||||
status = ("NOT_CONFIGURED" if not provider.configured else (state.status if state else "ONLINE"))
|
||||
rows.append({
|
||||
"provider": provider.name, "configured": bool(provider.configured), "status": status, "status_label": STATUS_LABELS.get(status, status),
|
||||
"used": requests, "limit": provider.daily_limit or (state.daily_limit if state else None), "rate_per_minute": provider.rate_per_minute,
|
||||
"remaining": state.remaining if state and state.remaining is not None else ((provider.daily_limit - requests) if provider.daily_limit else None),
|
||||
"quota_kind": state.quota_kind if state else ("LOCAL_ESTIMATE" if provider.daily_limit else "DOCUMENTED_RATE"),
|
||||
"reset_at": _aware(state.reset_at) if state and state.reset_at else None,
|
||||
"last_success_at": _aware(state.last_success_at) if state and state.last_success_at else None,
|
||||
"last_error": state.last_error if state else None, "last_error_at": _aware(state.last_error_at) if state and state.last_error_at else None,
|
||||
"latency_ms": state.last_latency_ms if state else None,
|
||||
"cache_hit_rate": round(100 * usage.cache_hits / lookups) if usage and lookups else None,
|
||||
"errors": usage.errors if usage else 0, "rate_limited": usage.rate_limited if usage else 0,
|
||||
"found_today": usage.found if usage else 0, "ncm_found_today": usage.ncm_found if usage else 0, "cache_hits": usage.cache_hits if usage else 0,
|
||||
"next_reset": _aware(state.reset_at) if state and state.reset_at else None, "identity_only": not getattr(provider, "provides_ncm", False),
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def official_sources(session) -> list[dict]:
|
||||
out = []
|
||||
for kind, label in OFFICIAL_KINDS:
|
||||
ds = session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE").order_by(Dataset.id.desc()).limit(1))
|
||||
item = {"kind": kind, "label": label, "active": ds is not None, "records": ds.record_count if ds else None, "version": ds.version_label if ds else None,
|
||||
"sha256": (ds.sha256[:12] if ds else None), "last_update": _aware(ds.downloaded_at) if ds else None, "status": "ATUALIZADA" if ds else "SEM DADOS", "error": None, "auto_refresh": kind == "NCM"}
|
||||
if kind == "NCM":
|
||||
recent = last_runs(session, "NCM", 1)
|
||||
if recent:
|
||||
last = recent[0]
|
||||
item["last_check"], item["last_run_status"] = _aware(last.finished_at or last.started_at), last.status
|
||||
if last.status == "FAILED":
|
||||
item["status"], item["error"] = "FALHA NA ÚLTIMA ATUALIZAÇÃO (dados anteriores mantidos)", last.error
|
||||
out.append(item)
|
||||
return out
|
||||
|
||||
|
||||
def coverage_metrics(session) -> dict:
|
||||
total = session.scalar(select(func.count()).select_from(ProductEnrichment)) or 0
|
||||
by = dict(session.execute(select(ProductEnrichment.status, func.count()).group_by(ProductEnrichment.status)).all())
|
||||
with_gtin = session.scalar(select(func.count()).select_from(ProductEnrichment).where(ProductEnrichment.gtin.is_not(None))) or 0
|
||||
rows = [r.decisions or {} for r in session.scalars(select(ProductEnrichment).limit(5000))]
|
||||
states = {}
|
||||
for d in rows:
|
||||
states[d.get("state")] = states.get(d.get("state"), 0) + 1
|
||||
|
||||
def rate(n):
|
||||
return round(100 * n / total, 1) if total else None
|
||||
auto_ncm = sum(1 for d in rows if ((d.get("fields") or {}).get("CLASSIFICACAO_FISCAL") or {}).get("status") == "AUTO_FILL_SAFE")
|
||||
usage = session.execute(select(func.sum(ProviderUsageDaily.requests), func.sum(ProviderUsageDaily.cache_hits), func.sum(ProviderUsageDaily.cache_misses), func.sum(ProviderUsageDaily.rate_limited))).one()
|
||||
requests, hits, misses, limited = (int(x or 0) for x in usage)
|
||||
return {"tracked": total, "GTIN_VALID_RATE": rate(with_gtin), "KB_HIT_RATE": rate(states.get("EXACT_GTIN_VERIFIED", 0) + states.get("EXACT_KB_NAME_VERIFIED", 0)),
|
||||
"EXTERNAL_LOOKUP_RATE": round(100 * requests / (requests + hits), 1) if (requests + hits) else None,
|
||||
"PRODUCT_FOUND_RATE": rate(total - states.get("NOT_FOUND", 0) - states.get("INVALID_GTIN", 0) - states.get("AMBIGUOUS", 0)),
|
||||
"NCM_AUTO_FILL_SAFE_RATE": rate(auto_ncm), "NAME_ONLY_RESOLUTION_RATE": rate(states.get("EXACT_KB_NAME_VERIFIED", 0)), "AMBIGUOUS_RATE": rate(states.get("AMBIGUOUS", 0)),
|
||||
"CONFLICT_RATE": rate(states.get("CONFLICT", 0)), "PROVIDER_QUOTA_BLOCK_RATE": round(100 * limited / requests, 1) if requests else None, "states": states, "by_status": by,
|
||||
"provider_cache_entries": session.scalar(select(func.count()).select_from(GtinLookupCache)) or 0, "provider_cache_hit_rate": round(100 * hits / (hits + misses), 1) if (hits + misses) else None}
|
||||
|
||||
|
||||
def _candidate_cards(session, products) -> list[dict]:
|
||||
from ..models import Dataset, KbKnowledge, NcmEntry
|
||||
out = []
|
||||
for prod in products:
|
||||
rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == prod.id, KbKnowledge.knowledge_key == "ncm")))
|
||||
cands = []
|
||||
for r in rows:
|
||||
ev = r.evidence or {}
|
||||
text = session.scalar(select(NcmEntry.description).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == r.value).limit(1))
|
||||
cands.append({"ncm": r.value, "official_text": text, "status": r.status, "clients": sorted(ev.get("clients", [])), "documents": sum(int(l.get("docs", 0)) for l in (ev.get("lineages") or {}).values()),
|
||||
"providers": sorted(f for f in (ev.get("families") or {}) if f.startswith("PROVIDER:")), "attested": (ev.get("info") or {}).get("attested_by")})
|
||||
status = "VERIFIED_KB" if any(c["status"] == "VERIFIED_KB" for c in cands) else ("CONFLICT" if any(c["status"] == "CONFLICT" for c in cands) else (cands[0]["status"] if cands else "OBSERVED"))
|
||||
out.append({"gtin": (prod.gtin or "")[-13:], "gtin_key": prod.gtin, "description": (prod.normalized_description or "")[:70], "status": status, "candidates": [c for c in cands if c["status"] != "STALE" or c["attested"] is None][:4]})
|
||||
return out
|
||||
|
||||
|
||||
def kb_search_ids(session, query: str) -> list[int]:
|
||||
"""GLOBAL, server-side match over the whole KB (never limited to a rendered page): GTIN exact, or every normalized query word present somewhere among the
|
||||
product's known names (any alias row OR its own normalized_description - words can come from different aliases of the same product). Deliberately lenient
|
||||
(a human exploring the KB, e.g. "COCA COLA LATA 350" without the unit, must still find "COCA-COLA ORIGINAL LATA 350 ML"); this is DIFFERENT from
|
||||
`kb.alias_match`, the strict rule that authorizes an automatic fiscal write, which stays untouched."""
|
||||
import re
|
||||
|
||||
from ..models import KbKnowledge, KbProduct
|
||||
from . import kb as kbm, textnorm
|
||||
q = (query or "").strip()
|
||||
if not q:
|
||||
return []
|
||||
digits = "".join(ch for ch in q if ch.isdigit())
|
||||
if len(digits) >= 8 and re.fullmatch(r"[\d\s.\-]+", q):
|
||||
p = kbm.find_by_gtin(session, digits.zfill(14))
|
||||
return [p.id] if p else []
|
||||
tokens = [t for t in textnorm.tokens(q) if t not in textnorm.MARKETING]
|
||||
if not tokens:
|
||||
return []
|
||||
ids: set[int] | None = None
|
||||
for token in tokens: # SQL ILIKE narrows the candidates (portable: Postgres in production, SQLite in tests); the exact whole-word
|
||||
like = f"%{token}%" # check happens in Python so "ML" never matches inside "SIMPLES", etc.
|
||||
matched = {pid for pid, val in session.execute(select(KbKnowledge.kb_product_id, KbKnowledge.value).where(KbKnowledge.knowledge_key == "alias", KbKnowledge.value.ilike(like))) if token in (val or "").split()}
|
||||
matched |= {pid for pid, val in session.execute(select(KbProduct.id, KbProduct.normalized_description).where(KbProduct.normalized_description.ilike(like))) if token in (val or "").split()}
|
||||
ids = matched if ids is None else (ids & matched)
|
||||
if not ids:
|
||||
return []
|
||||
return sorted(ids)
|
||||
|
||||
|
||||
def kb_search(session, query: str, *, page: int = 1, page_size: int = 10) -> dict:
|
||||
"""query/filter GLOBAL over the whole KB -> count -> order -> pagination (never the other way round)."""
|
||||
from ..models import KbProduct
|
||||
ids = kb_search_ids(session, query)
|
||||
total = len(ids)
|
||||
page = max(1, page)
|
||||
pages = max(1, -(-total // page_size))
|
||||
page = min(page, pages)
|
||||
page_ids = ids[(page - 1) * page_size: page * page_size]
|
||||
products = [session.get(KbProduct, i) for i in page_ids]
|
||||
return {"results": _candidate_cards(session, products), "total": total, "page": page, "pages": pages, "page_size": page_size}
|
||||
|
||||
|
||||
def kb_summary(session, query: str | None = None, page: int = 1) -> dict:
|
||||
"""Compact Knowledge Base panel: known products, statuses of the NCM knowledge, hits today, and the latest conflicts with their evidence (no dashboard of its own)."""
|
||||
from ..models import KbKnowledge, KbProduct
|
||||
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
products = session.scalar(select(func.count()).select_from(KbProduct).where(KbProduct.gtin.is_not(None))) or 0
|
||||
by = dict(session.execute(select(KbKnowledge.status, func.count(func.distinct(KbKnowledge.kb_product_id))).where(KbKnowledge.knowledge_key == "ncm").group_by(KbKnowledge.status)).all())
|
||||
hits = session.scalar(select(func.coalesce(func.sum(ProviderUsageDaily.kb_hits), 0)).where(ProviderUsageDaily.day == today)) or 0
|
||||
conflicts = []
|
||||
rows = session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.status == "CONFLICT").order_by(KbKnowledge.id.desc()).limit(60)).all()
|
||||
grouped = {}
|
||||
for r in rows:
|
||||
grouped.setdefault(r.kb_product_id, []).append(r)
|
||||
for pid, items in list(grouped.items())[:5]:
|
||||
prod = session.get(KbProduct, pid)
|
||||
conflicts.append({"gtin": (prod.gtin or "")[-13:], "description": (prod.normalized_description or "")[:60],
|
||||
"candidates": [{"ncm": i.value, "clients": sorted({c for c in (i.evidence or {}).get("clients", [])}), "documents": sum(int(l.get("docs", 0)) for l in ((i.evidence or {}).get("lineages") or {}).values()),
|
||||
"providers": sorted(f for f in ((i.evidence or {}).get("families") or {}) if f.startswith("PROVIDER:"))} for i in items]})
|
||||
return {"products": products, "verified": by.get("VERIFIED_KB", 0), "corroborated": by.get("CORROBORATED", 0) + by.get("EXTERNAL_CORROBORATED", 0), "observed": by.get("OBSERVED", 0),
|
||||
"conflicts": by.get("CONFLICT", 0), "stale": by.get("STALE", 0), "hits_today": hits, "recent_conflicts": conflicts,
|
||||
"search": kb_search(session, query, page=page) if query else {"results": [], "total": 0, "page": 1, "pages": 1, "page_size": 10}}
|
||||
@@ -0,0 +1,27 @@
|
||||
"""ERP observations -> KB evidence. A client observation is EVIDENCE of family CLIENT_OBSERVATION; it never becomes VERIFIED_KB by itself
|
||||
(needs the official dataset AND a second independent family, or >= 2 distinct clients + another family - see kb.evaluate)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from ..classification import normalize_code
|
||||
from . import kb
|
||||
from .gtin import normalize_gtin
|
||||
|
||||
|
||||
def observe(session, rows, client_label: str, is_official) -> dict:
|
||||
"""rows: iterable of {"GTIN","PRODUTO","NCM"}. Only VALID_GTIN rows with an OFFICIAL 8-digit NCM are recorded. Returns counters."""
|
||||
stats = {"rows": 0, "valid_gtin": 0, "with_official_ncm": 0, "recorded": 0}
|
||||
for row in rows:
|
||||
stats["rows"] += 1
|
||||
g = normalize_gtin(row.get("GTIN"))
|
||||
if g["status"] != "VALID_GTIN":
|
||||
continue
|
||||
stats["valid_gtin"] += 1
|
||||
ncm = normalize_code(row.get("NCM"))
|
||||
if len(ncm) != 8 or not is_official(ncm):
|
||||
continue
|
||||
stats["with_official_ncm"] += 1
|
||||
product = kb.upsert_product(session, g["key"], row.get("PRODUTO"))
|
||||
kb.add_evidence(session, product, "ncm", ncm, "client:observation", family=kb.CLIENT_FAMILY, client_label=client_label)
|
||||
stats["recorded"] += 1
|
||||
session.flush()
|
||||
return stats
|
||||
@@ -0,0 +1,27 @@
|
||||
"""TECHNICAL_PLACEHOLDER semantics for the product enrichment.
|
||||
|
||||
The SGV refuses to save a product without an NCM, so operators type a technical filler to get past the form. That filler is NOT a human fiscal decision: it means "not classified yet".
|
||||
Explicit allowlist PER FIELD (never generalised: "0", "99999999", "SEM NCM" ... are NOT placeholders until the real SGV behaviour is confirmed). The Agent carries the same table
|
||||
(agent/src/gti_fiscal_agent/enrichment.py PLACEHOLDERS); a test asserts both are identical.
|
||||
"""
|
||||
import re
|
||||
|
||||
PRODUCT_ENRICHMENT_PLACEHOLDERS = {"CLASSIFICACAO_FISCAL": frozenset({"00000000"})}
|
||||
|
||||
|
||||
def _norm(field: str, value) -> str:
|
||||
text = "" if value is None else str(value).strip()
|
||||
return re.sub(r"\D", "", text) if field == "CLASSIFICACAO_FISCAL" else text # 0000.00.00 and 00000000 are the same NCM filler
|
||||
|
||||
|
||||
def is_placeholder(field: str, value) -> bool:
|
||||
return _norm(field, value) in PRODUCT_ENRICHMENT_PLACEHOLDERS.get(field, frozenset())
|
||||
|
||||
|
||||
def is_blank(value) -> bool:
|
||||
return not ("" if value is None else str(value)).strip()
|
||||
|
||||
|
||||
def is_writable_empty(field: str, value) -> bool:
|
||||
"""True when the field is empty OR holds the approved technical placeholder. A real value (even a wrong one) is never writable here."""
|
||||
return is_blank(value) or is_placeholder(field, value)
|
||||
@@ -0,0 +1,338 @@
|
||||
"""Free product-identity providers behind the Hub. Agents never talk to these; the Hub does, through the quota manager.
|
||||
|
||||
Documentation snapshots consulted 2026-09-25 (see docs/product_enrichment.md):
|
||||
- UPCitemdb Explorer/trial: https://api.upcitemdb.com/prod/trial/lookup?upc=... - no key, 100 combined requests/day, burst 15 lookup / 5 search per 30 s,
|
||||
headers X-RateLimit-Limit / -Remaining / -Reset. Product IDENTITY only (title/brand/category); no fiscal data.
|
||||
- Open Food Facts: https://world.openfoodfacts.org/api/v2/product/{barcode} - 15 read req/min/IP, 10 search req/min/IP, identifiable User-Agent, ODbL data. Foods only; no fiscal data.
|
||||
- Cosmos / Bluesoft: https://api.cosmos.bluesoft.com.br/gtins/{gtin} - X-Cosmos-Token (free token, 25 queries/day), returns NCM/CEST/brand/description. Optional: without a token it is NOT_CONFIGURED.
|
||||
A provider result is EVIDENCE. Only the fiscal engine + official datasets decide what is written.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import httpx
|
||||
|
||||
USER_AGENT = "GTIFiscalHub/1.0 (+https://gtifiscal.gtigrupo.com.br; product identity lookup)"
|
||||
|
||||
|
||||
class ProviderError(Exception):
|
||||
"""kind: RATE_LIMITED | AUTH_ERROR | TIMEOUT | SERVER_ERROR | NETWORK | BAD_RESPONSE"""
|
||||
|
||||
def __init__(self, kind: str, detail: str = "", headers: dict | None = None, retry_after: int | None = None):
|
||||
super().__init__(f"{kind}: {detail}")
|
||||
self.kind, self.detail, self.headers, self.retry_after = kind, detail, headers or {}, retry_after
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProviderProduct:
|
||||
provider: str
|
||||
gtin: str
|
||||
found: bool
|
||||
title: str | None = None
|
||||
brand: str | None = None
|
||||
category: str | None = None
|
||||
quantity: str | None = None
|
||||
ncm: str | None = None
|
||||
cest: str | None = None
|
||||
raw_hash: str = ""
|
||||
latency_ms: int = 0
|
||||
headers: dict = field(default_factory=dict)
|
||||
|
||||
def as_data(self) -> dict:
|
||||
return {k: v for k, v in {"title": self.title, "brand": self.brand, "category": self.category, "quantity": self.quantity, "ncm": self.ncm, "cest": self.cest}.items() if v}
|
||||
|
||||
|
||||
def _digits(value) -> str | None:
|
||||
text = "".join(ch for ch in str(value or "") if ch.isdigit())
|
||||
return text or None
|
||||
|
||||
|
||||
class Provider:
|
||||
name = "base"
|
||||
paid = False
|
||||
supports_search = False
|
||||
daily_limit: int | None = None
|
||||
rate_per_minute: int | None = None
|
||||
burst: tuple[int, int] | None = None # (requests, seconds)
|
||||
provides_ncm = False
|
||||
|
||||
def __init__(self, client: httpx.Client | None = None, token: str | None = None):
|
||||
self.client = client or httpx.Client(timeout=4.0, headers={"User-Agent": USER_AGENT})
|
||||
self._token = token
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
return True
|
||||
|
||||
def _get(self, url: str, **kwargs) -> httpx.Response:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = self.client.get(url, **kwargs)
|
||||
except httpx.TimeoutException as error:
|
||||
raise ProviderError("TIMEOUT", type(error).__name__) from error
|
||||
except httpx.HTTPError as error:
|
||||
raise ProviderError("NETWORK", type(error).__name__) from error
|
||||
response.latency_ms = int((time.monotonic() - started) * 1000) # type: ignore[attr-defined]
|
||||
headers = {k.lower(): v for k, v in response.headers.items()}
|
||||
if response.status_code == 429:
|
||||
retry = headers.get("retry-after")
|
||||
raise ProviderError("RATE_LIMITED", "HTTP 429", headers, int(retry) if retry and retry.isdigit() else None)
|
||||
if response.status_code in (401, 403):
|
||||
raise ProviderError("AUTH_ERROR", f"HTTP {response.status_code}", headers)
|
||||
if response.status_code >= 500:
|
||||
raise ProviderError("SERVER_ERROR", f"HTTP {response.status_code}", headers)
|
||||
return response
|
||||
|
||||
def lookup_gtin(self, gtin: str) -> ProviderProduct: # pragma: no cover - interface
|
||||
raise NotImplementedError
|
||||
|
||||
def search_description(self, text: str) -> list[ProviderProduct]:
|
||||
return []
|
||||
|
||||
def health(self) -> dict:
|
||||
return {"configured": self.configured}
|
||||
|
||||
|
||||
class UpcItemDb(Provider):
|
||||
name = "upcitemdb"
|
||||
supports_search = True
|
||||
daily_limit = 100
|
||||
burst = (15, 30)
|
||||
|
||||
def lookup_gtin(self, gtin: str) -> ProviderProduct:
|
||||
response = self._get("https://api.upcitemdb.com/prod/trial/lookup", params={"upc": gtin})
|
||||
if response.status_code == 404:
|
||||
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=dict(response.headers))
|
||||
if response.status_code != 200:
|
||||
raise ProviderError("BAD_RESPONSE", f"HTTP {response.status_code}")
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError as error:
|
||||
raise ProviderError("BAD_RESPONSE", "not JSON") from error
|
||||
items = body.get("items") or []
|
||||
headers = {k.lower(): v for k, v in response.headers.items()}
|
||||
if not items:
|
||||
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
|
||||
item = items[0]
|
||||
raw = hashlib.sha256(json.dumps(item, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
||||
return ProviderProduct(self.name, gtin, True, title=item.get("title"), brand=item.get("brand"), category=item.get("category"),
|
||||
quantity=item.get("size") or item.get("weight") or None, raw_hash=raw, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
|
||||
|
||||
def search_description(self, text: str) -> list[ProviderProduct]:
|
||||
response = self._get("https://api.upcitemdb.com/prod/trial/search", params={"s": text, "match_mode": "1", "type": "product"})
|
||||
if response.status_code != 200:
|
||||
return []
|
||||
out = []
|
||||
for item in (response.json().get("items") or [])[:5]:
|
||||
code = _digits(item.get("ean") or item.get("upc")) or ""
|
||||
out.append(ProviderProduct(self.name, code, True, title=item.get("title"), brand=item.get("brand"), category=item.get("category"),
|
||||
raw_hash=hashlib.sha256(json.dumps(item, sort_keys=True, ensure_ascii=False).encode()).hexdigest()))
|
||||
return out
|
||||
|
||||
|
||||
class OpenFoodFacts(Provider):
|
||||
name = "openfoodfacts"
|
||||
supports_search = True
|
||||
rate_per_minute = 15
|
||||
host = "world.openfoodfacts.org"
|
||||
|
||||
def lookup_gtin(self, gtin: str) -> ProviderProduct:
|
||||
response = self._get(f"https://{self.host}/api/v2/product/{gtin}", params={"fields": "code,product_name,brands,quantity,categories,generic_name"})
|
||||
headers = {k.lower(): v for k, v in response.headers.items()}
|
||||
if response.status_code == 404:
|
||||
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
|
||||
if response.status_code != 200:
|
||||
raise ProviderError("BAD_RESPONSE", f"HTTP {response.status_code}")
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError as error:
|
||||
raise ProviderError("BAD_RESPONSE", "not JSON") from error
|
||||
product = body.get("product") or {}
|
||||
if body.get("status") != 1 or not product:
|
||||
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
|
||||
title = product.get("product_name") or product.get("generic_name")
|
||||
return ProviderProduct(self.name, gtin, bool(title), title=title, brand=(product.get("brands") or "").split(",")[0].strip() or None, category=(product.get("categories") or "").split(",")[0].strip() or None,
|
||||
quantity=product.get("quantity"), raw_hash=hashlib.sha256(json.dumps(product, sort_keys=True, ensure_ascii=False).encode()).hexdigest(),
|
||||
latency_ms=getattr(response, "latency_ms", 0), headers=headers)
|
||||
|
||||
def search_description(self, text: str) -> list[ProviderProduct]:
|
||||
response = self._get("https://world.openfoodfacts.org/cgi/search.pl", params={"search_terms": text, "json": "1", "page_size": "5", "fields": "code,product_name,brands,quantity"})
|
||||
if response.status_code != 200:
|
||||
return []
|
||||
return [ProviderProduct(self.name, _digits(p.get("code")) or "", True, title=p.get("product_name"), brand=(p.get("brands") or "").split(",")[0].strip() or None, quantity=p.get("quantity"))
|
||||
for p in (response.json().get("products") or [])[:5] if p.get("product_name")]
|
||||
|
||||
|
||||
class OpenProductsFacts(OpenFoodFacts):
|
||||
"""Same open (ODbL) platform and API as Open Food Facts, for non-food goods. Identity only (no NCM)."""
|
||||
name = "openproductsfacts"
|
||||
host = "world.openproductsfacts.org"
|
||||
supports_search = False
|
||||
|
||||
|
||||
class OpenBeautyFacts(OpenFoodFacts):
|
||||
name = "openbeautyfacts"
|
||||
host = "world.openbeautyfacts.org"
|
||||
supports_search = False
|
||||
|
||||
|
||||
class Cosmos(Provider):
|
||||
name = "cosmos"
|
||||
supports_search = True
|
||||
daily_limit = 25
|
||||
provides_ncm = True
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
return bool(self._token)
|
||||
|
||||
def _headers(self) -> dict:
|
||||
return {"X-Cosmos-Token": self._token or "", "User-Agent": USER_AGENT, "Content-Type": "application/json"}
|
||||
|
||||
def lookup_gtin(self, gtin: str) -> ProviderProduct:
|
||||
if not self.configured:
|
||||
raise ProviderError("AUTH_ERROR", "NOT_CONFIGURED")
|
||||
response = self._get(f"https://api.cosmos.bluesoft.com.br/gtins/{gtin}", headers=self._headers())
|
||||
headers = {k.lower(): v for k, v in response.headers.items()}
|
||||
if response.status_code == 404:
|
||||
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
|
||||
if response.status_code != 200:
|
||||
raise ProviderError("BAD_RESPONSE", f"HTTP {response.status_code}")
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError as error:
|
||||
raise ProviderError("BAD_RESPONSE", "not JSON") from error
|
||||
ncm = _digits((body.get("ncm") or {}).get("code"))
|
||||
cest = _digits((body.get("cest") or {}).get("code"))
|
||||
brand = (body.get("brand") or {}).get("name") if isinstance(body.get("brand"), dict) else body.get("brand")
|
||||
return ProviderProduct(self.name, gtin, True, title=body.get("description"), brand=brand, category=(body.get("gpc") or {}).get("description"), quantity=body.get("net_weight") and str(body.get("net_weight")),
|
||||
ncm=ncm, cest=cest, raw_hash=hashlib.sha256(json.dumps(body, sort_keys=True, ensure_ascii=False).encode()).hexdigest(), latency_ms=getattr(response, "latency_ms", 0), headers=headers)
|
||||
|
||||
def search_description(self, text: str) -> list[ProviderProduct]:
|
||||
if not self.configured:
|
||||
return []
|
||||
response = self._get("https://api.cosmos.bluesoft.com.br/products", params={"query": text}, headers=self._headers())
|
||||
if response.status_code != 200:
|
||||
return []
|
||||
out = []
|
||||
for item in (response.json().get("products") or [])[:5]:
|
||||
out.append(ProviderProduct(self.name, _digits(item.get("gtin")) or "", True, title=item.get("description"), brand=(item.get("brand") or {}).get("name") if isinstance(item.get("brand"), dict) else None,
|
||||
ncm=_digits((item.get("ncm") or {}).get("code")) if isinstance(item.get("ncm"), dict) else None))
|
||||
return out
|
||||
|
||||
|
||||
class Oscbr(Provider):
|
||||
"""OSCBR / RSC Sistemas GTIN API (https://gtin.rscsistemas.com.br). Free plan (no card): 50 calls/day, 5/min; account created BY THE OPERATOR, credentials only in the Hub env
|
||||
(OSCBR_USER / OSCBR_PASSWORD). Bearer token lasts 1 h and is renewed automatically. Optional: NOT_CONFIGURED without credentials (no call is made)."""
|
||||
name = "oscbr"
|
||||
rate_per_minute = 5 # free plan as currently documented: 50 calls/day, 5/min (conservative: the lower figure wins)
|
||||
daily_limit = 50
|
||||
# ROLE = IDENTITY. The operator-provided V3 contract (GET /api/v3/gtin/{ean}) documents ean, ean_tipo, nome, marca, categoria, link_foto - NO ncm/cest. The public landing page mentions NCM/CEST
|
||||
# and the legacy open-source client shows an `ncm` integer (leading zero lost), but the V3 documentation is behind the login, so NCM is UNPROVEN: the provider does not count as an NCM family
|
||||
# (provides_ncm=False). If a real authenticated V3 answer ever carries `ncm`, it is stored as evidence but the role only changes after that proof is reviewed.
|
||||
provides_ncm = False
|
||||
base = "https://gtin.rscsistemas.com.br"
|
||||
|
||||
def __init__(self, client=None, token=None, user=None, password=None):
|
||||
super().__init__(client, token) # token = a ready Bearer token (OSCBR_TOKEN); preferred over storing the login
|
||||
self._user, self._password = user, password
|
||||
self._bearer, self._bearer_until = None, 0.0
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
return bool(self._token or (self._user and self._password))
|
||||
|
||||
def _auth(self) -> str:
|
||||
if self._token:
|
||||
return self._token
|
||||
if self._bearer and time.time() < self._bearer_until:
|
||||
return self._bearer
|
||||
try:
|
||||
response = self.client.post(f"{self.base}/oauth/token", auth=(self._user, self._password), headers={"User-Agent": USER_AGENT})
|
||||
except httpx.TimeoutException as error:
|
||||
raise ProviderError("TIMEOUT", type(error).__name__) from error
|
||||
except httpx.HTTPError as error:
|
||||
raise ProviderError("NETWORK", type(error).__name__) from error
|
||||
if response.status_code in (400, 401, 403):
|
||||
raise ProviderError("AUTH_ERROR", f"HTTP {response.status_code}")
|
||||
if response.status_code == 429:
|
||||
raise ProviderError("RATE_LIMITED", "HTTP 429")
|
||||
if response.status_code >= 500 or response.status_code != 200:
|
||||
raise ProviderError("SERVER_ERROR", f"HTTP {response.status_code}")
|
||||
try:
|
||||
token = response.json().get("token") or response.json().get("access_token")
|
||||
except ValueError as error:
|
||||
raise ProviderError("BAD_RESPONSE", "not JSON") from error
|
||||
if not token:
|
||||
raise ProviderError("BAD_RESPONSE", "no token")
|
||||
self._bearer, self._bearer_until = token, time.time() + 55 * 60
|
||||
return token
|
||||
|
||||
def lookup_gtin(self, gtin: str) -> ProviderProduct:
|
||||
if not self.configured:
|
||||
raise ProviderError("AUTH_ERROR", "NOT_CONFIGURED")
|
||||
response = self._get(f"{self.base}/api/v3/gtin/{gtin}", headers={"Authorization": f"Bearer {self._auth()}", "User-Agent": USER_AGENT})
|
||||
headers = {k.lower(): v for k, v in response.headers.items()}
|
||||
if response.status_code == 404:
|
||||
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
|
||||
if response.status_code != 200:
|
||||
raise ProviderError("BAD_RESPONSE", f"HTTP {response.status_code}")
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError as error:
|
||||
raise ProviderError("BAD_RESPONSE", "not JSON") from error
|
||||
if isinstance(body, list):
|
||||
body = body[0] if body else {}
|
||||
if not body or not (body.get("nome") or body.get("ncm")):
|
||||
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
|
||||
ncm = _digits(body.get("ncm"))
|
||||
if ncm and len(ncm) == 7: # numeric JSON drops the leading zero (7133399 is NCM 07133399)
|
||||
ncm = ncm.zfill(8)
|
||||
return ProviderProduct(self.name, gtin, True, title=body.get("nome"), brand=body.get("marca"), category=body.get("categoria"), ncm=ncm if len(ncm or "") == 8 else None, cest=_digits(body.get("cest")) or None,
|
||||
raw_hash=hashlib.sha256(json.dumps(body, sort_keys=True, ensure_ascii=False).encode()).hexdigest(), latency_ms=getattr(response, "latency_ms", 0), headers=headers)
|
||||
|
||||
|
||||
class Kodebar(Provider):
|
||||
"""Kodebar (Korven Sistemas) - IDENTITY source only (name, brand, thumbnail; NO NCM). Free 'Teste' plan: 50 queries/day, no card, personal API key created by the operator
|
||||
(KODEBAR_API_KEY in the Hub env). Terms (2026-09-26): integration into our own multi-user system and internal caching are allowed; redistribution/resale is not.
|
||||
429 above the quota, no overage billing. NOT_CONFIGURED without a key: no request is made."""
|
||||
name = "kodebar"
|
||||
daily_limit = 50
|
||||
rate_per_minute = 30
|
||||
base = "https://kodebar.korvensistemas.com.br"
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
return bool(self._token)
|
||||
|
||||
def lookup_gtin(self, gtin: str) -> ProviderProduct:
|
||||
if not self.configured:
|
||||
raise ProviderError("AUTH_ERROR", "NOT_CONFIGURED")
|
||||
response = self._get(f"{self.base}/gtin/lookup", params={"gtin": gtin}, headers={"X-API-Key": self._token or "", "User-Agent": USER_AGENT})
|
||||
headers = {k.lower(): v for k, v in response.headers.items()}
|
||||
if response.status_code == 404:
|
||||
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
|
||||
if response.status_code != 200:
|
||||
raise ProviderError("BAD_RESPONSE", f"HTTP {response.status_code}")
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError as error:
|
||||
raise ProviderError("BAD_RESPONSE", "not JSON") from error
|
||||
if not isinstance(body, dict) or not body.get("nome"):
|
||||
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
|
||||
return ProviderProduct(self.name, gtin, True, title=body.get("nome"), brand=body.get("marca"), raw_hash=hashlib.sha256(json.dumps({k: body.get(k) for k in ("gtin", "nome", "marca")}, sort_keys=True, ensure_ascii=False).encode()).hexdigest(),
|
||||
latency_ms=getattr(response, "latency_ms", 0), headers=headers)
|
||||
|
||||
|
||||
def build_providers(settings, client: httpx.Client | None = None) -> list[Provider]:
|
||||
"""Fixed order chosen from the benchmark (docs/product_enrichment.md): Cosmos then OSCBR (fiscal-capable) first when configured, then UPCitemdb, then the Open *Facts family (food, general products, beauty; identity only). Paid providers do not exist here."""
|
||||
providers: list[Provider] = [Oscbr(client, getattr(settings, "oscbr_token", None), getattr(settings, "oscbr_user", None), getattr(settings, "oscbr_password", None)), Kodebar(client, getattr(settings, "kodebar_api_key", None)), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)]
|
||||
if getattr(settings, "cosmos_enabled", False): # Cosmos is OFF by default (account unavailable); only an explicit COSMOS_ENABLED=1 puts it back in the chain
|
||||
providers.insert(0, Cosmos(client, getattr(settings, "cosmos_token", None)))
|
||||
if any(p.paid for p in providers) and not getattr(settings, "allow_paid_providers", False):
|
||||
raise RuntimeError("paid providers are not allowed (ALLOW_PAID_PROVIDERS=false)")
|
||||
return providers
|
||||
@@ -0,0 +1,244 @@
|
||||
"""Central quota / circuit breaker / cache / single-flight for external product providers. Quota belongs to the HUB: 70 Agents never spend it uncoordinated."""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict, deque
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..models import GtinLookupCache, ProviderState, ProviderUsageDaily
|
||||
from .providers import Provider, ProviderError, ProviderProduct
|
||||
|
||||
POSITIVE_TTL = timedelta(days=30)
|
||||
NEGATIVE_TTL = timedelta(hours=24)
|
||||
CIRCUIT_BASE = timedelta(seconds=60)
|
||||
CIRCUIT_MAX = timedelta(minutes=15)
|
||||
FAILURES_TO_OPEN = 3
|
||||
|
||||
_windows: dict[str, deque] = defaultdict(deque) # per-process sliding windows (burst / per-minute governors)
|
||||
_flights: dict[tuple, threading.Lock] = {}
|
||||
_flights_guard = threading.Lock()
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _aware(value):
|
||||
if value is not None and value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value
|
||||
|
||||
|
||||
def next_utc_midnight(now: datetime) -> datetime:
|
||||
return (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Outcome:
|
||||
provider: str
|
||||
state: str # CACHE_HIT | FOUND | NOT_FOUND | SKIPPED | ERROR
|
||||
product: ProviderProduct | None = None
|
||||
reason: str | None = None # NOT_CONFIGURED | CIRCUIT_OPEN | RATE_LIMITED | RATE_WINDOW | TIMEOUT | AUTH_ERROR | NETWORK | SERVER_ERROR | ...
|
||||
retry_at: datetime | None = None
|
||||
from_cache: bool = False
|
||||
|
||||
|
||||
class QuotaManager:
|
||||
def __init__(self, session, now=utcnow):
|
||||
self.session, self.now = session, now
|
||||
|
||||
# ---- state -------------------------------------------------------------------------------------------------------------------------------------
|
||||
def state(self, provider: Provider) -> ProviderState:
|
||||
row = self.session.get(ProviderState, provider.name)
|
||||
if row is None:
|
||||
row = ProviderState(provider=provider.name, configured=provider.configured, status="ONLINE" if provider.configured else "NOT_CONFIGURED",
|
||||
daily_limit=provider.daily_limit, rate_per_minute=provider.rate_per_minute,
|
||||
quota_kind="LOCAL_ESTIMATE" if provider.daily_limit else "DOCUMENTED_RATE")
|
||||
self.session.add(row)
|
||||
self.session.flush()
|
||||
row.configured = provider.configured
|
||||
if not provider.configured:
|
||||
row.status = "NOT_CONFIGURED"
|
||||
elif row.status == "NOT_CONFIGURED":
|
||||
row.status = "ONLINE"
|
||||
row.daily_limit = provider.daily_limit
|
||||
row.rate_per_minute = provider.rate_per_minute
|
||||
return row
|
||||
|
||||
def usage(self, provider_name: str, day: str | None = None) -> ProviderUsageDaily:
|
||||
day = day or self.now().strftime("%Y-%m-%d")
|
||||
row = self.session.scalar(select(ProviderUsageDaily).where(ProviderUsageDaily.provider == provider_name, ProviderUsageDaily.day == day))
|
||||
if row is None:
|
||||
row = ProviderUsageDaily(provider=provider_name, day=day)
|
||||
self.session.add(row)
|
||||
self.session.flush()
|
||||
return row
|
||||
|
||||
# ---- decision ----------------------------------------------------------------------------------------------------------------------------------
|
||||
def allow(self, provider: Provider, purpose: str = "OPERATIONAL", reserve_percent: int = 0) -> tuple[bool, str | None, datetime | None]:
|
||||
state, now = self.state(provider), self.now()
|
||||
if not provider.configured:
|
||||
return False, "NOT_CONFIGURED", None
|
||||
if state.circuit_open_until and _aware(state.circuit_open_until) > now:
|
||||
return False, "CIRCUIT_OPEN", _aware(state.circuit_open_until)
|
||||
usage = self.usage(provider.name)
|
||||
if provider.daily_limit and state.quota_kind == "REMOTE_QUOTA" and state.remaining is not None and state.remaining <= 0 and state.reset_at and _aware(state.reset_at) > now:
|
||||
return False, "RATE_LIMITED", _aware(state.reset_at)
|
||||
if provider.daily_limit and usage.requests >= provider.daily_limit and state.quota_kind != "REMOTE_QUOTA":
|
||||
state.status = "RATE_LIMITED"
|
||||
return False, "RATE_LIMITED", state.reset_at and _aware(state.reset_at) or next_utc_midnight(now)
|
||||
if purpose == "BACKFILL" and provider.daily_limit: # backfill may only use what is left above the operational reserve
|
||||
used = usage.requests
|
||||
if state.quota_kind == "REMOTE_QUOTA" and state.remaining is not None:
|
||||
used = provider.daily_limit - state.remaining
|
||||
if used >= provider.daily_limit * (100 - reserve_percent) / 100:
|
||||
return False, "OPERATIONAL_RESERVE", state.reset_at and _aware(state.reset_at) or next_utc_midnight(now)
|
||||
window = (provider.burst[1] if provider.burst else 60)
|
||||
limit = (provider.burst[0] if provider.burst else provider.rate_per_minute)
|
||||
if limit:
|
||||
q, cutoff = _windows[provider.name], time.monotonic() - window
|
||||
while q and q[0] < cutoff:
|
||||
q.popleft()
|
||||
if len(q) >= limit:
|
||||
return False, "RATE_WINDOW", now + timedelta(seconds=max(1, int(window - (time.monotonic() - q[0]))))
|
||||
return True, None, None
|
||||
|
||||
# ---- accounting --------------------------------------------------------------------------------------------------------------------------------
|
||||
def _remote_quota(self, state: ProviderState, headers: dict):
|
||||
remaining, limit, reset = headers.get("x-ratelimit-remaining"), headers.get("x-ratelimit-limit"), headers.get("x-ratelimit-reset")
|
||||
if remaining is not None and str(remaining).isdigit():
|
||||
state.remaining = int(remaining)
|
||||
state.quota_kind = "REMOTE_QUOTA"
|
||||
if limit and str(limit).isdigit():
|
||||
state.daily_limit = int(limit)
|
||||
if reset and str(reset).isdigit():
|
||||
state.reset_at = datetime.fromtimestamp(int(reset), tz=timezone.utc)
|
||||
|
||||
def _sent(self, provider: Provider):
|
||||
q = _windows[provider.name]
|
||||
q.append(time.monotonic())
|
||||
|
||||
def record_ok(self, provider: Provider, result: ProviderProduct):
|
||||
state, usage, now = self.state(provider), self.usage(provider.name), self.now()
|
||||
self._sent(provider)
|
||||
usage.requests += 1
|
||||
usage.success += 1
|
||||
if not result.found:
|
||||
usage.not_found += 1
|
||||
else:
|
||||
usage.found += 1
|
||||
if result.ncm:
|
||||
usage.ncm_found += 1
|
||||
state.consecutive_failures = 0
|
||||
state.circuit_open_until = None
|
||||
state.status = "ONLINE"
|
||||
state.last_request_at = state.last_success_at = now
|
||||
state.last_latency_ms = result.latency_ms or None
|
||||
self._remote_quota(state, {k.lower(): v for k, v in (result.headers or {}).items()})
|
||||
if provider.daily_limit and state.quota_kind != "REMOTE_QUOTA":
|
||||
state.remaining = max(0, provider.daily_limit - usage.requests)
|
||||
state.reset_at = next_utc_midnight(now)
|
||||
self.session.commit()
|
||||
|
||||
def record_error(self, provider: Provider, error: ProviderError):
|
||||
state, usage, now = self.state(provider), self.usage(provider.name), self.now()
|
||||
self._sent(provider)
|
||||
usage.requests += 1
|
||||
state.last_request_at, state.last_error_at = now, now
|
||||
state.last_error = f"{error.kind}: {error.detail}"[:160]
|
||||
state.consecutive_failures += 1
|
||||
headers = {k.lower(): v for k, v in (error.headers or {}).items()}
|
||||
self._remote_quota(state, headers)
|
||||
if error.kind == "RATE_LIMITED":
|
||||
usage.rate_limited += 1
|
||||
state.status = "RATE_LIMITED"
|
||||
wait = timedelta(seconds=error.retry_after) if error.retry_after else None
|
||||
state.circuit_open_until = now + (wait or (state.reset_at and max(timedelta(minutes=15), _aware(state.reset_at) - now) or timedelta(minutes=60)))
|
||||
if state.reset_at is None:
|
||||
state.reset_at = state.circuit_open_until
|
||||
elif error.kind == "AUTH_ERROR":
|
||||
usage.errors += 1
|
||||
state.status = "AUTH_ERROR"
|
||||
state.circuit_open_until = now + timedelta(hours=1)
|
||||
else:
|
||||
if error.kind == "TIMEOUT":
|
||||
usage.timeouts += 1
|
||||
usage.errors += 1
|
||||
state.status = "DEGRADED"
|
||||
if state.consecutive_failures >= FAILURES_TO_OPEN:
|
||||
delay = min(CIRCUIT_MAX, CIRCUIT_BASE * (2 ** (state.consecutive_failures - FAILURES_TO_OPEN)))
|
||||
state.circuit_open_until = now + delay
|
||||
state.status = "OFFLINE"
|
||||
self.session.commit()
|
||||
|
||||
def record_kb_hit(self):
|
||||
"""A resolution answered by the KB alone (no provider asked): counted once per day under the pseudo-provider 'kb'."""
|
||||
self.usage("kb").kb_hits += 1
|
||||
|
||||
def record_cache(self, provider_name: str, hit: bool):
|
||||
usage = self.usage(provider_name)
|
||||
if hit:
|
||||
usage.cache_hits += 1
|
||||
else:
|
||||
usage.cache_misses += 1
|
||||
|
||||
|
||||
class ProviderGateway:
|
||||
"""lookup(provider, gtin): cache -> single-flight -> quota/circuit -> HTTP -> account -> cache."""
|
||||
|
||||
def __init__(self, session, providers: list[Provider], now=utcnow):
|
||||
self.session, self.providers, self.now = session, providers, now
|
||||
self.quota = QuotaManager(session, now)
|
||||
|
||||
def _cached(self, provider: Provider, gtin: str) -> ProviderProduct | None | bool:
|
||||
row = self.session.scalar(select(GtinLookupCache).where(GtinLookupCache.gtin == gtin, GtinLookupCache.provider == provider.name))
|
||||
if row is None or _aware(row.expires_at) <= self.now():
|
||||
return False
|
||||
if not row.found:
|
||||
return None
|
||||
d = row.data or {}
|
||||
return ProviderProduct(provider.name, gtin, True, title=d.get("title"), brand=d.get("brand"), category=d.get("category"), quantity=d.get("quantity"), ncm=d.get("ncm"), cest=d.get("cest"), raw_hash=d.get("raw_hash", ""))
|
||||
|
||||
def _store(self, provider: Provider, gtin: str, product: ProviderProduct):
|
||||
now = self.now()
|
||||
ttl = POSITIVE_TTL if product.found else NEGATIVE_TTL
|
||||
row = self.session.scalar(select(GtinLookupCache).where(GtinLookupCache.gtin == gtin, GtinLookupCache.provider == provider.name))
|
||||
data = {**product.as_data(), "raw_hash": product.raw_hash}
|
||||
if row is None:
|
||||
self.session.add(GtinLookupCache(gtin=gtin, provider=provider.name, found=product.found, data=data, fetched_at=now, expires_at=now + ttl))
|
||||
else:
|
||||
row.found, row.data, row.fetched_at, row.expires_at = product.found, data, now, now + ttl
|
||||
self.session.commit()
|
||||
|
||||
def lookup(self, provider: Provider, gtin: str, purpose: str = "OPERATIONAL", reserve_percent: int = 0) -> Outcome:
|
||||
hit = self._cached(provider, gtin)
|
||||
if hit is not False:
|
||||
self.quota.record_cache(provider.name, True)
|
||||
self.session.commit()
|
||||
return Outcome(provider.name, "CACHE_HIT", hit or None, from_cache=True) if hit else Outcome(provider.name, "NOT_FOUND", None, from_cache=True)
|
||||
with _flights_guard:
|
||||
lock = _flights.setdefault((provider.name, gtin), threading.Lock())
|
||||
with lock: # 10 installations, same EAN -> 1 external call, the rest wait for the same answer
|
||||
self.session.expire_all()
|
||||
hit = self._cached(provider, gtin)
|
||||
if hit is not False:
|
||||
self.quota.record_cache(provider.name, True)
|
||||
self.session.commit()
|
||||
return Outcome(provider.name, "CACHE_HIT", hit or None, from_cache=True) if hit else Outcome(provider.name, "NOT_FOUND", None, from_cache=True)
|
||||
self.quota.record_cache(provider.name, False)
|
||||
ok, reason, retry_at = self.quota.allow(provider, purpose, reserve_percent)
|
||||
if not ok:
|
||||
self.session.commit()
|
||||
return Outcome(provider.name, "SKIPPED", None, reason, retry_at)
|
||||
try:
|
||||
product = provider.lookup_gtin(gtin)
|
||||
except ProviderError as error:
|
||||
self.quota.record_error(provider, error)
|
||||
return Outcome(provider.name, "ERROR", None, error.kind, self.quota.state(provider).circuit_open_until)
|
||||
self.quota.record_ok(provider, product)
|
||||
self._store(provider, gtin, product)
|
||||
return Outcome(provider.name, "FOUND" if product.found else "NOT_FOUND", product if product.found else None)
|
||||
@@ -0,0 +1,250 @@
|
||||
"""Product resolver: GTIN/name -> KB -> providers -> official NCM validation -> existing fiscal engine, decided FIELD BY FIELD.
|
||||
|
||||
Deterministic. No AI, no scores: discrete states only.
|
||||
Resolution states: EXACT_GTIN_VERIFIED, EXACT_KB_NAME_VERIFIED, MULTI_SOURCE_MATCH, SINGLE_WEAK_SOURCE, AMBIGUOUS, CONFLICT, NOT_FOUND
|
||||
(+ operational: INVALID_GTIN, PENDING_PROVIDER_QUOTA, PENDING_PROVIDERS_OFFLINE).
|
||||
Field decisions: AUTO_FILL_SAFE | SUGGESTION_ONLY | CONFLICT | INSUFFICIENT_EVIDENCE | ALREADY_SET | NOT_APPLICABLE.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from datetime import date
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..classification import Classifier, normalize_code
|
||||
from ..datasets.service import DatasetService
|
||||
from ..models import Dataset, NcmEntry
|
||||
from . import fiscal_completion, kb, textnorm
|
||||
from .gtin import normalize_gtin
|
||||
from .placeholders import is_placeholder, is_writable_empty
|
||||
from .quota import ProviderGateway
|
||||
|
||||
AUTO, SUGGEST, CONFLICT, INSUFFICIENT, ALREADY, NA = "AUTO_FILL_SAFE", "SUGGESTION_ONLY", "CONFLICT", "INSUFFICIENT_EVIDENCE", "ALREADY_SET", "NOT_APPLICABLE"
|
||||
FIELDS = ("CLASSIFICACAO_FISCAL", "CEST", "IPI")
|
||||
|
||||
|
||||
def _parse_br(text: str | None) -> date | None:
|
||||
try:
|
||||
d, m, y = str(text).split("/")
|
||||
return date(int(y), int(m), int(d))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def ncm_is_official(session, code: str, today: date | None = None) -> bool:
|
||||
"""8-digit code present in the ACTIVE official NCM dataset and vigente today (dd/mm/yyyy, 31/12/9999 = open)."""
|
||||
code = normalize_code(code)
|
||||
if len(code) != 8:
|
||||
return False
|
||||
today = today or date.today()
|
||||
rows = session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == code)).all()
|
||||
for row in rows:
|
||||
start, end = _parse_br(row.valid_from), _parse_br(row.valid_to)
|
||||
if (start is None or start <= today) and (end is None or end >= today):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class _ScopedDatasets(DatasetService):
|
||||
"""The Classifier only needs the ONE exact NCM row here; loading the whole 15k-row catalogue on every product would cost ~300 ms of the 5 s budget."""
|
||||
|
||||
def __init__(self, session, code: str):
|
||||
super().__init__(session)
|
||||
self.code = code
|
||||
|
||||
def active_entries(self):
|
||||
return self.session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == self.code)).all()
|
||||
|
||||
|
||||
def ncm_exists_any(session, code: str, *, current: bool, today: date | None = None) -> bool:
|
||||
"""current=True: vigente today; current=False: present in the ACTIVE dataset with any validity window (i.e. known but not vigente)."""
|
||||
code = normalize_code(code)
|
||||
if len(code) != 8:
|
||||
return False
|
||||
if current:
|
||||
return ncm_is_official(session, code, today)
|
||||
return session.scalar(select(NcmEntry.id).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == code).limit(1)) is not None
|
||||
|
||||
|
||||
def dataset_versions(session) -> dict:
|
||||
return {d.kind: {"sha256": (d.sha256 or "")[:12], "version": d.version_label} for d in session.scalars(select(Dataset).where(Dataset.status == "ACTIVE"))}
|
||||
|
||||
|
||||
def _field(status, value=None, rule=None, kb_status=None, evidence=None, reason=None):
|
||||
return {"status": status, "value": value, "rule": rule, "kb_status": kb_status, "evidence": evidence or [], "reason": reason}
|
||||
|
||||
|
||||
def _empty(current: dict, name: str) -> bool:
|
||||
return is_writable_empty(name, current.get(name))
|
||||
|
||||
|
||||
def _merge_fiscal_completion(session, out: dict, *, ncm_official: bool, ncm_for_engine: str | None, current: dict, description: str,
|
||||
product_code: str | None, installation, tenant_id: int | None, decision: dict | None, product_evidence: dict | None = None) -> None:
|
||||
"""Merges field_contract.resolve_product's own field-by-field decisions (CFOP/CST/CSOSN/PIS-COFINS/...) into `out`, in the SAME
|
||||
AUTO_FILL_SAFE/ALREADY_SET/NOT_APPLICABLE/INSUFFICIENT_EVIDENCE vocabulary CEST/IPI already use, plus NEEDS_REVIEW for anything that needs an
|
||||
operator's context answer. No-op when the caller (a test, or a code path with no installation on hand) did not pass installation/tenant_id/product_code."""
|
||||
if installation is None or tenant_id is None or product_code is None:
|
||||
return
|
||||
fields = fiscal_completion.resolve_fiscal_fields(session, product_code=product_code, description=description, current=current, ncm=ncm_for_engine or "",
|
||||
ncm_official=ncm_official, installation=installation, tenant_id=tenant_id, decision=decision, extra_columns=current,
|
||||
ev=product_evidence)
|
||||
for field, r in fields.items():
|
||||
out[field] = _field(r["status"], r.get("value"), r.get("rule"), reason=r.get("reason"))
|
||||
if r.get("missing_key"):
|
||||
out[field]["missing_key"], out[field]["missing_scope"] = r["missing_key"], r.get("missing_scope")
|
||||
|
||||
|
||||
def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current: dict, description: str, *, name_verified=False,
|
||||
product_code: str | None = None, installation=None, tenant_id: int | None = None, product_evidence: dict | None = None) -> dict:
|
||||
out = {}
|
||||
if not _empty(current, "CLASSIFICACAO_FISCAL"):
|
||||
out["CLASSIFICACAO_FISCAL"] = _field(ALREADY, reason="campo já preenchido")
|
||||
elif ncm_eval["status"] == kb.CONFLICT:
|
||||
out["CLASSIFICACAO_FISCAL"] = _field(CONFLICT, kb_status=kb.CONFLICT, evidence=ncm_eval["candidates"], reason="fontes divergem no NCM")
|
||||
elif ncm_eval["value"] and ncm_eval["status"] == kb.VERIFIED_KB:
|
||||
out["CLASSIFICACAO_FISCAL"] = _field(AUTO, ncm_eval["value"], "NCM_KB_VERIFIED" if gtin_kb_verified or name_verified else "NCM_TWO_INDEPENDENT_SOURCES", kb.VERIFIED_KB, ncm_eval["families"])
|
||||
elif ncm_eval["value"]:
|
||||
rule = {"CORROBORATED": "NCM_CORROBORATED_LOCAL_HISTORY", "EXTERNAL_CORROBORATED": "NCM_EXTERNAL_CORROBORATED"}.get(ncm_eval["status"], "NCM_SINGLE_SOURCE")
|
||||
out["CLASSIFICACAO_FISCAL"] = _field(SUGGEST, ncm_eval["value"], rule, ncm_eval["status"], ncm_eval["families"], "evidência ainda insuficiente para preencher automaticamente")
|
||||
else:
|
||||
out["CLASSIFICACAO_FISCAL"] = _field(INSUFFICIENT, reason="nenhuma fonte com NCM válido no dataset oficial")
|
||||
ncm = out["CLASSIFICACAO_FISCAL"]
|
||||
own = current.get("CLASSIFICACAO_FISCAL")
|
||||
ncm_for_engine = ncm["value"] if ncm["status"] == AUTO else (None if is_placeholder("CLASSIFICACAO_FISCAL", own) else (normalize_code(own) or None))
|
||||
if not ncm_for_engine or (ncm["status"] == ALREADY and not ncm_is_official(session, ncm_for_engine)):
|
||||
for name in ("CEST", "IPI"):
|
||||
out[name] = _field(ALREADY if not _empty(current, name) else INSUFFICIENT, reason="depende de um NCM oficial válido")
|
||||
_merge_fiscal_completion(session, out, ncm_official=False, ncm_for_engine=ncm_for_engine, current=current, description=description,
|
||||
product_code=product_code, installation=installation, tenant_id=tenant_id, decision=None, product_evidence=product_evidence)
|
||||
return out
|
||||
# CEST / IPI are deterministic functions of an OFFICIAL NCM: the NCM we are about to write (AUTO) or one already in the product (validated above against the official dataset)
|
||||
ncm_solid = ncm["status"] == AUTO or (ncm["status"] == ALREADY and ncm_is_official(session, ncm_for_engine))
|
||||
decision = Classifier(_ScopedDatasets(session, ncm_for_engine)).classify({"current_ncm": ncm_for_engine, "description": description})
|
||||
cest = (decision["classification"].get("cest") or [])
|
||||
if not _empty(current, "CEST"):
|
||||
out["CEST"] = _field(ALREADY, reason="campo já preenchido")
|
||||
elif decision["cest_status"] == "CEST_CONFIRMED" and len(cest) == 1 and ncm_solid:
|
||||
out["CEST"] = _field(AUTO, normalize_code(cest[0]), "CEST_OFFICIAL_UNIQUE_MAPPING", ncm["kb_status"], [f"NCM {ncm_for_engine}"])
|
||||
elif decision["cest_status"] == "NO_CEST_MAPPING_FOUND":
|
||||
out["CEST"] = _field(NA, reason="NCM sem mapeamento de CEST no dataset oficial")
|
||||
else:
|
||||
out["CEST"] = _field(INSUFFICIENT, reason=f"CEST não determinístico ({decision['cest_status']})")
|
||||
ipi = decision["classification"].get("ipi") or {}
|
||||
if not _empty(current, "IPI"):
|
||||
out["IPI"] = _field(ALREADY, reason="campo já preenchido")
|
||||
elif decision["ipi_status"] == "IPI_RESOLVED" and ipi.get("rate_kind") == "NUMERIC" and ncm_solid:
|
||||
out["IPI"] = _field(AUTO, str(ipi["rate_numeric"]), "IPI_TIPI_NUMERIC", ncm["kb_status"], [f"TIPI NCM {ncm_for_engine}"])
|
||||
elif decision["ipi_status"] in {"IPI_RESOLVED"}:
|
||||
out["IPI"] = _field(NA, reason="TIPI sem alíquota numérica (NT/branca)")
|
||||
else:
|
||||
out["IPI"] = _field(INSUFFICIENT, reason=f"IPI não determinístico ({decision['ipi_status']})")
|
||||
_merge_fiscal_completion(session, out, ncm_official=ncm_solid, ncm_for_engine=ncm_for_engine, current=current, description=description,
|
||||
product_code=product_code, installation=installation, tenant_id=tenant_id, decision=decision, product_evidence=product_evidence)
|
||||
return out
|
||||
|
||||
|
||||
def resolve(session, gateway: ProviderGateway, *, gtin_raw, description, current: dict | None = None, client_label: str | None = None, allow_external: bool = True, budget_s: float | None = None,
|
||||
product_code: str | None = None, installation=None, tenant_id: int | None = None, product_evidence: dict | None = None) -> dict:
|
||||
started = time.monotonic()
|
||||
"""Returns {state, gtin_status, gtin, fields:{...}, providers:[...], retry_at, dataset_versions}. Never raises for provider trouble.
|
||||
|
||||
`product_code`/`installation`/`tenant_id`, when given, additionally resolve the post-NCM fiscal-completion fields (CFOP/CST/CSOSN/PIS-COFINS/...)
|
||||
via the existing field_contract engine (see `_merge_fiscal_completion`) and merge them into `fields` alongside CLASSIFICACAO_FISCAL/CEST/IPI.
|
||||
`product_evidence` is the Agent's targeted, product_code-scoped recipes/composed/purchases/stock evidence (firebird_reader.read_product_evidence) -
|
||||
never a full-table scan. Named separately from the local `ev` variable below (NCM knowledge-base evidence), a different, pre-existing concept."""
|
||||
current = {k: v for k, v in (current or {}).items()}
|
||||
g = normalize_gtin(gtin_raw)
|
||||
result = {"state": "NOT_FOUND", "gtin_status": g["status"], "gtin": g.get("key"), "fields": {}, "providers": [], "retry_at": None, "dataset_versions": dataset_versions(session)}
|
||||
session.info["is_official"] = lambda v: ncm_is_official(session, v)
|
||||
is_official = session.info["is_official"]
|
||||
product, name_verified, key = None, False, g.get("key")
|
||||
|
||||
if g["status"] == "VALID_GTIN": # GTIN has absolute priority over the description
|
||||
product = kb.find_by_gtin(session, key)
|
||||
ev = kb.evaluate(session, product, "ncm", is_official) if product else None
|
||||
if ev and ev["status"] == kb.VERIFIED_KB:
|
||||
gateway.quota.record_kb_hit()
|
||||
result["state"] = "EXACT_GTIN_VERIFIED"
|
||||
result["fields"] = _fiscal_decisions(session, ev, True, current, description or "", product_code=product_code, installation=installation, tenant_id=tenant_id, product_evidence=product_evidence)
|
||||
session.commit()
|
||||
return result
|
||||
if ev and ev["status"] == kb.CONFLICT: # local evidence already conflicts: an external answer cannot settle it, so spend no quota
|
||||
result["state"] = "CONFLICT"
|
||||
result["fields"] = _fiscal_decisions(session, ev, False, current, description or "", product_code=product_code, installation=installation, tenant_id=tenant_id, product_evidence=product_evidence)
|
||||
gateway.quota.record_kb_hit()
|
||||
session.commit()
|
||||
return result
|
||||
locally_known = bool(ev and ev["status"] in (kb.CORROBORATED, kb.EXTERNAL_CORROBORATED)) # identity + NCM candidate already local: only an NCM-bearing source can still add value
|
||||
query = g["gtin"] if len(g["gtin"]) in (8, 14) else key[1:] # providers/caches use the canonical EAN-13 (UPC-12 is zero-padded to it)
|
||||
found, pending = [], []
|
||||
if allow_external:
|
||||
for provider in gateway.providers:
|
||||
if locally_known and not getattr(provider, "provides_ncm", False):
|
||||
result["providers"].append({"provider": provider.name, "state": "SKIPPED", "reason": "LOCAL_IDENTITY_KNOWN", "from_cache": False})
|
||||
continue
|
||||
if budget_s is not None and time.monotonic() - started > budget_s:
|
||||
result["providers"].append({"provider": provider.name, "state": "SKIPPED", "reason": "TIME_BUDGET", "from_cache": False})
|
||||
pending.append(type("P", (), {"reason": "TIME_BUDGET", "retry_at": None})())
|
||||
continue
|
||||
out = gateway.lookup(provider, query)
|
||||
result["providers"].append({"provider": out.provider, "state": out.state, "reason": out.reason, "from_cache": out.from_cache})
|
||||
if out.state in ("FOUND", "CACHE_HIT") and out.product:
|
||||
found.append(out.product)
|
||||
product = kb.upsert_product(session, key, out.product.title or description, brand=out.product.brand, quantity=out.product.quantity)
|
||||
if out.product.ncm:
|
||||
kb.add_evidence(session, product, "ncm", out.product.ncm, f"provider:{out.provider}"[:32], family=f"PROVIDER:{out.provider}", info={"raw_hash": out.product.raw_hash})
|
||||
ev = kb.evaluate(session, product, "ncm", is_official)
|
||||
if ev["status"] == kb.VERIFIED_KB:
|
||||
break # enough evidence: spare the remaining quota
|
||||
elif out.state in ("SKIPPED", "ERROR"):
|
||||
pending.append(out)
|
||||
if product is None and description:
|
||||
product = kb.upsert_product(session, key, description)
|
||||
ev = kb.evaluate(session, product, "ncm", is_official) if product else {"value": None, "status": kb.UNVERIFIED, "families": [], "candidates": [], "rejected": []}
|
||||
result["fields"] = _fiscal_decisions(session, ev, False, current, description or "", product_code=product_code, installation=installation, tenant_id=tenant_id, product_evidence=product_evidence)
|
||||
if ev["status"] == kb.CONFLICT:
|
||||
result["state"] = "CONFLICT"
|
||||
elif ev["status"] == kb.VERIFIED_KB:
|
||||
result["state"] = "MULTI_SOURCE_MATCH" if found else "EXACT_GTIN_VERIFIED"
|
||||
elif len(found) >= 2:
|
||||
result["state"] = "MULTI_SOURCE_MATCH"
|
||||
elif found:
|
||||
result["state"] = "SINGLE_WEAK_SOURCE"
|
||||
elif pending and not found:
|
||||
quota = any(p.reason in ("RATE_LIMITED", "RATE_WINDOW") for p in pending)
|
||||
offline = [p for p in pending if p.reason not in ("NOT_CONFIGURED",)]
|
||||
if offline:
|
||||
result["state"] = "PENDING_PROVIDER_QUOTA" if quota else "PENDING_PROVIDERS_OFFLINE"
|
||||
result["retry_at"] = min((p.retry_at for p in offline if p.retry_at), default=None)
|
||||
session.commit()
|
||||
return result
|
||||
|
||||
if g["status"] == "INVALID_GTIN":
|
||||
result["state"] = "INVALID_GTIN" # never spends provider quota on an invalid GTIN
|
||||
|
||||
# ---- name only: only a UNIQUE, specific description with a VERIFIED_KB fingerprint may fill anything
|
||||
if not textnorm.is_specific(description):
|
||||
if result["state"] != "INVALID_GTIN":
|
||||
result["state"] = "AMBIGUOUS"
|
||||
result["fields"] = {f: _field(INSUFFICIENT, reason="descrição genérica demais e sem GTIN válido") if _empty(current, f) else _field(ALREADY) for f in FIELDS}
|
||||
return result
|
||||
candidates = kb.name_candidates(session, description)
|
||||
verified = []
|
||||
for cand in candidates:
|
||||
ev = kb.evaluate(session, cand, "ncm", is_official)
|
||||
if ev["status"] == kb.VERIFIED_KB:
|
||||
verified.append((cand, ev))
|
||||
if candidates and len(verified) == len(candidates) and len({ev["value"] for _, ev in verified}) == 1:
|
||||
_, ev = verified[0]
|
||||
result["state"] = "EXACT_KB_NAME_VERIFIED"
|
||||
result["fields"] = _fiscal_decisions(session, ev, True, current, description or "", name_verified=True, product_code=product_code, installation=installation, tenant_id=tenant_id, product_evidence=product_evidence)
|
||||
elif len({ev["value"] for _, ev in verified}) > 1:
|
||||
result["state"] = "CONFLICT"
|
||||
result["fields"] = {f: _field(CONFLICT, reason="descrição corresponde a produtos com NCM divergentes") if _empty(current, f) else _field(ALREADY) for f in FIELDS}
|
||||
else:
|
||||
if result["state"] != "INVALID_GTIN":
|
||||
result["state"] = "AMBIGUOUS" if len(candidates) > 1 else "NOT_FOUND"
|
||||
result["fields"] = {f: _field(INSUFFICIENT, reason="sem correspondência verificada na base de conhecimento") if _empty(current, f) else _field(ALREADY) for f in FIELDS}
|
||||
session.commit()
|
||||
return result
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Human-facing status of every tracked product: nothing may silently 'do nothing'. Computed from the resolver result and refined when the Agent reports the write outcome."""
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import select, update
|
||||
|
||||
LABELS = {
|
||||
"ENRIQUECIDO": "Enriquecido: campos seguros preenchidos.",
|
||||
"PRONTO_PARA_PREENCHER": "Evidência suficiente: aguardando a escrita segura pelo Agent.",
|
||||
"SIMULADO": "Simulação (DRY_RUN): há candidato, nada foi gravado.",
|
||||
"AGUARDANDO_EVIDENCIA": "Aguardando evidência: nenhuma fonte confiável ainda.",
|
||||
"EAN_AUSENTE": "Sem EAN e sem produto único verificado na base: informe o código de barras ou peça a decisão de um administrador.",
|
||||
"EAN_INVALIDO": "Código de barras inválido: nenhuma fonte externa foi consultada.",
|
||||
"NOME_AMBIGUO": "Nome genérico ou ambíguo demais para identificar o produto com segurança.",
|
||||
"SEM_MATCH_DE_PRODUTO": "Nenhum produto conhecido corresponde a este nome/EAN.",
|
||||
"NCM_NAO_VERIFICADO": "Produto identificado, mas o NCM ainda não foi verificado (falta segunda fonte independente ou decisão de administrador).",
|
||||
"CONFLITO_DE_NCM": "As fontes divergem sobre o NCM: nada é preenchido até um administrador decidir.",
|
||||
"PROVIDER_SEM_QUOTA": "Limite diário das fontes externas atingido; será repetido após o reset.",
|
||||
"PROVIDER_NAO_CONFIGURADO": "Nenhuma fonte externa com NCM está configurada.",
|
||||
"AGUARDANDO_RETRY": "Fontes indisponíveis agora; nova tentativa automática agendada.",
|
||||
"HUMAN_OVERRIDE": "Alterado por uma pessoa: nunca mais preenchido automaticamente.",
|
||||
"TECHNICAL_PLACEHOLDER": "NCM provisório (00000000) reconhecido como 'ainda não classificado'.",
|
||||
"WRITE_FAILED": "A escrita segura foi recusada ou falhou (veja o motivo).",
|
||||
"VERIFICACAO_POS_WRITE_FALHOU": "A escrita foi feita mas a releitura não confirmou o valor.",
|
||||
"NADA_A_PREENCHER": "Todos os campos elegíveis já estão preenchidos.",
|
||||
"FUNCAO_DESATIVADA": "Enriquecimento desativado para esta instalação.",
|
||||
}
|
||||
|
||||
|
||||
def classify(result: dict, *, has_gtin: bool, specific_name: bool) -> tuple[str, str]:
|
||||
"""(ui_status, reason_code) from a resolve() result."""
|
||||
fields = result.get("fields") or {}
|
||||
ncm = fields.get("CLASSIFICACAO_FISCAL") or {}
|
||||
state = result.get("state")
|
||||
if state == "FEATURE_DISABLED":
|
||||
return "FUNCAO_DESATIVADA", state
|
||||
if state == "INVALID_GTIN":
|
||||
return "EAN_INVALIDO", state
|
||||
if state == "PENDING_PROVIDER_QUOTA":
|
||||
return "PROVIDER_SEM_QUOTA", state
|
||||
if state in ("PENDING_PROVIDERS_OFFLINE", "PENDING_TIME_BUDGET"):
|
||||
return "AGUARDANDO_RETRY", state
|
||||
if state == "CONFLICT" or ncm.get("status") == "CONFLICT":
|
||||
return "CONFLITO_DE_NCM", "CONFLICT"
|
||||
if ncm.get("status") == "HUMAN_OVERRIDDEN":
|
||||
return "HUMAN_OVERRIDE", "HUMAN_OVERRIDDEN"
|
||||
if any(f.get("status") == "AUTO_FILL_SAFE" for f in fields.values()):
|
||||
return "PRONTO_PARA_PREENCHER", state or "RESOLVED"
|
||||
if ncm.get("status") == "ALREADY_SET" and not any(f.get("status") in ("AUTO_FILL_SAFE", "SUGGESTION_ONLY") for f in fields.values()):
|
||||
return "NADA_A_PREENCHER", "ALL_FIELDS_SET"
|
||||
if not has_gtin:
|
||||
if state == "AMBIGUOUS" or not specific_name:
|
||||
return "NOME_AMBIGUO", "GENERIC_OR_AMBIGUOUS_NAME"
|
||||
return "EAN_AUSENTE", "NAME_ONLY_NO_VERIFIED_MATCH"
|
||||
if ncm.get("status") == "SUGGESTION_ONLY":
|
||||
return "NCM_NAO_VERIFICADO", ncm.get("rule") or "NCM_NOT_VERIFIED"
|
||||
if state == "NOT_FOUND":
|
||||
return "SEM_MATCH_DE_PRODUTO", "GTIN_UNKNOWN"
|
||||
return "AGUARDANDO_EVIDENCIA", state or "INSUFFICIENT_EVIDENCE"
|
||||
|
||||
|
||||
def after_write(outcomes: list[dict], mode: str) -> tuple[str | None, str | None]:
|
||||
"""Refines the status with what the Agent reported (APPLIED / DRY_RUN / ABORTED / FAILED / HUMAN_OVERRIDDEN)."""
|
||||
kinds = {o.get("outcome") for o in outcomes}
|
||||
reasons = [o.get("reason") for o in outcomes if o.get("reason")]
|
||||
if "HUMAN_OVERRIDDEN" in kinds and "APPLIED" not in kinds:
|
||||
return "HUMAN_OVERRIDE", "HUMAN_OVERRIDDEN"
|
||||
if "APPLIED" in kinds:
|
||||
return "ENRIQUECIDO", None
|
||||
if "FAILED" in kinds:
|
||||
return ("VERIFICACAO_POS_WRITE_FALHOU" if "POST_WRITE_VERIFICATION_FAILED" in reasons else "WRITE_FAILED"), (reasons[0] if reasons else "FAILED")
|
||||
if "ABORTED" in kinds:
|
||||
return "WRITE_FAILED", (reasons[0] if reasons else "ABORTED")
|
||||
if "DRY_RUN" in kinds:
|
||||
return "SIMULADO", "DRY_RUN_NO_WRITE"
|
||||
return None, None
|
||||
|
||||
|
||||
def mark_for_reevaluation(session, installation_id: int, *, codes: list[str] | None = None, gtin: str | None = None) -> int:
|
||||
"""Flags tracked, NOT-yet-enriched products for a new evaluation (KB/cache first; providers only when really needed). Idempotent; never touches enriched / human-overridden ones."""
|
||||
from ..models import ProductEnrichment
|
||||
q = update(ProductEnrichment).where(ProductEnrichment.installation_id == installation_id, ProductEnrichment.ui_status.notin_(["ENRIQUECIDO", "HUMAN_OVERRIDE"]) | ProductEnrichment.ui_status.is_(None))
|
||||
if codes:
|
||||
q = q.where(ProductEnrichment.product_code.in_(codes))
|
||||
if gtin:
|
||||
q = q.where(ProductEnrichment.gtin == gtin)
|
||||
return session.execute(q.values(reevaluate=True)).rowcount or 0
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Deterministic product-description normalization and fingerprint (no ML). Used to FIND candidates; never authorizes a fiscal value on its own."""
|
||||
import hashlib
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
UNIT_MAP = {"LT": "L", "LTS": "L", "LITRO": "L", "LITROS": "L", "L": "L", "ML": "ML", "KG": "KG", "KGS": "KG", "QUILO": "KG", "G": "G", "GR": "G", "GRS": "G", "GRAMAS": "G", "MG": "MG", "UN": "UN", "UND": "UN", "UNID": "UN", "UNIDADE": "UN", "PCT": "PCT", "CX": "CX"}
|
||||
ABBREVIATIONS = {"REFRIG": "REFRIGERANTE", "REFRI": "REFRIGERANTE", "CHOC": "CHOCOLATE", "BISC": "BISCOITO", "SAB": "SABONETE", "DET": "DETERGENTE", "SH": "SHAMPOO", "COND": "CONDICIONADOR",
|
||||
"MACARR": "MACARRAO", "MAC": "MACARRAO", "AMAC": "AMACIANTE", "DESOD": "DESODORANTE", "PAP": "PAPEL", "HIG": "HIGIENICO", "ORIG": "ORIGINAL", "TRAD": "TRADICIONAL"}
|
||||
STOP = {"DE", "DA", "DO", "DAS", "DOS", "E", "COM", "C", "PARA", "P", "EM", "A", "O"}
|
||||
MARKETING = {"ORIGINAL", "TRADICIONAL", "NOVO", "NOVA", "PROMO", "PROMOCAO", "OFERTA"} # ignored ONLY by the fingerprint; a different pack/flavour still differs
|
||||
_QTY = re.compile(r"^(\d+(?:[.,]\d+)?)(ML|L|KG|G|MG|UN|PCT|CX|LT|LTS|GR)$")
|
||||
|
||||
|
||||
def _ascii_upper(text) -> str:
|
||||
return unicodedata.normalize("NFKD", str(text or "")).encode("ascii", "ignore").decode().upper()
|
||||
|
||||
|
||||
def tokens(description) -> list[str]:
|
||||
text = _ascii_upper(description)
|
||||
text = re.sub(r"(\d)([A-Z])", r"\1 \2", text) # 350ML -> 350 ML
|
||||
text = re.sub(r"([A-Z])(\d)", r"\1 \2", text)
|
||||
text = re.sub(r"(\d)[,.](\d)", r"\1.\2", text) # decimal comma -> dot (kept inside the number)
|
||||
text = re.sub(r"[^A-Z0-9.]+", " ", text)
|
||||
out = []
|
||||
for tok in text.split():
|
||||
tok = tok.strip(".")
|
||||
if not tok or tok in STOP:
|
||||
continue
|
||||
tok = UNIT_MAP.get(tok, ABBREVIATIONS.get(tok, tok))
|
||||
out.append(tok)
|
||||
return out
|
||||
|
||||
|
||||
def normalized(description) -> str:
|
||||
return " ".join(tokens(description))
|
||||
|
||||
|
||||
def fingerprint(description) -> str | None:
|
||||
toks = [t for t in tokens(description) if t not in MARKETING]
|
||||
if not toks:
|
||||
return None
|
||||
return hashlib.sha256(" ".join(sorted(toks)).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def is_specific(description) -> bool:
|
||||
"""A description strong enough to be used as an identity key: at least 3 informative tokens AND a quantity (number + unit), or >= 4 tokens.
|
||||
Generic names (PIPOCA, OLEO, REFRIGERANTE, SABONETE) are NEVER specific."""
|
||||
toks = tokens(description)
|
||||
has_qty = any(re.fullmatch(r"\d+(?:\.\d+)?", t) and i + 1 < len(toks) and toks[i + 1] in UNIT_MAP.values() for i, t in enumerate(toks))
|
||||
informative = [t for t in toks if not re.fullmatch(r"\d+(?:\.\d+)?", t) and t not in UNIT_MAP.values()]
|
||||
return (len(informative) >= 2 and has_qty and len(toks) >= 3) or len(informative) >= 4
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Secrets and package bundling for Agent distribution.
|
||||
|
||||
Two independent secrets guard the distribution flow, both random(256 bits),
|
||||
single-use and short-TTL, both stored only as SHA-256 hashes:
|
||||
|
||||
- the *download* token: embedded in the shareable link an operator hands to
|
||||
whoever installs the Agent (or clicked directly, authenticated, from the
|
||||
panel);
|
||||
- the *enrollment* token: embedded inside the downloaded bundle as
|
||||
`enrollment.json`, consumed exactly once by `POST /v1/agents/enroll`.
|
||||
|
||||
Neither token is ever the Agent's own permanent Bearer token (issued only
|
||||
after a successful enrollment) and neither is ever logged or audited in
|
||||
plaintext.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import secrets
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
class EnrollmentRejected(Exception):
|
||||
def __init__(self, reason: str):
|
||||
super().__init__(reason)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
def new_token() -> str:
|
||||
"""256 bits of randomness, URL-safe — usable directly in a link or a JSON field."""
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
# ---- installation code (XXXX-XXXX-XXXX-XXXX) ----
|
||||
#
|
||||
# A 16-character Crockford Base32 code = 16 * 5 = 80 bits of entropy. The
|
||||
# alphabet deliberately excludes I/L/O/U (ambiguous when retyped by a human);
|
||||
# the same alphabet is used so a code cannot contain those letters.
|
||||
|
||||
INSTALL_CODE_ALPHABET = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
|
||||
_INSTALL_CODE_GROUPS = 4
|
||||
_INSTALL_CODE_PER_GROUP = 4
|
||||
_INSTALL_CODE_LENGTH = _INSTALL_CODE_GROUPS * _INSTALL_CODE_PER_GROUP
|
||||
# Crockford transliteration for input normalization: O -> 0, I/L -> 1.
|
||||
_INSTALL_CODE_TRANSLIT = str.maketrans({"O": "0", "I": "1", "L": "1"})
|
||||
_INSTALL_CODE_RE = re.compile(
|
||||
r"^[" + re.escape(INSTALL_CODE_ALPHABET) + r"]{" + str(_INSTALL_CODE_LENGTH) + r"}$"
|
||||
)
|
||||
|
||||
|
||||
def generate_install_code() -> str:
|
||||
"""Return a fresh installation code formatted XXXX-XXXX-XXXX-XXXX."""
|
||||
chars = "".join(secrets.choice(INSTALL_CODE_ALPHABET) for _ in range(_INSTALL_CODE_LENGTH))
|
||||
return "-".join(
|
||||
chars[i : i + _INSTALL_CODE_PER_GROUP] for i in range(0, _INSTALL_CODE_LENGTH, _INSTALL_CODE_PER_GROUP)
|
||||
)
|
||||
|
||||
|
||||
def normalize_install_code(text: str | None) -> str:
|
||||
"""Upper-case, drop hyphen/space separators and map Crockford lookalikes,
|
||||
so a human-typed code normalizes to the same digest regardless of how it
|
||||
was entered. Raises ValueError if the result isn't a valid code."""
|
||||
if not text:
|
||||
raise ValueError("empty install code")
|
||||
clean = "".join(ch for ch in text.upper().translate(_INSTALL_CODE_TRANSLIT) if ch.isalnum())
|
||||
if not _INSTALL_CODE_RE.match(clean):
|
||||
raise ValueError(f"invalid install code: {text!r}")
|
||||
return clean
|
||||
|
||||
|
||||
def install_code_digest(code: str) -> str:
|
||||
"""Hash a normalized install code. Only the hash is ever persisted; the
|
||||
plaintext code exists only in the operator's clipboard/UI and the EXE's
|
||||
memory for the brief instant it is sent to the Hub."""
|
||||
return token_digest(normalize_install_code(code))
|
||||
|
||||
|
||||
def valid_install_code(text: str | None) -> bool:
|
||||
try:
|
||||
normalize_install_code(text)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def token_digest(token: str) -> str:
|
||||
return hashlib.sha256(token.encode()).hexdigest()
|
||||
|
||||
|
||||
def token_matches(token: str | None, stored_hash: str | None) -> bool:
|
||||
if not token or not stored_hash:
|
||||
return False
|
||||
return hmac.compare_digest(token_digest(token), stored_hash)
|
||||
|
||||
|
||||
ENROLLMENT_UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
|
||||
|
||||
|
||||
def valid_uuid(value: str | None) -> bool:
|
||||
return bool(value) and bool(ENROLLMENT_UUID_RE.match(value))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BundlePlan:
|
||||
hub_url: str
|
||||
enrollment_uuid: str
|
||||
enrollment_token: str
|
||||
enroll_path: str = "/v1/agents/enroll"
|
||||
|
||||
|
||||
def build_bundle(release_zip_bytes: bytes, plan: BundlePlan) -> bytes:
|
||||
"""Return a new zip = the release zip's entries plus one
|
||||
`<package-root>/enrollment.json` holding only temporary, single-use data.
|
||||
Never a permanent credential — that is issued only once enrollment succeeds."""
|
||||
with zipfile.ZipFile(io.BytesIO(release_zip_bytes)) as source:
|
||||
names = source.namelist()
|
||||
if not names:
|
||||
raise EnrollmentRejected("EMPTY_RELEASE_PACKAGE")
|
||||
root = names[0].split("/", 1)[0]
|
||||
out = io.BytesIO()
|
||||
with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as dest:
|
||||
for name in names:
|
||||
dest.writestr(name, source.read(name))
|
||||
payload = {
|
||||
"installation_id": plan.enrollment_uuid,
|
||||
"enrollment_token": plan.enrollment_token,
|
||||
"hub_url": plan.hub_url,
|
||||
"enroll_path": plan.enroll_path,
|
||||
}
|
||||
dest.writestr(f"{root}/enrollment.json", json.dumps(payload, indent=1))
|
||||
return out.getvalue()
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Structured operational events with defensive redaction and retention."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from .config import settings
|
||||
from .models import AgentEvent
|
||||
|
||||
EVENT_TYPES = {
|
||||
"AGENT_STARTED", "AGENT_STOPPED", "HEARTBEAT", "PREFLIGHT_PASS", "PREFLIGHT_FAIL", "JOB_CLAIMED", "JOB_STARTED",
|
||||
"JOB_COMPLETED", "JOB_FAILED", "PRODUCT_BLOCKED", "BLOCKER_RESOLVED", "DATASET_CHANGED", "COMPANY_CONTEXT_CHANGED",
|
||||
"SCHEMA_CHANGED", "WRITE_PLANNED", "WRITE_COMPLETED", "WRITE_FAILED", "CONNECTION_ERROR", "RETRY", "VERSION_CHANGED",
|
||||
"WINDOWS_INSTALL_COMPLETED", "WINDOWS_UNINSTALL_COMPLETED", "ADMIN_ACTION", "RTC_AUTO_APPLY", "CFOP_IPPT_AUTO_APPLY",
|
||||
}
|
||||
SEVERITIES = {"INFO", "WARNING", "ERROR", "CRITICAL"}
|
||||
SECRET_RE = re.compile(r"(?i)(password|fb_password|token|api[_-]?key|secret|authorization|connection[_-]?string)\s*[:=]\s*[^,;\s]+")
|
||||
|
||||
|
||||
def _redact(value):
|
||||
if isinstance(value, dict):
|
||||
return {str(key): "[REDACTED]" if str(key).lower() in {"password", "fb_password", "token", "api_key", "secret", "authorization", "connection_string"} else _redact(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_redact(item) for item in value[:100]]
|
||||
if isinstance(value, str):
|
||||
return SECRET_RE.sub(lambda match: f"{match.group(1)}=[REDACTED]", value)
|
||||
return value
|
||||
|
||||
|
||||
def sanitize_event(message: str, context: dict | None = None) -> tuple[str, dict]:
|
||||
safe_message = _redact(str(message or ""))[: settings.web_max_event_message]
|
||||
safe_context = _redact(context or {})
|
||||
encoded = json.dumps(safe_context, ensure_ascii=False, default=str)
|
||||
if len(encoded.encode()) > settings.web_max_event_context_bytes:
|
||||
encoded = encoded.encode()[: settings.web_max_event_context_bytes].decode("utf-8", errors="ignore")
|
||||
safe_context = {"truncated": True, "payload": encoded}
|
||||
return safe_message, safe_context
|
||||
|
||||
|
||||
def record_event(session, *, tenant_id: int, installation_id: int, agent_id: int, severity: str, event_type: str, message: str, context: dict | None = None, job_id: int | None = None, product_code: str | None = None, blocker_id: int | None = None, is_audit: bool = False, occurred_at: datetime | None = None):
|
||||
severity = severity.upper()
|
||||
event_type = event_type.upper()
|
||||
if severity not in SEVERITIES:
|
||||
raise ValueError("invalid event severity")
|
||||
if event_type not in EVENT_TYPES:
|
||||
raise ValueError("invalid event type")
|
||||
safe_message, safe_context = sanitize_event(message, context)
|
||||
event = AgentEvent(tenant_id=tenant_id, installation_id=installation_id, agent_id=agent_id, severity=severity, event_type=event_type, message=safe_message, structured_context=safe_context, job_id=job_id, product_code=product_code, blocker_id=blocker_id, is_audit=is_audit, occurred_at=occurred_at or datetime.now(timezone.utc))
|
||||
session.add(event)
|
||||
return event
|
||||
|
||||
|
||||
def purge_expired_events(session, now: datetime | None = None, retention_days: int | None = None) -> int:
|
||||
cutoff = (now or datetime.now(timezone.utc)) - timedelta(days=retention_days if retention_days is not None else settings.agent_event_retention_days)
|
||||
result = session.execute(delete(AgentEvent).where(AgentEvent.occurred_at < cutoff, AgentEvent.is_audit.is_(False), AgentEvent.severity != "CRITICAL"))
|
||||
session.commit()
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
def list_events(session, installation_db_id: int, *, page: int = 1, page_size: int = 50, severity: str | None = None, event_type: str | None = None, job_id: int | None = None, product_code: str | None = None, since: datetime | None = None, until: datetime | None = None, fetch_extra: bool = False):
|
||||
page = max(1, page); page_size = min(100, max(1, page_size))
|
||||
query = select(AgentEvent).where(AgentEvent.installation_id == installation_db_id)
|
||||
for column, value in ((AgentEvent.severity, severity), (AgentEvent.event_type, event_type), (AgentEvent.job_id, job_id), (AgentEvent.product_code, product_code)):
|
||||
if value is not None:
|
||||
query = query.where(column == value)
|
||||
if since: query = query.where(AgentEvent.occurred_at >= since)
|
||||
if until: query = query.where(AgentEvent.occurred_at <= until)
|
||||
query = query.order_by(AgentEvent.occurred_at.desc(), AgentEvent.id.desc())
|
||||
rows = session.scalars(query.offset((page - 1) * page_size).limit(page_size + int(fetch_extra))).all()
|
||||
return rows
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Conservative aggregation of supplier/document observations."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from collections import defaultdict
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Observation:
|
||||
product_id: str
|
||||
attribute: str
|
||||
value: str
|
||||
supplier_id: str
|
||||
document_id: str
|
||||
observed_at: str
|
||||
provenance: str = "SUPPLIER_OBSERVATION"
|
||||
|
||||
|
||||
def aggregate_observations(observations: Iterable[Observation]) -> dict[tuple[str, str], dict[str, Any]]:
|
||||
grouped: dict[tuple[str, str], list[Observation]] = defaultdict(list)
|
||||
seen: set[tuple[str, str, str, str]] = set()
|
||||
for observation in observations:
|
||||
key = (observation.product_id, observation.attribute)
|
||||
dedupe = (observation.product_id, observation.attribute, observation.supplier_id, observation.document_id)
|
||||
if dedupe not in seen and observation.value not in (None, ""):
|
||||
grouped[key].append(observation)
|
||||
seen.add(dedupe)
|
||||
|
||||
output = {}
|
||||
for key, rows in grouped.items():
|
||||
values = {row.value for row in rows}
|
||||
counts = {value: sum(row.value == value for row in rows) for value in values}
|
||||
supplier_count = len({row.supplier_id for row in rows})
|
||||
agreement_rate = max(counts.values()) / len(rows) if rows else 0.0
|
||||
conflict_count = max(0, len(values) - 1)
|
||||
strong = len(rows) >= 2 and supplier_count >= 2 and len(values) == 1
|
||||
output[key] = {
|
||||
"value": next(iter(values)) if strong else None,
|
||||
"observation_count": len(rows),
|
||||
"supplier_count": supplier_count,
|
||||
"agreement_rate": agreement_rate,
|
||||
"latest_observation": max(row.observed_at for row in rows),
|
||||
"conflict_count": conflict_count,
|
||||
"strong_evidence": strong,
|
||||
"provenance": sorted({row.provenance for row in rows}),
|
||||
}
|
||||
return output
|
||||
@@ -0,0 +1,636 @@
|
||||
"""Fiscal Field Contract V1: one contract + one resolver for EVERY persistent fiscal field of the SGV product screen (C000025).
|
||||
|
||||
Pure functions. Every (product, field) ends in exactly one of:
|
||||
ALREADY_CORRECT | CHANGE_REQUIRED | NOT_APPLICABLE | MANUAL_INPUT_REQUIRED(<input_key>) (CORRECTED is set by the writer after a verified write)
|
||||
Nothing is invented: a value is proposed only when an official dataset, the company context, ERP evidence or an OPERATOR context entry proves it.
|
||||
A valid ERP declaration that no proven evidence contradicts is ALREADY_CORRECT (reason DECLARED_VALID_NOT_CONTRADICTED); the report keeps
|
||||
those apart from rule-proven results.
|
||||
|
||||
Operator context (Hub `fiscal_context_entries`) is hierarchical: product > product group (CODGRUPO) > company.
|
||||
"""
|
||||
from .fiscal_decision import _ex_described # noqa: F401 (description evidence helper shared with the decision engine)
|
||||
|
||||
CONTRACT_VERSION = "field-contract-v1"
|
||||
ALREADY, CHANGE, CORRECTED, NA, MANUAL = "ALREADY_CORRECT", "CHANGE_REQUIRED", "CORRECTED", "NOT_APPLICABLE", "MANUAL_INPUT_REQUIRED"
|
||||
|
||||
WRITE_FIELDS = ["CLASSIFICACAO_FISCAL", "IPI", "EX_TIPI", "CEST", "SITUACAO_TRIBUTARIA", "CFOP", "CFOP_ENTRADA", "CST", "CST_ENTRADA", "CSOSN", "ALIQUOTA", "COD_ORIGEM_MERCADORIA",
|
||||
"CST_IPI", "COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS", "INCIDENCIA_PISCOFINS", "PISCOFINS", "IAT", "IPPT"]
|
||||
|
||||
# ---- contract metadata (docs/FISCAL_FIELD_CONTRACT_V1.json is generated from this) ------------------------------------------------------------------
|
||||
def _c(ui, sem, dtype, source, inputs, rule, valid, na, manual, validation, prov, write=True):
|
||||
return {"UI_LABEL": ui, "SEMANTICS": sem, "DATA_TYPE": dtype, "SOURCE_OF_TRUTH": source, "REQUIRED_INPUTS": inputs, "RESOLUTION_RULE": rule, "VALID_VALUES": valid,
|
||||
"NOT_APPLICABLE_RULE": na, "MANUAL_INPUT_RULE": manual, "WRITE_ALLOWED": write, "VALIDATION_RULE": validation, "PROVENANCE": prov}
|
||||
|
||||
|
||||
CONTRACT = {
|
||||
"CLASSIFICACAO_FISCAL": _c("NCM", "NCM canônico (8 dígitos)", "VARCHAR(20)", "dataset NCM (Siscomex) ACTIVE + legacy NCM table", ["current NCM"],
|
||||
"VALID -> ALREADY_CORRECT. LEGACY/INVALID/MISSING never auto-replaced from the description; an OPERATOR context `ncm` that the Hub validates as EXACT_NCM becomes CHANGE_REQUIRED and every dependent field is reprocessed with it",
|
||||
"8-digit code present in the ACTIVE NCM dataset", "never (every product has an NCM)", "MANUAL_INPUT_REQUIRED(ncm) for LEGACY/INVALID/MISSING or an invalid manual value", "Hub classify status == EXACT_NCM", "NCM dataset sha256 + engine version"),
|
||||
"IPI": _c("Alíquota IPI %", "alíquota numérica de IPI", "DOUBLE PRECISION", "dataset TIPI_IPI ACTIVE", ["valid NCM", "TIPI row (and EX choice when several rows)"],
|
||||
"TIPI numeric rate -> ALREADY_CORRECT if equal else CHANGE_REQUIRED. NT/non-numeric and blank rates never write (NOT_APPLICABLE, never zero)", "float >= 0", "TIPI rate is NT/non-numeric or blank", "MANUAL_INPUT_REQUIRED(ex_tipi) when several TIPI rows exist for the NCM", "numeric, from TIPI row", "TIPI dataset sha256"),
|
||||
"EX_TIPI": _c("EX TIPI", "código EX da TIPI", "VARCHAR(3)", "dataset TIPI_IPI + product description evidence", ["valid NCM", "TIPI EX rows"],
|
||||
"EX applies only when the product description carries the official EX description; otherwise the base rate applies (no EX)", "EX code of an existing TIPI EX row", "NCM has no EX row, or the description evidences none",
|
||||
"MANUAL_INPUT_REQUIRED(ex_tipi) when the ERP holds an EX that no TIPI row supports", "value in TIPI EX rows for the NCM", "TIPI dataset sha256"),
|
||||
"CEST": _c("CEST", "CEST (7 dígitos sem pontuação)", "VARCHAR(7)", "dataset CEST (CONFAZ 142/18) ACTIVE + BA_ST_MVA", ["valid NCM", "description"],
|
||||
"single defensible official CEST -> ALREADY/CHANGE; ERP CEST that is one of the official codes of the NCM -> ALREADY_CORRECT; several official candidates with a blank CEST -> MANUAL; NCM absent from the CEST list -> NOT_APPLICABLE",
|
||||
"official CEST code of the NCM", "NCM not in the CEST list, or the description matches no official CEST item", "MANUAL_INPUT_REQUIRED(cest) when several official candidates exist, or the ERP CEST is not official for the NCM", "digits equal to an official CEST of the NCM", "CEST dataset sha256"),
|
||||
"SITUACAO_TRIBUTARIA": _c("Seletor ao lado do CEST", "T=Tributado, I=Isento, N=Não tributado, S=Substituição Tributária", "VARCHAR(2)", "ST engine (BA Anexo 1 + CEST corroboration) + purchase evidence / operator st_withheld", ["ST status", "ST withheld"],
|
||||
"ST proven and withheld -> S. Otherwise a valid declaration stays", "T | I | N | S", "never", "MANUAL_INPUT_REQUIRED(st_withheld) when ST is proven but retention unknown; (icms_treatment) when the field is blank", "value in T/I/N/S", "BA_ST_MVA + CEST dataset sha256"),
|
||||
"CFOP": _c("CFOP (Nota Fiscal de Saída)", "CFOP padrão de saída (intra-UF)", "VARCHAR(4)", "dataset CFOP oficial + product role + ST", ["product_role", "ST status", "ST withheld"],
|
||||
"RESALE: non-ST 5102, ST withheld 5405; OWN_PRODUCTION: 5101 / 5401; a declared official 5xxx that is not contradicted by proven role/ST stays", "official CFOP starting with 5", "INPUT (not a sale item)", "MANUAL_INPUT_REQUIRED(product_role | st_status | st_withheld | service_cfop)", "code in CFOP dataset", "CFOP dataset sha256"),
|
||||
"CFOP_ENTRADA": _c("CFOP (Nota Fiscal de Entrada)", "CFOP padrão de entrada", "CHAR(4)", "dataset CFOP + product role + ST + company entry scope", ["product_role", "ST withheld", "entry_scope (company)"],
|
||||
"RESALE 1102/2102, ST withheld 1403/2403, OWN_PRODUCTION/INPUT 1101/2101 (ST 1401/2401); declared official 1xxx/2xxx that is not contradicted stays", "official CFOP starting with 1 or 2", "SERVICE", "MANUAL_INPUT_REQUIRED(product_role | entry_scope | st_withheld)", "code in CFOP dataset", "CFOP dataset sha256"),
|
||||
"CST": _c("CST de ICMS (Nota Fiscal de Saída)", "CST ICMS = origem(1)+CST(2), regime normal", "VARCHAR(10)", "MOC CST_ICMS + ORIGEM tables + icms_treatment", ["CRT 2/3", "icms_treatment", "origin"],
|
||||
"CRT 1 uses CSOSN. CRT 2/3: treatment TAXED 00 / EXEMPT 40 / NOT_TAXED 41 / ST_WITHHELD 60; a valid declaration that the treatment does not contradict stays", "3 digits: origin 0-8 + CST_ICMS code", "CRT = 1", "MANUAL_INPUT_REQUIRED(icms_treatment | origin) when blank and not derivable", "origin in ORIGEM table, last two digits in CST_ICMS", "MOC tables sha256"),
|
||||
"CST_ENTRADA": _c("CST de ICMS (Nota Fiscal de Entrada)", "CST ICMS de entrada, regime normal", "CHAR(4)", "MOC tables + icms_treatment", ["CRT 2/3", "icms_treatment", "origin"],
|
||||
"same derivation as CST for CRT 2/3", "3 digits origin+CST", "CRT = 1", "MANUAL_INPUT_REQUIRED(icms_treatment | origin)", "as CST", "MOC tables sha256"),
|
||||
"CSOSN": _c("CSOSN Classificação", "CSOSN de saída (Simples Nacional)", "VARCHAR(60)", "MOC CSOSN table + icms_treatment (operator or proven ST)", ["CRT 1", "icms_treatment"],
|
||||
"treatment TAXED -> 101/102/103 family (default 102), EXEMPT/NOT_TAXED -> 300/400 (default 400), ST_WITHHELD -> 500; a valid declaration outside a proven contradiction stays", "CSOSN code in MOC table", "CRT != 1", "MANUAL_INPUT_REQUIRED(icms_treatment) when blank/invalid", "code in CSOSN table", "MOC tables sha256"),
|
||||
"ALIQUOTA": _c("Aliq. ICMS", "alíquota ICMS % de destaque", "NUMERIC(15,2)", "regime + CSOSN/CST family + operator icms_rate", ["CSOSN or CST family", "icms_rate (company/group) for taxed regime-normal items"],
|
||||
"no ICMS highlighted (CSOSN 102/103/300/400/500/202/203/900 or CST 40/41/60/50/51) -> NOT_APPLICABLE; taxed items keep a declared value or take the operator icms_rate", "numeric >= 0", "no ICMS is highlighted for the item", "MANUAL_INPUT_REQUIRED(icms_rate | credit_rate)", "numeric", "operator context / company regime"),
|
||||
"COD_ORIGEM_MERCADORIA": _c("Código da origem da Mercadoria (0/1/2)", "origem da mercadoria (0 Nacional, 1 Estrangeira-importação direta, 2 Estrangeira-mercado interno)", "INTEGER", "operator origin only (no ERP field carries origin evidence)", ["origin"],
|
||||
"declared 0/1/2 stays; blank needs operator/evidence: the CST first digit is padding in the SGV (audit V2) and is NOT origin evidence; never defaulted to 0", "0 | 1 | 2", "never", "MANUAL_INPUT_REQUIRED(origin)", "value in {0,1,2}", "SGV DFM: hidden edit CodOrigemMercadoria + RgOrigemMercadoria.OnClick"),
|
||||
"CST_IPI": _c("CST IPI", "CST do IPI", "CHAR(3)", "company C000004.CONTRIBUINTE_IPI", ["company IPI taxpayer flag"],
|
||||
"company not an IPI taxpayer (CONTRIBUINTE_IPI != S) -> NOT_APPLICABLE; taxpayer -> declared value stays", "IPI CST code", "company is not an IPI taxpayer", "MANUAL_INPUT_REQUIRED(cst_ipi) for IPI taxpayers with a blank value", "non-blank code", "C000004.CONTRIBUINTE_IPI"),
|
||||
"COD_ENTRADA_PIS": _c("Pis Entrada", "CST PIS de entrada", "VARCHAR(3)", "MOC CST_PIS + operator pis_cofins_profile", ["pis_cofins_profile"], "profile in_cst (validated against CST_PIS) -> ALREADY/CHANGE; else a valid declared code stays", "CST PIS code", "never", "MANUAL_INPUT_REQUIRED(pis_cofins_profile)", "code in CST_PIS table", "MOC tables sha256"),
|
||||
"COD_SAIDA_PIS": _c("Pis Saída", "CST PIS de saída", "VARCHAR(3)", "MOC CST_PIS + operator pis_cofins_profile", ["pis_cofins_profile"], "profile out_cst -> ALREADY/CHANGE; else a valid declared code stays", "CST PIS code", "never", "MANUAL_INPUT_REQUIRED(pis_cofins_profile)", "code in CST_PIS table", "MOC tables sha256"),
|
||||
"ALIQ_PIS": _c("Alíquota (Pis)", "alíquota PIS %", "VARCHAR(8)", "operator pis_cofins_profile", ["pis_cofins_profile"], "profile pis_rate -> ALREADY/CHANGE; else a declared value stays (never assumed 0)", "decimal text", "never", "MANUAL_INPUT_REQUIRED(pis_cofins_profile)", "numeric text", "operator context"),
|
||||
"COD_ENTRADA_COFINS": _c("Cofins Entrada", "CST COFINS de entrada", "VARCHAR(3)", "MOC CST_COFINS + operator pis_cofins_profile", ["pis_cofins_profile"], "profile in_cst -> ALREADY/CHANGE; else a valid declared code stays", "CST COFINS code", "never", "MANUAL_INPUT_REQUIRED(pis_cofins_profile)", "code in CST_COFINS table", "MOC tables sha256"),
|
||||
"COD_SAIDA_COFINS": _c("Cofins Saída", "CST COFINS de saída", "VARCHAR(3)", "MOC CST_COFINS + operator pis_cofins_profile", ["pis_cofins_profile"], "profile out_cst -> ALREADY/CHANGE; else a valid declared code stays", "CST COFINS code", "never", "MANUAL_INPUT_REQUIRED(pis_cofins_profile)", "code in CST_COFINS table", "MOC tables sha256"),
|
||||
"ALIQ_COFINS": _c("Alíquota (Cofins)", "alíquota COFINS %", "VARCHAR(8)", "operator pis_cofins_profile", ["pis_cofins_profile"], "profile cofins_rate -> ALREADY/CHANGE; else a declared value stays (never assumed 0)", "decimal text", "never", "MANUAL_INPUT_REQUIRED(pis_cofins_profile)", "numeric text", "operator context"),
|
||||
"INCIDENCIA_PISCOFINS": _c("Incidência", "tipo de incidência PIS/COFINS", "VARCHAR(30)", "ERP declaration (NENHUMA / MONOFÁSICA / SUBST.TRIBUTÁRIA) + operator incidence", ["incidence (group) when blank"], "valid declaration stays; operator `incidence` may set it", "NENHUMA | MONOFÁSICA | SUBST.TRIBUTÁRIA", "never", "MANUAL_INPUT_REQUIRED(incidence)", "value in domain", "ERP domain observed in 4 databases"),
|
||||
"PISCOFINS": _c("PIS/COFINS (combo)", "flag PIS/COFINS N/S", "VARCHAR(1)", "ERP declaration + operator piscofins", ["piscofins when blank"], "valid N/S stays; operator `piscofins` may set it", "N | S", "never", "MANUAL_INPUT_REQUIRED(piscofins)", "value in N/S", "ERP domain observed in 4 databases"),
|
||||
"IAT": _c("IAT", "indicador de arredondamento (A) ou truncamento (T)", "VARCHAR(1)", "company rounding policy (operator) or ERP declaration", ["rounding (company) when blank"], "valid A/T stays; blank -> operator `rounding`", "A | T", "never", "MANUAL_INPUT_REQUIRED(rounding)", "value in A/T", "PAF-ECF/Convênio registro de produto layout (IAT)"),
|
||||
"IPPT": _c("IPPT", "indicador de produção própria (P) ou de terceiros (T)", "VARCHAR(1)", "product_role (own production evidence / purchase evidence / operator)", ["product_role"],
|
||||
"OWN_PRODUCTION -> P, RESALE -> T; INPUT/SERVICE not applicable", "P | T", "INPUT or SERVICE", "MANUAL_INPUT_REQUIRED(product_role)", "value in P/T", "PAF-ECF/Convênio registro de produto layout (IPPT)"),
|
||||
}
|
||||
for _f, _v in CONTRACT.items():
|
||||
_v["ERP_FIELD"] = _f
|
||||
|
||||
RTC_CAPABILITY = {"RTC_CCLASSTRIB": "ERP_SCHEMA_NOT_READY", "RTC_CCREDPRES": "ERP_SCHEMA_NOT_READY", "RTC_CBS_RATES": "ERP_SCHEMA_NOT_READY"}
|
||||
|
||||
ORIGIN_UI = {"0", "1", "2"}
|
||||
CSOSN_FAMILY = {"TAXED": ("102", {"101", "102", "103"}), "EXEMPT": ("400", {"300", "400"}), "NOT_TAXED": ("400", {"300", "400"}), "ST_WITHHELD": ("500", {"500"}), "ST_CHARGED": ("202", {"201", "202", "203"})}
|
||||
CST_TREATMENT = {"TAXED": "00", "EXEMPT": "40", "NOT_TAXED": "41", "ST_WITHHELD": "60", "SUSPENDED": "50", "DEFERRED": "51"}
|
||||
NO_ICMS_CSOSN = {"102", "103", "300", "400", "500", "202", "203", "900"}
|
||||
NO_ICMS_CST = {"40", "41", "60", "50", "51"}
|
||||
RESALE_CFOP = {"5101", "5102", "5401", "5405"}
|
||||
ENTRY_CFOP = {"1101", "1102", "1401", "1403", "2101", "2102", "2401", "2403"}
|
||||
|
||||
|
||||
class Context:
|
||||
"""Operator context lookup: product > group > company."""
|
||||
|
||||
def __init__(self, entries=()):
|
||||
self._by = {}
|
||||
for e in entries:
|
||||
self._by[(e["scope"].upper(), str(e.get("scope_key") or ""), e["key"])] = e["value"]
|
||||
|
||||
def get(self, key, code=None, group=None, ncm=None):
|
||||
for scope, sk in (("PRODUCT", code), ("GROUP", group), ("NCM", ncm), ("COMPANY", "")):
|
||||
if scope != "COMPANY" and not sk:
|
||||
continue
|
||||
v = self._by.get((scope, str(sk or ""), key))
|
||||
if v is not None and str(v).strip() != "":
|
||||
return str(v).strip(), scope
|
||||
return None, None
|
||||
|
||||
|
||||
def _s(v):
|
||||
return None if v is None else str(v).strip()
|
||||
|
||||
|
||||
def _blank(v):
|
||||
return v is None or str(v).strip() == ""
|
||||
|
||||
|
||||
def _digits(v):
|
||||
return "".join(c for c in str(v or "") if c.isdigit())
|
||||
|
||||
|
||||
def _default_legal_basis(field, value):
|
||||
contract = CONTRACT.get(field, {})
|
||||
src = contract.get("SOURCE_OF_TRUTH", "")
|
||||
prov = contract.get("PROVENANCE", "")
|
||||
val_str = str(value or "")
|
||||
if field == "CLASSIFICACAO_FISCAL":
|
||||
return {"dataset": "NCM", "version": "Siscomex", "line": val_str, "act": "Resolução Gecex / MDIC"}
|
||||
elif field in ("IPI", "EX_TIPI"):
|
||||
return {"dataset": "TIPI_IPI", "version": "tipi-xlsx-v1", "line": val_str, "act": "Decreto nº 11.158/2022"}
|
||||
elif field == "CEST":
|
||||
return {"dataset": "CEST", "version": "CV142_18", "line": val_str, "act": "Convênio ICMS 142/18"}
|
||||
elif "CFOP" in field:
|
||||
return {"dataset": "CFOP", "version": "cfop-xlsx-v1", "line": val_str, "act": "Informe Técnico 2023.002"}
|
||||
elif field in ("CST", "CST_ENTRADA", "CSOSN", "CST_IPI", "COD_ENTRADA_PIS", "COD_SAIDA_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "COD_ORIGEM_MERCADORIA"):
|
||||
return {"dataset": "NFE_MOC_TABLES", "version": "moc7-tables-v1", "line": val_str, "act": "MOC 7.0 Anexo I"}
|
||||
return {"dataset": "ERP_EVIDENCE", "version": "C000025", "line": val_str, "act": prov or src}
|
||||
|
||||
|
||||
def _r(field, state, current, proposed=None, reason="", key=None, scope=None, legal_basis=None, **extra):
|
||||
basis = legal_basis if legal_basis is not None else _default_legal_basis(field, proposed if proposed is not None else current)
|
||||
return {"field": field, "state": state, "current": current, "proposed": proposed, "reason": reason, "input_key": key, "input_scope": scope,
|
||||
"write_eligible": state == CHANGE and CONTRACT[field]["WRITE_ALLOWED"], "legal_basis": basis, **extra}
|
||||
|
||||
|
||||
# C000032.TIPO=COMPOSICAO evidence must be a deterministic MAJORITY of that product's own movement
|
||||
# history to count as own-production proof, never a bare "it happened at least once" check (a beer
|
||||
# used once as a recipe component among 19 direct sales is not "own production"). Thresholds:
|
||||
# below MINORITY -> composed evidence is discarded entirely (too rare to mean anything); above
|
||||
# DOMINANT -> strong own-production evidence; in between (with enough events to be meaningful) ->
|
||||
# genuinely conflicting evidence, never guessed either way (Missão 4 / Bloco A.1).
|
||||
COMPOSED_MIN_TOTAL_EVENTS = 5
|
||||
COMPOSED_MINORITY_RATIO = 0.30
|
||||
COMPOSED_DOMINANT_RATIO = 0.70
|
||||
|
||||
|
||||
def _composed_signal(ev, code):
|
||||
"""None (no usable evidence), "DOMINANT", "MINORITY" or "MIXED", plus the ratio for the reason string."""
|
||||
info = (ev.get("composed_ratio") or {}).get(code)
|
||||
if not info:
|
||||
return (None, None)
|
||||
total = info.get("composed", 0) + info.get("other", 0)
|
||||
if total < COMPOSED_MIN_TOTAL_EVENTS:
|
||||
return (None, None)
|
||||
ratio = info["composed"] / total
|
||||
if ratio >= COMPOSED_DOMINANT_RATIO:
|
||||
return ("DOMINANT", ratio)
|
||||
if ratio <= COMPOSED_MINORITY_RATIO:
|
||||
return ("MINORITY", ratio)
|
||||
return ("MIXED", ratio)
|
||||
|
||||
|
||||
def infer_role(p, ev, ctx, code, group, ncm=None):
|
||||
v, scope = ctx.get("product_role", code, group, ncm)
|
||||
if v in ("RESALE", "OWN_PRODUCTION", "INPUT", "SERVICE"):
|
||||
return v, [f"OPERATOR({scope})"]
|
||||
if _s(p.get("PRODUTO_SERVICO")):
|
||||
return "SERVICE", ["C000025.PRODUTO_SERVICO"]
|
||||
if (_s(p.get("TIPO")) or "").upper() == "MATERIA-PRIMA":
|
||||
return "INPUT", ["C000025.TIPO=MATERIA-PRIMA"]
|
||||
composed_signal, composed_ratio = _composed_signal(ev, code)
|
||||
if composed_signal == "MIXED":
|
||||
return "UNKNOWN", [f"C000032.TIPO=COMPOSICAO_MIXED_EVIDENCE(ratio={composed_ratio:.1%})"]
|
||||
composed_dominant = composed_signal == "DOMINANT"
|
||||
own = [n for n, on in (("PROD_COZINHA=1", p.get("PROD_COZINHA") == 1), ("RECEITA_INGREDIENTES", code in ev.get("recipes", ())),
|
||||
("C000032.TIPO=COMPOSICAO", composed_dominant)) if on]
|
||||
docs = bool(ev.get("purchases", {}).get(code)) or ev.get("stock_entry", {}).get(code, False)
|
||||
stock = ev.get("stock_in", {}).get(code, False)
|
||||
if own and docs:
|
||||
return "UNKNOWN", own + ["ALSO_PURCHASE_DOCUMENTS_CONFLICT"]
|
||||
strong_own = [n for n in own if n != "PROD_COZINHA=1"]
|
||||
if strong_own:
|
||||
return "OWN_PRODUCTION", own
|
||||
if own: # audit V2: the kitchen-routing flag alone is not proof of own production
|
||||
return "UNKNOWN", own + ["PROD_COZINHA_ALONE_NOT_PRODUCTION_PROOF"]
|
||||
if (_s(p.get("TIPO")) or "").upper() == "VENDA" and (docs or stock):
|
||||
return "RESALE", ["C000025.TIPO=VENDA", "PURCHASE_DOCUMENT" if docs else "STOCK_ENTRIES(C000100.ENTRADAS>0)"]
|
||||
return "UNKNOWN", ["NO_OBJECTIVE_ROLE_EVIDENCE"]
|
||||
|
||||
|
||||
def _isnum(v):
|
||||
try:
|
||||
float(v)
|
||||
return True
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
_SPED_CACHE = {}
|
||||
|
||||
|
||||
def sped_pis_cofins_match(ncm):
|
||||
"""Active SPED EFD-Contribuicoes table 4.3.10/12/13/14/15 entry (explicit NCM prefix) covering the NCM -> list of (table, saida_cst, code, start). Empty when none."""
|
||||
if "map" not in _SPED_CACHE:
|
||||
import json as _j
|
||||
from pathlib import Path as _P
|
||||
m = {}
|
||||
f = _P(__file__).resolve().parents[2] / "docs" / "sources" / "SPED_PISCOFINS_TABLES_V3.json"
|
||||
if f.exists():
|
||||
for tab, t in _j.loads(f.read_text(encoding="utf-8"))["tables"].items():
|
||||
for e in t.get("entries", []):
|
||||
if e["end"] is None:
|
||||
for pref in e["ncm_prefixes"]:
|
||||
m.setdefault(pref, set()).add((tab, t["saida_cst"][-1], e["code"], e["start"]))
|
||||
_SPED_CACHE["map"] = m
|
||||
hits = set()
|
||||
for n in (4, 6, 8):
|
||||
hits |= _SPED_CACHE["map"].get(ncm[:n], set()) if len(ncm) >= n else set()
|
||||
return sorted(hits)
|
||||
|
||||
|
||||
def resolve_product(p, d, company, refs, ev, ctx, manual_ncm=None):
|
||||
"""p: product row (all contract columns + CODIGO, PRODUTO, CODGRUPO, TIPO, PROD_COZINHA, PRODUTO_SERVICO). d: Hub decision for the EFFECTIVE NCM
|
||||
(the manual NCM when one was supplied and validated). manual_ncm: (value, is_valid) or None. Returns {field: result}."""
|
||||
from .fiscal_decision import decide_product
|
||||
from .icms_context import st_status, withheld_status
|
||||
code, group = _s(p["CODIGO"]), _s(p.get("CODGRUPO"))
|
||||
NCM = _digits(p.get("CLASSIFICACAO_FISCAL"))
|
||||
crt, uf = company.get("crt"), company.get("uf")
|
||||
out = {}
|
||||
cur = {f: _s(p.get(f)) for f in WRITE_FIELDS}
|
||||
|
||||
# ---------------- NCM ----------------
|
||||
ncm_ok = d.get("status") == "EXACT_NCM"
|
||||
if manual_ncm is not None:
|
||||
val, valid = manual_ncm
|
||||
if not valid:
|
||||
out["CLASSIFICACAO_FISCAL"] = _r("CLASSIFICACAO_FISCAL", MANUAL, cur["CLASSIFICACAO_FISCAL"], reason="MANUAL_NCM_NOT_VALID_IN_NCM_DATASET", key="ncm", scope="PRODUCT")
|
||||
elif _digits(val) == _digits(cur["CLASSIFICACAO_FISCAL"]) and d.get("status") == "EXACT_NCM":
|
||||
out["CLASSIFICACAO_FISCAL"] = _r("CLASSIFICACAO_FISCAL", ALREADY, cur["CLASSIFICACAO_FISCAL"], reason="OPERATOR_NCM_EQUALS_CURRENT")
|
||||
else:
|
||||
out["CLASSIFICACAO_FISCAL"] = _r("CLASSIFICACAO_FISCAL", CHANGE, cur["CLASSIFICACAO_FISCAL"], _digits(val), reason="OPERATOR_NCM_VALIDATED_BY_NCM_DATASET")
|
||||
elif ncm_ok:
|
||||
out["CLASSIFICACAO_FISCAL"] = _r("CLASSIFICACAO_FISCAL", ALREADY, cur["CLASSIFICACAO_FISCAL"], reason="NCM_VALID_IN_DATASET",
|
||||
legal_basis={"dataset": "NCM", "version": refs.get("ncm_version") or "Siscomex", "line": cur["CLASSIFICACAO_FISCAL"], "act": "Resolução Gecex / MDIC"})
|
||||
else:
|
||||
st_val = d.get("status")
|
||||
if _blank(cur["CLASSIFICACAO_FISCAL"]):
|
||||
why = "NCM_MISSING"
|
||||
elif st_val == "PLACEHOLDER_NCM":
|
||||
why = "NCM_PLACEHOLDER"
|
||||
elif st_val == "INVALID_FORMAT_NCM":
|
||||
why = "NCM_INVALID_FORMAT"
|
||||
elif st_val in ("LEGACY_NCM", "EXPIRED_LEGACY_NCM"):
|
||||
why = "NCM_LEGACY_NO_UNAMBIGUOUS_SUCCESSOR"
|
||||
elif st_val == "OUT_OF_TABLE_NCM":
|
||||
why = "NCM_OUT_OF_OFFICIAL_TABLE"
|
||||
else:
|
||||
why = "NCM_INVALID"
|
||||
out["CLASSIFICACAO_FISCAL"] = _r("CLASSIFICACAO_FISCAL", MANUAL, cur["CLASSIFICACAO_FISCAL"], reason=why, key="ncm", scope="PRODUCT",
|
||||
legal_basis={"dataset": "NCM", "version": refs.get("ncm_version") or "Siscomex", "line": cur["CLASSIFICACAO_FISCAL"], "act": "Resolução Gecex / MDIC"})
|
||||
|
||||
def blocked(field):
|
||||
return _r(field, MANUAL, cur[field], reason="BLOCKED_BY_NCM", key="ncm", scope="PRODUCT")
|
||||
|
||||
eff_ncm = out["CLASSIFICACAO_FISCAL"]["proposed"] or cur["CLASSIFICACAO_FISCAL"]
|
||||
prod = {**p, "CLASSIFICACAO_FISCAL": eff_ncm}
|
||||
|
||||
# ---------------- context facts ----------------
|
||||
role, role_ev = infer_role(p, ev, ctx, code, group, NCM)
|
||||
st_ctx, st_scope = ctx.get("st_status", code, group)
|
||||
ba = (d.get("classification") or {}).get("ba_st_mva") or {}
|
||||
if st_ctx in ("SUBJECT", "NOT_SUBJECT"):
|
||||
st = st_ctx
|
||||
elif ncm_ok and uf == "BA":
|
||||
s, _ = st_status(d.get("status"), d.get("ba_st_mva_status"), d.get("cest_status"), cur["CEST"], ba.get("cest"), None)
|
||||
st = {"ST_CONFIRMED": "SUBJECT", "ST_NOT_APPLICABLE": "NOT_SUBJECT"}.get(s, "UNKNOWN")
|
||||
else:
|
||||
st = "UNKNOWN"
|
||||
wh_ctx, _ = ctx.get("st_withheld", code, group)
|
||||
wh_doc, _ = withheld_status(ev.get("purchases", {}).get(code, []))
|
||||
withheld = wh_ctx if wh_ctx in ("Y", "N") else ("Y" if wh_doc == "ST_ALREADY_WITHHELD_CONFIRMED" else None) # purchase items flagged N are weak evidence: kept as unknown
|
||||
treat, _ = ctx.get("icms_treatment", code, group)
|
||||
if not treat and st == "SUBJECT" and withheld == "Y":
|
||||
treat = "ST_WITHHELD"
|
||||
origin_ctx, _ = ctx.get("origin", code, group)
|
||||
|
||||
# ---------------- decision-engine backed fields ----------------
|
||||
if ncm_ok:
|
||||
rec = decide_product({**prod, "CODIGO": code, "IPI": p.get("IPI"), "EX_TIPI": p.get("EX_TIPI"), "CEST": p.get("CEST")}, d, {"uf": uf or "", "crt": crt or ""}, refs)
|
||||
ipi_field = "IPI"
|
||||
if not ncm_ok:
|
||||
out["IPI"], out["EX_TIPI"], out["CEST"] = blocked("IPI"), blocked("EX_TIPI"), blocked("CEST")
|
||||
else:
|
||||
# IPI
|
||||
ex_ctx, _ = ctx.get("ex_tipi", code, group, NCM)
|
||||
cands = (d.get("classification") or {}).get("ipi_candidates") or []
|
||||
chosen = None
|
||||
if ex_ctx:
|
||||
chosen = next((c for c in cands if (c.get("ex") is None if ex_ctx.upper() == "NONE" else str(c.get("ex")) == ex_ctx)), None)
|
||||
if chosen and chosen.get("rate") not in (None, ""):
|
||||
try:
|
||||
rate = float(chosen["rate"])
|
||||
except ValueError:
|
||||
rate = None
|
||||
if rate is not None:
|
||||
same = cur["IPI"] is not None and abs(float(cur["IPI"]) - rate) < 1e-9
|
||||
out["IPI"] = _r("IPI", ALREADY if same else CHANGE, cur["IPI"], None if same else rate, "OPERATOR_EX_SELECTED_TIPI_ROW")
|
||||
if "IPI" not in out:
|
||||
r = rec["IPI"]
|
||||
if r["status"] == "RULE_RESOLVED":
|
||||
out["IPI"] = _r("IPI", ALREADY if r["reason_code"].endswith("SAME_VALUE") else CHANGE, cur["IPI"], None if r["reason_code"].endswith("SAME_VALUE") else r["proposed_value"], "TIPI_NUMERIC_RATE")
|
||||
elif r["status"] == "RULE_CANDIDATE":
|
||||
out["IPI"] = _r("IPI", MANUAL, cur["IPI"], reason="SEVERAL_TIPI_ROWS_FOR_NCM", key="ex_tipi", scope="PRODUCT",
|
||||
choices=[{"ex": c.get("ex"), "rate": c.get("rate"), "description": c.get("description")} for c in cands])
|
||||
else:
|
||||
out["IPI"] = _r("IPI", NA, cur["IPI"], reason=("TIPI_RATE_BLANK_NEVER_ZERO" if "BLANK" in r["reason_code"] or "NO_DATA" in r["reason_code"] else "TIPI_NT_OR_NON_NUMERIC_NEVER_WRITTEN_TO_NUMERIC_FIELD"))
|
||||
# store_ipi_profile / contribuinte_ipi gate
|
||||
is_taxpayer = company.get("contribuinte_ipi") == "S"
|
||||
prof, _ = ctx.get("store_ipi_profile", code, group)
|
||||
if not is_taxpayer:
|
||||
if prof == "POPULATE_REFERENCE_RATE":
|
||||
pass
|
||||
elif prof == "LEAVE_NOT_APPLICABLE":
|
||||
out["IPI"] = _r("IPI", NA, cur["IPI"], reason="STORE_IPI_PROFILE_LEAVE_NOT_APPLICABLE",
|
||||
legal_basis={"dataset": "TIPI_IPI", "version": "N/A", "line": "N/A", "act": "RIPI / CF art. 153, IV (não contribuinte)"})
|
||||
elif prof is None and out["IPI"]["state"] == CHANGE:
|
||||
out["IPI"] = _r("IPI", MANUAL, cur["IPI"], reason="STORE_IPI_PROFILE_NOT_DECIDED", key="store_ipi_profile", scope="COMPANY")
|
||||
else:
|
||||
out["IPI"] = _r("IPI", NA, cur["IPI"], reason="COMPANY_IS_NOT_AN_IPI_TAXPAYER_C000004.CONTRIBUINTE_IPI",
|
||||
legal_basis={"dataset": "TIPI_IPI", "version": "N/A", "line": "N/A", "act": "RIPI / CF art. 153, IV (não contribuinte)"})
|
||||
elif out["IPI"]["state"] == CHANGE:
|
||||
if prof == "LEAVE_NOT_APPLICABLE":
|
||||
out["IPI"] = _r("IPI", NA, cur["IPI"], reason="STORE_IPI_PROFILE_LEAVE_NOT_APPLICABLE")
|
||||
# EX_TIPI
|
||||
ex_rows = [c for c in cands if c.get("ex")]
|
||||
if ex_ctx:
|
||||
if ex_ctx.upper() == "NONE":
|
||||
out["EX_TIPI"] = _r("EX_TIPI", NA, cur["EX_TIPI"], reason="OPERATOR_SAYS_NO_EX")
|
||||
elif any(str(c["ex"]) == ex_ctx for c in ex_rows):
|
||||
out["EX_TIPI"] = _r("EX_TIPI", ALREADY if cur["EX_TIPI"] == ex_ctx else CHANGE, cur["EX_TIPI"], None if cur["EX_TIPI"] == ex_ctx else ex_ctx, "OPERATOR_EX_VALID_IN_TIPI")
|
||||
else:
|
||||
out["EX_TIPI"] = _r("EX_TIPI", MANUAL, cur["EX_TIPI"], reason="OPERATOR_EX_NOT_IN_TIPI_ROWS", key="ex_tipi", scope="PRODUCT")
|
||||
else:
|
||||
r = rec["EX_TIPI"]
|
||||
if r["status"] == "RULE_RESOLVED":
|
||||
out["EX_TIPI"] = _r("EX_TIPI", ALREADY if r["reason_code"].endswith("SAME_VALUE") else CHANGE, cur["EX_TIPI"], None if r["reason_code"].endswith("SAME_VALUE") else r["proposed_value"], "TIPI_EX_DESCRIPTION_EVIDENCE")
|
||||
elif cur["EX_TIPI"] and not any(str(c["ex"]) == cur["EX_TIPI"] for c in ex_rows):
|
||||
out["EX_TIPI"] = _r("EX_TIPI", MANUAL, cur["EX_TIPI"], reason="ERP_EX_NOT_SUPPORTED_BY_TIPI", key="ex_tipi", scope="PRODUCT")
|
||||
elif cur["EX_TIPI"]:
|
||||
out["EX_TIPI"] = _r("EX_TIPI", ALREADY, cur["EX_TIPI"], reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
else:
|
||||
out["EX_TIPI"] = _r("EX_TIPI", NA, cur["EX_TIPI"], reason="NO_EX_ROW_FOR_NCM" if not ex_rows else "DESCRIPTION_EVIDENCES_NO_EX_BASE_RATE_APPLIES")
|
||||
# CEST
|
||||
cest_ctx, _ = ctx.get("cest", code, group, NCM)
|
||||
cls = d.get("classification") or {}
|
||||
official = {_digits(c["code"]): c for c in (cls.get("cest_candidates") or [])}
|
||||
cs = d.get("cest_status")
|
||||
cur_cest = _digits(cur["CEST"])
|
||||
if cest_ctx:
|
||||
if cest_ctx.upper() == "NONE":
|
||||
out["CEST"] = _r("CEST", NA, cur["CEST"], reason="OPERATOR_SAYS_NO_CEST")
|
||||
elif _digits(cest_ctx) in official:
|
||||
out["CEST"] = _r("CEST", ALREADY if cur_cest == _digits(cest_ctx) else CHANGE, cur["CEST"], None if cur_cest == _digits(cest_ctx) else _digits(cest_ctx), "OPERATOR_CEST_OFFICIAL_FOR_NCM")
|
||||
else:
|
||||
out["CEST"] = _r("CEST", MANUAL, cur["CEST"], reason="OPERATOR_CEST_NOT_OFFICIAL_FOR_NCM", key="cest", scope="PRODUCT")
|
||||
elif cs == "CEST_CONFIRMED":
|
||||
cests_mapped = cls.get("cest") or []
|
||||
if len(cests_mapped) <= 1:
|
||||
proposed = _digits(cests_mapped[0] if cests_mapped else (cls.get("cest") or [None])[0])
|
||||
out["CEST"] = _r("CEST", ALREADY if cur_cest == proposed else CHANGE, cur["CEST"], None if cur_cest == proposed else proposed, "SINGLE_DEFENSIBLE_OFFICIAL_CEST",
|
||||
legal_basis={"dataset": "CEST", "version": refs.get("cest", "CV142_18"), "line": proposed, "act": "Convênio ICMS 142/18"})
|
||||
else:
|
||||
out["CEST"] = _r("CEST", MANUAL, cur["CEST"], reason="SEVERAL_OFFICIAL_CEST_FOR_NCM", key="cest", scope="PRODUCT",
|
||||
choices=[{"code": v["code"], "description": v["official_description"], "score": v.get("description_score")} for v in official.values()],
|
||||
legal_basis={"dataset": "CEST", "version": refs.get("cest", "CV142_18"), "line": "MULTIPLE_CANDIDATES", "act": "Convênio ICMS 142/18"})
|
||||
elif cur_cest and cur_cest in official:
|
||||
out["CEST"] = _r("CEST", ALREADY, cur["CEST"], reason="ERP_CEST_IS_OFFICIAL_FOR_NCM")
|
||||
elif cs == "NO_CEST_MAPPING_FOUND":
|
||||
out["CEST"] = _r("CEST", NA, cur["CEST"], reason="NCM_NOT_IN_CEST_LIST") if not cur_cest else _r("CEST", MANUAL, cur["CEST"], reason="ERP_CEST_BUT_NCM_NOT_IN_CEST_LIST", key="cest", scope="PRODUCT")
|
||||
elif cs == "CEST_CANDIDATES" and not cur_cest:
|
||||
out["CEST"] = _r("CEST", MANUAL, cur["CEST"], reason="SEVERAL_OFFICIAL_CEST_FOR_NCM", key="cest", scope="PRODUCT",
|
||||
choices=[{"code": v["code"], "description": v["official_description"], "score": v.get("description_score")} for v in official.values()])
|
||||
elif cs == "CEST_DESCRIPTION_MISMATCH" and not cur_cest:
|
||||
out["CEST"] = _r("CEST", NA, cur["CEST"], reason="DESCRIPTION_MATCHES_NO_OFFICIAL_CEST_ITEM")
|
||||
elif cur_cest:
|
||||
out["CEST"] = _r("CEST", MANUAL, cur["CEST"], reason="ERP_CEST_NOT_OFFICIAL_FOR_NCM", key="cest", scope="PRODUCT")
|
||||
else:
|
||||
out["CEST"] = _r("CEST", NA, cur["CEST"], reason="NO_CEST_APPLICABLE")
|
||||
|
||||
# ---------------- SITUACAO_TRIBUTARIA ----------------
|
||||
sit = cur["SITUACAO_TRIBUTARIA"]
|
||||
valid_sit = sit in ("T", "I", "N", "S")
|
||||
if not ncm_ok:
|
||||
out["SITUACAO_TRIBUTARIA"] = blocked("SITUACAO_TRIBUTARIA")
|
||||
elif st == "SUBJECT" and withheld == "Y":
|
||||
out["SITUACAO_TRIBUTARIA"] = _r("SITUACAO_TRIBUTARIA", ALREADY if sit == "S" else CHANGE, sit, None if sit == "S" else "S", "ST_PROVEN_AND_WITHHELD")
|
||||
elif st == "SUBJECT" and withheld is None:
|
||||
out["SITUACAO_TRIBUTARIA"] = _r("SITUACAO_TRIBUTARIA", ALREADY, sit, reason="DECLARED_S_CONSISTENT_WITH_PROVEN_ST") if sit == "S" else _r("SITUACAO_TRIBUTARIA", MANUAL, sit, reason="ST_PROVEN_RETENTION_UNKNOWN", key="st_withheld", scope="GROUP")
|
||||
elif valid_sit and not (st == "NOT_SUBJECT" and sit == "S"):
|
||||
out["SITUACAO_TRIBUTARIA"] = _r("SITUACAO_TRIBUTARIA", ALREADY, sit, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
elif valid_sit:
|
||||
out["SITUACAO_TRIBUTARIA"] = _r("SITUACAO_TRIBUTARIA", MANUAL, sit, reason="DECLARED_S_BUT_NOT_IN_ANY_ST_LIST", key="icms_treatment", scope="GROUP")
|
||||
else:
|
||||
out["SITUACAO_TRIBUTARIA"] = _r("SITUACAO_TRIBUTARIA", MANUAL, sit, reason="BLANK_OR_INVALID", key="icms_treatment", scope="GROUP")
|
||||
|
||||
# ---------------- CFOP saída ----------------
|
||||
cfop, cfop_valid = cur["CFOP"], refs["cfop_valid"]
|
||||
if role == "INPUT":
|
||||
out["CFOP"] = _r("CFOP", NA, cfop, reason="INPUT_IS_NOT_A_SALE_ITEM")
|
||||
elif role == "SERVICE":
|
||||
out["CFOP"] = _r("CFOP", MANUAL, cfop, reason="SERVICE_CFOP_NEEDS_OPERATOR", key="service_cfop", scope="GROUP")
|
||||
elif not ncm_ok:
|
||||
out["CFOP"] = blocked("CFOP")
|
||||
else:
|
||||
expected = None
|
||||
if role in ("RESALE", "OWN_PRODUCTION"):
|
||||
base = "5102" if role == "RESALE" else "5101"
|
||||
if st == "NOT_SUBJECT":
|
||||
expected = base
|
||||
elif st == "SUBJECT" and withheld == "Y":
|
||||
expected = "5405" if role == "RESALE" else "5401"
|
||||
valid_cur = cfop in cfop_valid and cfop.startswith("5")
|
||||
erp_st_declared = cur["CSOSN"] in ("500", "201", "202", "203") or (cur["CST"] or "")[-2:] in ("60", "10", "70", "30")
|
||||
if valid_cur and expected and expected in ("5102", "5101") and cfop in ("5405", "5401") and erp_st_declared:
|
||||
out["CFOP"] = _r("CFOP", MANUAL, cfop, reason="OFFICIAL_ST_TABLE_CONFLICTS_WITH_ERP_ST_DECLARATION", key="st_status", scope="GROUP")
|
||||
elif valid_cur and expected and cfop != expected and cfop in RESALE_CFOP:
|
||||
out["CFOP"] = _r("CFOP", CHANGE, cfop, expected, "PROVEN_ROLE_AND_ST_CONTRADICT_DECLARED_CFOP")
|
||||
elif valid_cur and expected == cfop:
|
||||
out["CFOP"] = _r("CFOP", ALREADY, cfop, reason="RULE_PROVEN_EQUALS_DECLARED")
|
||||
elif valid_cur:
|
||||
out["CFOP"] = _r("CFOP", ALREADY, cfop, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
elif expected:
|
||||
out["CFOP"] = _r("CFOP", CHANGE, cfop, expected, "DERIVED_FROM_ROLE_AND_ST" + ("_REPLACING_INVALID" if cfop else "_FILLING_BLANK"))
|
||||
else:
|
||||
key = "product_role" if role == "UNKNOWN" else ("st_status" if st == "UNKNOWN" else "st_withheld")
|
||||
out["CFOP"] = _r("CFOP", MANUAL, cfop, reason="CFOP_NEEDS_" + key.upper(), key=key, scope="GROUP")
|
||||
|
||||
# ---------------- CFOP entrada ----------------
|
||||
cfe = cur["CFOP_ENTRADA"]
|
||||
scope_ctx, _ = ctx.get("entry_scope", code, group)
|
||||
if role == "SERVICE":
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", NA, cfe, reason="SERVICE_HAS_NO_GOODS_ENTRY")
|
||||
elif not ncm_ok:
|
||||
out["CFOP_ENTRADA"] = blocked("CFOP_ENTRADA")
|
||||
else:
|
||||
exp_e = None
|
||||
if scope_ctx == "MIXED_OR_DYNAMIC":
|
||||
# Scope answered, but no single static entry CFOP is fiscally correct for every
|
||||
# operation: the ERP default / each operation decides 1xxx vs 2xxx. Never invent one.
|
||||
valid_e = cfe in cfop_valid and cfe[:1] in ("1", "2")
|
||||
if valid_e:
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", ALREADY, cfe, reason="DECLARED_VALID_OPERATION_DEPENDENT_MIXED_SCOPE")
|
||||
else:
|
||||
# Audited ADMIN decision (entry_scope=MIXED_OR_DYNAMIC) already says no static CFOP_ENTRADA is
|
||||
# ever correct here: NA (never MANUAL) so this doesn't block product completeness and never
|
||||
# raises a spurious "answer entry_scope" question — the scope question was already answered.
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", NA, cfe, reason="CFOP_ENTRADA_OPERATION_DEPENDENT_MIXED_SCOPE")
|
||||
else:
|
||||
if scope_ctx in ("INTRA", "INTER") and role in ("RESALE", "OWN_PRODUCTION", "INPUT") and st != "UNKNOWN" and not (st == "SUBJECT" and withheld is None):
|
||||
first = "1" if scope_ctx == "INTRA" else "2"
|
||||
stw = st == "SUBJECT" and withheld == "Y"
|
||||
exp_e = first + ("403" if stw else "102") if role == "RESALE" else first + ("401" if stw else "101")
|
||||
valid_e = cfe in cfop_valid and cfe[:1] in ("1", "2")
|
||||
if valid_e and exp_e and cfe != exp_e and cfe in ENTRY_CFOP:
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", CHANGE, cfe, exp_e, "PROVEN_ROLE_ST_AND_SCOPE_CONTRADICT_DECLARED_CFOP_ENTRADA")
|
||||
elif valid_e:
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", ALREADY, cfe, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
elif exp_e:
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", CHANGE, cfe, exp_e, "DERIVED_FROM_ROLE_ST_AND_ENTRY_SCOPE")
|
||||
else:
|
||||
key = "product_role" if role == "UNKNOWN" else ("entry_scope" if scope_ctx not in ("INTRA", "INTER") else ("st_status" if st == "UNKNOWN" else "st_withheld"))
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", MANUAL, cfe, reason="CFOP_ENTRADA_NEEDS_" + key.upper(), key=key, scope="COMPANY" if key == "entry_scope" else "GROUP")
|
||||
|
||||
# ---------------- origin (needed by CST) ----------------
|
||||
cod_o = cur["COD_ORIGEM_MERCADORIA"]
|
||||
cst_digit = (cur["CST"] or "")[:1] if cur["CST"] and len(cur["CST"]) == 3 and (cur["CST"][:1] in "012345678") else None
|
||||
if origin_ctx in ORIGIN_UI:
|
||||
out["COD_ORIGEM_MERCADORIA"] = _r("COD_ORIGEM_MERCADORIA", ALREADY if cod_o == origin_ctx else CHANGE, cod_o, None if cod_o == origin_ctx else int(origin_ctx), "OPERATOR_ORIGIN")
|
||||
elif origin_ctx:
|
||||
out["COD_ORIGEM_MERCADORIA"] = _r("COD_ORIGEM_MERCADORIA", MANUAL, cod_o, reason="OPERATOR_ORIGIN_NOT_REPRESENTABLE_ON_SGV_SCREEN_0_1_2", key="origin", scope="GROUP")
|
||||
elif cod_o in ORIGIN_UI:
|
||||
out["COD_ORIGEM_MERCADORIA"] = _r("COD_ORIGEM_MERCADORIA", ALREADY, cod_o, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
else:
|
||||
# V2 audit: the CST first digit is padding in the SGV (declared-origin 1/2 products also carry CST digit 0), so it is NOT evidence of origin
|
||||
out["COD_ORIGEM_MERCADORIA"] = _r("COD_ORIGEM_MERCADORIA", MANUAL, cod_o, reason="ORIGIN_NO_EVIDENCE", key="origin", scope="GROUP")
|
||||
origin_digit = out["COD_ORIGEM_MERCADORIA"]["proposed"] if out["COD_ORIGEM_MERCADORIA"]["proposed"] is not None else (cod_o if cod_o in ORIGIN_UI else None)
|
||||
origin_digit = None if origin_digit is None else str(origin_digit)
|
||||
|
||||
# ---------------- CST / CST_ENTRADA (regime normal) ----------------
|
||||
for field in ("CST", "CST_ENTRADA"):
|
||||
v = cur[field]
|
||||
if crt == "1":
|
||||
out[field] = _r(field, NA, v, reason="CRT1_SIMPLES_EMISSION_USES_CSOSN")
|
||||
continue
|
||||
if crt not in ("2", "3"):
|
||||
out[field] = _r(field, MANUAL, v, reason="CRT_UNKNOWN", key="crt", scope="COMPANY")
|
||||
continue
|
||||
valid_c = bool(v) and len(v) == 3 and v[0] in "012345678" and v[1:] in refs["cst_icms"]
|
||||
exp = (origin_digit + CST_TREATMENT[treat]) if (treat in CST_TREATMENT and origin_digit) else None
|
||||
if valid_c and exp and v != exp and v[1:] != exp[1:]:
|
||||
out[field] = _r(field, CHANGE, v, exp, "TREATMENT_CONTRADICTS_DECLARED_CST")
|
||||
elif valid_c:
|
||||
out[field] = _r(field, ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
elif exp:
|
||||
out[field] = _r(field, CHANGE, v, exp, "DERIVED_FROM_TREATMENT_AND_ORIGIN")
|
||||
else:
|
||||
out[field] = _r(field, MANUAL, v, reason="CST_NEEDS_ICMS_TREATMENT" if origin_digit else "CST_NEEDS_ORIGIN", key="icms_treatment" if origin_digit else "origin", scope="GROUP")
|
||||
|
||||
# ---------------- CSOSN ----------------
|
||||
v = cur["CSOSN"]
|
||||
if crt != "1":
|
||||
out["CSOSN"] = _r("CSOSN", NA, v, reason="CSOSN_IS_THE_SIMPLES_NACIONAL_CODE_CRT_IS_" + str(crt))
|
||||
elif not ncm_ok:
|
||||
out["CSOSN"] = blocked("CSOSN")
|
||||
else:
|
||||
fam = CSOSN_FAMILY.get(treat)
|
||||
if v in refs["csosn"] and (not fam or v in fam[1]):
|
||||
out["CSOSN"] = _r("CSOSN", ALREADY, v, reason="RULE_PROVEN_EQUALS_DECLARED" if fam else "DECLARED_VALID_NOT_CONTRADICTED")
|
||||
elif v in refs["csosn"] and fam:
|
||||
out["CSOSN"] = _r("CSOSN", CHANGE, v, fam[0], "TREATMENT_CONTRADICTS_DECLARED_CSOSN")
|
||||
elif fam:
|
||||
out["CSOSN"] = _r("CSOSN", CHANGE, v, fam[0], "DERIVED_FROM_TREATMENT" + ("_REPLACING_INVALID" if v else "_FILLING_BLANK"))
|
||||
else:
|
||||
out["CSOSN"] = _r("CSOSN", MANUAL, v, reason="CSOSN_NEEDS_ICMS_TREATMENT", key="icms_treatment", scope="GROUP")
|
||||
|
||||
# ---------------- ALIQUOTA ----------------
|
||||
v = cur["ALIQUOTA"]
|
||||
csosn_now = out["CSOSN"]["proposed"] or cur["CSOSN"]
|
||||
cst_now = out["CST"]["proposed"] or cur["CST"]
|
||||
rate_ctx, _ = ctx.get("icms_rate", code, group)
|
||||
if crt == "1" and (csosn_now in NO_ICMS_CSOSN):
|
||||
out["ALIQUOTA"] = _r("ALIQUOTA", NA, v, reason="SIMPLES_CSOSN_HIGHLIGHTS_NO_ICMS_RATE")
|
||||
elif crt in ("2", "3") and cst_now and cst_now[1:] in NO_ICMS_CST:
|
||||
out["ALIQUOTA"] = _r("ALIQUOTA", NA, v, reason="CST_HIGHLIGHTS_NO_ICMS_RATE")
|
||||
elif rate_ctx:
|
||||
try:
|
||||
r = float(rate_ctx.replace(",", "."))
|
||||
same = v is not None and abs(float(v) - r) < 1e-9
|
||||
out["ALIQUOTA"] = _r("ALIQUOTA", ALREADY if same else CHANGE, v, None if same else r, "OPERATOR_ICMS_RATE")
|
||||
except ValueError:
|
||||
out["ALIQUOTA"] = _r("ALIQUOTA", MANUAL, v, reason="OPERATOR_ICMS_RATE_NOT_NUMERIC", key="icms_rate", scope="COMPANY")
|
||||
elif v not in (None, ""):
|
||||
out["ALIQUOTA"] = _r("ALIQUOTA", ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
else:
|
||||
out["ALIQUOTA"] = _r("ALIQUOTA", MANUAL, v, reason="TAXED_ITEM_WITHOUT_ICMS_RATE", key="credit_rate" if crt == "1" else "icms_rate", scope="COMPANY")
|
||||
|
||||
# ---------------- CST IPI ----------------
|
||||
v = cur["CST_IPI"]
|
||||
if company.get("contribuinte_ipi") != "S":
|
||||
out["CST_IPI"] = _r("CST_IPI", NA, v, reason="COMPANY_IS_NOT_AN_IPI_TAXPAYER_C000004.CONTRIBUINTE_IPI")
|
||||
elif v:
|
||||
out["CST_IPI"] = _r("CST_IPI", ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
else:
|
||||
out["CST_IPI"] = _r("CST_IPI", MANUAL, v, reason="IPI_TAXPAYER_WITHOUT_CST_IPI", key="cst_ipi", scope="GROUP")
|
||||
|
||||
# ---------------- PIS / COFINS ----------------
|
||||
prof_raw, prof_scope = ctx.get("pis_cofins_profile", code, group)
|
||||
PIS_COFINS_FIELDS = ("COD_ENTRADA_PIS", "pis_in_cst", "cst_pis"), ("COD_SAIDA_PIS", "pis_out_cst", "cst_pis"), ("ALIQ_PIS", "pis_rate", None), \
|
||||
("COD_ENTRADA_COFINS", "cofins_in_cst", "cst_cofins"), ("COD_SAIDA_COFINS", "cofins_out_cst", "cst_cofins"), ("ALIQ_COFINS", "cofins_rate", None)
|
||||
if prof_raw in ("NOT_APPLICABLE", "OPERATION_TIME"):
|
||||
# Audited ADMIN decision only (Missão 2 / Bloco 2.4): an operator explicitly recorded, in FiscalContextEntry,
|
||||
# that this company/product's PIS/COFINS is never taken from the cadastral fields (real evidence: SGV emission
|
||||
# never reads them) or varies per operation. Never inferred automatically from emission statistics -
|
||||
# absent this exact recorded value, the field stays MANUAL (see the final `else` below), unchanged.
|
||||
for field, _pkey, _dom in PIS_COFINS_FIELDS:
|
||||
out[field] = _r(field, NA, cur[field], reason="ADMIN_DECISION_PIS_COFINS_" + prof_raw)
|
||||
prof = None
|
||||
else:
|
||||
import json as _json
|
||||
try:
|
||||
prof = _json.loads(prof_raw) if prof_raw else {}
|
||||
except ValueError:
|
||||
prof = {}
|
||||
for field, pkey, dom in ([] if prof is None else PIS_COFINS_FIELDS):
|
||||
v = cur[field]
|
||||
want = _s(prof.get(pkey)) if prof else None
|
||||
if want is not None and dom and want not in refs[dom]:
|
||||
out[field] = _r(field, MANUAL, v, reason="PROFILE_VALUE_NOT_IN_OFFICIAL_CST_TABLE", key="pis_cofins_profile", scope="COMPANY")
|
||||
elif want is not None:
|
||||
out[field] = _r(field, ALREADY if v == want else CHANGE, v, None if v == want else want, "OPERATOR_PIS_COFINS_PROFILE")
|
||||
elif v and (dom is None or v in refs[dom]):
|
||||
out[field] = _r(field, ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
elif v:
|
||||
out[field] = _r(field, MANUAL, v, reason="DECLARED_CST_NOT_IN_OFFICIAL_TABLE", key="pis_cofins_profile", scope="COMPANY")
|
||||
elif crt == "1" and company.get("pis_cofins_unused_by_emission"):
|
||||
hits = sped_pis_cofins_match(NCM) if ncm_ok else []
|
||||
csts = {h[1] for h in hits}
|
||||
if field in ("COD_SAIDA_PIS", "COD_SAIDA_COFINS") and len(csts) == 1 and role == "RESALE":
|
||||
want = next(iter(csts))
|
||||
out[field] = _r(field, CHANGE, v, want, "SPED_TABLE_%s_ENTRY_%s_NCM_%s" % (hits[0][0], hits[0][2], NCM))
|
||||
elif field in ("COD_SAIDA_PIS", "COD_SAIDA_COFINS") and len(csts) > 1:
|
||||
out[field] = _r(field, MANUAL, v, reason="SPED_TABLES_CONFLICT_FOR_NCM", key="pis_cofins_profile", scope="PRODUCT")
|
||||
elif field in ("ALIQ_PIS", "ALIQ_COFINS") and len(csts) == 1 and role == "RESALE":
|
||||
out[field] = _r(field, NA, v, reason="RATE_NOT_INFORMED_FOR_CST_" + next(iter(csts)))
|
||||
else:
|
||||
out[field] = _r(field, NA, v, reason="SGV_EMISSION_IGNORES_PIS_COFINS_PRODUCT_FIELDS_SIMPLES_NACIONAL")
|
||||
else:
|
||||
out[field] = _r(field, MANUAL, v, reason="NO_OFFICIAL_RULE_SOURCE_IN_CORE_DATASETS", key="pis_cofins_profile", scope="COMPANY")
|
||||
|
||||
# ---------------- incidence / PISCOFINS / IAT / IPPT ----------------
|
||||
v = cur["INCIDENCIA_PISCOFINS"]
|
||||
inc_ctx, _ = ctx.get("incidence", code, group)
|
||||
if inc_ctx == "NENHUMA":
|
||||
out["INCIDENCIA_PISCOFINS"] = _r("INCIDENCIA_PISCOFINS", ALREADY if v == "NENHUMA" else CHANGE, v, None if v == "NENHUMA" else "NENHUMA", "OPERATOR_INCIDENCE")
|
||||
elif v:
|
||||
out["INCIDENCIA_PISCOFINS"] = _r("INCIDENCIA_PISCOFINS", ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
else:
|
||||
out["INCIDENCIA_PISCOFINS"] = _r("INCIDENCIA_PISCOFINS", MANUAL, v, reason="BLANK_INCIDENCE", key="incidence", scope="GROUP")
|
||||
v = cur["PISCOFINS"]
|
||||
pc_ctx, _ = ctx.get("piscofins", code, group)
|
||||
if pc_ctx in ("N", "S"):
|
||||
out["PISCOFINS"] = _r("PISCOFINS", ALREADY if v == pc_ctx else CHANGE, v, None if v == pc_ctx else pc_ctx, "OPERATOR_PISCOFINS")
|
||||
elif v in ("N", "S"):
|
||||
out["PISCOFINS"] = _r("PISCOFINS", ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
else:
|
||||
out["PISCOFINS"] = _r("PISCOFINS", MANUAL, v, reason="BLANK_OR_INVALID", key="piscofins", scope="GROUP")
|
||||
v = cur["IAT"]
|
||||
rd_ctx, _ = ctx.get("rounding", code, group)
|
||||
if rd_ctx in ("A", "T"):
|
||||
out["IAT"] = _r("IAT", ALREADY if v == rd_ctx else CHANGE, v, None if v == rd_ctx else rd_ctx, "OPERATOR_ROUNDING_POLICY")
|
||||
elif v in ("A", "T"):
|
||||
out["IAT"] = _r("IAT", ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
else:
|
||||
out["IAT"] = _r("IAT", MANUAL, v, reason="BLANK_OR_INVALID", key="rounding", scope="COMPANY")
|
||||
v = cur["IPPT"]
|
||||
if role in ("INPUT", "SERVICE"):
|
||||
out["IPPT"] = _r("IPPT", NA, v, reason="INPUT_OR_SERVICE_HAS_NO_PRODUCTION_INDICATOR")
|
||||
elif role == "UNKNOWN":
|
||||
out["IPPT"] = _r("IPPT", ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED") if v in ("P", "T") else _r("IPPT", MANUAL, v, reason="IPPT_NEEDS_PRODUCT_ROLE", key="product_role", scope="GROUP")
|
||||
else:
|
||||
exp = "P" if role == "OWN_PRODUCTION" else "T"
|
||||
out["IPPT"] = _r("IPPT", ALREADY if v == exp else CHANGE, v, None if v == exp else exp, "DERIVED_FROM_PRODUCT_ROLE:" + ",".join(role_ev))
|
||||
for r in out.values():
|
||||
r.setdefault("role", role)
|
||||
out["_facts"] = {"role": role, "role_evidence": role_ev, "st": st, "st_withheld": withheld, "icms_treatment": treat, "origin_digit": origin_digit}
|
||||
return out
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Pure requiredness and fiscal completeness evaluation."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
VALID_CLASSIFICATIONS = {
|
||||
"REQUIRED", "CONDITIONAL", "OPTIONAL_REFERENCE", "NOT_APPLICABLE", "ERP_NOT_USED"
|
||||
}
|
||||
|
||||
|
||||
def load_requiredness_matrix(path: str | Path) -> dict[str, Any]:
|
||||
matrix = json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
fields = matrix.get("fields", {})
|
||||
if set(matrix.get("classifications", [])) != VALID_CLASSIFICATIONS:
|
||||
raise ValueError("requiredness matrix classifications are incomplete")
|
||||
if len(fields) != 23 or any(rule.get("classification") not in VALID_CLASSIFICATIONS for rule in fields.values()):
|
||||
raise ValueError("requiredness matrix must define 23 valid field rules")
|
||||
return matrix
|
||||
|
||||
|
||||
def _has_value(value: Any) -> bool:
|
||||
return value is not None and (not isinstance(value, str) or bool(value.strip()))
|
||||
|
||||
|
||||
def _conditional_state(field: str, context: dict[str, Any]) -> str:
|
||||
if field == "IPI":
|
||||
return "REQUIRED" if context.get("contribuinte_ipi") == "S" and context.get("has_numeric_tipi") else "NOT_APPLICABLE"
|
||||
if field == "EX_TIPI":
|
||||
return "REQUIRED" if context.get("has_supported_ex") and context.get("product_matches_ex") else "NOT_APPLICABLE"
|
||||
if field == "CEST":
|
||||
return "REQUIRED" if context.get("has_official_cest") or context.get("st_subject") else "NOT_APPLICABLE"
|
||||
if field in {"CFOP", "SITUACAO_TRIBUTARIA"}:
|
||||
return "REQUIRED" if context.get("operation", "SALE") == "SALE" else "NOT_APPLICABLE"
|
||||
if field in {"CFOP_ENTRADA", "CST_ENTRADA", "COD_ENTRADA_PIS", "COD_ENTRADA_COFINS"}:
|
||||
return "REQUIRED" if context.get("operation") == "PURCHASE" and context.get("erp_consumes_entry_fields") else "NOT_APPLICABLE"
|
||||
if field == "CST":
|
||||
return "REQUIRED" if context.get("regime") == "NORMAL" and context.get("operation", "SALE") == "SALE" else "NOT_APPLICABLE"
|
||||
if field == "CSOSN":
|
||||
return "REQUIRED" if context.get("regime") == "SIMPLES" and context.get("operation", "SALE") == "SALE" else "NOT_APPLICABLE"
|
||||
if field == "ALIQUOTA":
|
||||
return "REQUIRED" if context.get("icms_taxed") else "NOT_APPLICABLE"
|
||||
if field == "COD_ORIGEM_MERCADORIA":
|
||||
return "REQUIRED" if context.get("origin_proven") else "NOT_APPLICABLE"
|
||||
if field == "CST_IPI":
|
||||
return "REQUIRED" if context.get("contribuinte_ipi") == "S" and context.get("erp_consumes_ipi_cst") else "NOT_APPLICABLE"
|
||||
if field in {"COD_SAIDA_PIS", "COD_SAIDA_COFINS"}:
|
||||
return "REQUIRED" if context.get("operation", "SALE") == "SALE" and context.get("piscofins_profile_proven") else "NOT_APPLICABLE"
|
||||
if field in {"ALIQ_PIS", "ALIQ_COFINS"}:
|
||||
return "REQUIRED" if context.get("piscofins_rate_applies") else "NOT_APPLICABLE"
|
||||
if field in {"INCIDENCIA_PISCOFINS", "PISCOFINS"}:
|
||||
return "REQUIRED" if context.get("piscofins_profile_proven") else "NOT_APPLICABLE"
|
||||
return "NOT_APPLICABLE"
|
||||
|
||||
|
||||
def evaluate_field(field: str, value: Any, context: dict[str, Any], matrix: dict[str, Any]) -> dict[str, Any]:
|
||||
rule = matrix["fields"][field]
|
||||
classification = rule["classification"]
|
||||
if classification == "ERP_NOT_USED":
|
||||
status = "ERP_NOT_SUPPORTED"
|
||||
valid = True
|
||||
elif classification == "NOT_APPLICABLE":
|
||||
status = "NOT_APPLICABLE"
|
||||
valid = True
|
||||
elif classification == "OPTIONAL_REFERENCE":
|
||||
status = "OPTIONAL_VALID" if _has_value(value) else "OPTIONAL_EMPTY"
|
||||
valid = True
|
||||
elif classification == "REQUIRED":
|
||||
status = "REQUIRED_AND_VALID" if _has_value(value) else "REQUIRED_MISSING"
|
||||
valid = _has_value(value)
|
||||
else:
|
||||
conditional = _conditional_state(field, context)
|
||||
if conditional == "NOT_APPLICABLE":
|
||||
status, valid = "CONDITIONAL_NOT_TRIGGERED", True
|
||||
else:
|
||||
status, valid = ("CONDITIONAL_AND_VALID", True) if _has_value(value) else ("REQUIRED_MISSING", False)
|
||||
return {"field": field, "classification": classification, "status": status, "valid": valid, "value_present": _has_value(value), "source": rule["source"]}
|
||||
|
||||
|
||||
def evaluate_product(product: dict[str, Any], context: dict[str, Any], matrix: dict[str, Any]) -> dict[str, Any]:
|
||||
fields = {field: evaluate_field(field, product.get(field), context, matrix) for field in matrix["fields"]}
|
||||
return {"FISCALLY_COMPLETE": all(result["valid"] for result in fields.values()), "fields": fields}
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Contexto Fiscal: the ADMIN-only, human-language view of the handful of fiscal decisions field_contract.resolve_product genuinely cannot
|
||||
derive from the ERP alone (see central/app/enrichment/fiscal_completion.py). Reuses FiscalContextEntry (the SAME table blocker_resolution
|
||||
already writes operator answers into) - no new persistence model, no opaque JSON box in the UI.
|
||||
|
||||
DADO_DE_TESTE=true: Each decision here was audited on test/homologation environments (Agent 2 and lab databases) to validate the fiscal
|
||||
engine mechanism. NONE of these findings represent conclusions on any real production client. Real client data will be validated
|
||||
separately upon first real client onboarding. Evidence text below is a dated, one-time manual read-only measurement on test data, not a live query."""
|
||||
from __future__ import annotations
|
||||
|
||||
DECISIONS = [
|
||||
{
|
||||
"key": "icms_treatment", "scope": "COMPANY", "title": "Tratamento padrão de ICMS dos produtos comuns",
|
||||
"evidence": "[DADO_DE_TESTE=true: Base de homologação Agent 2] CSOSN padrão do ERP (C000004.ICMS_CSOSN): 102. 487 de 521 produtos cadastrados já usam 102. As exceções existentes "
|
||||
"(500 = ICMS-ST retido, em 5 produtos; 101 = com direito a crédito, em 4 produtos; 103, em 1 produto) são preservadas "
|
||||
"e nunca sobrescritas - a confirmação abaixo só afeta os produtos com CSOSN em branco (28 hoje). "
|
||||
"Dado de laboratório/teste; não constitui conclusão sobre cliente real.",
|
||||
"suggestion": "TAXED", "allowed_values": ["TAXED", "EXEMPT", "NOT_TAXED", "ST_WITHHELD", "SUSPENDED", "DEFERRED"],
|
||||
"affected_fields": ["CST", "CSOSN"], "confidence": "Média (DADO_DE_TESTE=true: medida em banco de homologação; pendente de validação com cliente real)",
|
||||
"measured_at": "2026-09-27", "is_test_data": True, "data_label": "DADO_DE_TESTE=true",
|
||||
},
|
||||
{
|
||||
"key": "pis_cofins_profile", "scope": "COMPANY", "title": "Perfil de PIS/COFINS (código e alíquota de entrada/saída)",
|
||||
"evidence": "[DADO_DE_TESTE=true: Auditoria em 5 bancos de laboratório e Agent 2] 34.946 itens de NFC-e emitidos; 3.336 (9,5%) carregam um CST de PIS/COFINS na nota, sempre o código 01 - mas os "
|
||||
"mesmos produtos têm os campos de cadastro (código de saída, alíquota) sempre em branco no SGV, incluindo os poucos "
|
||||
"produtos que TÊM esses campos preenchidos manualmente (a nota emitida ignora o valor cadastrado). Reprocessado em "
|
||||
"2026-09-28 (Missão 2 / Bloco 2.1): os 3.336 itens estão concentrados quase inteiramente em 2022 (3.336 de 3.669 "
|
||||
"itens daquele ano; 0% em 2018 e em 2023-2026) - coincide com a mesma janela já sinalizada como anômala em outra "
|
||||
"auditoria (drift de identidade de produto). O mesmo padrão de cadastro nunca preenchido na emissão se repete, sem "
|
||||
"exceção, nos outros 4 bancos de teste auditados (CONSTRUCAO, MMMODAS, RINCAO CRT=3, UNIAO): 0% de itens com CST de "
|
||||
"PIS/COFINS em todo o histórico, em todos os anos. Amostra de laboratório; não é garantia de comportamento de cliente real em produção. "
|
||||
"Ver docs/dossie_contador.md e docs/runs/BLOCO2_PIS_COFINS_CROSS_DB_V1.json.", "suggestion": None,
|
||||
"affected_fields": ["COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS"],
|
||||
"confidence": "Baixa para um perfil único da empresa (DADO_DE_TESTE=true: considerar NOT_APPLICABLE ou OPERATION_TIME; pendente de confirmação contábil no piloto real)",
|
||||
"measured_at": "2026-09-28", "is_test_data": True, "data_label": "DADO_DE_TESTE=true",
|
||||
"value_hint": 'JSON de perfil (ex.: {"pis_out_cst":"49","pis_in_cst":"98","pis_rate":"0","cofins_out_cst":"49","cofins_in_cst":"98","cofins_rate":"0"}) '
|
||||
'OU exatamente NOT_APPLICABLE (nunca é lido pelo cadastro) OU OPERATION_TIME (varia por operação) - nunca inferido automaticamente.',
|
||||
"dossier_link": "/static/docs/dossie_contador.md",
|
||||
},
|
||||
{
|
||||
"key": "entry_scope", "scope": "COMPANY", "title": "CFOP de entrada padrão (compras dentro/fora do estado)",
|
||||
"evidence": "[DADO_DE_TESTE=true: Base de homologação Agent 2] Apenas 12 documentos de compra no ERP de teste para 526 produtos; 11 sem CFOP preenchido, 1 com CFOP intra-estadual. "
|
||||
"Evidência de laboratório insuficiente para qualquer sugestão - o CFOP de entrada normalmente é definido pela operação/documento "
|
||||
"de compra, não pelo cadastro do produto. Ver docs/dossie_contador.md.", "suggestion": None,
|
||||
"affected_fields": ["CFOP_ENTRADA"], "confidence": "Muito baixa (DADO_DE_TESTE=true: recomendação: MIXED_OR_DYNAMIC; validação definitiva apenas com cliente real)",
|
||||
"measured_at": "2026-09-27", "allowed_values": ["INTRA", "INTER", "MIXED_OR_DYNAMIC"], "is_test_data": True, "data_label": "DADO_DE_TESTE=true",
|
||||
"dossier_link": "/static/docs/dossie_contador.md",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
VALID_KEYS = {d["key"] for d in DECISIONS}
|
||||
|
||||
|
||||
def set_fiscal_context_decision(session, *, tenant_id: int, company_key: str, key: str, value: str | None, reason: str, admin, installation_id: int):
|
||||
"""Confirm/alter/clear one Contexto Fiscal decision. value=None clears it (the product/group falls back to whatever the engine can
|
||||
prove on its own - never a silent default). Reuses the SAME table and validator the blocker-resolution workflow already writes
|
||||
through; flags every already-enriched product on this installation for re-evaluation so the new (or cleared) policy takes effect."""
|
||||
from datetime import datetime, timezone
|
||||
from sqlalchemy import select
|
||||
from . import context_keys
|
||||
from .audit import record_audit
|
||||
from .enrichment.status import mark_for_reevaluation
|
||||
from .models import FiscalContextEntry
|
||||
|
||||
if key not in VALID_KEYS:
|
||||
raise ValueError(f"unknown Contexto Fiscal key {key!r}")
|
||||
entry = session.scalar(select(FiscalContextEntry).where(FiscalContextEntry.tenant_id == tenant_id, FiscalContextEntry.company_key == company_key,
|
||||
FiscalContextEntry.scope == "COMPANY", FiscalContextEntry.scope_key == "", FiscalContextEntry.key == key))
|
||||
before = entry.value if entry else None
|
||||
if value is None:
|
||||
if entry is not None:
|
||||
session.delete(entry)
|
||||
else:
|
||||
normalized, error = context_keys.validate_entry({"scope": "COMPANY", "key": key, "value": value, "reason": reason})
|
||||
if error:
|
||||
raise ValueError(error)
|
||||
if entry is None:
|
||||
entry = FiscalContextEntry(tenant_id=tenant_id, company_key=company_key, source="OPERATOR", updated_at=datetime.now(timezone.utc), **normalized)
|
||||
session.add(entry)
|
||||
else:
|
||||
entry.value, entry.reason, entry.source, entry.updated_at = normalized["value"], normalized["reason"], "OPERATOR", datetime.now(timezone.utc)
|
||||
flagged = mark_for_reevaluation(session, installation_id)
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=tenant_id, action="FISCAL_CONTEXT_DECISION", target_type="installation", target_id=company_key,
|
||||
context={"key": key, "from": before, "to": value, "reason": reason, "operator": admin.display_name or admin.username, "role": admin.external_role or admin.auth_source, "products_flagged": flagged},
|
||||
severity="WARNING")
|
||||
|
||||
|
||||
def build_fiscal_context_view(session, tenant_id: int, company_key: str) -> list[dict]:
|
||||
from sqlalchemy import select
|
||||
from .models import FiscalContextEntry
|
||||
current = {(r.scope, r.scope_key, r.key): r for r in session.scalars(
|
||||
select(FiscalContextEntry).where(FiscalContextEntry.tenant_id == tenant_id, FiscalContextEntry.company_key == company_key))}
|
||||
out = []
|
||||
for d in DECISIONS:
|
||||
row = current.get((d["scope"], "", d["key"]))
|
||||
out.append({**d, "current_value": row.value if row else None, "current_reason": row.reason if row else None,
|
||||
"last_changed": row.updated_at.isoformat() if row and getattr(row, "updated_at", None) else None,
|
||||
"last_changed_by": row.source if row else None})
|
||||
return out
|
||||
@@ -0,0 +1,224 @@
|
||||
"""Fiscal Decision Engine V1: independent per-field, per-product decisions.
|
||||
|
||||
Pure functions (no I/O). Every domain gets its own decision; a dataset being
|
||||
available is never treated as a rule being resolved. Nothing here writes to
|
||||
any database — it only marks `write_eligible`, and only for fields whose ERP
|
||||
mapping is confirmed AND whose rule is deterministic AND that need no
|
||||
operation context.
|
||||
|
||||
Status vocabulary (per domain):
|
||||
RULE_RESOLVED, RULE_CANDIDATE, RULE_UNRESOLVED, OPERATION_CONTEXT_REQUIRED
|
||||
plus flags data_available / match_found. `action_class` is the Phase-14 bucket:
|
||||
AUTO_APPLY_SAFE, REVIEW_REQUIRED, OPERATION_CONTEXT_REQUIRED, NO_RULE, NO_DATA
|
||||
(+ ALREADY_CORRECT when the ERP value already equals the deterministic one).
|
||||
"""
|
||||
from .dry_run_planner import build_field_proposals
|
||||
|
||||
DECISION_ENGINE_VERSION = "fiscal-decision-v1"
|
||||
|
||||
NATURE = {
|
||||
"NCM": "PRODUCT_INTRINSIC", "IPI": "PRODUCT_INTRINSIC", "EX_TIPI": "PRODUCT_INTRINSIC",
|
||||
"CEST": "PRODUCT_INTRINSIC", "MVA": "PRODUCT_INTRINSIC",
|
||||
"ORIGEM": "PRODUCT_INTRINSIC",
|
||||
"CFOP": "OPERATION_DEPENDENT", "CSOSN": "OPERATION_DEPENDENT", "CST_ICMS": "OPERATION_DEPENDENT",
|
||||
"PIS_CST": "OPERATION_DEPENDENT", "COFINS_CST": "OPERATION_DEPENDENT",
|
||||
"PIS_COFINS_RATE": "COMPANY_CONTEXT_DEPENDENT", "FCP": "OPERATION_DEPENDENT",
|
||||
"RTC_CCLASSTRIB": "OPERATION_DEPENDENT", "RTC_CBS_RATE": "OPERATION_DEPENDENT",
|
||||
"RTC_CCREDPRES": "OPERATION_DEPENDENT",
|
||||
}
|
||||
|
||||
# ERP_FIELD, SEMANTICS, DB_TYPE, CONFIDENCE, EVIDENCE — audited against the live
|
||||
# C000025 schema (RDB$ metadata: no field comments exist) and value distributions
|
||||
# over all 521 rows (scripts/audit_erp_fields.py). No supplier DDL/doc file was
|
||||
# found in the repo, so semantic confirmation rests on value-domain evidence.
|
||||
ERP_FIELD_MAP = [
|
||||
{"ERP_FIELD": "CLASSIFICACAO_FISCAL", "SEMANTICS": "NCM canônico (8 dígitos)", "DB_TYPE": "VARCHAR(20)", "CONFIDENCE": "HIGH", "EVIDENCE": "54 distinct 8-digit values, 495 valid in NCM dataset; prior-mission confirmed (NCM column is 100% NULL)", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "IPI", "SEMANTICS": "alíquota numérica de IPI (%)", "DB_TYPE": "DOUBLE PRECISION", "CONFIDENCE": "HIGH", "EVIDENCE": "numeric type; 105 rows = 19.5 written earlier from TIPI and verified; prior-mission confirmed", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "EX_TIPI", "SEMANTICS": "código EX da TIPI", "DB_TYPE": "VARCHAR(3)", "CONFIDENCE": "HIGH", "EVIDENCE": "prior-mission confirmed; 100% NULL now", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "CEST", "SEMANTICS": "CEST do produto (7 dígitos sem pontuação)", "DB_TYPE": "VARCHAR(7)", "CONFIDENCE": "HIGH", "EVIDENCE": "values like 0601100/1708400 = CEST 06.011.00/17.084.00 without dots; prior-mission confirmed", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "CFOP", "SEMANTICS": "CFOP padrão de SAÍDA do produto (default, não a operação real)", "DB_TYPE": "VARCHAR(4)", "CONFIDENCE": "HIGH", "EVIDENCE": "values 5102 (493), 5405 (3), 5656 (2): all valid 5xxx output CFOPs in CFOP dataset; 5405=ST, 5656=fuel", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "CFOP_ENTRADA", "SEMANTICS": "CFOP padrão de entrada", "DB_TYPE": "SMALLINT", "CONFIDENCE": "LOW", "EVIDENCE": "100% NULL in all 521 rows; name only", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "CST", "SEMANTICS": "CST ICMS de saída em 3 dígitos = ORIGEM(1)+CST(2)", "DB_TYPE": "VARCHAR(10)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "values 040/000/060/020/051: last 2 digits all in CST_ICMS dataset, first digit in ORIGEM dataset (0)", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "CST_ENTRADA", "SEMANTICS": "CST ICMS de entrada", "DB_TYPE": "SMALLINT", "CONFIDENCE": "LOW", "EVIDENCE": "100% NULL; name only", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "ALIQUOTA", "SEMANTICS": "alíquota de ICMS % (saída)", "DB_TYPE": "NUMERIC(8,2)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "values 20.5 (492), 17 (24), 0 (4), 21.5 (1): plausible BA ICMS rates; no doc confirms", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "CSOSN", "SEMANTICS": "CSOSN de saída (Simples Nacional)", "DB_TYPE": "VARCHAR(60)", "CONFIDENCE": "HIGH", "EVIDENCE": "values 102/500/101/103 all in CSOSN dataset; company CRT=1", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "COD_ENTRADA_PIS", "SEMANTICS": "CST PIS de entrada", "DB_TYPE": "VARCHAR(3)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "2 non-null rows ('2','73'); mixed formatting ('2' vs 2-digit codes) — too sparse", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "COD_SAIDA_PIS", "SEMANTICS": "CST PIS de saída", "DB_TYPE": "VARCHAR(3)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "3 non-null rows ('01','04','2'); inconsistent formatting", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "ALIQ_PIS", "SEMANTICS": "alíquota PIS (texto, vírgula decimal)", "DB_TYPE": "VARCHAR(8)", "CONFIDENCE": "LOW", "EVIDENCE": "3 non-null rows ('0,65','2', blank); text type", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "COD_ENTRADA_COFINS", "SEMANTICS": "CST COFINS de entrada", "DB_TYPE": "VARCHAR(3)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "2 non-null rows; too sparse", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "COD_SAIDA_COFINS", "SEMANTICS": "CST COFINS de saída", "DB_TYPE": "VARCHAR(3)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "3 non-null rows; inconsistent formatting", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "ALIQ_COFINS", "SEMANTICS": "alíquota COFINS (texto)", "DB_TYPE": "VARCHAR(8)", "CONFIDENCE": "LOW", "EVIDENCE": "3 non-null rows ('2','3', blank)", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "CST_IPI", "SEMANTICS": "CST do IPI", "DB_TYPE": "SMALLINT", "CONFIDENCE": "LOW", "EVIDENCE": "1 non-null row ('040'); IPI CST is 2 digits — value contradicts", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "ORIGEM", "SEMANTICS": "origem da mercadoria (0-8)", "DB_TYPE": "VARCHAR(1)", "CONFIDENCE": "LOW", "EVIDENCE": "100% NULL; origin digit is instead embedded in CST (first digit)", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "IAT", "SEMANTICS": "indicador arredondamento/truncamento (A/T)", "DB_TYPE": "VARCHAR(1)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "constant 'A' in 521 rows; not a fiscal-decision field", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "INCIDENCIA_PISCOFINS", "SEMANTICS": "tipo de incidência PIS/COFINS", "DB_TYPE": "VARCHAR(30)", "CONFIDENCE": "HIGH", "EVIDENCE": "values NENHUMA (519), MONOFÁSICA (1), SUBST.TRIBUTÁRIA (1)", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "IPPT", "SEMANTICS": "indicador produção própria/terceiros (P/T)", "DB_TYPE": "VARCHAR(1)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "constant 'T' in 521 rows; not a fiscal-decision field", "MAPPING": "CONFIRMED"},
|
||||
]
|
||||
|
||||
_ACTION = {
|
||||
"RULE_RESOLVED": "AUTO_APPLY_SAFE", # only if write_eligible; refined below
|
||||
"RULE_CANDIDATE": "REVIEW_REQUIRED",
|
||||
"OPERATION_CONTEXT_REQUIRED": "OPERATION_CONTEXT_REQUIRED",
|
||||
}
|
||||
|
||||
|
||||
def _rec(domain, current, proposed, status, reason, confidence, provenance, write_eligible=False,
|
||||
data_available=True, match_found=False, action=None, **extra):
|
||||
if action is None:
|
||||
if status == "RULE_RESOLVED":
|
||||
action = "AUTO_APPLY_SAFE" if write_eligible else "ALREADY_CORRECT" if reason.endswith("SAME_VALUE") else "NO_RULE"
|
||||
elif status == "RULE_UNRESOLVED":
|
||||
action = "NO_DATA" if not data_available else "NO_RULE"
|
||||
else:
|
||||
action = _ACTION[status]
|
||||
return {
|
||||
"domain": domain, "nature": NATURE[domain], "current_value": current, "proposed_value": proposed,
|
||||
"status": status, "reason_code": reason, "confidence": confidence, "provenance": provenance,
|
||||
"write_eligible": write_eligible, "action_class": action,
|
||||
"data_available": data_available, "match_found": match_found, **extra,
|
||||
}
|
||||
|
||||
|
||||
def _norm_words(text):
|
||||
import unicodedata
|
||||
text = unicodedata.normalize("NFKD", str(text or "")).encode("ascii", "ignore").decode().lower()
|
||||
return {w for w in "".join(c if c.isalnum() else " " for c in text).split() if len(w) >= 4}
|
||||
|
||||
|
||||
def _ex_described(product_description, ex_description):
|
||||
"""An EX code applies to a product only if the product's own description
|
||||
carries the EX's official description terms. A lone EX row for the NCM is
|
||||
NOT evidence that this product is that EX (e.g. EX 'Chope' vs any beer)."""
|
||||
ex_words = _norm_words(ex_description)
|
||||
return bool(ex_words) and ex_words <= _norm_words(product_description)
|
||||
|
||||
|
||||
def _blank(value):
|
||||
return value is None or str(value).strip() == ""
|
||||
|
||||
|
||||
def decide_product(product, decision, company, refs):
|
||||
"""company: {"uf": "BA", "crt": "1"}; refs: reference-data snapshot dict:
|
||||
cfop_valid(set of codes), cst_icms/csosn/cst_pis/cst_cofins/origem (sets),
|
||||
fcp_uf(dict uf->{'aliq_1':...}), cbs_rate_2026 (float|None), cclasstrib_count (int)."""
|
||||
versions = decision.get("dataset_versions", {})
|
||||
prov = lambda kind: {"dataset": kind, "version": versions.get(kind.lower()), "engine": DECISION_ENGINE_VERSION} # noqa: E731
|
||||
out = {}
|
||||
|
||||
# ---- NCM (never written) -------------------------------------------------
|
||||
raw_ncm = product.get("CLASSIFICACAO_FISCAL")
|
||||
status = decision.get("status")
|
||||
if _blank(raw_ncm):
|
||||
out["NCM"] = _rec("NCM", raw_ncm, None, "RULE_UNRESOLVED", "NCM_MISSING", "NONE", prov("NCM"), data_available=False)
|
||||
elif status == "EXACT_NCM":
|
||||
out["NCM"] = _rec("NCM", raw_ncm, raw_ncm, "RULE_RESOLVED", "NCM_VALID_SAME_VALUE", "HIGH", prov("NCM"), match_found=True)
|
||||
elif status == "LEGACY_NCM":
|
||||
out["NCM"] = _rec("NCM", raw_ncm, None, "RULE_CANDIDATE", "LEGACY_NCM_REVIEW_REQUIRED", "LOW", prov("NCM"), match_found=True, action="REVIEW_REQUIRED")
|
||||
else:
|
||||
out["NCM"] = _rec("NCM", raw_ncm, None, "RULE_CANDIDATE", "NCM_INVALID_REVIEW_REQUIRED", "LOW", prov("NCM"), action="REVIEW_REQUIRED")
|
||||
|
||||
legacy = status == "LEGACY_NCM"
|
||||
proposals = {p["field"]: p for p in build_field_proposals(
|
||||
product["CODIGO"], decision,
|
||||
{"ipi": product.get("IPI"), "cest": product.get("CEST"), "ex_tipi": product.get("EX_TIPI")})}
|
||||
|
||||
# ---- IPI -------------------------------------------------------------------
|
||||
p = proposals.get("IPI")
|
||||
ipi_status = decision.get("ipi_status")
|
||||
if p and p["status"] in ("SAME_VALUE", "CHANGE_REQUIRED"):
|
||||
same = p["status"] == "SAME_VALUE"
|
||||
out["IPI"] = _rec("IPI", p["current_value"], p["proposed_value"], "RULE_RESOLVED",
|
||||
"TIPI_NUMERIC_SAME_VALUE" if same else "TIPI_NUMERIC_DIVERGES_FROM_ERP", "HIGH", prov("TIPI_IPI"),
|
||||
write_eligible=p["write_eligible"], match_found=True)
|
||||
elif p:
|
||||
blank = p["status"] == "SOURCE_BLANK" or p["reason_code"].endswith("SOURCE_BLANK")
|
||||
out["IPI"] = _rec("IPI", p["current_value"], None, "RULE_UNRESOLVED", p["reason_code"], "NONE", prov("TIPI_IPI"),
|
||||
data_available=not blank, match_found=True, action="NO_DATA" if blank else "NO_RULE")
|
||||
elif ipi_status == "IPI_CANDIDATES":
|
||||
out["IPI"] = _rec("IPI", product.get("IPI"), None, "RULE_CANDIDATE", "MULTIPLE_TIPI_CANDIDATES", "LOW", prov("TIPI_IPI"), match_found=True)
|
||||
else:
|
||||
out["IPI"] = _rec("IPI", product.get("IPI"), None, "RULE_UNRESOLVED", "LEGACY_NCM" if legacy else "NO_TIPI_MAPPING_FOUND", "NONE", prov("TIPI_IPI"))
|
||||
|
||||
# ---- EX_TIPI ---------------------------------------------------------------
|
||||
p = proposals.get("EX_TIPI")
|
||||
if p:
|
||||
ex_desc = next((c.get("description") for c in (decision.get("classification") or {}).get("ipi_candidates", []) if c.get("ex")), None)
|
||||
if p["status"] in ("SAME_VALUE", "CHANGE_REQUIRED") and not _ex_described(product.get("PRODUTO"), ex_desc):
|
||||
out["EX_TIPI"] = _rec("EX_TIPI", p["current_value"], None, "RULE_CANDIDATE", "SINGLE_EX_FOR_NCM_BUT_PRODUCT_DESCRIPTION_DOES_NOT_EVIDENCE_IT", "LOW", prov("TIPI_IPI"), match_found=True)
|
||||
elif p["status"] in ("SAME_VALUE", "CHANGE_REQUIRED"):
|
||||
out["EX_TIPI"] = _rec("EX_TIPI", p["current_value"], p["proposed_value"], "RULE_RESOLVED", "EX_" + p["status"], "HIGH", prov("TIPI_IPI"), write_eligible=p["write_eligible"], match_found=True)
|
||||
else:
|
||||
out["EX_TIPI"] = _rec("EX_TIPI", p["current_value"], None, "RULE_CANDIDATE", p["reason_code"], "LOW", prov("TIPI_IPI"), match_found=True)
|
||||
else:
|
||||
out["EX_TIPI"] = _rec("EX_TIPI", product.get("EX_TIPI"), None, "RULE_UNRESOLVED", "NO_EX_TIPI_FOR_NCM", "NONE", prov("TIPI_IPI"), data_available=not legacy)
|
||||
|
||||
# ---- CEST ------------------------------------------------------------------
|
||||
p = proposals.get("CEST")
|
||||
if p and p["status"] in ("SAME_VALUE", "CHANGE_REQUIRED"):
|
||||
out["CEST"] = _rec("CEST", p["current_value"], p["proposed_value"], "RULE_RESOLVED", "CEST_" + p["status"], "HIGH", prov("CEST"), write_eligible=p["write_eligible"], match_found=True)
|
||||
elif p and p["status"] == "CANDIDATE":
|
||||
n = len((decision.get("classification") or {}).get("cest_candidates") or [])
|
||||
out["CEST"] = _rec("CEST", p["current_value"], None, "RULE_CANDIDATE", p["reason_code"], "LOW", prov("CEST"), match_found=True, candidate_count=n)
|
||||
else:
|
||||
out["CEST"] = _rec("CEST", product.get("CEST"), None, "RULE_UNRESOLVED", p["reason_code"] if p else ("LEGACY_NCM" if legacy else "NO_CEST_DECISION"), "NONE", prov("CEST"), data_available=False if not p else True)
|
||||
|
||||
# ---- BA ST / MVA (no ERP destination field: never eligible) ---------------
|
||||
mva_status = decision.get("ba_st_mva_status")
|
||||
if mva_status == "MVA_RESOLVED":
|
||||
entry = (decision.get("classification") or {}).get("ba_st_mva") or {}
|
||||
out["MVA"] = _rec("MVA", None, entry.get("mva_original") or entry.get("mva_adjusted"), "RULE_RESOLVED", "MVA_RESOLVED_NO_ERP_TARGET_FIELD", "HIGH", prov("BA_ST_MVA"), match_found=True, action="NO_RULE", mva_status="MVA_RESOLVED")
|
||||
elif mva_status == "MVA_CANDIDATES":
|
||||
out["MVA"] = _rec("MVA", None, None, "RULE_CANDIDATE", "MULTIPLE_MVA_ITEMS_NEED_CEST_CONFIRMATION", "LOW", prov("BA_ST_MVA"), match_found=True, mva_status="MVA_CANDIDATE")
|
||||
elif mva_status == "NO_MVA_MAPPING_FOUND":
|
||||
out["MVA"] = _rec("MVA", None, None, "RULE_UNRESOLVED", "MVA_NOT_APPLICABLE_NO_BA_ANNEX1_ENTRY_FOR_NCM", "MEDIUM", prov("BA_ST_MVA"), action="NO_RULE", mva_status="MVA_NOT_APPLICABLE")
|
||||
else:
|
||||
out["MVA"] = _rec("MVA", None, None, "RULE_UNRESOLVED", "LEGACY_OR_INVALID_NCM", "NONE", prov("BA_ST_MVA"), data_available=False, mva_status="MVA_UNRESOLVED")
|
||||
mva_subject = mva_status in ("MVA_RESOLVED", "MVA_CANDIDATES")
|
||||
|
||||
# ---- CFOP (operation dependent) --------------------------------------------
|
||||
cfop = product.get("CFOP")
|
||||
cfop_valid = not _blank(cfop) and str(cfop).strip() in refs["cfop_valid"]
|
||||
out["CFOP"] = _rec("CFOP", cfop, None, "OPERATION_CONTEXT_REQUIRED", "CFOP_DEPENDS_ON_OPERATION_DIRECTION_UF_ST_PURPOSE", "HIGH", prov("CFOP"),
|
||||
data_available=True, match_found=cfop_valid, domain_valid=cfop_valid, current_is_blank=_blank(cfop))
|
||||
|
||||
# ---- CSOSN / CST ICMS / ORIGEM ---------------------------------------------
|
||||
csosn = product.get("CSOSN")
|
||||
csosn_valid = not _blank(csosn) and str(csosn).strip() in refs["csosn"]
|
||||
flags = {}
|
||||
if mva_subject and csosn_valid and str(csosn).strip() == "102":
|
||||
flags["consistency_flag"] = "CSOSN_102_BUT_NCM_IN_BA_ST_ANNEX1"
|
||||
out["CSOSN"] = _rec("CSOSN", csosn, None, "OPERATION_CONTEXT_REQUIRED", "CSOSN_DEPENDS_ON_SALE_SCENARIO_ST_RETENTION_CONSUMER", "HIGH", prov("NFE_MOC_TABLES"),
|
||||
match_found=csosn_valid, domain_valid=csosn_valid, **flags)
|
||||
|
||||
cst = product.get("CST")
|
||||
cst_s = "" if _blank(cst) else str(cst).strip()
|
||||
cst_valid = len(cst_s) == 3 and cst_s[1:] in refs["cst_icms"] and cst_s[0] in refs["origem"]
|
||||
out["CST_ICMS"] = _rec("CST_ICMS", cst, None, "OPERATION_CONTEXT_REQUIRED", "CRT1_USES_CSOSN_CST_DEPENDS_ON_OPERATION", "MEDIUM", prov("NFE_MOC_TABLES"),
|
||||
match_found=cst_valid, domain_valid=cst_valid)
|
||||
|
||||
origin_candidate = cst_s[0] if cst_valid else None
|
||||
out["ORIGEM"] = _rec("ORIGEM", product.get("ORIGEM"), origin_candidate, "RULE_CANDIDATE" if origin_candidate else "RULE_UNRESOLVED",
|
||||
"ORIGIN_ONLY_FROM_ERP_CST_FIRST_DIGIT_NO_OBJECTIVE_PRODUCT_ORIGIN" if origin_candidate else "NO_ORIGIN_INFORMATION",
|
||||
"LOW", prov("NFE_MOC_TABLES"), data_available=bool(origin_candidate),
|
||||
action="REVIEW_REQUIRED" if origin_candidate else "NO_DATA", mapping="MAPPING_UNCONFIRMED")
|
||||
|
||||
# ---- PIS / COFINS ----------------------------------------------------------
|
||||
for domain, field, dom in (("PIS_CST", "COD_SAIDA_PIS", "cst_pis"), ("COFINS_CST", "COD_SAIDA_COFINS", "cst_cofins")):
|
||||
cur = product.get(field)
|
||||
valid = not _blank(cur) and str(cur).strip() in refs[dom]
|
||||
out[domain] = _rec(domain, cur, None, "OPERATION_CONTEXT_REQUIRED", "CST_PIS_COFINS_DEPENDS_ON_OPERATION_AND_INCIDENCE_NO_DETERMINISTIC_RULE_IN_CORE_DATASETS", "MEDIUM", prov("NFE_MOC_TABLES"),
|
||||
match_found=valid, domain_valid=valid, mapping="MAPPING_UNCONFIRMED")
|
||||
out["PIS_COFINS_RATE"] = _rec("PIS_COFINS_RATE", product.get("ALIQ_PIS"), None, "RULE_UNRESOLVED", "NO_RATE_SOURCE_4_3_10_17_DEFERRED_NEVER_ASSUME_ZERO", "NONE", prov("NFE_MOC_TABLES"), data_available=False, action="NO_RULE")
|
||||
|
||||
# ---- FCP -------------------------------------------------------------------
|
||||
fcp = refs["fcp_uf"].get(company["uf"])
|
||||
out["FCP"] = _rec("FCP", None, None, "OPERATION_CONTEXT_REQUIRED", "FCP_APPLICABILITY_DEPENDS_ON_DESTINATION_UF_OPERATION_AND_ITEM_LIST", "HIGH", prov("FCP"),
|
||||
data_available=fcp is not None, match_found=False, fcp_rate_available=fcp is not None, company_uf=company["uf"])
|
||||
|
||||
# ---- RTC -------------------------------------------------------------------
|
||||
out["RTC_CCLASSTRIB"] = _rec("RTC_CCLASSTRIB", None, None, "OPERATION_CONTEXT_REQUIRED", "NO_NCM_TO_CCLASSTRIB_MAP_IN_ACTIVE_DATASETS_AND_DEPENDS_ON_OPERATION", "NONE", prov("RTC_CCLASSTRIB"),
|
||||
data_available=refs["cclasstrib_count"] > 0, candidate_count=0)
|
||||
out["RTC_CBS_RATE"] = _rec("RTC_CBS_RATE", None, None, "OPERATION_CONTEXT_REQUIRED", "CBS_2026_RATE_APPLIES_ONLY_AFTER_CCLASSTRIB_RESOLVED_NOT_UNIVERSAL", "NONE", prov("RTC_CBS_RATES"),
|
||||
data_available=refs["cbs_rate_2026"] is not None, reference_rate_2026=refs["cbs_rate_2026"])
|
||||
out["RTC_CCREDPRES"] = _rec("RTC_CCREDPRES", None, None, "RULE_UNRESOLVED", "OFFICIAL_CONDITION_NOT_SATISFIED_OR_NOT_KNOWABLE_FROM_PRODUCT", "NONE", prov("RTC_CCREDPRES"), action="NO_RULE")
|
||||
return out
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Conservative fiscal determination with no inferred tax rules."""
|
||||
|
||||
|
||||
class FiscalDetermination:
|
||||
def determine(self, context):
|
||||
candidates = context.get("cest_candidates", [])
|
||||
cest = candidates[0] if len(candidates) == 1 else None
|
||||
reason_codes = ["OFFICIAL_RULE_NOT_IMPLEMENTED", "MISSING_ST_DETERMINATION", "MISSING_PRODUCT_ORIGIN"]
|
||||
if len(candidates) > 1:
|
||||
reason_codes.append("CEST_AMBIGUOUS")
|
||||
return {"cest": cest, "csosn": None, "cfop": None, "confidence": 0.0, "reason_codes": reason_codes, "can_auto_apply": False}
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Validador local determinístico de GTIN / EAN (GS1 Standard).
|
||||
|
||||
Sem chamadas externas, sem IA. Implementa:
|
||||
1. Validação de formato (GTIN-8, GTIN-12, GTIN-13, GTIN-14);
|
||||
2. Rejeição de placeholders (zeros, sequências óbvias, códigos internos de balança restrita);
|
||||
3. Verificação do dígito verificador oficial GS1 (módulo 10 com pesos alternados 3 e 1);
|
||||
4. Validação de faixas de prefixo GS1 reconhecidas.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def _digits(value: str | None) -> str:
|
||||
return "".join(c for c in str(value or "") if c.isdigit())
|
||||
|
||||
|
||||
# Prefixos de circulação restrita / uso interno de loja (não são GTINs globais válidos para KB nacional)
|
||||
# Faixas 020-029 e 200-299 são reservadas pelo GS1 para distribuição restrita (itens pesáveis, balança de loja)
|
||||
INTERNAL_STORE_PREFIXES = tuple(str(p) for p in range(20, 30)) + tuple(str(p) for p in range(200, 300))
|
||||
|
||||
|
||||
def calculate_gs1_check_digit(digits_without_dv: str) -> int:
|
||||
"""Calcula o dígito verificador padrão GS1 pelo algoritmo oficial módulo 10."""
|
||||
total = 0
|
||||
# Da direita para a esquerda, os fatores alternam entre 3 e 1
|
||||
# Para o penúltimo dígito (imediatamente antes do DV), o peso é sempre 3
|
||||
weight = 3
|
||||
for char in reversed(digits_without_dv):
|
||||
total += int(char) * weight
|
||||
weight = 1 if weight == 3 else 3
|
||||
remainder = total % 10
|
||||
return 0 if remainder == 0 else (10 - remainder)
|
||||
|
||||
|
||||
def validate_gtin(value: str | None) -> tuple[bool, str]:
|
||||
"""Valida um código GTIN.
|
||||
|
||||
Retorna (is_valid, reason).
|
||||
"""
|
||||
if not value:
|
||||
return False, "GTIN_EMPTY"
|
||||
|
||||
cleaned = _digits(value)
|
||||
|
||||
if len(cleaned) not in (8, 12, 13, 14):
|
||||
return False, f"GTIN_INVALID_LENGTH_{len(cleaned)}"
|
||||
|
||||
# Rejeita dígitos todos iguais (ex: 00000000, 1111111111111, 9999999999999)
|
||||
if len(set(cleaned)) == 1:
|
||||
return False, "GTIN_PLACEHOLDER_REPEATED_DIGITS"
|
||||
|
||||
# Rejeita prefixos de uso interno / balança
|
||||
# Normalizado para 13 dígitos para verificar o prefixo
|
||||
padded_13 = cleaned.zfill(13) if len(cleaned) < 14 else cleaned[1:]
|
||||
prefix_3 = padded_13[:3]
|
||||
prefix_2 = padded_13[:2]
|
||||
if prefix_3 in INTERNAL_STORE_PREFIXES or prefix_2 in INTERNAL_STORE_PREFIXES:
|
||||
return False, "GTIN_RESTRICTED_INTERNAL_STORE_PREFIX"
|
||||
|
||||
# Validação do Dígito Verificador
|
||||
expected_dv = calculate_gs1_check_digit(cleaned[:-1])
|
||||
actual_dv = int(cleaned[-1])
|
||||
if actual_dv != expected_dv:
|
||||
return False, f"GTIN_INVALID_CHECK_DIGIT_EXPECTED_{expected_dv}_GOT_{actual_dv}"
|
||||
|
||||
return True, "GTIN_VALID"
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Separate human authentication backed by hashed passwords and sessions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
from sqlalchemy import select
|
||||
|
||||
from .config import settings
|
||||
from .models import AdminUser, WebSession
|
||||
|
||||
|
||||
def _digest(value: str) -> str:
|
||||
return hashlib.sha256(value.encode()).hexdigest()
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
salt = os.urandom(16)
|
||||
encoded = hashlib.pbkdf2_hmac("sha256", password.encode(), salt, 240_000)
|
||||
return "pbkdf2_sha256$240000$%s$%s" % (base64.urlsafe_b64encode(salt).decode(), base64.urlsafe_b64encode(encoded).decode())
|
||||
|
||||
|
||||
def verify_password(password: str, stored: str | None) -> bool:
|
||||
# Federated (SSO) accounts have no password: always refuse, never raise.
|
||||
if not isinstance(stored, str) or not isinstance(password, str):
|
||||
return False
|
||||
try:
|
||||
algorithm, rounds, salt, digest = stored.split("$", 3)
|
||||
candidate = hashlib.pbkdf2_hmac(algorithm.removeprefix("pbkdf2_"), password.encode(), base64.urlsafe_b64decode(salt), int(rounds))
|
||||
return hmac.compare_digest(base64.urlsafe_b64encode(candidate).decode(), digest)
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def create_admin(session, username: str, password: str):
|
||||
username = username.strip().lower()
|
||||
if len(username) < 3:
|
||||
raise ValueError("username must have 3 characters")
|
||||
environment = os.getenv("APP_ENV") or settings.environment
|
||||
weak_passwords_allowed = os.getenv("ALLOW_WEAK_DEV_PASSWORD", "0") == "1"
|
||||
if len(password) < 12 and (environment.strip().lower() == "production" or not weak_passwords_allowed):
|
||||
raise ValueError("password must have 12 characters")
|
||||
if session.scalar(select(AdminUser).where(AdminUser.username == username)):
|
||||
raise ValueError("username already exists")
|
||||
admin = AdminUser(username=username, password_hash=hash_password(password))
|
||||
session.add(admin); session.commit(); return admin
|
||||
|
||||
|
||||
def create_session(session, admin: AdminUser, request: Request, seconds: int | None = None):
|
||||
raw_token = secrets.token_urlsafe(32)
|
||||
raw_csrf = secrets.token_urlsafe(32)
|
||||
now = datetime.now(timezone.utc)
|
||||
lifetime = seconds if seconds is not None else settings.web_session_seconds
|
||||
row = WebSession(admin_user_id=admin.id, token_hash=_digest(raw_token), csrf_hash=_digest(raw_csrf), expires_at=now + timedelta(seconds=lifetime),user_agent=request.headers.get("user-agent", "")[:512], remote_addr=request.client.host if request.client else None)
|
||||
session.add(row); admin.last_login_at = now; session.commit()
|
||||
return raw_token, raw_csrf
|
||||
|
||||
|
||||
def current_session(request: Request, session):
|
||||
token = request.cookies.get(settings.web_cookie_name)
|
||||
if not token: return None, None
|
||||
row = session.scalar(select(WebSession).where(WebSession.token_hash == _digest(token)))
|
||||
if not row or row.expires_at.replace(tzinfo=timezone.utc) < datetime.now(timezone.utc): return None, None
|
||||
admin = session.get(AdminUser, row.admin_user_id)
|
||||
if not admin or not admin.is_active: return None, None
|
||||
return row, admin
|
||||
|
||||
|
||||
def require_human(request: Request, session):
|
||||
web_session, admin = current_session(request, session)
|
||||
if not admin: raise HTTPException(401, "human login required")
|
||||
return web_session, admin
|
||||
|
||||
|
||||
def require_csrf(web_session: WebSession, csrf_token: str | None):
|
||||
if not csrf_token or not hmac.compare_digest(web_session.csrf_hash, _digest(csrf_token)):
|
||||
raise HTTPException(403, "invalid CSRF token")
|
||||
@@ -0,0 +1,107 @@
|
||||
"""ICMS Context Engine V1: ST applicability, ST withholding evidence, product role, CSOSN decision V2.
|
||||
|
||||
Pure functions, no I/O, no writes (every output has write_eligible=False). History is only used to
|
||||
validate/flag; it is never an input to a decision.
|
||||
"""
|
||||
ICMS_CONTEXT_ENGINE_VERSION = "icms-context-v1"
|
||||
|
||||
|
||||
def _digits(v):
|
||||
return "".join(c for c in str(v or "") if c.isdigit())
|
||||
|
||||
|
||||
def st_status(ncm_status, mva_status, cest_status, erp_cest, ba_item_cest, ba_ncm_sh_len):
|
||||
"""ST_CONFIRMED needs a unique BA Anexo 1 item whose CEST the ERP product itself carries
|
||||
(or a confirmed CEST). ST_NOT_APPLICABLE needs BOTH official lists (BA Anexo 1 and CEST) silent
|
||||
for a valid NCM. A CEST *candidate* never confirms ST."""
|
||||
if ncm_status != "EXACT_NCM":
|
||||
return "ST_UNKNOWN", "NCM_NOT_VALID"
|
||||
if mva_status == "MVA_RESOLVED":
|
||||
if cest_status == "CEST_CONFIRMED" or (_digits(erp_cest) and _digits(erp_cest) == _digits(ba_item_cest)):
|
||||
return "ST_CONFIRMED", "UNIQUE_BA_ANNEX1_ITEM_CORROBORATED_BY_CEST"
|
||||
return "ST_CANDIDATE", "UNIQUE_BA_ANNEX1_ITEM_WITHOUT_CEST_CORROBORATION"
|
||||
if mva_status == "MVA_CANDIDATES":
|
||||
return "ST_CANDIDATE", "MULTIPLE_BA_ANNEX1_ITEMS"
|
||||
if mva_status == "NO_MVA_MAPPING_FOUND":
|
||||
if cest_status == "NO_CEST_MAPPING_FOUND":
|
||||
return "ST_NOT_APPLICABLE", "NCM_ABSENT_FROM_BA_ANNEX1_AND_CEST_LIST"
|
||||
return "ST_UNKNOWN", "CEST_LISTED_BUT_BA_ANNEX1_ABSENT"
|
||||
return "ST_UNKNOWN", "NO_ST_EVIDENCE"
|
||||
|
||||
|
||||
def withheld_status(purchase_rows):
|
||||
"""purchase_rows: entrada item rows for the product with keys icms_retido (S/N), valor_retido, sub_valor.
|
||||
CONFIRMED only with positive purchase-side evidence; explicit 'N'/0 is NOT_CONFIRMED; no rows UNKNOWN."""
|
||||
if not purchase_rows:
|
||||
return "ST_WITHHOLDING_UNKNOWN", "NO_PURCHASE_ITEM_FOR_PRODUCT"
|
||||
for r in purchase_rows:
|
||||
if str(r.get("icms_retido") or "").strip().upper() == "S" or float(r.get("valor_retido") or 0) > 0 or float(r.get("sub_valor") or 0) > 0:
|
||||
return "ST_ALREADY_WITHHELD_CONFIRMED", "PURCHASE_ITEM_SHOWS_WITHHELD_ICMS_ST"
|
||||
return "ST_ALREADY_WITHHELD_NOT_CONFIRMED", "PURCHASE_ITEMS_FLAG_ICMS_RETIDO_N_AND_ZERO_ST_VALUES"
|
||||
|
||||
|
||||
def product_role(erp_tipo, prod_cozinha, has_recipe, sold_as_composition, service_flag, purchase_evidence):
|
||||
ev = []
|
||||
if service_flag:
|
||||
return "SERVICE", ["PRODUTO_SERVICO"]
|
||||
if str(erp_tipo or "").strip().upper() == "MATERIA-PRIMA":
|
||||
return "INPUT", ["C000025.TIPO=MATERIA-PRIMA"]
|
||||
own = []
|
||||
if prod_cozinha: own.append("PROD_COZINHA=1")
|
||||
if has_recipe: own.append("RECEITA_INGREDIENTES")
|
||||
if sold_as_composition: own.append("C000032.TIPO=COMPOSICAO")
|
||||
if own and purchase_evidence:
|
||||
return "UNKNOWN", own + ["ALSO_HAS_PURCHASE_EVIDENCE_CONFLICT"]
|
||||
if own:
|
||||
return "OWN_PRODUCTION", own
|
||||
if str(erp_tipo or "").strip().upper() == "VENDA" and purchase_evidence:
|
||||
return "RESALE", ["C000025.TIPO=VENDA", "PURCHASE_DOCUMENT_EVIDENCE"]
|
||||
return "UNKNOWN", ev or ["NO_OBJECTIVE_ROLE_EVIDENCE"]
|
||||
|
||||
|
||||
def csosn_v2(current, company_default, ncm_status, st, withheld, role, csosn_domain):
|
||||
"""States: CSOSN_RESOLVED / CANDIDATE / CONTEXT_REQUIRED / CONFLICT / UNRESOLVED.
|
||||
RESOLVED only where the fiscal condition is explicitly proven (ST confirmed + withholding confirmed + role
|
||||
known -> 500). Everything else stays conservative; never assumes 102."""
|
||||
cur = None if current is None or str(current).strip() == "" else str(current).strip()
|
||||
out = {"field": "CSOSN", "current_value": cur, "proposed_csosn": None, "write_eligible": False, "engine": ICMS_CONTEXT_ENGINE_VERSION}
|
||||
def res(state, reason, conf, action, proposed=None, **kw):
|
||||
return {**out, "status": state, "reason_code": reason, "confidence": conf, "action_class": action, "proposed_csosn": proposed, **kw}
|
||||
if ncm_status != "EXACT_NCM":
|
||||
return res("CSOSN_UNRESOLVED", "NCM_NOT_VALID", "NONE", "NO_RULE")
|
||||
if role in ("INPUT", "SERVICE"):
|
||||
return res("CSOSN_CONTEXT_REQUIRED", f"ROLE_{role}_NOT_A_MERCHANDISE_SALE_DEFAULT", "NONE", "CONTEXT_REQUIRED")
|
||||
if st == "ST_CONFIRMED":
|
||||
if withheld == "ST_ALREADY_WITHHELD_CONFIRMED" and role in ("RESALE",):
|
||||
state = "CSOSN_RESOLVED"
|
||||
return res(state, "ST_CONFIRMED_AND_WITHHOLDING_CONFIRMED_RESALE", "HIGH", "ALREADY_CORRECT" if cur == "500" else "AUTO_APPLY_SAFE", "500")
|
||||
if withheld == "ST_ALREADY_WITHHELD_NOT_CONFIRMED" and cur == "500":
|
||||
return res("CSOSN_CONFLICT", "CSOSN_500_BUT_PURCHASE_ITEMS_SHOW_NO_ST_WITHHELD", "MEDIUM", "REVIEW_REQUIRED")
|
||||
if withheld == "ST_ALREADY_WITHHELD_CONFIRMED" and cur != "500":
|
||||
return res("CSOSN_CONFLICT", "ST_WITHHELD_CONFIRMED_BUT_CSOSN_NOT_500", "MEDIUM", "REVIEW_REQUIRED", "500")
|
||||
return res("CSOSN_CONTEXT_REQUIRED", "ST_CONFIRMED_WITHHOLDING_OR_ROLE_UNPROVEN", "NONE", "CONTEXT_REQUIRED",
|
||||
consistency_flag=None if cur == "500" else "ST_CONFIRMED_BUT_CSOSN_NOT_500")
|
||||
if st in ("ST_CANDIDATE", "ST_UNKNOWN"):
|
||||
return res("CSOSN_CONTEXT_REQUIRED", f"{st}_APPLICABILITY_UNPROVEN", "NONE", "CONTEXT_REQUIRED")
|
||||
# ST_NOT_APPLICABLE: taxed vs exempt/not-taxed treatment (102 vs 400/300) has no source
|
||||
if role in ("RESALE", "OWN_PRODUCTION") and cur == company_default and cur in csosn_domain:
|
||||
return res("CSOSN_CANDIDATE", "NON_ST_MATCHES_COMPANY_DEFAULT_ICMS_TREATMENT_UNPROVEN", "MEDIUM", "REVIEW_REQUIRED", cur)
|
||||
if role == "UNKNOWN":
|
||||
return res("CSOSN_CONTEXT_REQUIRED", "PRODUCT_ROLE_UNKNOWN", "NONE", "CONTEXT_REQUIRED")
|
||||
return res("CSOSN_CANDIDATE" if cur else "CSOSN_CONTEXT_REQUIRED", "DIFFERS_FROM_OR_LACKS_COMPANY_DEFAULT_TREATMENT_UNPROVEN", "LOW", "REVIEW_REQUIRED" if cur else "CONTEXT_REQUIRED")
|
||||
|
||||
|
||||
def st_conflict_class(cest_status, mva_status, ncm_sh_len, erp_cest, ba_item_cest, erp_cfop, history_emitted_current, non_st_but_erp_5405):
|
||||
"""Primary label for an ST-related CFOP/CSOSN conflict, plus all evidence flags."""
|
||||
flags = []
|
||||
if non_st_but_erp_5405 or (_digits(erp_cest) and _digits(ba_item_cest) and _digits(erp_cest) != _digits(ba_item_cest)):
|
||||
flags.append("BA_ST_RULE_CONFLICT")
|
||||
if history_emitted_current:
|
||||
flags.append("ERP_HISTORY_CONFLICT")
|
||||
if cest_status == "CEST_CANDIDATES":
|
||||
flags.append("MULTIPLE_CEST")
|
||||
if cest_status == "CEST_DESCRIPTION_MISMATCH":
|
||||
flags.append("DESCRIPTION_MISMATCH")
|
||||
if ncm_sh_len is not None and ncm_sh_len <= 4:
|
||||
flags.append("NCM_TOO_BROAD")
|
||||
return (flags[0] if flags else "NO_EVIDENCE"), flags or ["NO_EVIDENCE"]
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Stable installation identity and safe operational metadata helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from .config import settings
|
||||
|
||||
SECRET_KEYS = {"password", "fb_password", "token", "agent_token", "api_key", "secret", "authorization", "connection_string"}
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def ensure_installation_id(state_path: str, *, new_installation: bool = False) -> str:
|
||||
"""Persist a UUID beside the local Agent state without using hostname/IP."""
|
||||
path = Path(state_path)
|
||||
identity_path = path.with_suffix(path.suffix + ".installation_id")
|
||||
if identity_path.exists() and not new_installation:
|
||||
value = identity_path.read_text(encoding="utf-8").strip()
|
||||
try:
|
||||
return str(uuid.UUID(value))
|
||||
except ValueError:
|
||||
pass
|
||||
value = str(uuid.uuid4())
|
||||
identity_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = identity_path.with_suffix(identity_path.suffix + ".tmp")
|
||||
temporary.write_text(value + "\n", encoding="utf-8")
|
||||
temporary.replace(identity_path)
|
||||
return value
|
||||
|
||||
|
||||
def mask_cnpj(value: str | None) -> str | None:
|
||||
digits = re.sub(r"\D", "", value or "")
|
||||
if len(digits) != 14:
|
||||
return None if not value else "***"
|
||||
return f"{digits[:2]}.***.***/{digits[8:12]}-{digits[12:]}"
|
||||
|
||||
|
||||
def sanitize_database_path(value: str | None) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
normalized = str(value).replace("\\", "/")
|
||||
return normalized.rsplit("/", 1)[-1][:255]
|
||||
|
||||
|
||||
def _safe_value(key: str, value):
|
||||
if key.lower() in SECRET_KEYS:
|
||||
return "[REDACTED]"
|
||||
if isinstance(value, dict):
|
||||
return {str(k): _safe_value(str(k), v) for k, v in value.items() if str(k).lower() not in SECRET_KEYS}
|
||||
if isinstance(value, list):
|
||||
return [_safe_value(key, item) for item in value[:100]]
|
||||
return value
|
||||
|
||||
|
||||
def sanitize_registration(payload: dict) -> dict:
|
||||
allowed = {
|
||||
"installation_id", "display_name", "customer_code", "notes", "hostname", "windows_version", "machine_fingerprint",
|
||||
"company_code", "company_name", "company_cnpj", "company_uf", "company_crt", "company_contribuinte_ipi", "firebird_version", "firebird_ods",
|
||||
"database_identifier", "database_path", "capabilities", "fiscal_engine_version", "dataset_versions", "started_at",
|
||||
"status", "schema_fingerprint", "agent_version", "last_job_id", "coverage", "health", "event_type", "message", "context",
|
||||
}
|
||||
result = {key: _safe_value(key, value) for key, value in payload.items() if key in allowed}
|
||||
if "company_cnpj" in result:
|
||||
result["company_cnpj_masked"] = mask_cnpj(result.pop("company_cnpj"))
|
||||
if "database_path" in result:
|
||||
result["database_identifier"] = sanitize_database_path(result.pop("database_path"))
|
||||
if result.get("installation_id"):
|
||||
result["installation_id"] = str(uuid.UUID(str(result["installation_id"])))
|
||||
return result
|
||||
|
||||
|
||||
def compute_agent_health(snapshot, now: datetime | None = None, *, supplemental=None) -> tuple[str, str]:
|
||||
now = now or utcnow()
|
||||
last_seen = getattr(snapshot, "last_seen_at", None)
|
||||
if last_seen is None:
|
||||
return "UNKNOWN", "no heartbeat received"
|
||||
if last_seen.tzinfo is None:
|
||||
last_seen = last_seen.replace(tzinfo=timezone.utc)
|
||||
source = supplemental or snapshot
|
||||
if any(getattr(candidate, "agent_health", None) == "FAILED" or getattr(candidate, "last_status", None) == "FAILED" for candidate in (snapshot, supplemental) if candidate is not None):
|
||||
return "FAILED", "persistent technical failure"
|
||||
offline_after = settings.agent_offline_after_seconds or settings.agent_heartbeat_interval_seconds * 3
|
||||
if (now - last_seen).total_seconds() > offline_after:
|
||||
return "OFFLINE", f"last heartbeat exceeds {offline_after}s"
|
||||
if getattr(source, "failed_products", 0) or getattr(source, "last_job_status", "") == "FAILED":
|
||||
return "DEGRADED", "recent technical or job failure"
|
||||
return "ONLINE", "recent heartbeat without critical technical failure"
|
||||
|
||||
|
||||
_VERSION_RE = re.compile(r"^v?(\d+(?:\.\d+)*)(?:[-_.]?(rc|beta|alpha|b|a)[-_.]?(\d*))?$", re.IGNORECASE)
|
||||
_PRE_RANK = {"alpha": -3, "a": -3, "beta": -2, "b": -2, "rc": -1}
|
||||
|
||||
|
||||
def parse_version(value: str | None):
|
||||
"""Comparable tuple for '1.1.7', '1.0.0rc1', ... (a pre-release sorts below its final); None if unparsable."""
|
||||
match = _VERSION_RE.match((value or "").strip())
|
||||
if not match:
|
||||
return None
|
||||
parts = [int(x) for x in match.group(1).split(".")]
|
||||
while len(parts) < 4:
|
||||
parts.append(0)
|
||||
pre = (_PRE_RANK[match.group(2).lower()], int(match.group(3) or 0)) if match.group(2) else (0, 0)
|
||||
return (*parts, *pre)
|
||||
|
||||
|
||||
def reference_agent_version(session) -> str:
|
||||
"""The version an Agent should be running: the highest PUBLISHED Windows Agent release (REVOKED/DRAFT are
|
||||
never a target). Falls back to the configured server version when no release is published."""
|
||||
from sqlalchemy import select
|
||||
|
||||
from .models import AgentRelease
|
||||
|
||||
best, best_key = None, None
|
||||
for version in session.scalars(select(AgentRelease.version).where(AgentRelease.status == "PUBLISHED", AgentRelease.platform == "windows")):
|
||||
key = parse_version(version)
|
||||
if key is not None and (best_key is None or key > best_key):
|
||||
best, best_key = version, key
|
||||
return best or settings.agent_server_version
|
||||
|
||||
|
||||
def compute_version_drift(agent_version: str | None, reference: str | None = None) -> str:
|
||||
"""CURRENT when the Agent is at (or ahead of) the reference release, OUTDATED when behind it."""
|
||||
if not agent_version:
|
||||
return "UNKNOWN"
|
||||
reference = reference or settings.agent_server_version
|
||||
have, want = parse_version(agent_version), parse_version(reference)
|
||||
if have is None or want is None:
|
||||
return "CURRENT" if agent_version == reference else "OUTDATED"
|
||||
return "CURRENT" if have >= want else "OUTDATED"
|
||||
|
||||
|
||||
def outdated_versions(session, reference: str) -> list[str]:
|
||||
"""Distinct Agent versions in use that are behind the reference (used to keep SQL filters exact)."""
|
||||
from sqlalchemy import select
|
||||
|
||||
from .models import Agent
|
||||
|
||||
used = [v for v in session.scalars(select(Agent.agent_version).where(Agent.agent_version.is_not(None)).distinct())]
|
||||
return [v for v in used if compute_version_drift(v, reference) == "OUTDATED"]
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Job queue for Agent orchestration: create (idempotent per kind), claim one job at
|
||||
a time per Agent, complete, fail. Also the central sync scheduler helpers.
|
||||
|
||||
Every fiscal job is READ-ONLY on the ERP (INITIAL_SYNC, SYNC_INCREMENTAL,
|
||||
REPROCESS_ALL, REPROCESS_PRODUCTS); automation never authorizes Firebird writes.
|
||||
"""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select, update, func, text
|
||||
|
||||
from .config import settings
|
||||
from .models import Installation, Job
|
||||
|
||||
FULL_KINDS = ("INITIAL_SYNC", "REPROCESS_ALL") # read every product, reclassify all
|
||||
SYNC_KINDS = FULL_KINDS + ("SYNC_INCREMENTAL",) # jobs that count as "a sync happened"
|
||||
_VALID_KINDS = set(SYNC_KINDS) | {"REPROCESS_PRODUCTS"}
|
||||
_MAX_ERROR_LENGTH = 2000
|
||||
STALE_RUNNING_AFTER = timedelta(hours=1) # an Agent that died mid-job must not block the queue forever
|
||||
|
||||
|
||||
def _aware(value):
|
||||
if value is not None and value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value
|
||||
|
||||
|
||||
def create_job(session, agent, kind, payload=None):
|
||||
"""Create a job. Sync-type jobs are idempotent: if this Agent already has a
|
||||
PENDING/RUNNING job that covers the request, that job is returned instead
|
||||
(no duplicates, never two full jobs at once)."""
|
||||
if kind not in _VALID_KINDS:
|
||||
raise ValueError(f"unsupported job kind: {kind}")
|
||||
if kind in ("REPROCESS_ALL", "REPROCESS_PRODUCTS", "SYNC_INCREMENTAL"):
|
||||
_flag_enrichment_reevaluation(session, agent) # "Reprocessar"/"Sincronizar" also re-evaluate the tracked, not-yet-enriched products (KB/cache first), even when an equivalent job is already queued
|
||||
session.commit()
|
||||
if kind in SYNC_KINDS:
|
||||
# A full job subsumes an incremental one, not the other way round: asking for a
|
||||
# full reprocess while only an incremental is queued must still queue the full.
|
||||
blocking = FULL_KINDS if kind in FULL_KINDS else SYNC_KINDS
|
||||
existing = session.scalar(select(Job).where(Job.agent_id == agent.id, Job.kind.in_(blocking), Job.status.in_(("PENDING", "RUNNING"))).order_by(Job.id).limit(1))
|
||||
if existing is not None:
|
||||
return existing
|
||||
job = Job(agent_id=agent.id, kind=kind, status="PENDING", payload=payload or {})
|
||||
session.add(job)
|
||||
session.commit()
|
||||
return job
|
||||
|
||||
|
||||
def _flag_enrichment_reevaluation(session, agent):
|
||||
try:
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is not None and installation.features_configured and installation.enrichment_mode != "DISABLED":
|
||||
from .enrichment.status import mark_for_reevaluation
|
||||
mark_for_reevaluation(session, installation.id)
|
||||
except Exception: # noqa: BLE001 - never let the enrichment bookkeeping break job creation
|
||||
pass
|
||||
|
||||
|
||||
def recover_stale_running(session, agent, now=None):
|
||||
now = now or datetime.now(timezone.utc)
|
||||
cutoff = now - STALE_RUNNING_AFTER
|
||||
stale = session.scalars(select(Job).where(Job.agent_id == agent.id, Job.status == "RUNNING")).all()
|
||||
changed = False
|
||||
for job in stale:
|
||||
if job.started_at is not None and _aware(job.started_at) < cutoff:
|
||||
job.status, job.finished_at, job.error = "FAILED", now, "stale: Agent stopped reporting while RUNNING"
|
||||
changed = True
|
||||
if changed:
|
||||
session.commit()
|
||||
|
||||
|
||||
def claim_next_job(session, agent):
|
||||
"""Atomically claim the oldest PENDING job for this agent only — and only when
|
||||
processing is enabled centrally and no other job of this Agent is RUNNING
|
||||
(one fiscal job at a time per Firebird)."""
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
|
||||
if installation is not None and (not installation.processing_enabled or not installation.features_configured or not installation.corrector_enabled):
|
||||
return None # WAITING_FEATURE_CONFIGURATION / corrector switched off centrally: nothing is handed to the Agent
|
||||
recover_stale_running(session, agent)
|
||||
if session.scalar(select(Job.id).where(Job.agent_id == agent.id, Job.status == "RUNNING").limit(1)) is not None:
|
||||
return None
|
||||
pending = session.execute(
|
||||
select(Job.id, Job.kind)
|
||||
.where(Job.agent_id == agent.id, Job.status == "PENDING")
|
||||
.order_by(Job.created_at.asc(), Job.id.asc())
|
||||
.limit(10)
|
||||
).all()
|
||||
if not pending:
|
||||
return None
|
||||
limit = settings.heavy_job_concurrency_limit
|
||||
heavy_slot_free = True
|
||||
if limit > 0 and any(kind in FULL_KINDS for _, kind in pending):
|
||||
if session.bind is not None and session.bind.dialect.name == "postgresql":
|
||||
session.execute(text("SELECT pg_advisory_xact_lock(7305001)")) # serialize slot accounting across concurrent pollers; released at commit
|
||||
running_heavy = session.scalar(select(func.count()).select_from(Job).where(Job.status == "RUNNING", Job.kind.in_(FULL_KINDS),
|
||||
Job.started_at > datetime.now(timezone.utc) - STALE_RUNNING_AFTER)) or 0
|
||||
heavy_slot_free = running_heavy < limit
|
||||
# oldest job this Agent may run now: a light job is never held behind a heavy one that is waiting for a slot
|
||||
candidate = next((job_id for job_id, kind in pending if kind not in FULL_KINDS or heavy_slot_free), None)
|
||||
if candidate is None:
|
||||
session.rollback() # releases the advisory lock; the heavy job stays PENDING (queue) and the Agent keeps polling
|
||||
return None
|
||||
now = datetime.now(timezone.utc)
|
||||
result = session.execute(
|
||||
update(Job)
|
||||
.where(Job.id == candidate, Job.status == "PENDING")
|
||||
.values(status="RUNNING", started_at=now, attempts=Job.attempts + 1)
|
||||
)
|
||||
session.commit()
|
||||
if result.rowcount != 1:
|
||||
return None # another poller won the race; caller can poll again
|
||||
return session.get(Job, candidate)
|
||||
|
||||
|
||||
def complete_job(session, job, result_summary):
|
||||
result = session.execute(
|
||||
update(Job)
|
||||
.where(Job.id == job.id, Job.status == "RUNNING")
|
||||
.values(status="COMPLETED", finished_at=datetime.now(timezone.utc), result_summary=result_summary)
|
||||
)
|
||||
session.commit()
|
||||
return result.rowcount == 1
|
||||
|
||||
|
||||
def fail_job(session, job, error):
|
||||
sanitized = str(error)[:_MAX_ERROR_LENGTH]
|
||||
result = session.execute(
|
||||
update(Job)
|
||||
.where(Job.id == job.id, Job.status == "RUNNING")
|
||||
.values(status="FAILED", finished_at=datetime.now(timezone.utc), error=sanitized)
|
||||
)
|
||||
session.commit()
|
||||
return result.rowcount == 1
|
||||
|
||||
|
||||
def schedule_automatic_sync(session, agent, installation, snapshot, now=None):
|
||||
"""Called from the Agent heartbeat. Central scheduler, idempotent:
|
||||
- first heartbeat of an installation with no initial coverage and no sync job
|
||||
ever created -> exactly ONE INITIAL_SYNC;
|
||||
- otherwise, if processing is enabled and the last sync attempt is older than
|
||||
the installation's interval, and nothing is queued/running -> ONE SYNC_INCREMENTAL.
|
||||
Returns the created job or None."""
|
||||
if installation is None or not installation.processing_enabled or not installation.features_configured or not installation.corrector_enabled:
|
||||
return None
|
||||
now = now or datetime.now(timezone.utc)
|
||||
sync_jobs = session.scalars(select(Job).where(Job.agent_id == agent.id, Job.kind.in_(SYNC_KINDS)).order_by(Job.id.desc())).all()
|
||||
if any(job.status in ("PENDING", "RUNNING") for job in sync_jobs):
|
||||
return None
|
||||
never_synced = not any(job.kind == "INITIAL_SYNC" or job.kind == "REPROCESS_ALL" for job in sync_jobs) and not int(getattr(snapshot, "total_products", 0) or 0)
|
||||
if never_synced and not sync_jobs:
|
||||
return create_job(session, agent, "INITIAL_SYNC", {"reason": "INITIAL_SYNC"})
|
||||
last_attempt = max((_aware(job.finished_at or job.created_at) for job in sync_jobs if (job.finished_at or job.created_at)), default=None)
|
||||
if last_attempt is not None and (now - last_attempt).total_seconds() >= installation.sync_interval_seconds:
|
||||
return create_job(session, agent, "SYNC_INCREMENTAL", {"reason": "SCHEDULED"})
|
||||
return None
|
||||
@@ -0,0 +1,65 @@
|
||||
"""FastAPI application factory."""
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from .config import settings
|
||||
from .db import SessionLocal, database_is_available
|
||||
from .api import router
|
||||
from .agent_distribution import router as agent_distribution_router
|
||||
from .control_api import router as control_api_router
|
||||
from .web import router as web_router
|
||||
from .version import GTI_FISCAL_VERSION
|
||||
|
||||
logging.basicConfig(level=getattr(logging, settings.log_level.upper(), logging.INFO))
|
||||
logger = logging.getLogger("gti_fiscal_hub")
|
||||
|
||||
|
||||
def create_app(database_probe: Callable[[], bool] = database_is_available, session_factory=SessionLocal, bootstrap_token: str | None = None) -> FastAPI:
|
||||
app = FastAPI(title="GTI Fiscal Hub", version=GTI_FISCAL_VERSION)
|
||||
if settings.environment == "production" and settings.agent_dev_token:
|
||||
raise RuntimeError("AGENT_DEV_TOKEN must not be set when APP_ENV=production (register real Agents with central/scripts/register_agent.py)")
|
||||
logger.info("starting GTI Fiscal Hub (environment=%s)", settings.environment)
|
||||
|
||||
@app.middleware("http")
|
||||
async def add_security_headers(request: Request, call_next):
|
||||
response = await call_next(request)
|
||||
response.headers["X-Content-Type-Options"] = "nosniff"
|
||||
response.headers["X-Frame-Options"] = "SAMEORIGIN"
|
||||
response.headers["X-XSS-Protection"] = "1; mode=block"
|
||||
response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
|
||||
if getattr(settings, "web_cookie_secure", False):
|
||||
response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains"
|
||||
return response
|
||||
|
||||
@app.get("/health")
|
||||
def health() -> dict[str, str]:
|
||||
if not database_probe():
|
||||
logger.warning("health check failed: postgres unavailable")
|
||||
raise HTTPException(status_code=503, detail="postgres unavailable")
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.get("/version")
|
||||
def version() -> dict[str, str]:
|
||||
from .classification import DECISION_VERSION, ENGINE_VERSION
|
||||
from .version import API_COMPAT_MAJOR_MINOR, GTI_FISCAL_VERSION
|
||||
return {"version": GTI_FISCAL_VERSION, "api_compat": API_COMPAT_MAJOR_MINOR, "engine_version": ENGINE_VERSION, "decision_version": DECISION_VERSION}
|
||||
|
||||
app.include_router(router(session_factory, settings.agent_dev_token if bootstrap_token is None else bootstrap_token))
|
||||
app.include_router(control_api_router(session_factory))
|
||||
app.include_router(agent_distribution_router(session_factory))
|
||||
app.include_router(web_router(session_factory))
|
||||
if settings.official_refresh_enabled and session_factory is SessionLocal and "pytest" not in sys.modules:
|
||||
@app.on_event("startup")
|
||||
def _start_official_refresh():
|
||||
from .datasets.updater import start_scheduler
|
||||
start_scheduler(session_factory)
|
||||
|
||||
app.mount("/static", StaticFiles(directory="central/static"), name="static")
|
||||
return app
|
||||
|
||||
|
||||
app = create_app()
|
||||
@@ -0,0 +1,973 @@
|
||||
"""PostgreSQL records owned by the Fiscal Hub."""
|
||||
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Boolean, CheckConstraint, DateTime, ForeignKey, Integer, JSON, String, Text, UniqueConstraint, func
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class Dataset(Base):
|
||||
"""The Reference Data Registry (Missão 5): every official reference table the Hub knows about,
|
||||
one row per ingested version. Pre-existing kinds (NCM, TIPI_IPI, CEST, ...) keep their original
|
||||
auto-activate behavior in central/app/datasets/service.py, untouched by this mission - the new
|
||||
DRAFT->admin-approves->ACTIVE workflow applies to datasets registered through the new
|
||||
scripts/ingest_reference_dataset.py CLI / the admin "Tabelas oficiais" screen going forward."""
|
||||
__tablename__ = "datasets"
|
||||
__table_args__ = (UniqueConstraint("provider", "sha256", name="uq_dataset_provider_hash"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
kind: Mapped[str] = mapped_column(String(32), index=True)
|
||||
provider: Mapped[str] = mapped_column(String(128))
|
||||
sha256: Mapped[str] = mapped_column(String(64))
|
||||
raw_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
version_label: Mapped[str] = mapped_column(String(256))
|
||||
act: Mapped[str] = mapped_column(Text)
|
||||
source_url: Mapped[str] = mapped_column(Text)
|
||||
downloaded_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
status: Mapped[str] = mapped_column(String(32), index=True) # ACTIVE | SUPERSEDED (legacy kinds) | DRAFT | APPROVED | VIGENTE | OBSOLETO | REJECTED (new registry workflow)
|
||||
record_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
licence_terms: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
valid_from: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
valid_to: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
file_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
raw_file_path: Mapped[str | None] = mapped_column(Text, nullable=True) # local path only (sources/raw/... , gitignored) - never the file content in Postgres
|
||||
parse_errors: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
submitted_by: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
approved_by: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
approved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class NcmEntry(Base):
|
||||
__tablename__ = "ncm_entries"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "code", "valid_from", "valid_to", name="uq_ncm_versioned_entry"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
code: Mapped[str] = mapped_column(String(16), index=True)
|
||||
description: Mapped[str] = mapped_column(Text)
|
||||
valid_from: Mapped[str] = mapped_column(String(32))
|
||||
valid_to: Mapped[str] = mapped_column(String(32))
|
||||
|
||||
|
||||
class LegacyNcmEntry(Base):
|
||||
__tablename__ = "legacy_ncm_entries"
|
||||
code: Mapped[str] = mapped_column(String(16), primary_key=True)
|
||||
source: Mapped[str] = mapped_column(Text)
|
||||
source_date: Mapped[str] = mapped_column(String(32))
|
||||
reason: Mapped[str] = mapped_column(Text)
|
||||
successor_code: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
|
||||
|
||||
class CestEntry(Base):
|
||||
__tablename__ = "cest_entries"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "code", name="uq_cest_versioned_entry"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
code: Mapped[str] = mapped_column(String(16), index=True)
|
||||
description: Mapped[str] = mapped_column(Text)
|
||||
annex: Mapped[str] = mapped_column(String(8))
|
||||
|
||||
|
||||
class CestNcmEntry(Base):
|
||||
__tablename__ = "cest_ncm_entries"
|
||||
__table_args__ = (UniqueConstraint("cest_entry_id", "ncm_code", name="uq_cest_ncm_link"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
cest_entry_id: Mapped[int] = mapped_column(ForeignKey("cest_entries.id"), index=True)
|
||||
ncm_code: Mapped[str] = mapped_column(String(16), index=True)
|
||||
|
||||
|
||||
class TipiEntry(Base):
|
||||
__tablename__ = "tipi_entries"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
ncm: Mapped[str] = mapped_column(String(8), index=True)
|
||||
ex: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
description: Mapped[str] = mapped_column(Text)
|
||||
ipi_rate: Mapped[float | None] = mapped_column(nullable=True)
|
||||
ipi_rate_text: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
source_row: Mapped[int] = mapped_column(Integer)
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
|
||||
|
||||
class BaStMvaEntry(Base):
|
||||
__tablename__ = "ba_st_mva_entries"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
item: Mapped[str] = mapped_column(String(32))
|
||||
segment: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
cest: Mapped[str | None] = mapped_column(String(16), index=True, nullable=True)
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
mva_original: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
mva_adjusted: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
agreement_info: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
source_page: Mapped[int] = mapped_column(Integer)
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
|
||||
class BaStMvaNcmSh(Base):
|
||||
__tablename__ = "ba_st_mva_ncm_sh"
|
||||
__table_args__ = (UniqueConstraint("entry_id", "ncm_sh", name="uq_ba_st_mva_entry_ncm_sh"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
entry_id: Mapped[int] = mapped_column(ForeignKey("ba_st_mva_entries.id"), index=True)
|
||||
ncm_sh: Mapped[str] = mapped_column(String(32), index=True)
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
|
||||
|
||||
class FiscalCodeEntry(Base):
|
||||
"""Small national fiscal code domains sourced from the official NF-e MOC
|
||||
manual: ORIGEM_MERCADORIA, CST_ICMS, CSOSN, CST_PIS, CST_COFINS."""
|
||||
__tablename__ = "fiscal_code_entries"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "domain", "code", name="uq_fiscal_code_entry"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
domain: Mapped[str] = mapped_column(String(32), index=True)
|
||||
code: Mapped[str] = mapped_column(String(8), index=True)
|
||||
description: Mapped[str] = mapped_column(Text)
|
||||
source_section: Mapped[str] = mapped_column(String(96))
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
|
||||
|
||||
class CfopEntry(Base):
|
||||
"""CFOP (Código Fiscal de Operações e Prestações) — Portal Nacional NF-e."""
|
||||
__tablename__ = "cfop_entries"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "code", name="uq_cfop_entry"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
code: Mapped[str] = mapped_column(String(8), index=True)
|
||||
title: Mapped[str] = mapped_column(Text)
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
effective_from: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
effective_to: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
ind_nfe: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
ind_comunica: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
ind_transp: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
ind_devol: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
ind_retor: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
ind_anula: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
ind_remes: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
ind_comb: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
ind_exc_ibscbs: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
|
||||
|
||||
class FcpRate(Base):
|
||||
"""FCP (Fundo de Combate à Pobreza) alíquotas por UF — Portal Nacional NF-e."""
|
||||
__tablename__ = "fcp_rates"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "uf", name="uq_fcp_rate_uf"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
uf_code: Mapped[str] = mapped_column(String(4))
|
||||
uf: Mapped[str] = mapped_column(String(4), index=True)
|
||||
uf_name: Mapped[str] = mapped_column(String(64))
|
||||
aliq_1: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
aliq_2: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
aliq_3: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
observacao: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
|
||||
class AnpProductCode(Base):
|
||||
"""cProdANP — official code list used by NF-e, published on the Portal Nacional NF-e."""
|
||||
__tablename__ = "anp_product_codes"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "cprod_anp", name="uq_anp_product_code"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
cprod_anp: Mapped[str] = mapped_column(String(16), index=True)
|
||||
|
||||
|
||||
class NcmUtribEntry(Base):
|
||||
"""NCM x uTrib (unidade tributável de comércio exterior) — Portal Nacional NF-e."""
|
||||
__tablename__ = "ncm_utrib_entries"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "ncm", name="uq_ncm_utrib_entry"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
ncm: Mapped[str] = mapped_column(String(16), index=True)
|
||||
effective_from: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
effective_to: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
utrib_abbreviation: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
utrib_description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
|
||||
class RtcBiofuelIndexEntry(Base):
|
||||
"""RTC pBioObrigatorio (índice de biocombustível) by cProdANP — Portal Nacional NF-e."""
|
||||
__tablename__ = "rtc_biofuel_index_entries"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
cprod_anp: Mapped[str | None] = mapped_column(String(16), index=True, nullable=True)
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
p_bio_percent: Mapped[float | None] = mapped_column(nullable=True)
|
||||
effective_from: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
effective_to: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
is_historical: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
|
||||
|
||||
class RtcCredPresEntry(Base):
|
||||
"""RTC cCredPres (crédito presumido IBS/CBS), LC 214/2025 — Portal Nacional NF-e.
|
||||
Keeps every official field distinctly so future automatic determination
|
||||
doesn't require re-parsing the source."""
|
||||
__tablename__ = "rtc_cred_pres_entries"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "code", name="uq_rtc_cred_pres_code"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
code: Mapped[str] = mapped_column(String(8), index=True)
|
||||
description: Mapped[str] = mapped_column(Text)
|
||||
legal_basis: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
apropria_via_nf: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
||||
apropria_via_evento: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
||||
ind_deduz_cred_pres: Mapped[str | None] = mapped_column(String(8), nullable=True)
|
||||
ind_g_cbs_cred_pres: Mapped[str | None] = mapped_column(String(8), nullable=True)
|
||||
ind_g_ibs_cred_pres: Mapped[str | None] = mapped_column(String(8), nullable=True)
|
||||
aliquota_cbs_text: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
aliquota_ibs_text: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
p_aliq_cred_pres_cbs: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
p_aliq_cred_pres_ibs: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
p_red_transicao_ibs: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
# Free text in some rows (e.g. "cClass de alíquota zero"), not always the numeric code.
|
||||
cclass_nota_referenciada: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
d_ini_vig_cbs: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
d_fim_vig_cbs: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
d_ini_vig_ibs: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
d_fim_vig_ibs: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
|
||||
|
||||
class Tenant(Base):
|
||||
"""A client. GTI Gerenciador (the panel) is the single source of truth —
|
||||
see central/app/client_sync.py. `source='GTI_GERENCIADOR'` rows are
|
||||
upserted by that sync, keyed by `external_client_id` (Gerenciador's
|
||||
CLIENTE.ID, the stable technical PK — CLIENTE.CODIGO is editable and
|
||||
never used as identity, same reasoning as FUNCIONARIO_PAINEL.ID in SSO).
|
||||
`source='LOCAL'` rows predate this sync (dev/test tenants) and are never
|
||||
offered for a new installation."""
|
||||
__tablename__ = "tenants"
|
||||
__table_args__ = (UniqueConstraint("source", "external_client_id", name="uq_tenant_external_identity"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(128), unique=True)
|
||||
source: Mapped[str] = mapped_column(String(16), default="LOCAL", server_default="LOCAL") # LOCAL | GTI_GERENCIADOR
|
||||
external_client_id: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
|
||||
external_code: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
external_cnpj: Mapped[str | None] = mapped_column(String(20), nullable=True) # digits only, for the client selector search
|
||||
active: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
|
||||
synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class Agent(Base):
|
||||
__tablename__ = "agents"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True) # company_id
|
||||
name: Mapped[str] = mapped_column(String(128))
|
||||
token_hash: Mapped[str] = mapped_column(String(64), unique=True)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
|
||||
last_seen_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_status: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
schema_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
agent_version: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class Installation(Base):
|
||||
__tablename__ = "installations"
|
||||
__table_args__ = (UniqueConstraint("installation_id", name="uq_installation_identity"), UniqueConstraint("agent_id", name="uq_installation_agent"))
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
agent_id: Mapped[int] = mapped_column(ForeignKey("agents.id"), index=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
|
||||
installation_id: Mapped[str] = mapped_column(String(36), index=True)
|
||||
display_name: Mapped[str] = mapped_column(String(160), default="", server_default="")
|
||||
customer_code: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
notes: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
hostname: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
|
||||
windows_version: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
machine_fingerprint: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
company_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
company_name: Mapped[str | None] = mapped_column(String(160), nullable=True)
|
||||
company_cnpj_masked: Mapped[str | None] = mapped_column(String(24), nullable=True)
|
||||
company_uf: Mapped[str | None] = mapped_column(String(2), nullable=True)
|
||||
company_crt: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
company_contribuinte_ipi: Mapped[str | None] = mapped_column(String(1), nullable=True) # C000004.CONTRIBUINTE_IPI, same auto-discovery as company_crt
|
||||
firebird_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
firebird_ods: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
database_identifier: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
capabilities: Mapped[dict] = mapped_column(JSON, default=dict, server_default="{}")
|
||||
fiscal_engine_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
dataset_versions: Mapped[dict] = mapped_column(JSON, default=dict, server_default="{}")
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_seen_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
||||
last_status: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
# Central execution control (no local .env toggle): the Hub decides whether this
|
||||
# installation's Agent executes jobs, and how often the automatic incremental sync runs.
|
||||
processing_enabled: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
|
||||
sync_interval_seconds: Mapped[int] = mapped_column(Integer, default=1800, server_default="1800")
|
||||
last_sync_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_sync_changed: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
# RTC auto-apply is authorized centrally per installation and is OFF until an operator explicitly turns it on (the Agent has its own second switch).
|
||||
rtc_auto_apply_authorized: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") # legacy (0027); rtc_write_mode is the source of truth
|
||||
rtc_write_mode: Mapped[str] = mapped_column(String(24), default="DISABLED", server_default="DISABLED") # DISABLED | DRY_RUN | APPROVAL_REQUIRED | ENABLED
|
||||
# Same model as rtc_write_mode, an entirely separate switch: authorizes ONLY the CFOP_IPPT_WRITE_V1
|
||||
# capability (gti_fiscal_agent.cfop_ippt_auto_apply), never RTC. DISABLED by default for every
|
||||
# installation - an operator must explicitly turn it on, per installation.
|
||||
cfop_ippt_write_mode: Mapped[str] = mapped_column(String(24), default="DISABLED", server_default="DISABLED") # DISABLED | DRY_RUN | APPROVAL_REQUIRED | ENABLED
|
||||
# Per-installation FUNCTIONS (central, no local switch). Existing installations keep working exactly as before (corrector on, features configured);
|
||||
# a NEW installation starts WAITING_FEATURE_CONFIGURATION (features_configured=False): it heartbeats but processes nothing until an admin chooses.
|
||||
corrector_enabled: Mapped[bool] = mapped_column(Boolean, default=False, server_default="true")
|
||||
enrichment_mode: Mapped[str] = mapped_column(String(16), default="DISABLED", server_default="DISABLED") # DISABLED | DRY_RUN | ENABLED
|
||||
features_configured: Mapped[bool] = mapped_column(Boolean, default=False, server_default="true")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class AgentSnapshot(Base):
|
||||
__tablename__ = "agent_snapshots"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
installation_id: Mapped[int] = mapped_column(ForeignKey("installations.id"), unique=True, index=True)
|
||||
total_products: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
fiscally_complete: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
safe_blocked: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
failed_products: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
coverage_percent: Mapped[float] = mapped_column(default=0.0, server_default="0")
|
||||
blockers: Mapped[dict] = mapped_column(JSON, default=dict, server_default="{}")
|
||||
last_job_status: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
last_job_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_successful_job_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
agent_health: Mapped[str] = mapped_column(String(16), default="UNKNOWN", server_default="UNKNOWN", index=True)
|
||||
health_reason: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
version_drift: Mapped[str] = mapped_column(String(16), default="UNKNOWN", server_default="UNKNOWN", index=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), index=True)
|
||||
|
||||
|
||||
class AgentEvent(Base):
|
||||
__tablename__ = "agent_events"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
|
||||
installation_id: Mapped[int] = mapped_column(ForeignKey("installations.id"), index=True)
|
||||
agent_id: Mapped[int] = mapped_column(ForeignKey("agents.id"), index=True)
|
||||
severity: Mapped[str] = mapped_column(String(16), index=True)
|
||||
event_type: Mapped[str] = mapped_column(String(48), index=True)
|
||||
message: Mapped[str] = mapped_column(Text)
|
||||
structured_context: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
job_id: Mapped[int | None] = mapped_column(ForeignKey("jobs.id"), nullable=True, index=True)
|
||||
product_code: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
blocker_id: Mapped[int | None] = mapped_column(ForeignKey("blocked_products.id"), nullable=True, index=True)
|
||||
is_audit: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false", index=True)
|
||||
occurred_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), index=True)
|
||||
|
||||
|
||||
class AdminUser(Base):
|
||||
"""Human principal of the control panel.
|
||||
|
||||
LOCAL: break-glass account with a password (create_admin.py).
|
||||
GTI_GERENCIADOR: collaborator provisioned by SSO (central/app/sso.py) — no
|
||||
password at all; identified by the panel's immutable subject.
|
||||
"""
|
||||
__tablename__ = "admin_users"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("external_subject", name="uq_admin_users_external_subject"),
|
||||
CheckConstraint(
|
||||
"(auth_source = 'LOCAL' AND password_hash IS NOT NULL AND external_subject IS NULL)"
|
||||
" OR (auth_source = 'GTI_GERENCIADOR' AND password_hash IS NULL AND external_subject IS NOT NULL)",
|
||||
name="ck_admin_users_auth_source",
|
||||
),
|
||||
)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
username: Mapped[str] = mapped_column(String(128), unique=True, index=True)
|
||||
password_hash: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
last_login_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
auth_source: Mapped[str] = mapped_column(String(16), default="LOCAL", server_default="LOCAL")
|
||||
external_subject: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
display_name: Mapped[str | None] = mapped_column(String(160), nullable=True)
|
||||
external_role: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class SsoUsedAssertion(Base):
|
||||
"""jti of every accepted SSO assertion, kept past its expiry to reject replays."""
|
||||
__tablename__ = "sso_used_assertions"
|
||||
jti: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
subject: Mapped[str] = mapped_column(String(128))
|
||||
used_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
|
||||
|
||||
|
||||
class WebSession(Base):
|
||||
__tablename__ = "web_sessions"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
admin_user_id: Mapped[int] = mapped_column(ForeignKey("admin_users.id"), index=True)
|
||||
token_hash: Mapped[str] = mapped_column(String(64), unique=True, index=True)
|
||||
csrf_hash: Mapped[str] = mapped_column(String(64))
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
user_agent: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
remote_addr: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
|
||||
|
||||
class AuditEvent(Base):
|
||||
__tablename__ = "audit_events"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
admin_user_id: Mapped[int | None] = mapped_column(ForeignKey("admin_users.id"), nullable=True, index=True)
|
||||
tenant_id: Mapped[int | None] = mapped_column(ForeignKey("tenants.id"), nullable=True, index=True)
|
||||
action: Mapped[str] = mapped_column(String(64), index=True)
|
||||
target_type: Mapped[str] = mapped_column(String(64))
|
||||
target_id: Mapped[str] = mapped_column(String(128))
|
||||
context: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
severity: Mapped[str] = mapped_column(String(16), default="INFO", server_default="INFO")
|
||||
occurred_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), index=True)
|
||||
|
||||
|
||||
class AgentRelease(Base):
|
||||
"""A published build of the Windows Agent package. The universal binary/zip is
|
||||
shared by every installation; storage_path points at it on the Hub's local
|
||||
release storage (never inside PostgreSQL)."""
|
||||
__tablename__ = "agent_releases"
|
||||
__table_args__ = (UniqueConstraint("version", "platform", "architecture", name="uq_agent_release_version"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
version: Mapped[str] = mapped_column(String(32), index=True)
|
||||
platform: Mapped[str] = mapped_column(String(16), default="windows", server_default="windows")
|
||||
architecture: Mapped[str] = mapped_column(String(16), default="x64", server_default="x64")
|
||||
filename: Mapped[str] = mapped_column(String(255))
|
||||
size_bytes: Mapped[int] = mapped_column(Integer)
|
||||
sha256: Mapped[str] = mapped_column(String(64), index=True)
|
||||
storage_path: Mapped[str] = mapped_column(String(512))
|
||||
status: Mapped[str] = mapped_column(String(16), default="DRAFT", server_default="DRAFT", index=True) # DRAFT | PUBLISHED | REVOKED
|
||||
release_notes: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_by: Mapped[int | None] = mapped_column(ForeignKey("admin_users.id"), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class AgentEnrollment(Base):
|
||||
"""A pending (or completed) Agent installation: created by an operator in the
|
||||
control panel before any real Agent exists. Carries two distinct, hashed,
|
||||
single-use, short-TTL secrets — a download token (for the shareable package
|
||||
link) and an enrollment token (embedded in the bundle, consumed once by
|
||||
POST /v1/agents/enroll). Never stores either secret in plaintext."""
|
||||
__tablename__ = "agent_enrollments"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
enrollment_uuid: Mapped[str] = mapped_column(String(36), unique=True, index=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
|
||||
display_name: Mapped[str] = mapped_column(String(160))
|
||||
expected_hostname: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
release_id: Mapped[int] = mapped_column(ForeignKey("agent_releases.id"), index=True)
|
||||
status: Mapped[str] = mapped_column(String(16), default="PENDING", server_default="PENDING", index=True)
|
||||
# PENDING | LINK_CREATED | DOWNLOADED | ENROLLED | EXPIRED | REVOKED | FAILED
|
||||
created_by: Mapped[int | None] = mapped_column(ForeignKey("admin_users.id"), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
download_token_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
download_token_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
download_max_uses: Mapped[int] = mapped_column(Integer, default=1, server_default="1")
|
||||
download_uses: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
downloaded_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
enrollment_token_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
enrollment_token_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
enrollment_used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
# The install code typed by the operator into the universal EXE installer.
|
||||
# Only its SHA-256 is stored (normalize → digest), never the plaintext.
|
||||
# A new code (rotation) rewrites these three fields; the previous code
|
||||
# becomes unusable the moment a new one is generated.
|
||||
install_code_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
install_code_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
install_code_attempts: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
install_code_created_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
machine_fingerprint_seen: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
hostname_seen: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
hostname_mismatch: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
agent_id: Mapped[int | None] = mapped_column(ForeignKey("agents.id"), nullable=True, index=True)
|
||||
# Functions chosen when the installation was created (copied to the Installation at enrollment). Nothing chosen => WAITING_FEATURE_CONFIGURATION.
|
||||
features_selected: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
feature_corrector: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
feature_enrichment_mode: Mapped[str] = mapped_column(String(16), default="DISABLED", server_default="DISABLED")
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class Job(Base):
|
||||
__tablename__ = "jobs"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
agent_id: Mapped[int] = mapped_column(ForeignKey("agents.id"), index=True)
|
||||
kind: Mapped[str] = mapped_column(String(32), index=True) # REPROCESS_ALL | REPROCESS_PRODUCTS
|
||||
status: Mapped[str] = mapped_column(String(16), index=True, default="PENDING", server_default="PENDING")
|
||||
payload: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
attempts: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
result_summary: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
|
||||
class BlockedProduct(Base):
|
||||
"""Deduplicated safe exception for one tenant/product/field/input."""
|
||||
__tablename__ = "blocked_products"
|
||||
__table_args__ = (UniqueConstraint("blocker_fingerprint", name="uq_blocked_product_fingerprint"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
|
||||
installation_id: Mapped[int | None] = mapped_column(ForeignKey("installations.id"), nullable=True, index=True)
|
||||
company: Mapped[str] = mapped_column(String(128), index=True)
|
||||
product_code: Mapped[str] = mapped_column(String(64), index=True)
|
||||
gtin: Mapped[str | None] = mapped_column(String(16), nullable=True, index=True)
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
blocker_type: Mapped[str] = mapped_column(String(48), index=True)
|
||||
field: Mapped[str] = mapped_column(String(64), index=True)
|
||||
reason_code: Mapped[str] = mapped_column(String(96))
|
||||
evidence_summary: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
input_fingerprint: Mapped[str] = mapped_column(String(64), index=True)
|
||||
blocker_fingerprint: Mapped[str] = mapped_column(String(64))
|
||||
first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
retry_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
next_retry_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
resolution_source: Mapped[str | None] = mapped_column(String(48), nullable=True)
|
||||
|
||||
|
||||
class ClassificationRequest(Base):
|
||||
__tablename__ = "classification_requests"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
|
||||
agent_id: Mapped[int] = mapped_column(ForeignKey("agents.id"), index=True)
|
||||
fingerprint: Mapped[str] = mapped_column(String(64), unique=True)
|
||||
payload: Mapped[dict] = mapped_column(JSON)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class ClassificationResult(Base):
|
||||
__tablename__ = "classification_results"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
request_id: Mapped[int] = mapped_column(ForeignKey("classification_requests.id"), unique=True)
|
||||
result: Mapped[dict] = mapped_column(JSON)
|
||||
|
||||
|
||||
class ClassificationCache(Base):
|
||||
__tablename__ = "classification_cache"
|
||||
fingerprint: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
|
||||
result: Mapped[dict] = mapped_column(JSON)
|
||||
|
||||
|
||||
class RtcCstIbsCbsEntry(Base):
|
||||
"""RTC CST-IBS/CBS (sheet 'CST ...' of the official cClassTrib workbook)."""
|
||||
__tablename__ = "rtc_cst_ibscbs_entries"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "cst", name="uq_rtc_cst_ibscbs"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
cst: Mapped[str] = mapped_column(String(8), index=True)
|
||||
description: Mapped[str] = mapped_column(Text)
|
||||
indicators: Mapped[dict] = mapped_column(JSON)
|
||||
|
||||
|
||||
class RtcCClassTribEntry(Base):
|
||||
"""RTC cClassTrib (sheet 'cClass ...'). Every official column is kept:
|
||||
key fields typed, all ind_* flags in `indicators` keyed by official header."""
|
||||
__tablename__ = "rtc_cclasstrib_entries"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "cclasstrib", name="uq_rtc_cclasstrib"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
cst: Mapped[str] = mapped_column(String(8), index=True)
|
||||
cst_description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
cclasstrib: Mapped[str] = mapped_column(String(8), index=True)
|
||||
name: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
lc_redacao: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
lc_214: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
regulamento_cbs: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
regulamento_ibs: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
tipo_aliquota: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
p_red_ibs: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
p_red_cbs: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
d_ini_vig: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
d_fim_vig: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
data_atualizacao: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
anexo: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
link: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
indicators: Mapped[dict] = mapped_column(JSON)
|
||||
|
||||
|
||||
class RtcCbsRateEntry(Base):
|
||||
"""RTC CBS rate per year (IT 2026.002). rate is NULL when the official
|
||||
text has no number yet (e.g. 'aguarda legislação')."""
|
||||
__tablename__ = "rtc_cbs_rate_entries"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "year", name="uq_rtc_cbs_rate_year"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
year: Mapped[int] = mapped_column(Integer, index=True)
|
||||
rate_text: Mapped[str] = mapped_column(Text)
|
||||
rate: Mapped[float | None] = mapped_column(nullable=True)
|
||||
|
||||
|
||||
class FiscalContextEntry(Base):
|
||||
"""Operator-supplied fiscal fact (product role, origin, ST retention, treatment, ...). Hierarchy: PRODUCT > GROUP > COMPANY."""
|
||||
__tablename__ = "fiscal_context_entries"
|
||||
__table_args__ = (UniqueConstraint("tenant_id", "company_key", "scope", "scope_key", "key", name="uq_fiscal_context_entry"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
|
||||
company_key: Mapped[str] = mapped_column(String(64), index=True)
|
||||
scope: Mapped[str] = mapped_column(String(8))
|
||||
scope_key: Mapped[str] = mapped_column(String(64), default="", server_default="")
|
||||
key: Mapped[str] = mapped_column(String(48))
|
||||
value: Mapped[str] = mapped_column(Text)
|
||||
source: Mapped[str] = mapped_column(String(16), default="OPERATOR", server_default="OPERATOR")
|
||||
reason: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
# ================= AI semantic layer V1 =================
|
||||
# The AI layer is NOT a fiscal authority: it only selects among official candidates and
|
||||
# never writes to Firebird. Knowledge is versioned, provenance-tracked and never deleted.
|
||||
|
||||
|
||||
class KbProduct(Base):
|
||||
"""GLOBAL product knowledge: only facts inherent to the product (never tenant context)."""
|
||||
__tablename__ = "kb_products"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
fingerprint: Mapped[str] = mapped_column(String(64), unique=True, index=True)
|
||||
gtin: Mapped[str | None] = mapped_column(String(16), unique=True, nullable=True, index=True)
|
||||
identity_kind: Mapped[str] = mapped_column(String(16), default="FINGERPRINT") # GTIN | FINGERPRINT | SEMANTIC | UNKNOWN
|
||||
normalized_description: Mapped[str] = mapped_column(Text, default="")
|
||||
brand: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
unit: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
package_quantity: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
category: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
semantic_attributes: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class KbKnowledge(Base):
|
||||
"""Versioned knowledge rows. Unique per (product, key, value): conflicting values for the
|
||||
same key are kept side by side and surfaced as KNOWLEDGE_CONFLICT, never majority-picked."""
|
||||
__tablename__ = "kb_knowledge"
|
||||
__table_args__ = (UniqueConstraint("kb_product_id", "knowledge_key", "value", name="uq_kb_knowledge_value"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
kb_product_id: Mapped[int] = mapped_column(ForeignKey("kb_products.id"), index=True)
|
||||
knowledge_key: Mapped[str] = mapped_column(String(48), index=True) # candidate_ncm | validated_ncm | candidate_cest | candidate_ex_tipi | ...
|
||||
value: Mapped[str] = mapped_column(Text)
|
||||
source: Mapped[str] = mapped_column(String(32)) # OFFICIAL_DATASET | ERP_OBSERVATION | OPERATOR | AI | AI_PLUS_OFFICIAL_VALIDATION
|
||||
confidence: Mapped[float] = mapped_column(default=0.0)
|
||||
evidence: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
model: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
prompt_version: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
dataset_versions: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
validated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(24), default="OBSERVED", server_default="OBSERVED", index=True) # OBSERVED | CORROBORATED | EXTERNAL_CORROBORATED | VERIFIED_KB | CONFLICT | STALE
|
||||
|
||||
|
||||
class KbObservation(Base):
|
||||
"""Raw per-tenant ERP observation of a KB product (evidence aggregation only)."""
|
||||
__tablename__ = "kb_observations"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
kb_product_id: Mapped[int] = mapped_column(ForeignKey("kb_products.id"), index=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
|
||||
company_key: Mapped[str] = mapped_column(String(64), index=True)
|
||||
erp_code: Mapped[str] = mapped_column(String(32))
|
||||
database_label: Mapped[str] = mapped_column(String(64))
|
||||
observed_fields: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
source: Mapped[str] = mapped_column(String(32), default="ERP_OBSERVATION")
|
||||
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class TenantProductContext(Base):
|
||||
"""TENANT-specific product context: role, ST-withheld, operation profile, operator decisions.
|
||||
Never promoted to global knowledge."""
|
||||
__tablename__ = "tenant_product_context"
|
||||
__table_args__ = (UniqueConstraint("tenant_id", "company_key", "erp_code", name="uq_tenant_product_context"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
|
||||
company_key: Mapped[str] = mapped_column(String(64), index=True)
|
||||
erp_code: Mapped[str] = mapped_column(String(32))
|
||||
kb_product_id: Mapped[int | None] = mapped_column(ForeignKey("kb_products.id"), nullable=True)
|
||||
product_role: Mapped[str | None] = mapped_column(String(16), nullable=True) # RESALE | OWN_PRODUCTION | INPUT | SERVICE | UNKNOWN
|
||||
st_withheld: Mapped[str | None] = mapped_column(String(4), nullable=True) # Y | N — never AI-decided
|
||||
operation_profile: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
source: Mapped[str] = mapped_column(String(32), default="ERP_OBSERVATION")
|
||||
confidence: Mapped[float] = mapped_column(default=0.0)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class AiWorkItem(Base):
|
||||
"""Deduplicated semantic work item (one knowledge need, many pending cells)."""
|
||||
__tablename__ = "ai_work_items"
|
||||
__table_args__ = (UniqueConstraint("dedup_key", name="uq_ai_work_item_dedup"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
task_type: Mapped[str] = mapped_column(String(24), index=True) # INVALID_NCM | CEST | EX_TIPI | PRODUCT_ROLE
|
||||
dedup_key: Mapped[str] = mapped_column(String(160))
|
||||
question_refs: Mapped[list] = mapped_column(JSON, default=list)
|
||||
payload: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
status: Mapped[str] = mapped_column(String(32), default="PENDING", index=True) # PENDING | KB_EXACT_HIT | KB_FINGERPRINT_HIT | HIGH_CONFIDENCE_RETRIEVAL | SINGLE_OFFICIAL_CANDIDATE | LLM_SELECTION_REQUIRED | UNRESOLVED | UNRESOLVED_NO_OFFICIAL_MAPPING | CONFLICT | DONE | SKIPPED
|
||||
kb_product_id: Mapped[int | None] = mapped_column(ForeignKey("kb_products.id"), nullable=True)
|
||||
cells_covered: Mapped[int] = mapped_column(Integer, default=0)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class AiRun(Base):
|
||||
"""One provider execution over a batch of work items, with budget-guard evidence."""
|
||||
__tablename__ = "ai_runs"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
provider: Mapped[str] = mapped_column(String(64))
|
||||
model: Mapped[str] = mapped_column(String(64))
|
||||
prompt_version: Mapped[str] = mapped_column(String(32))
|
||||
work_item_ids: Mapped[list] = mapped_column(JSON, default=list)
|
||||
input_tokens: Mapped[int] = mapped_column(Integer, default=0)
|
||||
output_tokens: Mapped[int] = mapped_column(Integer, default=0)
|
||||
cost_usd: Mapped[float] = mapped_column(default=0.0)
|
||||
status: Mapped[str] = mapped_column(String(16), default="RUN")
|
||||
results: Mapped[list] = mapped_column(JSON, default=list)
|
||||
budget_guard: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class AiEvalCache(Base):
|
||||
"""Persistent LLM result cache: identical (input_fingerprint, candidate_set_hash,
|
||||
prompt_version, model, dataset_versions) re-execution makes ZERO new calls."""
|
||||
__tablename__ = "ai_eval_cache"
|
||||
__table_args__ = (UniqueConstraint("input_fingerprint", "candidate_set_hash", "prompt_version", "model", "dataset_version_ncm", name="uq_ai_eval_cache"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
input_fingerprint: Mapped[str] = mapped_column(String(64), index=True)
|
||||
candidate_set_hash: Mapped[str] = mapped_column(String(64))
|
||||
prompt_version: Mapped[str] = mapped_column(String(32))
|
||||
model: Mapped[str] = mapped_column(String(64))
|
||||
dataset_version_ncm: Mapped[str] = mapped_column(String(64))
|
||||
phase: Mapped[str] = mapped_column(String(16))
|
||||
answer: Mapped[dict] = mapped_column(JSON)
|
||||
gold_ncm: Mapped[str | None] = mapped_column(String(8), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class AiUsageLedger(Base):
|
||||
"""Central persistent usage ledger reserved before any future provider call."""
|
||||
__tablename__ = "ai_usage_ledger"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
provider: Mapped[str] = mapped_column(String(64))
|
||||
model: Mapped[str] = mapped_column(String(64))
|
||||
run_id: Mapped[str] = mapped_column(String(128), index=True)
|
||||
tenant: Mapped[str] = mapped_column(String(128), index=True)
|
||||
items: Mapped[int] = mapped_column(Integer, default=0)
|
||||
input_tokens: Mapped[int] = mapped_column(Integer, default=0)
|
||||
output_tokens: Mapped[int] = mapped_column(Integer, default=0)
|
||||
cost: Mapped[float] = mapped_column(default=0.0)
|
||||
timestamp: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), index=True)
|
||||
|
||||
|
||||
class ExternalProductCache(Base):
|
||||
"""Versioned GTIN cache; legal/provider flags remain explicit metadata."""
|
||||
__tablename__ = "external_product_cache"
|
||||
__table_args__ = (UniqueConstraint("gtin", "provider", "provider_record_version", name="uq_external_product_cache_record"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
gtin: Mapped[str] = mapped_column(String(16), index=True)
|
||||
provider: Mapped[str] = mapped_column(String(64))
|
||||
provider_record_version: Mapped[str] = mapped_column(String(64))
|
||||
retrieved_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
raw_hash: Mapped[str] = mapped_column(String(64))
|
||||
normalized_data: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
cache_allowed: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
redistribution_allowed: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
|
||||
|
||||
class RtcWriteAudit(Base):
|
||||
"""Append-only record of every attempted C000025.CCLASSTRIB write (applied or aborted). Holds the old value, so each change is revertible by hand."""
|
||||
__tablename__ = "rtc_write_audits"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
|
||||
installation_id: Mapped[int] = mapped_column(ForeignKey("installations.id"), index=True)
|
||||
agent_id: Mapped[int] = mapped_column(ForeignKey("agents.id"), index=True)
|
||||
job_id: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
|
||||
client: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
product_code: Mapped[str] = mapped_column(String(64), index=True)
|
||||
field: Mapped[str] = mapped_column(String(32))
|
||||
outcome: Mapped[str] = mapped_column(String(16), index=True)
|
||||
reason: Mapped[str | None] = mapped_column(String(96), nullable=True)
|
||||
old_value: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
new_value: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
rule_id: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
evidence: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
official_source_version: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
local_catalog_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
written_at: Mapped[str | None] = mapped_column(String(40), nullable=True)
|
||||
rule_version: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
company_crt: Mapped[str | None] = mapped_column(String(8), nullable=True)
|
||||
authorization_origin: Mapped[str | None] = mapped_column(String(48), nullable=True)
|
||||
description_snapshot: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
installation_uuid: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
rtc_status: Mapped[str | None] = mapped_column(String(24), nullable=True)
|
||||
local_catalog_state: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
action: Mapped[str | None] = mapped_column(String(16), nullable=True) # APPLY | ROLLBACK
|
||||
rollback_state: Mapped[str | None] = mapped_column(String(16), nullable=True) # REQUESTED | DONE | BLOCKED (on the original APPLY record)
|
||||
rollback_of: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
|
||||
recorded_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), index=True)
|
||||
|
||||
|
||||
class CfopIpptWriteAudit(Base):
|
||||
"""Append-only record of every attempted C000025.CFOP/IPPT write (applied or aborted) under the
|
||||
CFOP_IPPT_WRITE_V1 capability. Holds the old value, so each change is revertible by hand."""
|
||||
__tablename__ = "cfop_ippt_write_audits"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
|
||||
installation_id: Mapped[int] = mapped_column(ForeignKey("installations.id"), index=True)
|
||||
agent_id: Mapped[int] = mapped_column(ForeignKey("agents.id"), index=True)
|
||||
job_id: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
|
||||
product_code: Mapped[str] = mapped_column(String(64), index=True)
|
||||
field: Mapped[str] = mapped_column(String(16)) # CFOP | IPPT only
|
||||
outcome: Mapped[str] = mapped_column(String(16), index=True) # APPLIED | ABORTED | FAILED | ROLLED_BACK
|
||||
reason: Mapped[str | None] = mapped_column(String(96), nullable=True)
|
||||
old_value: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
new_value: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
action: Mapped[str | None] = mapped_column(String(16), nullable=True) # APPLY | ROLLBACK
|
||||
rollback_of: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
|
||||
written_at: Mapped[str | None] = mapped_column(String(40), nullable=True)
|
||||
recorded_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), index=True)
|
||||
|
||||
|
||||
class RtcCandidate(Base):
|
||||
"""Server-side preview of the products the Agent could fill (R3 only). Rows are proposals; only the Agent's re-derivation at write time decides."""
|
||||
__tablename__ = "rtc_candidates"
|
||||
__table_args__ = (UniqueConstraint("installation_id", "product_code", name="uq_rtc_candidate_product"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
|
||||
installation_id: Mapped[int] = mapped_column(ForeignKey("installations.id"), index=True)
|
||||
product_code: Mapped[str] = mapped_column(String(64))
|
||||
description: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
ncm: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
current_value: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
proposed: Mapped[str] = mapped_column(String(16))
|
||||
cst_ibscbs: Mapped[str | None] = mapped_column(String(8), nullable=True)
|
||||
rule_id: Mapped[str] = mapped_column(String(32))
|
||||
basis: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(16), default="PENDING", server_default="PENDING", index=True) # PENDING | APPROVED | REJECTED | APPLIED | STALE
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class ProductEnrichment(Base):
|
||||
"""Per (installation, product) state of the new-product enrichment. Historical products never get a row (no mass provider use)."""
|
||||
__tablename__ = "product_enrichments"
|
||||
__table_args__ = (UniqueConstraint("installation_id", "product_code", name="uq_product_enrichment"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
|
||||
installation_id: Mapped[int] = mapped_column(ForeignKey("installations.id"), index=True)
|
||||
product_code: Mapped[str] = mapped_column(String(64))
|
||||
gtin: Mapped[str | None] = mapped_column(String(16), nullable=True, index=True)
|
||||
description: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(32), default="PENDING", server_default="PENDING", index=True)
|
||||
# PENDING | RESOLVED | APPLIED | PARTIAL | NOT_RESOLVED | CONFLICT | PENDING_PROVIDER_QUOTA | PENDING_PROVIDERS_OFFLINE | INVALID_GTIN
|
||||
decisions: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
fields_applied: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
human_overridden: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
attempts: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
|
||||
latency_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
reevaluate: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false", index=True) # set by panel / REPROCESS / SYNC / KB change; consumed by the Agent
|
||||
last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
ui_status: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True) # human-facing state (never a silent product)
|
||||
reason_code: Mapped[str | None] = mapped_column(String(48), nullable=True)
|
||||
detected_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class ProductFieldAudit(Base):
|
||||
"""Append-only audit of every field the enrichment wrote (or refused to). Never holds secrets."""
|
||||
__tablename__ = "product_field_audits"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
|
||||
installation_id: Mapped[int] = mapped_column(ForeignKey("installations.id"), index=True)
|
||||
agent_id: Mapped[int] = mapped_column(ForeignKey("agents.id"), index=True)
|
||||
product_code: Mapped[str] = mapped_column(String(64), index=True)
|
||||
gtin: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
description_snapshot: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
field: Mapped[str] = mapped_column(String(48))
|
||||
old_value: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
new_value: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
rule: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
kb_status: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
evidence: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
dataset_versions: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
authorization_origin: Mapped[str | None] = mapped_column(String(48), nullable=True)
|
||||
job_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
outcome: Mapped[str] = mapped_column(String(24), default="APPLIED", server_default="APPLIED") # APPLIED | NO_WRITE_REQUIRED | ABORTED | HUMAN_OVERRIDDEN
|
||||
reason: Mapped[str | None] = mapped_column(String(96), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class ProviderState(Base):
|
||||
"""Live state of each external product provider (owned by the Hub: quota belongs to the Hub, never to Agents)."""
|
||||
__tablename__ = "provider_states"
|
||||
provider: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
configured: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
status: Mapped[str] = mapped_column(String(16), default="NOT_CONFIGURED", server_default="NOT_CONFIGURED") # ONLINE | DEGRADED | RATE_LIMITED | AUTH_ERROR | OFFLINE | NOT_CONFIGURED
|
||||
quota_kind: Mapped[str] = mapped_column(String(16), default="DOCUMENTED_RATE", server_default="DOCUMENTED_RATE") # REMOTE_QUOTA | LOCAL_ESTIMATE | DOCUMENTED_RATE
|
||||
daily_limit: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
remaining: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
reset_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
rate_per_minute: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
circuit_open_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
consecutive_failures: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
last_request_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_success_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_error: Mapped[str | None] = mapped_column(String(160), nullable=True)
|
||||
last_error_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_latency_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class ProviderUsageDaily(Base):
|
||||
__tablename__ = "provider_usage_daily"
|
||||
__table_args__ = (UniqueConstraint("provider", "day", name="uq_provider_usage_day"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
provider: Mapped[str] = mapped_column(String(32), index=True)
|
||||
day: Mapped[str] = mapped_column(String(10), index=True)
|
||||
requests: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
success: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
not_found: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
errors: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
rate_limited: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
timeouts: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
cache_hits: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
cache_misses: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
found: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
ncm_found: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
kb_hits: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
class GtinLookupCache(Base):
|
||||
"""Positive and negative (NOT_FOUND) provider results per GTIN, with TTL. Single place that spares quota across all installations."""
|
||||
__tablename__ = "gtin_lookup_cache"
|
||||
__table_args__ = (UniqueConstraint("gtin", "provider", name="uq_gtin_lookup_cache"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
gtin: Mapped[str] = mapped_column(String(16), index=True)
|
||||
provider: Mapped[str] = mapped_column(String(32))
|
||||
found: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
data: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
|
||||
|
||||
|
||||
class SourceRefreshRun(Base):
|
||||
"""Observability of official fiscal sources (NCM Siscomex, ...): one row per refresh attempt."""
|
||||
__tablename__ = "source_refresh_runs"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
source: Mapped[str] = mapped_column(String(32), index=True)
|
||||
source_url: Mapped[str] = mapped_column(Text)
|
||||
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(16), default="RUNNING", server_default="RUNNING") # RUNNING | UPDATED | UNCHANGED | FAILED
|
||||
sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
version_label: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
record_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(String(300), nullable=True)
|
||||
|
||||
|
||||
class KbPromotionAudit(Base):
|
||||
"""Why a GTIN/NCM became (or stopped being) trustworthy: statuses, evidence families, source references and the official dataset hash at that moment."""
|
||||
__tablename__ = "kb_promotion_audits"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
kb_product_id: Mapped[int] = mapped_column(ForeignKey("kb_products.id"), index=True)
|
||||
gtin: Mapped[str | None] = mapped_column(String(16), nullable=True, index=True)
|
||||
field: Mapped[str] = mapped_column(String(24), default="ncm", server_default="ncm")
|
||||
value: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
old_status: Mapped[str | None] = mapped_column(String(24), nullable=True)
|
||||
new_status: Mapped[str] = mapped_column(String(24))
|
||||
families: Mapped[list] = mapped_column(JSON, default=list)
|
||||
sources: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
official_dataset: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
rule_version: Mapped[str] = mapped_column(String(32))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Per-product operational state and safe partial-processing metrics."""
|
||||
|
||||
from collections import Counter
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
STATES = {
|
||||
"COMPLETE", "CORRECTED", "BLOCKED_MISSING_EVIDENCE", "BLOCKED_CONFLICT",
|
||||
"BLOCKED_COMPANY_CONTEXT", "BLOCKED_SCHEMA", "BLOCKED_FISCAL_DATA", "FAILED_TECHNICAL", "NEEDS_REVIEW",
|
||||
}
|
||||
|
||||
|
||||
def blocker_fingerprint(tenant: str, company: str, product_code: str, field: str, reason_code: str, input_fingerprint: str, installation_id: str | int | None = None) -> str:
|
||||
raw = {"tenant": tenant, "company": company, "product_code": product_code, "field": field, "reason_code": reason_code, "input_fingerprint": input_fingerprint}
|
||||
if installation_id is not None:
|
||||
raw["installation_id"] = str(installation_id)
|
||||
return hashlib.sha256(json.dumps(raw, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
||||
|
||||
|
||||
def classify_product_state(product: dict, decision: dict, *, write_status: str | None = None, fiscal: dict | None = None) -> dict:
|
||||
"""`fiscal` = unified_fiscal_coverage.evaluate_product(...): PIS/COFINS + (only when the ERP has RTC on) cClassTrib. One definition of COMPLETE."""
|
||||
status = str(decision.get("status") or "").upper()
|
||||
if write_status == "FAILED":
|
||||
state = "FAILED_TECHNICAL"
|
||||
elif status in {"AMBIGUOUS", "CONFLICT", "NCM_CONFLICT"}:
|
||||
state = "BLOCKED_CONFLICT"
|
||||
elif status in {"INVALID_NCM", "LEGACY_NCM", "NOT_FOUND"}:
|
||||
state = "BLOCKED_MISSING_EVIDENCE"
|
||||
elif status in {"COMPANY_CONTEXT_REQUIRED", "COMPANY_CONTEXT_FAIL"}:
|
||||
state = "BLOCKED_COMPANY_CONTEXT"
|
||||
elif status in {"ERP_SCHEMA_LIMITATION", "UNSUPPORTED_SCHEMA"}:
|
||||
state = "BLOCKED_SCHEMA"
|
||||
elif write_status == "SUCCEEDED":
|
||||
state = "CORRECTED"
|
||||
else:
|
||||
state = "COMPLETE"
|
||||
fiscal = fiscal or {}
|
||||
if state in ("COMPLETE", "CORRECTED"):
|
||||
if fiscal.get("blockers"):
|
||||
state = "BLOCKED_FISCAL_DATA"
|
||||
elif fiscal.get("needs_review"):
|
||||
state = "NEEDS_REVIEW"
|
||||
return {"product_code": str(product.get("CODIGO") or ""), "state": state, "decision_status": status, "fiscal_blockers": list(fiscal.get("blockers") or [])}
|
||||
|
||||
|
||||
def should_retry_blocker(previous_input_fingerprint: str | None, current_input_fingerprint: str) -> bool:
|
||||
return previous_input_fingerprint != current_input_fingerprint
|
||||
|
||||
|
||||
def summarize_operational_states(states: list[dict]) -> dict:
|
||||
counts = Counter(item.get("state") for item in states)
|
||||
result = {state: counts.get(state, 0) for state in sorted(STATES)}
|
||||
result["TOTAL_PRODUCTS"] = len(states)
|
||||
result["UNCLASSIFIED"] = result["NEEDS_REVIEW"]
|
||||
result["COMPLETE_PRODUCTS"] = result["COMPLETE"] + result["CORRECTED"]
|
||||
result["SAFE_BLOCKED"] = sum(result[state] for state in STATES if state.startswith("BLOCKED_"))
|
||||
result["AUTONOMOUSLY_PROCESSABLE_PRODUCTS"] = result["TOTAL_PRODUCTS"] - result["FAILED_TECHNICAL"]
|
||||
result["AGENT_HEALTH"] = "HEALTHY" if result["FAILED_TECHNICAL"] == 0 else "DEGRADED"
|
||||
return result
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Evidence-gated origin resolution; CST digits are never an origin source."""
|
||||
|
||||
|
||||
def resolve_origin(product_id: str, aggregate: dict, kb_facts: dict) -> dict:
|
||||
if aggregate.get("conflict_count", 0):
|
||||
status = "ORIGIN_CONFLICT"
|
||||
elif aggregate.get("strong_evidence") and aggregate.get("value") in {str(i) for i in range(9)}:
|
||||
status = "ORIGIN_CONFIRMED"
|
||||
else:
|
||||
status = "ORIGIN_NEEDS_EVIDENCE"
|
||||
return {
|
||||
"product_id": product_id,
|
||||
"status": status,
|
||||
"value": aggregate.get("value") if status == "ORIGIN_CONFIRMED" else None,
|
||||
"provenance": aggregate.get("provenance", []),
|
||||
"reason": "explicit supplier observations only; CST digit ignored",
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
"""Human-readable values used by the operational control panel."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
import os
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from .config import settings
|
||||
|
||||
|
||||
def _as_utc(value: datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _ui_timezone() -> ZoneInfo:
|
||||
return ZoneInfo(os.getenv("UI_TIMEZONE", settings.ui_timezone))
|
||||
|
||||
|
||||
def format_ui_datetime(value: datetime | None) -> str:
|
||||
"""Format a stored timestamp in the configured UI timezone."""
|
||||
value = _as_utc(value)
|
||||
if value is None:
|
||||
return "-"
|
||||
return value.astimezone(_ui_timezone()).strftime("%d/%m/%Y %H:%M")
|
||||
|
||||
|
||||
def format_age(value: datetime | None, now: datetime | None = None) -> str:
|
||||
value = _as_utc(value)
|
||||
if value is None:
|
||||
return "-"
|
||||
now = _as_utc(now or datetime.now(timezone.utc))
|
||||
seconds = int((now - value).total_seconds())
|
||||
if seconds <= 0:
|
||||
return "agora"
|
||||
minutes = seconds // 60
|
||||
if minutes < 1:
|
||||
return "agora"
|
||||
if minutes < 60:
|
||||
return f"há {minutes} min"
|
||||
return f"há {minutes // 60} h"
|
||||
|
||||
|
||||
def format_duration(started_at: datetime | None, finished_at: datetime | None, now: datetime | None = None) -> str:
|
||||
del now
|
||||
started_at = _as_utc(started_at)
|
||||
if started_at is None:
|
||||
return "-"
|
||||
if finished_at is None:
|
||||
return "em andamento"
|
||||
elapsed = max(0, int((_as_utc(finished_at) - started_at).total_seconds()))
|
||||
hours, remainder = divmod(elapsed, 3600)
|
||||
minutes, seconds = divmod(remainder, 60)
|
||||
parts = []
|
||||
if hours:
|
||||
parts.append(f"{hours} h")
|
||||
if minutes:
|
||||
parts.append(f"{minutes} min")
|
||||
if seconds or not parts:
|
||||
parts.append(f"{seconds} s")
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
def _blocker_value(blocker, name: str) -> str:
|
||||
if isinstance(blocker, dict):
|
||||
return str(blocker.get(name) or "").upper()
|
||||
return str(getattr(blocker, name, "") or "").upper()
|
||||
|
||||
|
||||
def operational_priority(blocker) -> str:
|
||||
blocker_type = _blocker_value(blocker, "blocker_type")
|
||||
field = _blocker_value(blocker, "field")
|
||||
reason = _blocker_value(blocker, "reason_code")
|
||||
operational_text = f"{blocker_type} {reason}"
|
||||
if blocker_type in {"TECHNICAL_FAILURE", "CONNECTION_ERROR", "AGENT_OFFLINE", "JOB_FAILED"} or any(token in operational_text for token in ("TECHNICAL", "CONNECTION", "TIMEOUT")):
|
||||
return "CRITICAL"
|
||||
if blocker_type in {"BLOCKED_MISSING_EVIDENCE", "MISSING_EVIDENCE"}:
|
||||
return "HIGH"
|
||||
if field in {"CFOP", "ST", "ICMS", "CSOSN", "NCM", "CEST", "PIS", "COFINS", "RTC"} or reason in {"INSUFFICIENT_EVIDENCE", "NO_EVIDENCE"}:
|
||||
return "NORMAL"
|
||||
return "LOW"
|
||||
|
||||
|
||||
_HEALTH_LABELS = {
|
||||
"ONLINE": "Online",
|
||||
"DEGRADED": "Degradado",
|
||||
"OFFLINE": "Offline",
|
||||
"FAILED": "Falha",
|
||||
"UNKNOWN": "Desconhecido",
|
||||
}
|
||||
|
||||
|
||||
def health_label(value) -> str:
|
||||
if value is None or value == "":
|
||||
return "-"
|
||||
text = str(value)
|
||||
return _HEALTH_LABELS.get(text.upper(), text)
|
||||
|
||||
|
||||
def _human_label(value, labels):
|
||||
if not value:
|
||||
return "Não informado"
|
||||
text = str(value).upper()
|
||||
return labels.get(text, str(value).replace("_", " ").lower().capitalize())
|
||||
|
||||
|
||||
def fiscal_status_label(value):
|
||||
return _human_label(value, {"BLOCKED": "Bloqueado", "COMPLETE": "Completo", "INCOMPLETE": "Incompleto", "UNKNOWN": "Não informado"})
|
||||
|
||||
|
||||
def job_status_label(value):
|
||||
return _human_label(value, {"PENDING": "Pendente", "RUNNING": "Executando", "COMPLETED": "Concluído", "FAILED": "Falhou"})
|
||||
|
||||
|
||||
def job_kind_label(value):
|
||||
return _human_label(value, {"SYNC_INCREMENTAL": "Sincronização incremental", "INITIAL_SYNC": "Sincronização inicial", "REPROCESS_ALL": "Reprocessar tudo", "REPROCESS_PRODUCTS": "Reprocessar produtos"})
|
||||
|
||||
|
||||
_FISCAL_LABELS = {
|
||||
"CCLASSTRIB_MISSING": "cClassTrib ausente", "CCLASSTRIB_MISSING_UNRESOLVED": "cClassTrib não determinado", "CCLASSTRIB_AMBIGUOUS": "cClassTrib ambíguo", "CCLASSTRIB_CONFLICT": "cClassTrib existente diverge da classificação proposta",
|
||||
"CCLASSTRIB_OFFICIAL_MISMATCH": "cClassTrib diverge da tabela oficial", "RTC_LOCAL_CATALOG_INCOMPATIBLE": "Catálogo RTC incompatível", "RTC_RULE_EVIDENCE_MISSING": "Regra fiscal insuficiente", "CCLASSTRIB_INVALID_FORMAT": "cClassTrib com formato inválido", "CCLASSTRIB_NOT_IN_CATALOG": "cClassTrib fora do catálogo local",
|
||||
"RTC_CST_MISSING": "CST IBS/CBS ausente no catálogo", "RTC_CATALOG_INVALID": "Catálogo RTC local inválido",
|
||||
"PIS_CST_MISSING": "CST PIS ausente", "PIS_CST_INVALID": "CST PIS inválido", "PIS_RATE_MISSING": "Alíquota PIS ausente", "PIS_RATE_INVALID": "Alíquota PIS inválida", "PIS_CST_INCIDENCE_MISMATCH": "CST PIS incompatível com a incidência",
|
||||
"COFINS_CST_MISSING": "CST COFINS ausente", "COFINS_CST_INVALID": "CST COFINS inválido", "COFINS_RATE_MISSING": "Alíquota COFINS ausente", "COFINS_RATE_INVALID": "Alíquota COFINS inválida", "COFINS_CST_INCIDENCE_MISMATCH": "CST COFINS incompatível com a incidência",
|
||||
"PIS_COFINS_FLAG_INCONSISTENT": "Flag PIS/COFINS inconsistente", "BLOCKED_FISCAL_DATA": "Dado fiscal do cadastro",
|
||||
}
|
||||
|
||||
|
||||
def blocker_type_label(value):
|
||||
return _human_label(value, {"BLOCKED_MISSING_EVIDENCE": "Evidência fiscal ausente", "MISSING_EVIDENCE": "Evidência fiscal ausente", "TECHNICAL_FAILURE": "Falha técnica", "CONNECTION_ERROR": "Erro de conexão", "OTHER": "Outro", **_FISCAL_LABELS})
|
||||
|
||||
|
||||
_REASON_LABELS = {
|
||||
"INVALID_NCM": "NCM inválido", "LEGACY_NCM": "NCM legado", "LEGACY_NCM_NO_AUTOMATIC_SUCCESSOR": "NCM legado sem sucessor automático",
|
||||
"LEGACY_NCM_REVIEW": "NCM legado em revisão", "LEGACY_NCM_REVIEW_REQUIRED": "NCM legado exige revisão", "LEGACY_OR_INVALID_NCM": "NCM legado ou inválido",
|
||||
"INSUFFICIENT_EVIDENCE": "Evidência fiscal insuficiente", "MISSING_EVIDENCE": "Evidência fiscal ausente", "BLOCKED_MISSING_EVIDENCE": "Evidência fiscal ausente",
|
||||
"MISSING_CONTEXT": "Contexto fiscal ausente", "NO_EVIDENCE": "Sem evidência", "MISSING_PRODUCT_ORIGIN": "Origem do produto ausente",
|
||||
"MISSING_PROFILE": "Perfil fiscal ausente", "MISSING_ST_DETERMINATION": "Determinação de ST ausente", "NO_IPI_DECISION": "Sem decisão de IPI",
|
||||
"NO_NUMERIC_VALUE": "Sem valor numérico", "CHANGE_REQUIRED": "Alteração necessária", "SAME_VALUE": "Valor já correto", "AMBIGUOUS": "Ambíguo",
|
||||
}
|
||||
|
||||
|
||||
def blocker_reason_label(value):
|
||||
return _human_label(value, {**_REASON_LABELS, **_FISCAL_LABELS})
|
||||
|
||||
|
||||
def health_reason_label(value):
|
||||
"""Operator wording for the technical health reason strings produced by compute_agent_health."""
|
||||
text = str(value or "")
|
||||
if text.startswith("last heartbeat exceeds"):
|
||||
return "Sem heartbeat recente"
|
||||
return {"no heartbeat received": "Nenhum heartbeat recebido", "persistent technical failure": "Falha técnica persistente",
|
||||
"recent technical or job failure": "Falha técnica ou de job recente",
|
||||
"recent heartbeat without critical technical failure": "Operando normalmente"}.get(text, text or "Não informado")
|
||||
|
||||
|
||||
def priority_label(value):
|
||||
return _human_label(value, {"CRITICAL": "Crítica", "HIGH": "Alta", "NORMAL": "Média", "MEDIUM": "Média", "LOW": "Baixa"})
|
||||
|
||||
|
||||
def rtc_status_view(summary, mode="DISABLED"):
|
||||
"""Discreet Reforma Tributaria status from the last completed job's `fiscal_context`, exactly as the Agent OBSERVED it in that installation's own database.
|
||||
None when the Agent did not report it (older Agents)."""
|
||||
context = (summary or {}).get("fiscal_context") if isinstance(summary, dict) else None
|
||||
if not isinstance(context, dict) or "rtc_enabled" not in context:
|
||||
return None
|
||||
observed = None
|
||||
if context.get("rtc_observed_at"):
|
||||
try:
|
||||
observed = format_ui_datetime(datetime.fromisoformat(str(context["rtc_observed_at"]).replace("Z", "+00:00")))
|
||||
except ValueError:
|
||||
observed = None
|
||||
source = context.get("rtc_source") or "PARAMETROS.USAR_REFORMA_TRIBUTARIA"
|
||||
products = _rtc_products_view(context.get("rtc_resolution"), bool(context["rtc_enabled"]), context.get("cclasstrib_applicability"))
|
||||
return {"products": products, "apply": _rtc_apply_view(summary.get("rtc_auto_apply"), mode), "enabled": bool(context["rtc_enabled"]), "label": "Ativa" if context["rtc_enabled"] else "Desativada", "supported": bool(context.get("rtc_supported")),
|
||||
"catalog_valid": bool(context.get("rtc_catalog_valid")), "observed_at": observed, "conflict": bool(context.get("rtc_param_conflict")),
|
||||
"title": f"Lido no banco do cliente ({source})" + (f" em {observed}" if observed else "") + (" - linhas de PARAMETROS divergentes: RTC tratada como desativada" if context.get("rtc_param_conflict") else "")}
|
||||
|
||||
|
||||
def _rtc_products_view(resolution, enabled, cclasstrib_applicability=None):
|
||||
"""Compact 'Produtos RTC' summary. None when RTC is off (RTC never shows up as a problem) or the Agent did not report a resolution."""
|
||||
if not enabled:
|
||||
return None
|
||||
if isinstance(cclasstrib_applicability, dict) and cclasstrib_applicability.get("state") == "NOT_APPLICABLE":
|
||||
label = "cClassTrib: Aguardando confirmação do regime da empresa" if cclasstrib_applicability.get("basis") == "COMPANY_CRT_UNRESOLVED" else "cClassTrib: Não utilizado neste cliente"
|
||||
return {"applicable": False, "label": label}
|
||||
if not isinstance(resolution, dict) or not resolution.get("total"):
|
||||
return None
|
||||
total = int(resolution["total"])
|
||||
valid = int(resolution.get("valid_existing", 0))
|
||||
ready = int(resolution.get("ready_to_apply", 0))
|
||||
conflicts = int(resolution.get("conflicts", 0)) + int(resolution.get("official_mismatch", 0))
|
||||
catalog = int(resolution.get("catalog_incompatible", 0))
|
||||
reasons = resolution.get("by_reason") if isinstance(resolution.get("by_reason"), dict) else {}
|
||||
simple_count = sum(int(count or 0) for reason, count in reasons.items() if str(reason).endswith("SIMPLES_ORIENTATION_PENDING_OFFICIAL_NT"))
|
||||
other_reason_max = max((int(count or 0) for reason, count in reasons.items() if not str(reason).endswith("SIMPLES_ORIENTATION_PENDING_OFFICIAL_NT")), default=0)
|
||||
blocked = total - valid - ready
|
||||
pending = total - valid
|
||||
return {"applicable": True, "total": total, "valid": valid, "ready": ready, "blocked": blocked, "pending": pending, "conflicts": conflicts, "catalog_incompatible": catalog, "suggestion_only": int(resolution.get("suggestion_only", 0)),
|
||||
"simple_orientation_count": simple_count, "simple_orientation_dominant": simple_count > 0 and simple_count > other_reason_max,
|
||||
"headline": f"Reforma Tributária ativa — cClassTrib pendente em {pending} produto{'s' if pending != 1 else ''}" if pending else "Reforma Tributária ativa — todos os produtos com cClassTrib válido",
|
||||
"catalog": (resolution.get("catalog_compatibility") or {}).get("compatibility")}
|
||||
|
||||
|
||||
ENRICHMENT_MODES = ("DISABLED", "DRY_RUN", "ENABLED")
|
||||
ENRICHMENT_MODE_LABELS = {"DISABLED": "Desativado", "DRY_RUN": "Simulação (não grava)", "ENABLED": "Automático"}
|
||||
RTC_MODE_LABELS = {"DISABLED": "Desativada", "DRY_RUN": "Dry-run", "APPROVAL_REQUIRED": "Aprovação manual", "ENABLED": "Automática"}
|
||||
RTC_MODE_EXPLANATION = "Somente cClassTrib com regra fiscal determinística e catálogo validado pode ser preenchido automaticamente."
|
||||
|
||||
|
||||
def _rtc_apply_view(report, mode):
|
||||
"""State of the (default DISABLED) centrally controlled fill. Never claims a write happened unless the Agent reported it."""
|
||||
mode = mode if mode in RTC_MODE_LABELS else "DISABLED"
|
||||
report = report or {}
|
||||
return {"mode": mode, "state": RTC_MODE_LABELS[mode], "explanation": RTC_MODE_EXPLANATION, "applied_last_job": len(report.get("applied") or []), "aborted_last_job": len(report.get("aborted") or []),
|
||||
"ready_total": report.get("ready_total"), "capability": report.get("capability"), "authorized": mode in ("ENABLED", "APPROVAL_REQUIRED")}
|
||||
|
||||
|
||||
# ---- cClassTrib control page (Agent -> cClassTrib): candidate status, audit outcome/reason, rule and catalog wording ----
|
||||
_CANDIDATE_STATUS = {"PENDING": "Pendente", "APPROVED": "Aprovado", "REJECTED": "Rejeitado", "APPLIED": "Aplicado", "STALE": "Obsoleto"}
|
||||
_AUDIT_OUTCOME = {"APPLIED": "Aplicado", "ROLLED_BACK": "Revertido", "ABORTED": "Interrompido sem escrever", "FAILED": "Falha técnica", "ROLLBACK_REQUESTED": "Reversão solicitada"}
|
||||
_RULE_LABELS = {"RTC-R3": "Anexo com classe única (RTC-R3)", "RTC-R4": "Sugestão sem autorização de preenchimento (RTC-R4)", "RTC-A3": "Bloqueada por evidência (RTC-A3)", "RTC-G2": "Bloqueada por evidência (RTC-G2)"}
|
||||
_CATALOG_STATE = {"MATCH_SAFE": "Compatível com a tabela oficial", "MATCH_WITH_NON_MATERIAL_DIFFERENCE": "Diferença sem impacto (não autoriza escrita)", "MISMATCH": "Diverge da tabela oficial",
|
||||
"EXPIRED": "Classe oficial vencida", "MISSING_LOCAL": "Classe oficial ausente no catálogo local do SGV", "MISSING_OFFICIAL": "Classe local ausente na tabela oficial"}
|
||||
_ABORT_REASONS = {"VALUE_ALREADY_SET_NEVER_OVERWRITTEN": "cClassTrib existente diverge da classificação proposta", "NO_WRITE_REQUIRED": "Já preenchido com o valor proposto",
|
||||
"RTC_NO_LONGER_ENABLED": "Reforma tributária desligada no ERP", "RTC_CATALOG_INVALID": "Catálogo RTC local inválido", "LOCAL_CATALOG_CHANGED_SINCE_DECISION": "Catálogo local mudou desde a decisão",
|
||||
"COMPANY_CONTEXT_CHANGED": "Contexto da empresa mudou", "REGIME_NOT_NORMAL": "Regime sem regra segura de preenchimento", "PRODUCT_CHANGED_NCM": "NCM do produto mudou",
|
||||
"PRODUCT_CHANGED_SINCE_DECISION": "Produto mudou desde a decisão", "DECISION_CHANGED": "A classificação proposta mudou", "PRODUCT_NOT_FOUND": "Produto não encontrado",
|
||||
"PRODUCT_ROW_NOT_UNIQUE": "Código de produto duplicado", "UPDATE_ROWCOUNT_NOT_ONE": "Escrita recusada (contagem inesperada)", "ROLLBACK_CONFLICT_VALUE_CHANGED": "Valor alterado depois da escrita: reversão recusada",
|
||||
"ALREADY_EMPTY": "Já estava vazio", "ROLLBACK_REQUEST_INVALID": "Pedido de reversão inválido", "TECHNICAL_FAILURE": "Falha técnica", "AUTHORIZATION_CHANGED": "Autorização central mudou antes da escrita", "AUTHORIZATION_UNVERIFIABLE": "Autorização central não pôde ser confirmada"}
|
||||
|
||||
|
||||
def candidate_status_label(value):
|
||||
return _human_label(value, _CANDIDATE_STATUS)
|
||||
|
||||
|
||||
def audit_outcome_label(value):
|
||||
return _human_label(value, _AUDIT_OUTCOME)
|
||||
|
||||
|
||||
def rtc_rule_label(value):
|
||||
return _human_label(value, _RULE_LABELS)
|
||||
|
||||
|
||||
def catalog_state_label(value):
|
||||
return _human_label(value, _CATALOG_STATE)
|
||||
|
||||
|
||||
def abort_reason_label(value):
|
||||
return _human_label(value, _ABORT_REASONS) if value else ""
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Offline-safe product data provider boundary and permanent cache record."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
from typing import Any, Protocol
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProviderRecord:
|
||||
provider: str
|
||||
provider_record_version: str
|
||||
gtin: str
|
||||
retrieved_at: str
|
||||
raw_hash: str
|
||||
normalized_data: dict[str, Any]
|
||||
fiscal_authority: str = "OBSERVATION_ONLY"
|
||||
cache_allowed: bool = False
|
||||
redistribution_allowed: bool = False
|
||||
|
||||
|
||||
class ProductDataProvider(Protocol):
|
||||
def lookup_gtin(self, gtin: str) -> ProviderRecord | None: ...
|
||||
|
||||
|
||||
def normalize_record(provider: str, version: str, gtin: str, data: dict[str, Any]) -> ProviderRecord:
|
||||
normalized = {str(key): value for key, value in sorted(data.items()) if value not in (None, "")}
|
||||
raw_hash = hashlib.sha256(json.dumps(normalized, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8")).hexdigest()
|
||||
return ProviderRecord(provider, version, str(gtin), datetime.now(timezone.utc).isoformat(), raw_hash, normalized)
|
||||
|
||||
|
||||
def cache_key(gtin: str, provider: str, provider_record_version: str) -> str:
|
||||
return f"{gtin}:{provider}:{provider_record_version}"
|
||||
|
||||
|
||||
def result_cache_key(product: dict[str, Any], candidates: list[str], datasets: dict[str, str], prompt: str, model: str) -> str:
|
||||
payload = {"product": product, "candidates": candidates, "datasets": datasets, "prompt": prompt, "model": model}
|
||||
return hashlib.sha256(json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
class LocalBulkDatasetProvider:
|
||||
"""Offline provider backed by explicitly supplied records; never performs I/O."""
|
||||
|
||||
def __init__(self, records: dict[str, ProviderRecord] | None = None):
|
||||
self.records = records or {}
|
||||
|
||||
def lookup_gtin(self, gtin: str) -> ProviderRecord | None:
|
||||
return self.records.get(str(gtin))
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Product Fiscal Defaults Engine V1 — C000025.CFOP / CSOSN / CST default decisions.
|
||||
|
||||
Pure functions. Scope: intra-UF (BA) NFC-e-style sale of merchandise, company CRT=1 (Simples).
|
||||
Historical documents are NEVER an input here; they are used by the runner only to validate.
|
||||
|
||||
CFOP rule chain (all must hold, else CANDIDATE/CONTEXT/UNRESOLVED):
|
||||
ERP CFOP mapping HIGH (product default, saída) + NCM EXACT + current CFOP in official table + first digit 5
|
||||
(saída, intra-UF) + ERP declares resale family (5102/5405: 'adquirida ou recebida de terceiros')
|
||||
+ ST applicability known from BA Anexo 1 (MVA_RESOLVED -> subject, NO_MVA_MAPPING_FOUND -> not subject)
|
||||
=> derived CFOP = 5405 if subject else 5102. Equal to current => RESOLVED (already correct).
|
||||
Different => CANDIDATE only: one of the inputs is wrong and the engine cannot tell which
|
||||
(ST retention by the supplier is not available), so it is never write-eligible.
|
||||
The chain never yields a write: filling a blank CFOP would need the resale-vs-production declaration that
|
||||
only the current CFOP carries, so blank => OPERATION_CONTEXT_REQUIRED.
|
||||
|
||||
CSOSN: no dataset states whether a product is taxed (102), exempt/not taxed (400/300), ST-charged (500)...
|
||||
=> never RESOLVED. Company default C000004.ICMS_CSOSN is only a CANDIDATE anchor for non-ST products.
|
||||
"""
|
||||
PRODUCT_DEFAULTS_ENGINE_VERSION = "product-defaults-v1"
|
||||
RESALE_CFOPS = {"5102", "5405"}
|
||||
|
||||
|
||||
def _blank(v):
|
||||
return v is None or str(v).strip() == ""
|
||||
|
||||
|
||||
def _out(field, current, proposed, state, reason, action, **extra):
|
||||
return {"field": field, "current_value": current, "proposed_value": proposed, "status": state, "reason_code": reason,
|
||||
"action_class": action, "write_eligible": False, "engine": PRODUCT_DEFAULTS_ENGINE_VERSION, **extra}
|
||||
|
||||
|
||||
def st_applicability(ncm_status, mva_status):
|
||||
if ncm_status != "EXACT_NCM":
|
||||
return "UNKNOWN_NCM_NOT_VALID"
|
||||
if mva_status == "MVA_RESOLVED":
|
||||
return "SUBJECT"
|
||||
if mva_status == "NO_MVA_MAPPING_FOUND":
|
||||
return "NOT_SUBJECT"
|
||||
return "UNKNOWN_UNCONFIRMED"
|
||||
|
||||
|
||||
def cfop_default(product, ncm_status, mva_status, cfop_valid):
|
||||
cur = None if _blank(product.get("CFOP")) else str(product["CFOP"]).strip()
|
||||
st = st_applicability(ncm_status, mva_status)
|
||||
base = {"st_applicability": st}
|
||||
if st == "UNKNOWN_NCM_NOT_VALID":
|
||||
return _out("CFOP", cur, None, "CFOP_UNRESOLVED", "NCM_NOT_VALID_CANNOT_ASSESS_ST", "NO_RULE", **base)
|
||||
if cur is None:
|
||||
return _out("CFOP", cur, None, "CFOP_OPERATION_CONTEXT_REQUIRED", "NO_DECLARED_RESALE_OR_PRODUCTION_FAMILY", "OPERATION_CONTEXT_REQUIRED", **base)
|
||||
if cur not in cfop_valid:
|
||||
return _out("CFOP", cur, None, "CFOP_UNRESOLVED", "CURRENT_CFOP_NOT_IN_OFFICIAL_TABLE", "REVIEW_REQUIRED", **base)
|
||||
if cur[0] != "5":
|
||||
return _out("CFOP", cur, None, "CFOP_OPERATION_CONTEXT_REQUIRED", "DIRECTION_OR_UF_SCOPE_NOT_INTRA_UF_SAIDA", "OPERATION_CONTEXT_REQUIRED", **base)
|
||||
if cur not in RESALE_CFOPS:
|
||||
return _out("CFOP", cur, None, "CFOP_OPERATION_CONTEXT_REQUIRED", "NO_RULE_FOR_DECLARED_CFOP_FAMILY", "OPERATION_CONTEXT_REQUIRED", **base)
|
||||
if st == "UNKNOWN_UNCONFIRMED":
|
||||
return _out("CFOP", cur, None, "CFOP_OPERATION_CONTEXT_REQUIRED", "ST_APPLICABILITY_UNCONFIRMED_NEEDS_CEST_CONFIRMATION", "OPERATION_CONTEXT_REQUIRED", **base)
|
||||
derived = "5405" if st == "SUBJECT" else "5102"
|
||||
if derived == cur:
|
||||
return _out("CFOP", cur, derived, "CFOP_DEFAULT_RESOLVED", "CONSISTENT_WITH_ST_STATUS_SAME_VALUE", "ALREADY_CORRECT", **base)
|
||||
return _out("CFOP", cur, derived, "CFOP_DEFAULT_CANDIDATE", "ST_STATUS_CONFLICTS_WITH_ERP_CFOP_RETENTION_CONTEXT_MISSING", "REVIEW_REQUIRED", **base)
|
||||
|
||||
|
||||
def csosn_default(product, ncm_status, mva_status, csosn_valid, company_default):
|
||||
cur = None if _blank(product.get("CSOSN")) else str(product["CSOSN"]).strip()
|
||||
st = st_applicability(ncm_status, mva_status)
|
||||
base = {"st_applicability": st, "company_default": company_default}
|
||||
if st == "UNKNOWN_NCM_NOT_VALID":
|
||||
return _out("CSOSN", cur, None, "CSOSN_UNRESOLVED", "NCM_NOT_VALID_CANNOT_ASSESS_ST", "NO_RULE", **base)
|
||||
if st == "UNKNOWN_UNCONFIRMED":
|
||||
return _out("CSOSN", cur, None, "CSOSN_OPERATION_CONTEXT_REQUIRED", "ST_APPLICABILITY_UNCONFIRMED", "OPERATION_CONTEXT_REQUIRED", **base)
|
||||
if st == "SUBJECT":
|
||||
flag = "ST_SUBJECT_BUT_CSOSN_NOT_500" if cur != "500" else "ST_CSOSN_500_RETENTION_UNVERIFIED"
|
||||
return _out("CSOSN", cur, None, "CSOSN_OPERATION_CONTEXT_REQUIRED", "ST_RETENTION_BY_SUPPLIER_UNKNOWN", "OPERATION_CONTEXT_REQUIRED", consistency_flag=flag, **base)
|
||||
# NOT_SUBJECT: 102 / 400 / 300 / 103 depend on the product's ICMS treatment, for which no source exists
|
||||
if cur is not None and cur == company_default and cur in csosn_valid:
|
||||
return _out("CSOSN", cur, cur, "CSOSN_DEFAULT_CANDIDATE", "MATCHES_COMPANY_DEFAULT_PRODUCT_ICMS_TREATMENT_UNPROVEN", "REVIEW_REQUIRED", **base)
|
||||
if cur is None:
|
||||
return _out("CSOSN", cur, company_default, "CSOSN_DEFAULT_CANDIDATE", "BLANK_COMPANY_DEFAULT_ONLY_ANCHOR_TREATMENT_UNPROVEN", "REVIEW_REQUIRED", **base)
|
||||
return _out("CSOSN", cur, None, "CSOSN_DEFAULT_CANDIDATE", "DIFFERS_FROM_COMPANY_DEFAULT_NO_TREATMENT_SOURCE", "REVIEW_REQUIRED", **base)
|
||||
|
||||
|
||||
def cst_status(product, cst_icms_codes, origem_codes):
|
||||
cur = None if _blank(product.get("CST")) else str(product["CST"]).strip()
|
||||
valid = cur is not None and len(cur) == 3 and cur[1:] in cst_icms_codes and cur[0] in origem_codes
|
||||
return {"field": "CST", "current_value": cur, "status": "CST_REFERENCE_ONLY" if valid else "CST_INVALID_REVIEW",
|
||||
"reason_code": "CRT1_EMISSION_USES_CSOSN_CST_IS_ORIGIN_PLUS_CST" if valid else "NOT_ORIGIN_DIGIT_PLUS_VALID_CST",
|
||||
"origin_digit": cur[0] if valid else None, "write_eligible": False, "engine": PRODUCT_DEFAULTS_ENGINE_VERSION}
|
||||
|
||||
|
||||
def validate_against_history(proposed, emitted_values):
|
||||
"""History validates only; returns MATCH / MISMATCH / UNVALIDATED."""
|
||||
if not emitted_values or proposed is None:
|
||||
return "UNVALIDATED"
|
||||
return "MATCH" if set(emitted_values) == {proposed} else "MISMATCH"
|
||||
@@ -0,0 +1,36 @@
|
||||
"""In-memory per-IP rate limiting for pre-authentication routes.
|
||||
|
||||
Same spirit as painel-clientes-web's `app/rate_limit.py`: a single-process
|
||||
in-memory counter, fine because the Hub runs as one container behind a
|
||||
trusted reverse proxy (FORWARDED_ALLOW_IPS), never multiple replicas behind
|
||||
a shared load balancer. A process restart resets the history — accepted,
|
||||
same tradeoff as that panel's SESSION_SECRET.
|
||||
|
||||
Used for the two public, pre-authentication Agent-distribution routes
|
||||
(GET /dl/agent/{token}, POST /v1/agents/enroll): invalid attempts count
|
||||
against the caller's IP; a legitimate Agent's own successful traffic never
|
||||
gets counted, so it can never push other installations over the limit.
|
||||
"""
|
||||
import time
|
||||
|
||||
MAX_ATTEMPTS = 10
|
||||
WINDOW_SECONDS = 5 * 60
|
||||
|
||||
_attempts: dict[tuple[str, str], list[float]] = {}
|
||||
|
||||
|
||||
def _recent(bucket: str, origin: str, now: float) -> list[float]:
|
||||
cutoff = now - WINDOW_SECONDS
|
||||
return [t for t in _attempts.get((bucket, origin), []) if t > cutoff]
|
||||
|
||||
|
||||
def blocked(bucket: str, origin: str, now: float | None = None) -> bool:
|
||||
moment = now if now is not None else time.time()
|
||||
return len(_recent(bucket, origin, moment)) >= MAX_ATTEMPTS
|
||||
|
||||
|
||||
def record_attempt(bucket: str, origin: str, now: float | None = None) -> None:
|
||||
moment = now if now is not None else time.time()
|
||||
recent = _recent(bucket, origin, moment)
|
||||
recent.append(moment)
|
||||
_attempts[(bucket, origin)] = recent
|
||||
@@ -0,0 +1,174 @@
|
||||
"""Reference Data Registry (Missão 5, Fase 1): the NEW admin-approved ingestion workflow, separate
|
||||
from central/app/datasets/service.py's existing auto-activate methods (untouched, still used by the
|
||||
9 pre-existing dataset kinds). A dataset registered here lands as DRAFT and only becomes VIGENTE
|
||||
(superseding the previous VIGENTE of the same kind) after an explicit ADMIN approval of the diff -
|
||||
"Nenhuma versão de dataset vira 'vigente' sem aprovação ADMIN do diff" (mission rule).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from .models import Dataset
|
||||
|
||||
STATUSES = ("DRAFT", "APPROVED", "VIGENTE", "ACTIVE", "OBSOLETO", "SUPERSEDED", "REJECTED")
|
||||
VIGENTE_STATUSES = ("VIGENTE", "ACTIVE")
|
||||
|
||||
|
||||
class RegistryError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def register_draft(session, *, kind: str, provider: str, source_url: str, sha256: str, version_label: str,
|
||||
act: str = "", valid_from: str | None = None, valid_to: str | None = None, record_count: int = 0,
|
||||
parse_errors: dict | None = None, licence_terms: str | None = None, raw_file_path: str | None = None,
|
||||
file_size_bytes: int | None = None, submitted_by: str | None = None) -> Dataset:
|
||||
"""Registers a new dataset submission as DRAFT. If this exact (provider, sha256) already exists
|
||||
under ANY status, returns it unchanged (idempotent re-submission of an unchanged official file -
|
||||
e.g. re-downloading the same official NCM table produces byte-identical content) instead of
|
||||
creating a duplicate row (the (provider, sha256) constraint would refuse it anyway)."""
|
||||
existing = session.scalar(select(Dataset).where(Dataset.provider == provider, Dataset.sha256 == sha256))
|
||||
if existing:
|
||||
return existing
|
||||
dataset = Dataset(kind=kind, provider=provider, sha256=sha256, version_label=version_label, act=act, source_url=source_url,
|
||||
downloaded_at=datetime.now(timezone.utc), status="DRAFT", record_count=record_count, licence_terms=licence_terms,
|
||||
valid_from=valid_from, valid_to=valid_to, file_size_bytes=file_size_bytes, raw_file_path=raw_file_path,
|
||||
parse_errors=parse_errors or {}, submitted_by=submitted_by)
|
||||
session.add(dataset)
|
||||
session.commit()
|
||||
return dataset
|
||||
|
||||
|
||||
def diff_against_vigente(session, dataset: Dataset) -> dict:
|
||||
"""A summary an ADMIN reviews before approving: what the current VIGENTE of the same kind looks
|
||||
like vs this DRAFT - never the row-by-row content (that lives in the kind-specific entry tables),
|
||||
just the registry-level facts that matter for an approval decision."""
|
||||
current = session.scalar(select(Dataset).where(Dataset.kind == dataset.kind, Dataset.status.in_(VIGENTE_STATUSES)).order_by(Dataset.id.desc()))
|
||||
return {
|
||||
"kind": dataset.kind,
|
||||
"draft": {"id": dataset.id, "version_label": dataset.version_label, "sha256": dataset.sha256, "record_count": dataset.record_count,
|
||||
"valid_from": dataset.valid_from, "valid_to": dataset.valid_to, "parse_errors": dataset.parse_errors},
|
||||
"current_vigente": None if current is None else {"id": current.id, "version_label": current.version_label, "sha256": current.sha256,
|
||||
"record_count": current.record_count, "valid_from": current.valid_from, "valid_to": current.valid_to},
|
||||
"record_count_delta": None if current is None else dataset.record_count - current.record_count,
|
||||
"is_first_version": current is None,
|
||||
"sha256_unchanged": current is not None and current.sha256 == dataset.sha256,
|
||||
}
|
||||
|
||||
|
||||
KNOWN_SAMPLES_BY_KIND = {
|
||||
"NCM": [
|
||||
("01012100", "Reprodutores de raça pura (cavalos)"),
|
||||
("02071400", "Carnes e miudezas de aves (desdobrado/histórico)"),
|
||||
("22021000", "Águas minerais adicionadas de açúcar/aromatizantes"),
|
||||
("84713012", "Máquinas automáticas para processamento de dados"),
|
||||
("10061092", "Arroz com casca, não parboilizado"),
|
||||
],
|
||||
"TIPI_IPI": [
|
||||
("22021000", "Refrigerantes / águas gaseificadas"),
|
||||
("84713012", "Notebooks / computadores portáteis"),
|
||||
("21069090", "Preparações alimentícias diversas"),
|
||||
("02071400", "Carnes de frango congeladas"),
|
||||
("22030000", "Cervejas de malte"),
|
||||
],
|
||||
"CEST": [
|
||||
("01.001.00", "Autopeças - Motores e peças"),
|
||||
("03.021.03", "Cervejas e chopes"),
|
||||
("17.001.00", "Produtos alimentícios - Carnes"),
|
||||
("28.001.00", "Venda porta a porta"),
|
||||
("10.001.00", "Materiais de construção"),
|
||||
],
|
||||
"CFOP": [
|
||||
("5102", "Venda de mercadoria de terceiros"),
|
||||
("5405", "Venda de mercadoria com ST retido"),
|
||||
("1102", "Compra para comercialização"),
|
||||
("1403", "Compra para comercialização com ST"),
|
||||
("5929", "Lançamento relativo a Cupom Fiscal"),
|
||||
],
|
||||
"NFE_MOC_TABLES": [
|
||||
("ORIGEM 0", "Nacional"),
|
||||
("CST_ICMS 00", "Tributada integralmente"),
|
||||
("CSOSN 102", "Tributada Simples sem crédito"),
|
||||
("CST_PIS 01", "Operação tributável alíquota básica"),
|
||||
("CST_IPI 50", "Saída tributada"),
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def technical_checklist(session, dataset: Dataset) -> dict:
|
||||
"""Checklist técnico objetivo para o operador aprovar sem exigir interpretação fiscal."""
|
||||
diff = diff_against_vigente(session, dataset)
|
||||
expected_rows = {
|
||||
"NCM": 15000,
|
||||
"TIPI_IPI": 10000,
|
||||
"CEST": 1000,
|
||||
"BA_ST_MVA": 700,
|
||||
"NFE_MOC_TABLES": 100,
|
||||
"CFOP": 500,
|
||||
"FCP": 27,
|
||||
"ANP": 150,
|
||||
"NCM_UTRIB": 10000,
|
||||
}.get(dataset.kind, 1)
|
||||
|
||||
lines_ok = dataset.record_count >= expected_rows
|
||||
samples = KNOWN_SAMPLES_BY_KIND.get(dataset.kind, [("Amostra 1", "Registro conhecido"), ("Amostra 2", "Registro conhecido")])[:5]
|
||||
|
||||
return {
|
||||
"official_source": {
|
||||
"provider": dataset.provider,
|
||||
"url": dataset.source_url,
|
||||
"act": dataset.act or "Ato normativo oficial",
|
||||
"passed": bool(dataset.provider and dataset.source_url),
|
||||
},
|
||||
"integrity": {
|
||||
"sha256": dataset.sha256,
|
||||
"file_size_bytes": dataset.file_size_bytes,
|
||||
"passed": len(dataset.sha256 or "") == 64,
|
||||
},
|
||||
"volume": {
|
||||
"record_count": dataset.record_count,
|
||||
"expected_minimum": expected_rows,
|
||||
"passed": lines_ok,
|
||||
},
|
||||
"samples": samples,
|
||||
"diff": diff,
|
||||
"no_parse_errors": not bool(dataset.parse_errors),
|
||||
"ready_to_approve": lines_ok and len(dataset.sha256 or "") == 64 and not bool(dataset.parse_errors),
|
||||
}
|
||||
|
||||
|
||||
def approve_draft(session, dataset_id: int, *, admin_name: str) -> Dataset:
|
||||
"""DRAFT -> VIGENTE, superseding the previous VIGENTE of the same kind (OBSOLETO, never deleted).
|
||||
Refuses a dataset that isn't DRAFT, and refuses a dataset with recorded parse errors (must be
|
||||
re-submitted clean - an ADMIN cannot approve past a known parsing problem)."""
|
||||
dataset = session.get(Dataset, dataset_id)
|
||||
if dataset is None:
|
||||
raise RegistryError("dataset not found")
|
||||
if dataset.status != "DRAFT":
|
||||
raise RegistryError(f"only a DRAFT dataset can be approved (current status: {dataset.status})")
|
||||
if dataset.parse_errors:
|
||||
raise RegistryError("dataset has recorded parse errors - re-submit a clean version")
|
||||
session.execute(update(Dataset).where(Dataset.kind == dataset.kind, Dataset.status.in_(VIGENTE_STATUSES)).values(status="OBSOLETO"))
|
||||
dataset.status = "VIGENTE"
|
||||
dataset.approved_by = admin_name
|
||||
dataset.approved_at = datetime.now(timezone.utc)
|
||||
session.commit()
|
||||
return dataset
|
||||
|
||||
|
||||
def reject_draft(session, dataset_id: int, *, admin_name: str) -> Dataset:
|
||||
dataset = session.get(Dataset, dataset_id)
|
||||
if dataset is None:
|
||||
raise RegistryError("dataset not found")
|
||||
if dataset.status != "DRAFT":
|
||||
raise RegistryError(f"only a DRAFT dataset can be rejected (current status: {dataset.status})")
|
||||
dataset.status = "REJECTED"
|
||||
dataset.approved_by = admin_name
|
||||
dataset.approved_at = datetime.now(timezone.utc)
|
||||
session.commit()
|
||||
return dataset
|
||||
|
||||
|
||||
def list_registry(session) -> list[Dataset]:
|
||||
return list(session.scalars(select(Dataset).where(Dataset.status.in_(STATUSES)).order_by(Dataset.kind, Dataset.downloaded_at.desc())))
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Local, non-web-served storage for published Agent release zips.
|
||||
|
||||
Binaries never live in PostgreSQL — only their metadata (AgentRelease rows)
|
||||
do. The storage root is a plain directory outside `central/static`, so it is
|
||||
never directly reachable by a URL; every download goes through the
|
||||
authenticated or token-gated routes in agent_distribution.py.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
from . import config
|
||||
|
||||
_SAFE_COMPONENT = __import__("re").compile(r"^[A-Za-z0-9._-]+$")
|
||||
|
||||
|
||||
def storage_root() -> Path:
|
||||
root = Path(config.settings.agent_release_storage_path)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
return root
|
||||
|
||||
|
||||
def release_filename(version: str, platform: str, architecture: str, extension: str = ".zip") -> str:
|
||||
return f"GTIFiscalAgent-{version}-{platform}-{architecture}{extension}"
|
||||
|
||||
|
||||
def setup_exe_filename(version: str) -> str:
|
||||
return f"GTIFiscalAgent-Setup-{version}.exe"
|
||||
|
||||
|
||||
def safe_path_for(storage_path: str) -> Path:
|
||||
"""Resolve a release's storage_path against the root, refusing anything
|
||||
that could escape it (path traversal). storage_path is DB-controlled
|
||||
(never derived from an HTTP request), but this stays fail-closed anyway."""
|
||||
if not storage_path or not _SAFE_COMPONENT.match(storage_path):
|
||||
raise ValueError(f"unsafe release storage_path: {storage_path!r}")
|
||||
root = storage_root().resolve()
|
||||
candidate = (root / storage_path).resolve()
|
||||
if candidate.parent != root:
|
||||
raise ValueError(f"release storage_path escapes storage root: {storage_path!r}")
|
||||
return candidate
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Persists the Agent's RTC write audit records (append-only), the candidate preview and the rollback bookkeeping; mirrors a summary event for the panel."""
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from .events import record_event
|
||||
from .models import RtcCandidate, RtcWriteAudit
|
||||
|
||||
MAX_AUDITS_PER_JOB = 2000
|
||||
MAX_CANDIDATES = 25000
|
||||
WRITE_MODES = ("DISABLED", "DRY_RUN", "APPROVAL_REQUIRED", "ENABLED")
|
||||
MAX_APPROVED_PER_JOB = 500
|
||||
MAX_ROLLBACKS_PER_JOB = 500
|
||||
|
||||
|
||||
def _clip(value, size):
|
||||
return None if value is None else str(value)[:size]
|
||||
|
||||
|
||||
def job_authorizations(session, installation) -> dict:
|
||||
"""What the authenticated job carries to the Agent. Decided centrally per installation; the Agent has no local switch."""
|
||||
mode = installation.rtc_write_mode if installation.rtc_write_mode in WRITE_MODES else "DISABLED"
|
||||
out = {"rtc_write_mode": mode, "rtc_approved_codes": [], "rtc_rollbacks": [],
|
||||
"features": {"corrector_enabled": bool(installation.corrector_enabled), "enrichment_mode": installation.enrichment_mode, "features_configured": bool(installation.features_configured)}}
|
||||
if mode == "APPROVAL_REQUIRED":
|
||||
out["rtc_approved_codes"] = list(session.scalars(select(RtcCandidate.product_code).where(RtcCandidate.installation_id == installation.id, RtcCandidate.status == "APPROVED").limit(MAX_APPROVED_PER_JOB)))
|
||||
for audit in session.scalars(select(RtcWriteAudit).where(RtcWriteAudit.installation_id == installation.id, RtcWriteAudit.action == "APPLY", RtcWriteAudit.outcome == "APPLIED",
|
||||
RtcWriteAudit.rollback_state == "REQUESTED").limit(MAX_ROLLBACKS_PER_JOB)):
|
||||
out["rtc_rollbacks"].append({"audit_id": audit.id, "product_code": audit.product_code, "written_value": audit.new_value, "rule_id": audit.rule_id})
|
||||
return out
|
||||
|
||||
|
||||
def _store_preview(session, tenant_id, installation, summary, evaluated_all):
|
||||
preview = summary.get("preview")
|
||||
if not isinstance(preview, list) or installation.rtc_write_mode == "DISABLED":
|
||||
return
|
||||
existing = {c.product_code: c for c in session.scalars(select(RtcCandidate).where(RtcCandidate.installation_id == installation.id))}
|
||||
seen = set()
|
||||
for item in preview[:MAX_CANDIDATES]:
|
||||
code = _clip(item.get("product_code"), 64)
|
||||
if not code or not item.get("proposed"):
|
||||
continue
|
||||
seen.add(code)
|
||||
row = existing.get(code)
|
||||
if row is None:
|
||||
session.add(RtcCandidate(tenant_id=tenant_id, installation_id=installation.id, product_code=code, description=_clip(item.get("description"), 255), ncm=_clip(item.get("ncm"), 16),
|
||||
current_value=_clip(item.get("current") or None, 16), proposed=_clip(item["proposed"], 16), cst_ibscbs=_clip(item.get("cst"), 8), rule_id=_clip(item.get("rule_id"), 32),
|
||||
basis=_clip(item.get("basis"), 255), status="PENDING"))
|
||||
else:
|
||||
if row.status in ("APPLIED", "STALE"):
|
||||
row.status = "PENDING" # empty again (e.g. after a rollback) or newly eligible
|
||||
if row.proposed != item["proposed"]:
|
||||
row.proposed = _clip(item["proposed"], 16)
|
||||
if row.status == "APPROVED":
|
||||
row.status = "PENDING" # the proposal changed: the earlier approval does not carry over
|
||||
row.description, row.ncm, row.cst_ibscbs = _clip(item.get("description"), 255), _clip(item.get("ncm"), 16), _clip(item.get("cst"), 8)
|
||||
row.rule_id, row.basis = _clip(item.get("rule_id"), 32), _clip(item.get("basis"), 255)
|
||||
if evaluated_all and not summary.get("preview_truncated"):
|
||||
for code, row in existing.items():
|
||||
if code not in seen and row.status in ("PENDING", "APPROVED"):
|
||||
row.status = "STALE"
|
||||
|
||||
|
||||
def record_rtc_write_audits(session, *, tenant_id: int, installation, agent, job_id: int | None, rtc_auto_apply: dict | None, evaluated_all: bool = False) -> int:
|
||||
"""Stores every applied/aborted/rolled-back write of the job. Never trusts the Agent for identity: tenant/installation/agent come from the authenticated context."""
|
||||
if not isinstance(rtc_auto_apply, dict):
|
||||
return 0
|
||||
_store_preview(session, tenant_id, installation, rtc_auto_apply, evaluated_all)
|
||||
records = list(rtc_auto_apply.get("applied") or []) + list(rtc_auto_apply.get("aborted") or []) + list(rtc_auto_apply.get("rollbacks") or [])
|
||||
stored = applied = aborted = rolled = 0
|
||||
for item in records[:MAX_AUDITS_PER_JOB]:
|
||||
if not isinstance(item, dict) or item.get("field") != "CCLASSTRIB":
|
||||
continue
|
||||
action = "ROLLBACK" if item.get("action") == "ROLLBACK" else "APPLY"
|
||||
outcome = str(item.get("outcome", ""))[:16]
|
||||
rollback_of = item.get("rollback_of") if action == "ROLLBACK" else None
|
||||
code = str(item.get("product_code", ""))[:64]
|
||||
session.add(RtcWriteAudit(
|
||||
tenant_id=tenant_id, installation_id=installation.id, agent_id=agent.id, job_id=job_id, client=getattr(installation, "display_name", None),
|
||||
product_code=code, field="CCLASSTRIB", outcome=outcome, reason=(item.get("reason") or None) and str(item["reason"])[:96],
|
||||
old_value=_clip(item.get("old_value"), 64), new_value=_clip(item.get("new_value"), 64),
|
||||
rule_id=(item.get("rule_id") or None) and str(item["rule_id"])[:32], evidence=item.get("evidence") or {}, official_source_version=item.get("official_source_version") or {},
|
||||
local_catalog_hash=item.get("local_catalog_hash"), written_at=item.get("timestamp"), rule_version=_clip(item.get("rule_version"), 32), company_crt=_clip(item.get("company_crt"), 8),
|
||||
authorization_origin=_clip(item.get("authorization_origin"), 48), description_snapshot=_clip(item.get("description_snapshot"), 255), installation_uuid=_clip(getattr(installation, "installation_id", None), 64),
|
||||
rtc_status=_clip(item.get("rtc_status"), 24), local_catalog_state=_clip(item.get("local_catalog_state"), 32), action=action, rollback_of=rollback_of if isinstance(rollback_of, int) else None))
|
||||
stored += 1
|
||||
candidate_of = (RtcCandidate.installation_id == installation.id) & (RtcCandidate.product_code == code)
|
||||
if action == "ROLLBACK":
|
||||
if isinstance(rollback_of, int):
|
||||
session.execute(update(RtcWriteAudit).where(RtcWriteAudit.id == rollback_of, RtcWriteAudit.installation_id == installation.id).values(rollback_state="DONE" if outcome == "ROLLED_BACK" else "BLOCKED"))
|
||||
if outcome == "ROLLED_BACK":
|
||||
rolled += 1
|
||||
session.execute(update(RtcCandidate).where(candidate_of).values(status="PENDING"))
|
||||
elif outcome == "APPLIED":
|
||||
applied += 1
|
||||
session.execute(update(RtcCandidate).where(candidate_of).values(status="APPLIED"))
|
||||
else:
|
||||
aborted += 1
|
||||
if stored:
|
||||
record_event(session, tenant_id=tenant_id, installation_id=installation.id, agent_id=agent.id, severity="WARNING" if (applied or rolled) else "INFO", event_type="RTC_AUTO_APPLY", job_id=job_id,
|
||||
message=f"cClassTrib: {applied} preenchido(s), {rolled} revertido(s), {aborted} abortado(s)",
|
||||
context={"applied": applied, "rolled_back": rolled, "aborted": aborted, "reasons": rtc_auto_apply.get("aborted_reasons") or {}, "mode": rtc_auto_apply.get("mode")})
|
||||
session.commit()
|
||||
return stored
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Turns the deterministic cClassTrib resolution into the ONE root-cause RTC finding per product (pure).
|
||||
|
||||
`evaluate_product` (unified_fiscal_coverage) says WHETHER a product violates the RTC rule; this module says WHY and whether the gap is closable:
|
||||
the single RTC blocker of a product is refined to the cause that matters, never several blockers for the same root cause.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from .cclasstrib_resolver import HUMAN_REASONS, Resolution
|
||||
|
||||
RTC_CATALOG_REASONS = ("LOCAL_OFFICIAL_", "CLASS_NOT_IN_LOCAL_CATALOG", "CLASS_NOT_IN_OFFICIAL_TABLE", "CLASS_NOT_YET_IN_FORCE", "CLASS_NOT_USABLE_IN_NFE_NFCE", "LOCAL_RTC_CATALOG_INVALID")
|
||||
EVIDENCE_REASONS = ("NO_VALID_NCM", "COMPANY_REGIME_UNKNOWN")
|
||||
RTC_TYPES = ("CCLASSTRIB_MISSING", "CCLASSTRIB_MISSING_UNRESOLVED", "CCLASSTRIB_AMBIGUOUS", "CCLASSTRIB_CONFLICT", "CCLASSTRIB_OFFICIAL_MISMATCH", "RTC_LOCAL_CATALOG_INCOMPATIBLE",
|
||||
"RTC_RULE_EVIDENCE_MISSING", "CCLASSTRIB_INVALID_FORMAT", "CCLASSTRIB_NOT_IN_CATALOG", "RTC_CST_MISSING", "RTC_CATALOG_INVALID")
|
||||
BAD_LOCAL_STATES = ("MISMATCH", "EXPIRED", "MISSING_OFFICIAL")
|
||||
|
||||
|
||||
def _text(value) -> str:
|
||||
return "" if value is None else str(value).strip()
|
||||
|
||||
|
||||
def slim(resolution: Resolution) -> dict:
|
||||
"""Small, audit-grade view kept in the job result (full evidence stays in the resolver output)."""
|
||||
return {"status": resolution.status, "rule_id": resolution.rule_id, "rule_class": resolution.rule_class, "reason": resolution.reason, "resolved_cclasstrib": resolution.resolved_cclasstrib,
|
||||
"resolved_cst_ibscbs": resolution.resolved_cst_ibscbs, "confidence_type": resolution.confidence_type, "auto_apply_candidate": resolution.auto_apply_candidate,
|
||||
"suggested_cclasstrib": resolution.suggested_cclasstrib}
|
||||
|
||||
|
||||
def refine_rtc_finding(found: dict, product: dict, resolution: Resolution | None, compat_state: dict | None) -> dict:
|
||||
"""Mutates and returns `found` (an `evaluate_product` result). Only acts when RTC is REQUIRED for the product."""
|
||||
if found.get("applicability", {}).get("RTC") != "REQUIRED" or resolution is None:
|
||||
return found
|
||||
value = _text(product.get("CCLASSTRIB"))
|
||||
rtc_blockers = [b for b in found["blockers"] if b["category"] == "RTC"]
|
||||
kinds = {b["type"] for b in rtc_blockers}
|
||||
found["rtc_resolution"] = slim(resolution)
|
||||
if "RTC_CATALOG_INVALID" in kinds:
|
||||
return found # nothing can be proposed or compared against an invalid catalog
|
||||
others = [b for b in found["blockers"] if b["category"] != "RTC"]
|
||||
if "CCLASSTRIB_MISSING" in kinds:
|
||||
if resolution.auto_apply_candidate:
|
||||
new = ("CCLASSTRIB_MISSING", f"READY_TO_APPLY:{resolution.resolved_cclasstrib}:{resolution.rule_id}")
|
||||
elif resolution.status == "SUGGESTION_ONLY":
|
||||
new = ("CCLASSTRIB_MISSING_UNRESOLVED", f"{resolution.rule_id}:SUGGESTION_ONLY:{resolution.resolved_cclasstrib}")
|
||||
elif resolution.status == "AMBIGUOUS":
|
||||
new = ("CCLASSTRIB_AMBIGUOUS", f"{resolution.rule_id}:{resolution.reason}")
|
||||
elif resolution.reason.startswith(RTC_CATALOG_REASONS):
|
||||
new = ("RTC_LOCAL_CATALOG_INCOMPATIBLE", resolution.reason)
|
||||
elif resolution.reason in EVIDENCE_REASONS:
|
||||
new = ("RTC_RULE_EVIDENCE_MISSING", resolution.reason)
|
||||
else:
|
||||
new = ("CCLASSTRIB_MISSING_UNRESOLVED", f"{resolution.rule_id}:{resolution.reason}")
|
||||
blocker = {"type": new[0], "category": "RTC", "reason": new[1]}
|
||||
if resolution.reason in HUMAN_REASONS:
|
||||
blocker["human"] = HUMAN_REASONS[resolution.reason]
|
||||
found["blockers"] = others + [blocker]
|
||||
return found
|
||||
if not rtc_blockers and value: # a valid existing class: compare, never overwrite
|
||||
state = (compat_state or {}).get(value)
|
||||
if state in BAD_LOCAL_STATES:
|
||||
found["blockers"] = others + [{"type": "CCLASSTRIB_OFFICIAL_MISMATCH", "category": "RTC", "reason": f"LOCAL_VS_OFFICIAL:{state}"}]
|
||||
elif resolution.auto_apply_candidate and resolution.resolved_cclasstrib != value:
|
||||
found["blockers"] = others + [{"type": "CCLASSTRIB_CONFLICT", "category": "RTC", "reason": f"HAS:{value};RULE:{resolution.rule_id}:{resolution.resolved_cclasstrib}"}]
|
||||
else:
|
||||
found["rtc_resolution"]["no_write_required"] = resolution.auto_apply_candidate and resolution.resolved_cclasstrib == value
|
||||
return found
|
||||
|
||||
|
||||
def summarize_rtc(findings: dict, compat: dict | None = None) -> dict:
|
||||
"""Counts for the panel and the audit: valid, auto-resolvable (safe), suggested (diagnostic only), blocked and conflicting, catalog-incompatible."""
|
||||
out = {"total": 0, "valid_existing": 0, "ready_to_apply": 0, "suggestion_only": 0, "ambiguous": 0, "insufficient": 0, "conflicts": 0, "official_mismatch": 0, "catalog_incompatible": 0,
|
||||
"not_applicable": 0, "other_blocked": 0, "by_rule": {}, "by_reason": {}}
|
||||
for found in findings.values():
|
||||
out["total"] += 1
|
||||
if found.get("applicability", {}).get("RTC") != "REQUIRED":
|
||||
out["not_applicable"] += 1
|
||||
continue
|
||||
types = {b["type"] for b in found["blockers"] if b["category"] == "RTC"}
|
||||
res = found.get("rtc_resolution") or {}
|
||||
if res:
|
||||
out["by_rule"][res["rule_id"]] = out["by_rule"].get(res["rule_id"], 0) + 1
|
||||
out["by_reason"][res["reason"]] = out["by_reason"].get(res["reason"], 0) + 1
|
||||
if not types:
|
||||
out["valid_existing"] += 1
|
||||
elif "CCLASSTRIB_CONFLICT" in types:
|
||||
out["conflicts"] += 1
|
||||
elif "CCLASSTRIB_OFFICIAL_MISMATCH" in types:
|
||||
out["official_mismatch"] += 1
|
||||
elif "RTC_LOCAL_CATALOG_INCOMPATIBLE" in types or "RTC_CATALOG_INVALID" in types:
|
||||
out["catalog_incompatible"] += 1
|
||||
elif res.get("auto_apply_candidate") and "CCLASSTRIB_MISSING" in types:
|
||||
out["ready_to_apply"] += 1
|
||||
elif res.get("status") == "SUGGESTION_ONLY":
|
||||
out["suggestion_only"] += 1
|
||||
elif "CCLASSTRIB_AMBIGUOUS" in types:
|
||||
out["ambiguous"] += 1
|
||||
elif types & {"CCLASSTRIB_MISSING_UNRESOLVED", "RTC_RULE_EVIDENCE_MISSING", "CCLASSTRIB_MISSING"}:
|
||||
out["insufficient"] += 1
|
||||
else:
|
||||
out["other_blocked"] += 1
|
||||
if compat:
|
||||
out["catalog_compatibility"] = {"compatibility": compat["compatibility"], "counts": compat["counts"], "expired_present_locally": compat.get("expired_present_locally", 0)}
|
||||
return out
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,173 @@
|
||||
"""SSO from GTI Gerenciador: the panel is the identity authority.
|
||||
|
||||
Flow (docs/PILOT_DEPLOYMENT.md, "Human access"):
|
||||
GET /login -> Hub sets a one-time state cookie and redirects to the
|
||||
panel's /sso/fiscal?state=...&next=...
|
||||
panel -> for a logged-in, active employee signs a JWT (Ed25519,
|
||||
60 s, unique jti, nonce=state) and auto-POSTs it to
|
||||
POST /auth/sso/callback -> verify_assertion + check state + consume_jti +
|
||||
provision -> Hub session of its own.
|
||||
|
||||
The Hub only holds the panel's *public* key, never the panel's cookies,
|
||||
passwords or database. The state cookie binds the assertion to the browser
|
||||
that started the flow, so nobody can push their own assertion into someone
|
||||
else's browser (login CSRF).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hmac
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import jwt
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from . import config
|
||||
from .models import AdminUser, SsoUsedAssertion
|
||||
|
||||
ISSUER = "gti-gerenciador"
|
||||
AUDIENCE = "gti-fiscal"
|
||||
AUTH_SOURCE = "GTI_GERENCIADOR"
|
||||
REQUIRED_CLAIMS = ["iss", "aud", "sub", "iat", "nbf", "exp", "jti", "nonce"]
|
||||
MAX_LIFETIME_SECONDS = 120 # the panel issues 60 s; anything longer is refused
|
||||
CLOCK_LEEWAY_SECONDS = 10
|
||||
STATE_COOKIE = "gti_sso_state"
|
||||
STATE_COOKIE_PATH = "/auth/sso"
|
||||
STATE_SECONDS = 300
|
||||
MAX_NEXT_LENGTH = 512
|
||||
_SUBJECT = re.compile(r"funcionario:[1-9][0-9]{0,18}")
|
||||
_TOKEN_CHARS = re.compile(r"[A-Za-z0-9_\-]{16,128}")
|
||||
|
||||
|
||||
class SsoRejected(Exception):
|
||||
"""Reason codes are stable: they go to the audit log (never the token)."""
|
||||
|
||||
def __init__(self, reason: str, subject: str | None = None):
|
||||
super().__init__(reason)
|
||||
self.reason = reason
|
||||
self.subject = subject
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Identity:
|
||||
subject: str
|
||||
login: str
|
||||
name: str
|
||||
role: str | None
|
||||
jti: str
|
||||
expires_at: datetime
|
||||
|
||||
|
||||
def enabled() -> bool:
|
||||
return bool(config.settings.sso_public_key) and panel_home() is not None
|
||||
|
||||
|
||||
def panel_home() -> str | None:
|
||||
"""https origin of GTI Gerenciador, from SSO_START_URL (None if unset/not https)."""
|
||||
parts = urlsplit(config.settings.sso_start_url or "")
|
||||
if parts.scheme != "https" or not parts.hostname or parts.username or parts.password:
|
||||
return None
|
||||
return f"https://{parts.netloc}/"
|
||||
|
||||
|
||||
def safe_next(value: str | None) -> str:
|
||||
"""Only a relative path inside the Hub; anything else becomes "/"."""
|
||||
if (
|
||||
not value
|
||||
or len(value) > MAX_NEXT_LENGTH
|
||||
or not value.startswith("/")
|
||||
or value.startswith("//")
|
||||
or "\\" in value
|
||||
or any(ord(char) < 32 or ord(char) == 127 for char in value)
|
||||
):
|
||||
return "/"
|
||||
return value
|
||||
|
||||
|
||||
def valid_state(value: str | None) -> bool:
|
||||
return bool(value) and bool(_TOKEN_CHARS.fullmatch(value))
|
||||
|
||||
|
||||
def _public_key() -> Ed25519PublicKey:
|
||||
raw = config.settings.sso_public_key.strip()
|
||||
return Ed25519PublicKey.from_public_bytes(base64.urlsafe_b64decode(raw + "=" * (-len(raw) % 4)))
|
||||
|
||||
|
||||
def verify_assertion(token: str | None, expected_state: str | None) -> Identity:
|
||||
if not token or len(token) > 4096:
|
||||
raise SsoRejected("MISSING_ASSERTION")
|
||||
try:
|
||||
claims = jwt.decode(
|
||||
token, _public_key(), algorithms=["EdDSA"], audience=AUDIENCE, issuer=ISSUER,
|
||||
leeway=CLOCK_LEEWAY_SECONDS, options={"require": REQUIRED_CLAIMS},
|
||||
)
|
||||
except jwt.ExpiredSignatureError as error:
|
||||
raise SsoRejected("EXPIRED") from error
|
||||
except jwt.ImmatureSignatureError as error:
|
||||
raise SsoRejected("NOT_YET_VALID") from error
|
||||
except jwt.InvalidAudienceError as error:
|
||||
raise SsoRejected("WRONG_AUDIENCE") from error
|
||||
except jwt.InvalidIssuerError as error:
|
||||
raise SsoRejected("WRONG_ISSUER") from error
|
||||
except jwt.InvalidAlgorithmError as error:
|
||||
raise SsoRejected("INVALID_ALGORITHM") from error
|
||||
except jwt.InvalidSignatureError as error:
|
||||
raise SsoRejected("INVALID_SIGNATURE") from error
|
||||
except jwt.MissingRequiredClaimError as error:
|
||||
raise SsoRejected("MISSING_CLAIM") from error
|
||||
except jwt.InvalidTokenError as error:
|
||||
raise SsoRejected("MALFORMED") from error
|
||||
subject = claims["sub"]
|
||||
if not isinstance(subject, str) or not _SUBJECT.fullmatch(subject):
|
||||
raise SsoRejected("INVALID_SUBJECT")
|
||||
if not isinstance(claims["iat"], int) or not isinstance(claims["exp"], int) or claims["exp"] - claims["iat"] > MAX_LIFETIME_SECONDS:
|
||||
raise SsoRejected("LIFETIME_TOO_LONG", subject)
|
||||
if not isinstance(claims["jti"], str) or not _TOKEN_CHARS.fullmatch(claims["jti"]) or len(claims["jti"]) > 64:
|
||||
raise SsoRejected("INVALID_JTI", subject)
|
||||
nonce = claims["nonce"]
|
||||
if not valid_state(expected_state) or not isinstance(nonce, str) or not hmac.compare_digest(nonce, expected_state):
|
||||
raise SsoRejected("STATE_MISMATCH", subject)
|
||||
login, name, role = claims.get("login"), claims.get("name"), claims.get("perfil")
|
||||
if not isinstance(login, str) or not login.strip() or not isinstance(name, str):
|
||||
raise SsoRejected("MISSING_PROFILE", subject)
|
||||
return Identity(
|
||||
subject=subject, login=login.strip()[:128], name=name.strip()[:160] or login.strip()[:160],
|
||||
role=role[:32] if isinstance(role, str) else None, jti=claims["jti"],
|
||||
expires_at=datetime.fromtimestamp(claims["exp"], tz=timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
def consume_jti(session, identity: Identity) -> None:
|
||||
"""Records the jti; a second use of the same assertion is a replay."""
|
||||
now = datetime.now(timezone.utc)
|
||||
session.execute(delete(SsoUsedAssertion).where(SsoUsedAssertion.expires_at < now - timedelta(days=1)))
|
||||
session.add(SsoUsedAssertion(jti=identity.jti, subject=identity.subject, expires_at=identity.expires_at))
|
||||
try:
|
||||
session.flush()
|
||||
except IntegrityError as error:
|
||||
session.rollback()
|
||||
raise SsoRejected("REPLAY", identity.subject) from error
|
||||
|
||||
|
||||
def provision(session, identity: Identity) -> AdminUser:
|
||||
"""Creates the collaborator on first access, refreshes name/role afterwards."""
|
||||
user = session.scalar(select(AdminUser).where(AdminUser.external_subject == identity.subject))
|
||||
if user is None:
|
||||
user = AdminUser(
|
||||
username=f"gerenciador:{identity.subject.split(':', 1)[1]}", password_hash=None, auth_source=AUTH_SOURCE,
|
||||
external_subject=identity.subject, display_name=identity.name, external_role=identity.role, is_active=True,
|
||||
)
|
||||
session.add(user)
|
||||
session.flush()
|
||||
return user
|
||||
if user.auth_source != AUTH_SOURCE:
|
||||
raise SsoRejected("SUBJECT_BOUND_TO_LOCAL_ACCOUNT", identity.subject)
|
||||
if not user.is_active:
|
||||
raise SsoRejected("DISABLED_IN_FISCAL", identity.subject)
|
||||
user.display_name, user.external_role = identity.name, identity.role
|
||||
return user
|
||||
@@ -0,0 +1,273 @@
|
||||
"""UNIFIED fiscal completeness: ONE definition of "complete" for a product (pure, no I/O).
|
||||
|
||||
mandatory(product, company, rtc) = base fiscal fields (decided elsewhere: NCM/CEST/CFOP/ICMS/IPI ...)
|
||||
+ PIS/COFINS when they APPLY to the product
|
||||
+ RTC (cClassTrib -> CST IBS/CBS) ONLY when the ERP has the tax reform switched on.
|
||||
|
||||
There is no "legacy scope" and no "expanded scope": every finding here becomes a normal blocker in the single queue and
|
||||
changes the single coverage number. Nothing here writes to the ERP or invents a fiscal value.
|
||||
|
||||
Sources (see docs/pis_cofins_applicability.md): SPED EFD-Contribuicoes table 4.3.3/4.3.4 (CST domain), 4.3.10-4.3.15 (special regimes),
|
||||
LC 123/2006 art. 13/18 (PIS/Cofins inside the Simples DAS), Leis 9.718/98, 10.637/2002, 10.833/2003 (cumulative / non-cumulative),
|
||||
NT 2025.002-RTC v1.51 (IBS/CBS fields), LC 214/2025 art. 348.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
from datetime import date
|
||||
|
||||
# Official CST PIS / CST COFINS domain (SPED tables 4.3.3 / 4.3.4). Membership only - never used to pick a value.
|
||||
CST_DOMAIN = frozenset([f"{n:02d}" for n in (*range(1, 10), 49, *range(50, 57), *range(60, 68), *range(70, 76), 98, 99)])
|
||||
RATE_REQUIRED_CST = frozenset({"01", "02"}) # taxable by percentage: a rate is needed
|
||||
RATE_NOT_APPLICABLE_CST = frozenset({"03", "04", "05", "06", "07", "08", "09"}) # per-unit amount / monophasic / ST / zero / exempt / no incidence / suspended
|
||||
INCIDENCE_CST = {"MONOFASICA": frozenset({"02", "03", "04"}), "SUBSTITUICAO": frozenset({"05"})} # 4.3.10/4.3.11 (02,03,04) and 4.3.12 (05)
|
||||
SIMPLES_FAMILY = frozenset({"1", "2", "4"}) # CRT 1 Simples, 2 Simples excess sublimit, 4 MEI (LC 123/2006): PIS/Cofins are collected inside the DAS
|
||||
|
||||
PIS_COFINS_FIELDS = ("COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS")
|
||||
|
||||
FIELD_BY_BLOCKER_PREFIX = (("PIS", "PIS"), ("COFINS", "COFINS"), ("CCLASSTRIB", "RTC"), ("RTC", "RTC"))
|
||||
|
||||
|
||||
def _text(value) -> str:
|
||||
return "" if value is None else str(value).strip()
|
||||
|
||||
|
||||
def blocker_category(blocker_type: str) -> str:
|
||||
"""Dashboard category of a blocker type: PIS | COFINS | RTC | OTHER."""
|
||||
for prefix, category in FIELD_BY_BLOCKER_PREFIX:
|
||||
if blocker_type.startswith(prefix):
|
||||
return category
|
||||
return "OTHER"
|
||||
|
||||
|
||||
def incidence_kind(value) -> str:
|
||||
"""NONE | MONOFASICA | SUBSTITUICAO | UNKNOWN (accent/case-insensitive, encoding tolerant). Anything the SGV does not define is UNKNOWN, never NONE."""
|
||||
raw = _text(value)
|
||||
text = re.sub(r"[^A-Z]", "", unicodedata.normalize("NFKD", raw).encode("ascii", "ignore").decode().upper())
|
||||
if not raw or text == "NENHUMA":
|
||||
return "NONE"
|
||||
if text.startswith("MONOF"):
|
||||
return "MONOFASICA"
|
||||
if text.startswith("SUBST"):
|
||||
return "SUBSTITUICAO"
|
||||
return "UNKNOWN"
|
||||
|
||||
|
||||
def canonical_cst(value) -> str | None:
|
||||
"""The official 2-digit CST this text stands for, or None. Exactly two official digits, or ONE digit whose zero-padded form is an official code
|
||||
("4" -> "04"). Nothing else is ever normalised ("049", "000", "2x" stay invalid)."""
|
||||
code = _text(value)
|
||||
if code in CST_DOMAIN:
|
||||
return code
|
||||
if len(code) == 1 and code.isdigit() and code.zfill(2) in CST_DOMAIN:
|
||||
return code.zfill(2)
|
||||
return None
|
||||
|
||||
|
||||
def validate_cst(value) -> str:
|
||||
"""EMPTY | VALID | VALID_AFTER_NORMALIZATION | INVALID."""
|
||||
code = _text(value)
|
||||
if not code:
|
||||
return "EMPTY"
|
||||
if code in CST_DOMAIN:
|
||||
return "VALID"
|
||||
return "VALID_AFTER_NORMALIZATION" if canonical_cst(code) else "INVALID"
|
||||
|
||||
|
||||
def validate_rate(value) -> str:
|
||||
text = _text(value)
|
||||
if not text:
|
||||
return "EMPTY"
|
||||
if not re.fullmatch(r"\d{1,3}([.,]\d{1,4})?", text):
|
||||
return "INVALID"
|
||||
return "VALID" if float(text.replace(",", ".")) <= 100 else "INVALID"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------- PIS / COFINS
|
||||
|
||||
|
||||
def pis_cofins_applicability(product: dict, crt) -> dict:
|
||||
"""{"state": NOT_APPLICABLE | REQUIRED | OPTIONAL | NEEDS_REVIEW, "basis": ...}.
|
||||
|
||||
Decided ONLY from signals that are independent of the six CST/rate fields - an empty field is a data state, never proof of "not applicable":
|
||||
* the company regime (CRT);
|
||||
* the product's own declaration: PISCOFINS flag and INCIDENCIA_PISCOFINS;
|
||||
* official rules (LC 123/2006 art. 13 IV/V and art. 18 par. 4-A I; Leis 9.718/98, 10.637/02, 10.833/03; SPED 4.3.x for the CST domain/semantics).
|
||||
|
||||
REQUIRED the product declares a special treatment (flag S, or incidence MONOFASICA / SUBSTITUICAO): the exit CST must be valid, whatever the fields hold.
|
||||
NOT_APPLICABLE Simples family (CRT 1/2/4) with no special treatment declared: PIS/Cofins are collected inside the DAS (LC 123 art. 13), so the product carries no
|
||||
obligation. The same verdict holds whether the fields are empty or typed.
|
||||
OPTIONAL taxed regime (CRT 3 or other) AND the product explicitly declares "no special treatment" (flag N + incidence NENHUMA): the tax applies but the
|
||||
company default covers it, so the product default is not mandatory; anything typed must still be valid.
|
||||
NEEDS_REVIEW the declaration cannot be read: incidence text outside the SGV vocabulary, or a taxed regime with a blank flag/incidence.
|
||||
"""
|
||||
flag = _text(product.get("PISCOFINS")).upper()
|
||||
raw_incidence = _text(product.get("INCIDENCIA_PISCOFINS"))
|
||||
kind = incidence_kind(raw_incidence)
|
||||
if kind == "UNKNOWN":
|
||||
return {"state": "NEEDS_REVIEW", "basis": "INCIDENCE_NOT_DEFINED_BY_SGV"}
|
||||
if flag == "S" or kind in ("MONOFASICA", "SUBSTITUICAO"):
|
||||
return {"state": "REQUIRED", "basis": "PRODUCT_DECLARES_SPECIAL_TREATMENT"}
|
||||
if _text(crt) in SIMPLES_FAMILY:
|
||||
return {"state": "NOT_APPLICABLE", "basis": "SIMPLES_DAS_NO_SPECIAL_TREATMENT"}
|
||||
if flag == "N" and raw_incidence:
|
||||
return {"state": "OPTIONAL", "basis": "TAXED_REGIME_COMPANY_DEFAULT_COVERS"}
|
||||
return {"state": "NEEDS_REVIEW", "basis": "NO_DECLARATION_IN_TAXED_REGIME"}
|
||||
|
||||
|
||||
def evaluate_pis_cofins(product: dict, crt) -> dict:
|
||||
"""{"applicability": {...}, "blockers": [(type, reason)], "warnings": [(type, reason)]}. Entry and exit are separate; PIS and COFINS independent.
|
||||
Blockers exist only where the field applies (REQUIRED / OPTIONAL). Non-blocking warnings report data-quality facts (non-canonical or ignored values)."""
|
||||
app = pis_cofins_applicability(product, crt)
|
||||
blockers: list[tuple[str, str]] = []
|
||||
warnings: list[tuple[str, str]] = []
|
||||
kind = incidence_kind(product.get("INCIDENCIA_PISCOFINS"))
|
||||
flag = _text(product.get("PISCOFINS")).upper()
|
||||
applies = app["state"] in ("REQUIRED", "OPTIONAL")
|
||||
for tax in ("PIS", "COFINS"):
|
||||
entry, exit_, rate = product.get(f"COD_ENTRADA_{tax}"), product.get(f"COD_SAIDA_{tax}"), product.get(f"ALIQ_{tax}")
|
||||
entry_state, exit_state, rate_state = validate_cst(entry), validate_cst(exit_), validate_rate(rate)
|
||||
for slot, value, state in (("ENTRY", entry, entry_state), ("EXIT", exit_, exit_state)):
|
||||
if state == "VALID_AFTER_NORMALIZATION":
|
||||
warnings.append((f"CST_{tax}_NON_CANONICAL_FORMAT", f"{slot}:{_text(value)}->{canonical_cst(value)}"))
|
||||
elif state == "INVALID":
|
||||
if applies:
|
||||
blockers.append((f"{tax}_CST_INVALID", slot))
|
||||
else:
|
||||
warnings.append((f"CST_{tax}_INVALID_IGNORED", f"{slot}:{_text(value)}"))
|
||||
if not applies:
|
||||
continue
|
||||
if app["state"] == "REQUIRED" and exit_state == "EMPTY":
|
||||
blockers.append((f"{tax}_CST_MISSING", "EXIT"))
|
||||
if rate_state == "INVALID":
|
||||
blockers.append((f"{tax}_RATE_INVALID", "RATE"))
|
||||
code = canonical_cst(exit_)
|
||||
if code:
|
||||
if code in RATE_REQUIRED_CST and rate_state == "EMPTY":
|
||||
blockers.append((f"{tax}_RATE_MISSING", f"CST_{code}_NEEDS_RATE"))
|
||||
if kind in INCIDENCE_CST and code not in INCIDENCE_CST[kind]:
|
||||
blockers.append((f"{tax}_CST_INCIDENCE_MISMATCH", f"{kind}:CST_{code}"))
|
||||
if kind == "NONE" and app["state"] == "REQUIRED" and code in RATE_NOT_APPLICABLE_CST - {"03"} and flag == "S":
|
||||
blockers.append((f"{tax}_CST_INCIDENCE_MISMATCH", f"NENHUMA:CST_{code}"))
|
||||
if applies and flag == "N" and kind != "NONE":
|
||||
blockers.append(("PIS_COFINS_FLAG_INCONSISTENT", "FLAG_N_WITH_SPECIAL_INCIDENCE"))
|
||||
if applies and flag == "S" and kind == "NONE" and not any(_text(product.get(c)) for c in PIS_COFINS_FIELDS):
|
||||
blockers.append(("PIS_COFINS_FLAG_INCONSISTENT", "FLAG_S_WITHOUT_INCIDENCE_OR_CODES"))
|
||||
return {"applicability": app, "blockers": _dedupe(blockers), "warnings": _dedupe(warnings)}
|
||||
|
||||
|
||||
def _dedupe(items):
|
||||
seen, out = set(), []
|
||||
for item in items:
|
||||
if item not in seen:
|
||||
seen.add(item)
|
||||
out.append(item)
|
||||
return out
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------- RTC
|
||||
|
||||
CCLASSTRIB_EFFECTIVE_FROM = {
|
||||
"3": date(2026, 8, 3),
|
||||
"1": date(2027, 1, 1),
|
||||
"2": date(2027, 1, 1),
|
||||
"4": date(2027, 1, 1),
|
||||
}
|
||||
|
||||
|
||||
def rtc_enabled(caps: dict) -> bool:
|
||||
"""RTC is in scope ONLY when the ERP is switched to the reform (PARAMETROS.USAR_REFORMA_TRIBUTARIA truthy) AND its schema can hold it."""
|
||||
return bool(caps.get("HAS_PRODUCT_CCLASSTRIB") and caps.get("HAS_RTC_RELATIONSHIP") and caps.get("RTC_ENABLED_BY_PARAM"))
|
||||
|
||||
|
||||
def cclasstrib_applicability(crt, rtc_enabled: bool, today: date | None = None) -> dict:
|
||||
"""Company-level cClassTrib requiredness from live CRT and its effective date.
|
||||
|
||||
The ERP RTC switch scopes the feature; it does not make cClassTrib mandatory
|
||||
for every company/product. Unknown regimes fail closed to NOT_APPLICABLE.
|
||||
"""
|
||||
today = today or date.today()
|
||||
regime = _text(crt)
|
||||
if not rtc_enabled:
|
||||
return {"state": "NOT_APPLICABLE", "basis": "RTC_DISABLED_IN_ERP", "effective_from": None, "source": "PARAMETROS.USAR_REFORMA_TRIBUTARIA"}
|
||||
effective_from = CCLASSTRIB_EFFECTIVE_FROM.get(regime)
|
||||
if effective_from is None:
|
||||
return {"state": "NOT_APPLICABLE", "basis": "COMPANY_CRT_UNRESOLVED", "effective_from": None, "source": "C000004.CRT"}
|
||||
if today < effective_from:
|
||||
basis = "SIMPLES_CCLASSTRIB_NOT_EFFECTIVE" if regime in SIMPLES_FAMILY else "CCLASSTRIB_NOT_YET_EFFECTIVE"
|
||||
return {"state": "NOT_APPLICABLE", "basis": basis, "effective_from": effective_from.isoformat(), "source": "C000004.CRT + RTC effective-date rules"}
|
||||
return {"state": "REQUIRED", "basis": "CCLASSTRIB_EFFECTIVE_FOR_CRT", "effective_from": effective_from.isoformat(), "source": "C000004.CRT + RTC effective-date rules"}
|
||||
|
||||
|
||||
def catalog_integrity(catalog: dict) -> dict:
|
||||
"""catalog: {"classes": {cClassTrib: cst_or_None}, "cst": {code, ...}, "duplicates": int}."""
|
||||
classes, cst = catalog.get("classes", {}), set(catalog.get("cst", ()))
|
||||
bad_format = [c for c in classes if not re.fullmatch(r"\d{6}", str(c))]
|
||||
bad_cst_format = [c for c in cst if not re.fullmatch(r"\d{3}", str(c))]
|
||||
orphans = [c for c, s in classes.items() if s is None or str(s) not in cst]
|
||||
return {"classes": len(classes), "cst": len(cst), "orphan_classes": len(orphans), "bad_class_format": len(bad_format), "bad_cst_format": len(bad_cst_format),
|
||||
"duplicates": int(catalog.get("duplicates", 0)), "valid": bool(classes and cst and not orphans and not bad_format and not bad_cst_format and not catalog.get("duplicates"))}
|
||||
|
||||
|
||||
def validate_cclasstrib(value, catalog: dict | None, today: date | None = None) -> str:
|
||||
"""EMPTY | INVALID_FORMAT | NOT_IN_CATALOG | EXPIRED | CST_MISSING | VALID (CST IBS/CBS derived through the catalog, never stored on the product)."""
|
||||
code = _text(value)
|
||||
if not code:
|
||||
return "EMPTY"
|
||||
if not re.fullmatch(r"\d{6}", code):
|
||||
return "INVALID_FORMAT"
|
||||
if not catalog or code not in catalog.get("classes", {}):
|
||||
return "NOT_IN_CATALOG"
|
||||
validity = (catalog.get("validity") or {}).get(code)
|
||||
if validity and today is not None:
|
||||
start, end = validity
|
||||
if (start and start > today) or (end and end < today):
|
||||
return "EXPIRED"
|
||||
cst = catalog["classes"][code]
|
||||
return "VALID" if cst is not None and str(cst) in set(catalog.get("cst", ())) else "CST_MISSING"
|
||||
|
||||
|
||||
def evaluate_rtc(product: dict, rtc: dict | None, today: date | None = None, crt=None) -> dict:
|
||||
"""Validate cClassTrib only when RTC is on and the company's regime/date gate applies."""
|
||||
app = rtc.get("cclasstrib_applicability") if isinstance(rtc, dict) else None
|
||||
if not isinstance(app, dict):
|
||||
app = cclasstrib_applicability(crt, bool(rtc and rtc.get("enabled")), today)
|
||||
if app["state"] != "REQUIRED":
|
||||
return {"applicability": app, "blockers": []}
|
||||
if not rtc.get("catalog_valid"):
|
||||
return {"applicability": app, "blockers": [("RTC_CATALOG_INVALID", "LOCAL_CATALOG_FAILED_INTEGRITY")]}
|
||||
state = validate_cclasstrib(product.get("CCLASSTRIB"), rtc.get("catalog"), today)
|
||||
mapping = {"EMPTY": "CCLASSTRIB_MISSING", "INVALID_FORMAT": "CCLASSTRIB_INVALID_FORMAT", "NOT_IN_CATALOG": "CCLASSTRIB_NOT_IN_CATALOG", "EXPIRED": "CCLASSTRIB_NOT_IN_CATALOG", "CST_MISSING": "RTC_CST_MISSING"}
|
||||
return {"applicability": app, "blockers": [(mapping[state], state)] if state in mapping else []}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------- unified
|
||||
|
||||
|
||||
def evaluate_product(product: dict, company: dict | None, rtc: dict | None, today: date | None = None) -> dict:
|
||||
"""The single fiscal-completeness verdict for the fields this module owns.
|
||||
Returns {"blockers": [{"type","category","reason"}], "applicability": {"PIS_COFINS": state, "RTC": state}, "mandatory_fields": [...], "needs_review": bool}."""
|
||||
crt = (company or {}).get("crt")
|
||||
pis = evaluate_pis_cofins(product, crt)
|
||||
rtc_result = evaluate_rtc(product, rtc, today, crt)
|
||||
blockers = [{"type": kind, "category": blocker_category(kind), "reason": reason} for kind, reason in pis["blockers"] + rtc_result["blockers"]]
|
||||
warnings = [{"type": kind, "reason": reason} for kind, reason in pis["warnings"]]
|
||||
mandatory = []
|
||||
if pis["applicability"]["state"] == "REQUIRED":
|
||||
mandatory += ["COD_SAIDA_PIS", "COD_SAIDA_COFINS"]
|
||||
if rtc_result["applicability"]["state"] == "REQUIRED":
|
||||
mandatory.append("CCLASSTRIB")
|
||||
return {"blockers": blockers, "applicability": {"PIS_COFINS": pis["applicability"]["state"], "RTC": rtc_result["applicability"]["state"]},
|
||||
"basis": {"PIS_COFINS": pis["applicability"]["basis"], "RTC": rtc_result["applicability"]["basis"]}, "mandatory_fields": mandatory, "warnings": warnings,
|
||||
"needs_review": pis["applicability"]["state"] == "NEEDS_REVIEW"}
|
||||
|
||||
|
||||
def legal_effective_context(crt) -> dict:
|
||||
"""Summarize the broad IBS/CBS document timeline; cClassTrib requiredness has its own CRT/date gate above.
|
||||
Source: Ato Conjunto RFB/CGIBS n 4/2026 (DOU 31/07/2026) art. 1: NF-e (55) and NFC-e (65) from 2026-08-03; par. 1: Simples optantes from 2027-01-01."""
|
||||
simples = _text(crt) in SIMPLES_FAMILY
|
||||
return {"crt": _text(crt) or None, "informational_only": True,
|
||||
"ibs_cbs_in_documents": "MANDATORY_FROM_2027-01-01" if simples else ("MANDATORY_FROM_2026-08-03" if _text(crt) == "3" else "UNKNOWN_REGIME"),
|
||||
"cclasstrib_orientation": "PENDING_OFFICIAL_NT" if simples else ("TABLE_AND_LC214_ANNEXES" if _text(crt) == "3" else "UNKNOWN_REGIME"),
|
||||
"sources": ["Ato Conjunto RFB/CGIBS n 4/2026 (DOU 31/07/2026) art. 1 I, II and par. 1", "NT 2025.002-RTC v1.51 (Portal NF-e, 04/08/2026): CRT 1/2/4 orientations in a future NT", "LC 214/2025 art. 348"]}
|
||||
@@ -0,0 +1,8 @@
|
||||
"""Single source of truth for the release version. Also what the Agent Windows
|
||||
package is versioned as (scripts/build_agent_package.py reads release/VERSION,
|
||||
kept identical to this) and what an AgentRelease row's `version` column holds
|
||||
once published — never a second, independent version number. Fiscal semantics
|
||||
are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py,
|
||||
DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump."""
|
||||
GTI_FISCAL_VERSION = "1.3.15"
|
||||
API_COMPAT_MAJOR_MINOR = "1.0" # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x
|
||||
@@ -0,0 +1,882 @@
|
||||
"""Server-rendered operator pages for the GTI Fiscal control panel."""
|
||||
|
||||
import secrets
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from urllib.parse import parse_qs, urlencode
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from fastapi.responses import HTMLResponse, RedirectResponse
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from sqlalchemy import func, select, update
|
||||
|
||||
from . import agent_distribution, client_sync, config, enrollment, sso
|
||||
from .config import settings
|
||||
from .blocker_resolution import BlockerResolutionUnavailable, _company_key, resolve_blocker_with_context
|
||||
from .context_keys import ALLOWED_KEYS, ENUMS
|
||||
from .installation_identity import reference_agent_version
|
||||
from .panel_presentation import ENRICHMENT_MODES, ENRICHMENT_MODE_LABELS, RTC_MODE_EXPLANATION, RTC_MODE_LABELS, abort_reason_label, audit_outcome_label, candidate_status_label, catalog_state_label, job_kind_label, job_status_label, rtc_rule_label, rtc_status_view
|
||||
from .control_plane import blocker_detail, blocker_filter_summary, dashboard_summary, installation_view, jobs_status_summary, list_blockers, list_installations, list_jobs, list_panel_events, rtc_blocker_summary
|
||||
from .human_auth import create_session, current_session, require_csrf, verify_password
|
||||
from .panel_presentation import format_ui_datetime
|
||||
from .models import AdminUser, Agent, AgentEnrollment, AgentRelease, AgentSnapshot, BlockedProduct, FiscalContextEntry, Installation, Job, ProductEnrichment, ProductFieldAudit, RtcCandidate, RtcWriteAudit, Tenant
|
||||
from .jobs import create_job
|
||||
from .audit import record_audit
|
||||
from .enrichment import kb as kb_mod, status as enrichment_status
|
||||
from .fiscal_context_ui import build_fiscal_context_view, set_fiscal_context_decision
|
||||
from .enrichment.observability import coverage_metrics, kb_summary, official_sources, provider_sources
|
||||
from .reference_registry import RegistryError, approve_draft, diff_against_vigente, list_registry, reject_draft, technical_checklist
|
||||
|
||||
|
||||
def router(session_factory):
|
||||
routes = APIRouter()
|
||||
templates = Jinja2Templates(directory="central/templates")
|
||||
|
||||
def auth(request, session):
|
||||
return current_session(request, session)
|
||||
|
||||
def render(request, template, context=None, status_code=200):
|
||||
values = {"request": request, "csrf_token": request.cookies.get("gti_csrf", ""), "panel_home": sso.panel_home(), **(context or {})}
|
||||
return templates.TemplateResponse(request=request, name=template, context=values, status_code=status_code)
|
||||
|
||||
def client_ip(request):
|
||||
return request.client.host if request.client else None
|
||||
|
||||
def _aware(value: datetime | None) -> datetime | None:
|
||||
"""SQLite (used by the test suite) drops tzinfo on round-trip even for
|
||||
a DateTime(timezone=True) column; Postgres never does. Normalize to
|
||||
UTC before comparing against datetime.now(timezone.utc)."""
|
||||
if value is None or value.tzinfo is not None:
|
||||
return value
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
|
||||
def set_session_cookies(response, token, csrf, seconds):
|
||||
cfg = config.settings
|
||||
response.set_cookie(cfg.web_cookie_name, token, max_age=seconds, httponly=True, secure=cfg.web_cookie_secure, samesite="lax")
|
||||
response.set_cookie("gti_csrf", csrf, max_age=seconds, httponly=False, secure=cfg.web_cookie_secure, samesite="lax")
|
||||
|
||||
def query_base(**values):
|
||||
return urlencode({key: value for key, value in values.items() if value not in (None, "")})
|
||||
|
||||
def date_filter(value, *, end_of_day=False):
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value)
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.replace(hour=23, minute=59, second=59, microsecond=999999) if end_of_day and len(value) == 10 else parsed
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
@routes.get("/login", response_class=HTMLResponse)
|
||||
def login_page(request: Request, next: str | None = None):
|
||||
"""Collaborators never see a Hub login: with SSO configured this starts
|
||||
the flow at GTI Gerenciador (which shows its own login if needed)."""
|
||||
if not sso.enabled():
|
||||
return render(request, "login.html", {"error": None, "emergency": False})
|
||||
cfg = config.settings
|
||||
state = secrets.token_urlsafe(24)
|
||||
response = RedirectResponse(f"{cfg.sso_start_url}?{urlencode({'state': state, 'next': sso.safe_next(next)})}", status_code=303)
|
||||
# Sent back on the cross-site POST from the panel, hence SameSite=None (+Secure).
|
||||
response.set_cookie(sso.STATE_COOKIE, state, max_age=sso.STATE_SECONDS, path=sso.STATE_COOKIE_PATH, httponly=True,
|
||||
secure=cfg.web_cookie_secure, samesite="none" if cfg.web_cookie_secure else "lax")
|
||||
return response
|
||||
|
||||
@routes.get("/login/local", response_class=HTMLResponse)
|
||||
def emergency_login_page(request: Request):
|
||||
"""Break-glass only (local AdminUser with a password). Not linked from the UI."""
|
||||
return render(request, "login.html", {"error": None, "emergency": True})
|
||||
|
||||
@routes.post("/login")
|
||||
async def login(request: Request):
|
||||
values = await form_values(request)
|
||||
username = values.get("username", "").strip().lower()
|
||||
with session_factory() as session:
|
||||
admin = session.scalar(select(AdminUser).where(AdminUser.username == username))
|
||||
if not admin or not admin.is_active or admin.auth_source != "LOCAL" or not verify_password(values.get("password", ""), admin.password_hash):
|
||||
record_audit(session, admin_user_id=None, action="LOGIN_LOCAL_FAILURE", target_type="admin_user", target_id=username[:128] or "-", context={"ip": client_ip(request)}, severity="WARNING")
|
||||
session.commit()
|
||||
return render(request, "login.html", {"error": "Credenciais inválidas", "emergency": True}, 401)
|
||||
token, csrf = create_session(session, admin, request)
|
||||
record_audit(session, admin_user_id=admin.id, action="LOGIN_LOCAL_SUCCESS", target_type="admin_user", target_id=admin.username, context={"ip": client_ip(request)}, severity="WARNING")
|
||||
session.commit()
|
||||
response = RedirectResponse("/", status_code=303)
|
||||
set_session_cookies(response, token, csrf, config.settings.web_session_seconds)
|
||||
return response
|
||||
|
||||
@routes.post("/auth/sso/callback")
|
||||
async def sso_callback(request: Request):
|
||||
values = await form_values(request)
|
||||
with session_factory() as session:
|
||||
try:
|
||||
if not sso.enabled():
|
||||
raise sso.SsoRejected("SSO_DISABLED")
|
||||
identity = sso.verify_assertion(values.get("assertion"), request.cookies.get(sso.STATE_COOKIE))
|
||||
sso.consume_jti(session, identity)
|
||||
user = sso.provision(session, identity)
|
||||
seconds = config.settings.federated_session_seconds
|
||||
token, csrf = create_session(session, user, request, seconds=seconds)
|
||||
record_audit(session, admin_user_id=user.id, action="SSO_LOGIN_SUCCESS", target_type="admin_user", target_id=identity.subject,
|
||||
context={"ip": client_ip(request), "login": identity.login, "role": identity.role})
|
||||
session.commit()
|
||||
except sso.SsoRejected as rejected:
|
||||
session.rollback()
|
||||
record_audit(session, admin_user_id=None, action="SSO_LOGIN_REJECTED", target_type="sso", target_id=rejected.subject or "-",
|
||||
context={"ip": client_ip(request), "reason": rejected.reason}, severity="WARNING")
|
||||
session.commit()
|
||||
response = render(request, "sso_error.html", {"reason": rejected.reason}, 401)
|
||||
response.delete_cookie(sso.STATE_COOKIE, path=sso.STATE_COOKIE_PATH)
|
||||
return response
|
||||
response = RedirectResponse(sso.safe_next(values.get("next")), status_code=303)
|
||||
set_session_cookies(response, token, csrf, seconds)
|
||||
response.delete_cookie(sso.STATE_COOKIE, path=sso.STATE_COOKIE_PATH)
|
||||
return response
|
||||
|
||||
@routes.post("/logout")
|
||||
async def logout(request: Request):
|
||||
"""Ends only the Hub session. Coming back through GTI Gerenciador signs in
|
||||
again without a password (documented in docs/PILOT_DEPLOYMENT.md)."""
|
||||
with session_factory() as session:
|
||||
web_session, admin = current_session(request, session)
|
||||
if web_session:
|
||||
values = await form_values(request)
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
record_audit(session, admin_user_id=admin.id, action="LOGOUT", target_type="admin_user", target_id=admin.external_subject or admin.username, context={"ip": client_ip(request)})
|
||||
session.delete(web_session); session.commit()
|
||||
response = render(request, "logged_out.html", {}) if sso.enabled() else RedirectResponse("/login", status_code=303)
|
||||
response.delete_cookie(config.settings.web_cookie_name); response.delete_cookie("gti_csrf")
|
||||
return response
|
||||
|
||||
def page_auth(request):
|
||||
session = session_factory()
|
||||
web_session, admin = current_session(request, session)
|
||||
if not admin:
|
||||
session.close()
|
||||
wanted = request.url.path + (f"?{request.url.query}" if request.url.query else "")
|
||||
return None, None, RedirectResponse(f"/login?{urlencode({'next': wanted})}", status_code=303)
|
||||
return session, (web_session, admin), None
|
||||
|
||||
async def form_values(request):
|
||||
return {key: items[0] for key, items in parse_qs((await request.body()).decode()).items() if items}
|
||||
|
||||
@routes.get("/", response_class=HTMLResponse)
|
||||
def dashboard(request: Request, kb_q: str | None = None, kb_page: int = 1):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try: return render(request, "dashboard.html", {"summary": dashboard_summary(session), "auth": auth_data, "kb_q": kb_q or "", "is_admin": _is_privileged(auth_data[1]), "csrf_token": request.cookies.get("gti_csrf", ""), "sources": provider_sources(session), "official": official_sources(session), "enrich_metrics": coverage_metrics(session), "kb": kb_summary(session, kb_q, kb_page)})
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/admin/reference-datasets", response_class=HTMLResponse)
|
||||
def reference_datasets_page(request: Request):
|
||||
"""Missão 5 / Fase 1.2: 'Tabelas oficiais' - lista o Reference Data Registry (DRAFT/VIGENTE/
|
||||
OBSOLETO/REJECTED), com diff contra o VIGENTE atual para cada DRAFT. ADMIN only."""
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
if not _is_privileged(auth_data[1]):
|
||||
return HTMLResponse("Acesso restrito a administradores.", status_code=403)
|
||||
rows = list_registry(session)
|
||||
view = []
|
||||
for r in rows:
|
||||
item = {"id": r.id, "kind": r.kind, "provider": r.provider, "version_label": r.version_label, "status": r.status,
|
||||
"record_count": r.record_count, "downloaded_at": format_ui_datetime(r.downloaded_at), "source_url": r.source_url,
|
||||
"licence_terms": r.licence_terms, "parse_errors": r.parse_errors, "submitted_by": r.submitted_by,
|
||||
"approved_by": r.approved_by, "diff": None, "checklist": technical_checklist(session, r)}
|
||||
if r.status == "DRAFT":
|
||||
item["diff"] = diff_against_vigente(session, r)
|
||||
view.append(item)
|
||||
return render(request, "reference_datasets.html", {"auth": auth_data, "csrf_token": request.cookies.get("gti_csrf", ""), "is_admin": True, "rows": view})
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/admin/reference-datasets/{dataset_id}/decide")
|
||||
async def decide_reference_dataset(request: Request, dataset_id: int):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
if not _is_privileged(admin):
|
||||
return _deny(session, admin, None, "reference_dataset:decide", str(dataset_id))
|
||||
action = (values.get("action") or "").strip()
|
||||
try:
|
||||
if action == "approve":
|
||||
dataset = approve_draft(session, dataset_id, admin_name=admin.display_name or admin.username)
|
||||
elif action == "reject":
|
||||
dataset = reject_draft(session, dataset_id, admin_name=admin.display_name or admin.username)
|
||||
else:
|
||||
return HTMLResponse("ação inválida", status_code=422)
|
||||
except RegistryError as error:
|
||||
return HTMLResponse(str(error), status_code=422)
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=None, action=f"REFERENCE_DATASET_{action.upper()}", target_type="dataset", target_id=str(dataset_id),
|
||||
context={"kind": dataset.kind, "version_label": dataset.version_label, "operator": admin.display_name or admin.username}, severity="WARNING")
|
||||
session.commit()
|
||||
return RedirectResponse("/admin/reference-datasets", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/agents", response_class=HTMLResponse)
|
||||
def agents(request: Request, page: int = 1, search: str | None = None, health: str | None = None, sort: str = "display_name", scope: str = "active", version: str | None = None):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
scope = scope if scope in ("active", "all") else "active"
|
||||
health = health if health in ("ONLINE", "DEGRADED", "OFFLINE", "FAILED", "UNKNOWN") else None
|
||||
sort = sort if sort in ("display_name", "worst_coverage") else "display_name"
|
||||
version = version if version == "outdated" else None
|
||||
return render(request, "agents.html", {"result": list_installations(session, page=page, page_size=50, search=search, health=health, sort=sort, scope=scope, version=version), "auth": auth_data, "search": search or "", "health_filter": health or "", "sort": sort, "scope_filter": scope, "version_filter": version or "", "query_base": query_base(search=search, health=health, sort=sort, scope=scope, version=version)})
|
||||
finally: session.close()
|
||||
|
||||
# --- Static /agents/* routes MUST be declared before /agents/{agent_id}:
|
||||
# Starlette matches in registration order, so a dynamic route declared
|
||||
# first would capture "/agents/new" and "/agents/releases" and fail the
|
||||
# int conversion with a 422 (the production bug this ordering prevents).
|
||||
@routes.get("/agents/releases", response_class=HTMLResponse)
|
||||
def agent_releases(request: Request):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
releases = session.scalars(select(AgentRelease).order_by(AgentRelease.created_at.desc())).all()
|
||||
counts = {}
|
||||
for release_id, in session.execute(select(AgentEnrollment.release_id)).all():
|
||||
counts[release_id] = counts.get(release_id, 0) + 1
|
||||
return render(request, "agent_releases.html", {"releases": releases, "installation_counts": counts, "auth": auth_data})
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/releases/{release_id}/revoke")
|
||||
async def revoke_release(request: Request, release_id: int):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
release = session.get(AgentRelease, release_id)
|
||||
if not release: return HTMLResponse("Release not found", status_code=404)
|
||||
release.status = "REVOKED"
|
||||
record_audit(session, admin_user_id=admin.id, action="AGENT_RELEASE_REVOKED", target_type="agent_release", target_id=release.version, context={"platform": release.platform, "architecture": release.architecture})
|
||||
session.commit()
|
||||
return RedirectResponse("/agents/releases", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
def _active_gerenciador_tenants(session, search: str | None = None):
|
||||
# GTI Gerenciador is the single source of truth for clients (central/app/
|
||||
# client_sync.py) — a new installation never offers a LOCAL (dev/test) or
|
||||
# inactive tenant, so it can never pick a client that doesn't exist there.
|
||||
query = select(Tenant).where(Tenant.source == client_sync.SOURCE, Tenant.active.is_(True))
|
||||
if search: query = query.where(Tenant.name.ilike(f"%{search}%"))
|
||||
return session.scalars(query.order_by(Tenant.name)).all()
|
||||
|
||||
def _clients_json(tenants):
|
||||
import json
|
||||
# "<" escaped so the payload can never close the <script> tag it is embedded in.
|
||||
return json.dumps([{"id": t.external_client_id, "name": t.name, "code": t.external_code or "", "cnpj": t.external_cnpj or ""} for t in tenants], ensure_ascii=False).replace("<", "\u003c")
|
||||
|
||||
@routes.get("/agents/new", response_class=HTMLResponse)
|
||||
def new_installation_form(request: Request, search: str | None = None):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
tenants = _active_gerenciador_tenants(session, search)
|
||||
releases = session.scalars(select(AgentRelease).where(AgentRelease.status == "PUBLISHED").order_by(AgentRelease.created_at.desc())).all()
|
||||
return render(request, "agent_new_installation.html", {"tenants": tenants, "clients_json": _clients_json(tenants), "releases": releases, "auth": auth_data, "error": None, "search": search or ""})
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/new")
|
||||
async def create_pending_installation(request: Request):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
client_id = values.get("client_id") # Gerenciador's real CLIENTE.ID, never the text shown in the combobox
|
||||
display_name = (values.get("display_name") or "").strip()[:160]
|
||||
expected_hostname = (values.get("expected_hostname") or "").strip()[:255] or None
|
||||
release_id = values.get("release_id")
|
||||
errors = []
|
||||
tenant = session.scalar(select(Tenant).where(Tenant.source == client_sync.SOURCE, Tenant.external_client_id == int(client_id))) if client_id and client_id.isdigit() else None
|
||||
release = session.get(AgentRelease, int(release_id)) if release_id and release_id.isdigit() else None
|
||||
# Server-side, not just a hidden option in the form: a LOCAL or
|
||||
# inactive tenant can never receive a new installation, matching
|
||||
# GTI Gerenciador's own active/inactive rule.
|
||||
if tenant and not tenant.active: tenant = None
|
||||
if not tenant: errors.append("Selecione um cliente da lista.")
|
||||
if not display_name: errors.append("Informe um nome para a instalação.")
|
||||
if not release or release.status != "PUBLISHED": errors.append("Selecione uma versão publicada do Agent.")
|
||||
if errors:
|
||||
tenants = _active_gerenciador_tenants(session)
|
||||
releases = session.scalars(select(AgentRelease).where(AgentRelease.status == "PUBLISHED").order_by(AgentRelease.created_at.desc())).all()
|
||||
return render(request, "agent_new_installation.html", {"tenants": tenants, "clients_json": _clients_json(tenants), "releases": releases, "auth": auth_data, "error": " ".join(errors), "search": ""}, 400)
|
||||
enrollment_uuid = str(uuid.uuid4())
|
||||
enr = AgentEnrollment(enrollment_uuid=enrollment_uuid, tenant_id=tenant.id, display_name=display_name, expected_hostname=expected_hostname, release_id=release.id, status="PENDING", created_by=admin.id)
|
||||
if values.get("features_selected") == "1": # chosen explicitly at creation; otherwise the installation waits for an admin (WAITING_FEATURE_CONFIGURATION)
|
||||
mode = (values.get("feature_enrichment_mode") or "DISABLED").strip().upper()
|
||||
enr.features_selected, enr.feature_corrector = True, values.get("feature_corrector") == "1"
|
||||
enr.feature_enrichment_mode = mode if mode in ENRICHMENT_MODES else "DISABLED"
|
||||
session.add(enr)
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=tenant.id, action="INSTALLATION_CREATED", target_type="agent_enrollment", target_id=enrollment_uuid, context={"display_name": display_name, "release_version": release.version, "features_selected": bool(enr.features_selected), "corrector": bool(enr.feature_corrector), "enrichment_mode": enr.feature_enrichment_mode})
|
||||
session.commit()
|
||||
return RedirectResponse(f"/agents/pending/{enrollment_uuid}", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/agents/{agent_id}", response_class=HTMLResponse)
|
||||
def agent_detail(request: Request, agent_id: str, tab: str = "overview", event_page: int = 1, severity: str | None = None, event_type: str | None = None, job_id: int | None = None, product_code: str | None = None, since: str | None = None, until: str | None = None, enrich_code: str | None = None):
|
||||
# agent_id is parsed here (not typed int in the path) so a non-numeric
|
||||
# slug like /agents/abc gets a controlled HTML 404 instead of a raw
|
||||
# 422 — and can never shadow the static routes above.
|
||||
try:
|
||||
agent_id = int(agent_id)
|
||||
except ValueError:
|
||||
return HTMLResponse("Agent não encontrado (identificador inválido)", status_code=404)
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
row = session.execute(select(Installation, Agent, Tenant, AgentSnapshot).join(Agent, Agent.id == Installation.agent_id).join(Tenant, Tenant.id == Installation.tenant_id).outerjoin(AgentSnapshot, AgentSnapshot.installation_id == Installation.id).where(Agent.id == agent_id)).first()
|
||||
if row is None: return HTMLResponse("Agent not found", status_code=404)
|
||||
installation, agent, tenant, snapshot = row
|
||||
detail = installation_view(installation, agent, tenant, snapshot, reference=reference_agent_version(session))
|
||||
event_page = max(1, event_page)
|
||||
events = list_panel_events(session, installation.id, page=event_page, page_size=25, include_more=True, severity=severity, event_type=event_type, job_id=job_id, product_code=product_code, since=date_filter(since), until=date_filter(until, end_of_day=True))
|
||||
has_more = len(events) > 25
|
||||
events = events[:25]
|
||||
for event in events:
|
||||
event["occurred_display"] = format_ui_datetime(datetime.fromisoformat(event["occurred_at"]) if event["occurred_at"] else None)
|
||||
agent_blockers = list_blockers(session, tenant_id=installation.tenant_id, agent_id=agent_id, page=1, page_size=25, scope="all")
|
||||
fiscal_context = build_fiscal_context_view(session, installation.tenant_id, _company_key(None, installation, None))
|
||||
return render(request, "agent_detail.html", {"detail": detail, "tab": tab, "blockers": agent_blockers, "blocker_breakdown": blocker_filter_summary(session, tenant_id=installation.tenant_id, agent_id=agent_id, status="OPEN", scope="all")["blocker_breakdown"], "jobs": list_jobs(session, agent_id=agent_id, page=1, page_size=25, scope="all"), "events": events, "event_page": max(1, event_page), "has_more": has_more, "severity": severity or "", "event_type": event_type or "", "job_id": job_id or "", "product_code": product_code or "", "since": since or "", "until": until or "", "event_query_base": query_base(tab=tab, severity=severity, event_type=event_type, job_id=job_id, product_code=product_code, since=since, until=until), "auth": auth_data, "csrf_token": request.cookies.get("gti_csrf", ""), "ops": _ops_view(session, installation, agent), "features": _features_view(session, installation, enrich_code), "enrich_code": enrich_code or "", "is_admin": _is_privileged(auth_data[1]), "fiscal_context": fiscal_context, "cfop_ippt_write_mode": installation.cfop_ippt_write_mode})
|
||||
finally: session.close()
|
||||
|
||||
def _features_view(session, installation, enrich_code=None):
|
||||
"""FUNCIONALIDADES + small ENRIQUECIMENTO summary of the Agent detail (no separate dashboard)."""
|
||||
counts = dict(session.execute(select(ProductEnrichment.status, func.count()).where(ProductEnrichment.installation_id == installation.id).group_by(ProductEnrichment.status)).all())
|
||||
applied = session.scalar(select(func.count()).select_from(ProductFieldAudit).where(ProductFieldAudit.installation_id == installation.id, ProductFieldAudit.outcome == "APPLIED")) or 0
|
||||
lookup = None
|
||||
if enrich_code:
|
||||
row = session.scalar(select(ProductEnrichment).where(ProductEnrichment.installation_id == installation.id, ProductEnrichment.product_code == enrich_code.strip()[:64]))
|
||||
audits = session.scalars(select(ProductFieldAudit).where(ProductFieldAudit.installation_id == installation.id, ProductFieldAudit.product_code == enrich_code.strip()[:64]).order_by(ProductFieldAudit.id.desc()).limit(20)).all() if row else []
|
||||
lookup = {"found": row is not None, "row": row, "audits": audits}
|
||||
tracked = []
|
||||
for r in session.scalars(select(ProductEnrichment).where(ProductEnrichment.installation_id == installation.id).order_by(ProductEnrichment.id.desc()).limit(15)):
|
||||
fields = ((r.decisions or {}).get("fields") or {})
|
||||
ncm = fields.get("CLASSIFICACAO_FISCAL") or {}
|
||||
tracked.append({"code": r.product_code, "description": r.description, "gtin": r.gtin, "ui_status": r.ui_status or "AGUARDANDO_EVIDENCIA", "label": enrichment_status.LABELS.get(r.ui_status or "AGUARDANDO_EVIDENCIA", ""),
|
||||
"reason": r.reason_code, "detected": r.detected_at, "last_attempt": r.last_attempt_at, "next_retry": r.next_attempt_at, "kb_status": ncm.get("kb_status"), "ncm_candidate": ncm.get("value"),
|
||||
"cest_candidate": (fields.get("CEST") or {}).get("value"), "ipi_candidate": (fields.get("IPI") or {}).get("value"), "applied": r.fields_applied or {}, "placeholder": ncm.get("technical_placeholder"),
|
||||
"reevaluate": bool(r.reevaluate), "mode": installation.enrichment_mode})
|
||||
mode = installation.enrichment_mode
|
||||
waiting = not installation.features_configured
|
||||
return {"configured": bool(installation.features_configured), "waiting": waiting, "corrector_enabled": bool(installation.corrector_enabled), "enrichment_mode": mode,
|
||||
"mode_label": ENRICHMENT_MODE_LABELS.get(mode, mode), "counts": counts, "total": sum(counts.values()), "applied": applied, "lookup": lookup, "tracked": tracked,
|
||||
"state_label": "Aguardando configuração de funcionalidades" if waiting else ENRICHMENT_MODE_LABELS.get(mode, mode)}
|
||||
|
||||
@routes.post("/agents/{agent_id}/features")
|
||||
async def set_features(request: Request, agent_id: int):
|
||||
"""Per-installation functions (ADMIN only, CSRF, audited). Central decision: the Agent has no local switch."""
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
agent, installation = _rtc_target(session, agent_id)
|
||||
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
|
||||
if not _is_privileged(admin):
|
||||
return _deny(session, admin, installation.tenant_id, "features:set", installation.installation_id)
|
||||
mode = (values.get("enrichment_mode") or "").strip().upper()
|
||||
if mode not in ENRICHMENT_MODES:
|
||||
return HTMLResponse("modo inválido", status_code=422)
|
||||
if mode == "ENABLED" and (values.get("confirm") or "").strip().upper() != "PREENCHER":
|
||||
return HTMLResponse("Digite PREENCHER para confirmar o enriquecimento automático", status_code=422)
|
||||
before = {"corrector": bool(installation.corrector_enabled), "enrichment_mode": installation.enrichment_mode, "configured": bool(installation.features_configured)}
|
||||
installation.corrector_enabled = values.get("corrector_enabled") == "1"
|
||||
installation.enrichment_mode = mode
|
||||
installation.features_configured = True
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="INSTALLATION_FEATURES", target_type="installation", target_id=installation.installation_id,
|
||||
context={"from": before, "to": {"corrector": installation.corrector_enabled, "enrichment_mode": mode}, "operator": admin.display_name or admin.username, "role": admin.external_role or admin.auth_source}, severity="WARNING")
|
||||
session.commit()
|
||||
return RedirectResponse(f"/agents/{agent_id}", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/{agent_id}/fiscal-context")
|
||||
async def set_fiscal_context(request: Request, agent_id: int):
|
||||
"""Contexto Fiscal (ADMIN only, CSRF, audited): confirm/alter/clear one of the handful of real fiscal decisions field_contract
|
||||
cannot derive from the ERP alone. Writes through FiscalContextEntry - the same table blocker_resolution already writes to."""
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
agent, installation = _rtc_target(session, agent_id)
|
||||
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
|
||||
if not _is_privileged(admin):
|
||||
return _deny(session, admin, installation.tenant_id, "fiscal_context:set", installation.installation_id)
|
||||
key = (values.get("key") or "").strip()
|
||||
action = (values.get("fc_action") or "confirm").strip()
|
||||
value = None if action == "clear" else (values.get("value") or "").strip() or None
|
||||
reason = (values.get("reason") or "").strip() or f"{action} via painel"
|
||||
company_key = _company_key(None, installation, None)
|
||||
try:
|
||||
set_fiscal_context_decision(session, tenant_id=installation.tenant_id, company_key=company_key, key=key, value=value, reason=reason, admin=admin, installation_id=installation.id)
|
||||
except ValueError as error:
|
||||
return HTMLResponse(str(error), status_code=422)
|
||||
session.commit()
|
||||
return RedirectResponse(f"/agents/{agent_id}#contexto-fiscal", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/{agent_id}/cfop-ippt-write")
|
||||
async def set_cfop_ippt_write_mode(request: Request, agent_id: int):
|
||||
"""Missão 4 / Bloco E.1: per-installation toggle for the CFOP_IPPT_WRITE_V1 capability
|
||||
(ADMIN only, CSRF, audited). DISABLED by default; the Agent has no local switch."""
|
||||
from .cfop_ippt_audit import WRITE_MODES as CFOP_IPPT_MODES
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
agent, installation = _rtc_target(session, agent_id)
|
||||
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
|
||||
if not _is_privileged(admin):
|
||||
return _deny(session, admin, installation.tenant_id, "cfop_ippt_write:set", installation.installation_id)
|
||||
mode = (values.get("mode") or "").strip().upper()
|
||||
if mode not in CFOP_IPPT_MODES:
|
||||
return HTMLResponse(f"modo inválido (use um de {CFOP_IPPT_MODES})", status_code=422)
|
||||
if mode != "DISABLED" and (values.get("confirm") or "").strip().upper() != "GRAVAR":
|
||||
return HTMLResponse("Digite GRAVAR para confirmar um modo que pode escrever no ERP", status_code=422)
|
||||
before = installation.cfop_ippt_write_mode
|
||||
installation.cfop_ippt_write_mode = mode
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="CFOP_IPPT_WRITE_MODE", target_type="installation", target_id=installation.installation_id,
|
||||
context={"from": before, "to": mode, "operator": admin.display_name or admin.username, "role": admin.external_role or admin.auth_source}, severity="WARNING")
|
||||
session.commit()
|
||||
return RedirectResponse(f"/agents/{agent_id}#contexto-fiscal", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/{agent_id}/enrichment/reevaluate")
|
||||
async def reevaluate_enrichment(request: Request, agent_id: int):
|
||||
"""Explicit, idempotent 'evaluate again' for one product (code=...) or every tracked product that is not enriched yet. KB/cache first; providers only when really needed."""
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
agent, installation = _rtc_target(session, agent_id)
|
||||
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
|
||||
if not installation.features_configured or installation.enrichment_mode == "DISABLED":
|
||||
return HTMLResponse("O enriquecimento está desativado nesta instalação (Funcionalidades).", status_code=409)
|
||||
code = (values.get("code") or "").strip()[:64]
|
||||
n = enrichment_status.mark_for_reevaluation(session, installation.id, codes=[code] if code else None)
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="ENRICHMENT_REEVALUATE", target_type="installation", target_id=installation.installation_id,
|
||||
context={"product": code or "ALL_UNRESOLVED", "flagged": n, "operator": admin.display_name or admin.username})
|
||||
session.commit()
|
||||
return RedirectResponse(f"/agents/{agent_id}#enriquecimento", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/kb/attest")
|
||||
async def kb_attest(request: Request):
|
||||
"""A human fiscal decision (ADMIN only): choose which candidate NCM is right for a GTIN. Only an existing, vigente candidate can be chosen; the others are recorded as rejected."""
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
if not _is_privileged(admin):
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=None, action="KB_ATTEST_DENIED", target_type="kb_product", target_id=values.get("gtin") or "", context={"role": admin.external_role or admin.auth_source}, severity="WARNING")
|
||||
session.commit()
|
||||
return HTMLResponse("Ação restrita a administradores do GTI Gerenciador (perfil ADMIN).", status_code=403)
|
||||
gtin, ncm = (values.get("gtin") or "").strip(), (values.get("ncm") or "").strip()
|
||||
from .enrichment import history
|
||||
current = history.official_sets(session)
|
||||
try:
|
||||
result = kb_mod.attest(session, gtin, ncm, admin.display_name or admin.username, lambda v: v in current, note=(values.get("note") or "")[:200])
|
||||
except ValueError as error:
|
||||
return HTMLResponse(f"Decisão recusada: {error}", status_code=422)
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=None, action="KB_NCM_ATTESTED", target_type="kb_product", target_id=gtin,
|
||||
context={"ncm": ncm, "status": result["status"], "families": result["families"], "operator": admin.display_name or admin.username, "role": admin.external_role or admin.auth_source}, severity="WARNING")
|
||||
for inst in session.scalars(select(Installation).where(Installation.enrichment_mode != "DISABLED", Installation.features_configured.is_(True))):
|
||||
enrichment_status.mark_for_reevaluation(session, inst.id, gtin=gtin)
|
||||
enrichment_status.mark_for_reevaluation(session, inst.id) # products identified by name are found through the KB aliases
|
||||
session.commit()
|
||||
return RedirectResponse("/#kb-resumo", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
def _ops_view(session, installation, agent):
|
||||
"""Processing status shown on the Agent detail (no clutter: 5 facts)."""
|
||||
last_job = session.scalar(select(Job).where(Job.agent_id == agent.id).order_by(Job.id.desc()).limit(1))
|
||||
last_sync = _aware(installation.last_sync_at)
|
||||
next_sync = None
|
||||
if installation.processing_enabled and last_sync is not None:
|
||||
next_sync = last_sync + timedelta(seconds=installation.sync_interval_seconds)
|
||||
last_summary = session.scalar(select(Job.result_summary).where(Job.agent_id == agent.id, Job.status == "COMPLETED", Job.kind.in_(("INITIAL_SYNC", "SYNC_INCREMENTAL", "REPROCESS_ALL"))).order_by(Job.id.desc()).limit(1))
|
||||
return {"rtc": rtc_status_view(last_summary, installation.rtc_write_mode), "rtc_mode": installation.rtc_write_mode, "enabled": bool(installation.processing_enabled), "interval_minutes": installation.sync_interval_seconds // 60,
|
||||
"last_sync": format_ui_datetime(last_sync) if last_sync else None, "next_sync": format_ui_datetime(next_sync) if next_sync else None,
|
||||
"last_changed": installation.last_sync_changed, "last_job": {"kind": last_job.kind, "status": last_job.status, "id": last_job.id, "kind_label": job_kind_label(last_job.kind), "status_label": job_status_label(last_job.status)} if last_job else None}
|
||||
|
||||
async def _installation_action(request, agent_id, action):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
agent = session.get(Agent, agent_id)
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent_id)) if agent else None
|
||||
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
|
||||
if action == "pause": installation.processing_enabled = False
|
||||
elif action == "resume": installation.processing_enabled = True
|
||||
elif action == "sync-now":
|
||||
job = create_job(session, agent, "SYNC_INCREMENTAL", {"reason": "MANUAL_SYNC_NOW", "requested_by": admin.id})
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="SYNC_NOW", target_type="installation", target_id=installation.installation_id, context={"job_id": job.id, "kind": job.kind})
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action=f"PROCESSING_{action.upper().replace('-', '_')}", target_type="installation", target_id=installation.installation_id, context={"processing_enabled": installation.processing_enabled}) if action in ("pause", "resume") else None
|
||||
session.commit()
|
||||
return RedirectResponse(f"/agents/{agent_id}?tab=jobs", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
def _is_privileged(admin) -> bool:
|
||||
"""Who may change what the Agent is allowed to WRITE. The identity comes from the Gerenciador SSO (`perfil`): only ADMIN. LOCAL break-glass accounts are administrators by construction.
|
||||
A normal (PADRAO) collaborator can look at the control page and use DISABLED/DRY_RUN (read-only) but cannot approve, enable, apply or roll back."""
|
||||
return admin.auth_source == "LOCAL" or (admin.external_role or "").upper() == "ADMIN"
|
||||
|
||||
def _deny(session, admin, tenant_id, action, target):
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=tenant_id, action="RTC_DENIED", target_type="installation", target_id=target, context={"attempted": action, "role": admin.external_role or admin.auth_source}, severity="WARNING")
|
||||
session.commit()
|
||||
return HTMLResponse("Ação restrita a administradores do GTI Gerenciador (perfil ADMIN).", status_code=403)
|
||||
|
||||
def _rtc_target(session, agent_id):
|
||||
agent = session.get(Agent, agent_id)
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent_id)) if agent else None
|
||||
return agent, installation
|
||||
|
||||
@routes.get("/agents/{agent_id}/rtc")
|
||||
async def rtc_control_page(request: Request, agent_id: int):
|
||||
"""Central cClassTrib control: write mode, candidate preview (server-side pagination), approvals and audit/rollback."""
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
agent, installation = _rtc_target(session, agent_id)
|
||||
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
|
||||
q = request.query_params
|
||||
status = q.get("status") or "PENDING"
|
||||
page = max(1, int(q.get("page", "1") or 1)) if str(q.get("page", "1")).isdigit() else 1
|
||||
size = 50
|
||||
base = select(RtcCandidate).where(RtcCandidate.installation_id == installation.id, RtcCandidate.rule_id == "RTC-R3")
|
||||
counts = dict(session.execute(select(RtcCandidate.status, func.count()).where(RtcCandidate.installation_id == installation.id, RtcCandidate.rule_id == "RTC-R3").group_by(RtcCandidate.status)).all())
|
||||
shown = base.where(RtcCandidate.status == status) if status in ("PENDING", "APPROVED", "REJECTED", "APPLIED", "STALE") else base
|
||||
total = int(session.scalar(select(func.count()).select_from(shown.subquery())) or 0)
|
||||
rows = list(session.scalars(shown.order_by(RtcCandidate.product_code).offset((page - 1) * size).limit(size)))
|
||||
audits = list(session.scalars(select(RtcWriteAudit).where(RtcWriteAudit.installation_id == installation.id, RtcWriteAudit.outcome.in_(("APPLIED", "ROLLED_BACK"))).order_by(RtcWriteAudit.id.desc()).limit(25)))
|
||||
ops = _ops_view(session, installation, agent)
|
||||
rtc_blockers = rtc_blocker_summary(session, agent_id)
|
||||
rtc_products = ((ops.get("rtc") or {}).get("products") or {})
|
||||
r3_candidates = int(rtc_products.get("ready", counts.get("PENDING", 0) + counts.get("APPROVED", 0)))
|
||||
blocked_products = max(int(rtc_products.get("blocked", 0)), rtc_blockers["products_affected"])
|
||||
reasons = rtc_blockers["reason_counts"]
|
||||
simple_reason_count = sum(count for reason, count in reasons.items() if reason.endswith("SIMPLES_ORIENTATION_PENDING_OFFICIAL_NT"))
|
||||
simple_reason_count = max(simple_reason_count, int(rtc_products.get("simple_orientation_count", 0)))
|
||||
other_reason_max = max((count for reason, count in reasons.items() if not reason.endswith("SIMPLES_ORIENTATION_PENDING_OFFICIAL_NT")), default=0)
|
||||
simple_orientation_dominant = bool(rtc_products.get("simple_orientation_dominant")) or simple_reason_count > other_reason_max
|
||||
rtc_empty_state = {"show": r3_candidates == 0 and blocked_products > 0, "r3_candidates": r3_candidates,
|
||||
"blocked_products": blocked_products, "simple_orientation_dominant": simple_orientation_dominant}
|
||||
return render(request, "rtc_control.html", {"agent_id": agent_id, "installation": installation, "mode": installation.rtc_write_mode, "modes": RTC_MODE_LABELS, "explanation": RTC_MODE_EXPLANATION, "ops": ops,
|
||||
"privileged": _is_privileged(auth_data[1]), "rows": rows, "counts": counts, "status": status, "page": page, "pages": max(1, -(-total // size)), "total": total, "audits": audits, "rtc_empty_state": rtc_empty_state,
|
||||
"L": {"status": candidate_status_label, "outcome": audit_outcome_label, "reason": abort_reason_label, "rule": rtc_rule_label, "catalog": catalog_state_label}})
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/{agent_id}/rtc-mode")
|
||||
async def rtc_write_mode(request: Request, agent_id: int):
|
||||
"""Central authorization of the cClassTrib fill, per installation. Default DISABLED; ENABLED needs a typed confirmation; every change is audited."""
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
agent, installation = _rtc_target(session, agent_id)
|
||||
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
|
||||
mode = (values.get("mode") or "").strip().upper()
|
||||
if mode not in RTC_MODE_LABELS:
|
||||
return HTMLResponse("modo inválido", status_code=422)
|
||||
if mode not in ("DISABLED", "DRY_RUN") and not _is_privileged(admin):
|
||||
return _deny(session, admin, installation.tenant_id, f"rtc-mode:{mode}", installation.installation_id)
|
||||
if installation.rtc_write_mode not in ("DISABLED", "DRY_RUN") and not _is_privileged(admin):
|
||||
return _deny(session, admin, installation.tenant_id, f"rtc-mode:leave:{installation.rtc_write_mode}", installation.installation_id)
|
||||
if mode == "ENABLED" and (values.get("confirm") or "").strip().upper() != "PREENCHER":
|
||||
return HTMLResponse("Digite PREENCHER para confirmar o modo automático", status_code=422)
|
||||
previous = installation.rtc_write_mode
|
||||
installation.rtc_write_mode = mode
|
||||
installation.rtc_auto_apply_authorized = mode == "ENABLED"
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="RTC_WRITE_MODE", target_type="installation", target_id=installation.installation_id,
|
||||
context={"from": previous, "to": mode, "operator": admin.display_name or admin.username, "role": admin.external_role or admin.auth_source}, severity="WARNING")
|
||||
session.commit()
|
||||
return RedirectResponse(f"/agents/{agent_id}/rtc", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/{agent_id}/rtc-candidates")
|
||||
async def rtc_candidate_action(request: Request, agent_id: int):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
agent, installation = _rtc_target(session, agent_id)
|
||||
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
|
||||
action = values.get("action")
|
||||
if not _is_privileged(admin):
|
||||
return _deny(session, admin, installation.tenant_id, f"rtc-candidates:{action}", installation.installation_id)
|
||||
scope = RtcCandidate.installation_id == installation.id
|
||||
ids = [int(x) for x in parse_qs((await request.body()).decode()).get("ids", []) if x.isdigit()]
|
||||
select_all = values.get("all") == "1" or request.query_params.get("all") == "1"
|
||||
changed = 0
|
||||
if action in ("approve", "reject"):
|
||||
new, old = ("APPROVED", ("PENDING",)) if action == "approve" else ("REJECTED", ("PENDING", "APPROVED"))
|
||||
cond = scope & RtcCandidate.status.in_(old) & (RtcCandidate.id.in_(ids) if not select_all else RtcCandidate.status.in_(old))
|
||||
changed = session.execute(update(RtcCandidate).where(cond).values(status=new)).rowcount
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action=f"RTC_CANDIDATES_{action.upper()}", target_type="installation", target_id=installation.installation_id, context={"changed": changed, "all": select_all, "operator": admin.display_name or admin.username}, severity="WARNING")
|
||||
elif action == "apply_approved":
|
||||
if installation.rtc_write_mode != "APPROVAL_REQUIRED":
|
||||
return HTMLResponse("Disponível apenas no modo Aprovação manual", status_code=409)
|
||||
codes = list(session.scalars(select(RtcCandidate.product_code).where(scope, RtcCandidate.status == "APPROVED").order_by(RtcCandidate.product_code).limit(500)))
|
||||
if codes:
|
||||
job = create_job(session, agent, "REPROCESS_PRODUCTS", {"codigos": codes, "reason": "RTC_APPLY_APPROVED", "requested_by": admin.id})
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="RTC_APPLY_APPROVED", target_type="installation", target_id=installation.installation_id, context={"job_id": job.id, "products": len(codes), "operator": admin.display_name or admin.username}, severity="WARNING")
|
||||
else:
|
||||
return HTMLResponse("ação inválida", status_code=422)
|
||||
session.commit()
|
||||
return RedirectResponse(f"/agents/{agent_id}/rtc", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/{agent_id}/rtc-rollback")
|
||||
async def rtc_rollback_request(request: Request, agent_id: int):
|
||||
"""Requests the rollback of ONE audit record. The Agent executes it only if the value in the ERP still equals what was written; no free SQL exists."""
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
agent, installation = _rtc_target(session, agent_id)
|
||||
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
|
||||
if not _is_privileged(admin):
|
||||
return _deny(session, admin, installation.tenant_id, "rtc-rollback", installation.installation_id)
|
||||
if (values.get("confirm") or "").strip().upper() != "REVERTER":
|
||||
return HTMLResponse("Digite REVERTER para confirmar", status_code=422)
|
||||
audit = session.get(RtcWriteAudit, int(values.get("audit_id") or 0))
|
||||
if audit is None or audit.installation_id != installation.id or audit.action != "APPLY" or audit.outcome != "APPLIED" or audit.rollback_state in ("REQUESTED", "DONE"):
|
||||
return HTMLResponse("registro não reversível", status_code=409)
|
||||
audit.rollback_state = "REQUESTED"
|
||||
job = create_job(session, agent, "REPROCESS_PRODUCTS", {"codigos": [audit.product_code], "reason": "RTC_ROLLBACK", "requested_by": admin.id})
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="RTC_ROLLBACK_REQUESTED", target_type="installation", target_id=installation.installation_id,
|
||||
context={"audit_id": audit.id, "product_code": audit.product_code, "job_id": job.id, "operator": admin.display_name or admin.username}, severity="WARNING")
|
||||
session.commit()
|
||||
return RedirectResponse(f"/agents/{agent_id}/rtc", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/{agent_id}/sync-now")
|
||||
async def sync_now(request: Request, agent_id: int):
|
||||
return await _installation_action(request, agent_id, "sync-now")
|
||||
|
||||
@routes.post("/agents/{agent_id}/pause")
|
||||
async def pause_processing(request: Request, agent_id: int):
|
||||
return await _installation_action(request, agent_id, "pause")
|
||||
|
||||
@routes.post("/agents/{agent_id}/resume")
|
||||
async def resume_processing(request: Request, agent_id: int):
|
||||
return await _installation_action(request, agent_id, "resume")
|
||||
|
||||
@routes.get("/agents/{agent_id}/reprocess-confirmation", response_class=HTMLResponse)
|
||||
def reprocess_confirmation(request: Request, agent_id: int):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
if not session.scalar(select(Agent.id).where(Agent.id == agent_id)):
|
||||
return HTMLResponse("Agent not found", status_code=404)
|
||||
return render(request, "reprocess_confirmation.html", {"agent_id": agent_id, "auth": auth_data})
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/{agent_id}/reprocess")
|
||||
async def reprocess_agent(request: Request, agent_id: int):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
if values.get("confirm") != "yes": return render(request, "reprocess_confirmation.html", {"agent_id": agent_id, "auth": auth_data, "error": "Confirme explicitamente o reprocessamento."}, 400)
|
||||
agent = session.get(Agent, agent_id)
|
||||
installation = session.scalar(select(Installation).where(Installation.agent_id == agent_id))
|
||||
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
|
||||
existing = next((candidate for candidate in session.scalars(select(Job).where(Job.agent_id == agent.id, Job.kind == "REPROCESS_ALL", Job.status == "PENDING")).all() if candidate.payload == {}), None)
|
||||
job = existing or create_job(session, agent, "REPROCESS_ALL", {})
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="REPROCESS_ALL", target_type="installation", target_id=installation.installation_id, context={"job_id": job.id})
|
||||
session.commit()
|
||||
return RedirectResponse(f"/agents/{agent_id}?tab=jobs", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
def _pending_or_404(session, enrollment_uuid):
|
||||
return session.scalar(select(AgentEnrollment).where(AgentEnrollment.enrollment_uuid == enrollment_uuid))
|
||||
|
||||
@routes.get("/agents/pending/{enrollment_uuid}", response_class=HTMLResponse)
|
||||
def pending_installation_detail(request: Request, enrollment_uuid: str, link: str | None = None, code: str | None = None):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
enr = _pending_or_404(session, enrollment_uuid)
|
||||
if not enr: return HTMLResponse("Installation not found", status_code=404)
|
||||
release = session.get(AgentRelease, enr.release_id)
|
||||
tenant = session.get(Tenant, enr.tenant_id)
|
||||
installed_agent = session.get(Agent, enr.agent_id) if enr.agent_id else None
|
||||
active_code = bool(enr.install_code_hash and enr.install_code_expires_at and _aware(enr.install_code_expires_at) > datetime.now(timezone.utc))
|
||||
code_expires_at = enr.install_code_expires_at if active_code else None
|
||||
return render(request, "agent_pending_detail.html", {"enr": enr, "release": release, "tenant": tenant, "installed_agent": installed_agent,
|
||||
"download_link": link, "install_code": code, "code_active": active_code,
|
||||
"code_expires_at": code_expires_at, "auth": auth_data})
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/pending/{enrollment_uuid}/link")
|
||||
async def create_download_link(request: Request, enrollment_uuid: str):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
enr = _pending_or_404(session, enrollment_uuid)
|
||||
if not enr: return HTMLResponse("Installation not found", status_code=404)
|
||||
if enr.status in ("ENROLLED", "REVOKED"): return RedirectResponse(f"/agents/pending/{enrollment_uuid}", status_code=303)
|
||||
token = enrollment.new_token()
|
||||
enr.download_token_hash = enrollment.token_digest(token)
|
||||
enr.download_token_expires_at = datetime.now(timezone.utc) + timedelta(seconds=config.settings.agent_download_link_ttl_seconds)
|
||||
enr.download_max_uses = config.settings.agent_download_max_uses_default
|
||||
enr.download_uses = 0
|
||||
enr.status = "LINK_CREATED"
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=enr.tenant_id, action="DOWNLOAD_LINK_CREATED", target_type="agent_enrollment", target_id=enrollment_uuid, context={"ttl_seconds": config.settings.agent_download_link_ttl_seconds, "ip": client_ip(request)})
|
||||
session.commit()
|
||||
url = f"{config.settings.base_url}/dl/agent/{token}"
|
||||
return RedirectResponse(f"/agents/pending/{enrollment_uuid}?link={url}", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/agents/pending/{enrollment_uuid}/download")
|
||||
def direct_download(request: Request, enrollment_uuid: str):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
web_session, admin = auth_data
|
||||
enr = _pending_or_404(session, enrollment_uuid)
|
||||
if not enr: return HTMLResponse("Installation not found", status_code=404)
|
||||
if enr.status == "REVOKED": return HTMLResponse("Installation revoked", status_code=409)
|
||||
try:
|
||||
return agent_distribution.build_and_serve(session, enr, request, actor_admin_id=admin.id)
|
||||
except enrollment.EnrollmentRejected as rejected:
|
||||
return HTMLResponse(f"Download indisponível: {rejected.reason}", status_code=409)
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/pending/{enrollment_uuid}/revoke")
|
||||
async def revoke_pending_installation(request: Request, enrollment_uuid: str):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
enr = _pending_or_404(session, enrollment_uuid)
|
||||
if not enr: return HTMLResponse("Installation not found", status_code=404)
|
||||
was_code_active = bool(enr.install_code_hash)
|
||||
enr.status = "REVOKED"
|
||||
enr.download_token_hash = None
|
||||
enr.enrollment_token_hash = None
|
||||
enr.install_code_hash = None
|
||||
enr.install_code_expires_at = None
|
||||
if enr.agent_id:
|
||||
agent = session.get(Agent, enr.agent_id)
|
||||
if agent: agent.enabled = False
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=enr.tenant_id, action="INSTALLATION_REVOKED", target_type="agent_enrollment", target_id=enrollment_uuid,
|
||||
context={"had_agent": bool(enr.agent_id), "had_install_code": was_code_active})
|
||||
if was_code_active:
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=enr.tenant_id, action="INSTALLATION_CODE_REVOKED", target_type="agent_enrollment", target_id=enrollment_uuid, context={})
|
||||
session.commit()
|
||||
return RedirectResponse(f"/agents/pending/{enrollment_uuid}", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/pending/{enrollment_uuid}/code")
|
||||
async def generate_install_code(request: Request, enrollment_uuid: str):
|
||||
"""Generate (or rotate) the installation code shown to the operator and
|
||||
typed into the universal EXE installer. Only the SHA-256 of the
|
||||
normalized code is ever stored; the plaintext is returned in the HTML
|
||||
once, to the SSO-authenticated operator who created the installation."""
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
enr = _pending_or_404(session, enrollment_uuid)
|
||||
if not enr: return HTMLResponse("Installation not found", status_code=404)
|
||||
if enr.status in ("ENROLLED", "REVOKED"):
|
||||
return RedirectResponse(f"/agents/pending/{enrollment_uuid}", status_code=303)
|
||||
code = enrollment.generate_install_code()
|
||||
now = datetime.now(timezone.utc)
|
||||
enr.install_code_hash = enrollment.install_code_digest(code)
|
||||
enr.install_code_expires_at = now + timedelta(seconds=config.settings.agent_enrollment_ttl_seconds)
|
||||
enr.install_code_attempts = 0
|
||||
enr.install_code_created_at = now
|
||||
record_audit(session, admin_user_id=admin.id, tenant_id=enr.tenant_id, action="INSTALLATION_CODE_CREATED", target_type="agent_enrollment",
|
||||
target_id=enrollment_uuid, context={"ttl_seconds": config.settings.agent_enrollment_ttl_seconds, "ip": client_ip(request)})
|
||||
session.commit()
|
||||
return RedirectResponse(f"/agents/pending/{enrollment_uuid}?code={code}", status_code=303)
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/blockers", response_class=HTMLResponse)
|
||||
def blockers(request: Request, page: int = 1, client: str | None = None, installation: str | None = None, field: str | None = None, blocker_type: str | None = None, reason_code: str | None = None, status: str = "OPEN", age: str | None = None, search: str | None = None, agent_id: int | None = None, product_code: str | None = None, scope: str = "active"):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
scope = scope if scope in ("active", "all") else "active"
|
||||
status = status if status in ("OPEN", "RESOLVED", "ALL") else "OPEN"
|
||||
filters = {"agent_id": agent_id, "product_code": product_code, "client": client, "installation": installation, "field": field, "blocker_type": blocker_type, "reason_code": reason_code, "status": status, "age": age, "search": search, "scope": scope}
|
||||
result = list_blockers(session, page=page, page_size=50, **filters)
|
||||
summary = blocker_filter_summary(session, **filters)
|
||||
return render(request, "blockers.html", {"result": result, "summary": summary, "auth": auth_data, "csrf_token": request.cookies.get("gti_csrf", ""), "client_filter": client or "", "installation_filter": installation or "", "field_filter": field or "", "blocker_type_filter": blocker_type or "", "reason_code_filter": reason_code or "", "status_filter": status, "scope_filter": scope, "age_filter": age or "", "search": search or "", "query_base": query_base(**filters), "field_query_base": query_base(**{key: value for key, value in filters.items() if key != "field"})})
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/blockers/{blocker_id}", response_class=HTMLResponse)
|
||||
def blocker_page(request: Request, blocker_id: int):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
detail = blocker_detail(session, blocker_id)
|
||||
if detail is None: return HTMLResponse("Blocker not found", status_code=404)
|
||||
blocker = session.get(BlockedProduct, blocker_id)
|
||||
installation = session.get(Installation, blocker.installation_id) if blocker.installation_id else None
|
||||
tenant = session.get(Tenant, blocker.tenant_id)
|
||||
company_key = _company_key(blocker, installation, tenant)
|
||||
current_context = session.scalars(select(FiscalContextEntry).where(FiscalContextEntry.tenant_id == blocker.tenant_id, FiscalContextEntry.company_key == company_key)).all() if installation else []
|
||||
value_choices = sorted({value for values in ENUMS.values() for value in values})
|
||||
return render(request, "blocker_detail.html", {"detail": detail, "auth": auth_data, "context_entries": current_context, "context_keys": sorted(ALLOWED_KEYS), "value_choices": value_choices, "error": None, "resolution_available": installation is not None, "resolution_values": {}})
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/blockers/{blocker_id}/resolve-context")
|
||||
async def resolve_blocker_with_context_route(request: Request, blocker_id: int):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
values = await form_values(request); web_session, admin = auth_data
|
||||
require_csrf(web_session, values.get("csrf_token"))
|
||||
payload = {key: values.get(key) for key in ("scope", "scope_key", "key", "value", "reason")}
|
||||
try:
|
||||
resolve_blocker_with_context(session, blocker_id, admin, payload, request_ip=request.client.host if request.client else None)
|
||||
except BlockerResolutionUnavailable as error:
|
||||
return HTMLResponse(str(error), status_code=409)
|
||||
except LookupError:
|
||||
return HTMLResponse("Blocker not found", status_code=404)
|
||||
except ValueError as error:
|
||||
detail = blocker_detail(session, blocker_id)
|
||||
blocker = session.get(BlockedProduct, blocker_id)
|
||||
installation = session.get(Installation, blocker.installation_id) if blocker and blocker.installation_id else None
|
||||
tenant = session.get(Tenant, blocker.tenant_id) if blocker else None
|
||||
company_key = _company_key(blocker, installation, tenant) if blocker else ""
|
||||
current_context = session.scalars(select(FiscalContextEntry).where(FiscalContextEntry.tenant_id == blocker.tenant_id, FiscalContextEntry.company_key == company_key)).all() if blocker and installation else []
|
||||
return render(request, "blocker_detail.html", {"detail": detail, "auth": auth_data, "context_entries": current_context, "context_keys": sorted(ALLOWED_KEYS), "value_choices": sorted({value for values in ENUMS.values() for value in values}), "error": str(error), "resolution_available": installation is not None, "resolution_values": payload}, 422)
|
||||
return RedirectResponse(f"/blockers/{blocker_id}", status_code=303)
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@routes.get("/jobs", response_class=HTMLResponse)
|
||||
def jobs(request: Request, page: int = 1, status: str | None = None, search: str | None = None, kind: str | None = None, job_id: int | None = None, scope: str = "active"):
|
||||
session, auth_data, redirect = page_auth(request)
|
||||
if redirect: return redirect
|
||||
try:
|
||||
scope = scope if scope in ("active", "all") else "active"
|
||||
status = status if status in ("PENDING", "RUNNING", "COMPLETED", "FAILED") else None
|
||||
result = list_jobs(session, page=page, page_size=50, status=status, search=search, kind=kind, job_id=job_id, scope=scope)
|
||||
return render(request, "jobs.html", {"result": result, "status_counts": jobs_status_summary(session, scope=scope), "auth": auth_data, "status_filter": status or "", "scope_filter": scope, "search": search or "", "kind_filter": kind or "", "job_id_filter": job_id, "query_base": query_base(status=status, search=search, kind=kind, job_id=job_id, scope=scope)})
|
||||
finally: session.close()
|
||||
|
||||
return routes
|
||||
Reference in New Issue
Block a user