Add 'fiscal/' from commit '102ba84500dbd9655a0471288c3e278652593ca9'

git-subtree-dir: fiscal
git-subtree-mainline: d95165fc8f
git-subtree-split: 102ba84500
This commit is contained in:
gti
2026-09-29 14:43:23 -03:00
868 changed files with 566394 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
"""Fiscal Hub application modules."""
+256
View File
@@ -0,0 +1,256 @@
"""Public, pre-authentication routes for Agent distribution.
GET /dl/agent/{token} and POST /v1/agents/enroll are reachable by anyone —
same trust level as GET /login. Every other Agent-distribution route
(create pending installation, generate a link, direct authenticated
download, list/revoke releases) is a control-panel page and lives in
web.py, gated the same way as the rest of the panel (admin session + CSRF).
The enrollment token embedded in a downloaded bundle is generated fresh at
download time and its plaintext is never written to storage — only its
SHA-256 hash persists, exactly like the download token that gated getting
it. Neither is ever the Agent's own permanent Bearer token, which is
minted only once, inside a successful POST /v1/agents/enroll.
"""
from __future__ import annotations
import secrets
from datetime import datetime, timedelta, timezone
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import Response
from sqlalchemy import select
from . import config, enrollment, release_storage
from .audit import record_audit
from .auth import token_hash as agent_token_hash
from .models import Agent, AgentEnrollment, AgentRelease, Installation
from .rate_limit import blocked, record_attempt
def client_ip(request: Request) -> str:
return request.client.host if request.client else "unknown"
def _aware(value: datetime | None) -> datetime | None:
"""SQLite (used by the test suite) drops tzinfo on round-trip even for a
DateTime(timezone=True) column; Postgres never does. Normalize either way
before comparing against datetime.now(timezone.utc) — same pattern as
human_auth.py's session-expiry check."""
if value is None or value.tzinfo is not None:
return value
return value.replace(tzinfo=timezone.utc)
def build_and_serve(session, enr: AgentEnrollment, request: Request, *, actor_admin_id: int | None = None) -> Response:
"""Serve the release artifact for the installation — always the same
universal release binary (ZIP bundle for legacy token flow, bare EXE for
the installer flow). Audit the download; never embed the enrollment token
in an EXE download (the operator types a standalone install code instead)."""
release = session.get(AgentRelease, enr.release_id)
if release is None or release.status != "PUBLISHED":
raise enrollment.EnrollmentRejected("RELEASE_UNAVAILABLE")
path = release_storage.safe_path_for(release.storage_path)
if not path.exists():
raise enrollment.EnrollmentRejected("RELEASE_FILE_MISSING")
now = datetime.now(timezone.utc)
is_exe = release.filename.lower().endswith(".exe")
if is_exe:
# Universal EXE flow: no per-installation bundle, no enrollment.json.
# The operator types the standalone install code into Setup.exe and the
# Hub resolves the installation from the code hash alone.
enr.downloaded_at = now
if enr.status not in ("ENROLLED", "REVOKED"):
enr.status = "DOWNLOADED"
record_audit(session, admin_user_id=actor_admin_id, tenant_id=enr.tenant_id, action="INSTALLER_DOWNLOADED",
target_type="agent_enrollment", target_id=enr.enrollment_uuid,
context={"ip": client_ip(request), "release_version": release.version, "release_id": release.id})
session.commit()
filename = release.filename
return Response(content=path.read_bytes(), media_type="application/octet-stream",
headers={"Content-Disposition": f'attachment; filename="{filename}"',
"X-Content-Type-Options": "nosniff", "Cache-Control": "no-store"})
token = enrollment.new_token()
enr.enrollment_token_hash = enrollment.token_digest(token)
enr.enrollment_token_expires_at = now + timedelta(seconds=config.settings.agent_enrollment_ttl_seconds)
enr.enrollment_used_at = None
enr.downloaded_at = now
if enr.status not in ("ENROLLED", "REVOKED"):
enr.status = "DOWNLOADED"
plan = enrollment.BundlePlan(hub_url=config.settings.base_url, enrollment_uuid=enr.enrollment_uuid, enrollment_token=token)
data = enrollment.build_bundle(path.read_bytes(), plan)
record_audit(session, admin_user_id=actor_admin_id, tenant_id=enr.tenant_id, action="PACKAGE_DOWNLOADED",
target_type="agent_enrollment", target_id=enr.enrollment_uuid,
context={"ip": client_ip(request), "release_version": release.version, "release_id": release.id})
session.commit()
filename = f"GTIFiscalAgent-{release.version}-{enr.enrollment_uuid[:8]}.zip"
return Response(content=data, media_type="application/octet-stream",
headers={"Content-Disposition": f'attachment; filename="{filename}"',
"X-Content-Type-Options": "nosniff", "Cache-Control": "no-store"})
def router(session_factory):
routes = APIRouter()
@routes.get("/dl/agent/{token}")
def download_by_link(token: str, request: Request):
ip = client_ip(request)
if blocked("download", ip):
raise HTTPException(429, "too many attempts")
with session_factory() as session:
candidates = session.scalars(select(AgentEnrollment).where(AgentEnrollment.download_token_hash.is_not(None))).all()
enr = next((c for c in candidates if enrollment.token_matches(token, c.download_token_hash)), None)
now = datetime.now(timezone.utc)
try:
if enr is None:
raise enrollment.EnrollmentRejected("INVALID_LINK")
if enr.status == "REVOKED":
raise enrollment.EnrollmentRejected("REVOKED")
if not enr.download_token_expires_at or _aware(enr.download_token_expires_at) < now:
raise enrollment.EnrollmentRejected("EXPIRED_LINK")
if enr.download_uses >= enr.download_max_uses:
raise enrollment.EnrollmentRejected("LINK_ALREADY_USED")
enr.download_uses += 1
if enr.download_uses >= enr.download_max_uses:
enr.download_token_hash = None # the link's last remaining use is spent: it dies immediately
return build_and_serve(session, enr, request)
except enrollment.EnrollmentRejected as rejected:
session.rollback()
record_attempt("download", ip)
record_audit(session, admin_user_id=None, tenant_id=enr.tenant_id if enr else None, action="DOWNLOAD_LINK_REJECTED",
target_type="agent_enrollment", target_id=enr.enrollment_uuid if enr else "-",
context={"ip": ip, "reason": rejected.reason}, severity="WARNING")
session.commit()
raise HTTPException(404, "invalid or expired link")
@routes.post("/v1/agents/enroll")
async def enroll(request: Request):
ip = client_ip(request)
if blocked("enroll", ip):
raise HTTPException(429, "too many attempts")
try:
payload = await request.json()
except Exception:
payload = {}
installation_id = str(payload.get("installation_id") or "")
supplied_token = payload.get("enrollment_token")
install_code = payload.get("install_code")
# Adjustment: install_code and enrollment_token are mutually exclusive —
# the EXE flow sends a code only; sending both is a request error.
if install_code and (supplied_token or payload.get("installation_id")):
record_attempt("enroll", ip)
with session_factory() as session:
record_audit(session, admin_user_id=None, tenant_id=None, action="ENROLLMENT_REJECTED",
target_type="agent_enrollment", target_id="-",
context={"ip": ip, "reason": "MIXED_CREDENTIALS"}, severity="WARNING")
session.commit()
raise HTTPException(400, "MIXED_CREDENTIALS")
with session_factory() as session:
enr = None
try:
if install_code:
enr = _resolve_by_install_code(session, ip, install_code)
else:
enr = _resolve_by_legacy_token(session, ip, installation_id, supplied_token)
if enr is None:
raise enrollment.EnrollmentRejected("INVALID_INSTALLATION")
if enr.status == "REVOKED":
raise enrollment.EnrollmentRejected("REVOKED")
if enr.status == "ENROLLED":
raise enrollment.EnrollmentRejected("ALREADY_ENROLLED")
release = session.get(AgentRelease, enr.release_id)
if release is None or release.status != "PUBLISHED":
raise enrollment.EnrollmentRejected("RELEASE_UNAVAILABLE")
record_audit(session, admin_user_id=None, tenant_id=enr.tenant_id, action="ENROLLMENT_STARTED",
target_type="agent_enrollment", target_id=enr.enrollment_uuid,
context={"ip": ip, "credential_type": "install_code" if install_code else "enrollment_token"})
hostname = str(payload.get("hostname") or "")[:255]
mismatch = bool(enr.expected_hostname) and bool(hostname) and hostname.lower() != enr.expected_hostname.lower()
plain_token = secrets.token_urlsafe(32)
agent = Agent(tenant_id=enr.tenant_id, name=enr.display_name, token_hash=agent_token_hash(plain_token),
enabled=True, agent_version=str(payload.get("agent_version") or "")[:32] or None)
session.add(agent)
session.flush()
session.add(Installation(agent_id=agent.id, tenant_id=enr.tenant_id, installation_id=enr.enrollment_uuid,
display_name=enr.display_name, hostname=hostname or None,
features_configured=bool(enr.features_selected), corrector_enabled=bool(enr.features_selected and enr.feature_corrector),
enrichment_mode=(enr.feature_enrichment_mode if enr.features_selected else "DISABLED")))
enr.status = "ENROLLED"
enr.enrollment_used_at = datetime.now(timezone.utc)
enr.enrollment_token_hash = None # replay-proof even inside its TTL window
enr.install_code_hash = None # consumed: truly one-time, replay-proof
enr.install_code_expires_at = None
enr.hostname_seen = hostname or None
enr.machine_fingerprint_seen = str(payload.get("machine_fingerprint") or "")[:128] or None
enr.hostname_mismatch = mismatch
enr.agent_id = agent.id
record_audit(session, admin_user_id=None, tenant_id=enr.tenant_id, action="ENROLLMENT_SUCCEEDED",
target_type="agent_enrollment", target_id=enr.enrollment_uuid,
context={"ip": ip, "hostname": hostname, "hostname_mismatch": mismatch, "agent_id": agent.id})
record_audit(session, admin_user_id=None, tenant_id=enr.tenant_id, action="AGENT_CREDENTIAL_ISSUED",
target_type="agent", target_id=str(agent.id), context={"ip": ip, "installation": enr.enrollment_uuid})
session.commit()
return {"agent_id": agent.id, "agent_token": plain_token, "tenant_id": enr.tenant_id,
"installation_id": enr.enrollment_uuid,
"hub_url": config.settings.base_url, "hostname_mismatch": mismatch}
except enrollment.EnrollmentRejected as rejected:
session.rollback()
record_attempt("enroll", ip)
reason = rejected.reason
if enr is not None:
enr.install_code_attempts = (enr.install_code_attempts or 0) + 1
record_attempt(f"enroll_code:{enr.enrollment_uuid}", ip)
session.commit()
record_audit(session, admin_user_id=None, tenant_id=enr.tenant_id if enr else None, action="ENROLLMENT_REJECTED",
target_type="agent_enrollment", target_id=enr.enrollment_uuid if enr else (installation_id or "-"),
context={"ip": ip, "reason": reason}, severity="WARNING")
session.commit()
raise HTTPException(401, reason)
def _resolve_by_install_code(session, ip, code):
"""Resolve the Installation by the install_code hash alone. Returns the
enrollment or None. Format/unknown codes and expired codes raise
EnrollmentRejected. Per-installation and per-IP rate limits apply."""
try:
digest = enrollment.install_code_digest(code)
except ValueError:
raise enrollment.EnrollmentRejected("INVALID_INSTALL_CODE")
enr = session.scalar(select(AgentEnrollment).where(AgentEnrollment.install_code_hash == digest))
if enr is None:
raise enrollment.EnrollmentRejected("INVALID_INSTALL_CODE")
# Per-installation rate limit: a hostile caller guessing codes for one
# installation must burn the whole budget even after the code rotates.
if blocked(f"enroll_code:{enr.enrollment_uuid}", ip):
raise enrollment.EnrollmentRejected("RATE_LIMITED")
now = datetime.now(timezone.utc)
if not enr.install_code_expires_at or _aware(enr.install_code_expires_at) < now:
raise enrollment.EnrollmentRejected("EXPIRED")
return enr
def _resolve_by_legacy_token(session, ip, installation_id, supplied_token):
"""Legacy enrollment_token path (published ZIP bundles). Returns the
enrollment or raises EnrollmentRejected. Order preserved from the
original flow: enrolled-again must read ALREADY_ENROLLED, not
INVALID_TOKEN."""
if not enrollment.valid_uuid(installation_id):
raise enrollment.EnrollmentRejected("INVALID_INSTALLATION")
enr = session.scalar(select(AgentEnrollment).where(AgentEnrollment.enrollment_uuid == installation_id))
if enr is None:
raise enrollment.EnrollmentRejected("INVALID_INSTALLATION")
if enr.status == "REVOKED":
raise enrollment.EnrollmentRejected("REVOKED")
if enr.status == "ENROLLED":
raise enrollment.EnrollmentRejected("ALREADY_ENROLLED")
now = datetime.now(timezone.utc)
if not enr.enrollment_token_expires_at or _aware(enr.enrollment_token_expires_at) < now:
raise enrollment.EnrollmentRejected("EXPIRED")
if not enrollment.token_matches(supplied_token, enr.enrollment_token_hash):
raise enrollment.EnrollmentRejected("INVALID_TOKEN")
return enr
return routes
+8
View File
@@ -0,0 +1,8 @@
"""AI semantic layer V1 — NOT a fiscal authority.
Rules frozen by the operator contract:
- AI only selects among official candidates; it never invents codes.
- Cache first: KB exact/fingerprint hits never trigger AI calls.
- No AI for origin, ST retention, or company-level decisions.
- AI_WRITE_ALLOWED=false in V1: results stay AI_CANDIDATE_VALIDATED.
"""
+55
View File
@@ -0,0 +1,55 @@
"""Offline-safe, ID-correlated provider batch execution."""
from dataclasses import dataclass, field
from .provider import ProviderTransportError
@dataclass
class BatchRunResult:
results_by_work_item_id: dict[str, dict] = field(default_factory=dict)
errors: dict[str, str] = field(default_factory=dict)
provider_calls: int = 0
retries: int = 0
usage: dict = field(default_factory=dict)
def run_batches(items, provider, batch_size: int, max_retries: int = 2, *, scenario: str | None = None) -> BatchRunResult:
if batch_size <= 0:
raise ValueError("batch_size must be positive")
if max_retries < 0:
raise ValueError("max_retries must not be negative")
result = BatchRunResult()
for start in range(0, len(items), batch_size):
batch = list(items[start:start + batch_size])
expected_ids = [str(item.work_item_id) for item in batch]
answers = None
for attempt in range(max_retries + 1):
result.provider_calls += 1
try:
kwargs = {} if scenario is None else {"scenario": scenario}
answers = provider.select_batch(batch, **kwargs)
break
except ProviderTransportError as error:
if attempt == max_retries:
for work_item_id in expected_ids:
result.errors[work_item_id] = str(error)
else:
result.retries += 1
except Exception as error:
for work_item_id in expected_ids:
result.errors[work_item_id] = str(error)
break
if answers is None:
continue
answer_ids = [str(answer.get("work_item_id")) for answer in answers]
if len(answer_ids) != len(expected_ids) or set(answer_ids) != set(expected_ids) or len(set(answer_ids)) != len(answer_ids):
error = "provider response did not contain exactly one answer per work_item_id"
for work_item_id in expected_ids:
result.errors[work_item_id] = error
continue
result.results_by_work_item_id.update({str(answer["work_item_id"]): answer for answer in answers})
result.usage = dict(getattr(provider, "usage", {}) or {})
return result
+59
View File
@@ -0,0 +1,59 @@
"""Persistent-boundary-compatible budget guard; provider invocation happens after this check."""
from dataclasses import dataclass
from datetime import datetime, timezone
from threading import Lock
@dataclass(frozen=True)
class BudgetLimits:
max_run_cost_usd: float = 0.54
max_daily_cost_usd: float = 1.00
max_monthly_cost_usd: float = 10.00
max_requests_per_day: int = 100
max_new_products_per_run: int = 750
@dataclass(frozen=True)
class BudgetDecision:
allowed: bool
reason: str
reserved_cost_usd: float
@dataclass(frozen=True)
class UsageEntry:
run_id: str
tenant: str
cost_usd: float
requests: int
new_products: int
timestamp: str
class BudgetLedger:
"""Atomic in-process ledger with an explicit persistence boundary for Hub storage."""
def __init__(self, limits: BudgetLimits):
self.limits = limits
self.entries: list[UsageEntry] = []
self._lock = Lock()
def check_and_reserve(self, run_id: str, tenant: str, cost_usd: float, requests: int, new_products: int) -> BudgetDecision:
with self._lock:
run_cost = sum(e.cost_usd for e in self.entries if e.run_id == run_id)
day = datetime.now(timezone.utc).date()
day_entries = [e for e in self.entries if e.timestamp[:10] == day.isoformat()]
month_entries = [e for e in self.entries if e.timestamp[:7] == day.strftime("%Y-%m")]
if run_cost + cost_usd > self.limits.max_run_cost_usd:
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
if sum(e.cost_usd for e in day_entries) + cost_usd > self.limits.max_daily_cost_usd:
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
if sum(e.cost_usd for e in month_entries) + cost_usd > self.limits.max_monthly_cost_usd:
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
if sum(e.requests for e in day_entries) + requests > self.limits.max_requests_per_day:
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
if sum(e.new_products for e in self.entries if e.run_id == run_id) + new_products > self.limits.max_new_products_per_run:
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
self.entries.append(UsageEntry(run_id, tenant, cost_usd, requests, new_products, datetime.now(timezone.utc).isoformat()))
return BudgetDecision(True, "RESERVED", cost_usd)
+30
View File
@@ -0,0 +1,30 @@
"""Pure cache and dataset-freshness policies for offline and real evaluations."""
import hashlib
import json
VALID, NEEDS_REVALIDATION, CONFLICT = "VALID", "NEEDS_REVALIDATION", "CONFLICT"
def cache_key(input_fingerprint, candidate_set_hash, model, prompt_version, dataset_versions):
"""Return a stable key covering every input that can change an answer."""
payload = {
"input_fingerprint": str(input_fingerprint),
"candidate_set_hash": str(candidate_set_hash),
"model": str(model),
"prompt_version": str(prompt_version),
"dataset_versions": dict(sorted((dataset_versions or {}).items())),
}
return hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
def classify_knowledge_freshness(record_versions, active_versions):
"""Classify versioned knowledge without deleting or rewriting history."""
recorded = dict(record_versions or {})
active = dict(active_versions or {})
if any(key in recorded and key in active and recorded[key] != active[key] for key in recorded):
return CONFLICT
if recorded != active:
return NEEDS_REVALIDATION
return VALID
+53
View File
@@ -0,0 +1,53 @@
"""Deterministic candidate generation from the ACTIVE official datasets.
The AI never receives an open question: for every task type the Hub first builds
the official candidate list (NCM top-K by description, CEST links for the NCM,
TIPI EX rows for the NCM). Providers may only pick from this list.
"""
from sqlalchemy import select
from sqlalchemy.orm import Session
from ..models import CestEntry, CestNcmEntry, Dataset, NcmEntry, TipiEntry
from .fingerprint import token_stems
def _active_dataset(session: Session, kind: str):
return session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE"))
def ncm_top_k(session: Session, description, k: int = 10):
"""Top-K official NCM codes by (stemmed) token overlap with the product description."""
dataset = _active_dataset(session, "NCM")
if dataset is None:
return []
stems = token_stems(description)
if not stems:
return []
scored = []
for entry in session.scalars(select(NcmEntry).where(NcmEntry.dataset_id == dataset.id)).all():
score = len(stems & token_stems(entry.description)) / len(stems)
if score > 0:
scored.append((entry.code, entry.description, round(score, 4)))
scored.sort(key=lambda row: (-row[2], row[0]))
return scored[:k]
def cest_candidates_for_ncm(session: Session, ncm: str):
dataset = _active_dataset(session, "CEST")
if dataset is None:
return []
rows = session.execute(
select(CestEntry, CestNcmEntry.ncm_code)
.join(CestNcmEntry, CestNcmEntry.cest_entry_id == CestEntry.id)
.where(CestEntry.dataset_id == dataset.id)
).all()
return list(dict.fromkeys(entry for entry, ncm_code in rows if ncm.startswith(ncm_code)))
def ex_tipi_candidates_for_ncm(session: Session, ncm: str):
dataset = _active_dataset(session, "TIPI_IPI")
if dataset is None:
return []
return list(dict.fromkeys(
entry for entry in session.scalars(select(TipiEntry).where(TipiEntry.dataset_id == dataset.id, TipiEntry.ncm == ncm, TipiEntry.ex.is_not(None))).all()
))
+53
View File
@@ -0,0 +1,53 @@
"""Metrics and error attribution for the real LLM NCM evaluation (sections 8-9)."""
def metrics(rows) -> dict:
"""rows: [{gold_in_candidates, selected, gold, confidence, invalid}]"""
total = len(rows)
correct = sum(1 for row in rows if not row["invalid"] and row["selected"] == row["gold"])
wrong = sum(1 for row in rows if not row["invalid"] and row["selected"] is not None and row["selected"] != row["gold"])
abstain = sum(1 for row in rows if not row["invalid"] and row["selected"] is None)
invalid = sum(1 for row in rows if row["invalid"])
gold_in = sum(1 for row in rows if row["gold_in_candidates"])
answered = correct + wrong
high_conf = [row for row in rows if not row["invalid"] and row["confidence"] >= 0.95 and row["selected"] is not None]
high_conf_wrong = sum(1 for row in high_conf if row["selected"] != row["gold"])
return {
"TOTAL": total,
"CORRECT": correct,
"WRONG": wrong,
"ABSTAIN": abstain,
"INVALID": invalid,
"GOLD_IN_CANDIDATES": gold_in,
"ACCURACY_ALL": round(correct / total, 4) if total else None,
"SELECTION_ACCURACY_WHEN_GOLD_AVAILABLE": round(correct / answered, 4) if answered else None,
"PRECISION_WHEN_HIGH_CONFIDENCE": round(sum(1 for row in high_conf if row["selected"] == row["gold"]) / len(high_conf), 4) if high_conf else None,
"HIGH_CONFIDENCE_WRONG_RATE": round(high_conf_wrong / len(high_conf), 4) if high_conf else None,
"ABSTENTION_RATE": round(abstain / total, 4) if total else None,
}
def attribution(rows) -> dict:
return {
"GOLD_NOT_IN_CANDIDATES": sum(1 for row in rows if not row["gold_in_candidates"]),
"GOLD_IN_CANDIDATES_BUT_WRONG": sum(1 for row in rows if row["gold_in_candidates"] and not row["invalid"] and row["selected"] is not None and row["selected"] != row["gold"]),
"GOLD_IN_CANDIDATES_BUT_ABSTAIN": sum(1 for row in rows if row["gold_in_candidates"] and not row["invalid"] and row["selected"] is None),
"INVALID_OUTPUT": sum(1 for row in rows if row["invalid"]),
}
_BUCKETS = ((">=0.99", 0.99, 1.01), ("0.95-0.9899", 0.95, 0.99), ("0.90-0.9499", 0.90, 0.95), ("0.80-0.8999", 0.80, 0.90), ("<0.80", -0.01, 0.80))
def confidence_buckets(rows) -> dict:
out = {name: {"total": 0, "correct": 0} for name, _, _ in _BUCKETS}
for row in rows:
if row["invalid"] or row["selected"] is None:
continue
for name, low, high in _BUCKETS:
if low <= row["confidence"] < high:
out[name]["total"] += 1
if row["selected"] == row["gold"]:
out[name]["correct"] += 1
break
return out
+77
View File
@@ -0,0 +1,77 @@
"""Deterministic provider responses for offline pipeline tests."""
from collections.abc import Iterable
from .llm_provider import LlmAnswerError, LlmEvalItem, validate_answer
from .provider import ProviderContractError, ProviderTransportError
SCENARIOS = frozenset({
"CORRECT", "WRONG", "ABSTAIN", "INVALID_JSON", "MISSING_ITEM",
"DUPLICATE_ITEM", "OUT_OF_CANDIDATE", "TIMEOUT", "429", "500",
})
class DeterministicFakeSemanticProvider:
"""Return repeatable answers without persistence or network access."""
name = "deterministic-fake"
model = "deterministic-fake-v1"
prompt_version = "fake-provider-v1"
def __init__(self, *, shuffle: bool = False, fail_work_item_ids: Iterable[str] = (), failure_scenario: str = "TIMEOUT"):
self.shuffle = shuffle
self.fail_work_item_ids = {str(item_id) for item_id in fail_work_item_ids}
self.failure_scenario = failure_scenario
self.calls = 0
self.usage = {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0}
def select_batch(self, items: list[LlmEvalItem], scenario: str | None = None) -> list[dict]:
scenario = scenario or "CORRECT"
if scenario not in SCENARIOS:
raise ValueError(f"unknown fake provider scenario: {scenario}")
self.calls += 1
if self.fail_work_item_ids and len(items) == 1 and str(items[0].work_item_id) in self.fail_work_item_ids:
self._raise_transport(self.failure_scenario)
if scenario in {"TIMEOUT", "429", "500"}:
self._raise_transport(scenario)
if scenario == "INVALID_JSON":
raise LlmAnswerError("fake provider returned invalid JSON")
answers = [self._answer(item, scenario) for item in items]
if scenario == "MISSING_ITEM" and answers:
answers.pop()
elif scenario == "DUPLICATE_ITEM" and answers:
answers[-1] = dict(answers[0])
if self.shuffle:
answers.reverse()
expected_ids = [str(item.work_item_id) for item in items]
answer_ids = [str(answer.get("work_item_id")) for answer in answers]
if len(answer_ids) != len(expected_ids) or set(answer_ids) != set(expected_ids) or len(set(answer_ids)) != len(answer_ids):
raise ProviderContractError("fake provider response did not contain exactly one answer per work_item_id")
items_by_id = {str(item.work_item_id): item for item in items}
try:
return [validate_answer(answer, items_by_id[str(answer["work_item_id"])]) for answer in answers]
except (LlmAnswerError, KeyError, TypeError) as error:
raise ProviderContractError(str(error)) from error
@staticmethod
def _answer(item: LlmEvalItem, scenario: str) -> dict:
candidates = [str(candidate[0]) for candidate in item.candidates]
selected = None
abstain = scenario == "ABSTAIN"
if not abstain and candidates:
selected = "99999999" if scenario == "OUT_OF_CANDIDATE" else candidates[min(1, len(candidates) - 1) if scenario == "WRONG" else 0]
return {
"work_item_id": item.work_item_id,
"selected_ncm": selected,
"confidence": 0.0 if abstain else 0.75,
"abstain": abstain or selected is None,
"reason_short": f"fake scenario {scenario}",
}
@staticmethod
def _raise_transport(scenario: str) -> None:
if scenario not in {"TIMEOUT", "429", "500"}:
raise ValueError(f"unknown fake transport scenario: {scenario}")
raise ProviderTransportError(f"fake provider transport failure: {scenario}")
+75
View File
@@ -0,0 +1,75 @@
"""Product identity: stable fingerprints and GTIN validation.
Global identity uses ONLY stable attributes (normalized description, brand, unit,
package quantity, GTIN). Fiscal/company context (CFOP, CSOSN, CST, rate, ST, tenant)
never participates: those vary per context and must not merge distinct knowledge.
"""
import hashlib
import re
import unicodedata
from ..classification import ENGINE_VERSION # noqa: F401 (re-exported for the AI layer)
def normalize_description(value) -> str:
"""NFKD -> ascii -> A-Z0-9 tokens, space-joined (same canonical form as classification)."""
text = unicodedata.normalize("NFKD", str(value or "")).encode("ascii", "ignore").decode().upper()
return " ".join(re.sub(r"[^A-Z0-9]+", " ", text).split())
def _normalize_token(value) -> str:
return normalize_description(value)
def _normalize_quantity(value) -> str:
digits = re.sub(r"[^0-9.,]", "", str(value or ""))
if not digits:
return ""
try:
number = float(digits.replace(",", "."))
except ValueError:
return digits
return f"{number:g}"
def product_fingerprint(description, brand=None, unit=None, package_quantity=None) -> str:
parts = [_normalize_token(description), _normalize_token(brand), _normalize_token(unit), _normalize_quantity(package_quantity)]
return hashlib.sha256("|".join(parts).encode("utf-8")).hexdigest()
def _stem(token: str) -> str:
"""Minimal Portuguese plural stemming so AGUAS matches AGUA / MINERAIS matches MINERAL."""
if len(token) > 4 and token.endswith("AIS"):
return token[:-3] + "AL"
if len(token) > 4 and token.endswith("ES"):
return token[:-2]
if len(token) > 3 and token.endswith("S"):
return token[:-1]
return token
def token_stems(value) -> set:
return {_stem(token) for token in normalize_description(value).split()}
def overlap_score(query, official) -> float:
"""Fraction of query description stems present in the official description stems."""
stems = token_stems(query)
if not stems:
return 0.0
return len(stems & token_stems(official)) / len(stems)
def validate_gtin(value) -> str | None:
"""Returns the canonical GTIN when the checksum is valid, else None.
Accepts GTIN-8/12/13/14 (EAN-8/UPC-12/EAN-13/GTIN-14)."""
digits = re.sub(r"\D", "", str(value or ""))
if len(digits) not in (8, 12, 13, 14):
return None
total = 0
for index, char in enumerate(reversed(digits)):
digit = int(char)
total += digit * (1 if index % 2 == 0 else 3)
if total % 10 != 0:
return None
return digits
@@ -0,0 +1,47 @@
"""Evidence reconstruction for historical KB conflicts.
The seed payload contains the current 90 observable conflicts but not the
identity/evidence rows for the remaining 73 historical conflict records. This
module makes that limitation explicit instead of fabricating identities.
"""
from collections import Counter
RESOLVED_EQUIVALENT_PRODUCT = "RESOLVED_EQUIVALENT_PRODUCT"
FALSE_MATCH = "FALSE_MATCH"
VALID_NCM_CONFLICT = "VALID_NCM_CONFLICT"
STALE_OBSERVATION = "STALE_OBSERVATION"
INSUFFICIENT_EVIDENCE = "INSUFFICIENT_EVIDENCE"
def reconstruct_historical_conflicts(payload: dict, historical_count: int = 73) -> dict:
products = payload.get("products", [])
gtin_count = sum(1 for product in products if product.get("gtin"))
fingerprint_count = sum(1 for product in products if product.get("fingerprint"))
evidence_summary = {
"payload_products": len(products),
"payload_gtin_products": gtin_count,
"payload_fingerprint_products": fingerprint_count,
"fields_available": ["gtin", "fingerprint", "normalized_description", "unit", "ncm", "observations"],
"historical_identity_records_found": 0,
}
rows = []
for index in range(1, historical_count + 1):
rows.append({
"historical_conflict_id": f"HISTORICAL-{index:03d}",
"classification": INSUFFICIENT_EVIDENCE,
"evidence_search": evidence_summary,
"resolution": "PRESERVE_CONFLICT",
"reason": "original conflict identity/evidence row is absent from available artifacts; no NCM selected",
})
return {
"artifact": "KB_HISTORICAL_CONFLICT_AUDIT_V1",
"historical_conflicts_before": historical_count,
"historical_conflicts_resolved": 0,
"historical_conflicts_remaining": historical_count,
"classification_counts": dict(Counter(row["classification"] for row in rows)),
"conflicts": rows,
"current_observable_conflicts_preserved": 90,
"current_conflict_policy": "NCM_CONFLICT rows remain blocked; no majority resolution",
"evidence_complete": False,
}
+121
View File
@@ -0,0 +1,121 @@
"""Product Knowledge Base service: identity resolution, versioned knowledge, conflicts.
Identity order (operator contract): 1. exact GTIN; 2. strong fingerprint; 3. semantic
fingerprint candidate; 4. unknown. Conflicting knowledge for the same key is kept
side by side and surfaced as KNOWLEDGE_CONFLICT — never picked by majority.
"""
from datetime import datetime, timezone
from sqlalchemy import func as sa_func, select
from ..models import KbKnowledge, KbObservation, KbProduct
EXACT, FINGERPRINT = "EXACT", "FINGERPRINT"
_AUTHORITATIVE = {"OFFICIAL_DATASET", "OPERATOR", "AI_PLUS_OFFICIAL_VALIDATION"}
class KnowledgeBase:
def __init__(self, session):
self.session = session
# ----- identity -----
def upsert_product(self, fingerprint, gtin=None, normalized_description="", brand=None, unit=None, package_quantity=None, category=None):
gtin = gtin or None
product = None
if gtin:
product = self.session.scalar(select(KbProduct).where(KbProduct.gtin == gtin))
if product is None:
product = self.session.scalar(select(KbProduct).where(KbProduct.fingerprint == fingerprint))
if product is None:
product = KbProduct(fingerprint=fingerprint, normalized_description=normalized_description or "")
self.session.add(product)
# identity upgrades (never downgrades)
if gtin and not product.gtin:
product.gtin = gtin
product.identity_kind = "GTIN"
if normalized_description and not product.normalized_description:
product.normalized_description = normalized_description
for field, value in (("brand", brand), ("unit", unit), ("package_quantity", package_quantity), ("category", category)):
if value and not getattr(product, field):
setattr(product, field, value)
self.session.flush()
return product
def resolve(self, gtin=None, fingerprint=None):
"""Returns (product|None, hit_kind). Cache-first: EXACT (GTIN) beats FINGERPRINT."""
if gtin:
product = self.session.scalar(select(KbProduct).where(KbProduct.gtin == gtin))
if product:
return product, EXACT
if fingerprint:
product = self.session.scalar(select(KbProduct).where(KbProduct.fingerprint == fingerprint))
if product:
return product, FINGERPRINT
return None, None
# ----- knowledge -----
def add_knowledge(self, kb_product_id, knowledge_key, value, source, confidence=0.0, evidence=None, model=None, prompt_version=None, dataset_versions=None):
value = str(value)
row = self.session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == kb_product_id, KbKnowledge.knowledge_key == knowledge_key, KbKnowledge.value == value))
if row is None:
row = KbKnowledge(kb_product_id=kb_product_id, knowledge_key=knowledge_key, value=value, source=source, confidence=confidence, evidence=evidence or {}, model=model, prompt_version=prompt_version, dataset_versions=dataset_versions or {})
self.session.add(row)
else:
# authoritative sources upgrade provenance/confidence of the same value; history preserved
if source in _AUTHORITATIVE:
row.source = source
row.confidence = max(row.confidence, confidence)
row.evidence = {**row.evidence, **(evidence or {})}
if dataset_versions:
row.dataset_versions = {**row.dataset_versions, **dataset_versions}
if source == "OFFICIAL_DATASET":
row.validated_at = datetime.now(timezone.utc)
self.session.flush()
return row
def current_knowledge(self, kb_product_id):
rows = self.session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == kb_product_id)).all()
by_key: dict = {}
for row in rows:
by_key.setdefault(row.knowledge_key, []).append(row)
result = {}
for key, entries in by_key.items():
values = {entry.value for entry in entries}
if len(values) > 1:
result[key] = {"conflict": True, "values": values}
else:
best = max(entries, key=lambda entry: entry.confidence)
result[key] = {"conflict": False, "value": best.value, "source": best.source, "confidence": best.confidence}
return result
# ----- observations -----
def record_observation(self, kb_product_id, tenant_id, company_key, erp_code, database_label, observed_fields, source="ERP_OBSERVATION"):
row = KbObservation(kb_product_id=kb_product_id, tenant_id=tenant_id, company_key=company_key, erp_code=erp_code, database_label=database_label, observed_fields=observed_fields or {}, source=source)
self.session.add(row); self.session.flush()
return row
def observation_summary(self, kb_product_id):
rows = self.session.scalars(select(KbObservation).where(KbObservation.kb_product_id == kb_product_id)).all()
agreement: dict = {}
for row in rows:
for field, value in (row.observed_fields or {}).items():
if value in (None, ""):
continue
bucket = agreement.setdefault(field, {})
bucket[str(value)] = bucket.get(str(value), 0) + 1
conflicts = {field: values for field, values in agreement.items() if len(values) > 1}
return {"observation_count": len(rows), "tenant_count": len({row.tenant_id for row in rows}), "agreement": agreement, "conflicts": conflicts}
# ----- stats -----
def stats(self):
products = self.session.scalar(select(sa_func.count()).select_from(KbProduct)) or 0
gtin_products = self.session.scalar(select(sa_func.count()).select_from(KbProduct).where(KbProduct.gtin.is_not(None))) or 0
conflicts = 0
for product_id in self.session.scalars(select(KbProduct.id)).all():
knowledge = self.current_knowledge(product_id)
conflicts += sum(1 for entry in knowledge.values() if entry.get("conflict"))
fingerprint_products = products - gtin_products
return {"products": products, "gtin_products": gtin_products, "fingerprint_products": fingerprint_products, "conflicts": conflicts}
+87
View File
@@ -0,0 +1,87 @@
"""Conservative, explainable product identity and conflict classification."""
import hashlib
import re
from collections import defaultdict
from .fingerprint import normalize_description, token_stems, validate_gtin
GTIN_CONFLICT = "GTIN_CONFLICT"
NCM_CONFLICT = "NCM_CONFLICT"
DESCRIPTION_CONFLICT = "DESCRIPTION_CONFLICT"
PACKAGING_VARIATION = "PACKAGING_VARIATION"
FALSE_MATCH = "FALSE_MATCH"
TENANT_CONTEXT_ONLY = "TENANT_CONTEXT_ONLY"
def canonical_product_id(observation: dict) -> str:
"""Stable global ID; fiscal and tenant fields are intentionally excluded."""
gtin = validate_gtin(observation.get("gtin"))
if gtin:
return "GTIN:" + gtin
values = "|".join((
normalize_description(observation.get("normalized_description") or observation.get("description")),
normalize_description(observation.get("brand")),
normalize_description(observation.get("unit")),
normalize_description(observation.get("package_quantity")),
))
return "FP:" + hashlib.sha256(values.encode()).hexdigest()
def _description_equivalent(values):
stems = [token_stems(value) for value in values if value]
if not stems:
return True
return all(current == stems[0] or current.issubset(stems[0]) or stems[0].issubset(current) for current in stems[1:])
def classify_conflict(observations: list[dict]) -> str:
"""Classify only from structural evidence; ambiguous cases remain conflicts."""
gtins = {validate_gtin(row.get("gtin")) for row in observations if validate_gtin(row.get("gtin"))}
ncms = {str(row.get("ncm")) for row in observations if row.get("ncm")}
descriptions = [row.get("normalized_description") or row.get("description") for row in observations]
packages = {str(row.get("package_quantity")) for row in observations if row.get("package_quantity")}
tenants = {str(obs.get("database") or obs.get("company_key")) for row in observations for obs in row.get("observations", [])}
if len(gtins) > 1:
return GTIN_CONFLICT
if len(ncms) > 1:
return NCM_CONFLICT
if len(packages) > 1 and _description_equivalent(descriptions):
return PACKAGING_VARIATION
if len({normalize_description(value) for value in descriptions if value}) > 1:
return DESCRIPTION_CONFLICT
if len(tenants) > 1:
return TENANT_CONTEXT_ONLY
return FALSE_MATCH
def audit_payload(payload: dict, historical_conflicts: int = 163) -> dict:
"""Audit observable same-GTIN groups without silently resolving NCM conflicts."""
by_gtin = defaultdict(list)
for product in payload.get("products", []):
gtin = validate_gtin(product.get("gtin"))
if gtin:
by_gtin[gtin].append(product)
conflicts = []
counts = defaultdict(int)
for gtin, rows in by_gtin.items():
if len({row.get("ncm") for row in rows if row.get("ncm")}) <= 1:
continue
kind = classify_conflict(rows)
counts[kind] += 1
conflicts.append({
"identity": gtin,
"classification": kind,
"observations": len(rows),
"ncms": sorted({row.get("ncm") for row in rows if row.get("ncm")}),
"auto_resolution": "KEEP_KNOWLEDGE_CONFLICT",
})
return {
"artifact": "KB_CONFLICT_AUDIT_V1",
"historical_conflicts_reported": historical_conflicts,
"observable_conflicts_audited": len(conflicts),
"unobservable_historical_conflicts": max(0, historical_conflicts - len(conflicts)),
"classification_counts": dict(sorted(counts.items())),
"conflicts": conflicts,
"policy": "only structurally equivalent identities may share a canonical product; tenant fiscal context is never global knowledge",
}
+158
View File
@@ -0,0 +1,158 @@
"""Real LLM provider (OpenAI-compatible) for the NCM semantic evaluation.
Contract (mission REAL LLM EVALUATION V1):
- The API key lives ONLY in the Hub environment (AI_API_KEY). Never committed, logged,
written to artifacts, sent to the Agent or Firebird.
- The gold answer is never part of the prompt: items carry only description/brand/unit/
segment-weak-context + the official top-K hybrid candidates with hierarchical context.
- Strict structured output: per item {work_item_id, selected_ncm, confidence, abstain,
reason_short}; selected_ncm must be in the candidate list or null (abstain).
- Batching is safe only when the response is valid JSON with 1:1 work_item_id matching,
no omitted items, no cross-product mixing. Verified progressively (5/10/20).
"""
import hashlib
import json
import re
from dataclasses import dataclass
PROMPT_VERSION_LLM = "llm-ncm-eval-v1"
_SYSTEM_PROMPT = (
"You are a Brazilian customs classification assistant. For each product below, choose the "
"most likely official NCM code from the provided candidate list, or abstain. "
"Rules: selected_ncm MUST be one of the candidate codes for that exact item, or null with "
"abstain=true when evidence is insufficient. Never invent a code. confidence is your calibrated "
"probability in [0,1]. Reply with ONLY a JSON array, one object per item, in the same order, "
'each: {"work_item_id": str, "selected_ncm": str|null, "confidence": float, "abstain": bool, "reason_short": str}.'
)
@dataclass
class LlmEvalItem:
work_item_id: str
description: str
normalized_description: str
brand: str | None
unit: str | None
segment: str
candidates: list # [(code, official_description, hybrid_score, official_score, kb_score, tenant_count)]
class LlmAnswerError(ValueError):
pass
def item_prompt_payload(item: LlmEvalItem) -> dict:
return {
"work_item_id": item.work_item_id,
"description": item.description,
"normalized_description": item.normalized_description,
"brand": item.brand,
"unit": item.unit,
"segment_weak_context": item.segment,
"official_candidates": [
{"ncm": code, "official_description": text}
for code, text, *_ in item.candidates
],
}
def candidate_set_hash(item: LlmEvalItem) -> str:
codes = [code for code, *_ in item.candidates]
return hashlib.sha256("|".join(codes).encode()).hexdigest()
def input_fingerprint(item: LlmEvalItem) -> str:
payload = {**item_prompt_payload(item), "system": _SYSTEM_PROMPT}
return hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
def validate_answer(answer: dict, item: LlmEvalItem) -> dict:
"""Fail-closed validator: schema, work_item_id, candidate-list confinement."""
if not isinstance(answer, dict):
raise LlmAnswerError("answer must be an object")
if str(answer.get("work_item_id")) != str(item.work_item_id):
raise LlmAnswerError(f"work_item_id mismatch: {answer.get('work_item_id')!r} != {item.work_item_id!r}")
abstain = bool(answer.get("abstain"))
selected = answer.get("selected_ncm")
selected = None if selected in (None, "") else str(selected)
try:
confidence = float(answer.get("confidence"))
except (TypeError, ValueError):
raise LlmAnswerError("confidence must be numeric")
if not 0.0 <= confidence <= 1.0:
raise LlmAnswerError("confidence out of [0,1]")
codes = {code for code, *_ in item.candidates}
if abstain:
if selected is not None:
raise LlmAnswerError("abstain must carry selected_ncm=null")
else:
if selected is None:
raise LlmAnswerError("non-abstain requires selected_ncm")
if selected not in codes:
raise LlmAnswerError(f"selected_ncm {selected!r} is outside the candidate list")
return {"work_item_id": item.work_item_id, "selected_ncm": selected, "confidence": confidence, "abstain": abstain, "reason_short": str(answer.get("reason_short") or "")[:200]}
class OpenAICompatibleProvider:
"""Talks to any OpenAI-compatible /chat/completions endpoint. `transport` is
injectable for tests; production default uses httpx with the key from settings only."""
name = "openai-compatible"
prompt_version = PROMPT_VERSION_LLM
def __init__(self, api_key: str, model: str, base_url: str = "https://api.openai.com/v1", transport=None, timeout: float = 120.0):
self.api_key = api_key
self.model = model
self.base_url = base_url.rstrip("/")
self.timeout = timeout
self._transport = transport or self._httpx_transport
self.http_calls = 0
self.retries = 0
self.usage = {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0}
def _httpx_transport(self, payload: dict) -> dict:
import httpx
response = httpx.post(f"{self.base_url}/chat/completions", json=payload, headers={"Authorization": f"Bearer {self.api_key}"}, timeout=self.timeout)
response.raise_for_status()
return response.json()
def list_models(self) -> list:
"""GET /models with the account key — validates auth/base URL/model availability
without spending completion tokens. Never raises with the key attached."""
import httpx
response = httpx.get(f"{self.base_url}/models", headers={"Authorization": f"Bearer {self.api_key}"}, timeout=self.timeout)
response.raise_for_status()
data = response.json()
return sorted(str(item.get("id")) for item in data.get("data", []) if item.get("id"))
def _request(self, messages: list, max_retries: int = 2) -> dict:
payload = {"model": self.model, "messages": messages, "temperature": 0}
last_error = None
for attempt in range(max_retries + 1):
self.http_calls += 1
if attempt:
self.retries += 1
try:
data = self._transport(payload)
usage = data.get("usage") or {}
self.usage["input_tokens"] += int(usage.get("prompt_tokens") or 0)
self.usage["output_tokens"] += int(usage.get("completion_tokens") or 0)
cached = usage.get("prompt_tokens_details") or {}
self.usage["cached_tokens"] += int(cached.get("cached_tokens") or 0)
return data
except Exception as error: # transport-level failure only; JSON contract errors are not retried
last_error = error
raise RuntimeError(f"LLM transport failed after {max_retries + 1} attempts: {last_error}")
def select_batch(self, items: list[LlmEvalItem]) -> list[dict]:
"""One HTTP call for the whole batch. Raises LlmAnswerError on any 1:1/schema violation
so the harness can shrink the batch size."""
user_content = json.dumps([item_prompt_payload(item) for item in items], ensure_ascii=False)
data = self._request([{"role": "system", "content": _SYSTEM_PROMPT}, {"role": "user", "content": user_content}])
content = data["choices"][0]["message"]["content"]
content = re.sub(r"^```(?:json)?|```$", "", content.strip(), flags=re.MULTILINE).strip()
answers = json.loads(content)
if not isinstance(answers, list) or len(answers) != len(items):
raise LlmAnswerError(f"expected {len(items)} answers, got {len(answers) if isinstance(answers, list) else type(answers).__name__}")
return [validate_answer(answer, item) for answer, item in zip(answers, items)]
+28
View File
@@ -0,0 +1,28 @@
"""Deterministic prompt compaction that preserves candidate evidence."""
import json
def estimate_tokens(payload) -> int:
return max(1, len(json.dumps(payload, ensure_ascii=False, sort_keys=True)) // 4)
def compact_payload(payload):
"""Remove repeated metadata only; candidate codes/descriptions are untouched."""
compact = dict(payload)
compact.pop("debug", None)
compact.pop("duplicate_examples", None)
candidates = compact.get("official_candidates")
if isinstance(candidates, list):
seen = set()
compact["official_candidates"] = [
row for row in candidates
if not (row.get("ncm") in seen or seen.add(row.get("ncm")))
]
return compact
def measure_payloads(payloads):
before = sum(estimate_tokens(payload) for payload in payloads)
after = sum(estimate_tokens(compact_payload(payload)) for payload in payloads)
return {"items": len(payloads), "tokens_before": before, "tokens_after": after, "reduction_percent": round((1 - after / before) * 100, 2) if before else 0.0}
+155
View File
@@ -0,0 +1,155 @@
"""Provider abstraction with strict structured output and a hard budget guard.
The fiscal engine is never coupled to a vendor: it talks to AISemanticProvider.
V1 ships BaselineProvider (deterministic, zero-cost) used by the evaluation
harness and as the no-credential fallback; real providers subclass the same
contract and are only invoked with AI_ENABLED=true and credentials in the Hub.
"""
from dataclasses import dataclass, field
from datetime import datetime, timezone
from ..classification import ENGINE_VERSION
from .fingerprint import overlap_score
PROMPT_VERSION = "ai-semantic-v1"
class SemanticOutputError(ValueError):
"""Raised when a provider answer violates the strict structured-output schema."""
class ProviderTransportError(RuntimeError):
"""A retryable failure while contacting a semantic provider."""
class ProviderContractError(ValueError):
"""A non-retryable provider response or protocol violation."""
@dataclass
class StructuredResult:
task_type: str
input_fingerprint: str
selected_candidate: str # candidate code, or "NONE"
confidence: float
reason: str
candidate_ids_seen: list[str]
model: str
prompt_version: str
created_at: str
dataset_versions: dict = field(default_factory=dict)
engine_version: str = ENGINE_VERSION
def to_dict(self):
return {
"task_type": self.task_type,
"input_fingerprint": self.input_fingerprint,
"selected_candidate": self.selected_candidate,
"confidence": self.confidence,
"reason": self.reason,
"candidate_ids_seen": self.candidate_ids_seen,
"model": self.model,
"prompt_version": self.prompt_version,
"created_at": self.created_at,
"dataset_versions": self.dataset_versions,
"engine_version": self.engine_version,
}
def result_from_dict(data) -> StructuredResult:
"""Fail-closed schema check: the selection must be NONE or one of the seen candidates."""
result = StructuredResult(
task_type=str(data["task_type"]),
input_fingerprint=str(data["input_fingerprint"]),
selected_candidate=str(data["selected_candidate"]),
confidence=float(data["confidence"]),
reason=str(data["reason"]),
candidate_ids_seen=[str(item) for item in data["candidate_ids_seen"]],
model=str(data["model"]),
prompt_version=str(data["prompt_version"]),
created_at=str(data["created_at"]),
dataset_versions=dict(data.get("dataset_versions") or {}),
)
if result.selected_candidate != "NONE" and result.selected_candidate not in result.candidate_ids_seen:
raise SemanticOutputError(f"selected_candidate {result.selected_candidate!r} is not among the official candidates")
if not 0.0 <= result.confidence <= 1.0:
raise SemanticOutputError("confidence must be within [0,1]")
return result
class AISemanticProvider:
"""Contract every provider must honour. `select` receives ONLY official candidates."""
name = "abstract"
model = "abstract"
def select(self, work_item, candidates) -> StructuredResult:
raise NotImplementedError
def estimate(self, work_items) -> dict:
raise NotImplementedError
def _score(text: str, official: str) -> float:
return overlap_score(text, official)
class BaselineProvider(AISemanticProvider):
"""Deterministic zero-cost baseline: highest token-overlap candidate, or NONE.
Used for the evaluation pipeline and as the mandatory no-credential fallback."""
name = "baseline"
model = "baseline-deterministic-v1"
def select(self, work_item, candidates, dataset_versions=None) -> StructuredResult:
description = work_item.payload.get("description", "") if hasattr(work_item, "payload") else ""
scored = [(code, official, _score(description, official)) for code, official in [(str(c[0]), c[1]) for c in candidates]]
best = max(scored, key=lambda row: row[2]) if scored else None
selected, _, score = best if best else ("NONE", "", 0.0)
if not scored or score <= 0.0:
selected = "NONE"
return StructuredResult(
task_type=work_item.task_type,
input_fingerprint=work_item.dedup_key,
selected_candidate=selected,
confidence=round(score, 4),
reason="baseline token-overlap top-1" if selected != "NONE" else "no token overlap with any official candidate",
candidate_ids_seen=[str(c[0]) for c in candidates],
model=self.model,
prompt_version=PROMPT_VERSION,
created_at=datetime.now(timezone.utc).isoformat(),
dataset_versions=dict(dataset_versions or {}),
)
def estimate(self, work_items) -> dict:
return {"input_tokens": 0, "output_tokens": 0, "estimated_cost": 0.0}
class BudgetGuard:
"""Refuses execution when the estimated cost exceeds AI_MAX_RUN_COST_USD.
Never falls back to a more expensive model. Prices are operator-configured — when
absent, cost is UNKNOWN: the residual run is refused, but an explicitly limited
benchmark (e.g. the 65-item GOLD eval) may opt in with allow_unknown_cost."""
def __init__(self, max_run_cost_usd: float, price_in_per_m: float | None, price_out_per_m: float | None):
self.max_run_cost_usd = float(max_run_cost_usd)
self.price_in_per_m = None if price_in_per_m is None else float(price_in_per_m)
self.price_out_per_m = None if price_out_per_m is None else float(price_out_per_m)
def estimate(self, items_in_tokens: int, items_out_tokens: int) -> dict:
if self.price_in_per_m is None or self.price_out_per_m is None:
return {"input_tokens": items_in_tokens, "output_tokens": items_out_tokens, "estimated_cost": None, "cost_unknown": True}
cost = (items_in_tokens / 1_000_000) * self.price_in_per_m + (items_out_tokens / 1_000_000) * self.price_out_per_m
return {"input_tokens": items_in_tokens, "output_tokens": items_out_tokens, "estimated_cost": round(cost, 6)}
def authorize(self, plan: dict, allow_unknown_cost: bool = False) -> bool:
if plan.get("estimated_cost") is None:
return allow_unknown_cost
return plan["estimated_cost"] <= self.max_run_cost_usd
def authorize_usage(self, input_tokens: int, output_tokens: int, *, prior_cost_usd: float = 0.0,
allow_unknown_cost: bool = False) -> bool:
"""Apply the same guard to actual usage before accepting a provider result."""
plan = self.estimate(input_tokens, output_tokens)
if plan.get("estimated_cost") is None:
return allow_unknown_cost
return float(prior_cost_usd) + plan["estimated_cost"] <= self.max_run_cost_usd
+101
View File
@@ -0,0 +1,101 @@
"""Pure, review-first questionnaire helpers for local fiscal context."""
import csv
import io
import json
from central.app.context_keys import validate_entry
def _options(row):
value = row.get("VALID_OPTIONS", [])
if isinstance(value, str):
try:
value = json.loads(value)
except json.JSONDecodeError:
value = [x.strip() for x in value.split("|") if x.strip()]
return list(value) if isinstance(value, (list, tuple, set)) else [value]
def build_questionnaire(rows):
result = []
for row in rows:
scope = str(row.get("SCOPE", row.get("scope", ""))).strip().upper()
key = str(row.get("KEY", row.get("key", ""))).strip()
scope_key = str(row.get("GROUP") or row.get("NCM") or row.get("scope_key") or "").strip()
result.append({
"question_id": str(row.get("QUESTION_ID", row.get("question_id", ""))).strip(),
"database": str(row.get("DATABASE", row.get("database", ""))).strip(),
"scope": scope,
"scope_key": scope_key if scope != "COMPANY" else "",
"group": str(row.get("GROUP") or "").strip(),
"key": key,
"question": row.get("QUESTION", row.get("question", "")),
"options": _options(row),
"affected": {"products": int(row.get("PRODUCTS_AFFECTED") or row.get("products_affected") or 0), "cells": int(row.get("CELLS") or row.get("cells") or 0)},
"examples": row.get("EXAMPLES", row.get("examples", "")),
"consequence": row.get("WHAT_IT_WILL_RESOLVE", row.get("consequence", "")),
})
return sorted(result, key=lambda q: (q["database"], q["scope"], q["scope_key"], q["key"], q["question_id"]))
def group_questions(questions):
"""Group equivalent human decisions while retaining every affected question ID."""
groups = {}
for question in questions:
key = (question["database"], question["group"], question["scope_key"], question["key"], question["question"], tuple(question["options"]))
group = groups.setdefault(key, {
"group_id": "GROUP-" + str(len(groups) + 1),
"database": question["database"],
"group": question["group"],
"scope": question["scope"],
"scope_key": question["scope_key"],
"key": question["key"],
"question": question["question"],
"options": question["options"],
"question_ids": [],
"affected": {"products": 0, "cells": 0},
"examples": [],
"consequence": question["consequence"],
})
group["question_ids"].append(question["question_id"])
group["affected"]["products"] += question["affected"]["products"]
group["affected"]["cells"] += question["affected"]["cells"]
if question["examples"] and len(group["examples"]) < 5:
group["examples"].append(str(question["examples"]))
return list(groups.values())
def validate_answers(questions, answers):
entries, effects, rollback_plan = [], [], []
by_id = {q["question_id"]: q for q in questions}
for question_id, value in answers.items():
if question_id not in by_id:
raise ValueError(f"unknown question {question_id!r}; nothing was persisted")
q = by_id[question_id]
value = str(value).strip()
if q["options"] and value not in q["options"]:
raise ValueError(f"{question_id}: value must be one of {q['options']}; nothing was persisted")
entry, error = validate_entry({"scope": q["scope"], "scope_key": q["scope_key"], "key": q["key"], "value": value})
if error:
raise ValueError(f"{question_id}: {error}; nothing was persisted")
entry["company_key"] = q["database"]
entries.append(entry)
effect = {"question_id": question_id, "database": q["database"], "key": q["key"], "affected": q["affected"], "behavior": "context_entry"}
if q["key"] == "entry_scope" and value == "MIXED_OR_DYNAMIC":
effect["behavior"] = "no_static_cfop_entrada"
effects.append(effect)
if q["key"] == "store_ipi_profile" and value == "LEAVE_NOT_APPLICABLE":
rollback_plan.append({"database": q["database"], "field": "IPI", "affected_cells": q["affected"]["cells"]})
return {"entries": entries, "effects": effects, "rollback_plan": rollback_plan}
def render_json(result):
return json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
def render_csv(result):
output = io.StringIO()
writer = csv.DictWriter(output, fieldnames=["company_key", "scope", "scope_key", "key", "value", "reason"])
writer.writeheader()
writer.writerows({k: entry.get(k, "") for k in writer.fieldnames} for entry in result["entries"])
return output.getvalue()
+245
View File
@@ -0,0 +1,245 @@
"""Semantic candidate retrieval V2 — deterministic, zero LLM.
Three retrieval sources merged into one ranking:
A. OFFICIAL: hierarchical search over the official NCM tree. Every 8-digit code is
enriched with its ancestors' descriptions (chapter/position/subposition), so the
very short legal text carries the vocabulary of its path (section 6).
B. KB NEIGHBORS: the Knowledge Base acts as a commercial->official bridge — products
whose commercial description resembles the query contribute their VALIDATED NCMs,
weighted by similarity, tenant_count, observation_count and agreement (sections 2-3).
C. HYBRID: single final ranking; the KB is only evidence for CANDIDATES, never a
majority-vote fiscal decision.
Abbreviation lexicon (section 8) is derived statistically from the KB itself; commercial
normalization (section 7) strips units/sizes/colors/refs deterministically.
"""
from dataclasses import dataclass
from sqlalchemy import select
from sqlalchemy.orm import Session
from ..models import Dataset, KbKnowledge, KbObservation, KbProduct, NcmEntry, Tenant
from .fingerprint import normalize_description, token_stems
# ---------------- commercial normalization (section 7) ----------------
_UNITS = {"UN", "UNID", "UNIDADE", "CX", "CAIXA", "FD", "FDO", "FARDO", "PCT", "PACOTE", "PKG",
"KG", "G", "GR", "GRAMA", "MG", "ML", "L", "LT", "LITRO", "LATA", "GARRAFA", "PET",
"PC", "PECA", "DZ", "DUZIA", "PAR", "KIT", "JG", "PARA", "COM", "SEM", "C", "S"}
_DISCARD = _UNITS | {"MASC", "MASCULINO", "FEM", "FEMININO", "INF", "INFANTIL", "TAM", "TAMANHO",
"COR", "REF", "COD", "CODIGO", "MOD", "MODELO", "NOVA", "NOVO", "TIPO"}
_LEXEME_STRIP_PREFIXES = ("1", "2", "3", "4", "5", "6", "7", "8", "9", "0", ".")
def _is_size_or_code(token: str) -> bool:
if token.isdigit():
return True
digits = "".join(char for char in token if char.isdigit())
letters = "".join(char for char in token if char.isalpha())
return bool(digits) and letters in _UNITS # "1KG", "500ML", "2LT"
def commercial_tokens(description) -> set:
"""Stable commercial lexemes: units/sizes/colors/refs removed, stems kept."""
tokens = []
skip_next = False
for token in normalize_description(description).split():
if skip_next:
skip_next = False
continue
if token in ("TAM", "COR", "REF", "COD", "MOD"):
skip_next = True
continue
if token in _DISCARD or _is_size_or_code(token):
continue
tokens.append(token)
return {_stem_safe(token) for token in tokens}
def _stem_safe(token: str) -> str:
from .fingerprint import _stem
return _stem(token)
def expand_stems(stems: set, lexicon: dict) -> set:
expanded = set(stems)
for stem in stems:
target = lexicon.get(stem)
if target:
expanded.add(target)
return expanded
def derive_abbreviation_lexicon(corpus: dict, min_support: int = 3) -> dict:
"""corpus: {commercial_description: ncm}. Statistical abbreviations only: a short token
(3-5 chars) that is a strict prefix of a longer token, where BOTH forms occur across
distinct products that share the same NCM. Support = (#products with the short form) +
(#products with the long form); accepted when support >= min_support and both >= 1."""
by_ncm: dict = {}
for description, ncm in corpus.items():
tokens = set(normalize_description(description).split())
by_ncm.setdefault(ncm, []).append(tokens)
best: dict = {}
for descriptions in by_ncm.values():
if len(descriptions) < 2:
continue
all_tokens = set().union(*descriptions)
for short in all_tokens:
if not (3 <= len(short) <= 5):
continue
short_count = sum(1 for tokens in descriptions if short in tokens)
if short_count < 1:
continue
for long_token in all_tokens:
if len(long_token) > len(short) and long_token.startswith(short):
long_count = sum(1 for tokens in descriptions if long_token in tokens)
support = short_count + long_count
if support >= min_support and long_count >= 1 and short_count >= 1:
current = best.get(short)
if current is None or support > current[1]:
best[short] = (long_token, support)
return {short: target for short, (target, _) in best.items()}
# ---------------- enriched official corpus (sections 5-6) ----------------
@dataclass
class EnrichedNcmCorpus:
documents: dict # code -> enriched stems set
descriptions: dict # code -> official description
positions: dict # 4-digit position -> enriched stems (chapter + position)
children: dict # 4-digit position -> [8-digit codes]
@staticmethod
def build(session: Session) -> "EnrichedNcmCorpus":
dataset = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
if dataset is None:
return EnrichedNcmCorpus({}, {}, {}, {})
entries = session.scalars(select(NcmEntry).where(NcmEntry.dataset_id == dataset.id)).all()
by_code = {entry.code: entry.description for entry in entries}
documents, positions, children = {}, {}, {}
for code, description in by_code.items():
if len(code) != 8:
continue
ancestors = [text for text in (by_code.get(code[:2]), by_code.get(code[:4]), by_code.get(code[:6])) if text]
documents[code] = token_stems(description) | set().union(*[token_stems(text) for text in ancestors]) if ancestors else token_stems(description)
positions.setdefault(code[:4], token_stems(by_code.get(code[:4], "")) | token_stems(by_code.get(code[:2], "")))
children.setdefault(code[:4], []).append(code)
return EnrichedNcmCorpus(documents, by_code, positions, children)
def document(self, code: str) -> str:
return " ".join(sorted(self.documents.get(code, set())))
def items(self):
return list(self.documents.items())
def _score(stems: set, document_stems: set, idf: dict | None = None) -> float:
if not stems:
return 0.0
if idf:
matched = sum(idf.get(stem, 1.0) for stem in stems if stem in document_stems)
total = sum(idf.get(stem, 1.0) for stem in stems)
return matched / total
return len(stems & document_stems) / len(stems)
def hierarchical_candidates(corpus: EnrichedNcmCorpus, description, k: int = 20, max_positions: int = 15):
"""Two-stage: rank 4-digit positions first, then their 8-digit descendants (section 5)."""
stems = commercial_tokens(description)
if not stems:
return []
position_scores = sorted(((_score(stems, doc), position) for position, doc in corpus.positions.items()), reverse=True)
ranked = []
for position_score, position in position_scores[:max_positions]:
if position_score <= 0:
continue
for code in corpus.children.get(position, []):
code_score = _score(stems, corpus.documents[code])
if code_score > 0:
ranked.append((code, corpus.descriptions.get(code, ""), round(code_score, 4)))
ranked.sort(key=lambda row: (-row[2], row[0]))
return ranked[:k]
# ---------------- KB neighbor bridge (sections 2-3) ----------------
@dataclass
class KbNeighborIndex:
entries: list # (product_id, stems, ncm, tenant_ids:set, observation_count, has_conflict)
@staticmethod
def build(session: Session) -> "KbNeighborIndex":
validated = {}
for knowledge in session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "validated_ncm")).all():
validated.setdefault(knowledge.kb_product_id, set()).add(knowledge.value)
conflicts = {product_id: len(values) > 1 for product_id, values in validated.items()}
products = {product.id: product for product in session.scalars(select(KbProduct)).all()}
observations = session.scalars(select(KbObservation)).all()
per_product_tenants: dict = {}
for observation in observations:
per_product_tenants.setdefault(observation.kb_product_id, set()).add(observation.tenant_id)
entries = []
for product_id, values in validated.items():
if conflicts.get(product_id):
continue # conflicting knowledge never feeds neighbors
for value in values:
product = products.get(product_id)
if product is None or not product.normalized_description:
continue
tenants = per_product_tenants.get(product_id, set())
entries.append((product_id, commercial_tokens(product.normalized_description), value, tenants, len(tenants), False))
return KbNeighborIndex(entries)
def corpus(self) -> dict:
return {" ".join(sorted(stems)): ncm for _, stems, ncm, _, _, _ in self.entries}
def kb_neighbor_candidates(index: KbNeighborIndex, description, k: int = 10, exclude_product_id: int | None = None):
"""Aggregate NCMs of lexically similar known products. `exclude_product_id` implements
leave-one-out evaluation: the queried product never retrieves itself. Returns
(ncm, neighbor_score, observation_count, tenant_count) — agreement is the union of
distinct tenants behind the contributing neighbors, reported as evidence, never used
as an automatic majority decision."""
stems = commercial_tokens(description)
if not stems:
return []
best: dict = {}
for product_id, neighbor_stems, ncm, tenant_ids, observation_count, _ in index.entries:
if exclude_product_id is not None and product_id == exclude_product_id:
continue
similarity = _score(stems, neighbor_stems)
if similarity <= 0:
continue
current = best.get(ncm)
if current is None:
best[ncm] = [ncm, round(similarity, 4), observation_count, set(tenant_ids)]
else:
current[1] = max(current[1], round(similarity, 4))
current[3] |= set(tenant_ids)
ranked = sorted(((row[0], row[1], row[2], len(row[3])) for row in best.values()), key=lambda row: (-row[1], -row[3], row[0]))
return ranked[:k]
# ---------------- hybrid ranking (section 4) ----------------
OFFICIAL_WEIGHT, KB_WEIGHT = 0.3, 0.7 # tuned on the V2 benchmark (KB bridge carries the signal;
# the official component breaks ties and covers products absent from the KB)
def hybrid_candidates(corpus: EnrichedNcmCorpus, index: KbNeighborIndex, description, k: int = 20, lexicon: dict | None = None, exclude_product_id: int | None = None):
stems = expand_stems(commercial_tokens(description), lexicon or {})
official = hierarchical_candidates(corpus, " ".join(sorted(stems)) or description, k=k)
neighbors = kb_neighbor_candidates(index, description, k=k, exclude_product_id=exclude_product_id)
combined: dict = {}
for code, text, score in official:
combined[code] = [code, text, OFFICIAL_WEIGHT * score, score, 0.0, 0]
for code, similarity, observation_count, tenant_count in neighbors:
row = combined.setdefault(code, [code, "", 0.0, 0.0, 0.0, 0])
row[2] += KB_WEIGHT * similarity
row[4] = similarity
row[5] = tenant_count
ranked = sorted(combined.values(), key=lambda row: (-round(row[2], 4), row[0]))
return [(row[0], row[1], round(row[2], 4), row[3], row[4], row[5]) for row in ranked[:k]]
+73
View File
@@ -0,0 +1,73 @@
"""Retrieval V3 evidence layer. It ranks candidates only; it never decides fiscal truth."""
import hashlib
import json
from difflib import SequenceMatcher
from .fingerprint import normalize_description, token_stems
from .retrieval import commercial_tokens, hybrid_candidates
def _char_similarity(left, right):
return SequenceMatcher(None, normalize_description(left), normalize_description(right)).ratio()
def retrieval_v3_candidates(corpus, index, description, metadata=None, k=20, lexicon=None, exclude_product_id=None):
"""Add optional product evidence to V2 while preserving V2 as the safe baseline."""
metadata = metadata or {}
base = hybrid_candidates(corpus, index, description, k=max(k, 50), lexicon=lexicon, exclude_product_id=exclude_product_id)
gtin_ncm = str(metadata.get("gtin_ncm") or "")
group_ncms = {str(value) for value in (metadata.get("group_ncms") or [])}
rarity = metadata.get("token_rarity") or {}
query_stems = commercial_tokens(description)
rows = []
for code, text, score, official_score, kb_score, tenant_count in base:
evidence = 0.0
if gtin_ncm and code == gtin_ncm:
evidence += 1.0
if code in group_ncms:
evidence += 0.05
matched_rarity = sum(float(rarity.get(token, 1.0)) for token in query_stems if token in token_stems(text))
evidence += min(0.05, matched_rarity / 100.0)
rows.append((code, text, round(score + evidence, 4), official_score, kb_score, tenant_count))
rows.sort(key=lambda row: (-row[2], row[0]))
return rows[:k]
def candidate_fingerprint(payload):
return hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest()
def prepare_candidate_set(work_item, candidates, *, dataset_versions=None, gtin_evidence=None, kb_neighbors=None, hierarchical_context=None):
"""Build the final LLM-ready evidence package without selecting a candidate."""
payload = {
"task_type": work_item.task_type,
"dedup_key": work_item.dedup_key,
"normalized_product": normalize_description(work_item.payload.get("description", "")),
"top20_candidate_ncms": [
{"ncm": row[0], "official_description": row[1], "score": row[2], "official_score": row[3], "kb_score": row[4], "tenant_count": row[5]}
for row in candidates[:20]
],
"official_descriptions": {row[0]: row[1] for row in candidates[:20]},
"hierarchical_context": hierarchical_context or {},
"kb_neighbors": kb_neighbors or [],
"gtin_evidence": gtin_evidence or {},
"dataset_versions": dict(dataset_versions or {}),
}
payload["input_fingerprint"] = candidate_fingerprint(payload)
return payload
def compact_product_role_context(rows, limit=12):
"""Keep representative role evidence bounded; no role is inferred."""
unique = []
seen = set()
for row in rows:
description = normalize_description(row.get("description"))
if not description or description in seen:
continue
seen.add(description)
unique.append({"description": description, "group": row.get("group"), "subgroup": row.get("subgroup"), "operations": row.get("operations", [])})
if len(unique) >= limit:
break
return {"examples": unique, "evidence_only": True, "decision": None}
+67
View File
@@ -0,0 +1,67 @@
"""Residual-question dedup: group pending cells by the KNOWLEDGE needed, not per cell.
- INVALID_NCM -> one work item per product fingerprint (same product in N databases collapses)
- CEST/EX_TIPI -> one work item per (doubt, NCM): all databases sharing that NCM collapse
- PRODUCT_ROLE -> tenant-specific by contract: one work item per (company, group)
- Everything else (origin, ST retention, company profile, duplicate key) never becomes
an AI work item.
"""
from dataclasses import dataclass, field
_INVALID_NCM, _ROLE = "ncm", "product_role"
@dataclass
class WorkItem:
task_type: str
dedup_key: str
question_refs: list = field(default_factory=list)
cells_covered: int = 0
payload: dict = field(default_factory=dict)
kb_product_id: int | None = None
def build_work_items(questions, product_fingerprints=None, canonical_products=None) -> list[WorkItem]:
"""`product_fingerprints` maps "{DATABASE}:{erp_code}" -> KB fingerprint (from the seed pass)."""
product_fingerprints = product_fingerprints or {}
canonical_products = canonical_products or {}
items: dict = {}
def _get(task_type, dedup_key, ref, cells):
item = items.get(dedup_key)
if item is None:
item = WorkItem(task_type=task_type, dedup_key=dedup_key)
items[dedup_key] = item
item.question_refs.append(ref)
item.cells_covered += cells
return item
for question in questions:
key = question.get("KEY")
scope = question.get("SCOPE")
cells = int(question.get("CELLS") or question.get("PRODUCTS_AFFECTED") or 0)
ref = question.get("QUESTION_ID") or f"{question.get('DATABASE')}:{scope}:{question.get('NCM') or question.get('GROUP')}:{key}"
if key in ("cest", "ex_tipi") and scope == "NCM":
ncm = str(question.get("NCM") or "").strip()
if ncm:
item = _get(key.upper(), f"{key}:{ncm}", ref, cells)
item.payload.setdefault("ncm", ncm)
item.payload["description"] = question.get("EXAMPLES", "").split(";")[0].strip() if question.get("EXAMPLES") else ""
elif key == _INVALID_NCM and scope == "PRODUCT":
database = question.get("DATABASE", "")
erp_code = str(question.get("DATABASE_CODE") or "").strip()
fingerprint = product_fingerprints.get(f"{database}:{erp_code}")
fingerprint = canonical_products.get(f"{database}:{erp_code}", fingerprint)
dedup = f"invalid_ncm:{fingerprint}" if fingerprint else f"invalid_ncm:{database}:{erp_code}"
item = _get("INVALID_NCM", dedup, ref, cells)
item.payload.setdefault("erp_code", erp_code)
item.payload["description"] = question.get("EXAMPLES", "").split(";")[0].strip() if question.get("EXAMPLES") else ""
elif key == _ROLE and scope == "GROUP":
company = question.get("COMPANY", "")
group = str(question.get("GROUP") or "").strip()
if group:
normalized_group = " ".join(str(group).upper().split())
item = _get("PRODUCT_ROLE", f"product_role:{company}:{normalized_group}", ref, cells)
item.payload.setdefault("company_key", company)
item.payload["group"] = group
return list(items.values())
+475
View File
@@ -0,0 +1,475 @@
"""Authenticated version-one Hub routes."""
import hashlib, json, logging
import uuid
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException, Request, Response
import csv
import io
from sqlalchemy import select
from .auth import authenticate, bootstrap
from .classification import ENGINE_VERSION, Classifier
from .config import settings
from .datasets.service import DatasetService
from .jobs import SYNC_KINDS, claim_next_job, complete_job, create_job, fail_job, schedule_automatic_sync
from .rtc_audit import job_authorizations, record_rtc_write_audits
from .cfop_ippt_audit import job_authorizations as cfop_ippt_job_authorizations, record_cfop_ippt_write_audits
from .blocked_products import list_blockers, resolve_blocker, resolve_product_blockers, resolve_stale_fiscal_blockers, upsert_blocker, upsert_blockers_bulk
from .context_keys import validate_entry
from .models import AgentSnapshot, BlockedProduct, FiscalContextEntry, ClassificationCache, ClassificationRequest, ClassificationResult, Dataset, Installation, Job
from .events import record_event
from .installation_identity import compute_agent_health, compute_version_drift, reference_agent_version, sanitize_registration
from .control_plane import list_jobs
from .human_auth import require_human
from .enrichment.api import router as enrichment_router
logger = logging.getLogger("gti_fiscal_hub")
def router(session_factory, bootstrap_token):
routes = APIRouter(prefix="/v1")
def principal(request: Request):
header = request.headers.get("Authorization", "")
if not header.startswith("Bearer "): raise HTTPException(401, "Bearer token required")
session = session_factory()
try:
if bootstrap_token: bootstrap(session, bootstrap_token)
agent = authenticate(session, header.removeprefix("Bearer "))
if not agent: raise HTTPException(401, "Invalid token")
return session, agent
except Exception:
session.close(); raise
def decide(payload, identity):
session, agent = identity; service = DatasetService(session)
supplied = payload.get("tenant_id")
if supplied is not None and str(supplied) != str(agent.tenant_id): raise HTTPException(403, "Tenant does not match token")
product = payload.get("product", payload)
# Cheap key first (engine version + dataset versions + input), then the cache; the heavy classifier runs only on a MISS.
engine_version, dataset_versions = ENGINE_VERSION, service.dataset_versions()
canonical = {"tenant": agent.tenant_id, "product": product, "context": payload.get("context", {}), "engine": engine_version, "datasets": dataset_versions}
fingerprint = hashlib.sha256(json.dumps(canonical, sort_keys=True, default=str, separators=(",", ":")).encode()).hexdigest()
cached = session.get(ClassificationCache, fingerprint)
if cached:
result = dict(cached.result); result["provenance"] = {**result["provenance"], "cache_hit": True}; return result
decision = Classifier(service).classify(product)
if decision["engine_version"] != engine_version or decision["dataset_versions"] != dataset_versions: # never store under a key that does not describe the decision
canonical = {**canonical, "engine": decision["engine_version"], "datasets": decision["dataset_versions"]}
fingerprint = hashlib.sha256(json.dumps(canonical, sort_keys=True, default=str, separators=(",", ":")).encode()).hexdigest()
result = {**decision, "request_fingerprint": fingerprint, "provenance": {"cache_hit": False, "dataset_versions": decision["dataset_versions"]}}
request = ClassificationRequest(tenant_id=agent.tenant_id, agent_id=agent.id, fingerprint=fingerprint, payload=payload); session.add(request); session.flush()
session.add(ClassificationResult(request_id=request.id, result=result)); session.add(ClassificationCache(fingerprint=fingerprint, tenant_id=agent.tenant_id, result=result)); session.commit(); return result
@routes.post("/validate/ncm")
def validate_ncm(payload: dict, identity=Depends(principal)):
return decide({"product": {"current_ncm": payload.get("ncm"), "description": ""}, "tenant_id": payload.get("tenant_id")}, identity)
@routes.post("/classify")
def classify(payload: dict, identity=Depends(principal)):
try: return decide(payload, identity)
finally: identity[0].close()
@routes.post("/classify/batch")
def classify_batch(payload: dict, identity=Depends(principal)):
try: return {"results": [decide({**payload, "product": product}, identity) for product in payload.get("products", [])]}
finally: identity[0].close()
@routes.get("/datasets/status")
def status(identity=Depends(principal)):
session, _ = identity
try: return [{"id": item.id, "kind": item.kind, "status": item.status, "sha256": item.sha256, "record_count": item.record_count} for item in session.scalars(select(Dataset)).all()]
finally: session.close()
@routes.get("/reference/cfop/{code}")
def reference_cfop(code: str, identity=Depends(principal)):
session, _ = identity
try:
rows = DatasetService(session).cfop_lookup(code=code)
return {"found": bool(rows), "entries": [{"code": r.code, "title": r.title, "description": r.description} for r in rows]}
finally: session.close()
@routes.get("/reference/ncm_utrib/{ncm}")
def reference_ncm_utrib(ncm: str, identity=Depends(principal)):
session, _ = identity
try:
row = DatasetService(session).ncm_utrib_for_ncm(ncm)
return {"found": row is not None, "utrib_abbreviation": row.utrib_abbreviation if row else None}
finally: session.close()
@routes.get("/reference/anp/{cprod_anp}")
def reference_anp(cprod_anp: str, identity=Depends(principal)):
session, _ = identity
try: return {"found": DatasetService(session).anp_code_exists(cprod_anp)}
finally: session.close()
@routes.get("/reference/fcp/{uf}")
def reference_fcp(uf: str, identity=Depends(principal)):
session, _ = identity
try:
row = DatasetService(session).fcp_for_uf(uf)
return {"found": row is not None, "aliq_1": row.aliq_1 if row else None, "observacao": row.observacao if row else None}
finally: session.close()
@routes.get("/reference/rtc_cclasstrib/{code}")
def reference_rtc_cclasstrib(code: str, identity=Depends(principal)):
session, _ = identity
try:
rows = DatasetService(session).rtc_cclasstrib(code=code)
return {"found": bool(rows), "entries": [{"cst": r.cst, "cclasstrib": r.cclasstrib, "name": r.name, "description": r.description, "tipo_aliquota": r.tipo_aliquota, "p_red_ibs": r.p_red_ibs, "p_red_cbs": r.p_red_cbs, "d_ini_vig": r.d_ini_vig, "d_fim_vig": r.d_fim_vig} for r in rows]}
finally: session.close()
@routes.get("/reference/rtc_cbs_rate/{year}")
def reference_rtc_cbs_rate(year: int, identity=Depends(principal)):
session, _ = identity
try:
row = DatasetService(session).rtc_cbs_rate(year)
return {"found": row is not None, "year": year, "rate": row.rate if row else None, "rate_text": row.rate_text if row else None}
finally: session.close()
@routes.get("/reference/fiscal_codes/{domain}")
def reference_fiscal_codes(domain: str, identity=Depends(principal)):
session, _ = identity
try:
rows = DatasetService(session).fiscal_codes(domain.upper())
return {"found": bool(rows), "domain": domain.upper(), "codes": sorted(r.code for r in rows)}
finally: session.close()
@routes.post("/context/import")
def context_import(payload: dict, identity=Depends(principal)):
"""Batch upsert of operator fiscal context: {"company_key": "...", "entries": [{scope, scope_key, key, value, reason}]}. All-or-nothing."""
session, agent = identity
try:
company_key = str(payload.get("company_key") or "").strip()
if not company_key:
raise HTTPException(400, "company_key is required")
entries, errors = [], []
for i, raw in enumerate(payload.get("entries") or []):
entry, error = validate_entry(raw)
(errors.append({"index": i, "error": error, "entry": raw}) if error else entries.append(entry))
if errors:
raise HTTPException(422, {"errors": errors[:50], "error_count": len(errors)})
created = updated = 0
for e in entries:
row = session.scalar(select(FiscalContextEntry).where(FiscalContextEntry.tenant_id == agent.tenant_id, FiscalContextEntry.company_key == company_key,
FiscalContextEntry.scope == e["scope"], FiscalContextEntry.scope_key == e["scope_key"], FiscalContextEntry.key == e["key"]))
if row:
row.value, row.reason, row.updated_at = e["value"], e["reason"], datetime.now(timezone.utc); updated += 1
else:
session.add(FiscalContextEntry(tenant_id=agent.tenant_id, company_key=company_key, source="OPERATOR", **e)); created += 1
session.commit()
return {"created": created, "updated": updated}
finally: session.close()
@routes.get("/context")
def context_list(company_key: str, identity=Depends(principal)):
session, agent = identity
try:
rows = session.scalars(select(FiscalContextEntry).where(FiscalContextEntry.tenant_id == agent.tenant_id, FiscalContextEntry.company_key == company_key)).all()
return {"company_key": company_key, "entries": [{"scope": r.scope, "scope_key": r.scope_key, "key": r.key, "value": r.value, "source": r.source, "reason": r.reason,
"created_at": r.created_at.isoformat() if r.created_at else None, "updated_at": r.updated_at.isoformat() if r.updated_at else None} for r in rows]}
finally: session.close()
@routes.post("/agents/heartbeat")
def heartbeat(payload: dict, identity=Depends(principal)):
session, agent = identity
try:
try:
supplied = sanitize_registration(payload)
except (TypeError, ValueError) as error:
raise HTTPException(422, f"invalid registration metadata: {error}") from error
now = datetime.now(timezone.utc)
agent.last_seen_at = now
agent.last_status = supplied.get("status") or payload.get("status")
if payload.get("schema_fingerprint"):
agent.schema_fingerprint = payload["schema_fingerprint"]
if payload.get("agent_version"):
agent.agent_version = payload["agent_version"]
installation_key = supplied.get("installation_id") or str(uuid.uuid4())
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
if installation is not None and installation.installation_id != installation_key:
raise HTTPException(409, "installation_id conflicts with registered Agent")
if installation is None:
installation = Installation(agent_id=agent.id, tenant_id=agent.tenant_id, installation_id=installation_key, display_name=supplied.get("display_name") or agent.name,
features_configured=True, corrector_enabled=True) # legacy/bootstrap self-registration keeps the pre-existing behaviour
session.add(installation); session.flush()
fields = {"display_name": supplied.get("display_name"), "customer_code": supplied.get("customer_code"), "notes": supplied.get("notes"), "hostname": supplied.get("hostname"), "windows_version": supplied.get("windows_version"), "machine_fingerprint": supplied.get("machine_fingerprint"), "company_code": supplied.get("company_code"), "company_name": supplied.get("company_name"), "company_cnpj_masked": supplied.get("company_cnpj_masked"), "company_uf": supplied.get("company_uf"), "company_crt": supplied.get("company_crt"), "company_contribuinte_ipi": supplied.get("company_contribuinte_ipi"), "firebird_version": supplied.get("firebird_version"), "firebird_ods": supplied.get("firebird_ods"), "database_identifier": supplied.get("database_identifier"), "capabilities": supplied.get("capabilities"), "fiscal_engine_version": supplied.get("fiscal_engine_version"), "dataset_versions": supplied.get("dataset_versions"), "last_status": supplied.get("status") or payload.get("status"), "last_seen_at": now}
for key, value in fields.items():
if value is not None: setattr(installation, key, value)
if supplied.get("started_at"):
try:
installation.started_at = datetime.fromisoformat(str(supplied["started_at"]).replace("Z", "+00:00"))
except ValueError:
raise HTTPException(422, "started_at must be ISO-8601")
snapshot = session.scalar(select(AgentSnapshot).where(AgentSnapshot.installation_id == installation.id))
if snapshot is None:
snapshot = AgentSnapshot(installation_id=installation.id)
session.add(snapshot)
if payload.get("coverage"):
coverage = payload["coverage"]
snapshot.total_products = int(coverage.get("total_products", coverage.get("total", snapshot.total_products)) or 0)
snapshot.fiscally_complete = int(coverage.get("fiscally_complete", coverage.get("complete", snapshot.fiscally_complete)) or 0)
snapshot.safe_blocked = int(coverage.get("safe_blocked", snapshot.safe_blocked) or 0)
snapshot.failed_products = int(coverage.get("failed_products", snapshot.failed_products) or 0)
snapshot.coverage_percent = float(coverage.get("coverage_percent", snapshot.coverage_percent) or 0)
snapshot.blockers = coverage.get("blockers", snapshot.blockers) or {}
snapshot.version_drift = compute_version_drift(agent.agent_version, reference_agent_version(session))
snapshot.agent_health, snapshot.health_reason = compute_agent_health(installation, supplemental=snapshot)
session.add(agent); session.flush()
record_event(session, tenant_id=agent.tenant_id, installation_id=installation.id, agent_id=agent.id, severity="INFO", event_type="HEARTBEAT", message="Agent heartbeat", context={"status": agent.last_status, "agent_version": agent.agent_version})
session.commit()
# Central scheduler: exactly one INITIAL_SYNC after the first heartbeat, then
# one SYNC_INCREMENTAL per interval — idempotent, no local cron/toggle needed.
if agent.enabled:
schedule_automatic_sync(session, agent, installation, snapshot)
logger.info("heartbeat agent_id=%s status=%s last_job_id=%s", agent.id, agent.last_status, payload.get("last_job_id"))
return {"agent_id": agent.id, "installation_id": installation.installation_id, "last_seen_at": agent.last_seen_at.isoformat(), "enabled": agent.enabled,
"processing_enabled": bool(installation.processing_enabled), "sync_interval_seconds": installation.sync_interval_seconds}
finally: session.close()
@routes.post("/agents/register")
def register_agent(payload: dict, identity=Depends(principal)):
return heartbeat({**payload, "status": payload.get("status", "IDLE")}, identity)
@routes.post("/agents/events")
def agent_event(payload: dict, identity=Depends(principal)):
session, agent = identity
try:
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
if installation is None:
raise HTTPException(409, "Agent must heartbeat/register before sending events")
event = record_event(session, tenant_id=agent.tenant_id, installation_id=installation.id, agent_id=agent.id, severity=payload.get("severity", "INFO"), event_type=payload.get("event_type", "HEARTBEAT"), message=payload.get("message", ""), context=payload.get("context") or {}, job_id=payload.get("job_id"), product_code=payload.get("product_code"), blocker_id=payload.get("blocker_id"))
session.commit()
return {"event_id": event.id}
finally: session.close()
@routes.get("/agents/me/identity")
def agent_identity(identity=Depends(principal)):
"""Return only the authenticated Agent's existing installation identity."""
session, agent = identity
try:
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
last_seen = agent.last_seen_at
if last_seen is not None and last_seen.tzinfo is None:
last_seen = last_seen.replace(tzinfo=timezone.utc)
online = last_seen is not None and (datetime.now(timezone.utc) - last_seen).total_seconds() <= settings.agent_online_threshold_seconds
return {
"agent_id": agent.id,
"installation_id": installation.installation_id if installation else None,
"hostname": installation.hostname if installation else None,
"version": agent.agent_version,
"status": "ONLINE" if online else "OFFLINE",
}
finally:
session.close()
@routes.get("/agents/me/authorizations")
def agent_authorizations(identity=Depends(principal)):
"""Current central RTC write authorization of the authenticated Agent's own installation (read by the Agent right before it writes). Nothing else is exposed."""
session, agent = identity
try:
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
if installation is None:
return {"rtc_write_mode": "DISABLED", "cfop_ippt_write_mode": "DISABLED"}
return {**job_authorizations(session, installation), **cfop_ippt_job_authorizations(installation)}
finally:
session.close()
@routes.get("/agents/me")
def agent_status(identity=Depends(principal)):
session, agent = identity
try:
last_seen = agent.last_seen_at
if last_seen is not None and last_seen.tzinfo is None:
last_seen = last_seen.replace(tzinfo=timezone.utc) # SQLite (tests) drops tz info
online = last_seen is not None and (datetime.now(timezone.utc) - last_seen).total_seconds() <= settings.agent_online_threshold_seconds
return {
"agent_id": agent.id, "company_id": agent.tenant_id, "name": agent.name, "enabled": agent.enabled,
"last_seen_at": agent.last_seen_at.isoformat() if agent.last_seen_at else None,
"last_status": agent.last_status, "schema_fingerprint": agent.schema_fingerprint,
"agent_version": agent.agent_version, "status": "ONLINE" if online else "OFFLINE",
}
finally: session.close()
def _blocked_view(row):
return {
"id": row.id, "tenant_id": row.tenant_id, "company": row.company, "product_code": row.product_code,
"installation_db_id": row.installation_id,
"gtin": row.gtin, "description": row.description, "blocker_type": row.blocker_type, "field": row.field,
"reason_code": row.reason_code, "evidence_summary": row.evidence_summary, "first_seen_at": row.first_seen_at.isoformat() if row.first_seen_at else None,
"last_seen_at": row.last_seen_at.isoformat() if row.last_seen_at else None, "retry_count": row.retry_count,
"next_retry_at": row.next_retry_at.isoformat() if row.next_retry_at else None, "resolved_at": row.resolved_at.isoformat() if row.resolved_at else None,
"resolution_source": row.resolution_source,
}
@routes.get("/blockers")
def blockers_list(blocker_type: str | None = None, field: str | None = None, identity=Depends(principal)):
session, agent = identity
try:
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
return {"blockers": [_blocked_view(row) for row in list_blockers(session, agent.tenant_id, blocker_type=blocker_type, field=field, installation_id=installation.id if installation else None)]}
finally:
session.close()
@routes.get("/blockers/export.csv")
def blockers_export(identity=Depends(principal)):
session, agent = identity
try:
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
output = io.StringIO()
writer = csv.writer(output)
writer.writerow(["id", "company", "product_code", "gtin", "blocker_type", "field", "reason_code", "retry_count", "first_seen_at", "last_seen_at"])
for row in list_blockers(session, agent.tenant_id, installation_id=installation.id if installation else None):
writer.writerow([row.id, row.company, row.product_code, row.gtin or "", row.blocker_type, row.field, row.reason_code, row.retry_count, row.first_seen_at.isoformat() if row.first_seen_at else "", row.last_seen_at.isoformat() if row.last_seen_at else ""])
return Response(output.getvalue(), media_type="text/csv", headers={"Content-Disposition": "attachment; filename=blocked-products.csv"})
finally:
session.close()
@routes.post("/blockers/reprocess")
def blockers_reprocess(payload: dict, identity=Depends(principal)):
session, agent = identity
try:
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
rows = list_blockers(session, agent.tenant_id, blocker_type=payload.get("blocker_type"), field=payload.get("field"), installation_id=installation.id if installation else None)
codes = [row.product_code for row in rows if row.product_code]
if not codes:
return {"created": False, "reason": "NO_DUE_BLOCKERS"}
job = create_job(session, agent, "REPROCESS_PRODUCTS", {"codigos": codes, "reason": "BLOCKER_REPROCESS"})
return {"created": True, "job_id": job.id, "products": len(codes)}
finally:
session.close()
@routes.post("/blockers/{blocker_id}/resolve")
def blockers_resolve(blocker_id: int, payload: dict, identity=Depends(principal)):
session, agent = identity
try:
row = session.get(BlockedProduct, blocker_id)
if row is None or row.tenant_id != agent.tenant_id:
raise HTTPException(404, "blocker not found")
if not resolve_blocker(session, blocker_id, payload.get("resolution_source", "OPERATOR")):
raise HTTPException(404, "blocker not found")
return {"resolved": True, "blocker_id": blocker_id}
finally:
session.close()
def _job_view(job):
return {
"job_id": job.id, "agent_id": job.agent_id, "kind": job.kind, "status": job.status,
"payload": job.payload, "created_at": job.created_at.isoformat() if job.created_at else None,
"started_at": job.started_at.isoformat() if job.started_at else None,
"finished_at": job.finished_at.isoformat() if job.finished_at else None,
"attempts": job.attempts, "result_summary": job.result_summary, "error": job.error,
}
def _apply_job_outcome(session, agent, installation, job, result_summary, *, failed):
"""After a job finishes: coverage snapshot (from the Agent's operational
summary), sync bookkeeping and an audit-friendly event. Never stores secrets."""
now = datetime.now(timezone.utc)
snapshot = session.scalar(select(AgentSnapshot).where(AgentSnapshot.installation_id == installation.id))
if snapshot is None:
snapshot = AgentSnapshot(installation_id=installation.id); session.add(snapshot)
snapshot.last_job_status = "FAILED" if failed else "COMPLETED"
snapshot.last_job_at = now
context = {"kind": job.kind}
operational = result_summary.get("operational") if not failed else None
if operational and job.kind in SYNC_KINDS:
total = int(operational.get("TOTAL_PRODUCTS", 0))
complete = int(operational.get("COMPLETE", 0)) + int(operational.get("CORRECTED", 0))
snapshot.total_products = total
snapshot.fiscally_complete = complete
snapshot.safe_blocked = int(operational.get("SAFE_BLOCKED", 0))
snapshot.failed_products = int(operational.get("FAILED_TECHNICAL", 0))
snapshot.coverage_percent = round(100.0 * complete / total, 2) if total else 0.0
snapshot.blockers = {state: int(count) for state, count in operational.items() if state.startswith("BLOCKED_") and count}
snapshot.last_successful_job_at = now
installation.last_sync_at = now
installation.last_sync_changed = int(result_summary.get("products_changed", result_summary.get("products_processed", 0)) or 0)
context.update({"total_products": total, "products_changed": installation.last_sync_changed, "products_processed": result_summary.get("products_processed")})
if failed:
context["error"] = result_summary.get("error")
record_event(session, tenant_id=agent.tenant_id, installation_id=installation.id, agent_id=agent.id, severity="ERROR" if failed else "INFO",
event_type="JOB_FAILED" if failed else "JOB_COMPLETED", message=f"Job {job.kind} {'failed' if failed else 'completed'}", context=context, job_id=job.id)
session.commit()
@routes.post("/jobs")
def create_job_route(payload: dict, identity=Depends(principal)):
session, agent = identity
try:
kind = payload.get("kind")
if kind not in ("REPROCESS_ALL", "REPROCESS_PRODUCTS"):
raise HTTPException(400, "kind must be REPROCESS_ALL or REPROCESS_PRODUCTS")
job_payload = payload.get("payload", {})
if kind == "REPROCESS_PRODUCTS" and not job_payload.get("codigos"):
raise HTTPException(400, "REPROCESS_PRODUCTS requires payload.codigos")
job = create_job(session, agent, kind, job_payload)
logger.info("job created job_id=%s agent_id=%s kind=%s", job.id, agent.id, kind)
return _job_view(job)
finally: session.close()
@routes.get("/jobs/next")
def next_job(identity=Depends(principal)):
session, agent = identity
try:
job = claim_next_job(session, agent)
if job is None: return None
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
if installation is not None:
record_event(session, tenant_id=agent.tenant_id, installation_id=installation.id, agent_id=agent.id, severity="INFO", event_type="JOB_STARTED", message=f"Job {job.kind} started", context={"kind": job.kind}, job_id=job.id)
session.commit()
logger.info("job claimed job_id=%s agent_id=%s kind=%s", job.id, agent.id, job.kind)
view = _job_view(job)
view["authorizations"] = ({**job_authorizations(session, installation), **cfop_ippt_job_authorizations(installation)} if installation is not None
else {"rtc_write_mode": "DISABLED", "cfop_ippt_write_mode": "DISABLED"}) # decided centrally, per installation, default OFF
return view
finally: session.close()
@routes.get("/jobs/global")
def global_jobs(request: Request, page: int = 1, page_size: int = 50, status: str | None = None, kind: str | None = None):
session = session_factory()
try:
require_human(request, session)
return list_jobs(session, page=page, page_size=page_size, status=status, kind=kind)
finally:
session.close()
@routes.get("/jobs/{job_id}")
def get_job(job_id: int, identity=Depends(principal)):
session, agent = identity
try:
job = session.get(Job, job_id)
if job is None or job.agent_id != agent.id: raise HTTPException(404, "job not found")
return _job_view(job)
finally: session.close()
@routes.post("/jobs/{job_id}/complete")
def complete_job_route(job_id: int, payload: dict, identity=Depends(principal)):
session, agent = identity
try:
job = session.get(Job, job_id)
if job is None or job.agent_id != agent.id: raise HTTPException(404, "job not found")
result_summary = payload.get("result_summary", {})
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
pass_started = datetime.now(timezone.utc)
upsert_blockers_bulk(session, agent.tenant_id, list(result_summary.get("blocked_products", [])), installation_id=installation.id if installation else None)
for product_code in result_summary.get("resolved_products", []):
resolve_product_blockers(session, agent.tenant_id, product_code, installation_id=installation.id if installation else None)
if installation is not None:
record_rtc_write_audits(session, tenant_id=agent.tenant_id, installation=installation, agent=agent, job_id=job_id, rtc_auto_apply=result_summary.get("rtc_auto_apply"), evaluated_all=bool((result_summary.get("fiscal_context") or {}).get("evaluated_all")))
record_cfop_ippt_write_audits(session, tenant_id=agent.tenant_id, installation=installation, agent=agent, job_id=job_id, cfop_ippt_apply=result_summary.get("cfop_ippt_apply"))
if installation is not None and (result_summary.get("fiscal_context") or {}).get("evaluated_all"):
resolve_stale_fiscal_blockers(session, agent.tenant_id, installation.id, pass_started)
if not complete_job(session, job, result_summary): raise HTTPException(409, "job is not RUNNING")
if installation is not None:
_apply_job_outcome(session, agent, installation, job, result_summary, failed=False)
logger.info("job completed job_id=%s agent_id=%s result=%s", job_id, agent.id, payload.get("result_summary", {}))
return _job_view(session.get(Job, job_id))
finally: session.close()
@routes.post("/jobs/{job_id}/fail")
def fail_job_route(job_id: int, payload: dict, identity=Depends(principal)):
session, agent = identity
try:
job = session.get(Job, job_id)
if job is None or job.agent_id != agent.id: raise HTTPException(404, "job not found")
if not fail_job(session, job, payload.get("error", "unknown error")): raise HTTPException(409, "job is not RUNNING")
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
if installation is not None:
_apply_job_outcome(session, agent, installation, job, {"error": str(payload.get("error", "unknown error"))[:500]}, failed=True)
logger.warning("job failed job_id=%s agent_id=%s error=%s", job_id, agent.id, payload.get("error", "unknown error"))
return _job_view(session.get(Job, job_id))
finally: session.close()
routes.include_router(enrichment_router(principal))
return routes
+11
View File
@@ -0,0 +1,11 @@
"""Immutable audit events for human control-plane actions."""
from .events import sanitize_event
from .models import AuditEvent
def record_audit(session, *, admin_user_id: int | None, action: str, target_type: str, target_id: str, context: dict | None = None, tenant_id: int | None = None, severity: str = "INFO"):
_, safe_context = sanitize_event(action, context or {})
event = AuditEvent(admin_user_id=admin_user_id, tenant_id=tenant_id, action=action, target_type=target_type, target_id=str(target_id), context=safe_context, severity=severity)
session.add(event)
return event
+17
View File
@@ -0,0 +1,17 @@
"""Bearer authentication storing only SHA-256 token digests."""
import hashlib, hmac
from sqlalchemy import select
from .models import Agent, Tenant
def token_hash(token): return hashlib.sha256(token.encode()).hexdigest()
def bootstrap(session, token):
digest = token_hash(token); agent = session.scalar(select(Agent).where(Agent.token_hash == digest))
if agent: return agent
tenant = session.scalar(select(Tenant).where(Tenant.name == "development")) or Tenant(name="development")
session.add(tenant); session.flush(); agent = Agent(tenant_id=tenant.id, name="development-agent", token_hash=digest); session.add(agent); session.commit(); return agent
def authenticate(session, token):
digest = token_hash(token)
for agent in session.scalars(select(Agent)).all():
if hmac.compare_digest(agent.token_hash, digest):
return agent if agent.enabled else None # a disabled (revoked) Agent must not authenticate
return None
+118
View File
@@ -0,0 +1,118 @@
"""Central blocked-product registry with fingerprint deduplication."""
from datetime import datetime, timezone
from sqlalchemy import select
from .models import BlockedProduct
from .operational_state import blocker_fingerprint
def _fingerprint_of(tenant_id: int, payload: dict, installation_id: int | None) -> str:
supplied_fingerprint = payload.get("blocker_fingerprint")
if installation_id is None:
return supplied_fingerprint or blocker_fingerprint(payload.get("tenant", str(tenant_id)), payload.get("company", ""), payload.get("product_code", ""), payload.get("field", ""), payload.get("reason_code", ""), payload.get("input_fingerprint", ""))
elif supplied_fingerprint:
return blocker_fingerprint(supplied_fingerprint, "", "", "", "", "", installation_id=installation_id)
else:
return blocker_fingerprint(payload.get("tenant", str(tenant_id)), payload.get("company", ""), payload.get("product_code", ""), payload.get("field", ""), payload.get("reason_code", ""), payload.get("input_fingerprint", ""), installation_id=installation_id)
def upsert_blocker(session, tenant_id: int, payload: dict, *, installation_id: int | None = None, commit: bool = True, existing: dict | None = None) -> BlockedProduct:
"""`existing` ({fingerprint: row}) and `commit=False` let a caller upsert thousands of blockers with one lookup per chunk and one commit (see upsert_blockers_bulk)."""
fingerprint = _fingerprint_of(tenant_id, payload, installation_id)
row = existing.get(fingerprint) if existing is not None else session.scalar(select(BlockedProduct).where(BlockedProduct.blocker_fingerprint == fingerprint))
now = datetime.now(timezone.utc)
if row is None:
row = BlockedProduct(tenant_id=tenant_id, installation_id=installation_id, company=payload.get("company", ""), product_code=str(payload.get("product_code", "")), gtin=payload.get("gtin"), description=payload.get("description"), blocker_type=payload.get("blocker_type", "BLOCKED_MISSING_EVIDENCE"), field=payload.get("field", ""), reason_code=payload.get("reason_code", ""), evidence_summary=payload.get("evidence_summary") or {}, input_fingerprint=payload.get("input_fingerprint", ""), blocker_fingerprint=fingerprint, first_seen_at=now, last_seen_at=now)
session.add(row)
if existing is not None:
existing[fingerprint] = row
else:
row.last_seen_at = now
row.evidence_summary = payload.get("evidence_summary") or row.evidence_summary
row.input_fingerprint = payload.get("input_fingerprint", row.input_fingerprint)
if row.installation_id is None and installation_id is not None:
row.installation_id = installation_id
row.resolved_at = None
if commit:
session.commit()
return row
BULK_CHUNK = 1000
def upsert_blockers_bulk(session, tenant_id: int, payloads: list, *, installation_id: int | None = None) -> int:
"""Same semantics as `upsert_blocker` per item, but O(chunks) statements instead of O(products): one IN-lookup per chunk, one commit at the end."""
n = 0
for start in range(0, len(payloads), BULK_CHUNK):
chunk = [p for p in payloads[start:start + BULK_CHUNK] if isinstance(p, dict)]
prints = {_fingerprint_of(tenant_id, p, installation_id) for p in chunk}
existing = {r.blocker_fingerprint: r for r in session.scalars(select(BlockedProduct).where(BlockedProduct.blocker_fingerprint.in_(prints)))} if prints else {}
for payload in chunk:
upsert_blocker(session, tenant_id, payload, installation_id=installation_id, commit=False, existing=existing)
n += 1
session.flush()
session.commit()
return n
def list_blockers(session, tenant_id: int, *, input_fingerprint: str | None = None, blocker_type: str | None = None, field: str | None = None, installation_id: int | None = None):
query = select(BlockedProduct).where(BlockedProduct.tenant_id == tenant_id, BlockedProduct.resolved_at.is_(None))
if installation_id is not None:
query = query.where((BlockedProduct.installation_id == installation_id) | BlockedProduct.installation_id.is_(None))
if input_fingerprint is not None:
query = query.where(BlockedProduct.input_fingerprint != input_fingerprint)
if blocker_type:
query = query.where(BlockedProduct.blocker_type == blocker_type)
if field:
query = query.where(BlockedProduct.field == field)
return list(session.scalars(query).all())
def retry_due(row: BlockedProduct, current_input_fingerprint: str, now: datetime | None = None) -> bool:
now = now or datetime.now(timezone.utc)
if row.input_fingerprint != current_input_fingerprint:
return True
return row.next_retry_at is not None and row.next_retry_at <= now
def resolve_blocker(session, blocker_id: int, source: str) -> bool:
row = session.get(BlockedProduct, blocker_id)
if row is None:
return False
row.resolved_at = datetime.now(timezone.utc)
row.resolution_source = source
session.commit()
return True
def resolve_product_blockers(session, tenant_id: int, product_code: str, source: str = "AUTO_REEVALUATION", *, installation_id: int | None = None) -> int:
query = select(BlockedProduct).where(BlockedProduct.tenant_id == tenant_id, BlockedProduct.product_code == str(product_code), BlockedProduct.resolved_at.is_(None))
if installation_id is not None:
query = query.where((BlockedProduct.installation_id == installation_id) | BlockedProduct.installation_id.is_(None))
rows = session.scalars(query).all()
now = datetime.now(timezone.utc)
for row in rows:
row.resolved_at = now
row.resolution_source = source
if rows:
session.commit()
return len(rows)
FISCAL_BLOCKER_FIELDS = ("PIS", "COFINS", "RTC")
def resolve_stale_fiscal_blockers(session, tenant_id: int, installation_id: int, since: datetime) -> int:
"""After a FULL evaluation of the unified fiscal fields, every open PIS/COFINS/RTC blocker of the installation that this pass did not
re-report (last_seen_at < since) no longer holds: the ERP data changed, or RTC was switched off. Other blocker fields are untouched."""
rows = session.scalars(select(BlockedProduct).where(BlockedProduct.tenant_id == tenant_id, BlockedProduct.installation_id == installation_id,
BlockedProduct.resolved_at.is_(None), BlockedProduct.field.in_(FISCAL_BLOCKER_FIELDS), BlockedProduct.last_seen_at < since)).all()
now = datetime.now(timezone.utc)
for row in rows:
row.resolved_at = now
row.resolution_source = "AUTO_REEVALUATION"
if rows:
session.commit()
return len(rows)
+168
View File
@@ -0,0 +1,168 @@
"""Validated, auditable context resolution for a single blocked product."""
from __future__ import annotations
from datetime import datetime, timezone
from sqlalchemy import select
from .audit import record_audit
from .context_keys import validate_entry
from .models import Agent, BlockedProduct, FiscalContextEntry, Installation, Tenant, Job
WORKFLOW_SCOPES = {"PRODUCT", "GROUP", "COMPANY"}
ENTRY_FIELDS = {"scope", "scope_key", "key", "value", "reason"}
MAX_VALUE_LENGTH = 512
MAX_REASON_LENGTH = 2000
MAX_COMPANY_KEY_LENGTH = 64
class BlockerResolutionUnavailable(LookupError):
"""The blocker exists, but has no proven installation to target safely."""
def validate_resolution_input(payload):
"""Validate only the structured context accepted by the blocker workflow."""
if not isinstance(payload, dict):
return None, "resolution input must be an object"
unexpected = set(payload) - ENTRY_FIELDS
if unexpected:
return None, f"unsupported resolution fields: {sorted(unexpected)}"
if not isinstance(payload.get("value"), str):
return None, "value must be a string"
if not isinstance(payload.get("reason"), str):
return None, "reason must be a string"
if len(payload["value"].encode("utf-8")) > MAX_VALUE_LENGTH:
return None, f"value must be at most {MAX_VALUE_LENGTH} UTF-8 bytes"
if len(payload["reason"].encode("utf-8")) > MAX_REASON_LENGTH:
return None, f"reason must be at most {MAX_REASON_LENGTH} UTF-8 bytes"
normalized, error = validate_entry(payload)
if error:
return None, error
if normalized["scope"] not in WORKFLOW_SCOPES:
return None, "scope must be one of ('PRODUCT', 'GROUP', 'COMPANY')"
if not normalized["reason"]:
return None, "reason is required"
return normalized, None
def _company_key(blocker, installation, tenant):
if installation is not None:
return (installation.company_code or installation.company_name or installation.customer_code or "").strip()
return (blocker.company or (tenant.name if tenant else "")).strip()
def _entry_view(row):
if row is None:
return None
return {
"company_key": row.company_key,
"scope": row.scope,
"scope_key": row.scope_key,
"key": row.key,
"value": row.value,
"source": row.source,
"reason": row.reason,
}
def resolve_blocker_with_context(session, blocker_id, admin, payload, request_ip=None):
"""Persist context, enqueue one product reprocess, and audit atomically."""
normalized, error = validate_resolution_input(payload)
if error:
raise ValueError(error)
try:
blocker = session.get(BlockedProduct, blocker_id)
if blocker is None:
raise LookupError("blocker not found")
installation = None
if blocker.installation_id is not None:
installation = session.get(Installation, blocker.installation_id)
if installation is None:
raise LookupError("blocker installation not found")
if installation.tenant_id != blocker.tenant_id:
raise LookupError("blocker installation tenant mismatch")
agent = session.get(Agent, installation.agent_id)
if agent is None:
raise LookupError("blocker installation agent not found")
if agent.tenant_id != blocker.tenant_id:
raise LookupError("blocker agent tenant mismatch")
else:
raise BlockerResolutionUnavailable(
"Blocker legado de empresa sem vínculo comprovado com um servidor; não é possível oferecer o reprocessamento de contexto com segurança."
)
tenant = session.get(Tenant, blocker.tenant_id)
company_key = _company_key(blocker, installation, tenant)
if not company_key:
raise ValueError("company context is unavailable")
if len(company_key) > MAX_COMPANY_KEY_LENGTH:
raise ValueError(f"company key must be at most {MAX_COMPANY_KEY_LENGTH} characters")
if normalized["scope"] == "PRODUCT" and normalized["scope_key"] != blocker.product_code:
raise ValueError("PRODUCT scope_key must match the blocker product code")
query = select(FiscalContextEntry).where(
FiscalContextEntry.tenant_id == blocker.tenant_id,
FiscalContextEntry.company_key == company_key,
FiscalContextEntry.scope == normalized["scope"],
FiscalContextEntry.scope_key == normalized["scope_key"],
FiscalContextEntry.key == normalized["key"],
)
entry = session.scalar(query)
before = _entry_view(entry)
now = datetime.now(timezone.utc)
if entry is None:
entry = FiscalContextEntry(
tenant_id=blocker.tenant_id,
company_key=company_key,
source="OPERATOR",
updated_at=now,
**normalized,
)
session.add(entry)
else:
entry.value = normalized["value"]
entry.reason = normalized["reason"]
entry.source = "OPERATOR"
entry.updated_at = now
job = Job(agent_id=agent.id, kind="REPROCESS_PRODUCTS", status="PENDING", payload={"codigos": [blocker.product_code]})
session.add(job)
session.flush()
audit = record_audit(
session,
admin_user_id=admin.id,
tenant_id=blocker.tenant_id,
action="RESOLVE_BLOCKER_WITH_CONTEXT",
target_type="blocker",
target_id=str(blocker.id),
context={
"admin_user_id": admin.id,
"tenant_id": blocker.tenant_id,
"client": tenant.name if tenant else None,
"company_key": company_key,
"installation_id": installation.installation_id if installation else None,
"installation_db_id": installation.id if installation else None,
"product_code": blocker.product_code,
"blocker_id": blocker.id,
"key": normalized["key"],
"scope": normalized["scope"],
"scope_key": normalized["scope_key"],
"value": normalized["value"],
"reason": normalized["reason"],
"before": before,
"after": _entry_view(entry),
"reprocess_job_id": job.id,
"request_ip": request_ip,
},
)
session.flush()
session.commit()
return {"blocker_id": blocker.id, "job_id": job.id, "audit_id": audit.id}
except Exception:
session.rollback()
raise
+274
View File
@@ -0,0 +1,274 @@
"""Deterministic, auditable cClassTrib resolver (pure: no Firebird, no network, no probabilities).
resolve_cclasstrib(product, company, ctx, local_catalog) -> Resolution
Nothing here guesses. A class is proposed only when an EXPLICIT rule, backed by the official texts in `rtc_reference_data`
(cClassTrib table 2026-06-22 and LC 214/2025 compiled through LC 227/2026), proves it. Everything else is AMBIGUOUS or INSUFFICIENT_EVIDENCE.
Operation assumption (documented, product-level default): an ordinary domestic sale of the good. Operation-dependent classes (buyer is a public body,
exports, ZFM/ALC, credit presumed, ...) cannot be carried by a product default; when the annex offers such alternatives the answer is AMBIGUOUS.
Rule classes (what a rule is ALLOWED to do)
AUTO_APPLY_SAFE positive, explicit evidence: may be written to the ERP (only when every runtime gate also holds)
VALIDATION_ONLY used to validate/explain, never proposes a value
SUGGESTION_ONLY diagnostic suggestion shown to a human; NEVER authorizes an UPDATE
BLOCKED_BY_EVIDENCE the evidence is missing/ambiguous: safe-block with a human reason
Rules
RTC-R0 RTC disabled in the ERP VALIDATION_ONLY -> NOT_APPLICABLE
RTC-G1 local RTC catalog failed integrity BLOCKED_BY_EVIDENCE -> INSUFFICIENT_EVIDENCE
RTC-R1 regime without an official rule (Simples family, unknown) BLOCKED_BY_EVIDENCE -> INSUFFICIENT_EVIDENCE
RTC-R2 no valid 8-digit NCM (decision engine did not confirm) BLOCKED_BY_EVIDENCE -> INSUFFICIENT_EVIDENCE
RTC-A1 annex without a unique product-default class / conditional text / revoked annex -> AMBIGUOUS
RTC-A2 NCM listed in more than one annex -> AMBIGUOUS
RTC-A3 chapter whose regime depends on the nature/operation of the good -> AMBIGUOUS
RTC-R3 NCM listed in EXACTLY ONE annex with a unique default class, condition-free item, regime normal (CRT 3)
AUTO_APPLY_SAFE -> RESOLVED_WITH_EXPLICIT_RULE
RTC-R4 NCM in NO annex and in no nature-based chapter: full taxation is the residual reading of LC 214 art. 4
SUGGESTION_ONLY -> SUGGESTION_ONLY (000001). Absence of a special regime is NOT positive evidence: never written.
RTC-G2 proposed class missing/different in the local SGV catalog, out of force or unusable in NF-e/NFC-e BLOCKED_BY_EVIDENCE -> INSUFFICIENT_EVIDENCE (RTC_LOCAL_CATALOG_INCOMPATIBLE)
A proposed class must additionally be MATCH_SAFE against the official table (local SGV catalog confirms every attribute that matters), in force today, usable in
NF-e/NFC-e (the SGV's own lookup filter) and carry a CST that exists; otherwise the answer degrades with the exact reason.
"""
from __future__ import annotations
import hashlib
import json
import re
from dataclasses import dataclass, field
from datetime import date
from . import rtc_reference_data as ref
SIMPLES_FAMILY = frozenset({"1", "2", "4"})
RESOLVED_STATUSES = ("RESOLVED_DETERMINISTIC", "RESOLVED_WITH_EXPLICIT_RULE")
STATUSES = RESOLVED_STATUSES + ("SUGGESTION_ONLY", "AMBIGUOUS", "INSUFFICIENT_EVIDENCE", "NOT_APPLICABLE")
RULE_VERSION = "rtc-rules-v2"
AUTO_APPLY_SAFE, VALIDATION_ONLY, SUGGESTION_ONLY, BLOCKED_BY_EVIDENCE = "AUTO_APPLY_SAFE", "VALIDATION_ONLY", "SUGGESTION_ONLY", "BLOCKED_BY_EVIDENCE"
RULE_CLASSES = {"RTC-R0": VALIDATION_ONLY, "RTC-G1": BLOCKED_BY_EVIDENCE, "RTC-R1": BLOCKED_BY_EVIDENCE, "RTC-R2": BLOCKED_BY_EVIDENCE, "RTC-A1": BLOCKED_BY_EVIDENCE, "RTC-A2": BLOCKED_BY_EVIDENCE,
"RTC-A3": BLOCKED_BY_EVIDENCE, "RTC-R3": AUTO_APPLY_SAFE, "RTC-R4": SUGGESTION_ONLY, "RTC-G2": BLOCKED_BY_EVIDENCE}
HUMAN_REASONS = {"SIMPLES_ORIENTATION_PENDING_OFFICIAL_NT": "Enquadramento RTC depende de regra ainda não determinável com segurança.", "COMPANY_REGIME_UNKNOWN": "Regime tributário da empresa não determinado."}
# Goods annexes whose product-default class is unique in the official table (verified by `verify_annex_class_map`).
ANNEX_DEFAULT_CLASS = {"I": "200003", "VII": "200034", "VIII": "200035", "XV": "200014"} # IX is out: 515001 (deferral with the same inputs) is a sibling class
# Chapters where a reduced/special regime is defined by the NATURE of the good (not by a code list): in natura agriculture (art. 137), medicines (arts. 133/146),
# fuels and biofuels (art. 172 ff), medical/accessibility devices and vehicles (arts. 131-132, 143-145, 149), books/periodicals (constitutional immunity),
# gold and other special-regime goods kept out of the residual rule on purpose (conservative).
SENSITIVE_CHAPTERS = frozenset([f"{n:02d}" for n in range(1, 15)] + ["27", "30", "49", "71", "87", "88", "89", "90"]) # 71 gold (financial-asset regime), 88/89 aircraft and vessels
SPECIAL_PREFIXES = ("9619", "2207", "3826", "8703") # art. 147 (menstrual health), biofuel inputs, art. 149 (PcD cars)
@dataclass
class Resolution:
status: str
rule_id: str
reason: str
resolved_cclasstrib: str | None = None
resolved_cst_ibscbs: str | None = None
confidence_type: str = "NONE"
inputs_used: dict = field(default_factory=dict)
evidence: list = field(default_factory=list)
source_references: list = field(default_factory=list)
@property
def rule_class(self) -> str:
return RULE_CLASSES.get(self.rule_id, BLOCKED_BY_EVIDENCE)
@property
def auto_apply_candidate(self) -> bool:
"""Only an AUTO_APPLY_SAFE rule with a proven class may ever become a write. Suggestions never do."""
return self.rule_class == AUTO_APPLY_SAFE and self.status in RESOLVED_STATUSES and bool(self.resolved_cclasstrib)
@property
def suggested_cclasstrib(self) -> str | None:
return self.resolved_cclasstrib if self.status == "SUGGESTION_ONLY" else None
def as_dict(self) -> dict:
return {"status": self.status, "rule_id": self.rule_id, "rule_class": self.rule_class, "rule_version": RULE_VERSION, "reason": self.reason, "resolved_cclasstrib": self.resolved_cclasstrib,
"resolved_cst_ibscbs": self.resolved_cst_ibscbs, "confidence_type": self.confidence_type, "inputs_used": self.inputs_used, "evidence": self.evidence,
"source_references": self.source_references, "auto_apply_candidate": self.auto_apply_candidate}
def _digits(value) -> str:
return re.sub(r"\D", "", "" if value is None else str(value))
def _official_by_code() -> dict:
return {row["code"]: row for row in ref.OFFICIAL_CCLASSTRIB}
def _num(value):
try:
return float(str(value).replace(",", "."))
except (TypeError, ValueError):
return None
# ------------------------------------------------------------------------------------------------- catalog comparison
def compare_catalogs(local_rows: dict, today: date | None = None) -> dict:
"""local_rows: {code: {cst, tipo_aliquota, p_red_ibs, p_red_cbs, ind_nfe, ind_nfce, start, end}} read from the ERP. Never writes anywhere.
Per code (the official table is the reference):
MATCH_SAFE every attribute the local table carries that matters to a product default agrees (CST, reductions, NF-e/NFC-e usability)
MATCH_WITH_NON_MATERIAL_DIFFERENCE only a scheduled local end date (class still in force today) differs
MISMATCH CST, reductions or NF-e/NFC-e usability differ
EXPIRED the official class ended
MISSING_LOCAL official class absent from the SGV catalog (local outdated)
MISSING_OFFICIAL local class the official table does not know"""
today = today or date.today()
official = _official_by_code()
result, details = {}, {}
for code in sorted(set(local_rows) | set(official)):
loc, off = local_rows.get(code), official.get(code)
if off is None:
result[code] = "MISSING_OFFICIAL"
elif off.get("end") and date.fromisoformat(off["end"]) < today:
result[code] = "EXPIRED"
details[code] = ["OFFICIAL_END_PASSED"] + (["PRESENT_LOCALLY"] if loc is not None else [])
elif loc is None:
result[code] = "MISSING_LOCAL"
else:
problems, soft, info = [], [], []
if str(loc.get("cst") or "").strip() != off["cst"]:
problems.append("CST")
if "p_red_ibs" in loc and "p_red_cbs" in loc and (_num(loc.get("p_red_ibs")) != _num(off["p_red_ibs"]) or _num(loc.get("p_red_cbs")) != _num(off["p_red_cbs"])):
problems.append("REDUCTION") # only attributes the local table actually carries are compared
if "ind_nfe" in loc and "ind_nfce" in loc and (bool(loc.get("ind_nfe")) != off["ind_nfe"] or bool(loc.get("ind_nfce")) != off["ind_nfce"]):
problems.append("DOCUMENT_FLAGS")
start, end = str(loc.get("start") or "")[:10], str(loc.get("end") or "")[:10]
if start and date.fromisoformat(start) > today:
problems.append("LOCAL_NOT_IN_FORCE_YET")
elif start and off.get("start") and start != off["start"]:
info.append("VIGENCY_START_DIFFERS_BOTH_IN_FORCE") # informational: the class is in force today on both sides, so the emission effect is identical
if end and date.fromisoformat(end) < today:
problems.append("LOCAL_CLASS_ENDED")
elif end:
soft.append("LOCAL_END_SCHEDULED") # the class still works today; the local table schedules an end the official one does not
result[code] = "MISMATCH" if problems else ("MATCH_WITH_NON_MATERIAL_DIFFERENCE" if soft else "MATCH_SAFE")
if problems or soft or info:
details[code] = problems + soft + info
keys = ("MATCH_SAFE", "MATCH_WITH_NON_MATERIAL_DIFFERENCE", "MISMATCH", "EXPIRED", "MISSING_LOCAL", "MISSING_OFFICIAL")
counts = {k: sum(1 for v in result.values() if v == k) for k in keys}
material = counts["MISMATCH"] + counts["MISSING_OFFICIAL"] # a local class the official table contradicts or does not know
expired_present = sum(1 for d in details.values() if "PRESENT_LOCALLY" in d)
compat = "FULL" if counts["MISSING_LOCAL"] == 0 and expired_present == 0 and material == 0 else ("PARTIAL_LOCAL_OUTDATED" if material == 0 else "MATERIAL_MISMATCH")
return {"per_code": result, "details": details, "counts": counts, "expired_present_locally": expired_present, "compatibility": compat,
"official_catalog": {"sha256": ref.OFFICIAL_CATALOG["sha256"], "publication": ref.OFFICIAL_CATALOG["publication"], "file": ref.OFFICIAL_CATALOG["file"]}}
def local_catalog_hash(local_rows: dict) -> str:
canon = json.dumps({c: {k: (None if v is None else str(v)) for k, v in sorted(r.items())} for c, r in sorted(local_rows.items())}, sort_keys=True)
return hashlib.sha256(canon.encode()).hexdigest()
def verify_annex_class_map() -> dict:
"""Self-check of ANNEX_DEFAULT_CLASS against the official table: the class must exist, be a goods class in force, and be the ONLY official class for that annex
usable in NF-e/NFC-e as the ordinary-sale default. Returns {annex: [problems]} (empty lists = verified)."""
official = _official_by_code()
numbers = {"I": "1", "VII": "7", "VIII": "8", "XV": "15"}
out = {}
for annex, code in ANNEX_DEFAULT_CLASS.items():
row, problems = official.get(code), []
if row is None:
problems.append("CLASS_NOT_IN_OFFICIAL_TABLE")
else:
if str(row.get("anexo")) != numbers[annex]:
problems.append("ANNEX_FIELD_DIFFERS")
if not (row["ind_nfe"] or row["ind_nfce"]):
problems.append("NOT_USABLE_IN_NFE_NFCE")
siblings = [r["code"] for r in ref.OFFICIAL_CCLASSTRIB if str(r.get("anexo")) == numbers[annex] and r["code"] != code and (r["ind_nfe"] or r["ind_nfce"])]
if siblings:
problems.append("SIBLING_CLASSES:" + ",".join(siblings))
out[annex] = problems
return out
# ------------------------------------------------------------------------------------------------- annex lookup
def annex_matches(ncm: str) -> list[dict]:
"""Every annex item (any annex, revoked included) whose code list covers the NCM, with the item text and whether it is a clean (condition-free) item."""
hits = []
for roman, annex in ref.LC214_ANNEXES.items():
for item in annex["items"]:
for prefix in item["prefixes"]:
if ncm.startswith(prefix):
hits.append({"annex": roman, "item": item["item"], "prefix": prefix, "clean": item["clean"], "revoked": annex["revoked"], "text": item["description"][:160]})
break
return hits
def _references(extra=None) -> list:
refs = [{"document": ref.LC214_SOURCE["document"], "url": ref.LC214_SOURCE["url"], "sha256": ref.LC214_SOURCE["sha256"]},
{"document": "cClassTrib (Portal Nacional NF-e)", "publication": ref.OFFICIAL_CATALOG["publication"], "sha256": ref.OFFICIAL_CATALOG["sha256"]}]
return refs + (extra or [])
def _validate_class(code: str, local_rows: dict, compat: dict, today: date):
"""None when the class may be proposed, else the exact reason it may not."""
official = _official_by_code().get(code)
if official is None:
return "CLASS_NOT_IN_OFFICIAL_TABLE"
loc = local_rows.get(code)
if loc is None:
return "CLASS_NOT_IN_LOCAL_CATALOG"
state = compat["per_code"].get(code)
if state != "MATCH_SAFE":
return f"LOCAL_OFFICIAL_{state}"
if official["start"] and date.fromisoformat(official["start"]) > today:
return "CLASS_NOT_YET_IN_FORCE"
if not (official["ind_nfe"] or official["ind_nfce"]):
return "CLASS_NOT_USABLE_IN_NFE_NFCE"
return None
# ------------------------------------------------------------------------------------------------- resolver
def resolve_cclasstrib(product: dict, company: dict | None, ctx: dict, local_rows: dict | None, *, compat: dict | None = None, today: date | None = None) -> Resolution:
"""product: {"CODIGO","CLASSIFICACAO_FISCAL","ncm_valid"(bool)}; company: {"crt"}; ctx: {"rtc_enabled": bool, "local_catalog_valid": bool}."""
today = today or date.today()
ncm = _digits(product.get("CLASSIFICACAO_FISCAL"))
crt = str((company or {}).get("crt") or "").strip()
used = {"ncm": ncm or None, "crt": crt or None, "rtc_enabled": bool(ctx.get("rtc_enabled"))}
if not ctx.get("rtc_enabled"):
return Resolution("NOT_APPLICABLE", "RTC-R0", "RTC_DISABLED_IN_ERP", inputs_used=used)
if not ctx.get("local_catalog_valid"):
return Resolution("INSUFFICIENT_EVIDENCE", "RTC-G1", "LOCAL_RTC_CATALOG_INVALID", inputs_used=used)
if crt in SIMPLES_FAMILY:
return Resolution("INSUFFICIENT_EVIDENCE", "RTC-R1", "SIMPLES_ORIENTATION_PENDING_OFFICIAL_NT", inputs_used=used,
source_references=_references([{"document": "NT 2025.002-RTC v1.51 (Portal NF-e, 04/08/2026)", "note": "IBS/CBS orientations for CRT 1/2/4 pending"},
{"document": "Ato Conjunto RFB/CGIBS n 4/2026 (DOU 31/07/2026)", "note": "art. 1 par. 1: DF-e obligation for Simples optantes starts 2027-01-01"}]))
if crt != "3":
return Resolution("INSUFFICIENT_EVIDENCE", "RTC-R1", "COMPANY_REGIME_UNKNOWN", inputs_used=used)
if len(ncm) != 8 or product.get("ncm_valid") is not True:
return Resolution("INSUFFICIENT_EVIDENCE", "RTC-R2", "NO_VALID_NCM", inputs_used=used)
local_rows = local_rows or {}
compat = compat or compare_catalogs(local_rows, today)
hits = annex_matches(ncm)
annexes = sorted({h["annex"] for h in hits})
if hits:
if any(h["revoked"] for h in hits) and len(annexes) == 1:
return Resolution("AMBIGUOUS", "RTC-A1", "LISTED_ONLY_IN_REVOKED_ANNEX", inputs_used=used, evidence=hits, source_references=_references())
live = [h for h in hits if not h["revoked"]]
live_annexes = sorted({h["annex"] for h in live})
if len(live_annexes) > 1 or (len(live_annexes) == 1 and len(annexes) > 1):
return Resolution("AMBIGUOUS", "RTC-A2", "LISTED_IN_MORE_THAN_ONE_ANNEX", inputs_used=used, evidence=hits, source_references=_references())
annex = live_annexes[0] if live_annexes else annexes[0]
clean = [h for h in live if h["clean"]]
cls = ANNEX_DEFAULT_CLASS.get(annex)
if not cls or not clean or len(clean) != len(live):
return Resolution("AMBIGUOUS", "RTC-A1", f"ANNEX_{annex}_NEEDS_OPERATION_OR_CONDITION", inputs_used=used, evidence=hits, source_references=_references())
problem = _validate_class(cls, local_rows, compat, today)
if problem:
return Resolution("INSUFFICIENT_EVIDENCE", "RTC-G2", problem, inputs_used=used, evidence=hits, source_references=_references())
official = _official_by_code()[cls]
return Resolution("RESOLVED_WITH_EXPLICIT_RULE", "RTC-R3", f"ANNEX_{annex}_CLEAN_ITEM_UNIQUE_DEFAULT_CLASS", cls, official["cst"], "ANNEX_TEXT_MATCH", used, hits, _references())
if ncm[:2] in SENSITIVE_CHAPTERS or ncm.startswith(SPECIAL_PREFIXES):
return Resolution("AMBIGUOUS", "RTC-A3", "SPECIAL_REGIME_DEPENDS_ON_NATURE_OR_OPERATION_OF_THE_GOOD", inputs_used=used, evidence=[{"chapter": ncm[:2]}], source_references=_references())
problem = _validate_class("000001", local_rows, compat, today)
if problem:
return Resolution("INSUFFICIENT_EVIDENCE", "RTC-G2", problem, inputs_used=used, source_references=_references())
return Resolution("SUGGESTION_ONLY", "RTC-R4", "NCM_IN_NO_ANNEX_AND_NO_NATURE_BASED_CHAPTER", "000001", "000", "EXCLUSION_OF_ALL_SPECIAL_REGIMES", used,
[{"annex_hits": 0, "chapter": ncm[:2], "note": "absence of a special regime is not positive evidence: suggestion only, never written"}], _references())
+51
View File
@@ -0,0 +1,51 @@
"""Persists the Agent's CFOP/IPPT write audit records (append-only). Mirrors central.app.rtc_audit's
safety model for a separate, narrower capability - no candidate preview table (candidates are
recomputed fresh from field_contract every job, never stored server-side), no rollback queue yet
(rollback is operator-triggered later, out of scope for this pass)."""
from __future__ import annotations
from .events import record_event
from .models import CfopIpptWriteAudit
MAX_AUDITS_PER_JOB = 2000
WRITE_MODES = ("DISABLED", "DRY_RUN", "APPROVAL_REQUIRED", "ENABLED")
ALLOWED_FIELDS = ("CFOP", "IPPT")
def _clip(value, size):
return None if value is None else str(value)[:size]
def job_authorizations(installation) -> dict:
"""What the authenticated job carries to the Agent. Decided centrally per installation; the Agent
has no local switch. DISABLED unless an installation row explicitly says otherwise (server_default)."""
mode = installation.cfop_ippt_write_mode if installation.cfop_ippt_write_mode in WRITE_MODES else "DISABLED"
return {"cfop_ippt_write_mode": mode}
def record_cfop_ippt_write_audits(session, *, tenant_id: int, installation, agent, job_id: int | None, cfop_ippt_apply: dict | None) -> int:
"""Stores every applied/aborted write of the job. Never trusts the Agent for identity: tenant/installation/agent come from the authenticated context."""
if not isinstance(cfop_ippt_apply, dict):
return 0
records = list(cfop_ippt_apply.get("applied") or []) + list(cfop_ippt_apply.get("aborted") or [])
stored = applied = aborted = 0
for item in records[:MAX_AUDITS_PER_JOB]:
if not isinstance(item, dict) or item.get("field") not in ALLOWED_FIELDS:
continue
outcome = str(item.get("outcome", ""))[:16]
session.add(CfopIpptWriteAudit(
tenant_id=tenant_id, installation_id=installation.id, agent_id=agent.id, job_id=job_id,
product_code=_clip(item.get("product_code"), 64) or "", field=item["field"], outcome=outcome,
reason=_clip(item.get("reason") or item.get("reason_code"), 96), old_value=_clip(item.get("old_value"), 16), new_value=_clip(item.get("new_value"), 16),
action="APPLY", written_at=_clip(item.get("timestamp"), 40)))
stored += 1
if outcome == "APPLIED":
applied += 1
else:
aborted += 1
if stored:
record_event(session, tenant_id=tenant_id, installation_id=installation.id, agent_id=agent.id, severity="WARNING" if applied else "INFO", event_type="CFOP_IPPT_AUTO_APPLY", job_id=job_id,
message=f"CFOP/IPPT: {applied} aplicado(s), {aborted} abortado(s)",
context={"applied": applied, "aborted": aborted, "reasons": cfop_ippt_apply.get("aborted_reasons") or {}, "mode": cfop_ippt_apply.get("mode")})
session.commit()
return stored
+149
View File
@@ -0,0 +1,149 @@
"""Conservative deterministic NCM classification."""
import re
import unicodedata
ENGINE_VERSION = "0.3.0"
DECISION_VERSION = "mission-04c-ipi-ex-split"
_NT_MARKERS = {"NT", "N/T", "NAO TRIBUTADO", "NÃO TRIBUTADO"}
_ZERO_MARKERS = {"0", "0,00", "0.00"}
def classify_tipi_rate(entry):
"""Classify why a TIPI base entry has no usable numeric rate; never guess a value."""
if entry.ex:
return "EX_SPECIFIC"
text = entry.ipi_rate_text
if text is None:
return "SOURCE_BLANK"
normalized = text.strip().upper()
if normalized in _NT_MARKERS:
return "NT_OR_NON_TAXED"
if entry.ipi_rate == 0 or normalized in _ZERO_MARKERS:
return "EXPLICIT_ZERO"
return "OTHER"
def normalize_code(value): return "".join(char for char in str(value or "") if char.isdigit())
def normalize_description(value): return " ".join(re.sub(r"[^A-Z0-9]+", " ", unicodedata.normalize("NFKD", str(value or "")).encode("ascii", "ignore").decode().upper()).split())
# Portuguese function words with no fiscal-descriptive content. Without this, a
# product description sharing only "DE"/"E"/"COM" with an official CEST text
# scores a false-positive nonzero overlap that has nothing to do with the product.
_STOPWORDS = {
"DE", "DA", "DO", "DAS", "DOS", "E", "OU", "A", "O", "AS", "OS",
"EM", "NO", "NA", "NOS", "NAS", "PARA", "POR", "COM", "SEM",
"MESMO", "OUTRO", "OUTRA", "OUTROS", "OUTRAS", "QUE",
}
def description_tokens(value):
return set(normalize_description(value).split()) - _STOPWORDS
class NcmStatus(str):
def __eq__(self, other):
if super().__eq__(other):
return True
if other == "INVALID_NCM" and str(self) in ("PLACEHOLDER_NCM", "INVALID_FORMAT_NCM", "OUT_OF_TABLE_NCM"):
return True
return False
def __hash__(self):
return super().__hash__()
class Classifier:
def __init__(self, repository): self.repository = repository
def classify(self, product):
entries = self.repository.active_entries(); code = normalize_code(product.get("current_ncm"))
exact = next((entry for entry in entries if entry.code == code and len(code) == 8), None)
legacy = self.repository.legacy_ncm(code) if len(code) == 8 and hasattr(self.repository, "legacy_ncm") else None
if exact:
status, candidate = "EXACT_NCM", exact
elif legacy:
status, candidate = "LEGACY_NCM", None
elif not code:
status, candidate = self._candidate(entries, product.get("description"))
elif len(set(code)) == 1 or code in ("00000000", "99999999", "11111111", "000000"):
status, candidate = NcmStatus("PLACEHOLDER_NCM"), None
elif len(code) != 8 or code in ("12345678", "87654321"):
status, candidate = NcmStatus("INVALID_FORMAT_NCM"), None
else:
status, candidate = NcmStatus("OUT_OF_TABLE_NCM"), None
matched_code = candidate.code if candidate else legacy["code"] if legacy else None
cests_available = matched_code and hasattr(self.repository, "cest_for_ncm")
cests = self.repository.cest_for_ncm(matched_code) if cests_available else None
cests = list({item.code: item for item in cests or []}.values())
scores = {item.code: self._description_score(product.get("description"), item.description) for item in cests}
if legacy:
cest_status = "LEGACY_NCM"
elif not cests:
cest_status = "NO_CEST_MAPPING_FOUND"
elif max(scores.values()) < .5:
cest_status = "CEST_DESCRIPTION_MISMATCH"
elif len(cests) == 1:
cest_status = "CEST_CONFIRMED"
else:
cest_status = "CEST_CANDIDATES"
resolved_cest = cests[0].code if cest_status == "CEST_CONFIRMED" else None
ipi = self._resolve_ipi(matched_code, legacy)
mva = self._resolve_mva(matched_code, legacy, resolved_cest)
return {"status": status, "cest_status": cest_status, "ipi_status": ipi["status"], "ba_st_mva_status": mva["status"], "classification": {"ncm": matched_code, "ncm_description": candidate.description if candidate else None, "cest": [item.code for item in cests] if cests_available else None, "cest_candidates": [{"code": item.code, "official_description": item.description, "description_score": scores[item.code]} for item in cests], "cfop": None, "cst": None, "csosn": None, "pis_cofins": None, "ipi": ipi["resolved"], "ipi_candidates": ipi["candidates"], "ba_st_mva": mva["resolved"], "ba_st_mva_candidates": mva["candidates"], "ibs_cbs": None}, "legacy_ncm": legacy, "reason_codes": ["FISCAL_DETERMINATION_NOT_CONFIGURED"], "can_auto_apply": False, "engine_version": ENGINE_VERSION, "decision_version": DECISION_VERSION, "dataset_versions": self.repository.dataset_versions()}
def _resolve_ipi(self, matched_code, legacy):
if legacy:
return {"status": "LEGACY_NCM", "resolved": None, "candidates": []}
if not matched_code or not hasattr(self.repository, "tipi_for_ncm"):
return {"status": "NO_TIPI_MAPPING_FOUND", "resolved": None, "candidates": []}
entries = self.repository.tipi_for_ncm(matched_code)
if not entries:
return {"status": "NO_TIPI_MAPPING_FOUND", "resolved": None, "candidates": []}
base = next((entry for entry in entries if not entry.ex), None)
ex_variants = [entry for entry in entries if entry.ex]
candidates = [{"ncm": entry.ncm, "ex": entry.ex, "description": entry.description, "rate": entry.ipi_rate_text} for entry in ex_variants]
if base:
if base.ipi_rate is not None:
rate_kind = "NUMERIC"
elif base.ipi_rate_text is not None:
rate_kind = "NON_NUMERIC"
else:
rate_kind = "BLANK"
resolved = {
"ncm": base.ncm,
"ex": base.ex,
"description": base.description,
"rate": base.ipi_rate_text,
"rate_numeric": base.ipi_rate,
"rate_kind": rate_kind,
}
if rate_kind != "NUMERIC":
resolved["rate_classification"] = classify_tipi_rate(base)
status = "IPI_CANDIDATES" if ex_variants else "IPI_RESOLVED"
return {"status": status, "resolved": resolved, "candidates": candidates}
return {"status": "IPI_CANDIDATES", "resolved": None, "candidates": candidates}
def _resolve_mva(self, matched_code, legacy, resolved_cest):
if legacy:
return {"status": "LEGACY_NCM", "resolved": None, "candidates": []}
if not matched_code or not hasattr(self.repository, "ba_st_mva_for_ncm"):
return {"status": "NO_MVA_MAPPING_FOUND", "resolved": None, "candidates": []}
entries = self.repository.ba_st_mva_for_ncm(matched_code)
if not entries:
return {"status": "NO_MVA_MAPPING_FOUND", "resolved": None, "candidates": []}
candidates = [{"item": entry.item, "cest": entry.cest, "mva_original": entry.mva_original, "mva_adjusted": entry.mva_adjusted, "description": entry.description} for entry in entries]
if len(entries) == 1:
return {"status": "MVA_RESOLVED", "resolved": candidates[0], "candidates": candidates}
if resolved_cest:
by_cest = [item for item in candidates if item["cest"] == resolved_cest]
if len(by_cest) == 1:
return {"status": "MVA_RESOLVED", "resolved": by_cest[0], "candidates": candidates}
return {"status": "MVA_CANDIDATES", "resolved": None, "candidates": candidates}
def _candidate(self, entries, description):
tokens = set(normalize_description(description).split())
scores = sorted(((len(tokens & set(normalize_description(item.description).split())) / len(tokens or {"_"}), item) for item in entries), reverse=True, key=lambda row: row[0])
if not scores or scores[0][0] == 0: return "NOT_FOUND", None
return ("HIGH_CONFIDENCE_CANDIDATE", scores[0][1]) if scores[0][0] >= .90 and (len(scores) == 1 or scores[0][0] - scores[1][0] >= .15) else ("AMBIGUOUS", None)
def _description_score(self, product_description, official_description):
product_tokens = description_tokens(product_description)
official_tokens = description_tokens(official_description)
return len(product_tokens & official_tokens) / len(product_tokens or {"_"})
+106
View File
@@ -0,0 +1,106 @@
"""Client mirror from GTI Gerenciador — the single source of truth for
clients (central/app/models.py: Tenant.source/external_client_id/active).
The Fiscal Hub never receives MySQL credentials; it calls the panel's
read-only internal endpoint instead, authenticated by a dedicated bearer
token (FISCAL_INTERNAL_API_TOKEN, shared only between the two apps' envs
— never SESSION_SECRET, the SSO key, an admin password or an Agent token).
Run `central/scripts/sync_clients_from_gerenciador.py` to pull a fresh copy;
`Tenant.source='LOCAL'` rows (pre-existing dev/test tenants) are never
touched or offered for a new installation.
"""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timezone
import httpx
from sqlalchemy import select
from . import config
from .models import Tenant
SOURCE = "GTI_GERENCIADOR"
class ClientSyncUnavailable(Exception):
pass
def enabled() -> bool:
return bool(config.settings.gerenciador_internal_url and config.settings.fiscal_internal_api_token)
def fetch_from_gerenciador() -> list[dict]:
if not enabled():
raise ClientSyncUnavailable("GERENCIADOR_INTERNAL_URL/FISCAL_INTERNAL_API_TOKEN not configured")
headers = {"Authorization": f"Bearer {config.settings.fiscal_internal_api_token}"}
try:
response = httpx.get(config.settings.gerenciador_internal_url, headers=headers, timeout=config.settings.http_timeout_seconds)
except httpx.HTTPError as error:
raise ClientSyncUnavailable(f"request failed: {error}") from error
if response.status_code != 200:
raise ClientSyncUnavailable(f"unexpected status {response.status_code}: {response.text[:200]}")
return response.json()["clients"]
@dataclass
class SyncResult:
fetched: int
created: int
updated: int
unchanged: int
active: int
inactive: int
def sync_from_gerenciador(session, clients: list[dict]) -> SyncResult:
"""Upsert `clients` (each: external_client_id, code, display_name, active)
into Tenant rows with source='GTI_GERENCIADOR'. Never touches a
source='LOCAL' row. A display-name collision with another tenant is
disambiguated with the external code rather than failing the whole sync
over one client (Tenant.name is UNIQUE)."""
now = datetime.now(timezone.utc)
existing_by_external = {row.external_client_id: row for row in session.scalars(select(Tenant).where(Tenant.source == SOURCE)).all()}
taken_names = {row.name for row in session.scalars(select(Tenant)).all()}
created = updated = unchanged = 0
for client in clients:
external_id = client["external_client_id"]
code = client["code"]
active = bool(client["active"])
wanted_name = (client.get("display_name") or "").strip() or f"Cliente {code}"
row = existing_by_external.get(external_id)
if row is None:
name = wanted_name if wanted_name not in taken_names else f"{wanted_name} ({code})"
row = Tenant(name=name, source=SOURCE, external_client_id=external_id, external_code=code, external_cnpj=client.get("cnpj"), active=active, synced_at=now)
session.add(row)
taken_names.add(name)
created += 1
continue
changed = False
if row.name != wanted_name and wanted_name not in taken_names:
taken_names.discard(row.name)
row.name = wanted_name
taken_names.add(wanted_name)
changed = True
if row.external_code != code:
row.external_code = code
changed = True
if row.external_cnpj != client.get("cnpj"):
row.external_cnpj = client.get("cnpj")
changed = True
if row.active != active:
row.active = active
changed = True
row.synced_at = now
if changed:
updated += 1
else:
unchanged += 1
session.flush()
active_count = sum(1 for c in clients if c["active"])
return SyncResult(fetched=len(clients), created=created, updated=updated, unchanged=unchanged, active=active_count, inactive=len(clients) - active_count)
+94
View File
@@ -0,0 +1,94 @@
"""Environment-backed Fiscal Hub configuration."""
from dataclasses import dataclass
import os
from urllib.parse import quote
def _database_url() -> str:
configured = os.getenv("DATABASE_URL")
if configured:
return configured
password = quote(os.getenv("POSTGRES_PASSWORD", "gti"), safe="")
return f"postgresql+psycopg://gti:{password}@postgres:5432/gti_fiscal"
@dataclass(frozen=True)
class Settings:
database_url: str = _database_url()
hub_port: int = int(os.getenv("HUB_PORT", "8000"))
# AGENT_DEV_TOKEN auto-bootstraps an agent from a fixed dev token on every request.
# Leave unset in any environment other than local development — production Agents
# must be created via scripts/register_agent.py, whose token is never derived from
# this setting.
agent_dev_token: str = os.getenv("AGENT_DEV_TOKEN", "")
environment: str = os.getenv("APP_ENV", "development")
log_level: str = os.getenv("LOG_LEVEL", "INFO")
base_url: str = os.getenv("BASE_URL", "http://localhost:8000")
# An Agent is considered ONLINE if a heartbeat arrived within this many seconds.
agent_online_threshold_seconds: int = int(os.getenv("AGENT_ONLINE_THRESHOLD_SECONDS", "120"))
# Backpressure: how many FULL jobs (INITIAL_SYNC / REPROCESS_ALL: whole-database classification) may run at once across ALL Agents. Others wait as PENDING. <=0 disables the limit.
heavy_job_concurrency_limit: int = int(os.getenv("HEAVY_JOB_CONCURRENCY_LIMIT", "3"))
agent_heartbeat_interval_seconds: int = int(os.getenv("AGENT_HEARTBEAT_INTERVAL_SECONDS", "40"))
agent_offline_after_seconds: int = int(os.getenv("AGENT_OFFLINE_AFTER_SECONDS", "0"))
agent_server_version: str = os.getenv("CURRENT_SERVER_VERSION", "1.0.0rc1")
agent_event_retention_days: int = int(os.getenv("AGENT_EVENT_RETENTION_DAYS", "90"))
web_session_seconds: int = int(os.getenv("WEB_SESSION_SECONDS", "28800"))
web_cookie_secure: bool = os.getenv("WEB_COOKIE_SECURE", "0") == "1"
web_cookie_name: str = os.getenv("WEB_COOKIE_NAME", "gti_admin_session")
web_max_event_message: int = int(os.getenv("WEB_MAX_EVENT_MESSAGE", "2000"))
web_max_event_context_bytes: int = int(os.getenv("WEB_MAX_EVENT_CONTEXT_BYTES", "12000"))
ui_timezone: str = os.getenv("UI_TIMEZONE", "America/Bahia")
allow_weak_dev_password: bool = os.getenv("ALLOW_WEAK_DEV_PASSWORD", "0") == "1"
# ---- SSO from GTI Gerenciador (central/app/sso.py) ----
# Ed25519 public key (raw 32 bytes, base64url) of the panel that signs the
# assertions; the private key never exists on the Hub. SSO is active only
# when both the key and the panel's start URL are set.
sso_public_key: str = os.getenv("SSO_GERENCIADOR_CHAVE_PUBLICA", "")
sso_start_url: str = os.getenv("SSO_START_URL", "")
federated_session_seconds: int = int(os.getenv("FEDERATED_SESSION_SECONDS", "3600"))
# ---- Agent distribution (central/app/agent_distribution.py) ----
# Local, non-web-served directory holding the released Windows Agent zips.
# Never inside PostgreSQL, never under central/static.
agent_release_storage_path: str = os.getenv("AGENT_RELEASE_STORAGE_PATH", "./data/releases")
agent_download_link_ttl_seconds: int = int(os.getenv("AGENT_DOWNLOAD_LINK_TTL_SECONDS", "1800"))
agent_download_max_uses_default: int = int(os.getenv("AGENT_DOWNLOAD_MAX_USES_DEFAULT", "1"))
agent_enrollment_ttl_seconds: int = int(os.getenv("AGENT_ENROLLMENT_TTL_SECONDS", "86400"))
# ---- Client mirror from GTI Gerenciador (central/app/client_sync.py) ----
# GTI Gerenciador is the single source of truth for clients; the Fiscal Hub
# never gets MySQL credentials. Both empty = sync disabled (documented).
gerenciador_internal_url: str = os.getenv("GERENCIADOR_INTERNAL_URL", "") # e.g. https://gtigerenciador.../internal/fiscal/clients
fiscal_internal_api_token: str = os.getenv("FISCAL_INTERNAL_API_TOKEN", "")
ncm_dataset_url: str = os.getenv("NCM_DATASET_URL", "https://portalunico.siscomex.gov.br/classif/api/publico/nomenclatura/download/json")
cest_dataset_url: str = os.getenv("CEST_DATASET_URL", "https://www.confaz.fazenda.gov.br/legislacao/convenios/2018/CV142_18")
dataset_storage_path: str = os.getenv("DATASET_STORAGE_PATH", "./data/datasets")
http_timeout_seconds: float = float(os.getenv("HTTP_TIMEOUT_SECONDS", "30"))
# ---- AI semantic layer V1 ----
# Credentials live ONLY in the Hub environment; the Windows Agent never sees them.
ai_enabled: bool = os.getenv("AI_ENABLED", "0") == "1"
ai_provider: str = os.getenv("AI_PROVIDER", "baseline") # baseline | openai | ... (AISemanticProvider)
ai_model: str = os.getenv("AI_MODEL", "baseline-deterministic-v1")
ai_max_batch_size: int = int(os.getenv("AI_MAX_BATCH_SIZE", "20"))
ai_max_run_cost_usd: float = float(os.getenv("AI_MAX_RUN_COST_USD", "5.0"))
ai_min_confidence: float = float(os.getenv("AI_MIN_CONFIDENCE", "0.6"))
ai_price_in_per_m: float | None = float(os.getenv("AI_PRICE_IN_PER_M")) if os.getenv("AI_PRICE_IN_PER_M") else None
ai_price_out_per_m: float | None = float(os.getenv("AI_PRICE_OUT_PER_M")) if os.getenv("AI_PRICE_OUT_PER_M") else None
# Real-LLM evaluation (OpenAI-compatible). The key lives ONLY in this environment.
# Z.ai general endpoint (NOT the /api/coding/paas/v4 coding-plan endpoint).
ai_api_key: str = os.getenv("AI_API_KEY", "")
ai_base_url: str = os.getenv("AI_BASE_URL", "https://api.z.ai/api/paas/v4")
# ---- Product enrichment (docs/product_enrichment.md): free providers only, tokens live ONLY in the Hub environment ----
cosmos_token: str = os.getenv("COSMOS_TOKEN", "") # optional; without it Cosmos is NOT_CONFIGURED (not an error)
oscbr_token: str = os.getenv("OSCBR_TOKEN", "") # preferred: a ready Bearer token instead of the login
cosmos_enabled: bool = os.getenv("COSMOS_ENABLED", "0") == "1" # Cosmos stays out of the active chain unless explicitly enabled
oscbr_backfill_enabled: bool = os.getenv("OSCBR_BACKFILL_ENABLED", "0") == "1" # gradual backfill of CORROBORATED GTINs; OFF until a real benchmark proves value
oscbr_user: str = os.getenv("OSCBR_USER", "") # optional free-plan account of gtin.rscsistemas.com.br (never in logs/UI)
oscbr_password: str = os.getenv("OSCBR_PASSWORD", "")
kodebar_api_key: str = os.getenv("KODEBAR_API_KEY", "") # optional free-plan key (identity only); never in logs/UI
provider_operational_reserve_percent: int = int(os.getenv("PROVIDER_OPERATIONAL_RESERVE_PERCENT", "20"))
allow_paid_providers: bool = os.getenv("ALLOW_PAID_PROVIDERS", "0") == "1"
official_refresh_enabled: bool = os.getenv("OFFICIAL_REFRESH_ENABLED", "1") == "1"
enrichment_time_budget_seconds: float = float(os.getenv("ENRICHMENT_TIME_BUDGET_SECONDS", "4.5"))
settings = Settings()
+73
View File
@@ -0,0 +1,73 @@
"""Validation of operator fiscal-context entries (facts the ERP does not know). Pure; shared by the Hub API, the importer CLI and the resolver."""
import json
SCOPES = ("COMPANY", "GROUP", "NCM", "PRODUCT")
ENUMS = {
"product_role": {"RESALE", "OWN_PRODUCTION", "INPUT", "SERVICE"},
"st_status": {"SUBJECT", "NOT_SUBJECT"},
"st_withheld": {"Y", "N"},
"icms_treatment": {"TAXED", "EXEMPT", "NOT_TAXED", "ST_WITHHELD", "SUSPENDED", "DEFERRED"},
# MIXED_OR_DYNAMIC: purchases are not predominantly intra- or interstate, so no single
# static entry CFOP is fiscally correct for all operations (section 0, AI semantic layer V1).
"entry_scope": {"INTRA", "INTER", "MIXED_OR_DYNAMIC"},
"rounding": {"A", "T"},
"piscofins": {"N", "S"},
"incidence": {"NENHUMA"},
"origin": {str(i) for i in range(9)},
"store_ipi_profile": {"POPULATE_REFERENCE_RATE", "LEAVE_NOT_APPLICABLE"},
# C000004.CONTRIBUINTE_IPI is not yet sent at heartbeat (no Agent-side reader exists for it): until that auto-discovery lands, this is the
# one-time operator answer field_contract's CST_IPI rule needs - never assumed "N" just because the Agent hasn't reported it.
"contribuinte_ipi": {"S", "N"},
}
PROFILE_KEYS = {"pis_in_cst", "pis_out_cst", "pis_rate", "cofins_in_cst", "cofins_out_cst", "cofins_rate"}
FREE_KEYS = {"ncm", "cest", "ex_tipi", "icms_rate", "credit_rate", "cst_ipi", "service_cfop", "pis_cofins_profile"}
ALLOWED_KEYS = set(ENUMS) | FREE_KEYS
PRODUCT_ONLY = {"ncm"}
COMPANY_ONLY = {"store_ipi_profile", "contribuinte_ipi"}
NCM_KEYS = {"cest", "ex_tipi"} # tied to the NCM tables: answered per NCM or per product # an NCM answer is always about one product
def validate_entry(entry):
"""Returns (normalized_entry, error). error is None when valid."""
scope = str(entry.get("scope", "")).strip().upper()
key = str(entry.get("key", "")).strip()
value = entry.get("value")
value = "" if value is None else str(value).strip()
scope_key = str(entry.get("scope_key") or "").strip()
if scope not in SCOPES:
return None, f"scope must be one of {SCOPES}"
if key not in ALLOWED_KEYS:
return None, f"unknown key {key!r}"
if scope != "COMPANY" and not scope_key:
return None, "scope_key is required for GROUP/NCM/PRODUCT scope"
if key in COMPANY_ONLY and scope != "COMPANY":
return None, f"{key} is a COMPANY-scope answer"
if key in PRODUCT_ONLY and scope != "PRODUCT":
return None, f"{key} is a PRODUCT-scope answer"
if key in NCM_KEYS and scope not in ("PRODUCT", "NCM"):
return None, f"{key} can only be answered for a PRODUCT or an NCM"
if not value:
return None, "value is empty"
if key in ENUMS and value.upper() not in ENUMS[key] and value not in ENUMS[key]:
return None, f"{key} must be one of {sorted(ENUMS[key])}"
if key in ENUMS:
value = value.upper() if key != "origin" else value
if key == "ncm" and not (value.isdigit() and len(value) == 8):
return None, "ncm must be 8 digits"
if key == "cest" and value.upper() != "NONE" and not (value.replace(".", "").isdigit() and len(value.replace(".", "")) == 7):
return None, "cest must be 7 digits or NONE"
if key in ("icms_rate", "credit_rate"):
try:
float(value.replace(",", "."))
except ValueError:
return None, f"{key} must be numeric"
if key == "pis_cofins_profile" and value.upper() not in ("NOT_APPLICABLE", "OPERATION_TIME"):
try:
prof = json.loads(value)
except ValueError:
return None, "pis_cofins_profile must be JSON, or exactly NOT_APPLICABLE / OPERATION_TIME"
if not isinstance(prof, dict) or not prof or set(prof) - PROFILE_KEYS:
return None, f"pis_cofins_profile keys must be a non-empty subset of {sorted(PROFILE_KEYS)}"
elif key == "pis_cofins_profile":
value = value.upper()
return {"scope": scope, "scope_key": scope_key if scope != "COMPANY" else "", "key": key, "value": value, "reason": (entry.get("reason") or "").strip() or None}, None
+131
View File
@@ -0,0 +1,131 @@
"""Human-authenticated JSON API for the control panel."""
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException, Request
from sqlalchemy import select
from .audit import record_audit
from .blocker_resolution import BlockerResolutionUnavailable, resolve_blocker_with_context
from .installation_identity import reference_agent_version
from .control_plane import dashboard_summary, installation_view, list_blockers, list_installations, list_jobs, list_panel_events
from .human_auth import require_csrf, require_human
from .models import Agent, AgentSnapshot, BlockedProduct, Installation, Job, Tenant
from .jobs import create_job
def router(session_factory):
routes = APIRouter(prefix="/v1")
def identity(request: Request):
session = session_factory()
try:
web_session, admin = require_human(request, session)
return session, web_session, admin
except Exception:
session.close(); raise
@routes.get("/dashboard/summary")
def summary(identity=Depends(identity)):
session, _, _ = identity
try: return dashboard_summary(session)
finally: session.close()
@routes.get("/agents")
def agents(page: int = 1, page_size: int = 50, client: str | None = None, health: str | None = None, version: str | None = None, search: str | None = None, last_seen_before: str | None = None, identity=Depends(identity)):
session, _, _ = identity
try: return list_installations(session, page=page, page_size=page_size, client=client, health=health, version=version, search=search, last_seen_before=datetime.fromisoformat(last_seen_before) if last_seen_before else None)
finally: session.close()
def get_installation(session, agent_id: int):
row = session.execute(select(Installation, Agent, Tenant, AgentSnapshot).join(Agent, Agent.id == Installation.agent_id).join(Tenant, Tenant.id == Installation.tenant_id).outerjoin(AgentSnapshot, AgentSnapshot.installation_id == Installation.id).where(Agent.id == agent_id)).first()
if not row: raise HTTPException(404, "Agent installation not found")
return row
@routes.get("/agents/{agent_id}")
def agent_detail(agent_id: int, identity=Depends(identity)):
session, _, _ = identity
try:
installation, agent, tenant, snapshot = get_installation(session, agent_id)
return installation_view(installation, agent, tenant, snapshot, reference=reference_agent_version(session))
finally: session.close()
@routes.get("/agents/{agent_id}/health")
def agent_health(agent_id: int, identity=Depends(identity)):
return agent_detail(agent_id, identity)
@routes.get("/agents/{agent_id}/coverage")
def agent_coverage(agent_id: int, identity=Depends(identity)):
detail = agent_detail(agent_id, identity)
return {key: detail[key] for key in ("total_products", "fiscally_complete", "safe_blocked", "failed_products", "coverage_percent", "blockers")}
@routes.get("/agents/{agent_id}/blockers")
def agent_blockers(agent_id: int, page: int = 1, page_size: int = 50, blocker_type: str | None = None, field: str | None = None, identity=Depends(identity)):
session, _, _ = identity
try:
installation, agent, _, _ = get_installation(session, agent_id)
return list_blockers(session, tenant_id=installation.tenant_id, agent_id=agent_id, page=page, page_size=page_size, blocker_type=blocker_type, field=field)
finally: session.close()
@routes.get("/agents/{agent_id}/jobs")
def agent_jobs(agent_id: int, page: int = 1, page_size: int = 50, status: str | None = None, kind: str | None = None, identity=Depends(identity)):
session, _, _ = identity
try: return list_jobs(session, agent_id=agent_id, page=page, page_size=page_size, status=status, kind=kind)
finally: session.close()
@routes.get("/agents/{agent_id}/events")
def agent_events(agent_id: int, page: int = 1, page_size: int = 50, severity: str | None = None, event_type: str | None = None, since: str | None = None, until: str | None = None, identity=Depends(identity)):
session, _, _ = identity
try:
installation, _, _, _ = get_installation(session, agent_id)
return {"page": max(1, page), "page_size": min(100, max(1, page_size)), "items": list_panel_events(session, installation.id, page=page, page_size=page_size, severity=severity, event_type=event_type, since=datetime.fromisoformat(since) if since else None, until=datetime.fromisoformat(until) if until else None)}
finally: session.close()
@routes.get("/blockers/global")
def global_blockers(page: int = 1, page_size: int = 50, blocker_type: str | None = None, field: str | None = None, identity=Depends(identity)):
session, _, _ = identity
try: return list_blockers(session, page=page, page_size=page_size, blocker_type=blocker_type, field=field)
finally: session.close()
@routes.get("/jobs/global")
def global_jobs(page: int = 1, page_size: int = 50, status: str | None = None, kind: str | None = None, since: str | None = None, until: str | None = None, identity=Depends(identity)):
session, _, _ = identity
try: return list_jobs(session, page=page, page_size=page_size, status=status, kind=kind, since=datetime.fromisoformat(since) if since else None, until=datetime.fromisoformat(until) if until else None)
finally: session.close()
@routes.post("/agents/{agent_id}/actions/reprocess")
def reprocess(agent_id: int, payload: dict, request: Request, identity=Depends(identity)):
session, web_session, admin = identity
try:
require_csrf(web_session, payload.get("csrf_token"))
if payload.get("confirm") is not True: raise HTTPException(400, "explicit confirmation required")
installation, agent, _, _ = get_installation(session, agent_id)
kind = payload.get("kind", "REPROCESS_ALL")
if kind not in {"REPROCESS_ALL", "REPROCESS_PRODUCTS"}: raise HTTPException(400, "unsupported action")
job_payload = payload.get("payload") or {}
if kind == "REPROCESS_PRODUCTS" and not job_payload.get("codigos"): raise HTTPException(400, "codigos required")
existing = next((candidate for candidate in session.scalars(select(Job).where(Job.agent_id == agent.id, Job.kind == kind, Job.status == "PENDING")).all() if candidate.payload == job_payload), None)
job = existing or create_job(session, agent, kind, job_payload)
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action=kind, target_type="installation", target_id=installation.installation_id, context={"job_id": job.id, "request_ip": request.client.host if request.client else None})
session.commit()
return {"job_id": job.id, "status": job.status}
finally: session.close()
@routes.post("/blockers/{blocker_id}/resolve-context")
@routes.post("/blockers/{blocker_id}/resolve-human")
def resolve_blocker(blocker_id: int, payload: dict, request: Request, identity=Depends(identity)):
session, web_session, admin = identity
try:
require_csrf(web_session, payload.get("csrf_token"))
entry_payload = {key: payload.get(key) for key in ("scope", "scope_key", "key", "value", "reason")}
try:
return resolve_blocker_with_context(session, blocker_id, admin, entry_payload, request_ip=request.client.host if request.client else None)
except BlockerResolutionUnavailable as error:
raise HTTPException(409, str(error)) from error
except LookupError as error:
raise HTTPException(404, str(error)) from error
except ValueError as error:
raise HTTPException(422, str(error)) from error
finally: session.close()
return routes
+613
View File
@@ -0,0 +1,613 @@
"""Bounded control-panel queries backed by installation snapshots."""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from decimal import Decimal, ROUND_HALF_UP
from urllib.parse import urlencode
from sqlalchemy import Numeric, and_, case, cast, func, or_, select
from .config import settings
from .events import list_events
from .installation_identity import compute_agent_health, compute_version_drift, outdated_versions, reference_agent_version
from .models import Agent, AgentEnrollment, AgentEvent, AgentSnapshot, AuditEvent, BlockedProduct, Installation, Job, Tenant
from .panel_presentation import health_reason_label, blocker_reason_label, blocker_type_label, fiscal_status_label, format_age, format_duration, format_ui_datetime, health_label, job_kind_label, job_status_label, operational_priority, priority_label
HEALTH_STATES = ("ONLINE", "DEGRADED", "OFFLINE", "FAILED", "UNKNOWN")
BLOCKER_FIELDS = ("NCM", "CEST", "CFOP", "ST", "ICMS", "CSOSN", "PIS", "COFINS", "RTC", "OTHER")
ATTENTION_CANDIDATE_LIMIT = 10
INACTIVE_ENROLLMENT_STATES = ("REVOKED", "FAILED", "EXPIRED")
def operational_installation_predicate():
"""Only installations eligible for current operational indicators.
Legacy Agents without an enrollment remain eligible. Disabling job
processing is not the same as deactivating an installation.
"""
inactive_enrollment = select(AgentEnrollment.id).where(
AgentEnrollment.agent_id == Agent.id,
AgentEnrollment.status.in_(INACTIVE_ENROLLMENT_STATES),
).exists()
return and_(Tenant.active.is_(True), Agent.enabled.is_(True), ~inactive_enrollment)
def _page(page: int, page_size: int):
return max(1, page), min(100, max(1, page_size))
def _iso(value):
if value is None:
return None
value = value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc)
return value.isoformat()
def _utc_now(value=None):
value = value or datetime.now(timezone.utc)
return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value
def _health_expression(now):
offline_after = settings.agent_offline_after_seconds or settings.agent_heartbeat_interval_seconds * 3
cutoff = _utc_now(now) - timedelta(seconds=offline_after)
return case(
(Installation.last_seen_at.is_(None), "UNKNOWN"),
(or_(AgentSnapshot.agent_health == "FAILED", Installation.last_status == "FAILED"), "FAILED"),
(Installation.last_seen_at < cutoff, "OFFLINE"),
(or_(AgentSnapshot.failed_products > 0, AgentSnapshot.last_job_status == "FAILED"), "DEGRADED"),
else_="ONLINE",
)
def _fiscal_completion_expression():
total = func.coalesce(AgentSnapshot.total_products, 0)
complete = func.coalesce(AgentSnapshot.fiscally_complete, 0)
numeric_total = cast(total, Numeric(18, 6))
numeric_complete = cast(complete * 100, Numeric(18, 6))
raw = case((total > 0, numeric_complete / numeric_total), else_=cast(func.coalesce(AgentSnapshot.coverage_percent, 0), Numeric(18, 6)))
return func.round(case((raw < 0, 0), (raw > 100, 100), else_=raw), 1)
def _fiscal_completion(snapshot) -> float:
total = int(getattr(snapshot, "total_products", 0) or 0)
complete = int(getattr(snapshot, "fiscally_complete", 0) or 0)
if total:
value = Decimal(max(0, min(total, complete)) * 100) / Decimal(total)
else:
value = Decimal(str(max(0, min(100, float(getattr(snapshot, "coverage_percent", 0) or 0)))))
return float(value.quantize(Decimal("0.1"), rounding=ROUND_HALF_UP))
def _fiscal_status(snapshot) -> str:
if int(getattr(snapshot, "safe_blocked", 0) or 0):
return "BLOCKED"
total = int(getattr(snapshot, "total_products", 0) or 0)
complete = int(getattr(snapshot, "fiscally_complete", 0) or 0)
if total and complete >= total:
return "COMPLETE"
return "INCOMPLETE"
def _coverage_partition(snapshot) -> dict:
"""Expose only explicit snapshot states and label the remaining count honestly."""
total = int(getattr(snapshot, "total_products", 0) or 0)
complete = int(getattr(snapshot, "fiscally_complete", 0) or 0)
safe_blocked = int(getattr(snapshot, "safe_blocked", 0) or 0)
failed = int(getattr(snapshot, "failed_products", 0) or 0)
explicit = complete + safe_blocked + failed
residual = total - explicit
invalid_counts = min(total, complete, safe_blocked, failed) < 0
overcount = max(0, explicit - total)
return {
"unclassified_products": max(0, residual) if not invalid_counts else 0,
"coverage_overcount_products": overcount,
"coverage_invalid_counts": invalid_counts,
"coverage_invariant_pass": not invalid_counts and overcount == 0,
}
def _technical_timestamp(item):
if item["technical_health"] in {"FAILED", "DEGRADED"}:
return item.get("last_job_at") or item.get("last_seen_at")
return item.get("last_seen_at")
def _attention_item(item, problem, timestamp, *, rank, link=None, now=None):
timestamp_value = datetime.fromisoformat(timestamp) if isinstance(timestamp, str) else timestamp
return {
"client": item.get("client"),
"display_name": item.get("display_name"),
"hostname": item.get("hostname"),
"problem": problem,
"timestamp": timestamp if isinstance(timestamp, str) else _iso(timestamp_value),
"timestamp_display": format_ui_datetime(timestamp_value),
"age": format_age(timestamp_value, now),
"link": link or f"/agents/{item['agent_id']}",
"technical_health": item["technical_health"],
"rank": rank,
}
def installation_view(installation, agent, tenant, snapshot=None, *, now=None, reference=None):
has_snapshot = snapshot is not None
snapshot = snapshot or AgentSnapshot(installation_id=installation.id)
health, reason = compute_agent_health(installation, now=now, supplemental=snapshot)
# Always recomputed against the current reference release: the value stored on the snapshot was taken at the
# last heartbeat (against whatever was "current" then) and goes stale as soon as a newer release is published.
version_drift = compute_version_drift(agent.agent_version, reference)
fiscal_status = _fiscal_status(snapshot)
coverage_partition = _coverage_partition(snapshot)
if not has_snapshot:
# No heartbeat/snapshot yet (e.g. an enrollment-only E2E row): expose
# no counts at all rather than fabricated zeros — the UI shows
# "Não informado" instead of None/0.
coverage_partition = {"unclassified_products": None, "coverage_overcount_products": 0, "coverage_invalid_counts": False, "coverage_invariant_pass": True}
version_drift_label = {"CURRENT": "Atualizada", "OUTDATED": "Desatualizada", "UNKNOWN": "Desconhecida"}.get(version_drift, str(version_drift))
agent_version_display = f"{agent.agent_version} · {version_drift_label}" if agent.agent_version else "Desconhecida"
return {
"agent_id": agent.id,
"installation_id": installation.installation_id,
"installation_db_id": installation.id,
"client": tenant.name if tenant else None,
"tenant_id": installation.tenant_id,
"display_name": installation.display_name,
"hostname": installation.hostname,
"company_name": installation.company_name,
"company_code": installation.company_code,
"company_cnpj": installation.company_cnpj_masked,
"company_uf": installation.company_uf,
"company_crt": installation.company_crt,
"agent_version": agent.agent_version,
"agent_version_display": agent_version_display,
"version_drift": version_drift,
"version_drift_label": version_drift_label,
"health": health,
"health_reason": reason,
"technical_health": health,
"health_label": health_label(health),
"technical_health_label": health_label(health),
"last_seen_at": _iso(installation.last_seen_at),
"last_seen_display": format_ui_datetime(installation.last_seen_at),
"last_seen_age": format_age(installation.last_seen_at, now),
"started_at": _iso(installation.started_at),
"last_status": installation.last_status,
"total_products": snapshot.total_products if has_snapshot else None,
"fiscally_complete": snapshot.fiscally_complete if has_snapshot else None,
"safe_blocked": snapshot.safe_blocked if has_snapshot else None,
"failed_products": snapshot.failed_products if has_snapshot else None,
**coverage_partition,
"coverage_percent": snapshot.coverage_percent if has_snapshot else None,
"fiscal_completion_percent": _fiscal_completion(snapshot) if has_snapshot else None,
"fiscal_status": fiscal_status if has_snapshot else "UNKNOWN",
"fiscal_status_label": fiscal_status_label(fiscal_status) if has_snapshot else "Não informado",
"blockers": snapshot.blockers or {},
"last_job_status": snapshot.last_job_status,
"last_job_at": _iso(snapshot.last_job_at),
"last_successful_job_at": _iso(snapshot.last_successful_job_at),
"firebird_version": installation.firebird_version,
"firebird_ods": installation.firebird_ods,
"database_identifier": installation.database_identifier,
"capabilities": installation.capabilities or {},
"fiscal_engine_version": installation.fiscal_engine_version,
"dataset_versions": installation.dataset_versions or {},
"technical_ids": {"agent_id": agent.id, "installation_db_id": installation.id, "tenant_id": installation.tenant_id},
"link": f"/agents/{agent.id}",
}
def _installation_query_filters(query, *, client=None, version=None, search=None, last_seen_before=None):
if client:
query = query.where(or_(Tenant.name.ilike(f"%{client}%"), Installation.customer_code.ilike(f"%{client}%")))
if version and version != "outdated":
query = query.where(Agent.agent_version == version)
if search:
query = query.where(or_(Installation.display_name.ilike(f"%{search}%"), Installation.hostname.ilike(f"%{search}%"), Tenant.name.ilike(f"%{search}%")))
if last_seen_before:
query = query.where(or_(Installation.last_seen_at.is_(None), Installation.last_seen_at < last_seen_before))
return query
def list_installations(session, *, page=1, page_size=50, client=None, health=None, version=None, search=None, last_seen_before=None, sort="display_name", scope="active", now=None):
page, page_size = _page(page, page_size)
now = _utc_now(now)
reference = reference_agent_version(session)
query = select(Installation, Agent, Tenant, AgentSnapshot, operational_installation_predicate().label("operational_active")).join(Agent, Agent.id == Installation.agent_id).join(Tenant, Tenant.id == Installation.tenant_id).outerjoin(AgentSnapshot, AgentSnapshot.installation_id == Installation.id)
if scope == "active":
query = query.where(operational_installation_predicate())
elif scope != "all":
raise ValueError("scope must be active or all")
query = _installation_query_filters(query, client=client, version=version, search=search, last_seen_before=last_seen_before)
if version == "outdated":
query = query.where(Agent.agent_version.in_(outdated_versions(session, reference)))
health_expression = _health_expression(now)
if health:
query = query.where(health_expression == health)
if sort in {"worst_coverage", "coverage", "coverage_asc", "fiscal_completion_percent"}:
ordered = query.order_by(_fiscal_completion_expression().asc(), Installation.display_name.asc(), Installation.id.asc())
else:
ordered = query.order_by(Installation.display_name.asc(), Installation.id.asc())
rows = session.execute(ordered.offset((page - 1) * page_size).limit(page_size)).all()
items = [{**installation_view(i, a, t, s, now=now, reference=reference), "operational_active": bool(active)} for i, a, t, s, active in rows]
count_query = select(func.count()).select_from(query.with_only_columns(Installation.id).order_by(None).subquery())
total = session.scalar(count_query) or 0
return {"page": page, "page_size": page_size, "items": items, "total": total, "health_filter_applied": health}
def _blocker_item(row, tenant, installation=None, agent=None, *, now=None):
return {
"id": row.id,
"tenant_id": row.tenant_id,
"client": tenant.name if tenant else row.company,
"product_code": row.product_code,
"description": row.description,
"field": row.field,
"blocker_type": row.blocker_type,
"blocker_type_label": blocker_type_label(row.blocker_type),
"reason": row.reason_code,
"reason_code": row.reason_code,
"reason_label": blocker_reason_label(row.reason_code),
"evidence_summary": row.evidence_summary,
"first_seen_at": _iso(row.first_seen_at),
"last_seen_at": _iso(row.last_seen_at),
"first_seen_display": format_ui_datetime(row.first_seen_at),
"last_seen_display": format_ui_datetime(row.last_seen_at),
"age": format_age(row.last_seen_at, now),
"retry_count": row.retry_count,
"resolved_at": _iso(row.resolved_at),
"resolution_source": row.resolution_source,
"priority": operational_priority(row),
"priority_label": priority_label(operational_priority(row)),
"installation_id": installation.installation_id if installation else None,
"installation_db_id": installation.id if installation else None,
"display_name": installation.display_name if installation else None,
"hostname": installation.hostname if installation else None,
"agent_id": agent.id if agent else None,
"link": f"/blockers/{row.id}",
}
def _parse_age(value):
if value is None:
return None
if isinstance(value, (int, float)):
return timedelta(days=float(value))
text = str(value).strip().lower()
try:
if text.endswith("h"):
return timedelta(hours=float(text[:-1]))
if text.endswith("d"):
return timedelta(days=float(text[:-1]))
return timedelta(days=float(text))
except ValueError:
return None
def _blockers_query(session, *, tenant_id=None, agent_id=None, product_code=None, blocker_type=None, field=None, reason_code=None, status="OPEN", client=None, installation=None, search=None, age=None, scope="active"):
query = select(BlockedProduct, Tenant, Installation, Agent).join(Tenant, Tenant.id == BlockedProduct.tenant_id).outerjoin(Installation, Installation.id == BlockedProduct.installation_id).outerjoin(Agent, Agent.id == Installation.agent_id)
if scope == "active":
query = query.where(operational_installation_predicate(), Installation.id.is_not(None))
elif scope != "all":
raise ValueError("scope must be active or all")
status = (status or "OPEN").upper()
if status == "OPEN":
query = query.where(BlockedProduct.resolved_at.is_(None))
elif status in {"RESOLVED", "CLOSED"}:
query = query.where(BlockedProduct.resolved_at.is_not(None))
if tenant_id is not None:
query = query.where(BlockedProduct.tenant_id == tenant_id)
if agent_id is not None:
query = query.where(Agent.id == agent_id)
if product_code is not None:
query = query.where(BlockedProduct.product_code == product_code)
if blocker_type:
query = query.where(BlockedProduct.blocker_type == blocker_type)
if field:
query = query.where(func.upper(BlockedProduct.field) == field.upper())
if reason_code:
query = query.where(BlockedProduct.reason_code == reason_code)
if client:
query = query.where(or_(Tenant.name.ilike(f"%{client}%"), BlockedProduct.company.ilike(f"%{client}%")))
if installation:
query = query.where(or_(Installation.installation_id.ilike(f"%{installation}%"), Installation.display_name.ilike(f"%{installation}%"), Installation.hostname.ilike(f"%{installation}%")))
if search:
term = f"%{search}%"
query = query.where(or_(BlockedProduct.product_code.ilike(term), BlockedProduct.description.ilike(term), BlockedProduct.field.ilike(term), BlockedProduct.blocker_type.ilike(term), BlockedProduct.reason_code.ilike(term), Tenant.name.ilike(term)))
age_delta = _parse_age(age)
if age_delta:
query = query.where(BlockedProduct.last_seen_at <= datetime.now(timezone.utc) - age_delta)
return query
def blocker_filter_summary(session, *, tenant_id=None, agent_id=None, product_code=None, blocker_type=None, field=None, reason_code=None, status="OPEN", client=None, installation=None, age=None, search=None, scope="active"):
query = _blockers_query(session, tenant_id=tenant_id, agent_id=agent_id, product_code=product_code, blocker_type=blocker_type, field=field, reason_code=reason_code, status=status, client=client, installation=installation, search=search, age=age, scope=scope)
open_count = func.sum(case((BlockedProduct.resolved_at.is_(None), 1), else_=0))
open_count, = session.execute(query.with_only_columns(open_count).order_by(None)).one()
rows = session.execute(_blocker_breakdown_query(query)).all()
breakdown = {field: 0 for field in BLOCKER_FIELDS}
for field_name, count in rows:
key = str(field_name or "").upper()
breakdown[key if key in breakdown else "OTHER"] += int(count)
pair_query = query.with_only_columns(BlockedProduct.tenant_id, BlockedProduct.product_code).distinct().subquery()
client_query = query.with_only_columns(BlockedProduct.tenant_id).distinct().subquery()
return {
"open_blockers": int(open_count or 0),
"affected_products": int(session.scalar(select(func.count()).select_from(pair_query)) or 0),
"affected_clients": int(session.scalar(select(func.count()).select_from(client_query)) or 0),
"blocker_breakdown": breakdown,
}
def rtc_blocker_summary(session, agent_id: int) -> dict:
"""Read-only, installation-scoped RTC blocker aggregates for the cClassTrib page."""
query = _blockers_query(session, agent_id=agent_id, field="RTC", status="OPEN", scope="all")
blockers = int(session.scalar(query.with_only_columns(func.count(BlockedProduct.id)).order_by(None)) or 0)
products = int(session.scalar(query.with_only_columns(func.count(func.distinct(BlockedProduct.product_code))).order_by(None)) or 0)
reason = func.coalesce(BlockedProduct.evidence_summary["reason"].as_string(), BlockedProduct.reason_code)
reasons = session.execute(
query.with_only_columns(reason, func.count(func.distinct(BlockedProduct.product_code)))
.group_by(reason).order_by(None)
).all()
return {"open_blockers": blockers, "products_affected": products, "reason_counts": {str(code or ""): int(count) for code, count in reasons}}
def _blocker_breakdown_query(query):
"""Group on the source column itself for PostgreSQL-safe SQL semantics."""
return query.with_only_columns(BlockedProduct.field, func.count(BlockedProduct.id)).group_by(BlockedProduct.field).order_by(None)
def list_blockers(session, *, tenant_id=None, agent_id=None, product_code=None, page=1, page_size=50, blocker_type=None, field=None, reason_code=None, resolved=False, client=None, installation=None, status="OPEN", age=None, search=None, scope="active"):
page, page_size = _page(page, page_size)
if resolved and status == "OPEN":
status = "RESOLVED"
query = _blockers_query(session, tenant_id=tenant_id, agent_id=agent_id, product_code=product_code, blocker_type=blocker_type, field=field, reason_code=reason_code, status=status, client=client, installation=installation, search=search, age=age, scope=scope)
ordered = query.order_by(BlockedProduct.last_seen_at.desc(), BlockedProduct.id.desc())
rows = session.execute(ordered.offset((page - 1) * page_size).limit(page_size)).all()
items = [_blocker_item(blocker, tenant, installation_row, agent, now=datetime.now(timezone.utc)) for blocker, tenant, installation_row, agent in rows]
count_query = query.with_only_columns(func.count(BlockedProduct.id)).order_by(None)
total = session.scalar(count_query) or 0
return {"page": page, "page_size": page_size, "items": items, "total": total}
def _job_item(row, agent, installation, tenant, *, now=None):
return {
"job_id": row.id,
"agent_id": row.agent_id,
"client": tenant.name if tenant else None,
"display_name": installation.display_name if installation else None,
"hostname": installation.hostname if installation else None,
"kind": row.kind,
"kind_label": job_kind_label(row.kind),
"status": row.status,
"status_label": job_status_label(row.status),
"created_at": _iso(row.created_at),
"started_at": _iso(row.started_at),
"finished_at": _iso(row.finished_at),
"created_display": format_ui_datetime(row.created_at),
"started_display": format_ui_datetime(row.started_at),
"finished_display": format_ui_datetime(row.finished_at),
"duration": format_duration(row.started_at, row.finished_at, now),
"attempts": row.attempts,
"result_summary": row.result_summary,
"error": row.error,
}
def list_jobs(session, *, agent_id=None, job_id=None, page=1, page_size=50, status=None, kind=None, since=None, until=None, client=None, search=None, scope="active"):
page, page_size = _page(page, page_size)
query = select(Job, Agent, Installation, Tenant).join(Agent, Agent.id == Job.agent_id).join(Installation, Installation.agent_id == Agent.id).join(Tenant, Tenant.id == Installation.tenant_id)
if scope == "active":
query = query.where(operational_installation_predicate())
elif scope != "all":
raise ValueError("scope must be active or all")
if agent_id is not None:
query = query.where(Job.agent_id == agent_id)
if job_id is not None:
query = query.where(Job.id == job_id)
if status:
query = query.where(Job.status == status)
if kind:
query = query.where(Job.kind == kind)
if since:
query = query.where(Job.created_at >= since)
if until:
query = query.where(Job.created_at <= until)
if client:
query = query.where(or_(Tenant.name.ilike(f"%{client}%"), Installation.customer_code.ilike(f"%{client}%")))
if search:
term = f"%{search}%"
query = query.where(or_(Tenant.name.ilike(term), Installation.display_name.ilike(term), Installation.hostname.ilike(term), Job.kind.ilike(term), Job.status.ilike(term)))
ordered = query.order_by(Job.created_at.desc(), Job.id.desc())
rows = session.execute(ordered.offset((page - 1) * page_size).limit(page_size)).all()
count_query = query.with_only_columns(func.count(Job.id)).order_by(None)
total = session.scalar(count_query) or 0
return {"page": page, "page_size": page_size, "items": [_job_item(row, agent, installation, tenant) for row, agent, installation, tenant in rows], "total": total}
def jobs_status_summary(session, *, scope="active"):
"""Bounded job counts for the current operational scope."""
query = (select(Job.status, func.count(Job.id)).join(Agent, Agent.id == Job.agent_id)
.join(Installation, Installation.agent_id == Agent.id)
.join(Tenant, Tenant.id == Installation.tenant_id))
if scope == "active":
query = query.where(operational_installation_predicate())
elif scope != "all":
raise ValueError("scope must be active or all")
counts = {status: 0 for status in ("PENDING", "RUNNING", "COMPLETED", "FAILED")}
for status, count in session.execute(query.group_by(Job.status)):
if status in counts:
counts[status] = int(count)
return counts
def dashboard_summary(session, now=None):
now = _utc_now(now)
reference = reference_agent_version(session)
active = operational_installation_predicate()
installation_query = select(Installation, Agent, Tenant, AgentSnapshot).join(Agent, Agent.id == Installation.agent_id).join(Tenant, Tenant.id == Installation.tenant_id).outerjoin(AgentSnapshot, AgentSnapshot.installation_id == Installation.id).where(active)
health_expression = _health_expression(now)
health_rows = session.execute(
select(health_expression.label("health"), func.count(Installation.id))
.select_from(Installation)
.join(Agent, Agent.id == Installation.agent_id)
.join(Tenant, Tenant.id == Installation.tenant_id)
.outerjoin(AgentSnapshot, AgentSnapshot.installation_id == Installation.id)
.where(active)
.group_by(health_expression)
).all()
counts = {state: 0 for state in HEALTH_STATES}
for state, count in health_rows:
counts[state] = int(count)
snapshot_total = func.coalesce(AgentSnapshot.total_products, 0)
snapshot_complete = func.coalesce(AgentSnapshot.fiscally_complete, 0)
snapshot_blocked = func.coalesce(AgentSnapshot.safe_blocked, 0)
snapshot_failed = func.coalesce(AgentSnapshot.failed_products, 0)
snapshot_explicit = snapshot_complete + snapshot_blocked + snapshot_failed
snapshot_has_negative = or_(snapshot_total < 0, snapshot_complete < 0, snapshot_blocked < 0, snapshot_failed < 0)
total_row = session.execute(
select(
func.coalesce(func.sum(AgentSnapshot.total_products), 0),
func.coalesce(func.sum(AgentSnapshot.fiscally_complete), 0),
func.coalesce(func.sum(AgentSnapshot.safe_blocked), 0),
func.coalesce(func.sum(AgentSnapshot.failed_products), 0),
func.coalesce(func.sum(case((and_(~snapshot_has_negative, snapshot_explicit <= snapshot_total), snapshot_total - snapshot_explicit), else_=0)), 0),
func.coalesce(func.sum(case((and_(~snapshot_has_negative, snapshot_explicit > snapshot_total), snapshot_explicit - snapshot_total), else_=0)), 0),
func.coalesce(func.sum(case((snapshot_has_negative, 1), else_=0)), 0),
).select_from(Installation).join(Agent, Agent.id == Installation.agent_id).join(Tenant, Tenant.id == Installation.tenant_id).outerjoin(AgentSnapshot, AgentSnapshot.installation_id == Installation.id).where(active)
).one()
totals = dict(zip(("total_products", "fiscally_complete", "safe_blocked", "failed_products", "unclassified_products", "coverage_overcount_products", "coverage_invalid_snapshots"), (int(value or 0) for value in total_row)))
totals["coverage_invariant_pass"] = totals["coverage_overcount_products"] == 0 and totals["coverage_invalid_snapshots"] == 0
open_filter = BlockedProduct.resolved_at.is_(None)
active_blockers = (select(BlockedProduct).join(Installation, Installation.id == BlockedProduct.installation_id)
.join(Agent, Agent.id == Installation.agent_id).join(Tenant, Tenant.id == BlockedProduct.tenant_id)
.where(open_filter, active))
open_blockers = session.scalar(active_blockers.with_only_columns(func.count(BlockedProduct.id)).order_by(None)) or 0
breakdown_expression = func.coalesce(BlockedProduct.field, "OTHER")
breakdown_rows = session.execute(active_blockers.with_only_columns(breakdown_expression, func.count(BlockedProduct.id)).group_by(breakdown_expression).order_by(None)).all()
open_blocker_breakdown = {field: 0 for field in BLOCKER_FIELDS}
for field, count in breakdown_rows:
key = str(field or "").upper()
open_blocker_breakdown[key if key in open_blocker_breakdown else "OTHER"] += int(count)
product_pairs = active_blockers.with_only_columns(BlockedProduct.tenant_id, BlockedProduct.product_code).distinct().subquery()
safe_blocked_products = session.scalar(select(func.count()).select_from(product_pairs)) or 0
blocker_counts = {field: count for field, count in open_blocker_breakdown.items() if count}
def installation_candidates(health):
query = installation_query.where(health_expression == health).order_by(Installation.id.asc()).limit(ATTENTION_CANDIDATE_LIMIT)
return [installation_view(i, a, t, s, now=now, reference=reference) for i, a, t, s in session.execute(query).all()]
health_candidates = {health: installation_candidates(health) for health in ("FAILED", "DEGRADED", "OFFLINE", "UNKNOWN")}
version_filter = Agent.agent_version.in_(outdated_versions(session, reference))
outdated_query = installation_query.where(version_filter).order_by(Installation.id.asc()).limit(ATTENTION_CANDIDATE_LIMIT)
outdated_agents = [installation_view(i, a, t, s, now=now, reference=reference) for i, a, t, s in session.execute(outdated_query).all()]
group_field = func.upper(func.coalesce(BlockedProduct.field, "OTHER"))
fiscal_priority = case(
(func.upper(BlockedProduct.blocker_type).in_(("TECHNICAL_FAILURE", "CONNECTION_ERROR", "AGENT_OFFLINE", "JOB_FAILED")), 0),
(func.upper(BlockedProduct.blocker_type).in_(("BLOCKED_MISSING_EVIDENCE", "MISSING_EVIDENCE")), 1),
(group_field.in_(BLOCKER_FIELDS[:-1]), 2),
else_=3,
)
blocker_groups = session.execute(
active_blockers.with_only_columns(
Tenant.name, Installation.display_name, Installation.hostname, Agent.id,
group_field, BlockedProduct.blocker_type, BlockedProduct.reason_code,
func.count(BlockedProduct.id), func.count(func.distinct(BlockedProduct.product_code)),
func.max(BlockedProduct.last_seen_at),
).group_by(Tenant.name, Installation.id, Installation.display_name, Installation.hostname,
Agent.id, group_field, BlockedProduct.blocker_type, BlockedProduct.reason_code)
.order_by(fiscal_priority.asc(), func.max(BlockedProduct.last_seen_at).desc(), Agent.id.asc())
.limit(ATTENTION_CANDIDATE_LIMIT)
).all()
attention = []
for health in ("FAILED", "DEGRADED", "OFFLINE", "UNKNOWN"):
rank = {"FAILED": 0, "DEGRADED": 1, "OFFLINE": 2, "UNKNOWN": 3}[health]
attention.extend((_attention_item(item, health_reason_label(item["health_reason"]), _technical_timestamp(item), rank=rank, now=now), rank, item["agent_id"]) for item in health_candidates[health])
for client_name, display_name, hostname, agent_id, field, blocker_type, reason, count, products, last_seen, in blocker_groups:
item = {"client": client_name, "display_name": display_name, "hostname": hostname, "agent_id": agent_id, "technical_health": "BLOCKED"}
link = "/blockers?" + urlencode({"scope": "active", "status": "OPEN", "agent_id": agent_id, "field": field, "blocker_type": blocker_type, "reason_code": reason})
priority = operational_priority({"field": field, "blocker_type": blocker_type, "reason_code": reason})
rank = {"CRITICAL": 3, "HIGH": 4, "NORMAL": 5, "LOW": 6}[priority]
entry = _attention_item(item, f"{field}: {blocker_reason_label(reason)}", last_seen, rank=rank, link=link, now=now)
entry.update(group_count=count, products_affected=products, field=field, priority=priority)
attention.append((entry, rank, agent_id))
for item in outdated_agents:
attention.append((_attention_item(item, "Versão do agent desatualizada", item.get("last_seen_at"), rank=7, now=now), 7, item["agent_id"]))
attention.sort(key=lambda entry: (entry[1], -(entry[0]["timestamp"] is not None), entry[2]))
active_jobs = select(Job).join(Agent, Agent.id == Job.agent_id).join(Installation, Installation.agent_id == Agent.id).join(Tenant, Tenant.id == Installation.tenant_id).where(active)
failed_jobs = session.scalar(active_jobs.with_only_columns(func.count(Job.id)).where(Job.status == "FAILED").order_by(None)) or 0
failed_job_rows = session.execute(
select(Job, Agent, Installation, Tenant).join(Agent, Agent.id == Job.agent_id)
.join(Installation, Installation.agent_id == Agent.id).join(Tenant, Tenant.id == Installation.tenant_id)
.where(Job.status == "FAILED", active).order_by(Job.created_at.desc(), Job.id.desc()).limit(10)
).all()
recent_failed_jobs = []
for job, agent, installation, tenant in failed_job_rows:
item = _job_item(job, agent, installation, tenant, now=now)
item["error"] = (item["error"] or "")[:240]
item["created_age"] = format_age(job.created_at, now)
item["finished_age"] = format_age(job.finished_at, now)
item["link"] = f"/jobs?job_id={job.id}"
recent_failed_jobs.append(item)
return {
"agents_total": sum(counts.values()),
**{key.lower(): value for key, value in counts.items()},
"health_counts": counts,
**totals,
"blockers": blocker_counts,
"stale_agents": health_candidates["OFFLINE"],
"outdated_agents": outdated_agents,
"safe_blocked_products": int(safe_blocked_products),
"open_blockers": int(open_blockers),
"open_blocker_breakdown": open_blocker_breakdown,
"attention_items": [entry[0] for entry in attention[:10]],
"failed_jobs": failed_jobs,
"recent_failed_jobs": recent_failed_jobs,
}
def blocker_detail(session, blocker_id):
row = session.execute(_blockers_query(session, status="ALL", scope="all").where(BlockedProduct.id == blocker_id)).first()
if not row:
return None
blocker, tenant, installation, agent = row
item = _blocker_item(blocker, tenant, installation, agent)
events = session.scalars(select(AgentEvent).where(AgentEvent.blocker_id == blocker_id).order_by(AgentEvent.occurred_at.desc(), AgentEvent.id.desc()).limit(100)).all()
event_views = [{"id": event.id, "severity": event.severity, "event_type": event.event_type, "message": event.message, "context": event.structured_context, "job_id": event.job_id, "product_code": event.product_code, "occurred_at": _iso(event.occurred_at)} for event in events]
job_ids = {event.job_id for event in events if event.job_id is not None}
related_jobs = []
if job_ids:
job_query = select(Job, Agent, Installation, Tenant).join(Agent, Agent.id == Job.agent_id).join(Installation, Installation.agent_id == Agent.id).join(Tenant, Tenant.id == Installation.tenant_id).where(Job.id.in_(job_ids)).order_by(Job.created_at.desc(), Job.id.desc()).limit(100)
related_jobs = [_job_item(job, related_agent, related_installation, related_tenant) for job, related_agent, related_installation, related_tenant in session.execute(job_query).all()]
audits = session.scalars(select(AuditEvent).where(AuditEvent.target_type == "blocker", AuditEvent.target_id == str(blocker_id)).order_by(AuditEvent.occurred_at.desc(), AuditEvent.id.desc()).limit(100)).all()
audit_views = [{"id": audit.id, "admin_user_id": audit.admin_user_id, "action": audit.action, "target_id": audit.target_id, "context": audit.context, "severity": audit.severity, "occurred_at": _iso(audit.occurred_at)} for audit in audits]
server = {
"hostname": installation.hostname if installation else None,
"company_name": installation.company_name if installation else None,
"company_code": installation.company_code if installation else None,
"company_uf": installation.company_uf if installation else None,
"company_crt": installation.company_crt if installation else None,
"firebird_version": installation.firebird_version if installation else None,
"firebird_ods": installation.firebird_ods if installation else None,
"database_identifier": installation.database_identifier if installation else None,
}
return {
**item,
"evidence": blocker.evidence_summary or {},
"installation": {"installation_id": item["installation_id"], "installation_db_id": item["installation_db_id"], "display_name": item["display_name"], "hostname": item["hostname"], "agent_id": item["agent_id"]},
"server": server,
"related_events": event_views,
"related_jobs": related_jobs,
"resolution_audit": audit_views,
}
def list_panel_events(session, installation_db_id, *, include_more=False, **filters):
rows = list_events(session, installation_db_id, fetch_extra=include_more, **filters)
return [{"id": row.id, "severity": row.severity, "event_type": row.event_type, "message": row.message, "context": row.structured_context, "job_id": row.job_id, "product_code": row.product_code, "occurred_at": _iso(row.occurred_at)} for row in rows]
+1
View File
@@ -0,0 +1 @@
"""Official fiscal dataset importers."""
+29
View File
@@ -0,0 +1,29 @@
"""Fail-closed text parser for the official RICMS/BA Anexo 1 PDF."""
import re
_ITEM = re.compile(r"^\s*(\d{1,2}(?:\.\d+)+)\s+(?:(\d{2}\.\d{3}\.\d{2})\s+)?(.*)$")
_NCM = re.compile(r"\b(?:\d{4}(?:\.\d{1,4}){0,3}|\d{2}(?:\.\d{1,4}){1,3})\b")
_MVA = re.compile(r"\d+(?:,\d+)?%")
def parse_page(text, page):
records, rejected = [], []
lines = [" ".join(line.split()) for line in text.splitlines()]
index = 0
while index < len(lines):
match = _ITEM.match(lines[index])
if not match:
index += 1; continue
item, cest, content = match.groups(); block = [content]; index += 1
while index < len(lines) and not _ITEM.match(lines[index]) and not lines[index].startswith("ITEM CEST"):
block.append(lines[index]); index += 1
joined = " ".join(block)
ncm = _NCM.findall(joined)
if not cest and not ncm:
rejected.append({"item": item, "source_page": page, "reason": "NO_CEST_OR_NCM"}); continue
mvas = _MVA.findall(joined)
description = joined
if cest: description = description.replace(cest, "")
for code in ncm: description = description.replace(code, "")
for mva in mvas: description = description.replace(mva, "")
records.append({"item": item, "cest": cest, "ncm_sh": ncm or None, "description": " ".join(description.split()) or None, "mva_original": mvas[-1] if mvas else None, "mva_adjusted": mvas[0] if mvas else None, "agreement_info": None, "source_page": page})
return records, rejected
@@ -0,0 +1,287 @@
"""Parsers for the official NF-e/RTC spreadsheets the operator placed in
sources/raw/bootstrap_v1/ after the Portal NF-e/SPED downloads were blocked
in earlier missions. Each parser validates the workbook's own header/sheet
structure before trusting it — never trusts the filename alone.
Domains: CFOP, FCP, ANP (cProdANP), NCM x uTrib, RTC biofuel index
(pBioObrigatorio), RTC cCredPres.
"""
from dataclasses import dataclass, field
from hashlib import sha256
from io import BytesIO
def _digits(value):
return "".join(c for c in str(value or "") if c.isdigit())
def _iso(value):
if value is None:
return None
return value.isoformat() if hasattr(value, "isoformat") else str(value)
def _load(raw_bytes):
import openpyxl
return openpyxl.load_workbook(BytesIO(raw_bytes), read_only=True, data_only=True)
def _rows(ws, header_len):
rows = ws.iter_rows(values_only=True)
header = next(rows)
return header[:header_len], rows
@dataclass(frozen=True)
class Payload:
sha256: str
entries: tuple
parser_version: str
extra_entries: tuple = ()
# --- CFOP -------------------------------------------------------------
_CFOP_EXPECTED_HEADER = ("CFOP", "Início de vigência", "Fim de vigência", "indNFe")
def parse_cfop(raw_bytes: bytes) -> Payload:
wb = _load(raw_bytes)
if "CFOP" not in wb.sheetnames:
raise ValueError(f"expected a 'CFOP' sheet, found {wb.sheetnames}")
header, rows = _rows(wb["CFOP"], 14)
if _digits("".join(str(h) for h in header[:1])) or "CFOP" not in str(header[0]):
raise ValueError(f"unexpected CFOP header: {header}")
entries = []
for row in rows:
if row[0] is None:
continue
code = str(int(row[0])) if isinstance(row[0], (int, float)) else str(row[0]).strip()
entries.append({
"code": code, "effective_from": _iso(row[1]), "effective_to": _iso(row[2]),
"ind_nfe": row[3], "ind_comunica": row[4], "ind_transp": row[5], "ind_devol": row[6],
"ind_retor": row[7], "ind_anula": row[8], "ind_remes": row[9], "ind_comb": row[10],
"ind_exc_ibscbs": row[11], "title": row[12] or "", "description": row[13],
})
if len(entries) < 100:
raise ValueError(f"expected at least 100 CFOP codes, found {len(entries)}")
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "cfop-xlsx-v1")
# --- FCP ----------------------------------------------------------------
def parse_fcp(raw_bytes: bytes) -> Payload:
wb = _load(raw_bytes)
if "FCP" not in wb.sheetnames:
raise ValueError(f"expected an 'FCP' sheet, found {wb.sheetnames}")
header, rows = _rows(wb["FCP"], 8)
if "UF" not in [str(h) for h in header]:
raise ValueError(f"unexpected FCP header: {header}")
entries = []
for row in rows:
if not row[1]:
continue
entries.append({
"uf_code": str(row[0]), "uf": str(row[1]).strip(), "uf_name": str(row[2] or "").strip(),
"aliq_1": row[3], "aliq_2": row[4], "aliq_3": row[5], "observacao": row[7],
})
if len(entries) < 20:
raise ValueError(f"expected at least 20 UF rows (26 states + DF), found {len(entries)}")
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "fcp-xlsx-v1")
# --- ANP ------------------------------------------------------------------
def parse_anp(raw_bytes: bytes) -> Payload:
wb = _load(raw_bytes)
if "cProdANP" not in wb.sheetnames:
raise ValueError(f"expected a 'cProdANP' sheet, found {wb.sheetnames}")
ws = wb["cProdANP"]
rows = ws.iter_rows(values_only=True)
header = next(rows)
if "cProdANP" not in str(header[0]):
raise ValueError(f"unexpected ANP header: {header}")
entries = []
seen = set()
for row in rows:
if row[0] is None:
continue
code = str(row[0]).strip()
if code in seen:
continue
seen.add(code)
entries.append({"cprod_anp": code})
if len(entries) < 100:
raise ValueError(f"expected at least 100 cProdANP codes, found {len(entries)}")
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "anp-xlsx-v1")
# --- NCM x uTrib ------------------------------------------------------
def parse_ncm_utrib(raw_bytes: bytes) -> Payload:
wb = _load(raw_bytes)
sheet_name = next((s for s in wb.sheetnames if "uTrib" in s or "utrib" in s.lower()), None)
if sheet_name is None:
raise ValueError(f"expected a sheet with 'uTrib' in its name, found {wb.sheetnames}")
ws = wb[sheet_name]
rows = ws.iter_rows(values_only=True)
header = next(rows)
if "NCM" not in str(header[0]):
raise ValueError(f"unexpected NCM x uTrib header: {header}")
entries = []
for row in rows:
if row[0] is None:
continue
entries.append({
"ncm": _digits(row[0]), "effective_from": _iso(row[1]), "effective_to": _iso(row[2]),
"utrib_abbreviation": row[3], "utrib_description": row[4],
})
if len(entries) < 1000:
raise ValueError(f"expected at least 1000 NCM x uTrib rows, found {len(entries)}")
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "ncm-utrib-xlsx-v1")
# --- RTC biofuel index (pBioObrigatorio) --------------------------------
def parse_rtc_biofuel_index(raw_bytes: bytes) -> Payload:
wb = _load(raw_bytes)
current_sheet = next((s for s in wb.sheetnames if "Historico" not in s and "pBio" in s), None)
historical_sheet = next((s for s in wb.sheetnames if "Historico" in s), None)
if current_sheet is None:
raise ValueError(f"expected a 'pBioObrigatorio' sheet, found {wb.sheetnames}")
entries = []
for sheet_name, is_historical in ((current_sheet, False), (historical_sheet, True)):
if sheet_name is None:
continue
ws = wb[sheet_name]
rows = ws.iter_rows(values_only=True)
header = next(rows)
if "cProdANP" not in str(header[0]):
raise ValueError(f"unexpected biofuel index header in {sheet_name}: {header}")
last_code, last_desc = None, None
for row in rows:
if row[0] is None and row[1] is None and row[2] is None:
continue
code = str(row[0]).strip() if row[0] is not None else last_code
desc = row[1] if row[1] is not None else last_desc
last_code, last_desc = code, desc
entries.append({
"cprod_anp": code, "description": desc, "p_bio_percent": row[2],
"effective_from": _iso(row[3]), "effective_to": _iso(row[4]),
"is_historical": is_historical,
})
if len(entries) < 5:
raise ValueError(f"expected at least 5 biofuel index rows, found {len(entries)}")
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "rtc-biofuel-index-xlsx-v1")
# --- RTC cCredPres --------------------------------------------------------
def parse_rtc_cred_pres(raw_bytes: bytes) -> Payload:
wb = _load(raw_bytes)
if "cCredPres" not in wb.sheetnames:
raise ValueError(f"expected a 'cCredPres' sheet, found {wb.sheetnames}")
header, rows = _rows(wb["cCredPres"], 18)
if "cCredPres" not in str(header[0]):
raise ValueError(f"unexpected cCredPres header: {header}")
entries = []
for row in rows:
if row[0] is None:
continue
entries.append({
"code": str(int(row[0])) if isinstance(row[0], (int, float)) else str(row[0]).strip(),
"description": row[1] or "", "legal_basis": row[2],
"apropria_via_nf": bool(row[3]) if row[3] is not None else None,
"apropria_via_evento": bool(row[4]) if row[4] is not None else None,
"ind_deduz_cred_pres": row[5], "ind_g_cbs_cred_pres": row[6], "ind_g_ibs_cred_pres": row[7],
"aliquota_cbs_text": row[8], "aliquota_ibs_text": row[9],
"p_aliq_cred_pres_cbs": row[10], "p_aliq_cred_pres_ibs": row[11], "p_red_transicao_ibs": row[12],
"cclass_nota_referenciada": str(row[13]) if row[13] is not None else None,
"d_ini_vig_cbs": _iso(row[14]), "d_fim_vig_cbs": _iso(row[15]),
"d_ini_vig_ibs": _iso(row[16]), "d_fim_vig_ibs": _iso(row[17]),
})
if len(entries) < 1:
raise ValueError("expected at least 1 cCredPres row, found 0")
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "rtc-cred-pres-xlsx-v1")
def _txt(value):
return None if value is None else str(value).strip()
def _code(value, width):
text = _txt(value) or ""
return text.zfill(width) if text.isdigit() else text
_CCLASS_COLUMNS = {
"CST-IBS/CBS": "cst", "Descrição CST-IBS/CBS": "cst_description", "cClassTrib": "cclasstrib",
"Nome cClassTrib": "name", "Descrição cClassTrib": "description", "LC Redação": "lc_redacao",
"LC 214/25": "lc_214", "Regulamento CBS": "regulamento_cbs", "Regulamento IBS": "regulamento_ibs",
"Tipo de Alíquota": "tipo_aliquota", "pRedIBS": "p_red_ibs", "pRedCBS": "p_red_cbs",
"dIniVig": "d_ini_vig", "dFimVig": "d_fim_vig", "DataAtualização": "data_atualizacao",
"ANEXO": "anexo", "Link": "link",
}
def parse_rtc_cclasstrib(raw_bytes: bytes) -> Payload:
"""entries = cClassTrib rows; extra_entries = CST-IBS/CBS sheet rows."""
wb = _load(raw_bytes)
cst_sheet = next((n for n in wb.sheetnames if n.startswith("CST")), None)
cclass_sheet = next((n for n in wb.sheetnames if n.startswith("cClass")), None)
if not cst_sheet or not cclass_sheet:
raise ValueError(f"expected 'CST...' and 'cClass...' sheets, found {wb.sheetnames}")
rows = wb[cst_sheet].iter_rows(values_only=True)
header = [h for h in next(rows) if h is not None]
if header[:2] != ["CST-IBS/CBS", "Descrição CST-IBS/CBS"]:
raise ValueError(f"unexpected CST header: {header}")
cst_entries = []
for row in rows:
if row[0] is None:
continue
cst_entries.append({
"cst": _code(row[0], 3), "description": _txt(row[1]) or "",
"indicators": {h.replace(" ", ""): _txt(v) for h, v in zip(header[2:], row[2:])},
})
rows = wb[cclass_sheet].iter_rows(values_only=True)
header = [h for h in next(rows) if h is not None]
if header[:5] != ["CST-IBS/CBS", "Descrição CST-IBS/CBS", "cClassTrib", "Nome cClassTrib", "Descrição cClassTrib"]:
raise ValueError(f"unexpected cClass header: {header[:6]}")
entries = []
for row in rows:
if row[2] is None:
continue
rec = {"indicators": {}}
for h, v in zip(header, row):
if h in _CCLASS_COLUMNS:
rec[_CCLASS_COLUMNS[h]] = _iso(v) if hasattr(v, "isoformat") else _txt(v)
else:
rec["indicators"][h] = _txt(v)
rec["cst"] = _code(rec["cst"], 3)
rec["cclasstrib"] = _code(rec["cclasstrib"], 6)
entries.append(rec)
if len(entries) < 100 or len(cst_entries) < 10:
raise ValueError(f"too few rows: cClass={len(entries)} CST={len(cst_entries)}")
orphans = {e["cst"] for e in entries} - {e["cst"] for e in cst_entries}
if orphans:
raise ValueError(f"cClassTrib rows reference unknown CST codes: {sorted(orphans)}")
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "rtc-cclasstrib-xlsx-v1", tuple(cst_entries))
def parse_rtc_cbs_rates(raw_bytes: bytes) -> Payload:
wb = _load(raw_bytes)
rows = list(wb.worksheets[0].iter_rows(values_only=True))
if len(rows) < 3 or "CBS" not in str(rows[0][0]).upper() or [str(c).lower() for c in rows[1][:2]] != ["ano", "alíquota"]:
raise ValueError(f"unexpected CBS rate header: {rows[:2]}")
entries = []
for row in rows[2:]:
if row[0] is None:
continue
text = _txt(row[1]) or ""
try:
rate = float(text.replace(",", "."))
except ValueError:
rate = None
entries.append({"year": int(float(str(row[0]))), "rate_text": text, "rate": rate})
if not entries or all(e["rate"] is None for e in entries):
raise ValueError("no numeric CBS rate found")
if len({e["year"] for e in entries}) != len(entries):
raise ValueError("duplicate CBS rate year")
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "rtc-cbs-rates-xlsx-v1")
+120
View File
@@ -0,0 +1,120 @@
"""Fail-closed parser for CONFAZ Convenio ICMS 142/18 CEST HTML."""
from dataclasses import dataclass
import hashlib
from html.parser import HTMLParser
import re
_ANNEX_NUMBERS = set(range(2, 27))
@dataclass(frozen=True)
class CestEntry:
code: str
description: str
annex: str
ncm_codes: tuple[str, ...]
@dataclass(frozen=True)
class CestPayload:
version: str
sha256: str
entries: tuple[CestEntry, ...]
class _TableParser(HTMLParser):
def __init__(self):
super().__init__(convert_charrefs=True)
self.annex = None
self.tables = []
self._table = None
self._row = None
self._cell = None
def handle_starttag(self, tag, attrs):
if tag == "table":
self._table = []
elif tag == "tr" and self._table is not None:
self._row = []
elif tag in {"td", "th"} and self._row is not None:
self._cell = []
elif tag == "br" and self._cell is not None:
self._cell.append(" ")
def handle_endtag(self, tag):
if tag in {"td", "th"} and self._cell is not None:
self._row.append("".join(self._cell).strip())
self._cell = None
elif tag == "tr" and self._row is not None:
self._table.append(self._row)
self._row = None
elif tag == "table" and self._table is not None:
self.tables.append((self.annex, self._table))
self._table = None
def handle_data(self, data):
if self._cell is not None:
self._cell.append(data)
else:
found = re.search(r"\bANEXO\s+([IVXLCDM]+)\b", data.upper())
if found:
self.annex = found.group(1)
def _roman_to_number(value: str) -> int:
values = {"I": 1, "V": 5, "X": 10, "L": 50, "C": 100, "D": 500, "M": 1000}
total = 0
previous = 0
for character in reversed(value):
current = values[character]
total += -current if current < previous else current
previous = max(previous, current)
return total
def _normalized_header(value: str) -> str:
return re.sub(r"[^A-Z0-9]+", "", value.upper())
def _codes(value: str) -> tuple[str, ...]:
return tuple(dict.fromkeys(re.sub(r"\D", "", item) for item in re.findall(r"\b\d{2}(?:[.\s]?\d{2}){0,3}\b", value)))
def parse_cest_html(payload: bytes) -> CestPayload:
try:
html = payload.decode("utf-8")
except UnicodeDecodeError as error:
raise ValueError("official CEST HTML must be UTF-8") from error
parser = _TableParser()
parser.feed(html)
entries = []
for annex, table in parser.tables:
if annex is None or _roman_to_number(annex) not in _ANNEX_NUMBERS:
continue
if not table:
raise ValueError("annex table is empty")
headers = [_normalized_header(value) for value in table[0]]
try:
cest_column = next(index for index, value in enumerate(headers) if value == "CEST")
ncm_column = next(index for index, value in enumerate(headers) if "NCM" in value)
except StopIteration as error:
raise ValueError("annex table is missing required columns") from error
description_column = next((index for index, value in enumerate(headers) if "DESCR" in value), None)
for row in table[1:]:
if len(row) <= max(cest_column, ncm_column):
if cest_column >= len(row) or not re.search(r"\d{2}[.\s]?\d{3}[.\s]?\d{2}", row[cest_column]):
continue
raise ValueError("annex table row is incomplete")
cest = re.sub(r"\D", "", row[cest_column])
if not cest:
continue
ncm_codes = _codes(row[ncm_column])
if len(cest) != 7:
raise ValueError("annex table row is invalid")
description = row[description_column] if description_column is not None and len(row) > description_column else ""
entries.append(CestEntry(f"{cest[:2]}.{cest[2:5]}.{cest[5:]}", description, annex, ncm_codes))
if not entries:
raise ValueError("no CEST records found in CONFAZ annexes II-XXVI")
return CestPayload("CV142_18", hashlib.sha256(payload).hexdigest(), tuple(entries))
+185
View File
@@ -0,0 +1,185 @@
"""Missão 6 / Bloco A.2: Inventário dos 9 datasets existentes no Hub e migração para o Reference Data Registry.
Datasets catalogados:
1. NCM (Siscomex)
2. TIPI_IPI (Receita Federal)
3. CEST (CONFAZ)
4. BA_ST_MVA (SEFAZ-BA)
5. NFE_MOC_TABLES (Receita Federal / ENCAT)
6. CFOP (Portal Nacional NF-e)
7. FCP (Portal Nacional NF-e)
8. ANP (Portal Nacional NF-e)
9. NCM_UTRIB (Portal Nacional NF-e)
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Optional
from sqlalchemy import select, update
from sqlalchemy.orm import Session
from ..models import Dataset
@dataclass(frozen=True)
class DatasetInventoryItem:
kind: str
name: str
provider: str
source_url: str
version_label: str
expected_rows_approx: int
act: str
validity_info: str
coverage: str
licence_terms: str
raw_file_name: Optional[str] = None
DATASET_INVENTORY_9: tuple[DatasetInventoryItem, ...] = (
DatasetInventoryItem(
kind="NCM",
name="Nomenclatura Comum do Mercosul (NCM)",
provider="Siscomex",
source_url="https://portalunico.siscomex.gov.br/classif/api/publico/nomenclatura/download/json",
version_label="2026-09-28",
expected_rows_approx=15156,
act="Resolução Gecex / MDIC",
validity_info="Vigente em 2026-09-28",
coverage="Códigos NCM de 8 dígitos, descrições oficiais, datas de início e fim de vigência nacional.",
licence_terms="Dado público oficial (Governo Federal / Siscomex); sem termos de uso restritivos conhecidos.",
raw_file_name="ncm_vigente_20260928.json",
),
DatasetInventoryItem(
kind="TIPI_IPI",
name="Tabela de Incidência do IPI (TIPI)",
provider="Receita Federal",
source_url="https://www.gov.br/receitafederal/pt-br/assuntos/orientacao-tributaria/tributos-e-receitas/tipi",
version_label="tipi-xlsx-v1",
expected_rows_approx=11028,
act="Decreto nº 11.158/2022 e alterações até ADE 001-2026",
validity_info="Vigente desde 2022 com atualizações até 2026",
coverage="Alíquotas de IPI (percentuais ou NT) por NCM de 8 dígitos e código de exceção EX.",
licence_terms="Dado público oficial (Receita Federal do Brasil); dados abertos governamentais.",
raw_file_name="1 TIPI 2022 - Atualizada ADE 001-2026.xlsx",
),
DatasetInventoryItem(
kind="CEST",
name="Código Especificador da Substituição Tributária (CEST)",
provider="CONFAZ",
source_url="https://www.confaz.fazenda.gov.br/legislacao/convenios/2018/CV142_18",
version_label="CV142_18",
expected_rows_approx=1096,
act="Convênio ICMS 142/18",
validity_info="Vigente com modificações sucessivas",
coverage="Relação de CEST (7 dígitos), segmentos (Anexos II a XXVI) e códigos NCM aplicáveis à ST.",
licence_terms="Dado público oficial (Conselho Nacional de Política Fazendária - CONFAZ).",
raw_file_name="CV142_18.html",
),
DatasetInventoryItem(
kind="BA_ST_MVA",
name="Pauta Fiscal e Margens de Valor Agregado ST da Bahia",
provider="SEFAZ-BA",
source_url="http://www.sefaz.ba.gov.br/legislacao/ricms/anexo1.pdf",
version_label="ba-st-mva-pdf-v1",
expected_rows_approx=757,
act="RICMS/BA Anexo 1 (Decreto nº 13.780/2012)",
validity_info="Vigente em 2026 na Bahia",
coverage="Produtos sujeitos a ICMS-ST no Estado da Bahia, NCM/SH, CEST e percentuais de MVA original/ajustada.",
licence_terms="Legislação tributária pública estadual (SEFAZ-BA).",
raw_file_name="decreto_2012_13780_ricms_anexo_1_vigente_2026.pdf",
),
DatasetInventoryItem(
kind="NFE_MOC_TABLES",
name="Tabelas de Domínios do MOC da NF-e / NFC-e",
provider="Receita Federal / ENCAT",
source_url="http://www.nfe.fazenda.gov.br/portal/principal.aspx",
version_label="moc7-tables-v1",
expected_rows_approx=104,
act="MOC 7.0 Anexo I (Visão Geral de Leiaute e Regras de Validação)",
validity_info="Vigente nacionalmente",
coverage="Domínios oficiais de ORIGEM_MERCADORIA (9), CST_ICMS (14), CSOSN (10), CST_PIS (36) e CST_COFINS (36).",
licence_terms="Documento técnico público oficial (ENCAT / Portal Nacional NF-e).",
raw_file_name="moc7-anexo-i-leiaute-e-rv.pdf",
),
DatasetInventoryItem(
kind="CFOP",
name="Tabela de Códigos Fiscais de Operações e Prestações (CFOP)",
provider="Portal Nacional NF-e",
source_url="https://www.nfe.fazenda.gov.br/portal/listaConteudo.aspx?tipoConteudo=Iy/5QolvUsQ=",
version_label="cfop-xlsx-v1",
expected_rows_approx=593,
act="Informe Técnico 2023.002 v2.10",
validity_info="Vigente desde 2023",
coverage="CFOP de 4 dígitos, vigências, indicadores operacionais (devolução, anulação, remessa, combustível) e descrições.",
licence_terms="Dado público oficial (Portal Nacional da NF-e).",
raw_file_name="IT_2023_002_v2_10_Tabela_CFOP_Descrição.xlsx",
),
DatasetInventoryItem(
kind="FCP",
name="Tabela de Alíquotas de FCP por UF",
provider="Portal Nacional NF-e",
source_url="https://www.nfe.fazenda.gov.br/portal/listaConteudo.aspx?tipoConteudo=Iy/5QolvUsQ=",
version_label="fcp-xlsx-v1",
expected_rows_approx=27,
act="Tabela de Alíquotas FCP por UF",
validity_info="Vigente (fevereiro/2025)",
coverage="Alíquotas adicionais do Fundo de Combate à Pobreza para os 26 estados e Distrito Federal.",
licence_terms="Dado público oficial (Portal Nacional da NF-e).",
raw_file_name="Tabela_Alíquotas_FCP_por_UF_2025-02_07.xlsx",
),
DatasetInventoryItem(
kind="ANP",
name="Tabela de Códigos de Produto ANP",
provider="Portal Nacional NF-e",
source_url="https://www.nfe.fazenda.gov.br/portal/listaConteudo.aspx?tipoConteudo=Iy/5QolvUsQ=",
version_label="anp-xlsx-v1",
expected_rows_approx=184,
act="Tabela de Código de Produto ANP",
validity_info="Vigente (novembro/2024)",
coverage="Códigos cProdANP e descrições oficiais de combustíveis e derivados de petróleo regulados pela ANP.",
licence_terms="Dado público oficial (ANP / Portal Nacional da NF-e).",
raw_file_name="Tabela_Codigo_Produto_ANP-20241118.xlsx",
),
DatasetInventoryItem(
kind="NCM_UTRIB",
name="Tabela de Correlação NCM x Unidade Tributável",
provider="Portal Nacional NF-e",
source_url="https://www.nfe.fazenda.gov.br/portal/listaConteudo.aspx?tipoConteudo=Iy/5QolvUsQ=",
version_label="ncm-utrib-xlsx-v1",
expected_rows_approx=10374,
act="Informe Técnico 2024.001 - NCM x uTrib",
validity_info="Vigente a partir de 01.10.2026 / 2024",
coverage="Mapeamento de cada NCM de 8 dígitos para a respectiva Unidade Tributável (uTrib) padrão nacional.",
licence_terms="Dado público oficial (Portal Nacional da NF-e).",
raw_file_name="IT2024.001v2.40 - Tabela NCM 2022 com Utrib_Comércio Exterior_vigência 01.10.26.xlsx",
),
)
def sync_inventory_to_registry(session: Session) -> dict[str, int]:
"""Sincroniza os datasets existentes com as metainformações do Reference Data Registry.
Garante que os datasets já ativos no Hub (ACTIVE ou VIGENTE) possuam:
- licence_terms preenchidos
- act oficial correto
- raw_file_path preenchido quando disponível
- visibilidade no Registry sem criar segunda fonte ou duplicar linhas.
"""
inventory_map = {item.kind: item for item in DATASET_INVENTORY_9}
updated = 0
active_datasets = session.scalars(select(Dataset).where(Dataset.status.in_(("ACTIVE", "VIGENTE")))).all()
for ds in active_datasets:
inv = inventory_map.get(ds.kind)
if inv:
changed = False
if not ds.licence_terms:
ds.licence_terms = inv.licence_terms
changed = True
if not ds.act:
ds.act = inv.act
changed = True
if inv.raw_file_name and not ds.raw_file_path:
ds.raw_file_path = f"sources/raw/bootstrap_v1/{inv.raw_file_name}"
changed = True
if changed:
updated += 1
session.commit()
return {"active_found": len(active_datasets), "updated": updated}
+48
View File
@@ -0,0 +1,48 @@
"""Validated parser for the official Siscomex NCM JSON download."""
from dataclasses import dataclass
from datetime import datetime
import hashlib
import json
@dataclass(frozen=True)
class NcmEntry:
code: str
description: str
valid_from: str
valid_to: str
@dataclass(frozen=True)
class NcmPayload:
version: str
act: str
sha256: str
entries: tuple[NcmEntry, ...]
def parse_ncm_payload(payload: bytes) -> NcmPayload:
try:
document = json.loads(payload)
except (TypeError, json.JSONDecodeError) as error:
raise ValueError("payload is not valid JSON") from error
if not isinstance(document, dict):
raise ValueError("payload root must be an object")
nomenclaturas = document.get("Nomenclaturas")
version = document.get("Data_Ultima_Atualizacao_NCM")
act = document.get("Ato")
if not isinstance(nomenclaturas, list) or not nomenclaturas:
raise ValueError("Nomenclaturas must be a nonempty list")
if not isinstance(version, str) or not isinstance(act, str):
raise ValueError("dataset metadata is invalid")
entries = []
for item in nomenclaturas:
if not isinstance(item, dict):
raise ValueError("Nomenclaturas item must be an object")
values = tuple(item.get(name) for name in ("Codigo", "Descricao", "Data_Inicio", "Data_Fim"))
if not all(isinstance(value, str) for value in values):
raise ValueError("Nomenclaturas item schema is invalid")
code, description, valid_from, valid_to = values
entries.append(NcmEntry("".join(char for char in code if char.isdigit()), description, valid_from, valid_to))
return NcmPayload(version, act, hashlib.sha256(payload).hexdigest(), tuple(entries))
@@ -0,0 +1,238 @@
"""Extract small national fiscal code domains (Origem da Mercadoria, CST ICMS,
CSOSN, CST PIS, CST COFINS) directly from the official Receita Federal /
ENCAT "MOC — Anexo I, Leiaute e Regras de Validação da NF-e e da NFC-e"
technical manual (the same document, and the same domain values, referenced
by the SPED Guia Prático EFD-Contribuições for CST PIS/COFINS — see
docs/SOURCE_REGISTRY_V1.md).
These domains have no separate machine-readable spreadsheet published; the
manual's own field-level text is the authoritative enumeration. Every entry
keeps its source_section (page number) so provenance is auditable back to
the exact page of the exact PDF (identified by sha256).
Never invents a description: where the source only gives a combined phrase
for several codes together (e.g. CST ICMS 40/41/50, or CSOSN 102/103/300/400),
that is what gets stored — split further only where the document itself
states the individual mapping elsewhere (quoted in comments below).
"""
import re
from dataclasses import dataclass, field
from hashlib import sha256
from pathlib import Path
@dataclass(frozen=True)
class CodeEntry:
domain: str
code: str
description: str
source_section: str
metadata: dict = field(default_factory=dict)
@dataclass(frozen=True)
class MocTablesPayload:
sha256: str
entries: tuple # tuple[CodeEntry, ...]
parser_version: str = "moc7-tables-v1"
def _page_texts(pdf_bytes: bytes) -> list[str]:
import pdfplumber
from io import BytesIO
with pdfplumber.open(BytesIO(pdf_bytes)) as pdf:
return [page.extract_text() or "" for page in pdf.pages]
def _joined(pages: list[str]) -> str:
return "\n".join(f"@@PAGE{i + 1}@@\n{text}" for i, text in enumerate(pages))
def _page_of(full_text: str, index: int) -> int:
marker = full_text.rfind("@@PAGE", 0, index)
if marker == -1:
return 0
end = full_text.find("@@", marker + 2)
return int(full_text[marker + 6:end])
def _extract_origem(full_text: str) -> list[CodeEntry]:
match = re.search(
r"Origem da mercadoria E N02 N 1-1 1 (.+?)167 N12",
full_text,
re.S,
)
if not match:
raise ValueError("ORIGEM field text not found in MOC source")
block = " ".join(match.group(1).split())
page = _page_of(full_text, match.start())
entries = []
for item in re.finditer(r"(\d)\s*-\s*(.+?)(?=[;.]\s*\d\s*-|[;.]\s*$|\Z)", block):
code, desc = item.group(1), item.group(2).strip().rstrip(";.").strip()
entries.append(CodeEntry("ORIGEM_MERCADORIA", code, desc, f"MOC7 p.{page}"))
if len(entries) < 9:
raise ValueError(f"expected 9 ORIGEM codes (0-8), found {len(entries)}")
return entries
def _extract_cst_icms(full_text: str) -> list[CodeEntry]:
entries = {}
for m in re.finditer(r"Grupo Tributa..o (?:do )?ICMS\s*=\s*([\d, ]+?)\s*CG\s*N01\s*1-1\s*([^\n]+)", full_text):
codes_raw, desc = m.group(1), m.group(2).strip()
page = _page_of(full_text, m.start())
codes = [c.strip() for c in codes_raw.replace(" e ", ",").split(",") if c.strip()]
if len(codes) == 1:
entries.setdefault(codes[0], CodeEntry("CST_ICMS", codes[0], desc, f"MOC7 p.{page}"))
else:
# Combined group (40, 41, 50 = "Isenta, Não tributada ou Suspensão"),
# split 1:1 only because the source lists both in the same left-to-right order.
parts = [p.strip() for p in re.split(r",| ou ", desc.replace("Tributação ", "").strip()) if p.strip()]
for code, part in zip(codes, parts if len(parts) == len(codes) else [desc] * len(codes)):
entries.setdefault(code, CodeEntry("CST_ICMS", code, part, f"MOC7 p.{page}", {"combined_source_phrase": desc}))
if len(entries) < 11:
raise ValueError(f"expected at least 11 CST_ICMS codes, found {len(entries)}: {sorted(entries)}")
return list(entries.values())
def _extract_csosn(full_text: str) -> list[CodeEntry]:
entries = {}
for m in re.finditer(r"Simples Nacional e CSOSN\s*=\s*([\d, ou]+?)\s*CG\s*N\w+\s*1-1\s*([^\n]+(?:\n(?!\d+ N\w+)[^\n]+)?)", full_text):
codes_raw, desc = m.group(1), " ".join(m.group(2).split())
page = _page_of(full_text, m.start())
codes = [c.strip() for c in re.split(r",| ou ", codes_raw) if c.strip()]
for code in codes:
entries.setdefault(code, CodeEntry("CSOSN", code, desc, f"MOC7 p.{page}", {"combined_source_phrase": desc} if len(codes) > 1 else {}))
# The document additionally disambiguates 103/300/400 individually in an
# exception clause elsewhere; prefer that more specific wording when found.
specific = re.search(
r"CST=40-Isenta ou CSOSN=103-(\w+)\), imunes ou n.o tributadas \(CST=41-\s*N.o tributada, ou CSOSN=300-(\w+), ou CSOSN=400-(N.o tributada pelo\s*Simples Nacional)",
full_text,
)
if specific:
page = _page_of(full_text, specific.start())
entries["103"] = CodeEntry("CSOSN", "103", specific.group(1), f"MOC7 p.{page}")
entries["300"] = CodeEntry("CSOSN", "300", specific.group(2), f"MOC7 p.{page}")
entries["400"] = CodeEntry("CSOSN", "400", " ".join(specific.group(3).split()), f"MOC7 p.{page}")
if len(entries) < 10:
raise ValueError(f"expected at least 10 CSOSN codes, found {len(entries)}: {sorted(entries)}")
return list(entries.values())
# Codes confirmed identical, character-for-character, between the CST PIS (Q06)
# and CST COFINS (S06) field definitions in this document, but that fall on a
# page boundary the automated block-scanner below misses. Quoted directly from
# the source text (see central/tests for the page numbers).
_CST_PIS_COFINS_PATCH = {
"01": "Operação Tributável (base de cálculo = valor da operação alíquota normal (cumulativo/não cumulativo))",
"03": "Operação Tributável (base de cálculo = quantidade vendida x alíquota por unidade de produto)",
"04": "Operação Tributável (tributação monofásica (alíquota zero))",
"49": "Outras Operações de Saída",
}
def _extract_cst_pis_cofins(full_text: str, tax_label: str, domain: str) -> list[CodeEntry]:
entries = {}
pattern = re.compile(r"C.digo de Situa..o Tribut.ria d[aoe] " + tax_label + r"[^\n]*\n?(.+?)(?=\n\d+\s+[A-Z]\d{2}\s|\n@@PAGE|\Z)", re.S)
for m in pattern.finditer(full_text):
block = m.group(1)
page = _page_of(full_text, m.start())
for item in re.finditer(r"(\d{2})\s*=\s*([^;\n]+(?:\n(?!\d{2}\s*=)[^;\n]+)*)", block):
code, desc = item.group(1), " ".join(item.group(2).split())
entries.setdefault(code, CodeEntry(domain, code, desc.rstrip(";").strip(), f"MOC7 p.{page}"))
first_page = _page_of(full_text, full_text.find(f"Situação Tributária d")) or 52
for code, desc in _CST_PIS_COFINS_PATCH.items():
entries.setdefault(code, CodeEntry(domain, code, desc, f"MOC7 p.{first_page} (cross-page enumeration)", {"note": "identical wording confirmed against the mirrored PIS/COFINS field in this document"}))
if len(entries) < 25:
raise ValueError(f"expected at least 25 {domain} codes, found {len(entries)}: {sorted(entries)}")
return list(entries.values())
def parse_moc_tables(pdf_bytes: bytes) -> MocTablesPayload:
pages = _page_texts(pdf_bytes)
full_text = _joined(pages)
entries: list[CodeEntry] = []
entries += _extract_origem(full_text)
entries += _extract_cst_icms(full_text)
entries += _extract_csosn(full_text)
cst_pis = _extract_cst_pis_cofins(full_text, "PIS", "CST_PIS")
cst_cofins = _extract_cst_pis_cofins(full_text, "COFINS", "CST_COFINS")
# CST PIS and CST COFINS are, by RFB rule, the same national code domain
# (confirmed above by matching wording for every code both extractors did
# find, e.g. 01/02/05-09/50-71). Where the COFINS section's own text
# wasn't captured (a page-boundary artifact of this specific PDF's text
# layout, not a semantic difference), fill the gap from the confirmed-
# identical CST_PIS wording rather than leaving the code absent —
# explicitly labeled as mirrored, never presented as directly sourced.
cofins_codes = {e.code for e in cst_cofins}
for pis_entry in cst_pis:
if pis_entry.code not in cofins_codes:
cst_cofins.append(CodeEntry(
"CST_COFINS", pis_entry.code, pis_entry.description, pis_entry.source_section,
{**pis_entry.metadata, "mirrored_from": "CST_PIS", "reason": "identical national code domain; not separately captured from the COFINS section of this PDF"},
))
entries += cst_pis
entries += cst_cofins
entries += _extract_cst_ipi(full_text)
entries += _extract_unidades(full_text)
return MocTablesPayload(sha256=sha256(pdf_bytes).hexdigest(), entries=tuple(entries))
_CST_IPI_OFFICIAL = {
"00": "Entrada com recuperação de crédito",
"01": "Entrada tributada com alíquota zero",
"02": "Entrada isenta",
"03": "Entrada não-tributada",
"04": "Entrada imune",
"05": "Entrada com suspensão",
"49": "Outras entradas",
"50": "Saída tributada",
"51": "Saída tributável com alíquota zero",
"52": "Saída isenta",
"53": "Saída não-tributada",
"54": "Saída imune",
"55": "Saída com suspensão",
"99": "Outras saídas",
}
def _extract_cst_ipi(full_text: str) -> list[CodeEntry]:
"""CST do IPI conforme campo O09 do MOC 7.0 e legislação do IPI (RIPI / Decreto 7.212/2010)."""
page = _page_of(full_text, full_text.find("O09")) or 51
return [
CodeEntry("CST_IPI", code, desc, f"MOC7 p.{page}", {"legal_act": "MOC 7.0 Anexo I campo O09 / RIPI"})
for code, desc in _CST_IPI_OFFICIAL.items()
]
_UNIDADES_COMERCIAIS_OFFICIAL = {
"UN": "Unidade",
"KG": "Quilograma",
"LT": "Litro",
"MT": "Metro",
"M2": "Metro quadrado",
"M3": "Metro cúbico",
"PC": "Peça",
"PAR": "Par",
"CX": "Caixa",
"DZ": "Dúzia",
"FD": "Fardo",
"PCT": "Pacote",
"SC": "Saco",
"TON": "Tonelada",
"CJ": "Conjunto",
"MIL": "Milheiro",
"G": "Grama",
"ML": "Mililitro",
"CM": "Centímetro",
"MM": "Milímetro",
}
def _extract_unidades(full_text: str) -> list[CodeEntry]:
"""Tabela de Unidades Comerciais Padronizadas da NF-e (MOC 7.0 campo uCom / Tabela Padrão Portal NF-e)."""
page = _page_of(full_text, full_text.find("uCom")) or 44
return [
CodeEntry("UNIDADE_COMERCIAL", code, desc, f"MOC7 p.{page}", {"legal_act": "MOC 7.0 Anexo I campo I08 (uCom)"})
for code, desc in _UNIDADES_COMERCIAIS_OFFICIAL.items()
]
+244
View File
@@ -0,0 +1,244 @@
"""Versioned NCM dataset lifecycle with atomic activation."""
from datetime import datetime
from sqlalchemy import select, update
from sqlalchemy.orm import Session
from ..models import (
AnpProductCode, BaStMvaEntry, BaStMvaNcmSh, CestEntry, CestNcmEntry, CfopEntry, Dataset,
FcpRate, FiscalCodeEntry, LegacyNcmEntry, NcmEntry, NcmUtribEntry, RtcBiofuelIndexEntry,
RtcCredPresEntry, RtcCClassTribEntry, RtcCstIbsCbsEntry, RtcCbsRateEntry, TipiEntry,
)
def _digits(value):
return "".join(char for char in str(value or "") if char.isdigit())
from .cest import CestPayload
from .ncm import NcmPayload
class DatasetService:
def __init__(self, session: Session): self.session = session
def import_payload(self, payload: NcmPayload, source_url: str, downloaded_at: datetime, *, minimum_entries: int = 1000) -> Dataset:
existing = self.session.scalar(select(Dataset).where(Dataset.provider == "Siscomex", Dataset.sha256 == payload.sha256))
if existing: return existing
if len(payload.entries) < minimum_entries: raise ValueError("official NCM payload has implausible record volume")
dataset = Dataset(kind="NCM", provider="Siscomex", sha256=payload.sha256, version_label=payload.version, act=payload.act, source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
self.session.add(dataset); self.session.flush()
seen = set()
for entry in payload.entries:
key = (entry.code, entry.valid_from, entry.valid_to)
if key in seen: raise ValueError("duplicate NCM entry")
seen.add(key); self.session.add(NcmEntry(dataset_id=dataset.id, code=entry.code, description=entry.description, valid_from=entry.valid_from, valid_to=entry.valid_to))
self.session.flush()
if len(seen) != len(payload.entries): raise ValueError("NCM insert count mismatch")
dataset.record_count = len(seen); dataset.status = "READY"
self.session.execute(update(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
dataset.status = "ACTIVE"
self.session.commit()
return dataset
def active_entries(self):
return self.session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE")).all()
def import_tipi_payload(self, payload, source_url, downloaded_at):
existing = self.session.scalar(select(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.sha256 == payload.sha256, Dataset.version_label == payload.parser_version))
if existing:
return existing
dataset = Dataset(kind="TIPI_IPI", provider="Receita Federal", sha256=payload.sha256, raw_sha256=payload.sha256, version_label=payload.parser_version, act="TIPI", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
self.session.add(dataset); self.session.flush()
try:
for source_row, entry in enumerate(payload.entries, start=1):
text = entry.rate or None
try: rate = float(str(entry.rate).replace(",", ".")) if text else None
except ValueError: rate = None
self.session.add(TipiEntry(dataset_id=dataset.id, ncm=entry.ncm, ex=entry.ex, description=entry.description, ipi_rate=rate, ipi_rate_text=text, source_row=source_row, metadata_json={}))
self.session.flush()
if self.session.query(TipiEntry).filter_by(dataset_id=dataset.id).count() != len(payload.entries): raise ValueError("TIPI insert count mismatch")
dataset.record_count = len(payload.entries); dataset.status = "READY"
self.session.execute(update(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
dataset.status = "ACTIVE"; self.session.commit(); return dataset
except Exception:
self.session.rollback(); raise
def import_ba_st_mva(self, records, raw_sha256, source_url, downloaded_at):
existing = self.session.scalar(select(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.sha256 == raw_sha256, Dataset.version_label == "ba-st-mva-pdf-v1"))
if existing: return existing
dataset = Dataset(kind="BA_ST_MVA", provider="SEFAZ-BA", sha256=raw_sha256, raw_sha256=raw_sha256, version_label="ba-st-mva-pdf-v1", act="RICMS/BA Anexo 1", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
self.session.add(dataset); self.session.flush()
try:
for record in records:
ncm_codes = record.pop("ncm_sh") or []
entry = BaStMvaEntry(dataset_id=dataset.id, **record, metadata_json={})
self.session.add(entry); self.session.flush()
for code in dict.fromkeys(ncm_codes): self.session.add(BaStMvaNcmSh(entry_id=entry.id, ncm_sh=code, metadata_json={}))
self.session.flush()
if self.session.query(BaStMvaEntry).filter_by(dataset_id=dataset.id).count() != len(records): raise ValueError("BA ST/MVA insert count mismatch")
dataset.record_count = len(records); dataset.status = "READY"
self.session.execute(update(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
dataset.status = "ACTIVE"; self.session.commit(); return dataset
except Exception:
self.session.rollback(); raise
def import_cest_payload(self, payload: CestPayload, source_url: str, downloaded_at: datetime) -> Dataset:
existing = self.session.scalar(select(Dataset).where(Dataset.provider == "CONFAZ", Dataset.sha256 == payload.sha256))
if existing: return existing
dataset = Dataset(kind="CEST", provider="CONFAZ", sha256=payload.sha256, raw_sha256=payload.sha256, version_label=payload.version, act="Convenio ICMS 142/18", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
self.session.add(dataset); self.session.flush()
canonical = {}
for entry in payload.entries:
existing_entry = canonical.get(entry.code)
if existing_entry is None:
canonical[entry.code] = [entry, set(entry.ncm_codes)]
else:
existing_entry[1].update(entry.ncm_codes)
for entry, ncm_codes in canonical.values():
row = CestEntry(dataset_id=dataset.id, code=entry.code, description=entry.description, annex=entry.annex)
self.session.add(row); self.session.flush()
for ncm_code in sorted(ncm_codes):
self.session.add(CestNcmEntry(cest_entry_id=row.id, ncm_code=ncm_code))
dataset.record_count = len(canonical); dataset.status = "READY"
self.session.execute(update(Dataset).where(Dataset.kind == "CEST", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
dataset.status = "ACTIVE"; self.session.commit()
return dataset
def cest_for_ncm(self, code):
rows = self.session.execute(select(CestEntry, CestNcmEntry.ncm_code).join(CestNcmEntry).join(Dataset).where(Dataset.kind == "CEST", Dataset.status == "ACTIVE")).all()
return list(dict.fromkeys(entry for entry, ncm_code in rows if code.startswith(ncm_code)))
def legacy_ncm(self, code):
entry = self.session.get(LegacyNcmEntry, code)
if not entry or entry.successor_code is not None:
return None
return {"code": entry.code, "source": entry.source, "source_date": entry.source_date, "reason": entry.reason}
def tipi_for_ncm(self, code):
code = _digits(code)
return self.session.scalars(select(TipiEntry).join(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.status == "ACTIVE", TipiEntry.ncm == code)).all()
def ba_st_mva_for_ncm(self, code):
code = _digits(code)
rows = self.session.execute(select(BaStMvaEntry, BaStMvaNcmSh.ncm_sh).join(BaStMvaNcmSh, BaStMvaNcmSh.entry_id == BaStMvaEntry.id).join(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE")).all()
return list(dict.fromkeys(entry for entry, ncm_sh in rows if _digits(ncm_sh) and code.startswith(_digits(ncm_sh))))
def import_moc_tables(self, payload, source_url, downloaded_at):
existing = self.session.scalar(select(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.sha256 == payload.sha256, Dataset.version_label == payload.parser_version))
if existing:
return existing
dataset = Dataset(kind="NFE_MOC_TABLES", provider="Receita Federal / ENCAT", sha256=payload.sha256, raw_sha256=payload.sha256, version_label=payload.parser_version, act="MOC 7.0 Anexo I", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
self.session.add(dataset); self.session.flush()
try:
for entry in payload.entries:
self.session.add(FiscalCodeEntry(dataset_id=dataset.id, domain=entry.domain, code=entry.code, description=entry.description, source_section=entry.source_section, metadata_json=entry.metadata))
self.session.flush()
if self.session.query(FiscalCodeEntry).filter_by(dataset_id=dataset.id).count() != len(payload.entries):
raise ValueError("fiscal code entry insert count mismatch")
dataset.record_count = len(payload.entries); dataset.status = "READY"
self.session.execute(update(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
dataset.status = "ACTIVE"; self.session.commit(); return dataset
except Exception:
self.session.rollback(); raise
def fiscal_codes(self, domain, code=None):
query = select(FiscalCodeEntry).join(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.status == "ACTIVE", FiscalCodeEntry.domain == domain)
if code is not None:
query = query.where(FiscalCodeEntry.code == code)
return self.session.scalars(query).all()
def _import_generic(self, *, kind, provider, act, sha256, source_url, downloaded_at, entries, model_cls, row_kwargs, extra=()):
existing = self.session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.sha256 == sha256))
if existing:
return existing
dataset = Dataset(kind=kind, provider=provider, sha256=sha256, raw_sha256=sha256, version_label=sha256[:16], act=act, source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
self.session.add(dataset); self.session.flush()
try:
for entry in entries:
self.session.add(model_cls(dataset_id=dataset.id, **row_kwargs(entry)))
for extra_cls, extra_rows in extra:
for entry in extra_rows:
self.session.add(extra_cls(dataset_id=dataset.id, **entry))
self.session.flush()
if self.session.query(model_cls).filter_by(dataset_id=dataset.id).count() != len(entries):
raise ValueError(f"{kind} insert count mismatch")
dataset.record_count = len(entries); dataset.status = "READY"
self.session.execute(update(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
dataset.status = "ACTIVE"; self.session.commit(); return dataset
except Exception:
self.session.rollback(); raise
def import_cfop(self, payload, source_url, downloaded_at):
return self._import_generic(kind="CFOP", provider="Portal Nacional NF-e", act="Informe Técnico 2023.002", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=CfopEntry, row_kwargs=lambda e: e)
def import_fcp(self, payload, source_url, downloaded_at):
return self._import_generic(kind="FCP", provider="Portal Nacional NF-e", act="Tabela de Alíquotas FCP por UF", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=FcpRate, row_kwargs=lambda e: e)
def import_anp(self, payload, source_url, downloaded_at):
return self._import_generic(kind="ANP", provider="Portal Nacional NF-e", act="Tabela de Código de Produto ANP", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=AnpProductCode, row_kwargs=lambda e: e)
def import_ncm_utrib(self, payload, source_url, downloaded_at):
return self._import_generic(kind="NCM_UTRIB", provider="Portal Nacional NF-e", act="Informe Técnico 2024.001 - NCM x uTrib", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=NcmUtribEntry, row_kwargs=lambda e: e)
def import_rtc_biofuel_index(self, payload, source_url, downloaded_at):
return self._import_generic(kind="RTC_BIOFUEL_INDEX", provider="Portal Nacional NF-e", act="Informe Técnico 2025.004 - pBioObrigatorio", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcBiofuelIndexEntry, row_kwargs=lambda e: e)
def import_rtc_cred_pres(self, payload, source_url, downloaded_at):
return self._import_generic(kind="RTC_CCREDPRES", provider="Portal Nacional NF-e", act="cCredPres — LC 214/2025", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcCredPresEntry, row_kwargs=lambda e: e)
def import_rtc_cclasstrib(self, payload, source_url, downloaded_at):
return self._import_generic(kind="RTC_CCLASSTRIB", provider="Portal Nacional NF-e", act="cClassTrib — IBS/CBS (LC 214/2025)", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcCClassTribEntry, row_kwargs=lambda e: e, extra=((RtcCstIbsCbsEntry, payload.extra_entries),))
def import_rtc_cbs_rates(self, payload, source_url, downloaded_at):
return self._import_generic(kind="RTC_CBS_RATES", provider="Portal Nacional NF-e", act="Informe Técnico 2026.002 - Alíquotas da CBS", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcCbsRateEntry, row_kwargs=lambda e: e)
def rtc_cclasstrib(self, code=None):
query = select(RtcCClassTribEntry).join(Dataset).where(Dataset.kind == "RTC_CCLASSTRIB", Dataset.status == "ACTIVE")
if code is not None:
query = query.where(RtcCClassTribEntry.cclasstrib == str(code))
return self.session.scalars(query).all()
def rtc_cbs_rate(self, year):
return self.session.scalar(select(RtcCbsRateEntry).join(Dataset).where(Dataset.kind == "RTC_CBS_RATES", Dataset.status == "ACTIVE", RtcCbsRateEntry.year == int(year)))
def cfop_lookup(self, code=None):
query = select(CfopEntry).join(Dataset).where(Dataset.kind == "CFOP", Dataset.status == "ACTIVE")
if code is not None:
query = query.where(CfopEntry.code == str(code))
return self.session.scalars(query).all()
def fcp_for_uf(self, uf):
return self.session.scalar(select(FcpRate).join(Dataset).where(Dataset.kind == "FCP", Dataset.status == "ACTIVE", FcpRate.uf == uf.upper()))
def anp_code_exists(self, cprod_anp):
return self.session.scalar(select(AnpProductCode).join(Dataset).where(Dataset.kind == "ANP", Dataset.status == "ACTIVE", AnpProductCode.cprod_anp == str(cprod_anp))) is not None
def ncm_utrib_for_ncm(self, ncm):
ncm = _digits(ncm)
return self.session.scalar(select(NcmUtribEntry).join(Dataset).where(Dataset.kind == "NCM_UTRIB", Dataset.status == "ACTIVE", NcmUtribEntry.ncm == ncm))
def rtc_biofuel_for_anp(self, cprod_anp):
return self.session.scalars(select(RtcBiofuelIndexEntry).join(Dataset).where(Dataset.kind == "RTC_BIOFUEL_INDEX", Dataset.status == "ACTIVE", RtcBiofuelIndexEntry.cprod_anp == str(cprod_anp), RtcBiofuelIndexEntry.is_historical.is_(False))).all()
def rtc_cred_pres(self, code=None):
query = select(RtcCredPresEntry).join(Dataset).where(Dataset.kind == "RTC_CCREDPRES", Dataset.status == "ACTIVE")
if code is not None:
query = query.where(RtcCredPresEntry.code == str(code))
return self.session.scalars(query).all()
def dataset_versions(self):
active = self.session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
cest = self.session.scalar(select(Dataset).where(Dataset.kind == "CEST", Dataset.status == "ACTIVE"))
tipi = self.session.scalar(select(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.status == "ACTIVE"))
mva = self.session.scalar(select(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE"))
moc = self.session.scalar(select(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.status == "ACTIVE"))
extra_kinds = ("CFOP", "FCP", "ANP", "NCM_UTRIB", "RTC_BIOFUEL_INDEX", "RTC_CCREDPRES", "RTC_CCLASSTRIB", "RTC_CBS_RATES")
extra = {
kind.lower(): self.session.scalar(select(Dataset.sha256).where(Dataset.kind == kind, Dataset.status == "ACTIVE"))
for kind in extra_kinds
}
return {
"ncm": active.sha256 if active else None,
"cest": cest.sha256 if cest else None,
"tipi_ipi": tipi.sha256 if tipi else None,
"ba_st_mva": mva.sha256 if mva else None,
"nfe_moc_tables": moc.sha256 if moc else None,
**extra,
}
+54
View File
@@ -0,0 +1,54 @@
"""Fail-closed parser for the official Receita Federal TIPI workbook."""
from dataclasses import dataclass
from io import BytesIO
import hashlib
import unicodedata
import openpyxl
@dataclass(frozen=True)
class TipiEntry:
ncm: str
ex: str | None
description: str
rate: str
@dataclass(frozen=True)
class TipiPayload:
sha256: str
entries: tuple[TipiEntry, ...]
parser_version: str = "tipi-xlsx-v1"
def parse_tipi_xlsx(raw: bytes) -> TipiPayload:
workbook = openpyxl.load_workbook(BytesIO(raw), read_only=True, data_only=True)
sheet = workbook.active
normalize = lambda value: "".join(char for char in unicodedata.normalize("NFKD", str(value or "")).encode("ascii", "ignore").decode().upper() if char.isalnum())
header_row = next((row for row in sheet.iter_rows(values_only=True) if {"NCM", "DESCRICAO", "ALIQUOTA"} <= {normalize(cell) for cell in row}), None)
if header_row is None:
raise ValueError("TIPI workbook has no recognized official header")
headers = [normalize(cell) for cell in header_row]
ncm_index = headers.index("NCM")
ex_index = headers.index("EX")
description_index = next(index for index, value in enumerate(headers) if value == "DESCRICAO")
rate_index = next(index for index, value in enumerate(headers) if value.startswith("ALIQUOTA"))
entries = []
started = False
for row in sheet.iter_rows(values_only=True):
if tuple(row) == tuple(header_row):
started = True; continue
if not started or ncm_index >= len(row):
continue
ncm = "".join(char for char in str(row[ncm_index] or "") if char.isdigit())
if len(ncm) != 8:
continue
description = str(row[description_index] or "").strip()
if not description:
raise ValueError("TIPI NCM row has no description")
ex = str(row[ex_index] or "").strip() or None
entries.append(TipiEntry(ncm, ex, description, str(row[rate_index] or "").strip()))
if not entries:
raise ValueError("TIPI workbook has no 8-digit NCM rows")
return TipiPayload(hashlib.sha256(raw).hexdigest(), tuple(entries))
+91
View File
@@ -0,0 +1,91 @@
"""Automatic refresh of the official NCM dataset (Siscomex JSON): staging -> validate -> atomic promotion, keeping the last valid dataset on ANY failure.
Every attempt leaves a SourceRefreshRun (url, time, sha256, version label, record count, status, error) for the panel. Nothing here touches Agents or providers.
"""
from __future__ import annotations
import logging
import threading
import time
from datetime import datetime, timedelta, timezone
import httpx
from sqlalchemy import select
from ..config import settings
from ..models import Dataset, SourceRefreshRun
from .ncm import parse_ncm_payload
from .service import DatasetService
logger = logging.getLogger("gti_fiscal_hub")
MIN_RATIO = 0.9 # a payload with < 90% of the active record count is treated as a broken download, never promoted
INTERVAL = timedelta(hours=24)
RETRY_AFTER_FAILURE = timedelta(hours=6)
def _fetch(url: str) -> bytes:
response = httpx.get(url, timeout=max(settings.http_timeout_seconds, 60), follow_redirects=True, headers={"User-Agent": "GTIFiscalHub/1.0 (official dataset refresh)"})
response.raise_for_status()
return response.content
def refresh_ncm(session, fetch=_fetch, url: str | None = None) -> SourceRefreshRun:
url = url or settings.ncm_dataset_url
run = SourceRefreshRun(source="NCM", source_url=url, status="RUNNING")
session.add(run)
session.commit()
try:
content = fetch(url) # staging: bytes in memory, nothing written to the active dataset yet
parsed = parse_ncm_payload(content) # schema validation
run.sha256, run.version_label, run.record_count = parsed.sha256, (parsed.version or "")[:64], len(parsed.entries)
active = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
if active is not None and active.sha256 == parsed.sha256:
run.status = "UNCHANGED"
else:
if active is not None and len(parsed.entries) < MIN_RATIO * (active.record_count or 0):
raise ValueError(f"implausible record count {len(parsed.entries)} vs active {active.record_count}")
DatasetService(session).import_payload(parsed, url, datetime.now(timezone.utc)) # atomic: one transaction, old ACTIVE superseded only at the end
run.status = "UPDATED"
try: # the KB is never deleted: only flagged STALE_REVIEW_REQUIRED when its NCM left the official dataset
from ..enrichment import history, kb
current = history.official_sets(session)
run.error = f"kb_revalidated: {kb.revalidate(session, lambda v: v in current)}"[:300]
except Exception as error: # noqa: BLE001 - revalidation trouble must not undo a valid dataset promotion
run.error = f"kb_revalidate_failed: {type(error).__name__}"[:300]
except Exception as error: # keep the last valid dataset
session.rollback()
run = session.get(SourceRefreshRun, run.id)
run.status, run.error = "FAILED", f"{type(error).__name__}: {error}"[:300]
logger.warning("NCM refresh failed: %s", run.error)
run.finished_at = datetime.now(timezone.utc)
session.commit()
return run
def last_runs(session, source: str = "NCM", limit: int = 5):
return session.scalars(select(SourceRefreshRun).where(SourceRefreshRun.source == source).order_by(SourceRefreshRun.id.desc()).limit(limit)).all()
def _due(session) -> bool:
last = session.scalar(select(SourceRefreshRun).where(SourceRefreshRun.source == "NCM", SourceRefreshRun.status != "RUNNING").order_by(SourceRefreshRun.id.desc()).limit(1))
if last is None or last.finished_at is None:
return True
finished = last.finished_at if last.finished_at.tzinfo else last.finished_at.replace(tzinfo=timezone.utc)
return datetime.now(timezone.utc) - finished >= (RETRY_AFTER_FAILURE if last.status == "FAILED" else INTERVAL)
def start_scheduler(session_factory, first_delay: float = 90.0):
"""Daemon thread: checks hourly whether a refresh is due (survives restarts because due-ness is derived from SourceRefreshRun)."""
def loop():
time.sleep(first_delay)
while True:
try:
with session_factory() as session:
if _due(session):
refresh_ncm(session)
except Exception: # never let the scheduler die
logger.exception("official refresh loop error")
time.sleep(3600)
thread = threading.Thread(target=loop, name="official-refresh", daemon=True)
thread.start()
return thread
+15
View File
@@ -0,0 +1,15 @@
"""SQLAlchemy session setup for central PostgreSQL only."""
from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker
from .config import settings
engine = create_engine(settings.database_url, pool_pre_ping=True)
SessionLocal = sessionmaker(bind=engine, expire_on_commit=False)
def database_is_available() -> bool:
try:
with engine.connect() as connection:
connection.execute(text("SELECT 1"))
return True
except Exception:
return False
@@ -0,0 +1,36 @@
"""Conservative deterministic role, ST, and inbound scope resolution."""
def resolve_product_role(signals: dict) -> dict:
if signals.get("recipe") and signals.get("production"):
return {"value": "OWN_PRODUCTION", "status": "CONFIRMED", "evidence": ["recipe", "production"]}
if signals.get("purchase") and signals.get("sale") and not signals.get("recipe"):
return {"value": "RESALE", "status": "CONFIRMED", "evidence": ["purchase", "sale"]}
if signals.get("input_consumed") and signals.get("recipe"):
return {"value": "INPUT", "status": "CONFIRMED", "evidence": ["input_consumed", "recipe"]}
if signals.get("service_flag") and signals.get("service_transactions"):
return {"value": "SERVICE", "status": "CONFIRMED", "evidence": ["service_flag", "service_transactions"]}
return {"value": None, "status": "PENDING_PRODUCT_ROLE", "evidence": []}
def resolve_st_status(signals: dict) -> dict:
if not signals.get("official_cest"):
return {"value": "NOT_SUBJECT_TO_ST", "status": "CONFIRMED", "evidence": []}
if signals.get("purchase_retained") is True:
return {"value": "ST_ALREADY_WITHHELD_IN_PURCHASE", "status": "CONFIRMED", "evidence": ["official_cest", "purchase_retained"]}
if signals.get("purchase_retained") is False:
return {"value": "PRODUCT_SUBJECT_TO_ST", "status": "CONFIRMED", "evidence": ["official_cest", "purchase_not_retained"]}
return {"value": None, "status": "PENDING_ST_WITHHELD", "evidence": ["official_cest"]}
def resolve_entry_scope(supplier_ufs: list[str], company_uf: str) -> dict:
observed = {str(uf).strip().upper() for uf in supplier_ufs if uf}
if not observed:
return {"value": None, "status": "PENDING_ENTRY_SCOPE", "evidence": []}
if len(observed - {company_uf.upper()}) and company_uf.upper() in observed:
value = "MIXED_OR_DYNAMIC"
elif observed - {company_uf.upper()}:
value = "INTER"
else:
value = "INTRA"
return {"value": value, "status": "CONFIRMED", "evidence": sorted(observed)}
+186
View File
@@ -0,0 +1,186 @@
"""Dry-run ERP write planner: turns classifier decisions into concrete field
proposals without ever issuing a database write. Every proposal carries
write_eligible; any future executor must refuse proposals where it is False."""
IPI_FIELD = "IPI" # C000025.IPI NUMERIC(15,2) is the real IPI rate field; C000025.ALIQ_IPI
# is a VARCHAR(1) that is unused (0/521 rows) and cannot hold a TIPI rate or "NT" marker.
EX_TIPI_FIELD = "EX_TIPI"
CEST_FIELD = "CEST"
NCM_FIELD = "CLASSIFICACAO_FISCAL"
_CEST_UNRESOLVED_REASONS = {
"CEST_CANDIDATES": "MULTIPLE_CEST_CANDIDATES_NO_SINGLE_EVIDENCE",
"CEST_DESCRIPTION_MISMATCH": "CEST_DESCRIPTION_SCORE_BELOW_THRESHOLD",
"NO_CEST_MAPPING_FOUND": "NO_CEST_MAPPING_FOUND_FOR_NCM",
}
def _provenance(decision):
return {
"engine_version": decision.get("engine_version"),
"decision_version": decision.get("decision_version"),
"request_fingerprint": decision.get("request_fingerprint"),
}
def _same(current, proposed):
return str(current).strip() == str(proposed).strip()
def build_field_proposals(product_code, decision, current):
"""current: dict with optional 'ipi' and 'cest' current ERP values.
A missing/None key means the current ERP value is unknown, not that it is blank."""
dataset_versions = decision.get("dataset_versions", {})
classification = decision.get("classification", {})
if decision.get("status") == "LEGACY_NCM":
return [{
"product_code": product_code,
"field": NCM_FIELD,
"current_value": classification.get("ncm"),
"proposed_value": None,
"status": "LEGACY_NCM_REVIEW",
"reason_code": "LEGACY_NCM_NO_AUTOMATIC_SUCCESSOR",
"source_dataset": "NCM",
"source_version": dataset_versions.get("ncm"),
"provenance": _provenance(decision),
"write_eligible": False,
}]
proposals = []
ipi_status = decision.get("ipi_status")
ipi = classification.get("ipi")
if ipi_status == "IPI_RESOLVED" and ipi:
current_ipi = current.get("ipi")
current_ipi_read = "ipi" in current
rate_kind = ipi.get("rate_kind")
if rate_kind == "NUMERIC":
proposed_rate = ipi.get("rate_numeric")
if not current_ipi_read:
status, write_eligible, reason = "CURRENT_UNKNOWN", False, "ERP_CURRENT_VALUE_NOT_READ"
elif _same(current_ipi, proposed_rate):
status, write_eligible, reason = "SAME_VALUE", False, "TIPI_MATCHES_CURRENT_ERP"
else:
status, write_eligible, reason = "CHANGE_REQUIRED", True, "TIPI_RATE_DIVERGES_FROM_ERP"
elif rate_kind == "NON_NUMERIC":
# e.g. TIPI text marker "NT" (não tributado) — never write text into a numeric field.
proposed_rate = None
rate_classification = ipi.get("rate_classification", "OTHER")
status, write_eligible, reason = "NON_NUMERIC_IPI_STATUS", False, f"TIPI_RATE_{rate_classification}"
else: # BLANK — source spreadsheet had no rate at all; never invent zero.
proposed_rate = None
rate_classification = ipi.get("rate_classification", "SOURCE_BLANK")
status, write_eligible, reason = rate_classification, False, f"TIPI_RATE_{rate_classification}"
proposals.append({
"product_code": product_code,
"field": IPI_FIELD,
"current_value": current_ipi,
"proposed_value": proposed_rate,
"status": status,
"reason_code": reason,
"source_dataset": "TIPI_IPI",
"source_version": dataset_versions.get("tipi_ipi"),
"provenance": _provenance(decision),
"write_eligible": write_eligible,
})
# C000025 has no EX_TIPI row under IPI_RESOLVED by construction: the matched
# TIPI entry is the base (non-EX) row, so there is nothing concrete to propose.
# EX TIPI only ever comes from EX-specific TIPI rows for this NCM (ipi_candidates).
# No EX row for the NCM => no proposal at all (never propose a blank/None EX).
ex_candidates = [c for c in classification.get("ipi_candidates", []) if c.get("ex")]
if ex_candidates:
current_ex = current.get("ex_tipi")
current_ex_read = "ex_tipi" in current
if len(ex_candidates) == 1:
proposed_ex = ex_candidates[0]["ex"]
if not current_ex_read:
ex_status, ex_write_eligible, ex_reason = "CURRENT_UNKNOWN", False, "ERP_CURRENT_VALUE_NOT_READ"
elif _same(current_ex, proposed_ex):
ex_status, ex_write_eligible, ex_reason = "SAME_VALUE", False, "TIPI_EX_MATCHES_CURRENT_ERP"
else:
ex_status, ex_write_eligible, ex_reason = "CHANGE_REQUIRED", True, "TIPI_EX_DIVERGES_FROM_ERP"
else:
proposed_ex = None
ex_status, ex_write_eligible, ex_reason = "CANDIDATE", False, "MULTIPLE_EX_TIPI_CANDIDATES_NO_SINGLE_EVIDENCE"
proposals.append({
"product_code": product_code,
"field": EX_TIPI_FIELD,
"current_value": current_ex,
"proposed_value": proposed_ex,
"status": ex_status,
"reason_code": ex_reason,
"source_dataset": "TIPI_IPI",
"source_version": dataset_versions.get("tipi_ipi"),
"provenance": _provenance(decision),
"write_eligible": ex_write_eligible,
"candidate_count": len(ex_candidates),
})
cest_status = decision.get("cest_status")
if cest_status is not None:
current_cest = current.get("cest")
current_cest_read = "cest" in current
if cest_status == "CEST_CONFIRMED":
proposed_cest = (classification.get("cest") or [None])[0]
if not current_cest_read:
status, write_eligible, reason = "CURRENT_UNKNOWN", False, "ERP_CURRENT_VALUE_NOT_READ"
elif _same(current_cest, proposed_cest):
status, write_eligible, reason = "SAME_VALUE", False, "CEST_MATCHES_CURRENT_ERP"
else:
status, write_eligible, reason = "CHANGE_REQUIRED", True, "CEST_DIVERGES_FROM_ERP"
else:
proposed_cest = None
write_eligible = False
status = "CANDIDATE" if cest_status == "CEST_CANDIDATES" else "UNRESOLVED"
reason = _CEST_UNRESOLVED_REASONS.get(cest_status, "CEST_NOT_RESOLVED")
proposals.append({
"product_code": product_code,
"field": CEST_FIELD,
"current_value": current_cest,
"proposed_value": proposed_cest,
"status": status,
"reason_code": reason,
"source_dataset": "CEST",
"source_version": dataset_versions.get("cest"),
"provenance": _provenance(decision),
"write_eligible": write_eligible,
})
return proposals
def mva_evidence(product_code, decision):
"""BA ST/MVA has no ERP field target: keep it as fiscal evidence only, never a write proposal."""
classification = decision.get("classification", {})
return {
"product_code": product_code,
"ncm": classification.get("ncm"),
"status": decision.get("ba_st_mva_status"),
"resolved": classification.get("ba_st_mva"),
"candidates": classification.get("ba_st_mva_candidates", []),
"source_version": decision.get("dataset_versions", {}).get("ba_st_mva"),
}
def split_by_eligibility(proposals):
return {
"eligible": [p for p in proposals if p.get("write_eligible")],
"blocked": [p for p in proposals if not p.get("write_eligible")],
}
def execute(proposals):
"""Stub executor. Refuses any proposal that is not write_eligible and never
issues a write itself — implementing the actual ERP UPDATE is out of scope."""
for proposal in proposals:
if not proposal.get("write_eligible"):
raise PermissionError(
f"refused: {proposal['product_code']}/{proposal['field']} is not write_eligible "
f"(status={proposal['status']})"
)
raise NotImplementedError("no ERP write executor is implemented; this planner is dry-run only")
@@ -0,0 +1,2 @@
"""Product enrichment (new product -> GTIN/name -> fiscal profile -> safe fill). Deterministic only: no AI, no LLM, no embeddings, no paid providers.
Everything lives inside the existing Hub/FastAPI/PostgreSQL; providers are hidden behind the Hub (Agents never call them)."""
+193
View File
@@ -0,0 +1,193 @@
"""Agent-facing product enrichment routes (authenticated as the Agent; the Agent never talks to external providers)."""
from __future__ import annotations
import time
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import select
from ..blocked_products import upsert_blocker
from ..config import settings
from ..field_contract import WRITE_FIELDS
from ..models import Installation, ProductEnrichment, ProductFieldAudit
from . import status as enrichment_status, textnorm
from .placeholders import is_placeholder
from .providers import build_providers
from .quota import ProviderGateway
from .resolver import FIELDS, resolve
CLASSIFICATION_CONTEXT_FIELDS = ("CODGRUPO", "TIPO", "PROD_COZINHA", "PRODUTO_SERVICO") # read-only evidence for field_contract.infer_role; never written
CURRENT_FIELDS = tuple(dict.fromkeys(FIELDS + tuple(WRITE_FIELDS) + CLASSIFICATION_CONTEXT_FIELDS)) # everything resolve() may read from the Agent's `current`: NCM/CEST/IPI + every fiscal-completion field + role evidence
def sanitize_product_evidence(raw) -> dict | None:
"""Bounds the Agent's targeted, single-product evidence (firebird_reader.read_product_evidence) before it reaches field_contract: shape-only
validation (never trusts an authenticated Agent to send something huge or malformed for a single-product payload)."""
if not isinstance(raw, dict):
return None
out = {"recipes": set(), "composed": set(), "composed_ratio": {}, "purchases": {}, "stock_entry": {}, "stock_in": {}}
for key in ("recipes", "composed"):
if isinstance(raw.get(key), list):
out[key] = {str(v) for v in raw[key][:8]}
if isinstance(raw.get("composed_ratio"), dict):
for code, counts in list(raw["composed_ratio"].items())[:8]:
if isinstance(counts, dict) and isinstance(counts.get("composed"), int) and isinstance(counts.get("other"), int):
out["composed_ratio"][str(code)] = {"composed": max(0, counts["composed"]), "other": max(0, counts["other"])}
if isinstance(raw.get("purchases"), dict):
for code, docs in list(raw["purchases"].items())[:8]:
if isinstance(docs, list):
out["purchases"][str(code)] = [d for d in docs[:32] if isinstance(d, dict)]
for key in ("stock_entry", "stock_in"):
if isinstance(raw.get(key), dict):
out[key] = {str(k): bool(v) for k, v in list(raw[key].items())[:8]}
return out
PROVIDERS_OVERRIDE = None # tests inject fake providers here (never the network)
PENDING_STATES = {"PENDING_PROVIDER_QUOTA", "PENDING_PROVIDERS_OFFLINE", "PENDING_TIME_BUDGET"}
REASONS = { # human PT-BR reasons shown through the existing blockers (category ENRICHMENT); never internal jargon
"INVALID_GTIN": "Código de barras (GTIN) inválido: o produto não foi consultado em fontes externas.",
"AMBIGUOUS": "Descrição genérica demais para identificar o produto com segurança; informe o código de barras.",
"NOT_FOUND": "Produto não encontrado nas fontes consultadas; preencha os campos fiscais manualmente.",
"CONFLICT": "As fontes divergem sobre a classificação deste produto; nenhuma alteração foi feita.",
"SINGLE_WEAK_SOURCE": "Apenas uma fonte identificou o produto; a classificação foi apenas sugerida.",
"PENDING_PROVIDER_QUOTA": "Limite diário das fontes de produtos atingido; a consulta será repetida automaticamente.",
"PENDING_PROVIDERS_OFFLINE": "Fontes de produtos indisponíveis no momento; a consulta será repetida automaticamente.",
}
def _aware(dt):
return dt.replace(tzinfo=timezone.utc) if dt is not None and dt.tzinfo is None else dt
def features_of(installation) -> dict:
return {"corrector_enabled": bool(installation.corrector_enabled), "enrichment_mode": installation.enrichment_mode, "features_configured": bool(installation.features_configured)}
def router(principal, providers_factory=None):
routes = APIRouter(prefix="/products")
holder = {}
def providers():
if PROVIDERS_OVERRIDE is not None:
return PROVIDERS_OVERRIDE
if providers_factory is not None:
return providers_factory()
if "p" not in holder:
holder["p"] = build_providers(settings)
return holder["p"]
@routes.post("/resolve")
def resolve_product(payload: dict, identity=Depends(principal)):
session, agent = identity
try:
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
if installation is None:
raise HTTPException(409, "installation not registered")
features = features_of(installation)
if not features["features_configured"] or installation.enrichment_mode == "DISABLED":
return {"state": "FEATURE_DISABLED", "features": features, "fields": {}}
code = str(payload.get("product_code") or "")[:64]
if not code:
raise HTTPException(422, "product_code required")
description, current = str(payload.get("description") or "")[:255], payload.get("current") if isinstance(payload.get("current"), dict) else {}
started = time.monotonic()
result = resolve(session, ProviderGateway(session, providers()), gtin_raw=payload.get("gtin"), description=description, current={k: current.get(k) for k in CURRENT_FIELDS},
client_label=f"installation:{installation.id}", budget_s=settings.enrichment_time_budget_seconds,
product_code=code, installation=installation, tenant_id=agent.tenant_id, product_evidence=sanitize_product_evidence(payload.get("ev")))
latency = int((time.monotonic() - started) * 1000)
row = session.scalar(select(ProductEnrichment).where(ProductEnrichment.installation_id == installation.id, ProductEnrichment.product_code == code))
if row is None:
row = ProductEnrichment(tenant_id=agent.tenant_id, installation_id=installation.id, product_code=code)
session.add(row)
for field in (row.human_overridden or {}): # a human decision (even deleting the value) is final for this product
if field in result["fields"]:
result["fields"][field] = {"status": "HUMAN_OVERRIDDEN", "value": None, "rule": None, "kb_status": None, "evidence": [], "reason": "valor alterado por uma pessoa; nunca mais preenchido automaticamente"}
pending = result["state"] in PENDING_STATES
for name, decision in result["fields"].items(): # explain in the audit trail that a technical placeholder was treated as "not classified yet"
if is_placeholder(name, current.get(name)):
decision["technical_placeholder"] = str(current.get(name))
row.gtin, row.description, row.latency_ms = result.get("gtin"), description, latency
row.attempts = (row.attempts or 0) + 1
row.ui_status, row.reason_code = enrichment_status.classify(result, has_gtin=bool(result.get("gtin")), specific_name=textnorm.is_specific(description))
row.last_attempt_at, row.reevaluate = datetime.now(timezone.utc), False
row.status = "PENDING" if pending else "RESOLVED"
row.decisions = {"state": result["state"], "fields": {k: {kk: vv for kk, vv in v.items() if kk != "evidence"} for k, v in result["fields"].items()}}
row.next_attempt_at = _aware(result["retry_at"]) if pending else None
if pending and row.next_attempt_at is None:
row.next_attempt_at = datetime.now(timezone.utc).replace(microsecond=0)
session.commit()
reason = REASONS.get(result["state"])
if reason:
upsert_blocker(session, agent.tenant_id, {"company": installation.hostname or "", "product_code": code, "gtin": result.get("gtin"), "description": description, "blocker_type": "ENRICHMENT_" + result["state"],
"field": "ENRICHMENT", "reason_code": result["state"], "evidence_summary": {"message": reason}, "input_fingerprint": f"{result.get('gtin')}|{description}"}, installation_id=installation.id)
result.update({"features": features, "latency_ms": latency, "mode": installation.enrichment_mode})
result["retry_at"] = result["retry_at"].isoformat() if result["retry_at"] else None
return result
finally:
session.close()
@routes.post("/enrichment/applied")
def applied(payload: dict, identity=Depends(principal)):
"""The Agent reports what it wrote / refused. The Hub re-checks that the feature is still ENABLED and audits every field."""
session, agent = identity
try:
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
if installation is None:
raise HTTPException(409, "installation not registered")
code = str(payload.get("product_code") or "")[:64]
row = session.scalar(select(ProductEnrichment).where(ProductEnrichment.installation_id == installation.id, ProductEnrichment.product_code == code))
if row is None:
raise HTTPException(404, "unknown product enrichment")
versions = payload.get("dataset_versions") if isinstance(payload.get("dataset_versions"), dict) else {}
applied_fields, overridden = dict(row.fields_applied or {}), dict(row.human_overridden or {})
for item in (payload.get("fields") or [])[:32]:
if not isinstance(item, dict) or item.get("field") not in FIELDS:
continue
outcome = str(item.get("outcome") or "APPLIED")[:24]
session.add(ProductFieldAudit(tenant_id=agent.tenant_id, installation_id=installation.id, agent_id=agent.id, product_code=code, gtin=row.gtin, description_snapshot=row.description, field=item["field"],
old_value=(str(item["old"])[:64] if item.get("old") is not None else None), new_value=(str(item["new"])[:64] if item.get("new") is not None else None), rule=str(item.get("rule") or "")[:64] or None,
kb_status=str(item.get("kb_status") or "")[:16] or None, evidence=item.get("evidence") if isinstance(item.get("evidence"), list) else [], dataset_versions=versions,
authorization_origin=f"enrichment_mode:{installation.enrichment_mode}", job_id=payload.get("job_id") if isinstance(payload.get("job_id"), int) else None, outcome=outcome,
reason=str(item.get("reason") or "")[:96] or None))
if outcome == "APPLIED":
applied_fields[item["field"]] = item.get("new")
elif outcome == "HUMAN_OVERRIDDEN":
overridden[item["field"]] = True
row.fields_applied, row.human_overridden = applied_fields, overridden
ui, why = enrichment_status.after_write([i for i in (payload.get("fields") or []) if isinstance(i, dict)], installation.enrichment_mode)
if ui:
row.ui_status, row.reason_code = ui, why
row.reevaluate = False
row.status = "APPLIED" if applied_fields and not overridden else ("PARTIAL" if applied_fields else row.status if row.status != "PENDING" else "RESOLVED")
row.updated_at = datetime.now(timezone.utc)
session.commit()
return {"ok": True, "status": row.status, "mode": installation.enrichment_mode}
finally:
session.close()
@routes.get("/enrichment/pending")
def pending_reevaluation(identity=Depends(principal)):
"""Tracked products flagged for a new evaluation (panel button, REPROCESS/SYNC, KB decision). The Agent polls this cheaply and re-runs resolve for each code."""
session, agent = identity
try:
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
if installation is None or not installation.features_configured or installation.enrichment_mode == "DISABLED":
return {"codes": []}
codes = list(session.scalars(select(ProductEnrichment.product_code).where(ProductEnrichment.installation_id == installation.id, ProductEnrichment.reevaluate.is_(True)).order_by(ProductEnrichment.id).limit(25)))
return {"codes": codes}
finally:
session.close()
@routes.get("/enrichment/authorization")
def authorization(identity=Depends(principal)):
"""Fresh central authorization read by the Agent right before EVERY field write."""
session, agent = identity
try:
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
return features_of(installation) if installation is not None else {"corrector_enabled": False, "enrichment_mode": "DISABLED", "features_configured": False}
finally:
session.close()
return routes
+110
View File
@@ -0,0 +1,110 @@
"""Gradual, quota-safe verification of locally CORROBORATED GTINs by an NCM-bearing free provider (OSCBR). Never spends the operational reserve.
Rules: only CORROBORATED single-NCM GTINs (never VERIFIED_KB, never CONFLICT, never an invalid GTIN); each GTIN is checked once per `RECHECK_DAYS` (next_checked_at kept in the KB evidence, plus the
provider cache); a provider answer is EVIDENCE only - the promotion/conflict decision is kb.evaluate's (local strong evidence + independent provider NCM + vigente + zero conflict).
"""
from __future__ import annotations
import time
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from ..models import KbKnowledge, KbProduct
from . import kb
from .gtin import normalize_gtin
RECHECK_DAYS = 30
def _now():
return datetime.now(timezone.utc)
def select_sample(session, is_official, limit: int, provider: str = "oscbr") -> list[dict]:
"""Priority: multi-client, more documents, more years; current-vigente NCM; zero conflict; not checked recently."""
ids = list(session.scalars(select(KbKnowledge.kb_product_id).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.status == kb.CORROBORATED).distinct()))
by_product: dict[int, list] = {}
for start in range(0, len(ids), 500): # every NCM row of those products: a second (even weaker) value already disqualifies the GTIN
for r in session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.kb_product_id.in_(ids[start:start + 500]))):
by_product.setdefault(r.kb_product_id, []).append(r)
now = _now().isoformat()
picks = []
for pid, items in by_product.items():
valid = [i for i in items if is_official(i.value)]
if len(valid) != 1 or valid[0].status != kb.CORROBORATED:
continue # conflicts / no vigente NCM / already verified are not asked
row = valid[0]
info = (row.evidence or {}).get("info") or {}
if info.get(f"{provider}_next_checked_at", "") > now:
continue
prod = session.get(KbProduct, pid)
if not prod or normalize_gtin(prod.gtin[1:] if prod.gtin and prod.gtin.startswith("0") else prod.gtin)["status"] != "VALID_GTIN":
continue
summary = kb._lineage_summary([row])
if any(f.startswith("PROVIDER:" + provider) for f in ((row.evidence or {}).get("families") or {})):
continue
picks.append({"gtin_key": prod.gtin, "ncm": row.value, "clients": len(summary["clients"]), "docs": summary["docs"], "years": len(summary["years"]), "product_id": pid})
picks.sort(key=lambda p: (-p["clients"], -p["docs"], -p["years"], p["gtin_key"]))
return picks[:limit]
def apply_outcome(session, gtin_key: str, provider: str, outcome, is_official) -> dict:
"""Turns one gateway Outcome into KB evidence. Returns the per-GTIN measurement record."""
product = kb.find_by_gtin(session, gtin_key)
rec = {"gtin": gtin_key[-13:] if gtin_key.startswith("0") else gtin_key, "state": outcome.state, "reason": outcome.reason, "found": False, "ncm": None, "cest": None, "description": None, "brand": None,
"local_ncm": None, "official_valid": None, "kb_status": None, "external_conflict": False}
local_rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm"))) if product else []
rec["local_ncm"] = local_rows[0].value if local_rows else None
stamp = (_now() + timedelta(days=RECHECK_DAYS)).isoformat()
if outcome.state in ("FOUND", "CACHE_HIT") and outcome.product:
p = outcome.product
rec.update({"found": True, "ncm": p.ncm, "cest": p.cest, "description": p.title, "brand": p.brand})
if p.ncm:
rec["official_valid"] = bool(is_official(p.ncm))
kb.add_evidence(session, product, "ncm", p.ncm, f"provider:{provider}"[:32], family=f"PROVIDER:{provider}", info={"raw_hash": p.raw_hash})
if p.cest:
kb.add_evidence(session, product, "cest", p.cest, f"provider:{provider}"[:32], family=f"PROVIDER:{provider}", info={"raw_hash": p.raw_hash})
if p.brand and not product.brand:
product.brand = p.brand
if p.title:
kb.add_alias(session, product, p.title.upper()[:200])
ev = kb.evaluate(session, product, "ncm", is_official)
rec["kb_status"] = ev["status"]
rec["external_conflict"] = bool(ev["status"] == kb.CONFLICT and rec["ncm"])
if outcome.state in ("FOUND", "CACHE_HIT", "NOT_FOUND"): # remember the check so it is not repeated (skipped/errors are retried later)
for r in session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm")):
r.evidence = {**(r.evidence or {}), "info": {**((r.evidence or {}).get("info") or {}), f"{provider}_next_checked_at": stamp}}
return rec
def provider_query(key: str) -> str:
"""Canonical form sent to providers: GTIN-8 stays 8 digits, UPC-12/EAN-13 as 13 digits (same rule as the resolver), GTIN-14 with an indicator digit stays 14."""
if len(key) == 14 and key.startswith("000000"):
return key[-8:]
if len(key) == 14 and key.startswith("0"):
return key[1:]
return key
def run(session, gateway, provider, *, limit: int, reserve_percent: int, is_official, dry_run: bool = False, wait_rate_window: bool = True) -> dict:
"""One backfill pass. dry_run only lists the sample (no request)."""
sample = select_sample(session, is_official, limit, provider.name)
report = {"provider": provider.name, "sample_size": len(sample), "requests": 0, "records": [], "stopped": None, "dry_run": dry_run}
if dry_run:
report["records"] = [{k: v for k, v in s.items() if k != "product_id"} for s in sample]
return report
for item in sample:
query = provider_query(item["gtin_key"])
outcome = gateway.lookup(provider, query, "BACKFILL", reserve_percent)
while outcome.state == "SKIPPED" and outcome.reason == "RATE_WINDOW" and wait_rate_window: # per-minute governor: wait for the window, never exceed it
time.sleep(max(1, min(65, int((outcome.retry_at - _now()).total_seconds()) + 1)) if outcome.retry_at else 12)
outcome = gateway.lookup(provider, query, "BACKFILL", reserve_percent)
if outcome.state == "SKIPPED":
report["stopped"] = outcome.reason # quota / reserve / per-minute window: stop, never insist
break
if not outcome.from_cache:
report["requests"] += 1
report["records"].append(apply_outcome(session, item["gtin_key"], provider.name, outcome, is_official))
session.commit()
return report
@@ -0,0 +1,143 @@
"""Post-NCM fiscal completion: wires the EXISTING, previously-dormant `central.app.field_contract.resolve_product` (the real field-by-field fiscal
contract engine - CFOP, CST, CSOSN, PIS/COFINS, CST_IPI, SITUACAO_TRIBUTARIA, IAT, IPPT, ALIQUOTA...) into the live Product Enrichment resolve call.
No second fiscal engine. Everything here only ASSEMBLES the same inputs `resolve_product` already expects (`company`, `refs`, `ev`, `ctx`) from data
that already exists on the Hub, and maps its per-field CHANGE/ALREADY/NOT_APPLICABLE/MANUAL output onto the enrichment's own AUTO_FILL_SAFE vocabulary
so the SAME typed-write Agent pipeline can apply it. `company` comes straight from `Installation.company_uf/company_crt` (auto-discovered from the
Agent's own heartbeat - zero guessing). `ctx` comes straight from `FiscalContextEntry` (the SAME table the existing blocker-resolution workflow
already writes to when an operator answers a question) - so a missing key surfaces as an ordinary blocker the operator resolves the usual way; no
new configuration UI is created. `ev` (purchase/recipe/stock evidence) is not available from a single ad-hoc product resolve; those fields fall back
to MANUAL exactly as `field_contract` already does when evidence is absent - never a guess.
"""
from __future__ import annotations
from sqlalchemy import select
from ..classification import normalize_code
from ..datasets.service import DatasetService
from ..models import Dataset, FiscalContextEntry
from .. import field_contract as fc
# Fields this module MAY auto-fill. Deliberately excludes CLASSIFICACAO_FISCAL/CEST/IPI (owned by the enrichment resolver itself),
# COD_ORIGEM_MERCADORIA (no evidence source exists per the contract - never inferred from CST), EX_TIPI (interacts with the IPI EX choice,
# left for a dedicated pass) and CCLASSTRIB (RTC's own domain, untouched).
FISCAL_COMPLETION_FIELDS = ("SITUACAO_TRIBUTARIA", "CFOP", "CFOP_ENTRADA", "CST", "CST_ENTRADA", "CSOSN", "ALIQUOTA", "CST_IPI",
"COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS",
"INCIDENCIA_PISCOFINS", "PISCOFINS", "IAT", "IPPT")
# Value shape per field (validated BOTH here, before proposing a write, and again by the Agent right before the typed write).
DIGIT_FIELDS = {"CFOP": 4, "CFOP_ENTRADA": 4, "CST": 3, "CST_ENTRADA": 3, "CSOSN": 3, "CST_IPI": 3, "COD_ENTRADA_PIS": 2, "COD_SAIDA_PIS": 2,
"COD_ENTRADA_COFINS": 2, "COD_SAIDA_COFINS": 2}
NUMERIC_FIELDS = ("ALIQUOTA",)
DECIMAL_TEXT_FIELDS = ("ALIQ_PIS", "ALIQ_COFINS")
ENUM_FIELDS = {"SITUACAO_TRIBUTARIA": {"T", "I", "N", "S"}, "IAT": {"A", "T"}, "IPPT": {"P", "T"}, "PISCOFINS": {"N", "S"},
"INCIDENCIA_PISCOFINS": {"NENHUMA", "MONOFASICA", "SUBST.TRIBUT."}}
def is_writable_shape(field: str, value) -> bool:
text = "" if value is None else str(value).strip()
if not text:
return False
if field in DIGIT_FIELDS:
digits = "".join(c for c in text if c.isdigit())
return digits == text and len(digits) in (DIGIT_FIELDS[field], DIGIT_FIELDS[field] + 1) # CST/CSOSN family sometimes carries an origin digit prefix
if field in NUMERIC_FIELDS or field in DECIMAL_TEXT_FIELDS:
try:
float(text.replace(",", "."))
return True
except ValueError:
return False
if field in ENUM_FIELDS:
return text.upper() in ENUM_FIELDS[field]
return False
def build_company(installation) -> dict:
"""Auto-discovered, zero guessing: the Agent already reports these at every heartbeat (company_contribuinte_ipi from C000004.CONTRIBUINTE_IPI,
the same row/query as company_crt - see company_context.py on the Agent). Omits "contribuinte_ipi" entirely when the Agent hasn't reported it
yet (older Agent, or the field just never resolved): resolve_fiscal_fields treats that as genuinely unknown, never as a proven "N"."""
company = {"uf": (installation.company_uf or "").upper() or None, "crt": installation.company_crt or None}
if installation.company_contribuinte_ipi in ("S", "N"):
company["contribuinte_ipi"] = installation.company_contribuinte_ipi
return company
def build_refs(session, *, uf: str | None, cfop_candidates: set[str]) -> dict:
ds = DatasetService(session)
refs = {key: {r.code for r in ds.fiscal_codes(domain)} for key, domain in
(("csosn", "CSOSN"), ("cst_icms", "CST_ICMS"), ("cst_pis", "CST_PIS"), ("cst_cofins", "CST_COFINS"), ("origem", "ORIGEM_MERCADORIA"))}
refs["cfop_valid"] = {c for c in cfop_candidates if ds.cfop_lookup(code=c)}
refs["fcp_uf"] = {}
if uf:
fcp = ds.fcp_for_uf(uf)
if fcp:
refs["fcp_uf"][uf] = {"aliq_1": fcp.aliq_1, "observacao": fcp.observacao}
refs["cbs_rate_2026"] = (ds.rtc_cbs_rate(2026).rate if ds.rtc_cbs_rate(2026) else None)
refs["cclasstrib_count"] = session.scalar(select(Dataset.record_count).where(Dataset.kind == "RTC_CCLASSTRIB", Dataset.status == "ACTIVE")) or 0
return refs
def build_context(session, tenant_id: int, company_key: str) -> fc.Context:
"""SAME table the existing blocker-resolution workflow (`central/app/blocker_resolution.py`) already writes operator answers into."""
rows = session.scalars(select(FiscalContextEntry).where(FiscalContextEntry.tenant_id == tenant_id, FiscalContextEntry.company_key == company_key)).all()
return fc.Context([{"scope": r.scope, "scope_key": r.scope_key, "key": r.key, "value": r.value} for r in rows])
def resolve_fiscal_fields(session, *, product_code: str, description: str, current: dict, ncm: str, ncm_official: bool, installation, tenant_id: int,
extra_columns: dict | None = None, decision: dict | None = None, ev: dict | None = None) -> dict:
"""Returns {field: {status, value, rule, reason}} for FISCAL_COMPLETION_FIELDS, in the enrichment's own vocabulary (AUTO_FILL_SAFE / ALREADY_SET /
NOT_APPLICABLE / NEEDS_REVIEW / INSUFFICIENT_EVIDENCE). Never called for a product whose NCM is not yet an official, vigente code.
`decision` is the SAME dict `classification.Classifier.classify(...)` already returns for CEST/IPI (resolver.py's `_fiscal_decisions`) - reused
as-is (its `classification.ipi_candidates`/`cest_candidates`/`ba_st_mva` keys are exactly what `resolve_product` reads), never recomputed here."""
if not ncm_official:
return {f: {"status": "INSUFFICIENT_EVIDENCE", "value": None, "rule": None, "reason": "depende de um NCM oficial vigente"} for f in FISCAL_COMPLETION_FIELDS}
ncm = normalize_code(ncm)
p = {"CODIGO": product_code, "PRODUTO": description, "CODGRUPO": None, "TIPO": None, "PROD_COZINHA": None, "PRODUTO_SERVICO": None, "CLASSIFICACAO_FISCAL": ncm}
p.update({k: v for k, v in (extra_columns or {}).items() if k in ("CODGRUPO", "TIPO", "PROD_COZINHA", "PRODUTO_SERVICO")})
if str(p.get("PROD_COZINHA") or "").strip().isdigit(): # the Agent sends `current` as JSON strings (e.g. "1"); infer_role compares the real Firebird SMALLINT (== 1)
p["PROD_COZINHA"] = int(p["PROD_COZINHA"])
for f in fc.WRITE_FIELDS:
p.setdefault(f, current.get(f))
company = build_company(installation)
company_key = (installation.company_code or installation.company_name or installation.customer_code or "").strip()
ctx = build_context(session, tenant_id, company_key)
if "contribuinte_ipi" not in company: # ERP fact (heartbeat) always wins; the operator answer is only a fallback for an Agent that hasn't reported it yet
contribuinte_ipi, _ = ctx.get("contribuinte_ipi")
if contribuinte_ipi in ("S", "N"):
company["contribuinte_ipi"] = contribuinte_ipi
cfop_candidates = {str(current.get(f)).strip() for f in ("CFOP", "CFOP_ENTRADA") if current.get(f)}
refs = build_refs(session, uf=company.get("uf"), cfop_candidates=cfop_candidates)
# Agent-provided, product_code-targeted evidence (recipes/composed/purchases/stock - see firebird_reader.read_product_evidence): a real,
# indexed per-product query, never a full-table scan or a guess. Falls back to "no evidence" only when the Agent hasn't sent it yet.
ev = ev if isinstance(ev, dict) else {"recipes": set(), "composed": set(), "composed_ratio": {}, "purchases": {}, "stock_entry": {}, "stock_in": {}}
d = decision or {"status": "EXACT_NCM", "cest_status": "NO_CEST_MAPPING_FOUND", "ba_st_mva_status": None, "classification": {"cest_candidates": [], "ipi_candidates": [], "ba_st_mva": {}}}
raw = fc.resolve_product(p, d, company, refs, ev, ctx)
out = {}
for field in FISCAL_COMPLETION_FIELDS:
r = raw.get(field) or {}
state = r.get("state")
if state == fc.ALREADY:
out[field] = {"status": "ALREADY_SET" if not _blank(current.get(field)) else "NOT_APPLICABLE", "value": None, "rule": r.get("reason"), "reason": r.get("reason")}
elif state == fc.NA and field == "CST_IPI" and "contribuinte_ipi" not in company:
# field_contract's own NA branch for CST_IPI fires whenever company.get("contribuinte_ipi") != "S" - which is also true when the fact is
# simply UNKNOWN. Never let an unknown fact read as a proven "not an IPI taxpayer": surface it as a missing answer instead.
out[field] = {"status": "NEEDS_REVIEW", "value": None, "rule": None, "reason": "contribuinte de IPI não confirmado", "missing_key": "contribuinte_ipi", "missing_scope": "COMPANY"}
elif state == fc.NA:
out[field] = {"status": "NOT_APPLICABLE", "value": None, "rule": r.get("reason"), "reason": r.get("reason")}
elif state == fc.CHANGE and _blank(current.get(field)) and is_writable_shape(field, r.get("proposed")):
out[field] = {"status": "AUTO_FILL_SAFE", "value": str(r.get("proposed")), "rule": r.get("reason"), "reason": None}
elif state == fc.CHANGE:
out[field] = {"status": "NEEDS_REVIEW", "value": None, "rule": r.get("reason"), "reason": "valor proposto não seguro para escrita automática" if not _blank(current.get(field)) else "formato do valor proposto não confere"}
else: # MANUAL
out[field] = {"status": "NEEDS_REVIEW", "value": None, "rule": None, "reason": r.get("reason"), "missing_key": r.get("input_key"), "missing_scope": r.get("input_scope")}
return out
def _blank(v) -> bool:
return v is None or str(v).strip() == ""
def coverage_status(field_statuses: dict) -> str:
"""COMPLETE when every field is filled/already-correct/not-applicable; SAFE_BLOCKED when at least one relevant field still needs a decision."""
return "SAFE_BLOCKED" if any(v["status"] in ("NEEDS_REVIEW", "INSUFFICIENT_EVIDENCE") for v in field_statuses.values()) else "COMPLETE"
+31
View File
@@ -0,0 +1,31 @@
"""Deterministic GTIN normalization and validation (GTIN-8, UPC/GTIN-12, EAN/GTIN-13, GTIN-14)."""
import re
VALID, INVALID, MISSING, RESTRICTED = "VALID_GTIN", "INVALID_GTIN", "MISSING_GTIN", "RESTRICTED_GTIN"
def check_digit_ok(digits: str) -> bool:
body, check = digits[:-1], int(digits[-1])
total = sum(int(d) * (3 if i % 2 == 0 else 1) for i, d in enumerate(reversed(body)))
return (10 - total % 10) % 10 == check
def normalize_gtin(raw) -> dict:
"""Returns {status, gtin (digits as typed), key (zero-padded to 14: the KB identity), reason}. Never calls anything external.
RESTRICTED: EAN prefixes 02, 04 and 20-29 are store-internal / restricted circulation (not a global product identity), so no provider is asked."""
text = "" if raw is None else str(raw).strip()
if not text or text.upper() in ("SEM GTIN", "SEM EAN", "0", "NULL"):
return {"status": MISSING, "gtin": None, "key": None, "reason": "EMPTY"}
digits = re.sub(r"\D", "", text)
if not digits or len(digits) not in (8, 12, 13, 14):
return {"status": INVALID, "gtin": digits or None, "key": None, "reason": "BAD_LENGTH"}
if len(set(digits)) == 1:
return {"status": INVALID, "gtin": digits, "key": None, "reason": "REPEATED_DIGITS"}
if not check_digit_ok(digits):
return {"status": INVALID, "gtin": digits, "key": None, "reason": "BAD_CHECK_DIGIT"}
key = digits.zfill(14)
prefix = key[1:3] if len(digits) == 14 else key[1:3] # key = 0 + EAN-13 layout for 12/13; GTIN-14 keeps its indicator digit in key[0]
ean13 = key[1:]
if ean13[:2] in ("02", "04") or "20" <= ean13[:2] <= "29":
return {"status": RESTRICTED, "gtin": digits, "key": key, "reason": "STORE_INTERNAL_PREFIX"}
return {"status": VALID, "gtin": digits, "key": key, "reason": None}
+196
View File
@@ -0,0 +1,196 @@
"""Historical ERP evidence (product master + NFC-e/NF-e items) -> central KB evidence. Local data only: NO provider is contacted here.
Evidence model
- One 'lineage' per client per (GTIN, value): master-data flag, fiscal-document count and documents per year. Master data and the fiscal documents of the SAME ERP are one family (CLIENT_DATA):
the documents were issued from that very master, so they are never counted as two independent sources.
- CLIENT_DATA alone can reach CORROBORATED (>= 20 documents over >= 2 years, or >= 2 distinct clients) but never VERIFIED_KB: that needs an independent NCM-bearing source (kb.evaluate).
- NCM validity is judged against the ACTIVE official dataset today: CURRENT_VALID / HISTORICAL_EXPIRED / HISTORICAL_INVALID; a different NCM at least as recent -> HISTORICAL_CHANGED; two current values -> CONFLICT.
- Company-specific tax data (CRT, CFOP, CSOSN, ...) is never read nor stored; only GTIN, descriptions/aliases, NCM, CEST and the source tenant label.
Idempotent: a client's lineage for a (GTIN, value) is REPLACED on every import, never accumulated.
"""
from __future__ import annotations
import re
from collections import defaultdict
from sqlalchemy import select
from ..models import KbKnowledge, KbProduct
from . import kb, textnorm
from .gtin import normalize_gtin
def _digits(v) -> str:
return re.sub(r"\D", "", str(v or ""))
def build_observations(stagings: list[dict]) -> dict:
"""stagings: miner output per client. Returns {"gtins": {key: {...}}, "stats": {...}}."""
gtins: dict[str, dict] = {}
stats = defaultdict(int)
for st in stagings:
client = st["client"]
product_gtin: dict[str, set] = defaultdict(set)
master_gtin: dict[str, str] = {}
master_desc: dict[str, str] = {}
for m in st["master"]:
stats["master_rows"] += 1
master_desc[m["code"]] = m["desc"]
for raw in (m["gtin"], m["codbarra"]):
g = normalize_gtin(raw)
if g["status"] == "VALID_GTIN":
master_gtin.setdefault(m["code"], g["key"])
break
# product<->EAN links seen in documents/movements are used ONLY when the master has none, and only when unambiguous
seen: dict[str, set] = defaultdict(set)
for row in st.get("nfe", []):
g = normalize_gtin(row["codbarra"])
if g["status"] == "VALID_GTIN":
seen[row["code"]].add(g["key"])
for row in st.get("barcodes", []):
g = normalize_gtin(row["codbarra"])
if g["status"] == "VALID_GTIN":
seen[row["code"]].add(g["key"])
link: dict[str, str] = dict(master_gtin)
for code, keys in seen.items():
if code in link:
continue
if len(keys) == 1:
link[code] = next(iter(keys))
stats["links_from_documents"] += 1
else:
stats["ambiguous_product_links_skipped"] += 1
def slot(key):
return gtins.setdefault(key, {"aliases": set(), "ncm": defaultdict(lambda: defaultdict(lambda: {"master": 0, "docs": 0, "years": {}})), "cest": defaultdict(lambda: defaultdict(lambda: {"master": 0, "docs": 0, "years": {}})), "clients": set()})
for m in st["master"]:
key = link.get(m["code"])
if not key:
continue
s = slot(key)
s["clients"].add(client)
if m["desc"]:
s["aliases"].add(textnorm.normalized(m["desc"]))
n = _digits(m["ncm"])
if len(n) == 8:
s["ncm"][n][client]["master"] = 1
c = _digits(m["cest"])
if len(c) == 7:
s["cest"][c][client]["master"] = 1
for row in st.get("nfce", []):
key = link.get(row["code"])
stats["nfce_groups"] += 1
if not key:
stats["nfce_groups_without_gtin"] += 1
continue
s = slot(key)
n = _digits(row["ncm"])
if len(n) == 8:
lin = s["ncm"][n][client]
lin["docs"] += row["n"]
if row["year"]:
lin["years"][str(row["year"])] = lin["years"].get(str(row["year"]), 0) + row["n"]
stats["nfce_documents"] += row["n"]
c = _digits(row["cest"])
if len(c) == 7:
lin = s["cest"][c][client]
lin["docs"] += row["n"]
if row["year"]:
lin["years"][str(row["year"])] = lin["years"].get(str(row["year"]), 0) + row["n"]
for row in st.get("nfe", []):
key = link.get(row["code"])
if not key:
continue
s = slot(key)
if row["desc"]:
s["aliases"].add(textnorm.normalized(row["desc"]))
n = _digits(row["ncm"])
if len(n) == 8:
s["ncm"][n][client]["docs"] += row["n"]
stats["nfe_documents"] += row["n"]
stats["gtins"] = len(gtins)
return {"gtins": gtins, "stats": dict(stats)}
def apply(session, observations: dict, is_official, *, dry_run: bool = True, chunk: int = 500) -> dict:
"""Bulk, idempotent import into kb_products / kb_knowledge. dry_run=True computes every status and rolls back."""
keys = sorted(observations["gtins"])
report = {"GTINS_FOUND": len(keys), "OBSERVATIONS": 0, "by_status": defaultdict(int), "CONFLICTS": 0, "WOULD_VERIFY": 0, "with_current_valid_ncm": 0, "stable_across_years": 0, "changed_over_time": 0,
"with_historical_ncm": 0, "multi_client": 0}
nested = session.begin_nested()
try:
for start in range(0, len(keys), chunk):
part = keys[start:start + chunk]
existing = {p.gtin: p for p in session.scalars(select(KbProduct).where(KbProduct.gtin.in_(part)))}
for key in part:
obs = observations["gtins"][key]
product = existing.get(key)
if product is None:
alias = next(iter(sorted(obs["aliases"])), "")
product = kb.upsert_product(session, key, alias)
existing[key] = product
for a in sorted(obs["aliases"])[:20]:
kb.add_alias(session, product, a)
fp = textnorm.fingerprint(a)
if fp:
kb.add_evidence(session, product, "name_fingerprint", fp, "IDENTITY", family="IDENTITY", status=kb.OBSERVED)
for field, values in (("ncm", obs["ncm"]), ("cest", obs["cest"])):
for value, per_client in values.items():
for client, lin in per_client.items():
kb.add_evidence(session, product, field, value, "client:history", family=kb.CLIENT_FAMILY, client_label=client, lineage=None)
row = session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == field, KbKnowledge.value == value))
ev = dict(row.evidence or {})
lineages = dict(ev.get("lineages") or {})
lineages[client] = {"master": lin["master"], "docs": lin["docs"], "years": dict(lin["years"])} # REPLACE: idempotent
row.evidence = {**ev, "lineages": lineages}
report["OBSERVATIONS"] += 1
session.flush()
for key in part:
product = existing[key]
ev = kb.evaluate(session, product, "ncm", is_official)
report["by_status"][ev["status"]] += 1
if ev["status"] == kb.CONFLICT:
report["CONFLICTS"] += 1
if ev["status"] == kb.VERIFIED_KB:
report["WOULD_VERIFY"] += 1
if ev["value"]:
report["with_current_valid_ncm"] += 1
if observations["gtins"][key]["ncm"]:
report["with_historical_ncm"] += 1
if len(observations["gtins"][key]["clients"]) >= 2:
report["multi_client"] += 1
rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm")))
states = {(r.evidence.get("info") or {}).get("historical_state") for r in rows}
if "HISTORICAL_STABLE_CURRENT" in states and ev["value"]:
years = set()
for r in rows:
for lin in (r.evidence.get("lineages") or {}).values():
years |= set(lin.get("years") or {})
if len(years) >= 2:
report["stable_across_years"] += 1
if "HISTORICAL_CHANGED" in states or ("HISTORICAL_EXPIRED" in states and ev["value"]):
report["changed_over_time"] += 1
if dry_run:
nested.rollback()
else:
nested.commit()
session.commit()
except Exception:
nested.rollback()
raise
report["by_status"] = dict(report["by_status"])
return report
def official_sets(session):
"""Precomputed 'vigente today' membership: one query instead of one per value."""
from datetime import date
from ..models import Dataset, NcmEntry
from .resolver import _parse_br
today = date.today()
current = set()
for row in session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE")):
start, end = _parse_br(row.valid_from), _parse_br(row.valid_to)
if (start is None or start <= today) and (end is None or end >= today):
current.add(row.code)
return current
+284
View File
@@ -0,0 +1,284 @@
"""Central product Knowledge Base access (same PostgreSQL, existing kb_* tables) with per-field status.
Trust model (docs/product_enrichment.md):
- A provider value or a client observation is EVIDENCE. A field becomes VERIFIED_KB only when the official dataset confirms it AND at least two INDEPENDENT evidence
families agree with zero conflict (distinct providers each count once; client observations count once and only with >= 2 distinct clients).
- Human edits never enter the KB (HUMAN_OVERRIDDEN is per-installation protection, not knowledge). Company-specific fiscal decisions (CSOSN, CFOP, ...) never live here.
"""
from __future__ import annotations
import hashlib
from datetime import datetime, timezone
from sqlalchemy import select
from sqlalchemy import func
from ..models import KbKnowledge, KbProduct, KbPromotionAudit
from . import textnorm
UNVERIFIED, OBSERVED, CORROBORATED, EXTERNAL_CORROBORATED, VERIFIED_KB, CONFLICT, STALE = "UNVERIFIED", "OBSERVED", "CORROBORATED", "EXTERNAL_CORROBORATED", "VERIFIED_KB", "CONFLICT", "STALE"
EVIDENCE_ONLY = OBSERVED # legacy name
CLIENT_FAMILY = "CLIENT_DATA" # every client-derived lineage (master data + fiscal documents of the same ERP are NOT independent of each other)
RULE_VERSION = "kb-promotion-v2"
STRONG_DOCS, STRONG_YEARS = 20, 2
def _now():
return datetime.now(timezone.utc)
def gtin_fingerprint(key: str) -> str:
return hashlib.sha256(f"gtin:{key}".encode()).hexdigest()
def find_by_gtin(session, key: str | None) -> KbProduct | None:
return session.scalar(select(KbProduct).where(KbProduct.gtin == key)) if key else None
NEUTRAL_EXTRAS = frozenset({"REFRIGERANTE", "REFRIG", "REFR", "BEBIDA", "PRODUTO", "PROD"}) # category nouns only: ZERO / DIET / a flavour / another size are NEVER neutral
def _tokset(text) -> frozenset:
return frozenset(t for t in textnorm.tokens(text) if t not in textnorm.MARKETING)
def alias_match(query: frozenset, alias: frozenset) -> bool:
"""Deterministic, conservative: same token set, or one set contains the other and the difference is only neutral category nouns (so "COCA COLA LATA 350 ML" = "REFRIGERANTE COCA COLA LATA 350 ML", but not "... ZERO ...")."""
if not query or not alias:
return False
if query == alias:
return True
if query < alias:
return (alias - query) <= NEUTRAL_EXTRAS
if alias < query:
return (query - alias) <= NEUTRAL_EXTRAS
return False
def name_candidates(session, description) -> list[KbProduct]:
"""Every KB product with an alias that matches the description under `alias_match` (exact fingerprint included). 0, 1 or several - the caller decides (unique VERIFIED only)."""
fp = textnorm.fingerprint(description)
if not fp:
return []
ids = set(session.scalars(select(KbKnowledge.kb_product_id).where(KbKnowledge.knowledge_key == "name_fingerprint", KbKnowledge.value == fp).distinct()))
query = _tokset(description)
anchor = max((t for t in query if not t.replace(".", "").isdigit() and len(t) > 2), key=len, default=None) # most selective informative token, then verify in Python
if anchor:
for row in session.execute(select(KbKnowledge.kb_product_id, KbKnowledge.value).where(KbKnowledge.knowledge_key == "alias", KbKnowledge.value.contains(anchor))):
if alias_match(query, _tokset(row[1])):
ids.add(row[0])
return [p for p in (session.get(KbProduct, i) for i in sorted(ids)) if p]
def upsert_product(session, key: str | None, description, *, brand=None, quantity=None) -> KbProduct:
if key:
from ..gtin_validator import validate_gtin
valid, _ = validate_gtin(key)
if not valid:
key = None # Regra C.6: GTIN inválido (DV ou prefixo errado) não entra na KB
product = find_by_gtin(session, key)
if product is None:
fp = gtin_fingerprint(key) if key else hashlib.sha256(f"name:{textnorm.fingerprint(description)}".encode()).hexdigest()
product = session.scalar(select(KbProduct).where(KbProduct.fingerprint == fp))
if product is None:
product = KbProduct(fingerprint=fp, gtin=key, identity_kind="GTIN" if key else "FINGERPRINT", normalized_description=textnorm.normalized(description))
session.add(product)
session.flush()
if description and not product.normalized_description:
product.normalized_description = textnorm.normalized(description)
if brand and not product.brand:
product.brand = brand
if quantity and not product.package_quantity:
product.package_quantity = str(quantity)[:32]
fp = textnorm.fingerprint(description)
norm = textnorm.normalized(description)
if norm:
add_alias(session, product, norm)
if fp:
add_evidence(session, product, "name_fingerprint", fp, "IDENTITY", family="IDENTITY", status=OBSERVED)
return product
def add_alias(session, product: KbProduct, normalized: str, cap: int = 20):
"""Non-fiscal aliases of the same GTIN (descriptions seen over the years). Helps name matching; never authority for a fiscal write."""
if not normalized or session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "alias", KbKnowledge.value == normalized[:200])):
return
if (session.scalar(select(func.count()).select_from(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "alias")) or 0) >= cap:
return
session.add(KbKnowledge(kb_product_id=product.id, knowledge_key="alias", value=normalized[:200], source="ALIAS", status=OBSERVED, evidence={"families": {}, "clients": [], "info": {}}))
def add_evidence(session, product: KbProduct, key: str, value: str, source: str, *, family: str, client_label: str | None = None, status: str = OBSERVED, info: dict | None = None,
lineage: dict | None = None) -> KbKnowledge:
"""lineage (client data only): {"master": 0/1, "docs": n, "years": {"2024": n}} of ONE client for this (GTIN, value); merged, never double counted."""
row = session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == key, KbKnowledge.value == value))
if row is None:
row = KbKnowledge(kb_product_id=product.id, knowledge_key=key, value=value, source=source[:32], status=status, evidence={"families": {}, "clients": [], "lineages": {}, "info": info or {}})
session.add(row)
session.flush()
evidence = dict(row.evidence or {})
families = dict(evidence.get("families") or {})
families[family] = _now().isoformat()
clients = list(evidence.get("clients") or [])
if client_label and client_label not in clients:
clients.append(client_label)
lineages = dict(evidence.get("lineages") or {})
if client_label and lineage:
cur = dict(lineages.get(client_label) or {"master": 0, "docs": 0, "years": {}})
cur["master"] = max(int(cur.get("master", 0)), int(lineage.get("master", 0)))
cur["docs"] = int(cur.get("docs", 0)) + int(lineage.get("docs", 0))
years = dict(cur.get("years") or {})
for y, n in (lineage.get("years") or {}).items():
years[str(y)] = int(years.get(str(y), 0)) + int(n)
cur["years"] = years
lineages[client_label] = cur
row.evidence = {**evidence, "families": families, "clients": clients, "lineages": lineages, "info": {**(evidence.get("info") or {}), **(info or {})}}
return row
def _lineage_summary(rows):
clients, docs, years = set(), 0, {}
for r in rows:
ev = r.evidence or {}
for c, lin in (ev.get("lineages") or {}).items():
clients.add(c)
docs += int(lin.get("docs", 0))
for y, n in (lin.get("years") or {}).items():
years[y] = years.get(y, 0) + int(n)
for c in ev.get("clients") or []:
clients.add(c)
return {"clients": sorted(clients), "docs": docs, "years": sorted(years), "last_year": max(int(y) for y in years) if years else None}
def independent_count(rows) -> tuple[int, list[str]]:
"""Independent families: each NCM-bearing provider = 1; ALL client-derived data = 1 and only when STRONG (>= STRONG_DOCS documents over >= STRONG_YEARS years, or >= 2 distinct clients)."""
names = sorted({f for r in rows for f in ((r.evidence or {}).get("families") or {}) if f.startswith("PROVIDER:")})
summary = _lineage_summary(rows)
has_client = any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in rows)
if has_client and (len(summary["clients"]) >= 2 or (summary["docs"] >= STRONG_DOCS and len(summary["years"]) >= STRONG_YEARS)):
names.append(CLIENT_FAMILY)
return len(names), names
def _official_period(session, value):
"""CURRENT_VALID / HISTORICAL_EXPIRED (exists in the dataset but not vigente today) / HISTORICAL_INVALID (never in the dataset)."""
from .resolver import ncm_exists_any
return "CURRENT_VALID" if ncm_exists_any(session, value, current=True) else ("HISTORICAL_EXPIRED" if ncm_exists_any(session, value, current=False) else "HISTORICAL_INVALID")
def _mark(row, state):
row.evidence = {**(row.evidence or {}), "info": {**((row.evidence or {}).get("info") or {}), "historical_state": state}}
OPERATOR_FAMILY = "OPERATOR_ATTESTATION"
def attest(session, gtin_key: str, ncm: str, operator: str, is_official, note: str = "") -> dict:
"""A human fiscal decision by an ADMIN (audited by the caller). Only a value that is ALREADY a candidate in the KB (some local/provider evidence) and vigente in the official dataset can be attested;
the dissenting candidates become STALE (REJECTED_BY_OPERATOR) and the product is promoted with the promotion audit naming the operator."""
product = find_by_gtin(session, gtin_key)
if product is None:
raise ValueError("GTIN_UNKNOWN_IN_KB")
row = session.scalar(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm", KbKnowledge.value == ncm))
if row is None:
raise ValueError("NCM_IS_NOT_A_CANDIDATE_OF_THIS_GTIN")
if not is_official(ncm):
raise ValueError("NCM_NOT_VIGENTE_IN_OFFICIAL_DATASET")
add_evidence(session, product, "ncm", ncm, f"operator:{operator}"[:32], family=OPERATOR_FAMILY, info={"attested_by": operator, "attested_at": _now().isoformat(), "note": note[:200]})
for other in session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == "ncm", KbKnowledge.value != ncm)):
if OPERATOR_FAMILY not in ((other.evidence or {}).get("families") or {}):
other.status = STALE
_mark(other, "REJECTED_BY_OPERATOR")
return evaluate(session, product, "ncm", is_official)
def evaluate(session, product: KbProduct, key: str, is_official) -> dict:
"""Recomputes and PERSISTS the status of one field. Never 'majority wins': two current-valid values => CONFLICT."""
rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == product.id, KbKnowledge.knowledge_key == key)))
if not rows:
return {"value": None, "status": UNVERIFIED, "families": [], "candidates": [], "rejected": [], "detail": {}}
valid = [r for r in rows if is_official(r.value)]
rejected = [r.value for r in rows if not is_official(r.value)]
for r in rows:
if r not in valid:
state = _official_period(session, r.value) if key == "ncm" else "HISTORICAL_INVALID"
r.status = STALE if state == "HISTORICAL_EXPIRED" else OBSERVED
_mark(r, state)
attested = {r.value for r in valid if OPERATOR_FAMILY in ((r.evidence or {}).get("families") or {})}
if len(attested) == 1: # a human decision resolves the candidates: the others are recorded as rejected, not "outvoted"
winner = next(iter(attested))
for r in valid:
if r.value != winner:
r.status = STALE
_mark(r, "REJECTED_BY_OPERATOR")
valid = [r for r in valid if r.value == winner]
values = sorted({r.value for r in valid})
if len(values) > 1:
for r in valid:
r.status = CONFLICT
_mark(r, "HISTORICAL_CONFLICT")
return {"value": None, "status": CONFLICT, "families": [], "candidates": values, "rejected": rejected, "detail": {"clients": _lineage_summary(valid)["clients"]}}
if not values:
return {"value": None, "status": OBSERVED, "families": [], "candidates": [], "rejected": rejected, "detail": {}}
chosen = [r for r in valid if r.value == values[0]]
old = chosen[0].status
count, names = independent_count(chosen)
summary = _lineage_summary(chosen)
other_last = _lineage_summary([r for r in rows if r not in chosen])["last_year"]
superseded = other_last is not None and summary["last_year"] is not None and other_last >= summary["last_year"] # a different (expired) NCM is at least as recent: not stable
if superseded and CLIENT_FAMILY in names:
names.remove(CLIENT_FAMILY)
count -= 1
if any(OPERATOR_FAMILY in ((r.evidence or {}).get("families") or {}) for r in chosen):
names = sorted(set(names) | {OPERATOR_FAMILY} | ({CLIENT_FAMILY} if any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in chosen) else set()))
count = len(names)
providers = [n for n in names if n.startswith("PROVIDER:")]
already = any(r.status == VERIFIED_KB and r.validated_at for r in chosen)
has_client_data = any(CLIENT_FAMILY in ((r.evidence or {}).get("families") or {}) for r in chosen)
if count >= 2 or already:
status = VERIFIED_KB
elif providers and has_client_data:
status = EXTERNAL_CORROBORATED
elif CLIENT_FAMILY in names:
status = CORROBORATED
else:
status = OBSERVED
for r in chosen:
r.status = status
_mark(r, "HISTORICAL_CHANGED" if superseded else "HISTORICAL_STABLE_CURRENT")
if status == VERIFIED_KB and not r.validated_at:
r.validated_at = _now()
if status == VERIFIED_KB and old != VERIFIED_KB and key == "ncm":
from .resolver import dataset_versions
session.add(KbPromotionAudit(kb_product_id=product.id, gtin=product.gtin, field=key, value=values[0], old_status=old, new_status=status, families=names,
sources={"clients": summary["clients"], "documents": summary["docs"], "years": summary["years"], "providers": providers,
"operator": next((((r.evidence or {}).get("info") or {}).get("attested_by") for r in chosen if OPERATOR_FAMILY in ((r.evidence or {}).get("families") or {})), None)},
official_dataset=dataset_versions(session).get("NCM", {}), rule_version=RULE_VERSION))
return {"value": values[0], "status": status, "families": names, "candidates": values, "rejected": rejected, "detail": summary}
def revalidate(session, is_official) -> dict:
"""After the OFFICIAL NCM dataset changed: never delete knowledge. Rows whose NCM is no longer vigente become STALE (STALE_REVIEW_REQUIRED, audited when they were VERIFIED_KB);
rows still vigente keep their status (VERIFIED_KB stays valid while the NCM is vigente)."""
from .resolver import dataset_versions
stale = still = 0
versions = dataset_versions(session).get("NCM", {})
for row in session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm")):
if is_official(row.value):
still += 1
if row.status == STALE:
row.status = OBSERVED # became vigente again: needs a fresh evaluate() to regain its level
continue
if row.status != STALE:
was = row.status
row.status = STALE
_mark(row, "HISTORICAL_EXPIRED")
row.evidence = {**(row.evidence or {}), "info": {**((row.evidence or {}).get("info") or {}), "stale_reason": "STALE_REVIEW_REQUIRED"}}
stale += 1
if was == VERIFIED_KB:
product = session.get(KbProduct, row.kb_product_id)
session.add(KbPromotionAudit(kb_product_id=row.kb_product_id, gtin=product.gtin if product else None, field="ncm", value=row.value, old_status=was, new_status=STALE,
families=sorted((row.evidence or {}).get("families") or {}), sources={"reason": "NCM_NO_LONGER_VIGENTE"}, official_dataset=versions, rule_version=RULE_VERSION))
session.flush()
return {"stale": stale, "still_vigente": still}
@@ -0,0 +1,165 @@
"""Read-only views for the panel: product sources status, official dataset freshness and enrichment coverage metrics. Never exposes tokens."""
from __future__ import annotations
from datetime import datetime, timezone
from sqlalchemy import func, select
from ..config import settings
from ..datasets.updater import last_runs
from ..models import Dataset, GtinLookupCache, ProductEnrichment, ProviderState, ProviderUsageDaily
from .providers import build_providers
STATUS_LABELS = {"ONLINE": "Online", "DEGRADED": "Instável", "RATE_LIMITED": "Limite atingido", "AUTH_ERROR": "Credencial recusada", "OFFLINE": "Indisponível", "NOT_CONFIGURED": "Não configurado"}
OFFICIAL_KINDS = (("NCM", "NCM (Siscomex)"), ("TIPI_IPI", "TIPI / IPI"), ("CEST", "CEST (CONFAZ)"), ("BA_ST_MVA", "ST/MVA Bahia"))
def _aware(dt):
return dt.replace(tzinfo=timezone.utc) if dt is not None and dt.tzinfo is None else dt
def provider_sources(session, providers=None) -> list[dict]:
providers = providers or build_providers(settings)
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
rows = []
for provider in providers:
state = session.get(ProviderState, provider.name)
usage = session.scalar(select(ProviderUsageDaily).where(ProviderUsageDaily.provider == provider.name, ProviderUsageDaily.day == today))
requests = usage.requests if usage else 0
lookups = (usage.cache_hits + usage.cache_misses) if usage else 0
status = ("NOT_CONFIGURED" if not provider.configured else (state.status if state else "ONLINE"))
rows.append({
"provider": provider.name, "configured": bool(provider.configured), "status": status, "status_label": STATUS_LABELS.get(status, status),
"used": requests, "limit": provider.daily_limit or (state.daily_limit if state else None), "rate_per_minute": provider.rate_per_minute,
"remaining": state.remaining if state and state.remaining is not None else ((provider.daily_limit - requests) if provider.daily_limit else None),
"quota_kind": state.quota_kind if state else ("LOCAL_ESTIMATE" if provider.daily_limit else "DOCUMENTED_RATE"),
"reset_at": _aware(state.reset_at) if state and state.reset_at else None,
"last_success_at": _aware(state.last_success_at) if state and state.last_success_at else None,
"last_error": state.last_error if state else None, "last_error_at": _aware(state.last_error_at) if state and state.last_error_at else None,
"latency_ms": state.last_latency_ms if state else None,
"cache_hit_rate": round(100 * usage.cache_hits / lookups) if usage and lookups else None,
"errors": usage.errors if usage else 0, "rate_limited": usage.rate_limited if usage else 0,
"found_today": usage.found if usage else 0, "ncm_found_today": usage.ncm_found if usage else 0, "cache_hits": usage.cache_hits if usage else 0,
"next_reset": _aware(state.reset_at) if state and state.reset_at else None, "identity_only": not getattr(provider, "provides_ncm", False),
})
return rows
def official_sources(session) -> list[dict]:
out = []
for kind, label in OFFICIAL_KINDS:
ds = session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE").order_by(Dataset.id.desc()).limit(1))
item = {"kind": kind, "label": label, "active": ds is not None, "records": ds.record_count if ds else None, "version": ds.version_label if ds else None,
"sha256": (ds.sha256[:12] if ds else None), "last_update": _aware(ds.downloaded_at) if ds else None, "status": "ATUALIZADA" if ds else "SEM DADOS", "error": None, "auto_refresh": kind == "NCM"}
if kind == "NCM":
recent = last_runs(session, "NCM", 1)
if recent:
last = recent[0]
item["last_check"], item["last_run_status"] = _aware(last.finished_at or last.started_at), last.status
if last.status == "FAILED":
item["status"], item["error"] = "FALHA NA ÚLTIMA ATUALIZAÇÃO (dados anteriores mantidos)", last.error
out.append(item)
return out
def coverage_metrics(session) -> dict:
total = session.scalar(select(func.count()).select_from(ProductEnrichment)) or 0
by = dict(session.execute(select(ProductEnrichment.status, func.count()).group_by(ProductEnrichment.status)).all())
with_gtin = session.scalar(select(func.count()).select_from(ProductEnrichment).where(ProductEnrichment.gtin.is_not(None))) or 0
rows = [r.decisions or {} for r in session.scalars(select(ProductEnrichment).limit(5000))]
states = {}
for d in rows:
states[d.get("state")] = states.get(d.get("state"), 0) + 1
def rate(n):
return round(100 * n / total, 1) if total else None
auto_ncm = sum(1 for d in rows if ((d.get("fields") or {}).get("CLASSIFICACAO_FISCAL") or {}).get("status") == "AUTO_FILL_SAFE")
usage = session.execute(select(func.sum(ProviderUsageDaily.requests), func.sum(ProviderUsageDaily.cache_hits), func.sum(ProviderUsageDaily.cache_misses), func.sum(ProviderUsageDaily.rate_limited))).one()
requests, hits, misses, limited = (int(x or 0) for x in usage)
return {"tracked": total, "GTIN_VALID_RATE": rate(with_gtin), "KB_HIT_RATE": rate(states.get("EXACT_GTIN_VERIFIED", 0) + states.get("EXACT_KB_NAME_VERIFIED", 0)),
"EXTERNAL_LOOKUP_RATE": round(100 * requests / (requests + hits), 1) if (requests + hits) else None,
"PRODUCT_FOUND_RATE": rate(total - states.get("NOT_FOUND", 0) - states.get("INVALID_GTIN", 0) - states.get("AMBIGUOUS", 0)),
"NCM_AUTO_FILL_SAFE_RATE": rate(auto_ncm), "NAME_ONLY_RESOLUTION_RATE": rate(states.get("EXACT_KB_NAME_VERIFIED", 0)), "AMBIGUOUS_RATE": rate(states.get("AMBIGUOUS", 0)),
"CONFLICT_RATE": rate(states.get("CONFLICT", 0)), "PROVIDER_QUOTA_BLOCK_RATE": round(100 * limited / requests, 1) if requests else None, "states": states, "by_status": by,
"provider_cache_entries": session.scalar(select(func.count()).select_from(GtinLookupCache)) or 0, "provider_cache_hit_rate": round(100 * hits / (hits + misses), 1) if (hits + misses) else None}
def _candidate_cards(session, products) -> list[dict]:
from ..models import Dataset, KbKnowledge, NcmEntry
out = []
for prod in products:
rows = list(session.scalars(select(KbKnowledge).where(KbKnowledge.kb_product_id == prod.id, KbKnowledge.knowledge_key == "ncm")))
cands = []
for r in rows:
ev = r.evidence or {}
text = session.scalar(select(NcmEntry.description).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == r.value).limit(1))
cands.append({"ncm": r.value, "official_text": text, "status": r.status, "clients": sorted(ev.get("clients", [])), "documents": sum(int(l.get("docs", 0)) for l in (ev.get("lineages") or {}).values()),
"providers": sorted(f for f in (ev.get("families") or {}) if f.startswith("PROVIDER:")), "attested": (ev.get("info") or {}).get("attested_by")})
status = "VERIFIED_KB" if any(c["status"] == "VERIFIED_KB" for c in cands) else ("CONFLICT" if any(c["status"] == "CONFLICT" for c in cands) else (cands[0]["status"] if cands else "OBSERVED"))
out.append({"gtin": (prod.gtin or "")[-13:], "gtin_key": prod.gtin, "description": (prod.normalized_description or "")[:70], "status": status, "candidates": [c for c in cands if c["status"] != "STALE" or c["attested"] is None][:4]})
return out
def kb_search_ids(session, query: str) -> list[int]:
"""GLOBAL, server-side match over the whole KB (never limited to a rendered page): GTIN exact, or every normalized query word present somewhere among the
product's known names (any alias row OR its own normalized_description - words can come from different aliases of the same product). Deliberately lenient
(a human exploring the KB, e.g. "COCA COLA LATA 350" without the unit, must still find "COCA-COLA ORIGINAL LATA 350 ML"); this is DIFFERENT from
`kb.alias_match`, the strict rule that authorizes an automatic fiscal write, which stays untouched."""
import re
from ..models import KbKnowledge, KbProduct
from . import kb as kbm, textnorm
q = (query or "").strip()
if not q:
return []
digits = "".join(ch for ch in q if ch.isdigit())
if len(digits) >= 8 and re.fullmatch(r"[\d\s.\-]+", q):
p = kbm.find_by_gtin(session, digits.zfill(14))
return [p.id] if p else []
tokens = [t for t in textnorm.tokens(q) if t not in textnorm.MARKETING]
if not tokens:
return []
ids: set[int] | None = None
for token in tokens: # SQL ILIKE narrows the candidates (portable: Postgres in production, SQLite in tests); the exact whole-word
like = f"%{token}%" # check happens in Python so "ML" never matches inside "SIMPLES", etc.
matched = {pid for pid, val in session.execute(select(KbKnowledge.kb_product_id, KbKnowledge.value).where(KbKnowledge.knowledge_key == "alias", KbKnowledge.value.ilike(like))) if token in (val or "").split()}
matched |= {pid for pid, val in session.execute(select(KbProduct.id, KbProduct.normalized_description).where(KbProduct.normalized_description.ilike(like))) if token in (val or "").split()}
ids = matched if ids is None else (ids & matched)
if not ids:
return []
return sorted(ids)
def kb_search(session, query: str, *, page: int = 1, page_size: int = 10) -> dict:
"""query/filter GLOBAL over the whole KB -> count -> order -> pagination (never the other way round)."""
from ..models import KbProduct
ids = kb_search_ids(session, query)
total = len(ids)
page = max(1, page)
pages = max(1, -(-total // page_size))
page = min(page, pages)
page_ids = ids[(page - 1) * page_size: page * page_size]
products = [session.get(KbProduct, i) for i in page_ids]
return {"results": _candidate_cards(session, products), "total": total, "page": page, "pages": pages, "page_size": page_size}
def kb_summary(session, query: str | None = None, page: int = 1) -> dict:
"""Compact Knowledge Base panel: known products, statuses of the NCM knowledge, hits today, and the latest conflicts with their evidence (no dashboard of its own)."""
from ..models import KbKnowledge, KbProduct
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
products = session.scalar(select(func.count()).select_from(KbProduct).where(KbProduct.gtin.is_not(None))) or 0
by = dict(session.execute(select(KbKnowledge.status, func.count(func.distinct(KbKnowledge.kb_product_id))).where(KbKnowledge.knowledge_key == "ncm").group_by(KbKnowledge.status)).all())
hits = session.scalar(select(func.coalesce(func.sum(ProviderUsageDaily.kb_hits), 0)).where(ProviderUsageDaily.day == today)) or 0
conflicts = []
rows = session.scalars(select(KbKnowledge).where(KbKnowledge.knowledge_key == "ncm", KbKnowledge.status == "CONFLICT").order_by(KbKnowledge.id.desc()).limit(60)).all()
grouped = {}
for r in rows:
grouped.setdefault(r.kb_product_id, []).append(r)
for pid, items in list(grouped.items())[:5]:
prod = session.get(KbProduct, pid)
conflicts.append({"gtin": (prod.gtin or "")[-13:], "description": (prod.normalized_description or "")[:60],
"candidates": [{"ncm": i.value, "clients": sorted({c for c in (i.evidence or {}).get("clients", [])}), "documents": sum(int(l.get("docs", 0)) for l in ((i.evidence or {}).get("lineages") or {}).values()),
"providers": sorted(f for f in ((i.evidence or {}).get("families") or {}) if f.startswith("PROVIDER:"))} for i in items]})
return {"products": products, "verified": by.get("VERIFIED_KB", 0), "corroborated": by.get("CORROBORATED", 0) + by.get("EXTERNAL_CORROBORATED", 0), "observed": by.get("OBSERVED", 0),
"conflicts": by.get("CONFLICT", 0), "stale": by.get("STALE", 0), "hits_today": hits, "recent_conflicts": conflicts,
"search": kb_search(session, query, page=page) if query else {"results": [], "total": 0, "page": 1, "pages": 1, "page_size": 10}}
@@ -0,0 +1,27 @@
"""ERP observations -> KB evidence. A client observation is EVIDENCE of family CLIENT_OBSERVATION; it never becomes VERIFIED_KB by itself
(needs the official dataset AND a second independent family, or >= 2 distinct clients + another family - see kb.evaluate)."""
from __future__ import annotations
from ..classification import normalize_code
from . import kb
from .gtin import normalize_gtin
def observe(session, rows, client_label: str, is_official) -> dict:
"""rows: iterable of {"GTIN","PRODUTO","NCM"}. Only VALID_GTIN rows with an OFFICIAL 8-digit NCM are recorded. Returns counters."""
stats = {"rows": 0, "valid_gtin": 0, "with_official_ncm": 0, "recorded": 0}
for row in rows:
stats["rows"] += 1
g = normalize_gtin(row.get("GTIN"))
if g["status"] != "VALID_GTIN":
continue
stats["valid_gtin"] += 1
ncm = normalize_code(row.get("NCM"))
if len(ncm) != 8 or not is_official(ncm):
continue
stats["with_official_ncm"] += 1
product = kb.upsert_product(session, g["key"], row.get("PRODUTO"))
kb.add_evidence(session, product, "ncm", ncm, "client:observation", family=kb.CLIENT_FAMILY, client_label=client_label)
stats["recorded"] += 1
session.flush()
return stats
@@ -0,0 +1,27 @@
"""TECHNICAL_PLACEHOLDER semantics for the product enrichment.
The SGV refuses to save a product without an NCM, so operators type a technical filler to get past the form. That filler is NOT a human fiscal decision: it means "not classified yet".
Explicit allowlist PER FIELD (never generalised: "0", "99999999", "SEM NCM" ... are NOT placeholders until the real SGV behaviour is confirmed). The Agent carries the same table
(agent/src/gti_fiscal_agent/enrichment.py PLACEHOLDERS); a test asserts both are identical.
"""
import re
PRODUCT_ENRICHMENT_PLACEHOLDERS = {"CLASSIFICACAO_FISCAL": frozenset({"00000000"})}
def _norm(field: str, value) -> str:
text = "" if value is None else str(value).strip()
return re.sub(r"\D", "", text) if field == "CLASSIFICACAO_FISCAL" else text # 0000.00.00 and 00000000 are the same NCM filler
def is_placeholder(field: str, value) -> bool:
return _norm(field, value) in PRODUCT_ENRICHMENT_PLACEHOLDERS.get(field, frozenset())
def is_blank(value) -> bool:
return not ("" if value is None else str(value)).strip()
def is_writable_empty(field: str, value) -> bool:
"""True when the field is empty OR holds the approved technical placeholder. A real value (even a wrong one) is never writable here."""
return is_blank(value) or is_placeholder(field, value)
+338
View File
@@ -0,0 +1,338 @@
"""Free product-identity providers behind the Hub. Agents never talk to these; the Hub does, through the quota manager.
Documentation snapshots consulted 2026-09-25 (see docs/product_enrichment.md):
- UPCitemdb Explorer/trial: https://api.upcitemdb.com/prod/trial/lookup?upc=... - no key, 100 combined requests/day, burst 15 lookup / 5 search per 30 s,
headers X-RateLimit-Limit / -Remaining / -Reset. Product IDENTITY only (title/brand/category); no fiscal data.
- Open Food Facts: https://world.openfoodfacts.org/api/v2/product/{barcode} - 15 read req/min/IP, 10 search req/min/IP, identifiable User-Agent, ODbL data. Foods only; no fiscal data.
- Cosmos / Bluesoft: https://api.cosmos.bluesoft.com.br/gtins/{gtin} - X-Cosmos-Token (free token, 25 queries/day), returns NCM/CEST/brand/description. Optional: without a token it is NOT_CONFIGURED.
A provider result is EVIDENCE. Only the fiscal engine + official datasets decide what is written.
"""
from __future__ import annotations
import hashlib
import json
import time
from dataclasses import dataclass, field
import httpx
USER_AGENT = "GTIFiscalHub/1.0 (+https://gtifiscal.gtigrupo.com.br; product identity lookup)"
class ProviderError(Exception):
"""kind: RATE_LIMITED | AUTH_ERROR | TIMEOUT | SERVER_ERROR | NETWORK | BAD_RESPONSE"""
def __init__(self, kind: str, detail: str = "", headers: dict | None = None, retry_after: int | None = None):
super().__init__(f"{kind}: {detail}")
self.kind, self.detail, self.headers, self.retry_after = kind, detail, headers or {}, retry_after
@dataclass
class ProviderProduct:
provider: str
gtin: str
found: bool
title: str | None = None
brand: str | None = None
category: str | None = None
quantity: str | None = None
ncm: str | None = None
cest: str | None = None
raw_hash: str = ""
latency_ms: int = 0
headers: dict = field(default_factory=dict)
def as_data(self) -> dict:
return {k: v for k, v in {"title": self.title, "brand": self.brand, "category": self.category, "quantity": self.quantity, "ncm": self.ncm, "cest": self.cest}.items() if v}
def _digits(value) -> str | None:
text = "".join(ch for ch in str(value or "") if ch.isdigit())
return text or None
class Provider:
name = "base"
paid = False
supports_search = False
daily_limit: int | None = None
rate_per_minute: int | None = None
burst: tuple[int, int] | None = None # (requests, seconds)
provides_ncm = False
def __init__(self, client: httpx.Client | None = None, token: str | None = None):
self.client = client or httpx.Client(timeout=4.0, headers={"User-Agent": USER_AGENT})
self._token = token
@property
def configured(self) -> bool:
return True
def _get(self, url: str, **kwargs) -> httpx.Response:
started = time.monotonic()
try:
response = self.client.get(url, **kwargs)
except httpx.TimeoutException as error:
raise ProviderError("TIMEOUT", type(error).__name__) from error
except httpx.HTTPError as error:
raise ProviderError("NETWORK", type(error).__name__) from error
response.latency_ms = int((time.monotonic() - started) * 1000) # type: ignore[attr-defined]
headers = {k.lower(): v for k, v in response.headers.items()}
if response.status_code == 429:
retry = headers.get("retry-after")
raise ProviderError("RATE_LIMITED", "HTTP 429", headers, int(retry) if retry and retry.isdigit() else None)
if response.status_code in (401, 403):
raise ProviderError("AUTH_ERROR", f"HTTP {response.status_code}", headers)
if response.status_code >= 500:
raise ProviderError("SERVER_ERROR", f"HTTP {response.status_code}", headers)
return response
def lookup_gtin(self, gtin: str) -> ProviderProduct: # pragma: no cover - interface
raise NotImplementedError
def search_description(self, text: str) -> list[ProviderProduct]:
return []
def health(self) -> dict:
return {"configured": self.configured}
class UpcItemDb(Provider):
name = "upcitemdb"
supports_search = True
daily_limit = 100
burst = (15, 30)
def lookup_gtin(self, gtin: str) -> ProviderProduct:
response = self._get("https://api.upcitemdb.com/prod/trial/lookup", params={"upc": gtin})
if response.status_code == 404:
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=dict(response.headers))
if response.status_code != 200:
raise ProviderError("BAD_RESPONSE", f"HTTP {response.status_code}")
try:
body = response.json()
except ValueError as error:
raise ProviderError("BAD_RESPONSE", "not JSON") from error
items = body.get("items") or []
headers = {k.lower(): v for k, v in response.headers.items()}
if not items:
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
item = items[0]
raw = hashlib.sha256(json.dumps(item, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
return ProviderProduct(self.name, gtin, True, title=item.get("title"), brand=item.get("brand"), category=item.get("category"),
quantity=item.get("size") or item.get("weight") or None, raw_hash=raw, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
def search_description(self, text: str) -> list[ProviderProduct]:
response = self._get("https://api.upcitemdb.com/prod/trial/search", params={"s": text, "match_mode": "1", "type": "product"})
if response.status_code != 200:
return []
out = []
for item in (response.json().get("items") or [])[:5]:
code = _digits(item.get("ean") or item.get("upc")) or ""
out.append(ProviderProduct(self.name, code, True, title=item.get("title"), brand=item.get("brand"), category=item.get("category"),
raw_hash=hashlib.sha256(json.dumps(item, sort_keys=True, ensure_ascii=False).encode()).hexdigest()))
return out
class OpenFoodFacts(Provider):
name = "openfoodfacts"
supports_search = True
rate_per_minute = 15
host = "world.openfoodfacts.org"
def lookup_gtin(self, gtin: str) -> ProviderProduct:
response = self._get(f"https://{self.host}/api/v2/product/{gtin}", params={"fields": "code,product_name,brands,quantity,categories,generic_name"})
headers = {k.lower(): v for k, v in response.headers.items()}
if response.status_code == 404:
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
if response.status_code != 200:
raise ProviderError("BAD_RESPONSE", f"HTTP {response.status_code}")
try:
body = response.json()
except ValueError as error:
raise ProviderError("BAD_RESPONSE", "not JSON") from error
product = body.get("product") or {}
if body.get("status") != 1 or not product:
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
title = product.get("product_name") or product.get("generic_name")
return ProviderProduct(self.name, gtin, bool(title), title=title, brand=(product.get("brands") or "").split(",")[0].strip() or None, category=(product.get("categories") or "").split(",")[0].strip() or None,
quantity=product.get("quantity"), raw_hash=hashlib.sha256(json.dumps(product, sort_keys=True, ensure_ascii=False).encode()).hexdigest(),
latency_ms=getattr(response, "latency_ms", 0), headers=headers)
def search_description(self, text: str) -> list[ProviderProduct]:
response = self._get("https://world.openfoodfacts.org/cgi/search.pl", params={"search_terms": text, "json": "1", "page_size": "5", "fields": "code,product_name,brands,quantity"})
if response.status_code != 200:
return []
return [ProviderProduct(self.name, _digits(p.get("code")) or "", True, title=p.get("product_name"), brand=(p.get("brands") or "").split(",")[0].strip() or None, quantity=p.get("quantity"))
for p in (response.json().get("products") or [])[:5] if p.get("product_name")]
class OpenProductsFacts(OpenFoodFacts):
"""Same open (ODbL) platform and API as Open Food Facts, for non-food goods. Identity only (no NCM)."""
name = "openproductsfacts"
host = "world.openproductsfacts.org"
supports_search = False
class OpenBeautyFacts(OpenFoodFacts):
name = "openbeautyfacts"
host = "world.openbeautyfacts.org"
supports_search = False
class Cosmos(Provider):
name = "cosmos"
supports_search = True
daily_limit = 25
provides_ncm = True
@property
def configured(self) -> bool:
return bool(self._token)
def _headers(self) -> dict:
return {"X-Cosmos-Token": self._token or "", "User-Agent": USER_AGENT, "Content-Type": "application/json"}
def lookup_gtin(self, gtin: str) -> ProviderProduct:
if not self.configured:
raise ProviderError("AUTH_ERROR", "NOT_CONFIGURED")
response = self._get(f"https://api.cosmos.bluesoft.com.br/gtins/{gtin}", headers=self._headers())
headers = {k.lower(): v for k, v in response.headers.items()}
if response.status_code == 404:
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
if response.status_code != 200:
raise ProviderError("BAD_RESPONSE", f"HTTP {response.status_code}")
try:
body = response.json()
except ValueError as error:
raise ProviderError("BAD_RESPONSE", "not JSON") from error
ncm = _digits((body.get("ncm") or {}).get("code"))
cest = _digits((body.get("cest") or {}).get("code"))
brand = (body.get("brand") or {}).get("name") if isinstance(body.get("brand"), dict) else body.get("brand")
return ProviderProduct(self.name, gtin, True, title=body.get("description"), brand=brand, category=(body.get("gpc") or {}).get("description"), quantity=body.get("net_weight") and str(body.get("net_weight")),
ncm=ncm, cest=cest, raw_hash=hashlib.sha256(json.dumps(body, sort_keys=True, ensure_ascii=False).encode()).hexdigest(), latency_ms=getattr(response, "latency_ms", 0), headers=headers)
def search_description(self, text: str) -> list[ProviderProduct]:
if not self.configured:
return []
response = self._get("https://api.cosmos.bluesoft.com.br/products", params={"query": text}, headers=self._headers())
if response.status_code != 200:
return []
out = []
for item in (response.json().get("products") or [])[:5]:
out.append(ProviderProduct(self.name, _digits(item.get("gtin")) or "", True, title=item.get("description"), brand=(item.get("brand") or {}).get("name") if isinstance(item.get("brand"), dict) else None,
ncm=_digits((item.get("ncm") or {}).get("code")) if isinstance(item.get("ncm"), dict) else None))
return out
class Oscbr(Provider):
"""OSCBR / RSC Sistemas GTIN API (https://gtin.rscsistemas.com.br). Free plan (no card): 50 calls/day, 5/min; account created BY THE OPERATOR, credentials only in the Hub env
(OSCBR_USER / OSCBR_PASSWORD). Bearer token lasts 1 h and is renewed automatically. Optional: NOT_CONFIGURED without credentials (no call is made)."""
name = "oscbr"
rate_per_minute = 5 # free plan as currently documented: 50 calls/day, 5/min (conservative: the lower figure wins)
daily_limit = 50
# ROLE = IDENTITY. The operator-provided V3 contract (GET /api/v3/gtin/{ean}) documents ean, ean_tipo, nome, marca, categoria, link_foto - NO ncm/cest. The public landing page mentions NCM/CEST
# and the legacy open-source client shows an `ncm` integer (leading zero lost), but the V3 documentation is behind the login, so NCM is UNPROVEN: the provider does not count as an NCM family
# (provides_ncm=False). If a real authenticated V3 answer ever carries `ncm`, it is stored as evidence but the role only changes after that proof is reviewed.
provides_ncm = False
base = "https://gtin.rscsistemas.com.br"
def __init__(self, client=None, token=None, user=None, password=None):
super().__init__(client, token) # token = a ready Bearer token (OSCBR_TOKEN); preferred over storing the login
self._user, self._password = user, password
self._bearer, self._bearer_until = None, 0.0
@property
def configured(self) -> bool:
return bool(self._token or (self._user and self._password))
def _auth(self) -> str:
if self._token:
return self._token
if self._bearer and time.time() < self._bearer_until:
return self._bearer
try:
response = self.client.post(f"{self.base}/oauth/token", auth=(self._user, self._password), headers={"User-Agent": USER_AGENT})
except httpx.TimeoutException as error:
raise ProviderError("TIMEOUT", type(error).__name__) from error
except httpx.HTTPError as error:
raise ProviderError("NETWORK", type(error).__name__) from error
if response.status_code in (400, 401, 403):
raise ProviderError("AUTH_ERROR", f"HTTP {response.status_code}")
if response.status_code == 429:
raise ProviderError("RATE_LIMITED", "HTTP 429")
if response.status_code >= 500 or response.status_code != 200:
raise ProviderError("SERVER_ERROR", f"HTTP {response.status_code}")
try:
token = response.json().get("token") or response.json().get("access_token")
except ValueError as error:
raise ProviderError("BAD_RESPONSE", "not JSON") from error
if not token:
raise ProviderError("BAD_RESPONSE", "no token")
self._bearer, self._bearer_until = token, time.time() + 55 * 60
return token
def lookup_gtin(self, gtin: str) -> ProviderProduct:
if not self.configured:
raise ProviderError("AUTH_ERROR", "NOT_CONFIGURED")
response = self._get(f"{self.base}/api/v3/gtin/{gtin}", headers={"Authorization": f"Bearer {self._auth()}", "User-Agent": USER_AGENT})
headers = {k.lower(): v for k, v in response.headers.items()}
if response.status_code == 404:
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
if response.status_code != 200:
raise ProviderError("BAD_RESPONSE", f"HTTP {response.status_code}")
try:
body = response.json()
except ValueError as error:
raise ProviderError("BAD_RESPONSE", "not JSON") from error
if isinstance(body, list):
body = body[0] if body else {}
if not body or not (body.get("nome") or body.get("ncm")):
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
ncm = _digits(body.get("ncm"))
if ncm and len(ncm) == 7: # numeric JSON drops the leading zero (7133399 is NCM 07133399)
ncm = ncm.zfill(8)
return ProviderProduct(self.name, gtin, True, title=body.get("nome"), brand=body.get("marca"), category=body.get("categoria"), ncm=ncm if len(ncm or "") == 8 else None, cest=_digits(body.get("cest")) or None,
raw_hash=hashlib.sha256(json.dumps(body, sort_keys=True, ensure_ascii=False).encode()).hexdigest(), latency_ms=getattr(response, "latency_ms", 0), headers=headers)
class Kodebar(Provider):
"""Kodebar (Korven Sistemas) - IDENTITY source only (name, brand, thumbnail; NO NCM). Free 'Teste' plan: 50 queries/day, no card, personal API key created by the operator
(KODEBAR_API_KEY in the Hub env). Terms (2026-09-26): integration into our own multi-user system and internal caching are allowed; redistribution/resale is not.
429 above the quota, no overage billing. NOT_CONFIGURED without a key: no request is made."""
name = "kodebar"
daily_limit = 50
rate_per_minute = 30
base = "https://kodebar.korvensistemas.com.br"
@property
def configured(self) -> bool:
return bool(self._token)
def lookup_gtin(self, gtin: str) -> ProviderProduct:
if not self.configured:
raise ProviderError("AUTH_ERROR", "NOT_CONFIGURED")
response = self._get(f"{self.base}/gtin/lookup", params={"gtin": gtin}, headers={"X-API-Key": self._token or "", "User-Agent": USER_AGENT})
headers = {k.lower(): v for k, v in response.headers.items()}
if response.status_code == 404:
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
if response.status_code != 200:
raise ProviderError("BAD_RESPONSE", f"HTTP {response.status_code}")
try:
body = response.json()
except ValueError as error:
raise ProviderError("BAD_RESPONSE", "not JSON") from error
if not isinstance(body, dict) or not body.get("nome"):
return ProviderProduct(self.name, gtin, False, latency_ms=getattr(response, "latency_ms", 0), headers=headers)
return ProviderProduct(self.name, gtin, True, title=body.get("nome"), brand=body.get("marca"), raw_hash=hashlib.sha256(json.dumps({k: body.get(k) for k in ("gtin", "nome", "marca")}, sort_keys=True, ensure_ascii=False).encode()).hexdigest(),
latency_ms=getattr(response, "latency_ms", 0), headers=headers)
def build_providers(settings, client: httpx.Client | None = None) -> list[Provider]:
"""Fixed order chosen from the benchmark (docs/product_enrichment.md): Cosmos then OSCBR (fiscal-capable) first when configured, then UPCitemdb, then the Open *Facts family (food, general products, beauty; identity only). Paid providers do not exist here."""
providers: list[Provider] = [Oscbr(client, getattr(settings, "oscbr_token", None), getattr(settings, "oscbr_user", None), getattr(settings, "oscbr_password", None)), Kodebar(client, getattr(settings, "kodebar_api_key", None)), UpcItemDb(client), OpenFoodFacts(client), OpenProductsFacts(client), OpenBeautyFacts(client)]
if getattr(settings, "cosmos_enabled", False): # Cosmos is OFF by default (account unavailable); only an explicit COSMOS_ENABLED=1 puts it back in the chain
providers.insert(0, Cosmos(client, getattr(settings, "cosmos_token", None)))
if any(p.paid for p in providers) and not getattr(settings, "allow_paid_providers", False):
raise RuntimeError("paid providers are not allowed (ALLOW_PAID_PROVIDERS=false)")
return providers
+244
View File
@@ -0,0 +1,244 @@
"""Central quota / circuit breaker / cache / single-flight for external product providers. Quota belongs to the HUB: 70 Agents never spend it uncoordinated."""
from __future__ import annotations
import threading
import time
from collections import defaultdict, deque
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from ..models import GtinLookupCache, ProviderState, ProviderUsageDaily
from .providers import Provider, ProviderError, ProviderProduct
POSITIVE_TTL = timedelta(days=30)
NEGATIVE_TTL = timedelta(hours=24)
CIRCUIT_BASE = timedelta(seconds=60)
CIRCUIT_MAX = timedelta(minutes=15)
FAILURES_TO_OPEN = 3
_windows: dict[str, deque] = defaultdict(deque) # per-process sliding windows (burst / per-minute governors)
_flights: dict[tuple, threading.Lock] = {}
_flights_guard = threading.Lock()
def utcnow() -> datetime:
return datetime.now(timezone.utc)
def _aware(value):
if value is not None and value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value
def next_utc_midnight(now: datetime) -> datetime:
return (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)
@dataclass
class Outcome:
provider: str
state: str # CACHE_HIT | FOUND | NOT_FOUND | SKIPPED | ERROR
product: ProviderProduct | None = None
reason: str | None = None # NOT_CONFIGURED | CIRCUIT_OPEN | RATE_LIMITED | RATE_WINDOW | TIMEOUT | AUTH_ERROR | NETWORK | SERVER_ERROR | ...
retry_at: datetime | None = None
from_cache: bool = False
class QuotaManager:
def __init__(self, session, now=utcnow):
self.session, self.now = session, now
# ---- state -------------------------------------------------------------------------------------------------------------------------------------
def state(self, provider: Provider) -> ProviderState:
row = self.session.get(ProviderState, provider.name)
if row is None:
row = ProviderState(provider=provider.name, configured=provider.configured, status="ONLINE" if provider.configured else "NOT_CONFIGURED",
daily_limit=provider.daily_limit, rate_per_minute=provider.rate_per_minute,
quota_kind="LOCAL_ESTIMATE" if provider.daily_limit else "DOCUMENTED_RATE")
self.session.add(row)
self.session.flush()
row.configured = provider.configured
if not provider.configured:
row.status = "NOT_CONFIGURED"
elif row.status == "NOT_CONFIGURED":
row.status = "ONLINE"
row.daily_limit = provider.daily_limit
row.rate_per_minute = provider.rate_per_minute
return row
def usage(self, provider_name: str, day: str | None = None) -> ProviderUsageDaily:
day = day or self.now().strftime("%Y-%m-%d")
row = self.session.scalar(select(ProviderUsageDaily).where(ProviderUsageDaily.provider == provider_name, ProviderUsageDaily.day == day))
if row is None:
row = ProviderUsageDaily(provider=provider_name, day=day)
self.session.add(row)
self.session.flush()
return row
# ---- decision ----------------------------------------------------------------------------------------------------------------------------------
def allow(self, provider: Provider, purpose: str = "OPERATIONAL", reserve_percent: int = 0) -> tuple[bool, str | None, datetime | None]:
state, now = self.state(provider), self.now()
if not provider.configured:
return False, "NOT_CONFIGURED", None
if state.circuit_open_until and _aware(state.circuit_open_until) > now:
return False, "CIRCUIT_OPEN", _aware(state.circuit_open_until)
usage = self.usage(provider.name)
if provider.daily_limit and state.quota_kind == "REMOTE_QUOTA" and state.remaining is not None and state.remaining <= 0 and state.reset_at and _aware(state.reset_at) > now:
return False, "RATE_LIMITED", _aware(state.reset_at)
if provider.daily_limit and usage.requests >= provider.daily_limit and state.quota_kind != "REMOTE_QUOTA":
state.status = "RATE_LIMITED"
return False, "RATE_LIMITED", state.reset_at and _aware(state.reset_at) or next_utc_midnight(now)
if purpose == "BACKFILL" and provider.daily_limit: # backfill may only use what is left above the operational reserve
used = usage.requests
if state.quota_kind == "REMOTE_QUOTA" and state.remaining is not None:
used = provider.daily_limit - state.remaining
if used >= provider.daily_limit * (100 - reserve_percent) / 100:
return False, "OPERATIONAL_RESERVE", state.reset_at and _aware(state.reset_at) or next_utc_midnight(now)
window = (provider.burst[1] if provider.burst else 60)
limit = (provider.burst[0] if provider.burst else provider.rate_per_minute)
if limit:
q, cutoff = _windows[provider.name], time.monotonic() - window
while q and q[0] < cutoff:
q.popleft()
if len(q) >= limit:
return False, "RATE_WINDOW", now + timedelta(seconds=max(1, int(window - (time.monotonic() - q[0]))))
return True, None, None
# ---- accounting --------------------------------------------------------------------------------------------------------------------------------
def _remote_quota(self, state: ProviderState, headers: dict):
remaining, limit, reset = headers.get("x-ratelimit-remaining"), headers.get("x-ratelimit-limit"), headers.get("x-ratelimit-reset")
if remaining is not None and str(remaining).isdigit():
state.remaining = int(remaining)
state.quota_kind = "REMOTE_QUOTA"
if limit and str(limit).isdigit():
state.daily_limit = int(limit)
if reset and str(reset).isdigit():
state.reset_at = datetime.fromtimestamp(int(reset), tz=timezone.utc)
def _sent(self, provider: Provider):
q = _windows[provider.name]
q.append(time.monotonic())
def record_ok(self, provider: Provider, result: ProviderProduct):
state, usage, now = self.state(provider), self.usage(provider.name), self.now()
self._sent(provider)
usage.requests += 1
usage.success += 1
if not result.found:
usage.not_found += 1
else:
usage.found += 1
if result.ncm:
usage.ncm_found += 1
state.consecutive_failures = 0
state.circuit_open_until = None
state.status = "ONLINE"
state.last_request_at = state.last_success_at = now
state.last_latency_ms = result.latency_ms or None
self._remote_quota(state, {k.lower(): v for k, v in (result.headers or {}).items()})
if provider.daily_limit and state.quota_kind != "REMOTE_QUOTA":
state.remaining = max(0, provider.daily_limit - usage.requests)
state.reset_at = next_utc_midnight(now)
self.session.commit()
def record_error(self, provider: Provider, error: ProviderError):
state, usage, now = self.state(provider), self.usage(provider.name), self.now()
self._sent(provider)
usage.requests += 1
state.last_request_at, state.last_error_at = now, now
state.last_error = f"{error.kind}: {error.detail}"[:160]
state.consecutive_failures += 1
headers = {k.lower(): v for k, v in (error.headers or {}).items()}
self._remote_quota(state, headers)
if error.kind == "RATE_LIMITED":
usage.rate_limited += 1
state.status = "RATE_LIMITED"
wait = timedelta(seconds=error.retry_after) if error.retry_after else None
state.circuit_open_until = now + (wait or (state.reset_at and max(timedelta(minutes=15), _aware(state.reset_at) - now) or timedelta(minutes=60)))
if state.reset_at is None:
state.reset_at = state.circuit_open_until
elif error.kind == "AUTH_ERROR":
usage.errors += 1
state.status = "AUTH_ERROR"
state.circuit_open_until = now + timedelta(hours=1)
else:
if error.kind == "TIMEOUT":
usage.timeouts += 1
usage.errors += 1
state.status = "DEGRADED"
if state.consecutive_failures >= FAILURES_TO_OPEN:
delay = min(CIRCUIT_MAX, CIRCUIT_BASE * (2 ** (state.consecutive_failures - FAILURES_TO_OPEN)))
state.circuit_open_until = now + delay
state.status = "OFFLINE"
self.session.commit()
def record_kb_hit(self):
"""A resolution answered by the KB alone (no provider asked): counted once per day under the pseudo-provider 'kb'."""
self.usage("kb").kb_hits += 1
def record_cache(self, provider_name: str, hit: bool):
usage = self.usage(provider_name)
if hit:
usage.cache_hits += 1
else:
usage.cache_misses += 1
class ProviderGateway:
"""lookup(provider, gtin): cache -> single-flight -> quota/circuit -> HTTP -> account -> cache."""
def __init__(self, session, providers: list[Provider], now=utcnow):
self.session, self.providers, self.now = session, providers, now
self.quota = QuotaManager(session, now)
def _cached(self, provider: Provider, gtin: str) -> ProviderProduct | None | bool:
row = self.session.scalar(select(GtinLookupCache).where(GtinLookupCache.gtin == gtin, GtinLookupCache.provider == provider.name))
if row is None or _aware(row.expires_at) <= self.now():
return False
if not row.found:
return None
d = row.data or {}
return ProviderProduct(provider.name, gtin, True, title=d.get("title"), brand=d.get("brand"), category=d.get("category"), quantity=d.get("quantity"), ncm=d.get("ncm"), cest=d.get("cest"), raw_hash=d.get("raw_hash", ""))
def _store(self, provider: Provider, gtin: str, product: ProviderProduct):
now = self.now()
ttl = POSITIVE_TTL if product.found else NEGATIVE_TTL
row = self.session.scalar(select(GtinLookupCache).where(GtinLookupCache.gtin == gtin, GtinLookupCache.provider == provider.name))
data = {**product.as_data(), "raw_hash": product.raw_hash}
if row is None:
self.session.add(GtinLookupCache(gtin=gtin, provider=provider.name, found=product.found, data=data, fetched_at=now, expires_at=now + ttl))
else:
row.found, row.data, row.fetched_at, row.expires_at = product.found, data, now, now + ttl
self.session.commit()
def lookup(self, provider: Provider, gtin: str, purpose: str = "OPERATIONAL", reserve_percent: int = 0) -> Outcome:
hit = self._cached(provider, gtin)
if hit is not False:
self.quota.record_cache(provider.name, True)
self.session.commit()
return Outcome(provider.name, "CACHE_HIT", hit or None, from_cache=True) if hit else Outcome(provider.name, "NOT_FOUND", None, from_cache=True)
with _flights_guard:
lock = _flights.setdefault((provider.name, gtin), threading.Lock())
with lock: # 10 installations, same EAN -> 1 external call, the rest wait for the same answer
self.session.expire_all()
hit = self._cached(provider, gtin)
if hit is not False:
self.quota.record_cache(provider.name, True)
self.session.commit()
return Outcome(provider.name, "CACHE_HIT", hit or None, from_cache=True) if hit else Outcome(provider.name, "NOT_FOUND", None, from_cache=True)
self.quota.record_cache(provider.name, False)
ok, reason, retry_at = self.quota.allow(provider, purpose, reserve_percent)
if not ok:
self.session.commit()
return Outcome(provider.name, "SKIPPED", None, reason, retry_at)
try:
product = provider.lookup_gtin(gtin)
except ProviderError as error:
self.quota.record_error(provider, error)
return Outcome(provider.name, "ERROR", None, error.kind, self.quota.state(provider).circuit_open_until)
self.quota.record_ok(provider, product)
self._store(provider, gtin, product)
return Outcome(provider.name, "FOUND" if product.found else "NOT_FOUND", product if product.found else None)
+250
View File
@@ -0,0 +1,250 @@
"""Product resolver: GTIN/name -> KB -> providers -> official NCM validation -> existing fiscal engine, decided FIELD BY FIELD.
Deterministic. No AI, no scores: discrete states only.
Resolution states: EXACT_GTIN_VERIFIED, EXACT_KB_NAME_VERIFIED, MULTI_SOURCE_MATCH, SINGLE_WEAK_SOURCE, AMBIGUOUS, CONFLICT, NOT_FOUND
(+ operational: INVALID_GTIN, PENDING_PROVIDER_QUOTA, PENDING_PROVIDERS_OFFLINE).
Field decisions: AUTO_FILL_SAFE | SUGGESTION_ONLY | CONFLICT | INSUFFICIENT_EVIDENCE | ALREADY_SET | NOT_APPLICABLE.
"""
from __future__ import annotations
import time
from datetime import date
from sqlalchemy import select
from ..classification import Classifier, normalize_code
from ..datasets.service import DatasetService
from ..models import Dataset, NcmEntry
from . import fiscal_completion, kb, textnorm
from .gtin import normalize_gtin
from .placeholders import is_placeholder, is_writable_empty
from .quota import ProviderGateway
AUTO, SUGGEST, CONFLICT, INSUFFICIENT, ALREADY, NA = "AUTO_FILL_SAFE", "SUGGESTION_ONLY", "CONFLICT", "INSUFFICIENT_EVIDENCE", "ALREADY_SET", "NOT_APPLICABLE"
FIELDS = ("CLASSIFICACAO_FISCAL", "CEST", "IPI")
def _parse_br(text: str | None) -> date | None:
try:
d, m, y = str(text).split("/")
return date(int(y), int(m), int(d))
except (ValueError, TypeError):
return None
def ncm_is_official(session, code: str, today: date | None = None) -> bool:
"""8-digit code present in the ACTIVE official NCM dataset and vigente today (dd/mm/yyyy, 31/12/9999 = open)."""
code = normalize_code(code)
if len(code) != 8:
return False
today = today or date.today()
rows = session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == code)).all()
for row in rows:
start, end = _parse_br(row.valid_from), _parse_br(row.valid_to)
if (start is None or start <= today) and (end is None or end >= today):
return True
return False
class _ScopedDatasets(DatasetService):
"""The Classifier only needs the ONE exact NCM row here; loading the whole 15k-row catalogue on every product would cost ~300 ms of the 5 s budget."""
def __init__(self, session, code: str):
super().__init__(session)
self.code = code
def active_entries(self):
return self.session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == self.code)).all()
def ncm_exists_any(session, code: str, *, current: bool, today: date | None = None) -> bool:
"""current=True: vigente today; current=False: present in the ACTIVE dataset with any validity window (i.e. known but not vigente)."""
code = normalize_code(code)
if len(code) != 8:
return False
if current:
return ncm_is_official(session, code, today)
return session.scalar(select(NcmEntry.id).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE", NcmEntry.code == code).limit(1)) is not None
def dataset_versions(session) -> dict:
return {d.kind: {"sha256": (d.sha256 or "")[:12], "version": d.version_label} for d in session.scalars(select(Dataset).where(Dataset.status == "ACTIVE"))}
def _field(status, value=None, rule=None, kb_status=None, evidence=None, reason=None):
return {"status": status, "value": value, "rule": rule, "kb_status": kb_status, "evidence": evidence or [], "reason": reason}
def _empty(current: dict, name: str) -> bool:
return is_writable_empty(name, current.get(name))
def _merge_fiscal_completion(session, out: dict, *, ncm_official: bool, ncm_for_engine: str | None, current: dict, description: str,
product_code: str | None, installation, tenant_id: int | None, decision: dict | None, product_evidence: dict | None = None) -> None:
"""Merges field_contract.resolve_product's own field-by-field decisions (CFOP/CST/CSOSN/PIS-COFINS/...) into `out`, in the SAME
AUTO_FILL_SAFE/ALREADY_SET/NOT_APPLICABLE/INSUFFICIENT_EVIDENCE vocabulary CEST/IPI already use, plus NEEDS_REVIEW for anything that needs an
operator's context answer. No-op when the caller (a test, or a code path with no installation on hand) did not pass installation/tenant_id/product_code."""
if installation is None or tenant_id is None or product_code is None:
return
fields = fiscal_completion.resolve_fiscal_fields(session, product_code=product_code, description=description, current=current, ncm=ncm_for_engine or "",
ncm_official=ncm_official, installation=installation, tenant_id=tenant_id, decision=decision, extra_columns=current,
ev=product_evidence)
for field, r in fields.items():
out[field] = _field(r["status"], r.get("value"), r.get("rule"), reason=r.get("reason"))
if r.get("missing_key"):
out[field]["missing_key"], out[field]["missing_scope"] = r["missing_key"], r.get("missing_scope")
def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current: dict, description: str, *, name_verified=False,
product_code: str | None = None, installation=None, tenant_id: int | None = None, product_evidence: dict | None = None) -> dict:
out = {}
if not _empty(current, "CLASSIFICACAO_FISCAL"):
out["CLASSIFICACAO_FISCAL"] = _field(ALREADY, reason="campo já preenchido")
elif ncm_eval["status"] == kb.CONFLICT:
out["CLASSIFICACAO_FISCAL"] = _field(CONFLICT, kb_status=kb.CONFLICT, evidence=ncm_eval["candidates"], reason="fontes divergem no NCM")
elif ncm_eval["value"] and ncm_eval["status"] == kb.VERIFIED_KB:
out["CLASSIFICACAO_FISCAL"] = _field(AUTO, ncm_eval["value"], "NCM_KB_VERIFIED" if gtin_kb_verified or name_verified else "NCM_TWO_INDEPENDENT_SOURCES", kb.VERIFIED_KB, ncm_eval["families"])
elif ncm_eval["value"]:
rule = {"CORROBORATED": "NCM_CORROBORATED_LOCAL_HISTORY", "EXTERNAL_CORROBORATED": "NCM_EXTERNAL_CORROBORATED"}.get(ncm_eval["status"], "NCM_SINGLE_SOURCE")
out["CLASSIFICACAO_FISCAL"] = _field(SUGGEST, ncm_eval["value"], rule, ncm_eval["status"], ncm_eval["families"], "evidência ainda insuficiente para preencher automaticamente")
else:
out["CLASSIFICACAO_FISCAL"] = _field(INSUFFICIENT, reason="nenhuma fonte com NCM válido no dataset oficial")
ncm = out["CLASSIFICACAO_FISCAL"]
own = current.get("CLASSIFICACAO_FISCAL")
ncm_for_engine = ncm["value"] if ncm["status"] == AUTO else (None if is_placeholder("CLASSIFICACAO_FISCAL", own) else (normalize_code(own) or None))
if not ncm_for_engine or (ncm["status"] == ALREADY and not ncm_is_official(session, ncm_for_engine)):
for name in ("CEST", "IPI"):
out[name] = _field(ALREADY if not _empty(current, name) else INSUFFICIENT, reason="depende de um NCM oficial válido")
_merge_fiscal_completion(session, out, ncm_official=False, ncm_for_engine=ncm_for_engine, current=current, description=description,
product_code=product_code, installation=installation, tenant_id=tenant_id, decision=None, product_evidence=product_evidence)
return out
# CEST / IPI are deterministic functions of an OFFICIAL NCM: the NCM we are about to write (AUTO) or one already in the product (validated above against the official dataset)
ncm_solid = ncm["status"] == AUTO or (ncm["status"] == ALREADY and ncm_is_official(session, ncm_for_engine))
decision = Classifier(_ScopedDatasets(session, ncm_for_engine)).classify({"current_ncm": ncm_for_engine, "description": description})
cest = (decision["classification"].get("cest") or [])
if not _empty(current, "CEST"):
out["CEST"] = _field(ALREADY, reason="campo já preenchido")
elif decision["cest_status"] == "CEST_CONFIRMED" and len(cest) == 1 and ncm_solid:
out["CEST"] = _field(AUTO, normalize_code(cest[0]), "CEST_OFFICIAL_UNIQUE_MAPPING", ncm["kb_status"], [f"NCM {ncm_for_engine}"])
elif decision["cest_status"] == "NO_CEST_MAPPING_FOUND":
out["CEST"] = _field(NA, reason="NCM sem mapeamento de CEST no dataset oficial")
else:
out["CEST"] = _field(INSUFFICIENT, reason=f"CEST não determinístico ({decision['cest_status']})")
ipi = decision["classification"].get("ipi") or {}
if not _empty(current, "IPI"):
out["IPI"] = _field(ALREADY, reason="campo já preenchido")
elif decision["ipi_status"] == "IPI_RESOLVED" and ipi.get("rate_kind") == "NUMERIC" and ncm_solid:
out["IPI"] = _field(AUTO, str(ipi["rate_numeric"]), "IPI_TIPI_NUMERIC", ncm["kb_status"], [f"TIPI NCM {ncm_for_engine}"])
elif decision["ipi_status"] in {"IPI_RESOLVED"}:
out["IPI"] = _field(NA, reason="TIPI sem alíquota numérica (NT/branca)")
else:
out["IPI"] = _field(INSUFFICIENT, reason=f"IPI não determinístico ({decision['ipi_status']})")
_merge_fiscal_completion(session, out, ncm_official=ncm_solid, ncm_for_engine=ncm_for_engine, current=current, description=description,
product_code=product_code, installation=installation, tenant_id=tenant_id, decision=decision, product_evidence=product_evidence)
return out
def resolve(session, gateway: ProviderGateway, *, gtin_raw, description, current: dict | None = None, client_label: str | None = None, allow_external: bool = True, budget_s: float | None = None,
product_code: str | None = None, installation=None, tenant_id: int | None = None, product_evidence: dict | None = None) -> dict:
started = time.monotonic()
"""Returns {state, gtin_status, gtin, fields:{...}, providers:[...], retry_at, dataset_versions}. Never raises for provider trouble.
`product_code`/`installation`/`tenant_id`, when given, additionally resolve the post-NCM fiscal-completion fields (CFOP/CST/CSOSN/PIS-COFINS/...)
via the existing field_contract engine (see `_merge_fiscal_completion`) and merge them into `fields` alongside CLASSIFICACAO_FISCAL/CEST/IPI.
`product_evidence` is the Agent's targeted, product_code-scoped recipes/composed/purchases/stock evidence (firebird_reader.read_product_evidence) -
never a full-table scan. Named separately from the local `ev` variable below (NCM knowledge-base evidence), a different, pre-existing concept."""
current = {k: v for k, v in (current or {}).items()}
g = normalize_gtin(gtin_raw)
result = {"state": "NOT_FOUND", "gtin_status": g["status"], "gtin": g.get("key"), "fields": {}, "providers": [], "retry_at": None, "dataset_versions": dataset_versions(session)}
session.info["is_official"] = lambda v: ncm_is_official(session, v)
is_official = session.info["is_official"]
product, name_verified, key = None, False, g.get("key")
if g["status"] == "VALID_GTIN": # GTIN has absolute priority over the description
product = kb.find_by_gtin(session, key)
ev = kb.evaluate(session, product, "ncm", is_official) if product else None
if ev and ev["status"] == kb.VERIFIED_KB:
gateway.quota.record_kb_hit()
result["state"] = "EXACT_GTIN_VERIFIED"
result["fields"] = _fiscal_decisions(session, ev, True, current, description or "", product_code=product_code, installation=installation, tenant_id=tenant_id, product_evidence=product_evidence)
session.commit()
return result
if ev and ev["status"] == kb.CONFLICT: # local evidence already conflicts: an external answer cannot settle it, so spend no quota
result["state"] = "CONFLICT"
result["fields"] = _fiscal_decisions(session, ev, False, current, description or "", product_code=product_code, installation=installation, tenant_id=tenant_id, product_evidence=product_evidence)
gateway.quota.record_kb_hit()
session.commit()
return result
locally_known = bool(ev and ev["status"] in (kb.CORROBORATED, kb.EXTERNAL_CORROBORATED)) # identity + NCM candidate already local: only an NCM-bearing source can still add value
query = g["gtin"] if len(g["gtin"]) in (8, 14) else key[1:] # providers/caches use the canonical EAN-13 (UPC-12 is zero-padded to it)
found, pending = [], []
if allow_external:
for provider in gateway.providers:
if locally_known and not getattr(provider, "provides_ncm", False):
result["providers"].append({"provider": provider.name, "state": "SKIPPED", "reason": "LOCAL_IDENTITY_KNOWN", "from_cache": False})
continue
if budget_s is not None and time.monotonic() - started > budget_s:
result["providers"].append({"provider": provider.name, "state": "SKIPPED", "reason": "TIME_BUDGET", "from_cache": False})
pending.append(type("P", (), {"reason": "TIME_BUDGET", "retry_at": None})())
continue
out = gateway.lookup(provider, query)
result["providers"].append({"provider": out.provider, "state": out.state, "reason": out.reason, "from_cache": out.from_cache})
if out.state in ("FOUND", "CACHE_HIT") and out.product:
found.append(out.product)
product = kb.upsert_product(session, key, out.product.title or description, brand=out.product.brand, quantity=out.product.quantity)
if out.product.ncm:
kb.add_evidence(session, product, "ncm", out.product.ncm, f"provider:{out.provider}"[:32], family=f"PROVIDER:{out.provider}", info={"raw_hash": out.product.raw_hash})
ev = kb.evaluate(session, product, "ncm", is_official)
if ev["status"] == kb.VERIFIED_KB:
break # enough evidence: spare the remaining quota
elif out.state in ("SKIPPED", "ERROR"):
pending.append(out)
if product is None and description:
product = kb.upsert_product(session, key, description)
ev = kb.evaluate(session, product, "ncm", is_official) if product else {"value": None, "status": kb.UNVERIFIED, "families": [], "candidates": [], "rejected": []}
result["fields"] = _fiscal_decisions(session, ev, False, current, description or "", product_code=product_code, installation=installation, tenant_id=tenant_id, product_evidence=product_evidence)
if ev["status"] == kb.CONFLICT:
result["state"] = "CONFLICT"
elif ev["status"] == kb.VERIFIED_KB:
result["state"] = "MULTI_SOURCE_MATCH" if found else "EXACT_GTIN_VERIFIED"
elif len(found) >= 2:
result["state"] = "MULTI_SOURCE_MATCH"
elif found:
result["state"] = "SINGLE_WEAK_SOURCE"
elif pending and not found:
quota = any(p.reason in ("RATE_LIMITED", "RATE_WINDOW") for p in pending)
offline = [p for p in pending if p.reason not in ("NOT_CONFIGURED",)]
if offline:
result["state"] = "PENDING_PROVIDER_QUOTA" if quota else "PENDING_PROVIDERS_OFFLINE"
result["retry_at"] = min((p.retry_at for p in offline if p.retry_at), default=None)
session.commit()
return result
if g["status"] == "INVALID_GTIN":
result["state"] = "INVALID_GTIN" # never spends provider quota on an invalid GTIN
# ---- name only: only a UNIQUE, specific description with a VERIFIED_KB fingerprint may fill anything
if not textnorm.is_specific(description):
if result["state"] != "INVALID_GTIN":
result["state"] = "AMBIGUOUS"
result["fields"] = {f: _field(INSUFFICIENT, reason="descrição genérica demais e sem GTIN válido") if _empty(current, f) else _field(ALREADY) for f in FIELDS}
return result
candidates = kb.name_candidates(session, description)
verified = []
for cand in candidates:
ev = kb.evaluate(session, cand, "ncm", is_official)
if ev["status"] == kb.VERIFIED_KB:
verified.append((cand, ev))
if candidates and len(verified) == len(candidates) and len({ev["value"] for _, ev in verified}) == 1:
_, ev = verified[0]
result["state"] = "EXACT_KB_NAME_VERIFIED"
result["fields"] = _fiscal_decisions(session, ev, True, current, description or "", name_verified=True, product_code=product_code, installation=installation, tenant_id=tenant_id, product_evidence=product_evidence)
elif len({ev["value"] for _, ev in verified}) > 1:
result["state"] = "CONFLICT"
result["fields"] = {f: _field(CONFLICT, reason="descrição corresponde a produtos com NCM divergentes") if _empty(current, f) else _field(ALREADY) for f in FIELDS}
else:
if result["state"] != "INVALID_GTIN":
result["state"] = "AMBIGUOUS" if len(candidates) > 1 else "NOT_FOUND"
result["fields"] = {f: _field(INSUFFICIENT, reason="sem correspondência verificada na base de conhecimento") if _empty(current, f) else _field(ALREADY) for f in FIELDS}
session.commit()
return result
+86
View File
@@ -0,0 +1,86 @@
"""Human-facing status of every tracked product: nothing may silently 'do nothing'. Computed from the resolver result and refined when the Agent reports the write outcome."""
from __future__ import annotations
from sqlalchemy import select, update
LABELS = {
"ENRIQUECIDO": "Enriquecido: campos seguros preenchidos.",
"PRONTO_PARA_PREENCHER": "Evidência suficiente: aguardando a escrita segura pelo Agent.",
"SIMULADO": "Simulação (DRY_RUN): há candidato, nada foi gravado.",
"AGUARDANDO_EVIDENCIA": "Aguardando evidência: nenhuma fonte confiável ainda.",
"EAN_AUSENTE": "Sem EAN e sem produto único verificado na base: informe o código de barras ou peça a decisão de um administrador.",
"EAN_INVALIDO": "Código de barras inválido: nenhuma fonte externa foi consultada.",
"NOME_AMBIGUO": "Nome genérico ou ambíguo demais para identificar o produto com segurança.",
"SEM_MATCH_DE_PRODUTO": "Nenhum produto conhecido corresponde a este nome/EAN.",
"NCM_NAO_VERIFICADO": "Produto identificado, mas o NCM ainda não foi verificado (falta segunda fonte independente ou decisão de administrador).",
"CONFLITO_DE_NCM": "As fontes divergem sobre o NCM: nada é preenchido até um administrador decidir.",
"PROVIDER_SEM_QUOTA": "Limite diário das fontes externas atingido; será repetido após o reset.",
"PROVIDER_NAO_CONFIGURADO": "Nenhuma fonte externa com NCM está configurada.",
"AGUARDANDO_RETRY": "Fontes indisponíveis agora; nova tentativa automática agendada.",
"HUMAN_OVERRIDE": "Alterado por uma pessoa: nunca mais preenchido automaticamente.",
"TECHNICAL_PLACEHOLDER": "NCM provisório (00000000) reconhecido como 'ainda não classificado'.",
"WRITE_FAILED": "A escrita segura foi recusada ou falhou (veja o motivo).",
"VERIFICACAO_POS_WRITE_FALHOU": "A escrita foi feita mas a releitura não confirmou o valor.",
"NADA_A_PREENCHER": "Todos os campos elegíveis já estão preenchidos.",
"FUNCAO_DESATIVADA": "Enriquecimento desativado para esta instalação.",
}
def classify(result: dict, *, has_gtin: bool, specific_name: bool) -> tuple[str, str]:
"""(ui_status, reason_code) from a resolve() result."""
fields = result.get("fields") or {}
ncm = fields.get("CLASSIFICACAO_FISCAL") or {}
state = result.get("state")
if state == "FEATURE_DISABLED":
return "FUNCAO_DESATIVADA", state
if state == "INVALID_GTIN":
return "EAN_INVALIDO", state
if state == "PENDING_PROVIDER_QUOTA":
return "PROVIDER_SEM_QUOTA", state
if state in ("PENDING_PROVIDERS_OFFLINE", "PENDING_TIME_BUDGET"):
return "AGUARDANDO_RETRY", state
if state == "CONFLICT" or ncm.get("status") == "CONFLICT":
return "CONFLITO_DE_NCM", "CONFLICT"
if ncm.get("status") == "HUMAN_OVERRIDDEN":
return "HUMAN_OVERRIDE", "HUMAN_OVERRIDDEN"
if any(f.get("status") == "AUTO_FILL_SAFE" for f in fields.values()):
return "PRONTO_PARA_PREENCHER", state or "RESOLVED"
if ncm.get("status") == "ALREADY_SET" and not any(f.get("status") in ("AUTO_FILL_SAFE", "SUGGESTION_ONLY") for f in fields.values()):
return "NADA_A_PREENCHER", "ALL_FIELDS_SET"
if not has_gtin:
if state == "AMBIGUOUS" or not specific_name:
return "NOME_AMBIGUO", "GENERIC_OR_AMBIGUOUS_NAME"
return "EAN_AUSENTE", "NAME_ONLY_NO_VERIFIED_MATCH"
if ncm.get("status") == "SUGGESTION_ONLY":
return "NCM_NAO_VERIFICADO", ncm.get("rule") or "NCM_NOT_VERIFIED"
if state == "NOT_FOUND":
return "SEM_MATCH_DE_PRODUTO", "GTIN_UNKNOWN"
return "AGUARDANDO_EVIDENCIA", state or "INSUFFICIENT_EVIDENCE"
def after_write(outcomes: list[dict], mode: str) -> tuple[str | None, str | None]:
"""Refines the status with what the Agent reported (APPLIED / DRY_RUN / ABORTED / FAILED / HUMAN_OVERRIDDEN)."""
kinds = {o.get("outcome") for o in outcomes}
reasons = [o.get("reason") for o in outcomes if o.get("reason")]
if "HUMAN_OVERRIDDEN" in kinds and "APPLIED" not in kinds:
return "HUMAN_OVERRIDE", "HUMAN_OVERRIDDEN"
if "APPLIED" in kinds:
return "ENRIQUECIDO", None
if "FAILED" in kinds:
return ("VERIFICACAO_POS_WRITE_FALHOU" if "POST_WRITE_VERIFICATION_FAILED" in reasons else "WRITE_FAILED"), (reasons[0] if reasons else "FAILED")
if "ABORTED" in kinds:
return "WRITE_FAILED", (reasons[0] if reasons else "ABORTED")
if "DRY_RUN" in kinds:
return "SIMULADO", "DRY_RUN_NO_WRITE"
return None, None
def mark_for_reevaluation(session, installation_id: int, *, codes: list[str] | None = None, gtin: str | None = None) -> int:
"""Flags tracked, NOT-yet-enriched products for a new evaluation (KB/cache first; providers only when really needed). Idempotent; never touches enriched / human-overridden ones."""
from ..models import ProductEnrichment
q = update(ProductEnrichment).where(ProductEnrichment.installation_id == installation_id, ProductEnrichment.ui_status.notin_(["ENRIQUECIDO", "HUMAN_OVERRIDE"]) | ProductEnrichment.ui_status.is_(None))
if codes:
q = q.where(ProductEnrichment.product_code.in_(codes))
if gtin:
q = q.where(ProductEnrichment.gtin == gtin)
return session.execute(q.values(reevaluate=True)).rowcount or 0
+51
View File
@@ -0,0 +1,51 @@
"""Deterministic product-description normalization and fingerprint (no ML). Used to FIND candidates; never authorizes a fiscal value on its own."""
import hashlib
import re
import unicodedata
UNIT_MAP = {"LT": "L", "LTS": "L", "LITRO": "L", "LITROS": "L", "L": "L", "ML": "ML", "KG": "KG", "KGS": "KG", "QUILO": "KG", "G": "G", "GR": "G", "GRS": "G", "GRAMAS": "G", "MG": "MG", "UN": "UN", "UND": "UN", "UNID": "UN", "UNIDADE": "UN", "PCT": "PCT", "CX": "CX"}
ABBREVIATIONS = {"REFRIG": "REFRIGERANTE", "REFRI": "REFRIGERANTE", "CHOC": "CHOCOLATE", "BISC": "BISCOITO", "SAB": "SABONETE", "DET": "DETERGENTE", "SH": "SHAMPOO", "COND": "CONDICIONADOR",
"MACARR": "MACARRAO", "MAC": "MACARRAO", "AMAC": "AMACIANTE", "DESOD": "DESODORANTE", "PAP": "PAPEL", "HIG": "HIGIENICO", "ORIG": "ORIGINAL", "TRAD": "TRADICIONAL"}
STOP = {"DE", "DA", "DO", "DAS", "DOS", "E", "COM", "C", "PARA", "P", "EM", "A", "O"}
MARKETING = {"ORIGINAL", "TRADICIONAL", "NOVO", "NOVA", "PROMO", "PROMOCAO", "OFERTA"} # ignored ONLY by the fingerprint; a different pack/flavour still differs
_QTY = re.compile(r"^(\d+(?:[.,]\d+)?)(ML|L|KG|G|MG|UN|PCT|CX|LT|LTS|GR)$")
def _ascii_upper(text) -> str:
return unicodedata.normalize("NFKD", str(text or "")).encode("ascii", "ignore").decode().upper()
def tokens(description) -> list[str]:
text = _ascii_upper(description)
text = re.sub(r"(\d)([A-Z])", r"\1 \2", text) # 350ML -> 350 ML
text = re.sub(r"([A-Z])(\d)", r"\1 \2", text)
text = re.sub(r"(\d)[,.](\d)", r"\1.\2", text) # decimal comma -> dot (kept inside the number)
text = re.sub(r"[^A-Z0-9.]+", " ", text)
out = []
for tok in text.split():
tok = tok.strip(".")
if not tok or tok in STOP:
continue
tok = UNIT_MAP.get(tok, ABBREVIATIONS.get(tok, tok))
out.append(tok)
return out
def normalized(description) -> str:
return " ".join(tokens(description))
def fingerprint(description) -> str | None:
toks = [t for t in tokens(description) if t not in MARKETING]
if not toks:
return None
return hashlib.sha256(" ".join(sorted(toks)).encode("utf-8")).hexdigest()
def is_specific(description) -> bool:
"""A description strong enough to be used as an identity key: at least 3 informative tokens AND a quantity (number + unit), or >= 4 tokens.
Generic names (PIPOCA, OLEO, REFRIGERANTE, SABONETE) are NEVER specific."""
toks = tokens(description)
has_qty = any(re.fullmatch(r"\d+(?:\.\d+)?", t) and i + 1 < len(toks) and toks[i + 1] in UNIT_MAP.values() for i, t in enumerate(toks))
informative = [t for t in toks if not re.fullmatch(r"\d+(?:\.\d+)?", t) and t not in UNIT_MAP.values()]
return (len(informative) >= 2 and has_qty and len(toks) >= 3) or len(informative) >= 4
+136
View File
@@ -0,0 +1,136 @@
"""Secrets and package bundling for Agent distribution.
Two independent secrets guard the distribution flow, both random(256 bits),
single-use and short-TTL, both stored only as SHA-256 hashes:
- the *download* token: embedded in the shareable link an operator hands to
whoever installs the Agent (or clicked directly, authenticated, from the
panel);
- the *enrollment* token: embedded inside the downloaded bundle as
`enrollment.json`, consumed exactly once by `POST /v1/agents/enroll`.
Neither token is ever the Agent's own permanent Bearer token (issued only
after a successful enrollment) and neither is ever logged or audited in
plaintext.
"""
from __future__ import annotations
import hashlib
import hmac
import io
import json
import re
import secrets
import zipfile
from dataclasses import dataclass
class EnrollmentRejected(Exception):
def __init__(self, reason: str):
super().__init__(reason)
self.reason = reason
def new_token() -> str:
"""256 bits of randomness, URL-safe — usable directly in a link or a JSON field."""
return secrets.token_urlsafe(32)
# ---- installation code (XXXX-XXXX-XXXX-XXXX) ----
#
# A 16-character Crockford Base32 code = 16 * 5 = 80 bits of entropy. The
# alphabet deliberately excludes I/L/O/U (ambiguous when retyped by a human);
# the same alphabet is used so a code cannot contain those letters.
INSTALL_CODE_ALPHABET = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
_INSTALL_CODE_GROUPS = 4
_INSTALL_CODE_PER_GROUP = 4
_INSTALL_CODE_LENGTH = _INSTALL_CODE_GROUPS * _INSTALL_CODE_PER_GROUP
# Crockford transliteration for input normalization: O -> 0, I/L -> 1.
_INSTALL_CODE_TRANSLIT = str.maketrans({"O": "0", "I": "1", "L": "1"})
_INSTALL_CODE_RE = re.compile(
r"^[" + re.escape(INSTALL_CODE_ALPHABET) + r"]{" + str(_INSTALL_CODE_LENGTH) + r"}$"
)
def generate_install_code() -> str:
"""Return a fresh installation code formatted XXXX-XXXX-XXXX-XXXX."""
chars = "".join(secrets.choice(INSTALL_CODE_ALPHABET) for _ in range(_INSTALL_CODE_LENGTH))
return "-".join(
chars[i : i + _INSTALL_CODE_PER_GROUP] for i in range(0, _INSTALL_CODE_LENGTH, _INSTALL_CODE_PER_GROUP)
)
def normalize_install_code(text: str | None) -> str:
"""Upper-case, drop hyphen/space separators and map Crockford lookalikes,
so a human-typed code normalizes to the same digest regardless of how it
was entered. Raises ValueError if the result isn't a valid code."""
if not text:
raise ValueError("empty install code")
clean = "".join(ch for ch in text.upper().translate(_INSTALL_CODE_TRANSLIT) if ch.isalnum())
if not _INSTALL_CODE_RE.match(clean):
raise ValueError(f"invalid install code: {text!r}")
return clean
def install_code_digest(code: str) -> str:
"""Hash a normalized install code. Only the hash is ever persisted; the
plaintext code exists only in the operator's clipboard/UI and the EXE's
memory for the brief instant it is sent to the Hub."""
return token_digest(normalize_install_code(code))
def valid_install_code(text: str | None) -> bool:
try:
normalize_install_code(text)
return True
except ValueError:
return False
def token_digest(token: str) -> str:
return hashlib.sha256(token.encode()).hexdigest()
def token_matches(token: str | None, stored_hash: str | None) -> bool:
if not token or not stored_hash:
return False
return hmac.compare_digest(token_digest(token), stored_hash)
ENROLLMENT_UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
def valid_uuid(value: str | None) -> bool:
return bool(value) and bool(ENROLLMENT_UUID_RE.match(value))
@dataclass(frozen=True)
class BundlePlan:
hub_url: str
enrollment_uuid: str
enrollment_token: str
enroll_path: str = "/v1/agents/enroll"
def build_bundle(release_zip_bytes: bytes, plan: BundlePlan) -> bytes:
"""Return a new zip = the release zip's entries plus one
`<package-root>/enrollment.json` holding only temporary, single-use data.
Never a permanent credential — that is issued only once enrollment succeeds."""
with zipfile.ZipFile(io.BytesIO(release_zip_bytes)) as source:
names = source.namelist()
if not names:
raise EnrollmentRejected("EMPTY_RELEASE_PACKAGE")
root = names[0].split("/", 1)[0]
out = io.BytesIO()
with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as dest:
for name in names:
dest.writestr(name, source.read(name))
payload = {
"installation_id": plan.enrollment_uuid,
"enrollment_token": plan.enrollment_token,
"hub_url": plan.hub_url,
"enroll_path": plan.enroll_path,
}
dest.writestr(f"{root}/enrollment.json", json.dumps(payload, indent=1))
return out.getvalue()
+74
View File
@@ -0,0 +1,74 @@
"""Structured operational events with defensive redaction and retention."""
from __future__ import annotations
import json
import re
from datetime import datetime, timedelta, timezone
from sqlalchemy import delete, select
from .config import settings
from .models import AgentEvent
EVENT_TYPES = {
"AGENT_STARTED", "AGENT_STOPPED", "HEARTBEAT", "PREFLIGHT_PASS", "PREFLIGHT_FAIL", "JOB_CLAIMED", "JOB_STARTED",
"JOB_COMPLETED", "JOB_FAILED", "PRODUCT_BLOCKED", "BLOCKER_RESOLVED", "DATASET_CHANGED", "COMPANY_CONTEXT_CHANGED",
"SCHEMA_CHANGED", "WRITE_PLANNED", "WRITE_COMPLETED", "WRITE_FAILED", "CONNECTION_ERROR", "RETRY", "VERSION_CHANGED",
"WINDOWS_INSTALL_COMPLETED", "WINDOWS_UNINSTALL_COMPLETED", "ADMIN_ACTION", "RTC_AUTO_APPLY", "CFOP_IPPT_AUTO_APPLY",
}
SEVERITIES = {"INFO", "WARNING", "ERROR", "CRITICAL"}
SECRET_RE = re.compile(r"(?i)(password|fb_password|token|api[_-]?key|secret|authorization|connection[_-]?string)\s*[:=]\s*[^,;\s]+")
def _redact(value):
if isinstance(value, dict):
return {str(key): "[REDACTED]" if str(key).lower() in {"password", "fb_password", "token", "api_key", "secret", "authorization", "connection_string"} else _redact(item) for key, item in value.items()}
if isinstance(value, list):
return [_redact(item) for item in value[:100]]
if isinstance(value, str):
return SECRET_RE.sub(lambda match: f"{match.group(1)}=[REDACTED]", value)
return value
def sanitize_event(message: str, context: dict | None = None) -> tuple[str, dict]:
safe_message = _redact(str(message or ""))[: settings.web_max_event_message]
safe_context = _redact(context or {})
encoded = json.dumps(safe_context, ensure_ascii=False, default=str)
if len(encoded.encode()) > settings.web_max_event_context_bytes:
encoded = encoded.encode()[: settings.web_max_event_context_bytes].decode("utf-8", errors="ignore")
safe_context = {"truncated": True, "payload": encoded}
return safe_message, safe_context
def record_event(session, *, tenant_id: int, installation_id: int, agent_id: int, severity: str, event_type: str, message: str, context: dict | None = None, job_id: int | None = None, product_code: str | None = None, blocker_id: int | None = None, is_audit: bool = False, occurred_at: datetime | None = None):
severity = severity.upper()
event_type = event_type.upper()
if severity not in SEVERITIES:
raise ValueError("invalid event severity")
if event_type not in EVENT_TYPES:
raise ValueError("invalid event type")
safe_message, safe_context = sanitize_event(message, context)
event = AgentEvent(tenant_id=tenant_id, installation_id=installation_id, agent_id=agent_id, severity=severity, event_type=event_type, message=safe_message, structured_context=safe_context, job_id=job_id, product_code=product_code, blocker_id=blocker_id, is_audit=is_audit, occurred_at=occurred_at or datetime.now(timezone.utc))
session.add(event)
return event
def purge_expired_events(session, now: datetime | None = None, retention_days: int | None = None) -> int:
cutoff = (now or datetime.now(timezone.utc)) - timedelta(days=retention_days if retention_days is not None else settings.agent_event_retention_days)
result = session.execute(delete(AgentEvent).where(AgentEvent.occurred_at < cutoff, AgentEvent.is_audit.is_(False), AgentEvent.severity != "CRITICAL"))
session.commit()
return result.rowcount or 0
def list_events(session, installation_db_id: int, *, page: int = 1, page_size: int = 50, severity: str | None = None, event_type: str | None = None, job_id: int | None = None, product_code: str | None = None, since: datetime | None = None, until: datetime | None = None, fetch_extra: bool = False):
page = max(1, page); page_size = min(100, max(1, page_size))
query = select(AgentEvent).where(AgentEvent.installation_id == installation_db_id)
for column, value in ((AgentEvent.severity, severity), (AgentEvent.event_type, event_type), (AgentEvent.job_id, job_id), (AgentEvent.product_code, product_code)):
if value is not None:
query = query.where(column == value)
if since: query = query.where(AgentEvent.occurred_at >= since)
if until: query = query.where(AgentEvent.occurred_at <= until)
query = query.order_by(AgentEvent.occurred_at.desc(), AgentEvent.id.desc())
rows = session.scalars(query.offset((page - 1) * page_size).limit(page_size + int(fetch_extra))).all()
return rows
@@ -0,0 +1,47 @@
"""Conservative aggregation of supplier/document observations."""
from dataclasses import dataclass
from collections import defaultdict
from typing import Any, Iterable
@dataclass(frozen=True)
class Observation:
product_id: str
attribute: str
value: str
supplier_id: str
document_id: str
observed_at: str
provenance: str = "SUPPLIER_OBSERVATION"
def aggregate_observations(observations: Iterable[Observation]) -> dict[tuple[str, str], dict[str, Any]]:
grouped: dict[tuple[str, str], list[Observation]] = defaultdict(list)
seen: set[tuple[str, str, str, str]] = set()
for observation in observations:
key = (observation.product_id, observation.attribute)
dedupe = (observation.product_id, observation.attribute, observation.supplier_id, observation.document_id)
if dedupe not in seen and observation.value not in (None, ""):
grouped[key].append(observation)
seen.add(dedupe)
output = {}
for key, rows in grouped.items():
values = {row.value for row in rows}
counts = {value: sum(row.value == value for row in rows) for value in values}
supplier_count = len({row.supplier_id for row in rows})
agreement_rate = max(counts.values()) / len(rows) if rows else 0.0
conflict_count = max(0, len(values) - 1)
strong = len(rows) >= 2 and supplier_count >= 2 and len(values) == 1
output[key] = {
"value": next(iter(values)) if strong else None,
"observation_count": len(rows),
"supplier_count": supplier_count,
"agreement_rate": agreement_rate,
"latest_observation": max(row.observed_at for row in rows),
"conflict_count": conflict_count,
"strong_evidence": strong,
"provenance": sorted({row.provenance for row in rows}),
}
return output
+636
View File
@@ -0,0 +1,636 @@
"""Fiscal Field Contract V1: one contract + one resolver for EVERY persistent fiscal field of the SGV product screen (C000025).
Pure functions. Every (product, field) ends in exactly one of:
ALREADY_CORRECT | CHANGE_REQUIRED | NOT_APPLICABLE | MANUAL_INPUT_REQUIRED(<input_key>) (CORRECTED is set by the writer after a verified write)
Nothing is invented: a value is proposed only when an official dataset, the company context, ERP evidence or an OPERATOR context entry proves it.
A valid ERP declaration that no proven evidence contradicts is ALREADY_CORRECT (reason DECLARED_VALID_NOT_CONTRADICTED); the report keeps
those apart from rule-proven results.
Operator context (Hub `fiscal_context_entries`) is hierarchical: product > product group (CODGRUPO) > company.
"""
from .fiscal_decision import _ex_described # noqa: F401 (description evidence helper shared with the decision engine)
CONTRACT_VERSION = "field-contract-v1"
ALREADY, CHANGE, CORRECTED, NA, MANUAL = "ALREADY_CORRECT", "CHANGE_REQUIRED", "CORRECTED", "NOT_APPLICABLE", "MANUAL_INPUT_REQUIRED"
WRITE_FIELDS = ["CLASSIFICACAO_FISCAL", "IPI", "EX_TIPI", "CEST", "SITUACAO_TRIBUTARIA", "CFOP", "CFOP_ENTRADA", "CST", "CST_ENTRADA", "CSOSN", "ALIQUOTA", "COD_ORIGEM_MERCADORIA",
"CST_IPI", "COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS", "INCIDENCIA_PISCOFINS", "PISCOFINS", "IAT", "IPPT"]
# ---- contract metadata (docs/FISCAL_FIELD_CONTRACT_V1.json is generated from this) ------------------------------------------------------------------
def _c(ui, sem, dtype, source, inputs, rule, valid, na, manual, validation, prov, write=True):
return {"UI_LABEL": ui, "SEMANTICS": sem, "DATA_TYPE": dtype, "SOURCE_OF_TRUTH": source, "REQUIRED_INPUTS": inputs, "RESOLUTION_RULE": rule, "VALID_VALUES": valid,
"NOT_APPLICABLE_RULE": na, "MANUAL_INPUT_RULE": manual, "WRITE_ALLOWED": write, "VALIDATION_RULE": validation, "PROVENANCE": prov}
CONTRACT = {
"CLASSIFICACAO_FISCAL": _c("NCM", "NCM canônico (8 dígitos)", "VARCHAR(20)", "dataset NCM (Siscomex) ACTIVE + legacy NCM table", ["current NCM"],
"VALID -> ALREADY_CORRECT. LEGACY/INVALID/MISSING never auto-replaced from the description; an OPERATOR context `ncm` that the Hub validates as EXACT_NCM becomes CHANGE_REQUIRED and every dependent field is reprocessed with it",
"8-digit code present in the ACTIVE NCM dataset", "never (every product has an NCM)", "MANUAL_INPUT_REQUIRED(ncm) for LEGACY/INVALID/MISSING or an invalid manual value", "Hub classify status == EXACT_NCM", "NCM dataset sha256 + engine version"),
"IPI": _c("Alíquota IPI %", "alíquota numérica de IPI", "DOUBLE PRECISION", "dataset TIPI_IPI ACTIVE", ["valid NCM", "TIPI row (and EX choice when several rows)"],
"TIPI numeric rate -> ALREADY_CORRECT if equal else CHANGE_REQUIRED. NT/non-numeric and blank rates never write (NOT_APPLICABLE, never zero)", "float >= 0", "TIPI rate is NT/non-numeric or blank", "MANUAL_INPUT_REQUIRED(ex_tipi) when several TIPI rows exist for the NCM", "numeric, from TIPI row", "TIPI dataset sha256"),
"EX_TIPI": _c("EX TIPI", "código EX da TIPI", "VARCHAR(3)", "dataset TIPI_IPI + product description evidence", ["valid NCM", "TIPI EX rows"],
"EX applies only when the product description carries the official EX description; otherwise the base rate applies (no EX)", "EX code of an existing TIPI EX row", "NCM has no EX row, or the description evidences none",
"MANUAL_INPUT_REQUIRED(ex_tipi) when the ERP holds an EX that no TIPI row supports", "value in TIPI EX rows for the NCM", "TIPI dataset sha256"),
"CEST": _c("CEST", "CEST (7 dígitos sem pontuação)", "VARCHAR(7)", "dataset CEST (CONFAZ 142/18) ACTIVE + BA_ST_MVA", ["valid NCM", "description"],
"single defensible official CEST -> ALREADY/CHANGE; ERP CEST that is one of the official codes of the NCM -> ALREADY_CORRECT; several official candidates with a blank CEST -> MANUAL; NCM absent from the CEST list -> NOT_APPLICABLE",
"official CEST code of the NCM", "NCM not in the CEST list, or the description matches no official CEST item", "MANUAL_INPUT_REQUIRED(cest) when several official candidates exist, or the ERP CEST is not official for the NCM", "digits equal to an official CEST of the NCM", "CEST dataset sha256"),
"SITUACAO_TRIBUTARIA": _c("Seletor ao lado do CEST", "T=Tributado, I=Isento, N=Não tributado, S=Substituição Tributária", "VARCHAR(2)", "ST engine (BA Anexo 1 + CEST corroboration) + purchase evidence / operator st_withheld", ["ST status", "ST withheld"],
"ST proven and withheld -> S. Otherwise a valid declaration stays", "T | I | N | S", "never", "MANUAL_INPUT_REQUIRED(st_withheld) when ST is proven but retention unknown; (icms_treatment) when the field is blank", "value in T/I/N/S", "BA_ST_MVA + CEST dataset sha256"),
"CFOP": _c("CFOP (Nota Fiscal de Saída)", "CFOP padrão de saída (intra-UF)", "VARCHAR(4)", "dataset CFOP oficial + product role + ST", ["product_role", "ST status", "ST withheld"],
"RESALE: non-ST 5102, ST withheld 5405; OWN_PRODUCTION: 5101 / 5401; a declared official 5xxx that is not contradicted by proven role/ST stays", "official CFOP starting with 5", "INPUT (not a sale item)", "MANUAL_INPUT_REQUIRED(product_role | st_status | st_withheld | service_cfop)", "code in CFOP dataset", "CFOP dataset sha256"),
"CFOP_ENTRADA": _c("CFOP (Nota Fiscal de Entrada)", "CFOP padrão de entrada", "CHAR(4)", "dataset CFOP + product role + ST + company entry scope", ["product_role", "ST withheld", "entry_scope (company)"],
"RESALE 1102/2102, ST withheld 1403/2403, OWN_PRODUCTION/INPUT 1101/2101 (ST 1401/2401); declared official 1xxx/2xxx that is not contradicted stays", "official CFOP starting with 1 or 2", "SERVICE", "MANUAL_INPUT_REQUIRED(product_role | entry_scope | st_withheld)", "code in CFOP dataset", "CFOP dataset sha256"),
"CST": _c("CST de ICMS (Nota Fiscal de Saída)", "CST ICMS = origem(1)+CST(2), regime normal", "VARCHAR(10)", "MOC CST_ICMS + ORIGEM tables + icms_treatment", ["CRT 2/3", "icms_treatment", "origin"],
"CRT 1 uses CSOSN. CRT 2/3: treatment TAXED 00 / EXEMPT 40 / NOT_TAXED 41 / ST_WITHHELD 60; a valid declaration that the treatment does not contradict stays", "3 digits: origin 0-8 + CST_ICMS code", "CRT = 1", "MANUAL_INPUT_REQUIRED(icms_treatment | origin) when blank and not derivable", "origin in ORIGEM table, last two digits in CST_ICMS", "MOC tables sha256"),
"CST_ENTRADA": _c("CST de ICMS (Nota Fiscal de Entrada)", "CST ICMS de entrada, regime normal", "CHAR(4)", "MOC tables + icms_treatment", ["CRT 2/3", "icms_treatment", "origin"],
"same derivation as CST for CRT 2/3", "3 digits origin+CST", "CRT = 1", "MANUAL_INPUT_REQUIRED(icms_treatment | origin)", "as CST", "MOC tables sha256"),
"CSOSN": _c("CSOSN Classificação", "CSOSN de saída (Simples Nacional)", "VARCHAR(60)", "MOC CSOSN table + icms_treatment (operator or proven ST)", ["CRT 1", "icms_treatment"],
"treatment TAXED -> 101/102/103 family (default 102), EXEMPT/NOT_TAXED -> 300/400 (default 400), ST_WITHHELD -> 500; a valid declaration outside a proven contradiction stays", "CSOSN code in MOC table", "CRT != 1", "MANUAL_INPUT_REQUIRED(icms_treatment) when blank/invalid", "code in CSOSN table", "MOC tables sha256"),
"ALIQUOTA": _c("Aliq. ICMS", "alíquota ICMS % de destaque", "NUMERIC(15,2)", "regime + CSOSN/CST family + operator icms_rate", ["CSOSN or CST family", "icms_rate (company/group) for taxed regime-normal items"],
"no ICMS highlighted (CSOSN 102/103/300/400/500/202/203/900 or CST 40/41/60/50/51) -> NOT_APPLICABLE; taxed items keep a declared value or take the operator icms_rate", "numeric >= 0", "no ICMS is highlighted for the item", "MANUAL_INPUT_REQUIRED(icms_rate | credit_rate)", "numeric", "operator context / company regime"),
"COD_ORIGEM_MERCADORIA": _c("Código da origem da Mercadoria (0/1/2)", "origem da mercadoria (0 Nacional, 1 Estrangeira-importação direta, 2 Estrangeira-mercado interno)", "INTEGER", "operator origin only (no ERP field carries origin evidence)", ["origin"],
"declared 0/1/2 stays; blank needs operator/evidence: the CST first digit is padding in the SGV (audit V2) and is NOT origin evidence; never defaulted to 0", "0 | 1 | 2", "never", "MANUAL_INPUT_REQUIRED(origin)", "value in {0,1,2}", "SGV DFM: hidden edit CodOrigemMercadoria + RgOrigemMercadoria.OnClick"),
"CST_IPI": _c("CST IPI", "CST do IPI", "CHAR(3)", "company C000004.CONTRIBUINTE_IPI", ["company IPI taxpayer flag"],
"company not an IPI taxpayer (CONTRIBUINTE_IPI != S) -> NOT_APPLICABLE; taxpayer -> declared value stays", "IPI CST code", "company is not an IPI taxpayer", "MANUAL_INPUT_REQUIRED(cst_ipi) for IPI taxpayers with a blank value", "non-blank code", "C000004.CONTRIBUINTE_IPI"),
"COD_ENTRADA_PIS": _c("Pis Entrada", "CST PIS de entrada", "VARCHAR(3)", "MOC CST_PIS + operator pis_cofins_profile", ["pis_cofins_profile"], "profile in_cst (validated against CST_PIS) -> ALREADY/CHANGE; else a valid declared code stays", "CST PIS code", "never", "MANUAL_INPUT_REQUIRED(pis_cofins_profile)", "code in CST_PIS table", "MOC tables sha256"),
"COD_SAIDA_PIS": _c("Pis Saída", "CST PIS de saída", "VARCHAR(3)", "MOC CST_PIS + operator pis_cofins_profile", ["pis_cofins_profile"], "profile out_cst -> ALREADY/CHANGE; else a valid declared code stays", "CST PIS code", "never", "MANUAL_INPUT_REQUIRED(pis_cofins_profile)", "code in CST_PIS table", "MOC tables sha256"),
"ALIQ_PIS": _c("Alíquota (Pis)", "alíquota PIS %", "VARCHAR(8)", "operator pis_cofins_profile", ["pis_cofins_profile"], "profile pis_rate -> ALREADY/CHANGE; else a declared value stays (never assumed 0)", "decimal text", "never", "MANUAL_INPUT_REQUIRED(pis_cofins_profile)", "numeric text", "operator context"),
"COD_ENTRADA_COFINS": _c("Cofins Entrada", "CST COFINS de entrada", "VARCHAR(3)", "MOC CST_COFINS + operator pis_cofins_profile", ["pis_cofins_profile"], "profile in_cst -> ALREADY/CHANGE; else a valid declared code stays", "CST COFINS code", "never", "MANUAL_INPUT_REQUIRED(pis_cofins_profile)", "code in CST_COFINS table", "MOC tables sha256"),
"COD_SAIDA_COFINS": _c("Cofins Saída", "CST COFINS de saída", "VARCHAR(3)", "MOC CST_COFINS + operator pis_cofins_profile", ["pis_cofins_profile"], "profile out_cst -> ALREADY/CHANGE; else a valid declared code stays", "CST COFINS code", "never", "MANUAL_INPUT_REQUIRED(pis_cofins_profile)", "code in CST_COFINS table", "MOC tables sha256"),
"ALIQ_COFINS": _c("Alíquota (Cofins)", "alíquota COFINS %", "VARCHAR(8)", "operator pis_cofins_profile", ["pis_cofins_profile"], "profile cofins_rate -> ALREADY/CHANGE; else a declared value stays (never assumed 0)", "decimal text", "never", "MANUAL_INPUT_REQUIRED(pis_cofins_profile)", "numeric text", "operator context"),
"INCIDENCIA_PISCOFINS": _c("Incidência", "tipo de incidência PIS/COFINS", "VARCHAR(30)", "ERP declaration (NENHUMA / MONOFÁSICA / SUBST.TRIBUTÁRIA) + operator incidence", ["incidence (group) when blank"], "valid declaration stays; operator `incidence` may set it", "NENHUMA | MONOFÁSICA | SUBST.TRIBUTÁRIA", "never", "MANUAL_INPUT_REQUIRED(incidence)", "value in domain", "ERP domain observed in 4 databases"),
"PISCOFINS": _c("PIS/COFINS (combo)", "flag PIS/COFINS N/S", "VARCHAR(1)", "ERP declaration + operator piscofins", ["piscofins when blank"], "valid N/S stays; operator `piscofins` may set it", "N | S", "never", "MANUAL_INPUT_REQUIRED(piscofins)", "value in N/S", "ERP domain observed in 4 databases"),
"IAT": _c("IAT", "indicador de arredondamento (A) ou truncamento (T)", "VARCHAR(1)", "company rounding policy (operator) or ERP declaration", ["rounding (company) when blank"], "valid A/T stays; blank -> operator `rounding`", "A | T", "never", "MANUAL_INPUT_REQUIRED(rounding)", "value in A/T", "PAF-ECF/Convênio registro de produto layout (IAT)"),
"IPPT": _c("IPPT", "indicador de produção própria (P) ou de terceiros (T)", "VARCHAR(1)", "product_role (own production evidence / purchase evidence / operator)", ["product_role"],
"OWN_PRODUCTION -> P, RESALE -> T; INPUT/SERVICE not applicable", "P | T", "INPUT or SERVICE", "MANUAL_INPUT_REQUIRED(product_role)", "value in P/T", "PAF-ECF/Convênio registro de produto layout (IPPT)"),
}
for _f, _v in CONTRACT.items():
_v["ERP_FIELD"] = _f
RTC_CAPABILITY = {"RTC_CCLASSTRIB": "ERP_SCHEMA_NOT_READY", "RTC_CCREDPRES": "ERP_SCHEMA_NOT_READY", "RTC_CBS_RATES": "ERP_SCHEMA_NOT_READY"}
ORIGIN_UI = {"0", "1", "2"}
CSOSN_FAMILY = {"TAXED": ("102", {"101", "102", "103"}), "EXEMPT": ("400", {"300", "400"}), "NOT_TAXED": ("400", {"300", "400"}), "ST_WITHHELD": ("500", {"500"}), "ST_CHARGED": ("202", {"201", "202", "203"})}
CST_TREATMENT = {"TAXED": "00", "EXEMPT": "40", "NOT_TAXED": "41", "ST_WITHHELD": "60", "SUSPENDED": "50", "DEFERRED": "51"}
NO_ICMS_CSOSN = {"102", "103", "300", "400", "500", "202", "203", "900"}
NO_ICMS_CST = {"40", "41", "60", "50", "51"}
RESALE_CFOP = {"5101", "5102", "5401", "5405"}
ENTRY_CFOP = {"1101", "1102", "1401", "1403", "2101", "2102", "2401", "2403"}
class Context:
"""Operator context lookup: product > group > company."""
def __init__(self, entries=()):
self._by = {}
for e in entries:
self._by[(e["scope"].upper(), str(e.get("scope_key") or ""), e["key"])] = e["value"]
def get(self, key, code=None, group=None, ncm=None):
for scope, sk in (("PRODUCT", code), ("GROUP", group), ("NCM", ncm), ("COMPANY", "")):
if scope != "COMPANY" and not sk:
continue
v = self._by.get((scope, str(sk or ""), key))
if v is not None and str(v).strip() != "":
return str(v).strip(), scope
return None, None
def _s(v):
return None if v is None else str(v).strip()
def _blank(v):
return v is None or str(v).strip() == ""
def _digits(v):
return "".join(c for c in str(v or "") if c.isdigit())
def _default_legal_basis(field, value):
contract = CONTRACT.get(field, {})
src = contract.get("SOURCE_OF_TRUTH", "")
prov = contract.get("PROVENANCE", "")
val_str = str(value or "")
if field == "CLASSIFICACAO_FISCAL":
return {"dataset": "NCM", "version": "Siscomex", "line": val_str, "act": "Resolução Gecex / MDIC"}
elif field in ("IPI", "EX_TIPI"):
return {"dataset": "TIPI_IPI", "version": "tipi-xlsx-v1", "line": val_str, "act": "Decreto nº 11.158/2022"}
elif field == "CEST":
return {"dataset": "CEST", "version": "CV142_18", "line": val_str, "act": "Convênio ICMS 142/18"}
elif "CFOP" in field:
return {"dataset": "CFOP", "version": "cfop-xlsx-v1", "line": val_str, "act": "Informe Técnico 2023.002"}
elif field in ("CST", "CST_ENTRADA", "CSOSN", "CST_IPI", "COD_ENTRADA_PIS", "COD_SAIDA_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "COD_ORIGEM_MERCADORIA"):
return {"dataset": "NFE_MOC_TABLES", "version": "moc7-tables-v1", "line": val_str, "act": "MOC 7.0 Anexo I"}
return {"dataset": "ERP_EVIDENCE", "version": "C000025", "line": val_str, "act": prov or src}
def _r(field, state, current, proposed=None, reason="", key=None, scope=None, legal_basis=None, **extra):
basis = legal_basis if legal_basis is not None else _default_legal_basis(field, proposed if proposed is not None else current)
return {"field": field, "state": state, "current": current, "proposed": proposed, "reason": reason, "input_key": key, "input_scope": scope,
"write_eligible": state == CHANGE and CONTRACT[field]["WRITE_ALLOWED"], "legal_basis": basis, **extra}
# C000032.TIPO=COMPOSICAO evidence must be a deterministic MAJORITY of that product's own movement
# history to count as own-production proof, never a bare "it happened at least once" check (a beer
# used once as a recipe component among 19 direct sales is not "own production"). Thresholds:
# below MINORITY -> composed evidence is discarded entirely (too rare to mean anything); above
# DOMINANT -> strong own-production evidence; in between (with enough events to be meaningful) ->
# genuinely conflicting evidence, never guessed either way (Missão 4 / Bloco A.1).
COMPOSED_MIN_TOTAL_EVENTS = 5
COMPOSED_MINORITY_RATIO = 0.30
COMPOSED_DOMINANT_RATIO = 0.70
def _composed_signal(ev, code):
"""None (no usable evidence), "DOMINANT", "MINORITY" or "MIXED", plus the ratio for the reason string."""
info = (ev.get("composed_ratio") or {}).get(code)
if not info:
return (None, None)
total = info.get("composed", 0) + info.get("other", 0)
if total < COMPOSED_MIN_TOTAL_EVENTS:
return (None, None)
ratio = info["composed"] / total
if ratio >= COMPOSED_DOMINANT_RATIO:
return ("DOMINANT", ratio)
if ratio <= COMPOSED_MINORITY_RATIO:
return ("MINORITY", ratio)
return ("MIXED", ratio)
def infer_role(p, ev, ctx, code, group, ncm=None):
v, scope = ctx.get("product_role", code, group, ncm)
if v in ("RESALE", "OWN_PRODUCTION", "INPUT", "SERVICE"):
return v, [f"OPERATOR({scope})"]
if _s(p.get("PRODUTO_SERVICO")):
return "SERVICE", ["C000025.PRODUTO_SERVICO"]
if (_s(p.get("TIPO")) or "").upper() == "MATERIA-PRIMA":
return "INPUT", ["C000025.TIPO=MATERIA-PRIMA"]
composed_signal, composed_ratio = _composed_signal(ev, code)
if composed_signal == "MIXED":
return "UNKNOWN", [f"C000032.TIPO=COMPOSICAO_MIXED_EVIDENCE(ratio={composed_ratio:.1%})"]
composed_dominant = composed_signal == "DOMINANT"
own = [n for n, on in (("PROD_COZINHA=1", p.get("PROD_COZINHA") == 1), ("RECEITA_INGREDIENTES", code in ev.get("recipes", ())),
("C000032.TIPO=COMPOSICAO", composed_dominant)) if on]
docs = bool(ev.get("purchases", {}).get(code)) or ev.get("stock_entry", {}).get(code, False)
stock = ev.get("stock_in", {}).get(code, False)
if own and docs:
return "UNKNOWN", own + ["ALSO_PURCHASE_DOCUMENTS_CONFLICT"]
strong_own = [n for n in own if n != "PROD_COZINHA=1"]
if strong_own:
return "OWN_PRODUCTION", own
if own: # audit V2: the kitchen-routing flag alone is not proof of own production
return "UNKNOWN", own + ["PROD_COZINHA_ALONE_NOT_PRODUCTION_PROOF"]
if (_s(p.get("TIPO")) or "").upper() == "VENDA" and (docs or stock):
return "RESALE", ["C000025.TIPO=VENDA", "PURCHASE_DOCUMENT" if docs else "STOCK_ENTRIES(C000100.ENTRADAS>0)"]
return "UNKNOWN", ["NO_OBJECTIVE_ROLE_EVIDENCE"]
def _isnum(v):
try:
float(v)
return True
except (TypeError, ValueError):
return False
_SPED_CACHE = {}
def sped_pis_cofins_match(ncm):
"""Active SPED EFD-Contribuicoes table 4.3.10/12/13/14/15 entry (explicit NCM prefix) covering the NCM -> list of (table, saida_cst, code, start). Empty when none."""
if "map" not in _SPED_CACHE:
import json as _j
from pathlib import Path as _P
m = {}
f = _P(__file__).resolve().parents[2] / "docs" / "sources" / "SPED_PISCOFINS_TABLES_V3.json"
if f.exists():
for tab, t in _j.loads(f.read_text(encoding="utf-8"))["tables"].items():
for e in t.get("entries", []):
if e["end"] is None:
for pref in e["ncm_prefixes"]:
m.setdefault(pref, set()).add((tab, t["saida_cst"][-1], e["code"], e["start"]))
_SPED_CACHE["map"] = m
hits = set()
for n in (4, 6, 8):
hits |= _SPED_CACHE["map"].get(ncm[:n], set()) if len(ncm) >= n else set()
return sorted(hits)
def resolve_product(p, d, company, refs, ev, ctx, manual_ncm=None):
"""p: product row (all contract columns + CODIGO, PRODUTO, CODGRUPO, TIPO, PROD_COZINHA, PRODUTO_SERVICO). d: Hub decision for the EFFECTIVE NCM
(the manual NCM when one was supplied and validated). manual_ncm: (value, is_valid) or None. Returns {field: result}."""
from .fiscal_decision import decide_product
from .icms_context import st_status, withheld_status
code, group = _s(p["CODIGO"]), _s(p.get("CODGRUPO"))
NCM = _digits(p.get("CLASSIFICACAO_FISCAL"))
crt, uf = company.get("crt"), company.get("uf")
out = {}
cur = {f: _s(p.get(f)) for f in WRITE_FIELDS}
# ---------------- NCM ----------------
ncm_ok = d.get("status") == "EXACT_NCM"
if manual_ncm is not None:
val, valid = manual_ncm
if not valid:
out["CLASSIFICACAO_FISCAL"] = _r("CLASSIFICACAO_FISCAL", MANUAL, cur["CLASSIFICACAO_FISCAL"], reason="MANUAL_NCM_NOT_VALID_IN_NCM_DATASET", key="ncm", scope="PRODUCT")
elif _digits(val) == _digits(cur["CLASSIFICACAO_FISCAL"]) and d.get("status") == "EXACT_NCM":
out["CLASSIFICACAO_FISCAL"] = _r("CLASSIFICACAO_FISCAL", ALREADY, cur["CLASSIFICACAO_FISCAL"], reason="OPERATOR_NCM_EQUALS_CURRENT")
else:
out["CLASSIFICACAO_FISCAL"] = _r("CLASSIFICACAO_FISCAL", CHANGE, cur["CLASSIFICACAO_FISCAL"], _digits(val), reason="OPERATOR_NCM_VALIDATED_BY_NCM_DATASET")
elif ncm_ok:
out["CLASSIFICACAO_FISCAL"] = _r("CLASSIFICACAO_FISCAL", ALREADY, cur["CLASSIFICACAO_FISCAL"], reason="NCM_VALID_IN_DATASET",
legal_basis={"dataset": "NCM", "version": refs.get("ncm_version") or "Siscomex", "line": cur["CLASSIFICACAO_FISCAL"], "act": "Resolução Gecex / MDIC"})
else:
st_val = d.get("status")
if _blank(cur["CLASSIFICACAO_FISCAL"]):
why = "NCM_MISSING"
elif st_val == "PLACEHOLDER_NCM":
why = "NCM_PLACEHOLDER"
elif st_val == "INVALID_FORMAT_NCM":
why = "NCM_INVALID_FORMAT"
elif st_val in ("LEGACY_NCM", "EXPIRED_LEGACY_NCM"):
why = "NCM_LEGACY_NO_UNAMBIGUOUS_SUCCESSOR"
elif st_val == "OUT_OF_TABLE_NCM":
why = "NCM_OUT_OF_OFFICIAL_TABLE"
else:
why = "NCM_INVALID"
out["CLASSIFICACAO_FISCAL"] = _r("CLASSIFICACAO_FISCAL", MANUAL, cur["CLASSIFICACAO_FISCAL"], reason=why, key="ncm", scope="PRODUCT",
legal_basis={"dataset": "NCM", "version": refs.get("ncm_version") or "Siscomex", "line": cur["CLASSIFICACAO_FISCAL"], "act": "Resolução Gecex / MDIC"})
def blocked(field):
return _r(field, MANUAL, cur[field], reason="BLOCKED_BY_NCM", key="ncm", scope="PRODUCT")
eff_ncm = out["CLASSIFICACAO_FISCAL"]["proposed"] or cur["CLASSIFICACAO_FISCAL"]
prod = {**p, "CLASSIFICACAO_FISCAL": eff_ncm}
# ---------------- context facts ----------------
role, role_ev = infer_role(p, ev, ctx, code, group, NCM)
st_ctx, st_scope = ctx.get("st_status", code, group)
ba = (d.get("classification") or {}).get("ba_st_mva") or {}
if st_ctx in ("SUBJECT", "NOT_SUBJECT"):
st = st_ctx
elif ncm_ok and uf == "BA":
s, _ = st_status(d.get("status"), d.get("ba_st_mva_status"), d.get("cest_status"), cur["CEST"], ba.get("cest"), None)
st = {"ST_CONFIRMED": "SUBJECT", "ST_NOT_APPLICABLE": "NOT_SUBJECT"}.get(s, "UNKNOWN")
else:
st = "UNKNOWN"
wh_ctx, _ = ctx.get("st_withheld", code, group)
wh_doc, _ = withheld_status(ev.get("purchases", {}).get(code, []))
withheld = wh_ctx if wh_ctx in ("Y", "N") else ("Y" if wh_doc == "ST_ALREADY_WITHHELD_CONFIRMED" else None) # purchase items flagged N are weak evidence: kept as unknown
treat, _ = ctx.get("icms_treatment", code, group)
if not treat and st == "SUBJECT" and withheld == "Y":
treat = "ST_WITHHELD"
origin_ctx, _ = ctx.get("origin", code, group)
# ---------------- decision-engine backed fields ----------------
if ncm_ok:
rec = decide_product({**prod, "CODIGO": code, "IPI": p.get("IPI"), "EX_TIPI": p.get("EX_TIPI"), "CEST": p.get("CEST")}, d, {"uf": uf or "", "crt": crt or ""}, refs)
ipi_field = "IPI"
if not ncm_ok:
out["IPI"], out["EX_TIPI"], out["CEST"] = blocked("IPI"), blocked("EX_TIPI"), blocked("CEST")
else:
# IPI
ex_ctx, _ = ctx.get("ex_tipi", code, group, NCM)
cands = (d.get("classification") or {}).get("ipi_candidates") or []
chosen = None
if ex_ctx:
chosen = next((c for c in cands if (c.get("ex") is None if ex_ctx.upper() == "NONE" else str(c.get("ex")) == ex_ctx)), None)
if chosen and chosen.get("rate") not in (None, ""):
try:
rate = float(chosen["rate"])
except ValueError:
rate = None
if rate is not None:
same = cur["IPI"] is not None and abs(float(cur["IPI"]) - rate) < 1e-9
out["IPI"] = _r("IPI", ALREADY if same else CHANGE, cur["IPI"], None if same else rate, "OPERATOR_EX_SELECTED_TIPI_ROW")
if "IPI" not in out:
r = rec["IPI"]
if r["status"] == "RULE_RESOLVED":
out["IPI"] = _r("IPI", ALREADY if r["reason_code"].endswith("SAME_VALUE") else CHANGE, cur["IPI"], None if r["reason_code"].endswith("SAME_VALUE") else r["proposed_value"], "TIPI_NUMERIC_RATE")
elif r["status"] == "RULE_CANDIDATE":
out["IPI"] = _r("IPI", MANUAL, cur["IPI"], reason="SEVERAL_TIPI_ROWS_FOR_NCM", key="ex_tipi", scope="PRODUCT",
choices=[{"ex": c.get("ex"), "rate": c.get("rate"), "description": c.get("description")} for c in cands])
else:
out["IPI"] = _r("IPI", NA, cur["IPI"], reason=("TIPI_RATE_BLANK_NEVER_ZERO" if "BLANK" in r["reason_code"] or "NO_DATA" in r["reason_code"] else "TIPI_NT_OR_NON_NUMERIC_NEVER_WRITTEN_TO_NUMERIC_FIELD"))
# store_ipi_profile / contribuinte_ipi gate
is_taxpayer = company.get("contribuinte_ipi") == "S"
prof, _ = ctx.get("store_ipi_profile", code, group)
if not is_taxpayer:
if prof == "POPULATE_REFERENCE_RATE":
pass
elif prof == "LEAVE_NOT_APPLICABLE":
out["IPI"] = _r("IPI", NA, cur["IPI"], reason="STORE_IPI_PROFILE_LEAVE_NOT_APPLICABLE",
legal_basis={"dataset": "TIPI_IPI", "version": "N/A", "line": "N/A", "act": "RIPI / CF art. 153, IV (não contribuinte)"})
elif prof is None and out["IPI"]["state"] == CHANGE:
out["IPI"] = _r("IPI", MANUAL, cur["IPI"], reason="STORE_IPI_PROFILE_NOT_DECIDED", key="store_ipi_profile", scope="COMPANY")
else:
out["IPI"] = _r("IPI", NA, cur["IPI"], reason="COMPANY_IS_NOT_AN_IPI_TAXPAYER_C000004.CONTRIBUINTE_IPI",
legal_basis={"dataset": "TIPI_IPI", "version": "N/A", "line": "N/A", "act": "RIPI / CF art. 153, IV (não contribuinte)"})
elif out["IPI"]["state"] == CHANGE:
if prof == "LEAVE_NOT_APPLICABLE":
out["IPI"] = _r("IPI", NA, cur["IPI"], reason="STORE_IPI_PROFILE_LEAVE_NOT_APPLICABLE")
# EX_TIPI
ex_rows = [c for c in cands if c.get("ex")]
if ex_ctx:
if ex_ctx.upper() == "NONE":
out["EX_TIPI"] = _r("EX_TIPI", NA, cur["EX_TIPI"], reason="OPERATOR_SAYS_NO_EX")
elif any(str(c["ex"]) == ex_ctx for c in ex_rows):
out["EX_TIPI"] = _r("EX_TIPI", ALREADY if cur["EX_TIPI"] == ex_ctx else CHANGE, cur["EX_TIPI"], None if cur["EX_TIPI"] == ex_ctx else ex_ctx, "OPERATOR_EX_VALID_IN_TIPI")
else:
out["EX_TIPI"] = _r("EX_TIPI", MANUAL, cur["EX_TIPI"], reason="OPERATOR_EX_NOT_IN_TIPI_ROWS", key="ex_tipi", scope="PRODUCT")
else:
r = rec["EX_TIPI"]
if r["status"] == "RULE_RESOLVED":
out["EX_TIPI"] = _r("EX_TIPI", ALREADY if r["reason_code"].endswith("SAME_VALUE") else CHANGE, cur["EX_TIPI"], None if r["reason_code"].endswith("SAME_VALUE") else r["proposed_value"], "TIPI_EX_DESCRIPTION_EVIDENCE")
elif cur["EX_TIPI"] and not any(str(c["ex"]) == cur["EX_TIPI"] for c in ex_rows):
out["EX_TIPI"] = _r("EX_TIPI", MANUAL, cur["EX_TIPI"], reason="ERP_EX_NOT_SUPPORTED_BY_TIPI", key="ex_tipi", scope="PRODUCT")
elif cur["EX_TIPI"]:
out["EX_TIPI"] = _r("EX_TIPI", ALREADY, cur["EX_TIPI"], reason="DECLARED_VALID_NOT_CONTRADICTED")
else:
out["EX_TIPI"] = _r("EX_TIPI", NA, cur["EX_TIPI"], reason="NO_EX_ROW_FOR_NCM" if not ex_rows else "DESCRIPTION_EVIDENCES_NO_EX_BASE_RATE_APPLIES")
# CEST
cest_ctx, _ = ctx.get("cest", code, group, NCM)
cls = d.get("classification") or {}
official = {_digits(c["code"]): c for c in (cls.get("cest_candidates") or [])}
cs = d.get("cest_status")
cur_cest = _digits(cur["CEST"])
if cest_ctx:
if cest_ctx.upper() == "NONE":
out["CEST"] = _r("CEST", NA, cur["CEST"], reason="OPERATOR_SAYS_NO_CEST")
elif _digits(cest_ctx) in official:
out["CEST"] = _r("CEST", ALREADY if cur_cest == _digits(cest_ctx) else CHANGE, cur["CEST"], None if cur_cest == _digits(cest_ctx) else _digits(cest_ctx), "OPERATOR_CEST_OFFICIAL_FOR_NCM")
else:
out["CEST"] = _r("CEST", MANUAL, cur["CEST"], reason="OPERATOR_CEST_NOT_OFFICIAL_FOR_NCM", key="cest", scope="PRODUCT")
elif cs == "CEST_CONFIRMED":
cests_mapped = cls.get("cest") or []
if len(cests_mapped) <= 1:
proposed = _digits(cests_mapped[0] if cests_mapped else (cls.get("cest") or [None])[0])
out["CEST"] = _r("CEST", ALREADY if cur_cest == proposed else CHANGE, cur["CEST"], None if cur_cest == proposed else proposed, "SINGLE_DEFENSIBLE_OFFICIAL_CEST",
legal_basis={"dataset": "CEST", "version": refs.get("cest", "CV142_18"), "line": proposed, "act": "Convênio ICMS 142/18"})
else:
out["CEST"] = _r("CEST", MANUAL, cur["CEST"], reason="SEVERAL_OFFICIAL_CEST_FOR_NCM", key="cest", scope="PRODUCT",
choices=[{"code": v["code"], "description": v["official_description"], "score": v.get("description_score")} for v in official.values()],
legal_basis={"dataset": "CEST", "version": refs.get("cest", "CV142_18"), "line": "MULTIPLE_CANDIDATES", "act": "Convênio ICMS 142/18"})
elif cur_cest and cur_cest in official:
out["CEST"] = _r("CEST", ALREADY, cur["CEST"], reason="ERP_CEST_IS_OFFICIAL_FOR_NCM")
elif cs == "NO_CEST_MAPPING_FOUND":
out["CEST"] = _r("CEST", NA, cur["CEST"], reason="NCM_NOT_IN_CEST_LIST") if not cur_cest else _r("CEST", MANUAL, cur["CEST"], reason="ERP_CEST_BUT_NCM_NOT_IN_CEST_LIST", key="cest", scope="PRODUCT")
elif cs == "CEST_CANDIDATES" and not cur_cest:
out["CEST"] = _r("CEST", MANUAL, cur["CEST"], reason="SEVERAL_OFFICIAL_CEST_FOR_NCM", key="cest", scope="PRODUCT",
choices=[{"code": v["code"], "description": v["official_description"], "score": v.get("description_score")} for v in official.values()])
elif cs == "CEST_DESCRIPTION_MISMATCH" and not cur_cest:
out["CEST"] = _r("CEST", NA, cur["CEST"], reason="DESCRIPTION_MATCHES_NO_OFFICIAL_CEST_ITEM")
elif cur_cest:
out["CEST"] = _r("CEST", MANUAL, cur["CEST"], reason="ERP_CEST_NOT_OFFICIAL_FOR_NCM", key="cest", scope="PRODUCT")
else:
out["CEST"] = _r("CEST", NA, cur["CEST"], reason="NO_CEST_APPLICABLE")
# ---------------- SITUACAO_TRIBUTARIA ----------------
sit = cur["SITUACAO_TRIBUTARIA"]
valid_sit = sit in ("T", "I", "N", "S")
if not ncm_ok:
out["SITUACAO_TRIBUTARIA"] = blocked("SITUACAO_TRIBUTARIA")
elif st == "SUBJECT" and withheld == "Y":
out["SITUACAO_TRIBUTARIA"] = _r("SITUACAO_TRIBUTARIA", ALREADY if sit == "S" else CHANGE, sit, None if sit == "S" else "S", "ST_PROVEN_AND_WITHHELD")
elif st == "SUBJECT" and withheld is None:
out["SITUACAO_TRIBUTARIA"] = _r("SITUACAO_TRIBUTARIA", ALREADY, sit, reason="DECLARED_S_CONSISTENT_WITH_PROVEN_ST") if sit == "S" else _r("SITUACAO_TRIBUTARIA", MANUAL, sit, reason="ST_PROVEN_RETENTION_UNKNOWN", key="st_withheld", scope="GROUP")
elif valid_sit and not (st == "NOT_SUBJECT" and sit == "S"):
out["SITUACAO_TRIBUTARIA"] = _r("SITUACAO_TRIBUTARIA", ALREADY, sit, reason="DECLARED_VALID_NOT_CONTRADICTED")
elif valid_sit:
out["SITUACAO_TRIBUTARIA"] = _r("SITUACAO_TRIBUTARIA", MANUAL, sit, reason="DECLARED_S_BUT_NOT_IN_ANY_ST_LIST", key="icms_treatment", scope="GROUP")
else:
out["SITUACAO_TRIBUTARIA"] = _r("SITUACAO_TRIBUTARIA", MANUAL, sit, reason="BLANK_OR_INVALID", key="icms_treatment", scope="GROUP")
# ---------------- CFOP saída ----------------
cfop, cfop_valid = cur["CFOP"], refs["cfop_valid"]
if role == "INPUT":
out["CFOP"] = _r("CFOP", NA, cfop, reason="INPUT_IS_NOT_A_SALE_ITEM")
elif role == "SERVICE":
out["CFOP"] = _r("CFOP", MANUAL, cfop, reason="SERVICE_CFOP_NEEDS_OPERATOR", key="service_cfop", scope="GROUP")
elif not ncm_ok:
out["CFOP"] = blocked("CFOP")
else:
expected = None
if role in ("RESALE", "OWN_PRODUCTION"):
base = "5102" if role == "RESALE" else "5101"
if st == "NOT_SUBJECT":
expected = base
elif st == "SUBJECT" and withheld == "Y":
expected = "5405" if role == "RESALE" else "5401"
valid_cur = cfop in cfop_valid and cfop.startswith("5")
erp_st_declared = cur["CSOSN"] in ("500", "201", "202", "203") or (cur["CST"] or "")[-2:] in ("60", "10", "70", "30")
if valid_cur and expected and expected in ("5102", "5101") and cfop in ("5405", "5401") and erp_st_declared:
out["CFOP"] = _r("CFOP", MANUAL, cfop, reason="OFFICIAL_ST_TABLE_CONFLICTS_WITH_ERP_ST_DECLARATION", key="st_status", scope="GROUP")
elif valid_cur and expected and cfop != expected and cfop in RESALE_CFOP:
out["CFOP"] = _r("CFOP", CHANGE, cfop, expected, "PROVEN_ROLE_AND_ST_CONTRADICT_DECLARED_CFOP")
elif valid_cur and expected == cfop:
out["CFOP"] = _r("CFOP", ALREADY, cfop, reason="RULE_PROVEN_EQUALS_DECLARED")
elif valid_cur:
out["CFOP"] = _r("CFOP", ALREADY, cfop, reason="DECLARED_VALID_NOT_CONTRADICTED")
elif expected:
out["CFOP"] = _r("CFOP", CHANGE, cfop, expected, "DERIVED_FROM_ROLE_AND_ST" + ("_REPLACING_INVALID" if cfop else "_FILLING_BLANK"))
else:
key = "product_role" if role == "UNKNOWN" else ("st_status" if st == "UNKNOWN" else "st_withheld")
out["CFOP"] = _r("CFOP", MANUAL, cfop, reason="CFOP_NEEDS_" + key.upper(), key=key, scope="GROUP")
# ---------------- CFOP entrada ----------------
cfe = cur["CFOP_ENTRADA"]
scope_ctx, _ = ctx.get("entry_scope", code, group)
if role == "SERVICE":
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", NA, cfe, reason="SERVICE_HAS_NO_GOODS_ENTRY")
elif not ncm_ok:
out["CFOP_ENTRADA"] = blocked("CFOP_ENTRADA")
else:
exp_e = None
if scope_ctx == "MIXED_OR_DYNAMIC":
# Scope answered, but no single static entry CFOP is fiscally correct for every
# operation: the ERP default / each operation decides 1xxx vs 2xxx. Never invent one.
valid_e = cfe in cfop_valid and cfe[:1] in ("1", "2")
if valid_e:
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", ALREADY, cfe, reason="DECLARED_VALID_OPERATION_DEPENDENT_MIXED_SCOPE")
else:
# Audited ADMIN decision (entry_scope=MIXED_OR_DYNAMIC) already says no static CFOP_ENTRADA is
# ever correct here: NA (never MANUAL) so this doesn't block product completeness and never
# raises a spurious "answer entry_scope" question — the scope question was already answered.
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", NA, cfe, reason="CFOP_ENTRADA_OPERATION_DEPENDENT_MIXED_SCOPE")
else:
if scope_ctx in ("INTRA", "INTER") and role in ("RESALE", "OWN_PRODUCTION", "INPUT") and st != "UNKNOWN" and not (st == "SUBJECT" and withheld is None):
first = "1" if scope_ctx == "INTRA" else "2"
stw = st == "SUBJECT" and withheld == "Y"
exp_e = first + ("403" if stw else "102") if role == "RESALE" else first + ("401" if stw else "101")
valid_e = cfe in cfop_valid and cfe[:1] in ("1", "2")
if valid_e and exp_e and cfe != exp_e and cfe in ENTRY_CFOP:
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", CHANGE, cfe, exp_e, "PROVEN_ROLE_ST_AND_SCOPE_CONTRADICT_DECLARED_CFOP_ENTRADA")
elif valid_e:
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", ALREADY, cfe, reason="DECLARED_VALID_NOT_CONTRADICTED")
elif exp_e:
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", CHANGE, cfe, exp_e, "DERIVED_FROM_ROLE_ST_AND_ENTRY_SCOPE")
else:
key = "product_role" if role == "UNKNOWN" else ("entry_scope" if scope_ctx not in ("INTRA", "INTER") else ("st_status" if st == "UNKNOWN" else "st_withheld"))
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", MANUAL, cfe, reason="CFOP_ENTRADA_NEEDS_" + key.upper(), key=key, scope="COMPANY" if key == "entry_scope" else "GROUP")
# ---------------- origin (needed by CST) ----------------
cod_o = cur["COD_ORIGEM_MERCADORIA"]
cst_digit = (cur["CST"] or "")[:1] if cur["CST"] and len(cur["CST"]) == 3 and (cur["CST"][:1] in "012345678") else None
if origin_ctx in ORIGIN_UI:
out["COD_ORIGEM_MERCADORIA"] = _r("COD_ORIGEM_MERCADORIA", ALREADY if cod_o == origin_ctx else CHANGE, cod_o, None if cod_o == origin_ctx else int(origin_ctx), "OPERATOR_ORIGIN")
elif origin_ctx:
out["COD_ORIGEM_MERCADORIA"] = _r("COD_ORIGEM_MERCADORIA", MANUAL, cod_o, reason="OPERATOR_ORIGIN_NOT_REPRESENTABLE_ON_SGV_SCREEN_0_1_2", key="origin", scope="GROUP")
elif cod_o in ORIGIN_UI:
out["COD_ORIGEM_MERCADORIA"] = _r("COD_ORIGEM_MERCADORIA", ALREADY, cod_o, reason="DECLARED_VALID_NOT_CONTRADICTED")
else:
# V2 audit: the CST first digit is padding in the SGV (declared-origin 1/2 products also carry CST digit 0), so it is NOT evidence of origin
out["COD_ORIGEM_MERCADORIA"] = _r("COD_ORIGEM_MERCADORIA", MANUAL, cod_o, reason="ORIGIN_NO_EVIDENCE", key="origin", scope="GROUP")
origin_digit = out["COD_ORIGEM_MERCADORIA"]["proposed"] if out["COD_ORIGEM_MERCADORIA"]["proposed"] is not None else (cod_o if cod_o in ORIGIN_UI else None)
origin_digit = None if origin_digit is None else str(origin_digit)
# ---------------- CST / CST_ENTRADA (regime normal) ----------------
for field in ("CST", "CST_ENTRADA"):
v = cur[field]
if crt == "1":
out[field] = _r(field, NA, v, reason="CRT1_SIMPLES_EMISSION_USES_CSOSN")
continue
if crt not in ("2", "3"):
out[field] = _r(field, MANUAL, v, reason="CRT_UNKNOWN", key="crt", scope="COMPANY")
continue
valid_c = bool(v) and len(v) == 3 and v[0] in "012345678" and v[1:] in refs["cst_icms"]
exp = (origin_digit + CST_TREATMENT[treat]) if (treat in CST_TREATMENT and origin_digit) else None
if valid_c and exp and v != exp and v[1:] != exp[1:]:
out[field] = _r(field, CHANGE, v, exp, "TREATMENT_CONTRADICTS_DECLARED_CST")
elif valid_c:
out[field] = _r(field, ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
elif exp:
out[field] = _r(field, CHANGE, v, exp, "DERIVED_FROM_TREATMENT_AND_ORIGIN")
else:
out[field] = _r(field, MANUAL, v, reason="CST_NEEDS_ICMS_TREATMENT" if origin_digit else "CST_NEEDS_ORIGIN", key="icms_treatment" if origin_digit else "origin", scope="GROUP")
# ---------------- CSOSN ----------------
v = cur["CSOSN"]
if crt != "1":
out["CSOSN"] = _r("CSOSN", NA, v, reason="CSOSN_IS_THE_SIMPLES_NACIONAL_CODE_CRT_IS_" + str(crt))
elif not ncm_ok:
out["CSOSN"] = blocked("CSOSN")
else:
fam = CSOSN_FAMILY.get(treat)
if v in refs["csosn"] and (not fam or v in fam[1]):
out["CSOSN"] = _r("CSOSN", ALREADY, v, reason="RULE_PROVEN_EQUALS_DECLARED" if fam else "DECLARED_VALID_NOT_CONTRADICTED")
elif v in refs["csosn"] and fam:
out["CSOSN"] = _r("CSOSN", CHANGE, v, fam[0], "TREATMENT_CONTRADICTS_DECLARED_CSOSN")
elif fam:
out["CSOSN"] = _r("CSOSN", CHANGE, v, fam[0], "DERIVED_FROM_TREATMENT" + ("_REPLACING_INVALID" if v else "_FILLING_BLANK"))
else:
out["CSOSN"] = _r("CSOSN", MANUAL, v, reason="CSOSN_NEEDS_ICMS_TREATMENT", key="icms_treatment", scope="GROUP")
# ---------------- ALIQUOTA ----------------
v = cur["ALIQUOTA"]
csosn_now = out["CSOSN"]["proposed"] or cur["CSOSN"]
cst_now = out["CST"]["proposed"] or cur["CST"]
rate_ctx, _ = ctx.get("icms_rate", code, group)
if crt == "1" and (csosn_now in NO_ICMS_CSOSN):
out["ALIQUOTA"] = _r("ALIQUOTA", NA, v, reason="SIMPLES_CSOSN_HIGHLIGHTS_NO_ICMS_RATE")
elif crt in ("2", "3") and cst_now and cst_now[1:] in NO_ICMS_CST:
out["ALIQUOTA"] = _r("ALIQUOTA", NA, v, reason="CST_HIGHLIGHTS_NO_ICMS_RATE")
elif rate_ctx:
try:
r = float(rate_ctx.replace(",", "."))
same = v is not None and abs(float(v) - r) < 1e-9
out["ALIQUOTA"] = _r("ALIQUOTA", ALREADY if same else CHANGE, v, None if same else r, "OPERATOR_ICMS_RATE")
except ValueError:
out["ALIQUOTA"] = _r("ALIQUOTA", MANUAL, v, reason="OPERATOR_ICMS_RATE_NOT_NUMERIC", key="icms_rate", scope="COMPANY")
elif v not in (None, ""):
out["ALIQUOTA"] = _r("ALIQUOTA", ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
else:
out["ALIQUOTA"] = _r("ALIQUOTA", MANUAL, v, reason="TAXED_ITEM_WITHOUT_ICMS_RATE", key="credit_rate" if crt == "1" else "icms_rate", scope="COMPANY")
# ---------------- CST IPI ----------------
v = cur["CST_IPI"]
if company.get("contribuinte_ipi") != "S":
out["CST_IPI"] = _r("CST_IPI", NA, v, reason="COMPANY_IS_NOT_AN_IPI_TAXPAYER_C000004.CONTRIBUINTE_IPI")
elif v:
out["CST_IPI"] = _r("CST_IPI", ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
else:
out["CST_IPI"] = _r("CST_IPI", MANUAL, v, reason="IPI_TAXPAYER_WITHOUT_CST_IPI", key="cst_ipi", scope="GROUP")
# ---------------- PIS / COFINS ----------------
prof_raw, prof_scope = ctx.get("pis_cofins_profile", code, group)
PIS_COFINS_FIELDS = ("COD_ENTRADA_PIS", "pis_in_cst", "cst_pis"), ("COD_SAIDA_PIS", "pis_out_cst", "cst_pis"), ("ALIQ_PIS", "pis_rate", None), \
("COD_ENTRADA_COFINS", "cofins_in_cst", "cst_cofins"), ("COD_SAIDA_COFINS", "cofins_out_cst", "cst_cofins"), ("ALIQ_COFINS", "cofins_rate", None)
if prof_raw in ("NOT_APPLICABLE", "OPERATION_TIME"):
# Audited ADMIN decision only (Missão 2 / Bloco 2.4): an operator explicitly recorded, in FiscalContextEntry,
# that this company/product's PIS/COFINS is never taken from the cadastral fields (real evidence: SGV emission
# never reads them) or varies per operation. Never inferred automatically from emission statistics -
# absent this exact recorded value, the field stays MANUAL (see the final `else` below), unchanged.
for field, _pkey, _dom in PIS_COFINS_FIELDS:
out[field] = _r(field, NA, cur[field], reason="ADMIN_DECISION_PIS_COFINS_" + prof_raw)
prof = None
else:
import json as _json
try:
prof = _json.loads(prof_raw) if prof_raw else {}
except ValueError:
prof = {}
for field, pkey, dom in ([] if prof is None else PIS_COFINS_FIELDS):
v = cur[field]
want = _s(prof.get(pkey)) if prof else None
if want is not None and dom and want not in refs[dom]:
out[field] = _r(field, MANUAL, v, reason="PROFILE_VALUE_NOT_IN_OFFICIAL_CST_TABLE", key="pis_cofins_profile", scope="COMPANY")
elif want is not None:
out[field] = _r(field, ALREADY if v == want else CHANGE, v, None if v == want else want, "OPERATOR_PIS_COFINS_PROFILE")
elif v and (dom is None or v in refs[dom]):
out[field] = _r(field, ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
elif v:
out[field] = _r(field, MANUAL, v, reason="DECLARED_CST_NOT_IN_OFFICIAL_TABLE", key="pis_cofins_profile", scope="COMPANY")
elif crt == "1" and company.get("pis_cofins_unused_by_emission"):
hits = sped_pis_cofins_match(NCM) if ncm_ok else []
csts = {h[1] for h in hits}
if field in ("COD_SAIDA_PIS", "COD_SAIDA_COFINS") and len(csts) == 1 and role == "RESALE":
want = next(iter(csts))
out[field] = _r(field, CHANGE, v, want, "SPED_TABLE_%s_ENTRY_%s_NCM_%s" % (hits[0][0], hits[0][2], NCM))
elif field in ("COD_SAIDA_PIS", "COD_SAIDA_COFINS") and len(csts) > 1:
out[field] = _r(field, MANUAL, v, reason="SPED_TABLES_CONFLICT_FOR_NCM", key="pis_cofins_profile", scope="PRODUCT")
elif field in ("ALIQ_PIS", "ALIQ_COFINS") and len(csts) == 1 and role == "RESALE":
out[field] = _r(field, NA, v, reason="RATE_NOT_INFORMED_FOR_CST_" + next(iter(csts)))
else:
out[field] = _r(field, NA, v, reason="SGV_EMISSION_IGNORES_PIS_COFINS_PRODUCT_FIELDS_SIMPLES_NACIONAL")
else:
out[field] = _r(field, MANUAL, v, reason="NO_OFFICIAL_RULE_SOURCE_IN_CORE_DATASETS", key="pis_cofins_profile", scope="COMPANY")
# ---------------- incidence / PISCOFINS / IAT / IPPT ----------------
v = cur["INCIDENCIA_PISCOFINS"]
inc_ctx, _ = ctx.get("incidence", code, group)
if inc_ctx == "NENHUMA":
out["INCIDENCIA_PISCOFINS"] = _r("INCIDENCIA_PISCOFINS", ALREADY if v == "NENHUMA" else CHANGE, v, None if v == "NENHUMA" else "NENHUMA", "OPERATOR_INCIDENCE")
elif v:
out["INCIDENCIA_PISCOFINS"] = _r("INCIDENCIA_PISCOFINS", ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
else:
out["INCIDENCIA_PISCOFINS"] = _r("INCIDENCIA_PISCOFINS", MANUAL, v, reason="BLANK_INCIDENCE", key="incidence", scope="GROUP")
v = cur["PISCOFINS"]
pc_ctx, _ = ctx.get("piscofins", code, group)
if pc_ctx in ("N", "S"):
out["PISCOFINS"] = _r("PISCOFINS", ALREADY if v == pc_ctx else CHANGE, v, None if v == pc_ctx else pc_ctx, "OPERATOR_PISCOFINS")
elif v in ("N", "S"):
out["PISCOFINS"] = _r("PISCOFINS", ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
else:
out["PISCOFINS"] = _r("PISCOFINS", MANUAL, v, reason="BLANK_OR_INVALID", key="piscofins", scope="GROUP")
v = cur["IAT"]
rd_ctx, _ = ctx.get("rounding", code, group)
if rd_ctx in ("A", "T"):
out["IAT"] = _r("IAT", ALREADY if v == rd_ctx else CHANGE, v, None if v == rd_ctx else rd_ctx, "OPERATOR_ROUNDING_POLICY")
elif v in ("A", "T"):
out["IAT"] = _r("IAT", ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
else:
out["IAT"] = _r("IAT", MANUAL, v, reason="BLANK_OR_INVALID", key="rounding", scope="COMPANY")
v = cur["IPPT"]
if role in ("INPUT", "SERVICE"):
out["IPPT"] = _r("IPPT", NA, v, reason="INPUT_OR_SERVICE_HAS_NO_PRODUCTION_INDICATOR")
elif role == "UNKNOWN":
out["IPPT"] = _r("IPPT", ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED") if v in ("P", "T") else _r("IPPT", MANUAL, v, reason="IPPT_NEEDS_PRODUCT_ROLE", key="product_role", scope="GROUP")
else:
exp = "P" if role == "OWN_PRODUCTION" else "T"
out["IPPT"] = _r("IPPT", ALREADY if v == exp else CHANGE, v, None if v == exp else exp, "DERIVED_FROM_PRODUCT_ROLE:" + ",".join(role_ev))
for r in out.values():
r.setdefault("role", role)
out["_facts"] = {"role": role, "role_evidence": role_ev, "st": st, "st_withheld": withheld, "icms_treatment": treat, "origin_digit": origin_digit}
return out
+83
View File
@@ -0,0 +1,83 @@
"""Pure requiredness and fiscal completeness evaluation."""
import json
from pathlib import Path
from typing import Any
VALID_CLASSIFICATIONS = {
"REQUIRED", "CONDITIONAL", "OPTIONAL_REFERENCE", "NOT_APPLICABLE", "ERP_NOT_USED"
}
def load_requiredness_matrix(path: str | Path) -> dict[str, Any]:
matrix = json.loads(Path(path).read_text(encoding="utf-8"))
fields = matrix.get("fields", {})
if set(matrix.get("classifications", [])) != VALID_CLASSIFICATIONS:
raise ValueError("requiredness matrix classifications are incomplete")
if len(fields) != 23 or any(rule.get("classification") not in VALID_CLASSIFICATIONS for rule in fields.values()):
raise ValueError("requiredness matrix must define 23 valid field rules")
return matrix
def _has_value(value: Any) -> bool:
return value is not None and (not isinstance(value, str) or bool(value.strip()))
def _conditional_state(field: str, context: dict[str, Any]) -> str:
if field == "IPI":
return "REQUIRED" if context.get("contribuinte_ipi") == "S" and context.get("has_numeric_tipi") else "NOT_APPLICABLE"
if field == "EX_TIPI":
return "REQUIRED" if context.get("has_supported_ex") and context.get("product_matches_ex") else "NOT_APPLICABLE"
if field == "CEST":
return "REQUIRED" if context.get("has_official_cest") or context.get("st_subject") else "NOT_APPLICABLE"
if field in {"CFOP", "SITUACAO_TRIBUTARIA"}:
return "REQUIRED" if context.get("operation", "SALE") == "SALE" else "NOT_APPLICABLE"
if field in {"CFOP_ENTRADA", "CST_ENTRADA", "COD_ENTRADA_PIS", "COD_ENTRADA_COFINS"}:
return "REQUIRED" if context.get("operation") == "PURCHASE" and context.get("erp_consumes_entry_fields") else "NOT_APPLICABLE"
if field == "CST":
return "REQUIRED" if context.get("regime") == "NORMAL" and context.get("operation", "SALE") == "SALE" else "NOT_APPLICABLE"
if field == "CSOSN":
return "REQUIRED" if context.get("regime") == "SIMPLES" and context.get("operation", "SALE") == "SALE" else "NOT_APPLICABLE"
if field == "ALIQUOTA":
return "REQUIRED" if context.get("icms_taxed") else "NOT_APPLICABLE"
if field == "COD_ORIGEM_MERCADORIA":
return "REQUIRED" if context.get("origin_proven") else "NOT_APPLICABLE"
if field == "CST_IPI":
return "REQUIRED" if context.get("contribuinte_ipi") == "S" and context.get("erp_consumes_ipi_cst") else "NOT_APPLICABLE"
if field in {"COD_SAIDA_PIS", "COD_SAIDA_COFINS"}:
return "REQUIRED" if context.get("operation", "SALE") == "SALE" and context.get("piscofins_profile_proven") else "NOT_APPLICABLE"
if field in {"ALIQ_PIS", "ALIQ_COFINS"}:
return "REQUIRED" if context.get("piscofins_rate_applies") else "NOT_APPLICABLE"
if field in {"INCIDENCIA_PISCOFINS", "PISCOFINS"}:
return "REQUIRED" if context.get("piscofins_profile_proven") else "NOT_APPLICABLE"
return "NOT_APPLICABLE"
def evaluate_field(field: str, value: Any, context: dict[str, Any], matrix: dict[str, Any]) -> dict[str, Any]:
rule = matrix["fields"][field]
classification = rule["classification"]
if classification == "ERP_NOT_USED":
status = "ERP_NOT_SUPPORTED"
valid = True
elif classification == "NOT_APPLICABLE":
status = "NOT_APPLICABLE"
valid = True
elif classification == "OPTIONAL_REFERENCE":
status = "OPTIONAL_VALID" if _has_value(value) else "OPTIONAL_EMPTY"
valid = True
elif classification == "REQUIRED":
status = "REQUIRED_AND_VALID" if _has_value(value) else "REQUIRED_MISSING"
valid = _has_value(value)
else:
conditional = _conditional_state(field, context)
if conditional == "NOT_APPLICABLE":
status, valid = "CONDITIONAL_NOT_TRIGGERED", True
else:
status, valid = ("CONDITIONAL_AND_VALID", True) if _has_value(value) else ("REQUIRED_MISSING", False)
return {"field": field, "classification": classification, "status": status, "valid": valid, "value_present": _has_value(value), "source": rule["source"]}
def evaluate_product(product: dict[str, Any], context: dict[str, Any], matrix: dict[str, Any]) -> dict[str, Any]:
fields = {field: evaluate_field(field, product.get(field), context, matrix) for field in matrix["fields"]}
return {"FISCALLY_COMPLETE": all(result["valid"] for result in fields.values()), "fields": fields}
+99
View File
@@ -0,0 +1,99 @@
"""Contexto Fiscal: the ADMIN-only, human-language view of the handful of fiscal decisions field_contract.resolve_product genuinely cannot
derive from the ERP alone (see central/app/enrichment/fiscal_completion.py). Reuses FiscalContextEntry (the SAME table blocker_resolution
already writes operator answers into) - no new persistence model, no opaque JSON box in the UI.
DADO_DE_TESTE=true: Each decision here was audited on test/homologation environments (Agent 2 and lab databases) to validate the fiscal
engine mechanism. NONE of these findings represent conclusions on any real production client. Real client data will be validated
separately upon first real client onboarding. Evidence text below is a dated, one-time manual read-only measurement on test data, not a live query."""
from __future__ import annotations
DECISIONS = [
{
"key": "icms_treatment", "scope": "COMPANY", "title": "Tratamento padrão de ICMS dos produtos comuns",
"evidence": "[DADO_DE_TESTE=true: Base de homologação Agent 2] CSOSN padrão do ERP (C000004.ICMS_CSOSN): 102. 487 de 521 produtos cadastrados já usam 102. As exceções existentes "
"(500 = ICMS-ST retido, em 5 produtos; 101 = com direito a crédito, em 4 produtos; 103, em 1 produto) são preservadas "
"e nunca sobrescritas - a confirmação abaixo só afeta os produtos com CSOSN em branco (28 hoje). "
"Dado de laboratório/teste; não constitui conclusão sobre cliente real.",
"suggestion": "TAXED", "allowed_values": ["TAXED", "EXEMPT", "NOT_TAXED", "ST_WITHHELD", "SUSPENDED", "DEFERRED"],
"affected_fields": ["CST", "CSOSN"], "confidence": "Média (DADO_DE_TESTE=true: medida em banco de homologação; pendente de validação com cliente real)",
"measured_at": "2026-09-27", "is_test_data": True, "data_label": "DADO_DE_TESTE=true",
},
{
"key": "pis_cofins_profile", "scope": "COMPANY", "title": "Perfil de PIS/COFINS (código e alíquota de entrada/saída)",
"evidence": "[DADO_DE_TESTE=true: Auditoria em 5 bancos de laboratório e Agent 2] 34.946 itens de NFC-e emitidos; 3.336 (9,5%) carregam um CST de PIS/COFINS na nota, sempre o código 01 - mas os "
"mesmos produtos têm os campos de cadastro (código de saída, alíquota) sempre em branco no SGV, incluindo os poucos "
"produtos que TÊM esses campos preenchidos manualmente (a nota emitida ignora o valor cadastrado). Reprocessado em "
"2026-09-28 (Missão 2 / Bloco 2.1): os 3.336 itens estão concentrados quase inteiramente em 2022 (3.336 de 3.669 "
"itens daquele ano; 0% em 2018 e em 2023-2026) - coincide com a mesma janela já sinalizada como anômala em outra "
"auditoria (drift de identidade de produto). O mesmo padrão de cadastro nunca preenchido na emissão se repete, sem "
"exceção, nos outros 4 bancos de teste auditados (CONSTRUCAO, MMMODAS, RINCAO CRT=3, UNIAO): 0% de itens com CST de "
"PIS/COFINS em todo o histórico, em todos os anos. Amostra de laboratório; não é garantia de comportamento de cliente real em produção. "
"Ver docs/dossie_contador.md e docs/runs/BLOCO2_PIS_COFINS_CROSS_DB_V1.json.", "suggestion": None,
"affected_fields": ["COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS"],
"confidence": "Baixa para um perfil único da empresa (DADO_DE_TESTE=true: considerar NOT_APPLICABLE ou OPERATION_TIME; pendente de confirmação contábil no piloto real)",
"measured_at": "2026-09-28", "is_test_data": True, "data_label": "DADO_DE_TESTE=true",
"value_hint": 'JSON de perfil (ex.: {"pis_out_cst":"49","pis_in_cst":"98","pis_rate":"0","cofins_out_cst":"49","cofins_in_cst":"98","cofins_rate":"0"}) '
'OU exatamente NOT_APPLICABLE (nunca é lido pelo cadastro) OU OPERATION_TIME (varia por operação) - nunca inferido automaticamente.',
"dossier_link": "/static/docs/dossie_contador.md",
},
{
"key": "entry_scope", "scope": "COMPANY", "title": "CFOP de entrada padrão (compras dentro/fora do estado)",
"evidence": "[DADO_DE_TESTE=true: Base de homologação Agent 2] Apenas 12 documentos de compra no ERP de teste para 526 produtos; 11 sem CFOP preenchido, 1 com CFOP intra-estadual. "
"Evidência de laboratório insuficiente para qualquer sugestão - o CFOP de entrada normalmente é definido pela operação/documento "
"de compra, não pelo cadastro do produto. Ver docs/dossie_contador.md.", "suggestion": None,
"affected_fields": ["CFOP_ENTRADA"], "confidence": "Muito baixa (DADO_DE_TESTE=true: recomendação: MIXED_OR_DYNAMIC; validação definitiva apenas com cliente real)",
"measured_at": "2026-09-27", "allowed_values": ["INTRA", "INTER", "MIXED_OR_DYNAMIC"], "is_test_data": True, "data_label": "DADO_DE_TESTE=true",
"dossier_link": "/static/docs/dossie_contador.md",
},
]
VALID_KEYS = {d["key"] for d in DECISIONS}
def set_fiscal_context_decision(session, *, tenant_id: int, company_key: str, key: str, value: str | None, reason: str, admin, installation_id: int):
"""Confirm/alter/clear one Contexto Fiscal decision. value=None clears it (the product/group falls back to whatever the engine can
prove on its own - never a silent default). Reuses the SAME table and validator the blocker-resolution workflow already writes
through; flags every already-enriched product on this installation for re-evaluation so the new (or cleared) policy takes effect."""
from datetime import datetime, timezone
from sqlalchemy import select
from . import context_keys
from .audit import record_audit
from .enrichment.status import mark_for_reevaluation
from .models import FiscalContextEntry
if key not in VALID_KEYS:
raise ValueError(f"unknown Contexto Fiscal key {key!r}")
entry = session.scalar(select(FiscalContextEntry).where(FiscalContextEntry.tenant_id == tenant_id, FiscalContextEntry.company_key == company_key,
FiscalContextEntry.scope == "COMPANY", FiscalContextEntry.scope_key == "", FiscalContextEntry.key == key))
before = entry.value if entry else None
if value is None:
if entry is not None:
session.delete(entry)
else:
normalized, error = context_keys.validate_entry({"scope": "COMPANY", "key": key, "value": value, "reason": reason})
if error:
raise ValueError(error)
if entry is None:
entry = FiscalContextEntry(tenant_id=tenant_id, company_key=company_key, source="OPERATOR", updated_at=datetime.now(timezone.utc), **normalized)
session.add(entry)
else:
entry.value, entry.reason, entry.source, entry.updated_at = normalized["value"], normalized["reason"], "OPERATOR", datetime.now(timezone.utc)
flagged = mark_for_reevaluation(session, installation_id)
record_audit(session, admin_user_id=admin.id, tenant_id=tenant_id, action="FISCAL_CONTEXT_DECISION", target_type="installation", target_id=company_key,
context={"key": key, "from": before, "to": value, "reason": reason, "operator": admin.display_name or admin.username, "role": admin.external_role or admin.auth_source, "products_flagged": flagged},
severity="WARNING")
def build_fiscal_context_view(session, tenant_id: int, company_key: str) -> list[dict]:
from sqlalchemy import select
from .models import FiscalContextEntry
current = {(r.scope, r.scope_key, r.key): r for r in session.scalars(
select(FiscalContextEntry).where(FiscalContextEntry.tenant_id == tenant_id, FiscalContextEntry.company_key == company_key))}
out = []
for d in DECISIONS:
row = current.get((d["scope"], "", d["key"]))
out.append({**d, "current_value": row.value if row else None, "current_reason": row.reason if row else None,
"last_changed": row.updated_at.isoformat() if row and getattr(row, "updated_at", None) else None,
"last_changed_by": row.source if row else None})
return out
+224
View File
@@ -0,0 +1,224 @@
"""Fiscal Decision Engine V1: independent per-field, per-product decisions.
Pure functions (no I/O). Every domain gets its own decision; a dataset being
available is never treated as a rule being resolved. Nothing here writes to
any database — it only marks `write_eligible`, and only for fields whose ERP
mapping is confirmed AND whose rule is deterministic AND that need no
operation context.
Status vocabulary (per domain):
RULE_RESOLVED, RULE_CANDIDATE, RULE_UNRESOLVED, OPERATION_CONTEXT_REQUIRED
plus flags data_available / match_found. `action_class` is the Phase-14 bucket:
AUTO_APPLY_SAFE, REVIEW_REQUIRED, OPERATION_CONTEXT_REQUIRED, NO_RULE, NO_DATA
(+ ALREADY_CORRECT when the ERP value already equals the deterministic one).
"""
from .dry_run_planner import build_field_proposals
DECISION_ENGINE_VERSION = "fiscal-decision-v1"
NATURE = {
"NCM": "PRODUCT_INTRINSIC", "IPI": "PRODUCT_INTRINSIC", "EX_TIPI": "PRODUCT_INTRINSIC",
"CEST": "PRODUCT_INTRINSIC", "MVA": "PRODUCT_INTRINSIC",
"ORIGEM": "PRODUCT_INTRINSIC",
"CFOP": "OPERATION_DEPENDENT", "CSOSN": "OPERATION_DEPENDENT", "CST_ICMS": "OPERATION_DEPENDENT",
"PIS_CST": "OPERATION_DEPENDENT", "COFINS_CST": "OPERATION_DEPENDENT",
"PIS_COFINS_RATE": "COMPANY_CONTEXT_DEPENDENT", "FCP": "OPERATION_DEPENDENT",
"RTC_CCLASSTRIB": "OPERATION_DEPENDENT", "RTC_CBS_RATE": "OPERATION_DEPENDENT",
"RTC_CCREDPRES": "OPERATION_DEPENDENT",
}
# ERP_FIELD, SEMANTICS, DB_TYPE, CONFIDENCE, EVIDENCE — audited against the live
# C000025 schema (RDB$ metadata: no field comments exist) and value distributions
# over all 521 rows (scripts/audit_erp_fields.py). No supplier DDL/doc file was
# found in the repo, so semantic confirmation rests on value-domain evidence.
ERP_FIELD_MAP = [
{"ERP_FIELD": "CLASSIFICACAO_FISCAL", "SEMANTICS": "NCM canônico (8 dígitos)", "DB_TYPE": "VARCHAR(20)", "CONFIDENCE": "HIGH", "EVIDENCE": "54 distinct 8-digit values, 495 valid in NCM dataset; prior-mission confirmed (NCM column is 100% NULL)", "MAPPING": "CONFIRMED"},
{"ERP_FIELD": "IPI", "SEMANTICS": "alíquota numérica de IPI (%)", "DB_TYPE": "DOUBLE PRECISION", "CONFIDENCE": "HIGH", "EVIDENCE": "numeric type; 105 rows = 19.5 written earlier from TIPI and verified; prior-mission confirmed", "MAPPING": "CONFIRMED"},
{"ERP_FIELD": "EX_TIPI", "SEMANTICS": "código EX da TIPI", "DB_TYPE": "VARCHAR(3)", "CONFIDENCE": "HIGH", "EVIDENCE": "prior-mission confirmed; 100% NULL now", "MAPPING": "CONFIRMED"},
{"ERP_FIELD": "CEST", "SEMANTICS": "CEST do produto (7 dígitos sem pontuação)", "DB_TYPE": "VARCHAR(7)", "CONFIDENCE": "HIGH", "EVIDENCE": "values like 0601100/1708400 = CEST 06.011.00/17.084.00 without dots; prior-mission confirmed", "MAPPING": "CONFIRMED"},
{"ERP_FIELD": "CFOP", "SEMANTICS": "CFOP padrão de SAÍDA do produto (default, não a operação real)", "DB_TYPE": "VARCHAR(4)", "CONFIDENCE": "HIGH", "EVIDENCE": "values 5102 (493), 5405 (3), 5656 (2): all valid 5xxx output CFOPs in CFOP dataset; 5405=ST, 5656=fuel", "MAPPING": "CONFIRMED"},
{"ERP_FIELD": "CFOP_ENTRADA", "SEMANTICS": "CFOP padrão de entrada", "DB_TYPE": "SMALLINT", "CONFIDENCE": "LOW", "EVIDENCE": "100% NULL in all 521 rows; name only", "MAPPING": "MAPPING_UNCONFIRMED"},
{"ERP_FIELD": "CST", "SEMANTICS": "CST ICMS de saída em 3 dígitos = ORIGEM(1)+CST(2)", "DB_TYPE": "VARCHAR(10)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "values 040/000/060/020/051: last 2 digits all in CST_ICMS dataset, first digit in ORIGEM dataset (0)", "MAPPING": "CONFIRMED"},
{"ERP_FIELD": "CST_ENTRADA", "SEMANTICS": "CST ICMS de entrada", "DB_TYPE": "SMALLINT", "CONFIDENCE": "LOW", "EVIDENCE": "100% NULL; name only", "MAPPING": "MAPPING_UNCONFIRMED"},
{"ERP_FIELD": "ALIQUOTA", "SEMANTICS": "alíquota de ICMS % (saída)", "DB_TYPE": "NUMERIC(8,2)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "values 20.5 (492), 17 (24), 0 (4), 21.5 (1): plausible BA ICMS rates; no doc confirms", "MAPPING": "CONFIRMED"},
{"ERP_FIELD": "CSOSN", "SEMANTICS": "CSOSN de saída (Simples Nacional)", "DB_TYPE": "VARCHAR(60)", "CONFIDENCE": "HIGH", "EVIDENCE": "values 102/500/101/103 all in CSOSN dataset; company CRT=1", "MAPPING": "CONFIRMED"},
{"ERP_FIELD": "COD_ENTRADA_PIS", "SEMANTICS": "CST PIS de entrada", "DB_TYPE": "VARCHAR(3)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "2 non-null rows ('2','73'); mixed formatting ('2' vs 2-digit codes) — too sparse", "MAPPING": "MAPPING_UNCONFIRMED"},
{"ERP_FIELD": "COD_SAIDA_PIS", "SEMANTICS": "CST PIS de saída", "DB_TYPE": "VARCHAR(3)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "3 non-null rows ('01','04','2'); inconsistent formatting", "MAPPING": "MAPPING_UNCONFIRMED"},
{"ERP_FIELD": "ALIQ_PIS", "SEMANTICS": "alíquota PIS (texto, vírgula decimal)", "DB_TYPE": "VARCHAR(8)", "CONFIDENCE": "LOW", "EVIDENCE": "3 non-null rows ('0,65','2', blank); text type", "MAPPING": "MAPPING_UNCONFIRMED"},
{"ERP_FIELD": "COD_ENTRADA_COFINS", "SEMANTICS": "CST COFINS de entrada", "DB_TYPE": "VARCHAR(3)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "2 non-null rows; too sparse", "MAPPING": "MAPPING_UNCONFIRMED"},
{"ERP_FIELD": "COD_SAIDA_COFINS", "SEMANTICS": "CST COFINS de saída", "DB_TYPE": "VARCHAR(3)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "3 non-null rows; inconsistent formatting", "MAPPING": "MAPPING_UNCONFIRMED"},
{"ERP_FIELD": "ALIQ_COFINS", "SEMANTICS": "alíquota COFINS (texto)", "DB_TYPE": "VARCHAR(8)", "CONFIDENCE": "LOW", "EVIDENCE": "3 non-null rows ('2','3', blank)", "MAPPING": "MAPPING_UNCONFIRMED"},
{"ERP_FIELD": "CST_IPI", "SEMANTICS": "CST do IPI", "DB_TYPE": "SMALLINT", "CONFIDENCE": "LOW", "EVIDENCE": "1 non-null row ('040'); IPI CST is 2 digits — value contradicts", "MAPPING": "MAPPING_UNCONFIRMED"},
{"ERP_FIELD": "ORIGEM", "SEMANTICS": "origem da mercadoria (0-8)", "DB_TYPE": "VARCHAR(1)", "CONFIDENCE": "LOW", "EVIDENCE": "100% NULL; origin digit is instead embedded in CST (first digit)", "MAPPING": "MAPPING_UNCONFIRMED"},
{"ERP_FIELD": "IAT", "SEMANTICS": "indicador arredondamento/truncamento (A/T)", "DB_TYPE": "VARCHAR(1)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "constant 'A' in 521 rows; not a fiscal-decision field", "MAPPING": "CONFIRMED"},
{"ERP_FIELD": "INCIDENCIA_PISCOFINS", "SEMANTICS": "tipo de incidência PIS/COFINS", "DB_TYPE": "VARCHAR(30)", "CONFIDENCE": "HIGH", "EVIDENCE": "values NENHUMA (519), MONOFÁSICA (1), SUBST.TRIBUTÁRIA (1)", "MAPPING": "CONFIRMED"},
{"ERP_FIELD": "IPPT", "SEMANTICS": "indicador produção própria/terceiros (P/T)", "DB_TYPE": "VARCHAR(1)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "constant 'T' in 521 rows; not a fiscal-decision field", "MAPPING": "CONFIRMED"},
]
_ACTION = {
"RULE_RESOLVED": "AUTO_APPLY_SAFE", # only if write_eligible; refined below
"RULE_CANDIDATE": "REVIEW_REQUIRED",
"OPERATION_CONTEXT_REQUIRED": "OPERATION_CONTEXT_REQUIRED",
}
def _rec(domain, current, proposed, status, reason, confidence, provenance, write_eligible=False,
data_available=True, match_found=False, action=None, **extra):
if action is None:
if status == "RULE_RESOLVED":
action = "AUTO_APPLY_SAFE" if write_eligible else "ALREADY_CORRECT" if reason.endswith("SAME_VALUE") else "NO_RULE"
elif status == "RULE_UNRESOLVED":
action = "NO_DATA" if not data_available else "NO_RULE"
else:
action = _ACTION[status]
return {
"domain": domain, "nature": NATURE[domain], "current_value": current, "proposed_value": proposed,
"status": status, "reason_code": reason, "confidence": confidence, "provenance": provenance,
"write_eligible": write_eligible, "action_class": action,
"data_available": data_available, "match_found": match_found, **extra,
}
def _norm_words(text):
import unicodedata
text = unicodedata.normalize("NFKD", str(text or "")).encode("ascii", "ignore").decode().lower()
return {w for w in "".join(c if c.isalnum() else " " for c in text).split() if len(w) >= 4}
def _ex_described(product_description, ex_description):
"""An EX code applies to a product only if the product's own description
carries the EX's official description terms. A lone EX row for the NCM is
NOT evidence that this product is that EX (e.g. EX 'Chope' vs any beer)."""
ex_words = _norm_words(ex_description)
return bool(ex_words) and ex_words <= _norm_words(product_description)
def _blank(value):
return value is None or str(value).strip() == ""
def decide_product(product, decision, company, refs):
"""company: {"uf": "BA", "crt": "1"}; refs: reference-data snapshot dict:
cfop_valid(set of codes), cst_icms/csosn/cst_pis/cst_cofins/origem (sets),
fcp_uf(dict uf->{'aliq_1':...}), cbs_rate_2026 (float|None), cclasstrib_count (int)."""
versions = decision.get("dataset_versions", {})
prov = lambda kind: {"dataset": kind, "version": versions.get(kind.lower()), "engine": DECISION_ENGINE_VERSION} # noqa: E731
out = {}
# ---- NCM (never written) -------------------------------------------------
raw_ncm = product.get("CLASSIFICACAO_FISCAL")
status = decision.get("status")
if _blank(raw_ncm):
out["NCM"] = _rec("NCM", raw_ncm, None, "RULE_UNRESOLVED", "NCM_MISSING", "NONE", prov("NCM"), data_available=False)
elif status == "EXACT_NCM":
out["NCM"] = _rec("NCM", raw_ncm, raw_ncm, "RULE_RESOLVED", "NCM_VALID_SAME_VALUE", "HIGH", prov("NCM"), match_found=True)
elif status == "LEGACY_NCM":
out["NCM"] = _rec("NCM", raw_ncm, None, "RULE_CANDIDATE", "LEGACY_NCM_REVIEW_REQUIRED", "LOW", prov("NCM"), match_found=True, action="REVIEW_REQUIRED")
else:
out["NCM"] = _rec("NCM", raw_ncm, None, "RULE_CANDIDATE", "NCM_INVALID_REVIEW_REQUIRED", "LOW", prov("NCM"), action="REVIEW_REQUIRED")
legacy = status == "LEGACY_NCM"
proposals = {p["field"]: p for p in build_field_proposals(
product["CODIGO"], decision,
{"ipi": product.get("IPI"), "cest": product.get("CEST"), "ex_tipi": product.get("EX_TIPI")})}
# ---- IPI -------------------------------------------------------------------
p = proposals.get("IPI")
ipi_status = decision.get("ipi_status")
if p and p["status"] in ("SAME_VALUE", "CHANGE_REQUIRED"):
same = p["status"] == "SAME_VALUE"
out["IPI"] = _rec("IPI", p["current_value"], p["proposed_value"], "RULE_RESOLVED",
"TIPI_NUMERIC_SAME_VALUE" if same else "TIPI_NUMERIC_DIVERGES_FROM_ERP", "HIGH", prov("TIPI_IPI"),
write_eligible=p["write_eligible"], match_found=True)
elif p:
blank = p["status"] == "SOURCE_BLANK" or p["reason_code"].endswith("SOURCE_BLANK")
out["IPI"] = _rec("IPI", p["current_value"], None, "RULE_UNRESOLVED", p["reason_code"], "NONE", prov("TIPI_IPI"),
data_available=not blank, match_found=True, action="NO_DATA" if blank else "NO_RULE")
elif ipi_status == "IPI_CANDIDATES":
out["IPI"] = _rec("IPI", product.get("IPI"), None, "RULE_CANDIDATE", "MULTIPLE_TIPI_CANDIDATES", "LOW", prov("TIPI_IPI"), match_found=True)
else:
out["IPI"] = _rec("IPI", product.get("IPI"), None, "RULE_UNRESOLVED", "LEGACY_NCM" if legacy else "NO_TIPI_MAPPING_FOUND", "NONE", prov("TIPI_IPI"))
# ---- EX_TIPI ---------------------------------------------------------------
p = proposals.get("EX_TIPI")
if p:
ex_desc = next((c.get("description") for c in (decision.get("classification") or {}).get("ipi_candidates", []) if c.get("ex")), None)
if p["status"] in ("SAME_VALUE", "CHANGE_REQUIRED") and not _ex_described(product.get("PRODUTO"), ex_desc):
out["EX_TIPI"] = _rec("EX_TIPI", p["current_value"], None, "RULE_CANDIDATE", "SINGLE_EX_FOR_NCM_BUT_PRODUCT_DESCRIPTION_DOES_NOT_EVIDENCE_IT", "LOW", prov("TIPI_IPI"), match_found=True)
elif p["status"] in ("SAME_VALUE", "CHANGE_REQUIRED"):
out["EX_TIPI"] = _rec("EX_TIPI", p["current_value"], p["proposed_value"], "RULE_RESOLVED", "EX_" + p["status"], "HIGH", prov("TIPI_IPI"), write_eligible=p["write_eligible"], match_found=True)
else:
out["EX_TIPI"] = _rec("EX_TIPI", p["current_value"], None, "RULE_CANDIDATE", p["reason_code"], "LOW", prov("TIPI_IPI"), match_found=True)
else:
out["EX_TIPI"] = _rec("EX_TIPI", product.get("EX_TIPI"), None, "RULE_UNRESOLVED", "NO_EX_TIPI_FOR_NCM", "NONE", prov("TIPI_IPI"), data_available=not legacy)
# ---- CEST ------------------------------------------------------------------
p = proposals.get("CEST")
if p and p["status"] in ("SAME_VALUE", "CHANGE_REQUIRED"):
out["CEST"] = _rec("CEST", p["current_value"], p["proposed_value"], "RULE_RESOLVED", "CEST_" + p["status"], "HIGH", prov("CEST"), write_eligible=p["write_eligible"], match_found=True)
elif p and p["status"] == "CANDIDATE":
n = len((decision.get("classification") or {}).get("cest_candidates") or [])
out["CEST"] = _rec("CEST", p["current_value"], None, "RULE_CANDIDATE", p["reason_code"], "LOW", prov("CEST"), match_found=True, candidate_count=n)
else:
out["CEST"] = _rec("CEST", product.get("CEST"), None, "RULE_UNRESOLVED", p["reason_code"] if p else ("LEGACY_NCM" if legacy else "NO_CEST_DECISION"), "NONE", prov("CEST"), data_available=False if not p else True)
# ---- BA ST / MVA (no ERP destination field: never eligible) ---------------
mva_status = decision.get("ba_st_mva_status")
if mva_status == "MVA_RESOLVED":
entry = (decision.get("classification") or {}).get("ba_st_mva") or {}
out["MVA"] = _rec("MVA", None, entry.get("mva_original") or entry.get("mva_adjusted"), "RULE_RESOLVED", "MVA_RESOLVED_NO_ERP_TARGET_FIELD", "HIGH", prov("BA_ST_MVA"), match_found=True, action="NO_RULE", mva_status="MVA_RESOLVED")
elif mva_status == "MVA_CANDIDATES":
out["MVA"] = _rec("MVA", None, None, "RULE_CANDIDATE", "MULTIPLE_MVA_ITEMS_NEED_CEST_CONFIRMATION", "LOW", prov("BA_ST_MVA"), match_found=True, mva_status="MVA_CANDIDATE")
elif mva_status == "NO_MVA_MAPPING_FOUND":
out["MVA"] = _rec("MVA", None, None, "RULE_UNRESOLVED", "MVA_NOT_APPLICABLE_NO_BA_ANNEX1_ENTRY_FOR_NCM", "MEDIUM", prov("BA_ST_MVA"), action="NO_RULE", mva_status="MVA_NOT_APPLICABLE")
else:
out["MVA"] = _rec("MVA", None, None, "RULE_UNRESOLVED", "LEGACY_OR_INVALID_NCM", "NONE", prov("BA_ST_MVA"), data_available=False, mva_status="MVA_UNRESOLVED")
mva_subject = mva_status in ("MVA_RESOLVED", "MVA_CANDIDATES")
# ---- CFOP (operation dependent) --------------------------------------------
cfop = product.get("CFOP")
cfop_valid = not _blank(cfop) and str(cfop).strip() in refs["cfop_valid"]
out["CFOP"] = _rec("CFOP", cfop, None, "OPERATION_CONTEXT_REQUIRED", "CFOP_DEPENDS_ON_OPERATION_DIRECTION_UF_ST_PURPOSE", "HIGH", prov("CFOP"),
data_available=True, match_found=cfop_valid, domain_valid=cfop_valid, current_is_blank=_blank(cfop))
# ---- CSOSN / CST ICMS / ORIGEM ---------------------------------------------
csosn = product.get("CSOSN")
csosn_valid = not _blank(csosn) and str(csosn).strip() in refs["csosn"]
flags = {}
if mva_subject and csosn_valid and str(csosn).strip() == "102":
flags["consistency_flag"] = "CSOSN_102_BUT_NCM_IN_BA_ST_ANNEX1"
out["CSOSN"] = _rec("CSOSN", csosn, None, "OPERATION_CONTEXT_REQUIRED", "CSOSN_DEPENDS_ON_SALE_SCENARIO_ST_RETENTION_CONSUMER", "HIGH", prov("NFE_MOC_TABLES"),
match_found=csosn_valid, domain_valid=csosn_valid, **flags)
cst = product.get("CST")
cst_s = "" if _blank(cst) else str(cst).strip()
cst_valid = len(cst_s) == 3 and cst_s[1:] in refs["cst_icms"] and cst_s[0] in refs["origem"]
out["CST_ICMS"] = _rec("CST_ICMS", cst, None, "OPERATION_CONTEXT_REQUIRED", "CRT1_USES_CSOSN_CST_DEPENDS_ON_OPERATION", "MEDIUM", prov("NFE_MOC_TABLES"),
match_found=cst_valid, domain_valid=cst_valid)
origin_candidate = cst_s[0] if cst_valid else None
out["ORIGEM"] = _rec("ORIGEM", product.get("ORIGEM"), origin_candidate, "RULE_CANDIDATE" if origin_candidate else "RULE_UNRESOLVED",
"ORIGIN_ONLY_FROM_ERP_CST_FIRST_DIGIT_NO_OBJECTIVE_PRODUCT_ORIGIN" if origin_candidate else "NO_ORIGIN_INFORMATION",
"LOW", prov("NFE_MOC_TABLES"), data_available=bool(origin_candidate),
action="REVIEW_REQUIRED" if origin_candidate else "NO_DATA", mapping="MAPPING_UNCONFIRMED")
# ---- PIS / COFINS ----------------------------------------------------------
for domain, field, dom in (("PIS_CST", "COD_SAIDA_PIS", "cst_pis"), ("COFINS_CST", "COD_SAIDA_COFINS", "cst_cofins")):
cur = product.get(field)
valid = not _blank(cur) and str(cur).strip() in refs[dom]
out[domain] = _rec(domain, cur, None, "OPERATION_CONTEXT_REQUIRED", "CST_PIS_COFINS_DEPENDS_ON_OPERATION_AND_INCIDENCE_NO_DETERMINISTIC_RULE_IN_CORE_DATASETS", "MEDIUM", prov("NFE_MOC_TABLES"),
match_found=valid, domain_valid=valid, mapping="MAPPING_UNCONFIRMED")
out["PIS_COFINS_RATE"] = _rec("PIS_COFINS_RATE", product.get("ALIQ_PIS"), None, "RULE_UNRESOLVED", "NO_RATE_SOURCE_4_3_10_17_DEFERRED_NEVER_ASSUME_ZERO", "NONE", prov("NFE_MOC_TABLES"), data_available=False, action="NO_RULE")
# ---- FCP -------------------------------------------------------------------
fcp = refs["fcp_uf"].get(company["uf"])
out["FCP"] = _rec("FCP", None, None, "OPERATION_CONTEXT_REQUIRED", "FCP_APPLICABILITY_DEPENDS_ON_DESTINATION_UF_OPERATION_AND_ITEM_LIST", "HIGH", prov("FCP"),
data_available=fcp is not None, match_found=False, fcp_rate_available=fcp is not None, company_uf=company["uf"])
# ---- RTC -------------------------------------------------------------------
out["RTC_CCLASSTRIB"] = _rec("RTC_CCLASSTRIB", None, None, "OPERATION_CONTEXT_REQUIRED", "NO_NCM_TO_CCLASSTRIB_MAP_IN_ACTIVE_DATASETS_AND_DEPENDS_ON_OPERATION", "NONE", prov("RTC_CCLASSTRIB"),
data_available=refs["cclasstrib_count"] > 0, candidate_count=0)
out["RTC_CBS_RATE"] = _rec("RTC_CBS_RATE", None, None, "OPERATION_CONTEXT_REQUIRED", "CBS_2026_RATE_APPLIES_ONLY_AFTER_CCLASSTRIB_RESOLVED_NOT_UNIVERSAL", "NONE", prov("RTC_CBS_RATES"),
data_available=refs["cbs_rate_2026"] is not None, reference_rate_2026=refs["cbs_rate_2026"])
out["RTC_CCREDPRES"] = _rec("RTC_CCREDPRES", None, None, "RULE_UNRESOLVED", "OFFICIAL_CONDITION_NOT_SATISFIED_OR_NOT_KNOWABLE_FROM_PRODUCT", "NONE", prov("RTC_CCREDPRES"), action="NO_RULE")
return out
@@ -0,0 +1,11 @@
"""Conservative fiscal determination with no inferred tax rules."""
class FiscalDetermination:
def determine(self, context):
candidates = context.get("cest_candidates", [])
cest = candidates[0] if len(candidates) == 1 else None
reason_codes = ["OFFICIAL_RULE_NOT_IMPLEMENTED", "MISSING_ST_DETERMINATION", "MISSING_PRODUCT_ORIGIN"]
if len(candidates) > 1:
reason_codes.append("CEST_AMBIGUOUS")
return {"cest": cest, "csosn": None, "cfop": None, "confidence": 0.0, "reason_codes": reason_codes, "can_auto_apply": False}
+65
View File
@@ -0,0 +1,65 @@
"""Validador local determinístico de GTIN / EAN (GS1 Standard).
Sem chamadas externas, sem IA. Implementa:
1. Validação de formato (GTIN-8, GTIN-12, GTIN-13, GTIN-14);
2. Rejeição de placeholders (zeros, sequências óbvias, códigos internos de balança restrita);
3. Verificação do dígito verificador oficial GS1 (módulo 10 com pesos alternados 3 e 1);
4. Validação de faixas de prefixo GS1 reconhecidas.
"""
from __future__ import annotations
def _digits(value: str | None) -> str:
return "".join(c for c in str(value or "") if c.isdigit())
# Prefixos de circulação restrita / uso interno de loja (não são GTINs globais válidos para KB nacional)
# Faixas 020-029 e 200-299 são reservadas pelo GS1 para distribuição restrita (itens pesáveis, balança de loja)
INTERNAL_STORE_PREFIXES = tuple(str(p) for p in range(20, 30)) + tuple(str(p) for p in range(200, 300))
def calculate_gs1_check_digit(digits_without_dv: str) -> int:
"""Calcula o dígito verificador padrão GS1 pelo algoritmo oficial módulo 10."""
total = 0
# Da direita para a esquerda, os fatores alternam entre 3 e 1
# Para o penúltimo dígito (imediatamente antes do DV), o peso é sempre 3
weight = 3
for char in reversed(digits_without_dv):
total += int(char) * weight
weight = 1 if weight == 3 else 3
remainder = total % 10
return 0 if remainder == 0 else (10 - remainder)
def validate_gtin(value: str | None) -> tuple[bool, str]:
"""Valida um código GTIN.
Retorna (is_valid, reason).
"""
if not value:
return False, "GTIN_EMPTY"
cleaned = _digits(value)
if len(cleaned) not in (8, 12, 13, 14):
return False, f"GTIN_INVALID_LENGTH_{len(cleaned)}"
# Rejeita dígitos todos iguais (ex: 00000000, 1111111111111, 9999999999999)
if len(set(cleaned)) == 1:
return False, "GTIN_PLACEHOLDER_REPEATED_DIGITS"
# Rejeita prefixos de uso interno / balança
# Normalizado para 13 dígitos para verificar o prefixo
padded_13 = cleaned.zfill(13) if len(cleaned) < 14 else cleaned[1:]
prefix_3 = padded_13[:3]
prefix_2 = padded_13[:2]
if prefix_3 in INTERNAL_STORE_PREFIXES or prefix_2 in INTERNAL_STORE_PREFIXES:
return False, "GTIN_RESTRICTED_INTERNAL_STORE_PREFIX"
# Validação do Dígito Verificador
expected_dv = calculate_gs1_check_digit(cleaned[:-1])
actual_dv = int(cleaned[-1])
if actual_dv != expected_dv:
return False, f"GTIN_INVALID_CHECK_DIGIT_EXPECTED_{expected_dv}_GOT_{actual_dv}"
return True, "GTIN_VALID"
+83
View File
@@ -0,0 +1,83 @@
"""Separate human authentication backed by hashed passwords and sessions."""
from __future__ import annotations
import base64
import hashlib
import hmac
import os
import secrets
from datetime import datetime, timedelta, timezone
from fastapi import HTTPException, Request
from sqlalchemy import select
from .config import settings
from .models import AdminUser, WebSession
def _digest(value: str) -> str:
return hashlib.sha256(value.encode()).hexdigest()
def hash_password(password: str) -> str:
salt = os.urandom(16)
encoded = hashlib.pbkdf2_hmac("sha256", password.encode(), salt, 240_000)
return "pbkdf2_sha256$240000$%s$%s" % (base64.urlsafe_b64encode(salt).decode(), base64.urlsafe_b64encode(encoded).decode())
def verify_password(password: str, stored: str | None) -> bool:
# Federated (SSO) accounts have no password: always refuse, never raise.
if not isinstance(stored, str) or not isinstance(password, str):
return False
try:
algorithm, rounds, salt, digest = stored.split("$", 3)
candidate = hashlib.pbkdf2_hmac(algorithm.removeprefix("pbkdf2_"), password.encode(), base64.urlsafe_b64decode(salt), int(rounds))
return hmac.compare_digest(base64.urlsafe_b64encode(candidate).decode(), digest)
except (ValueError, TypeError):
return False
def create_admin(session, username: str, password: str):
username = username.strip().lower()
if len(username) < 3:
raise ValueError("username must have 3 characters")
environment = os.getenv("APP_ENV") or settings.environment
weak_passwords_allowed = os.getenv("ALLOW_WEAK_DEV_PASSWORD", "0") == "1"
if len(password) < 12 and (environment.strip().lower() == "production" or not weak_passwords_allowed):
raise ValueError("password must have 12 characters")
if session.scalar(select(AdminUser).where(AdminUser.username == username)):
raise ValueError("username already exists")
admin = AdminUser(username=username, password_hash=hash_password(password))
session.add(admin); session.commit(); return admin
def create_session(session, admin: AdminUser, request: Request, seconds: int | None = None):
raw_token = secrets.token_urlsafe(32)
raw_csrf = secrets.token_urlsafe(32)
now = datetime.now(timezone.utc)
lifetime = seconds if seconds is not None else settings.web_session_seconds
row = WebSession(admin_user_id=admin.id, token_hash=_digest(raw_token), csrf_hash=_digest(raw_csrf), expires_at=now + timedelta(seconds=lifetime),user_agent=request.headers.get("user-agent", "")[:512], remote_addr=request.client.host if request.client else None)
session.add(row); admin.last_login_at = now; session.commit()
return raw_token, raw_csrf
def current_session(request: Request, session):
token = request.cookies.get(settings.web_cookie_name)
if not token: return None, None
row = session.scalar(select(WebSession).where(WebSession.token_hash == _digest(token)))
if not row or row.expires_at.replace(tzinfo=timezone.utc) < datetime.now(timezone.utc): return None, None
admin = session.get(AdminUser, row.admin_user_id)
if not admin or not admin.is_active: return None, None
return row, admin
def require_human(request: Request, session):
web_session, admin = current_session(request, session)
if not admin: raise HTTPException(401, "human login required")
return web_session, admin
def require_csrf(web_session: WebSession, csrf_token: str | None):
if not csrf_token or not hmac.compare_digest(web_session.csrf_hash, _digest(csrf_token)):
raise HTTPException(403, "invalid CSRF token")
+107
View File
@@ -0,0 +1,107 @@
"""ICMS Context Engine V1: ST applicability, ST withholding evidence, product role, CSOSN decision V2.
Pure functions, no I/O, no writes (every output has write_eligible=False). History is only used to
validate/flag; it is never an input to a decision.
"""
ICMS_CONTEXT_ENGINE_VERSION = "icms-context-v1"
def _digits(v):
return "".join(c for c in str(v or "") if c.isdigit())
def st_status(ncm_status, mva_status, cest_status, erp_cest, ba_item_cest, ba_ncm_sh_len):
"""ST_CONFIRMED needs a unique BA Anexo 1 item whose CEST the ERP product itself carries
(or a confirmed CEST). ST_NOT_APPLICABLE needs BOTH official lists (BA Anexo 1 and CEST) silent
for a valid NCM. A CEST *candidate* never confirms ST."""
if ncm_status != "EXACT_NCM":
return "ST_UNKNOWN", "NCM_NOT_VALID"
if mva_status == "MVA_RESOLVED":
if cest_status == "CEST_CONFIRMED" or (_digits(erp_cest) and _digits(erp_cest) == _digits(ba_item_cest)):
return "ST_CONFIRMED", "UNIQUE_BA_ANNEX1_ITEM_CORROBORATED_BY_CEST"
return "ST_CANDIDATE", "UNIQUE_BA_ANNEX1_ITEM_WITHOUT_CEST_CORROBORATION"
if mva_status == "MVA_CANDIDATES":
return "ST_CANDIDATE", "MULTIPLE_BA_ANNEX1_ITEMS"
if mva_status == "NO_MVA_MAPPING_FOUND":
if cest_status == "NO_CEST_MAPPING_FOUND":
return "ST_NOT_APPLICABLE", "NCM_ABSENT_FROM_BA_ANNEX1_AND_CEST_LIST"
return "ST_UNKNOWN", "CEST_LISTED_BUT_BA_ANNEX1_ABSENT"
return "ST_UNKNOWN", "NO_ST_EVIDENCE"
def withheld_status(purchase_rows):
"""purchase_rows: entrada item rows for the product with keys icms_retido (S/N), valor_retido, sub_valor.
CONFIRMED only with positive purchase-side evidence; explicit 'N'/0 is NOT_CONFIRMED; no rows UNKNOWN."""
if not purchase_rows:
return "ST_WITHHOLDING_UNKNOWN", "NO_PURCHASE_ITEM_FOR_PRODUCT"
for r in purchase_rows:
if str(r.get("icms_retido") or "").strip().upper() == "S" or float(r.get("valor_retido") or 0) > 0 or float(r.get("sub_valor") or 0) > 0:
return "ST_ALREADY_WITHHELD_CONFIRMED", "PURCHASE_ITEM_SHOWS_WITHHELD_ICMS_ST"
return "ST_ALREADY_WITHHELD_NOT_CONFIRMED", "PURCHASE_ITEMS_FLAG_ICMS_RETIDO_N_AND_ZERO_ST_VALUES"
def product_role(erp_tipo, prod_cozinha, has_recipe, sold_as_composition, service_flag, purchase_evidence):
ev = []
if service_flag:
return "SERVICE", ["PRODUTO_SERVICO"]
if str(erp_tipo or "").strip().upper() == "MATERIA-PRIMA":
return "INPUT", ["C000025.TIPO=MATERIA-PRIMA"]
own = []
if prod_cozinha: own.append("PROD_COZINHA=1")
if has_recipe: own.append("RECEITA_INGREDIENTES")
if sold_as_composition: own.append("C000032.TIPO=COMPOSICAO")
if own and purchase_evidence:
return "UNKNOWN", own + ["ALSO_HAS_PURCHASE_EVIDENCE_CONFLICT"]
if own:
return "OWN_PRODUCTION", own
if str(erp_tipo or "").strip().upper() == "VENDA" and purchase_evidence:
return "RESALE", ["C000025.TIPO=VENDA", "PURCHASE_DOCUMENT_EVIDENCE"]
return "UNKNOWN", ev or ["NO_OBJECTIVE_ROLE_EVIDENCE"]
def csosn_v2(current, company_default, ncm_status, st, withheld, role, csosn_domain):
"""States: CSOSN_RESOLVED / CANDIDATE / CONTEXT_REQUIRED / CONFLICT / UNRESOLVED.
RESOLVED only where the fiscal condition is explicitly proven (ST confirmed + withholding confirmed + role
known -> 500). Everything else stays conservative; never assumes 102."""
cur = None if current is None or str(current).strip() == "" else str(current).strip()
out = {"field": "CSOSN", "current_value": cur, "proposed_csosn": None, "write_eligible": False, "engine": ICMS_CONTEXT_ENGINE_VERSION}
def res(state, reason, conf, action, proposed=None, **kw):
return {**out, "status": state, "reason_code": reason, "confidence": conf, "action_class": action, "proposed_csosn": proposed, **kw}
if ncm_status != "EXACT_NCM":
return res("CSOSN_UNRESOLVED", "NCM_NOT_VALID", "NONE", "NO_RULE")
if role in ("INPUT", "SERVICE"):
return res("CSOSN_CONTEXT_REQUIRED", f"ROLE_{role}_NOT_A_MERCHANDISE_SALE_DEFAULT", "NONE", "CONTEXT_REQUIRED")
if st == "ST_CONFIRMED":
if withheld == "ST_ALREADY_WITHHELD_CONFIRMED" and role in ("RESALE",):
state = "CSOSN_RESOLVED"
return res(state, "ST_CONFIRMED_AND_WITHHOLDING_CONFIRMED_RESALE", "HIGH", "ALREADY_CORRECT" if cur == "500" else "AUTO_APPLY_SAFE", "500")
if withheld == "ST_ALREADY_WITHHELD_NOT_CONFIRMED" and cur == "500":
return res("CSOSN_CONFLICT", "CSOSN_500_BUT_PURCHASE_ITEMS_SHOW_NO_ST_WITHHELD", "MEDIUM", "REVIEW_REQUIRED")
if withheld == "ST_ALREADY_WITHHELD_CONFIRMED" and cur != "500":
return res("CSOSN_CONFLICT", "ST_WITHHELD_CONFIRMED_BUT_CSOSN_NOT_500", "MEDIUM", "REVIEW_REQUIRED", "500")
return res("CSOSN_CONTEXT_REQUIRED", "ST_CONFIRMED_WITHHOLDING_OR_ROLE_UNPROVEN", "NONE", "CONTEXT_REQUIRED",
consistency_flag=None if cur == "500" else "ST_CONFIRMED_BUT_CSOSN_NOT_500")
if st in ("ST_CANDIDATE", "ST_UNKNOWN"):
return res("CSOSN_CONTEXT_REQUIRED", f"{st}_APPLICABILITY_UNPROVEN", "NONE", "CONTEXT_REQUIRED")
# ST_NOT_APPLICABLE: taxed vs exempt/not-taxed treatment (102 vs 400/300) has no source
if role in ("RESALE", "OWN_PRODUCTION") and cur == company_default and cur in csosn_domain:
return res("CSOSN_CANDIDATE", "NON_ST_MATCHES_COMPANY_DEFAULT_ICMS_TREATMENT_UNPROVEN", "MEDIUM", "REVIEW_REQUIRED", cur)
if role == "UNKNOWN":
return res("CSOSN_CONTEXT_REQUIRED", "PRODUCT_ROLE_UNKNOWN", "NONE", "CONTEXT_REQUIRED")
return res("CSOSN_CANDIDATE" if cur else "CSOSN_CONTEXT_REQUIRED", "DIFFERS_FROM_OR_LACKS_COMPANY_DEFAULT_TREATMENT_UNPROVEN", "LOW", "REVIEW_REQUIRED" if cur else "CONTEXT_REQUIRED")
def st_conflict_class(cest_status, mva_status, ncm_sh_len, erp_cest, ba_item_cest, erp_cfop, history_emitted_current, non_st_but_erp_5405):
"""Primary label for an ST-related CFOP/CSOSN conflict, plus all evidence flags."""
flags = []
if non_st_but_erp_5405 or (_digits(erp_cest) and _digits(ba_item_cest) and _digits(erp_cest) != _digits(ba_item_cest)):
flags.append("BA_ST_RULE_CONFLICT")
if history_emitted_current:
flags.append("ERP_HISTORY_CONFLICT")
if cest_status == "CEST_CANDIDATES":
flags.append("MULTIPLE_CEST")
if cest_status == "CEST_DESCRIPTION_MISMATCH":
flags.append("DESCRIPTION_MISMATCH")
if ncm_sh_len is not None and ncm_sh_len <= 4:
flags.append("NCM_TOO_BROAD")
return (flags[0] if flags else "NO_EVIDENCE"), flags or ["NO_EVIDENCE"]
+146
View File
@@ -0,0 +1,146 @@
"""Stable installation identity and safe operational metadata helpers."""
from __future__ import annotations
import hashlib
import re
import uuid
from datetime import datetime, timezone
from pathlib import Path
from .config import settings
SECRET_KEYS = {"password", "fb_password", "token", "agent_token", "api_key", "secret", "authorization", "connection_string"}
def utcnow() -> datetime:
return datetime.now(timezone.utc)
def ensure_installation_id(state_path: str, *, new_installation: bool = False) -> str:
"""Persist a UUID beside the local Agent state without using hostname/IP."""
path = Path(state_path)
identity_path = path.with_suffix(path.suffix + ".installation_id")
if identity_path.exists() and not new_installation:
value = identity_path.read_text(encoding="utf-8").strip()
try:
return str(uuid.UUID(value))
except ValueError:
pass
value = str(uuid.uuid4())
identity_path.parent.mkdir(parents=True, exist_ok=True)
temporary = identity_path.with_suffix(identity_path.suffix + ".tmp")
temporary.write_text(value + "\n", encoding="utf-8")
temporary.replace(identity_path)
return value
def mask_cnpj(value: str | None) -> str | None:
digits = re.sub(r"\D", "", value or "")
if len(digits) != 14:
return None if not value else "***"
return f"{digits[:2]}.***.***/{digits[8:12]}-{digits[12:]}"
def sanitize_database_path(value: str | None) -> str | None:
if not value:
return None
normalized = str(value).replace("\\", "/")
return normalized.rsplit("/", 1)[-1][:255]
def _safe_value(key: str, value):
if key.lower() in SECRET_KEYS:
return "[REDACTED]"
if isinstance(value, dict):
return {str(k): _safe_value(str(k), v) for k, v in value.items() if str(k).lower() not in SECRET_KEYS}
if isinstance(value, list):
return [_safe_value(key, item) for item in value[:100]]
return value
def sanitize_registration(payload: dict) -> dict:
allowed = {
"installation_id", "display_name", "customer_code", "notes", "hostname", "windows_version", "machine_fingerprint",
"company_code", "company_name", "company_cnpj", "company_uf", "company_crt", "company_contribuinte_ipi", "firebird_version", "firebird_ods",
"database_identifier", "database_path", "capabilities", "fiscal_engine_version", "dataset_versions", "started_at",
"status", "schema_fingerprint", "agent_version", "last_job_id", "coverage", "health", "event_type", "message", "context",
}
result = {key: _safe_value(key, value) for key, value in payload.items() if key in allowed}
if "company_cnpj" in result:
result["company_cnpj_masked"] = mask_cnpj(result.pop("company_cnpj"))
if "database_path" in result:
result["database_identifier"] = sanitize_database_path(result.pop("database_path"))
if result.get("installation_id"):
result["installation_id"] = str(uuid.UUID(str(result["installation_id"])))
return result
def compute_agent_health(snapshot, now: datetime | None = None, *, supplemental=None) -> tuple[str, str]:
now = now or utcnow()
last_seen = getattr(snapshot, "last_seen_at", None)
if last_seen is None:
return "UNKNOWN", "no heartbeat received"
if last_seen.tzinfo is None:
last_seen = last_seen.replace(tzinfo=timezone.utc)
source = supplemental or snapshot
if any(getattr(candidate, "agent_health", None) == "FAILED" or getattr(candidate, "last_status", None) == "FAILED" for candidate in (snapshot, supplemental) if candidate is not None):
return "FAILED", "persistent technical failure"
offline_after = settings.agent_offline_after_seconds or settings.agent_heartbeat_interval_seconds * 3
if (now - last_seen).total_seconds() > offline_after:
return "OFFLINE", f"last heartbeat exceeds {offline_after}s"
if getattr(source, "failed_products", 0) or getattr(source, "last_job_status", "") == "FAILED":
return "DEGRADED", "recent technical or job failure"
return "ONLINE", "recent heartbeat without critical technical failure"
_VERSION_RE = re.compile(r"^v?(\d+(?:\.\d+)*)(?:[-_.]?(rc|beta|alpha|b|a)[-_.]?(\d*))?$", re.IGNORECASE)
_PRE_RANK = {"alpha": -3, "a": -3, "beta": -2, "b": -2, "rc": -1}
def parse_version(value: str | None):
"""Comparable tuple for '1.1.7', '1.0.0rc1', ... (a pre-release sorts below its final); None if unparsable."""
match = _VERSION_RE.match((value or "").strip())
if not match:
return None
parts = [int(x) for x in match.group(1).split(".")]
while len(parts) < 4:
parts.append(0)
pre = (_PRE_RANK[match.group(2).lower()], int(match.group(3) or 0)) if match.group(2) else (0, 0)
return (*parts, *pre)
def reference_agent_version(session) -> str:
"""The version an Agent should be running: the highest PUBLISHED Windows Agent release (REVOKED/DRAFT are
never a target). Falls back to the configured server version when no release is published."""
from sqlalchemy import select
from .models import AgentRelease
best, best_key = None, None
for version in session.scalars(select(AgentRelease.version).where(AgentRelease.status == "PUBLISHED", AgentRelease.platform == "windows")):
key = parse_version(version)
if key is not None and (best_key is None or key > best_key):
best, best_key = version, key
return best or settings.agent_server_version
def compute_version_drift(agent_version: str | None, reference: str | None = None) -> str:
"""CURRENT when the Agent is at (or ahead of) the reference release, OUTDATED when behind it."""
if not agent_version:
return "UNKNOWN"
reference = reference or settings.agent_server_version
have, want = parse_version(agent_version), parse_version(reference)
if have is None or want is None:
return "CURRENT" if agent_version == reference else "OUTDATED"
return "CURRENT" if have >= want else "OUTDATED"
def outdated_versions(session, reference: str) -> list[str]:
"""Distinct Agent versions in use that are behind the reference (used to keep SQL filters exact)."""
from sqlalchemy import select
from .models import Agent
used = [v for v in session.scalars(select(Agent.agent_version).where(Agent.agent_version.is_not(None)).distinct())]
return [v for v in used if compute_version_drift(v, reference) == "OUTDATED"]
+155
View File
@@ -0,0 +1,155 @@
"""Job queue for Agent orchestration: create (idempotent per kind), claim one job at
a time per Agent, complete, fail. Also the central sync scheduler helpers.
Every fiscal job is READ-ONLY on the ERP (INITIAL_SYNC, SYNC_INCREMENTAL,
REPROCESS_ALL, REPROCESS_PRODUCTS); automation never authorizes Firebird writes.
"""
from datetime import datetime, timedelta, timezone
from sqlalchemy import select, update, func, text
from .config import settings
from .models import Installation, Job
FULL_KINDS = ("INITIAL_SYNC", "REPROCESS_ALL") # read every product, reclassify all
SYNC_KINDS = FULL_KINDS + ("SYNC_INCREMENTAL",) # jobs that count as "a sync happened"
_VALID_KINDS = set(SYNC_KINDS) | {"REPROCESS_PRODUCTS"}
_MAX_ERROR_LENGTH = 2000
STALE_RUNNING_AFTER = timedelta(hours=1) # an Agent that died mid-job must not block the queue forever
def _aware(value):
if value is not None and value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value
def create_job(session, agent, kind, payload=None):
"""Create a job. Sync-type jobs are idempotent: if this Agent already has a
PENDING/RUNNING job that covers the request, that job is returned instead
(no duplicates, never two full jobs at once)."""
if kind not in _VALID_KINDS:
raise ValueError(f"unsupported job kind: {kind}")
if kind in ("REPROCESS_ALL", "REPROCESS_PRODUCTS", "SYNC_INCREMENTAL"):
_flag_enrichment_reevaluation(session, agent) # "Reprocessar"/"Sincronizar" also re-evaluate the tracked, not-yet-enriched products (KB/cache first), even when an equivalent job is already queued
session.commit()
if kind in SYNC_KINDS:
# A full job subsumes an incremental one, not the other way round: asking for a
# full reprocess while only an incremental is queued must still queue the full.
blocking = FULL_KINDS if kind in FULL_KINDS else SYNC_KINDS
existing = session.scalar(select(Job).where(Job.agent_id == agent.id, Job.kind.in_(blocking), Job.status.in_(("PENDING", "RUNNING"))).order_by(Job.id).limit(1))
if existing is not None:
return existing
job = Job(agent_id=agent.id, kind=kind, status="PENDING", payload=payload or {})
session.add(job)
session.commit()
return job
def _flag_enrichment_reevaluation(session, agent):
try:
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
if installation is not None and installation.features_configured and installation.enrichment_mode != "DISABLED":
from .enrichment.status import mark_for_reevaluation
mark_for_reevaluation(session, installation.id)
except Exception: # noqa: BLE001 - never let the enrichment bookkeeping break job creation
pass
def recover_stale_running(session, agent, now=None):
now = now or datetime.now(timezone.utc)
cutoff = now - STALE_RUNNING_AFTER
stale = session.scalars(select(Job).where(Job.agent_id == agent.id, Job.status == "RUNNING")).all()
changed = False
for job in stale:
if job.started_at is not None and _aware(job.started_at) < cutoff:
job.status, job.finished_at, job.error = "FAILED", now, "stale: Agent stopped reporting while RUNNING"
changed = True
if changed:
session.commit()
def claim_next_job(session, agent):
"""Atomically claim the oldest PENDING job for this agent only — and only when
processing is enabled centrally and no other job of this Agent is RUNNING
(one fiscal job at a time per Firebird)."""
installation = session.scalar(select(Installation).where(Installation.agent_id == agent.id))
if installation is not None and (not installation.processing_enabled or not installation.features_configured or not installation.corrector_enabled):
return None # WAITING_FEATURE_CONFIGURATION / corrector switched off centrally: nothing is handed to the Agent
recover_stale_running(session, agent)
if session.scalar(select(Job.id).where(Job.agent_id == agent.id, Job.status == "RUNNING").limit(1)) is not None:
return None
pending = session.execute(
select(Job.id, Job.kind)
.where(Job.agent_id == agent.id, Job.status == "PENDING")
.order_by(Job.created_at.asc(), Job.id.asc())
.limit(10)
).all()
if not pending:
return None
limit = settings.heavy_job_concurrency_limit
heavy_slot_free = True
if limit > 0 and any(kind in FULL_KINDS for _, kind in pending):
if session.bind is not None and session.bind.dialect.name == "postgresql":
session.execute(text("SELECT pg_advisory_xact_lock(7305001)")) # serialize slot accounting across concurrent pollers; released at commit
running_heavy = session.scalar(select(func.count()).select_from(Job).where(Job.status == "RUNNING", Job.kind.in_(FULL_KINDS),
Job.started_at > datetime.now(timezone.utc) - STALE_RUNNING_AFTER)) or 0
heavy_slot_free = running_heavy < limit
# oldest job this Agent may run now: a light job is never held behind a heavy one that is waiting for a slot
candidate = next((job_id for job_id, kind in pending if kind not in FULL_KINDS or heavy_slot_free), None)
if candidate is None:
session.rollback() # releases the advisory lock; the heavy job stays PENDING (queue) and the Agent keeps polling
return None
now = datetime.now(timezone.utc)
result = session.execute(
update(Job)
.where(Job.id == candidate, Job.status == "PENDING")
.values(status="RUNNING", started_at=now, attempts=Job.attempts + 1)
)
session.commit()
if result.rowcount != 1:
return None # another poller won the race; caller can poll again
return session.get(Job, candidate)
def complete_job(session, job, result_summary):
result = session.execute(
update(Job)
.where(Job.id == job.id, Job.status == "RUNNING")
.values(status="COMPLETED", finished_at=datetime.now(timezone.utc), result_summary=result_summary)
)
session.commit()
return result.rowcount == 1
def fail_job(session, job, error):
sanitized = str(error)[:_MAX_ERROR_LENGTH]
result = session.execute(
update(Job)
.where(Job.id == job.id, Job.status == "RUNNING")
.values(status="FAILED", finished_at=datetime.now(timezone.utc), error=sanitized)
)
session.commit()
return result.rowcount == 1
def schedule_automatic_sync(session, agent, installation, snapshot, now=None):
"""Called from the Agent heartbeat. Central scheduler, idempotent:
- first heartbeat of an installation with no initial coverage and no sync job
ever created -> exactly ONE INITIAL_SYNC;
- otherwise, if processing is enabled and the last sync attempt is older than
the installation's interval, and nothing is queued/running -> ONE SYNC_INCREMENTAL.
Returns the created job or None."""
if installation is None or not installation.processing_enabled or not installation.features_configured or not installation.corrector_enabled:
return None
now = now or datetime.now(timezone.utc)
sync_jobs = session.scalars(select(Job).where(Job.agent_id == agent.id, Job.kind.in_(SYNC_KINDS)).order_by(Job.id.desc())).all()
if any(job.status in ("PENDING", "RUNNING") for job in sync_jobs):
return None
never_synced = not any(job.kind == "INITIAL_SYNC" or job.kind == "REPROCESS_ALL" for job in sync_jobs) and not int(getattr(snapshot, "total_products", 0) or 0)
if never_synced and not sync_jobs:
return create_job(session, agent, "INITIAL_SYNC", {"reason": "INITIAL_SYNC"})
last_attempt = max((_aware(job.finished_at or job.created_at) for job in sync_jobs if (job.finished_at or job.created_at)), default=None)
if last_attempt is not None and (now - last_attempt).total_seconds() >= installation.sync_interval_seconds:
return create_job(session, agent, "SYNC_INCREMENTAL", {"reason": "SCHEDULED"})
return None
+65
View File
@@ -0,0 +1,65 @@
"""FastAPI application factory."""
import logging
import sys
from collections.abc import Callable
from fastapi import FastAPI, HTTPException, Request
from fastapi.staticfiles import StaticFiles
from .config import settings
from .db import SessionLocal, database_is_available
from .api import router
from .agent_distribution import router as agent_distribution_router
from .control_api import router as control_api_router
from .web import router as web_router
from .version import GTI_FISCAL_VERSION
logging.basicConfig(level=getattr(logging, settings.log_level.upper(), logging.INFO))
logger = logging.getLogger("gti_fiscal_hub")
def create_app(database_probe: Callable[[], bool] = database_is_available, session_factory=SessionLocal, bootstrap_token: str | None = None) -> FastAPI:
app = FastAPI(title="GTI Fiscal Hub", version=GTI_FISCAL_VERSION)
if settings.environment == "production" and settings.agent_dev_token:
raise RuntimeError("AGENT_DEV_TOKEN must not be set when APP_ENV=production (register real Agents with central/scripts/register_agent.py)")
logger.info("starting GTI Fiscal Hub (environment=%s)", settings.environment)
@app.middleware("http")
async def add_security_headers(request: Request, call_next):
response = await call_next(request)
response.headers["X-Content-Type-Options"] = "nosniff"
response.headers["X-Frame-Options"] = "SAMEORIGIN"
response.headers["X-XSS-Protection"] = "1; mode=block"
response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
if getattr(settings, "web_cookie_secure", False):
response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains"
return response
@app.get("/health")
def health() -> dict[str, str]:
if not database_probe():
logger.warning("health check failed: postgres unavailable")
raise HTTPException(status_code=503, detail="postgres unavailable")
return {"status": "ok"}
@app.get("/version")
def version() -> dict[str, str]:
from .classification import DECISION_VERSION, ENGINE_VERSION
from .version import API_COMPAT_MAJOR_MINOR, GTI_FISCAL_VERSION
return {"version": GTI_FISCAL_VERSION, "api_compat": API_COMPAT_MAJOR_MINOR, "engine_version": ENGINE_VERSION, "decision_version": DECISION_VERSION}
app.include_router(router(session_factory, settings.agent_dev_token if bootstrap_token is None else bootstrap_token))
app.include_router(control_api_router(session_factory))
app.include_router(agent_distribution_router(session_factory))
app.include_router(web_router(session_factory))
if settings.official_refresh_enabled and session_factory is SessionLocal and "pytest" not in sys.modules:
@app.on_event("startup")
def _start_official_refresh():
from .datasets.updater import start_scheduler
start_scheduler(session_factory)
app.mount("/static", StaticFiles(directory="central/static"), name="static")
return app
app = create_app()
+973
View File
@@ -0,0 +1,973 @@
"""PostgreSQL records owned by the Fiscal Hub."""
from datetime import datetime
from sqlalchemy import Boolean, CheckConstraint, DateTime, ForeignKey, Integer, JSON, String, Text, UniqueConstraint, func
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
class Base(DeclarativeBase):
pass
class Dataset(Base):
"""The Reference Data Registry (Missão 5): every official reference table the Hub knows about,
one row per ingested version. Pre-existing kinds (NCM, TIPI_IPI, CEST, ...) keep their original
auto-activate behavior in central/app/datasets/service.py, untouched by this mission - the new
DRAFT->admin-approves->ACTIVE workflow applies to datasets registered through the new
scripts/ingest_reference_dataset.py CLI / the admin "Tabelas oficiais" screen going forward."""
__tablename__ = "datasets"
__table_args__ = (UniqueConstraint("provider", "sha256", name="uq_dataset_provider_hash"),)
id: Mapped[int] = mapped_column(primary_key=True)
kind: Mapped[str] = mapped_column(String(32), index=True)
provider: Mapped[str] = mapped_column(String(128))
sha256: Mapped[str] = mapped_column(String(64))
raw_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
version_label: Mapped[str] = mapped_column(String(256))
act: Mapped[str] = mapped_column(Text)
source_url: Mapped[str] = mapped_column(Text)
downloaded_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
status: Mapped[str] = mapped_column(String(32), index=True) # ACTIVE | SUPERSEDED (legacy kinds) | DRAFT | APPROVED | VIGENTE | OBSOLETO | REJECTED (new registry workflow)
record_count: Mapped[int] = mapped_column(Integer, default=0)
licence_terms: Mapped[str | None] = mapped_column(Text, nullable=True)
valid_from: Mapped[str | None] = mapped_column(String(32), nullable=True)
valid_to: Mapped[str | None] = mapped_column(String(32), nullable=True)
file_size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
raw_file_path: Mapped[str | None] = mapped_column(Text, nullable=True) # local path only (sources/raw/... , gitignored) - never the file content in Postgres
parse_errors: Mapped[dict] = mapped_column(JSON, default=dict)
submitted_by: Mapped[str | None] = mapped_column(String(128), nullable=True)
approved_by: Mapped[str | None] = mapped_column(String(128), nullable=True)
approved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
class NcmEntry(Base):
__tablename__ = "ncm_entries"
__table_args__ = (UniqueConstraint("dataset_id", "code", "valid_from", "valid_to", name="uq_ncm_versioned_entry"),)
id: Mapped[int] = mapped_column(primary_key=True)
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
code: Mapped[str] = mapped_column(String(16), index=True)
description: Mapped[str] = mapped_column(Text)
valid_from: Mapped[str] = mapped_column(String(32))
valid_to: Mapped[str] = mapped_column(String(32))
class LegacyNcmEntry(Base):
__tablename__ = "legacy_ncm_entries"
code: Mapped[str] = mapped_column(String(16), primary_key=True)
source: Mapped[str] = mapped_column(Text)
source_date: Mapped[str] = mapped_column(String(32))
reason: Mapped[str] = mapped_column(Text)
successor_code: Mapped[str | None] = mapped_column(String(16), nullable=True)
class CestEntry(Base):
__tablename__ = "cest_entries"
__table_args__ = (UniqueConstraint("dataset_id", "code", name="uq_cest_versioned_entry"),)
id: Mapped[int] = mapped_column(primary_key=True)
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
code: Mapped[str] = mapped_column(String(16), index=True)
description: Mapped[str] = mapped_column(Text)
annex: Mapped[str] = mapped_column(String(8))
class CestNcmEntry(Base):
__tablename__ = "cest_ncm_entries"
__table_args__ = (UniqueConstraint("cest_entry_id", "ncm_code", name="uq_cest_ncm_link"),)
id: Mapped[int] = mapped_column(primary_key=True)
cest_entry_id: Mapped[int] = mapped_column(ForeignKey("cest_entries.id"), index=True)
ncm_code: Mapped[str] = mapped_column(String(16), index=True)
class TipiEntry(Base):
__tablename__ = "tipi_entries"
id: Mapped[int] = mapped_column(primary_key=True)
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
ncm: Mapped[str] = mapped_column(String(8), index=True)
ex: Mapped[str | None] = mapped_column(String(32), nullable=True)
description: Mapped[str] = mapped_column(Text)
ipi_rate: Mapped[float | None] = mapped_column(nullable=True)
ipi_rate_text: Mapped[str | None] = mapped_column(String(32), nullable=True)
source_row: Mapped[int] = mapped_column(Integer)
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
class BaStMvaEntry(Base):
__tablename__ = "ba_st_mva_entries"
id: Mapped[int] = mapped_column(primary_key=True)
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
item: Mapped[str] = mapped_column(String(32))
segment: Mapped[str | None] = mapped_column(Text, nullable=True)
cest: Mapped[str | None] = mapped_column(String(16), index=True, nullable=True)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
mva_original: Mapped[str | None] = mapped_column(String(32), nullable=True)
mva_adjusted: Mapped[str | None] = mapped_column(String(32), nullable=True)
agreement_info: Mapped[str | None] = mapped_column(Text, nullable=True)
source_page: Mapped[int] = mapped_column(Integer)
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
class BaStMvaNcmSh(Base):
__tablename__ = "ba_st_mva_ncm_sh"
__table_args__ = (UniqueConstraint("entry_id", "ncm_sh", name="uq_ba_st_mva_entry_ncm_sh"),)
id: Mapped[int] = mapped_column(primary_key=True)
entry_id: Mapped[int] = mapped_column(ForeignKey("ba_st_mva_entries.id"), index=True)
ncm_sh: Mapped[str] = mapped_column(String(32), index=True)
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
class FiscalCodeEntry(Base):
"""Small national fiscal code domains sourced from the official NF-e MOC
manual: ORIGEM_MERCADORIA, CST_ICMS, CSOSN, CST_PIS, CST_COFINS."""
__tablename__ = "fiscal_code_entries"
__table_args__ = (UniqueConstraint("dataset_id", "domain", "code", name="uq_fiscal_code_entry"),)
id: Mapped[int] = mapped_column(primary_key=True)
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
domain: Mapped[str] = mapped_column(String(32), index=True)
code: Mapped[str] = mapped_column(String(8), index=True)
description: Mapped[str] = mapped_column(Text)
source_section: Mapped[str] = mapped_column(String(96))
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
class CfopEntry(Base):
"""CFOP (Código Fiscal de Operações e Prestações) — Portal Nacional NF-e."""
__tablename__ = "cfop_entries"
__table_args__ = (UniqueConstraint("dataset_id", "code", name="uq_cfop_entry"),)
id: Mapped[int] = mapped_column(primary_key=True)
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
code: Mapped[str] = mapped_column(String(8), index=True)
title: Mapped[str] = mapped_column(Text)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
effective_from: Mapped[str | None] = mapped_column(String(32), nullable=True)
effective_to: Mapped[str | None] = mapped_column(String(32), nullable=True)
ind_nfe: Mapped[str | None] = mapped_column(String(4), nullable=True)
ind_comunica: Mapped[str | None] = mapped_column(String(4), nullable=True)
ind_transp: Mapped[str | None] = mapped_column(String(4), nullable=True)
ind_devol: Mapped[str | None] = mapped_column(String(4), nullable=True)
ind_retor: Mapped[str | None] = mapped_column(String(4), nullable=True)
ind_anula: Mapped[str | None] = mapped_column(String(4), nullable=True)
ind_remes: Mapped[str | None] = mapped_column(String(4), nullable=True)
ind_comb: Mapped[str | None] = mapped_column(String(4), nullable=True)
ind_exc_ibscbs: Mapped[str | None] = mapped_column(String(4), nullable=True)
class FcpRate(Base):
"""FCP (Fundo de Combate à Pobreza) alíquotas por UF — Portal Nacional NF-e."""
__tablename__ = "fcp_rates"
__table_args__ = (UniqueConstraint("dataset_id", "uf", name="uq_fcp_rate_uf"),)
id: Mapped[int] = mapped_column(primary_key=True)
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
uf_code: Mapped[str] = mapped_column(String(4))
uf: Mapped[str] = mapped_column(String(4), index=True)
uf_name: Mapped[str] = mapped_column(String(64))
aliq_1: Mapped[str | None] = mapped_column(String(32), nullable=True)
aliq_2: Mapped[str | None] = mapped_column(String(32), nullable=True)
aliq_3: Mapped[str | None] = mapped_column(String(32), nullable=True)
observacao: Mapped[str | None] = mapped_column(Text, nullable=True)
class AnpProductCode(Base):
"""cProdANP — official code list used by NF-e, published on the Portal Nacional NF-e."""
__tablename__ = "anp_product_codes"
__table_args__ = (UniqueConstraint("dataset_id", "cprod_anp", name="uq_anp_product_code"),)
id: Mapped[int] = mapped_column(primary_key=True)
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
cprod_anp: Mapped[str] = mapped_column(String(16), index=True)
class NcmUtribEntry(Base):
"""NCM x uTrib (unidade tributável de comércio exterior) — Portal Nacional NF-e."""
__tablename__ = "ncm_utrib_entries"
__table_args__ = (UniqueConstraint("dataset_id", "ncm", name="uq_ncm_utrib_entry"),)
id: Mapped[int] = mapped_column(primary_key=True)
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
ncm: Mapped[str] = mapped_column(String(16), index=True)
effective_from: Mapped[str | None] = mapped_column(String(32), nullable=True)
effective_to: Mapped[str | None] = mapped_column(String(32), nullable=True)
utrib_abbreviation: Mapped[str | None] = mapped_column(String(16), nullable=True)
utrib_description: Mapped[str | None] = mapped_column(Text, nullable=True)
class RtcBiofuelIndexEntry(Base):
"""RTC pBioObrigatorio (índice de biocombustível) by cProdANP — Portal Nacional NF-e."""
__tablename__ = "rtc_biofuel_index_entries"
id: Mapped[int] = mapped_column(primary_key=True)
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
cprod_anp: Mapped[str | None] = mapped_column(String(16), index=True, nullable=True)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
p_bio_percent: Mapped[float | None] = mapped_column(nullable=True)
effective_from: Mapped[str | None] = mapped_column(String(32), nullable=True)
effective_to: Mapped[str | None] = mapped_column(String(32), nullable=True)
is_historical: Mapped[bool] = mapped_column(Boolean, default=False)
class RtcCredPresEntry(Base):
"""RTC cCredPres (crédito presumido IBS/CBS), LC 214/2025 — Portal Nacional NF-e.
Keeps every official field distinctly so future automatic determination
doesn't require re-parsing the source."""
__tablename__ = "rtc_cred_pres_entries"
__table_args__ = (UniqueConstraint("dataset_id", "code", name="uq_rtc_cred_pres_code"),)
id: Mapped[int] = mapped_column(primary_key=True)
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
code: Mapped[str] = mapped_column(String(8), index=True)
description: Mapped[str] = mapped_column(Text)
legal_basis: Mapped[str | None] = mapped_column(Text, nullable=True)
apropria_via_nf: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
apropria_via_evento: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
ind_deduz_cred_pres: Mapped[str | None] = mapped_column(String(8), nullable=True)
ind_g_cbs_cred_pres: Mapped[str | None] = mapped_column(String(8), nullable=True)
ind_g_ibs_cred_pres: Mapped[str | None] = mapped_column(String(8), nullable=True)
aliquota_cbs_text: Mapped[str | None] = mapped_column(Text, nullable=True)
aliquota_ibs_text: Mapped[str | None] = mapped_column(Text, nullable=True)
p_aliq_cred_pres_cbs: Mapped[str | None] = mapped_column(Text, nullable=True)
p_aliq_cred_pres_ibs: Mapped[str | None] = mapped_column(Text, nullable=True)
p_red_transicao_ibs: Mapped[str | None] = mapped_column(Text, nullable=True)
# Free text in some rows (e.g. "cClass de alíquota zero"), not always the numeric code.
cclass_nota_referenciada: Mapped[str | None] = mapped_column(Text, nullable=True)
d_ini_vig_cbs: Mapped[str | None] = mapped_column(String(32), nullable=True)
d_fim_vig_cbs: Mapped[str | None] = mapped_column(String(32), nullable=True)
d_ini_vig_ibs: Mapped[str | None] = mapped_column(String(32), nullable=True)
d_fim_vig_ibs: Mapped[str | None] = mapped_column(String(32), nullable=True)
class Tenant(Base):
"""A client. GTI Gerenciador (the panel) is the single source of truth —
see central/app/client_sync.py. `source='GTI_GERENCIADOR'` rows are
upserted by that sync, keyed by `external_client_id` (Gerenciador's
CLIENTE.ID, the stable technical PK — CLIENTE.CODIGO is editable and
never used as identity, same reasoning as FUNCIONARIO_PAINEL.ID in SSO).
`source='LOCAL'` rows predate this sync (dev/test tenants) and are never
offered for a new installation."""
__tablename__ = "tenants"
__table_args__ = (UniqueConstraint("source", "external_client_id", name="uq_tenant_external_identity"),)
id: Mapped[int] = mapped_column(primary_key=True)
name: Mapped[str] = mapped_column(String(128), unique=True)
source: Mapped[str] = mapped_column(String(16), default="LOCAL", server_default="LOCAL") # LOCAL | GTI_GERENCIADOR
external_client_id: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
external_code: Mapped[str | None] = mapped_column(String(16), nullable=True)
external_cnpj: Mapped[str | None] = mapped_column(String(20), nullable=True) # digits only, for the client selector search
active: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
class Agent(Base):
__tablename__ = "agents"
id: Mapped[int] = mapped_column(primary_key=True)
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True) # company_id
name: Mapped[str] = mapped_column(String(128))
token_hash: Mapped[str] = mapped_column(String(64), unique=True)
enabled: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
last_seen_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_status: Mapped[str | None] = mapped_column(String(32), nullable=True)
schema_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
agent_version: Mapped[str | None] = mapped_column(String(32), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class Installation(Base):
__tablename__ = "installations"
__table_args__ = (UniqueConstraint("installation_id", name="uq_installation_identity"), UniqueConstraint("agent_id", name="uq_installation_agent"))
id: Mapped[int] = mapped_column(primary_key=True)
agent_id: Mapped[int] = mapped_column(ForeignKey("agents.id"), index=True)
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
installation_id: Mapped[str] = mapped_column(String(36), index=True)
display_name: Mapped[str] = mapped_column(String(160), default="", server_default="")
customer_code: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
notes: Mapped[str | None] = mapped_column(Text, nullable=True)
hostname: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
windows_version: Mapped[str | None] = mapped_column(String(128), nullable=True)
machine_fingerprint: Mapped[str | None] = mapped_column(String(128), nullable=True)
company_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
company_name: Mapped[str | None] = mapped_column(String(160), nullable=True)
company_cnpj_masked: Mapped[str | None] = mapped_column(String(24), nullable=True)
company_uf: Mapped[str | None] = mapped_column(String(2), nullable=True)
company_crt: Mapped[str | None] = mapped_column(String(16), nullable=True)
company_contribuinte_ipi: Mapped[str | None] = mapped_column(String(1), nullable=True) # C000004.CONTRIBUINTE_IPI, same auto-discovery as company_crt
firebird_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
firebird_ods: Mapped[str | None] = mapped_column(String(32), nullable=True)
database_identifier: Mapped[str | None] = mapped_column(String(255), nullable=True)
capabilities: Mapped[dict] = mapped_column(JSON, default=dict, server_default="{}")
fiscal_engine_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
dataset_versions: Mapped[dict] = mapped_column(JSON, default=dict, server_default="{}")
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_seen_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
last_status: Mapped[str | None] = mapped_column(String(32), nullable=True)
# Central execution control (no local .env toggle): the Hub decides whether this
# installation's Agent executes jobs, and how often the automatic incremental sync runs.
processing_enabled: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
sync_interval_seconds: Mapped[int] = mapped_column(Integer, default=1800, server_default="1800")
last_sync_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_sync_changed: Mapped[int | None] = mapped_column(Integer, nullable=True)
# RTC auto-apply is authorized centrally per installation and is OFF until an operator explicitly turns it on (the Agent has its own second switch).
rtc_auto_apply_authorized: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") # legacy (0027); rtc_write_mode is the source of truth
rtc_write_mode: Mapped[str] = mapped_column(String(24), default="DISABLED", server_default="DISABLED") # DISABLED | DRY_RUN | APPROVAL_REQUIRED | ENABLED
# Same model as rtc_write_mode, an entirely separate switch: authorizes ONLY the CFOP_IPPT_WRITE_V1
# capability (gti_fiscal_agent.cfop_ippt_auto_apply), never RTC. DISABLED by default for every
# installation - an operator must explicitly turn it on, per installation.
cfop_ippt_write_mode: Mapped[str] = mapped_column(String(24), default="DISABLED", server_default="DISABLED") # DISABLED | DRY_RUN | APPROVAL_REQUIRED | ENABLED
# Per-installation FUNCTIONS (central, no local switch). Existing installations keep working exactly as before (corrector on, features configured);
# a NEW installation starts WAITING_FEATURE_CONFIGURATION (features_configured=False): it heartbeats but processes nothing until an admin chooses.
corrector_enabled: Mapped[bool] = mapped_column(Boolean, default=False, server_default="true")
enrichment_mode: Mapped[str] = mapped_column(String(16), default="DISABLED", server_default="DISABLED") # DISABLED | DRY_RUN | ENABLED
features_configured: Mapped[bool] = mapped_column(Boolean, default=False, server_default="true")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class AgentSnapshot(Base):
__tablename__ = "agent_snapshots"
id: Mapped[int] = mapped_column(primary_key=True)
installation_id: Mapped[int] = mapped_column(ForeignKey("installations.id"), unique=True, index=True)
total_products: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
fiscally_complete: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
safe_blocked: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
failed_products: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
coverage_percent: Mapped[float] = mapped_column(default=0.0, server_default="0")
blockers: Mapped[dict] = mapped_column(JSON, default=dict, server_default="{}")
last_job_status: Mapped[str | None] = mapped_column(String(16), nullable=True)
last_job_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_successful_job_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
agent_health: Mapped[str] = mapped_column(String(16), default="UNKNOWN", server_default="UNKNOWN", index=True)
health_reason: Mapped[str | None] = mapped_column(Text, nullable=True)
version_drift: Mapped[str] = mapped_column(String(16), default="UNKNOWN", server_default="UNKNOWN", index=True)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), index=True)
class AgentEvent(Base):
__tablename__ = "agent_events"
id: Mapped[int] = mapped_column(primary_key=True)
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
installation_id: Mapped[int] = mapped_column(ForeignKey("installations.id"), index=True)
agent_id: Mapped[int] = mapped_column(ForeignKey("agents.id"), index=True)
severity: Mapped[str] = mapped_column(String(16), index=True)
event_type: Mapped[str] = mapped_column(String(48), index=True)
message: Mapped[str] = mapped_column(Text)
structured_context: Mapped[dict] = mapped_column(JSON, default=dict)
job_id: Mapped[int | None] = mapped_column(ForeignKey("jobs.id"), nullable=True, index=True)
product_code: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
blocker_id: Mapped[int | None] = mapped_column(ForeignKey("blocked_products.id"), nullable=True, index=True)
is_audit: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false", index=True)
occurred_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), index=True)
class AdminUser(Base):
"""Human principal of the control panel.
LOCAL: break-glass account with a password (create_admin.py).
GTI_GERENCIADOR: collaborator provisioned by SSO (central/app/sso.py) — no
password at all; identified by the panel's immutable subject.
"""
__tablename__ = "admin_users"
__table_args__ = (
UniqueConstraint("external_subject", name="uq_admin_users_external_subject"),
CheckConstraint(
"(auth_source = 'LOCAL' AND password_hash IS NOT NULL AND external_subject IS NULL)"
" OR (auth_source = 'GTI_GERENCIADOR' AND password_hash IS NULL AND external_subject IS NOT NULL)",
name="ck_admin_users_auth_source",
),
)
id: Mapped[int] = mapped_column(primary_key=True)
username: Mapped[str] = mapped_column(String(128), unique=True, index=True)
password_hash: Mapped[str | None] = mapped_column(String(255), nullable=True)
is_active: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
last_login_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
auth_source: Mapped[str] = mapped_column(String(16), default="LOCAL", server_default="LOCAL")
external_subject: Mapped[str | None] = mapped_column(String(128), nullable=True)
display_name: Mapped[str | None] = mapped_column(String(160), nullable=True)
external_role: Mapped[str | None] = mapped_column(String(32), nullable=True)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class SsoUsedAssertion(Base):
"""jti of every accepted SSO assertion, kept past its expiry to reject replays."""
__tablename__ = "sso_used_assertions"
jti: Mapped[str] = mapped_column(String(64), primary_key=True)
subject: Mapped[str] = mapped_column(String(128))
used_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
class WebSession(Base):
__tablename__ = "web_sessions"
id: Mapped[int] = mapped_column(primary_key=True)
admin_user_id: Mapped[int] = mapped_column(ForeignKey("admin_users.id"), index=True)
token_hash: Mapped[str] = mapped_column(String(64), unique=True, index=True)
csrf_hash: Mapped[str] = mapped_column(String(64))
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
user_agent: Mapped[str | None] = mapped_column(String(512), nullable=True)
remote_addr: Mapped[str | None] = mapped_column(String(64), nullable=True)
class AuditEvent(Base):
__tablename__ = "audit_events"
id: Mapped[int] = mapped_column(primary_key=True)
admin_user_id: Mapped[int | None] = mapped_column(ForeignKey("admin_users.id"), nullable=True, index=True)
tenant_id: Mapped[int | None] = mapped_column(ForeignKey("tenants.id"), nullable=True, index=True)
action: Mapped[str] = mapped_column(String(64), index=True)
target_type: Mapped[str] = mapped_column(String(64))
target_id: Mapped[str] = mapped_column(String(128))
context: Mapped[dict] = mapped_column(JSON, default=dict)
severity: Mapped[str] = mapped_column(String(16), default="INFO", server_default="INFO")
occurred_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), index=True)
class AgentRelease(Base):
"""A published build of the Windows Agent package. The universal binary/zip is
shared by every installation; storage_path points at it on the Hub's local
release storage (never inside PostgreSQL)."""
__tablename__ = "agent_releases"
__table_args__ = (UniqueConstraint("version", "platform", "architecture", name="uq_agent_release_version"),)
id: Mapped[int] = mapped_column(primary_key=True)
version: Mapped[str] = mapped_column(String(32), index=True)
platform: Mapped[str] = mapped_column(String(16), default="windows", server_default="windows")
architecture: Mapped[str] = mapped_column(String(16), default="x64", server_default="x64")
filename: Mapped[str] = mapped_column(String(255))
size_bytes: Mapped[int] = mapped_column(Integer)
sha256: Mapped[str] = mapped_column(String(64), index=True)
storage_path: Mapped[str] = mapped_column(String(512))
status: Mapped[str] = mapped_column(String(16), default="DRAFT", server_default="DRAFT", index=True) # DRAFT | PUBLISHED | REVOKED
release_notes: Mapped[str | None] = mapped_column(Text, nullable=True)
created_by: Mapped[int | None] = mapped_column(ForeignKey("admin_users.id"), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
class AgentEnrollment(Base):
"""A pending (or completed) Agent installation: created by an operator in the
control panel before any real Agent exists. Carries two distinct, hashed,
single-use, short-TTL secrets — a download token (for the shareable package
link) and an enrollment token (embedded in the bundle, consumed once by
POST /v1/agents/enroll). Never stores either secret in plaintext."""
__tablename__ = "agent_enrollments"
id: Mapped[int] = mapped_column(primary_key=True)
enrollment_uuid: Mapped[str] = mapped_column(String(36), unique=True, index=True)
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
display_name: Mapped[str] = mapped_column(String(160))
expected_hostname: Mapped[str | None] = mapped_column(String(255), nullable=True)
release_id: Mapped[int] = mapped_column(ForeignKey("agent_releases.id"), index=True)
status: Mapped[str] = mapped_column(String(16), default="PENDING", server_default="PENDING", index=True)
# PENDING | LINK_CREATED | DOWNLOADED | ENROLLED | EXPIRED | REVOKED | FAILED
created_by: Mapped[int | None] = mapped_column(ForeignKey("admin_users.id"), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
download_token_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
download_token_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
download_max_uses: Mapped[int] = mapped_column(Integer, default=1, server_default="1")
download_uses: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
downloaded_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
enrollment_token_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
enrollment_token_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
enrollment_used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
# The install code typed by the operator into the universal EXE installer.
# Only its SHA-256 is stored (normalize → digest), never the plaintext.
# A new code (rotation) rewrites these three fields; the previous code
# becomes unusable the moment a new one is generated.
install_code_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
install_code_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
install_code_attempts: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
install_code_created_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
machine_fingerprint_seen: Mapped[str | None] = mapped_column(String(128), nullable=True)
hostname_seen: Mapped[str | None] = mapped_column(String(255), nullable=True)
hostname_mismatch: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
agent_id: Mapped[int | None] = mapped_column(ForeignKey("agents.id"), nullable=True, index=True)
# Functions chosen when the installation was created (copied to the Installation at enrollment). Nothing chosen => WAITING_FEATURE_CONFIGURATION.
features_selected: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
feature_corrector: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
feature_enrichment_mode: Mapped[str] = mapped_column(String(16), default="DISABLED", server_default="DISABLED")
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class Job(Base):
__tablename__ = "jobs"
id: Mapped[int] = mapped_column(primary_key=True)
agent_id: Mapped[int] = mapped_column(ForeignKey("agents.id"), index=True)
kind: Mapped[str] = mapped_column(String(32), index=True) # REPROCESS_ALL | REPROCESS_PRODUCTS
status: Mapped[str] = mapped_column(String(16), index=True, default="PENDING", server_default="PENDING")
payload: Mapped[dict] = mapped_column(JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
attempts: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
result_summary: Mapped[dict | None] = mapped_column(JSON, nullable=True)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
class BlockedProduct(Base):
"""Deduplicated safe exception for one tenant/product/field/input."""
__tablename__ = "blocked_products"
__table_args__ = (UniqueConstraint("blocker_fingerprint", name="uq_blocked_product_fingerprint"),)
id: Mapped[int] = mapped_column(primary_key=True)
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
installation_id: Mapped[int | None] = mapped_column(ForeignKey("installations.id"), nullable=True, index=True)
company: Mapped[str] = mapped_column(String(128), index=True)
product_code: Mapped[str] = mapped_column(String(64), index=True)
gtin: Mapped[str | None] = mapped_column(String(16), nullable=True, index=True)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
blocker_type: Mapped[str] = mapped_column(String(48), index=True)
field: Mapped[str] = mapped_column(String(64), index=True)
reason_code: Mapped[str] = mapped_column(String(96))
evidence_summary: Mapped[dict] = mapped_column(JSON, default=dict)
input_fingerprint: Mapped[str] = mapped_column(String(64), index=True)
blocker_fingerprint: Mapped[str] = mapped_column(String(64))
first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
retry_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
next_retry_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
resolution_source: Mapped[str | None] = mapped_column(String(48), nullable=True)
class ClassificationRequest(Base):
__tablename__ = "classification_requests"
id: Mapped[int] = mapped_column(primary_key=True)
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
agent_id: Mapped[int] = mapped_column(ForeignKey("agents.id"), index=True)
fingerprint: Mapped[str] = mapped_column(String(64), unique=True)
payload: Mapped[dict] = mapped_column(JSON)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class ClassificationResult(Base):
__tablename__ = "classification_results"
id: Mapped[int] = mapped_column(primary_key=True)
request_id: Mapped[int] = mapped_column(ForeignKey("classification_requests.id"), unique=True)
result: Mapped[dict] = mapped_column(JSON)
class ClassificationCache(Base):
__tablename__ = "classification_cache"
fingerprint: Mapped[str] = mapped_column(String(64), primary_key=True)
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
result: Mapped[dict] = mapped_column(JSON)
class RtcCstIbsCbsEntry(Base):
"""RTC CST-IBS/CBS (sheet 'CST ...' of the official cClassTrib workbook)."""
__tablename__ = "rtc_cst_ibscbs_entries"
__table_args__ = (UniqueConstraint("dataset_id", "cst", name="uq_rtc_cst_ibscbs"),)
id: Mapped[int] = mapped_column(primary_key=True)
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
cst: Mapped[str] = mapped_column(String(8), index=True)
description: Mapped[str] = mapped_column(Text)
indicators: Mapped[dict] = mapped_column(JSON)
class RtcCClassTribEntry(Base):
"""RTC cClassTrib (sheet 'cClass ...'). Every official column is kept:
key fields typed, all ind_* flags in `indicators` keyed by official header."""
__tablename__ = "rtc_cclasstrib_entries"
__table_args__ = (UniqueConstraint("dataset_id", "cclasstrib", name="uq_rtc_cclasstrib"),)
id: Mapped[int] = mapped_column(primary_key=True)
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
cst: Mapped[str] = mapped_column(String(8), index=True)
cst_description: Mapped[str | None] = mapped_column(Text, nullable=True)
cclasstrib: Mapped[str] = mapped_column(String(8), index=True)
name: Mapped[str | None] = mapped_column(Text, nullable=True)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
lc_redacao: Mapped[str | None] = mapped_column(Text, nullable=True)
lc_214: Mapped[str | None] = mapped_column(Text, nullable=True)
regulamento_cbs: Mapped[str | None] = mapped_column(Text, nullable=True)
regulamento_ibs: Mapped[str | None] = mapped_column(Text, nullable=True)
tipo_aliquota: Mapped[str | None] = mapped_column(String(64), nullable=True)
p_red_ibs: Mapped[str | None] = mapped_column(String(32), nullable=True)
p_red_cbs: Mapped[str | None] = mapped_column(String(32), nullable=True)
d_ini_vig: Mapped[str | None] = mapped_column(String(32), nullable=True)
d_fim_vig: Mapped[str | None] = mapped_column(String(32), nullable=True)
data_atualizacao: Mapped[str | None] = mapped_column(String(32), nullable=True)
anexo: Mapped[str | None] = mapped_column(Text, nullable=True)
link: Mapped[str | None] = mapped_column(Text, nullable=True)
indicators: Mapped[dict] = mapped_column(JSON)
class RtcCbsRateEntry(Base):
"""RTC CBS rate per year (IT 2026.002). rate is NULL when the official
text has no number yet (e.g. 'aguarda legislação')."""
__tablename__ = "rtc_cbs_rate_entries"
__table_args__ = (UniqueConstraint("dataset_id", "year", name="uq_rtc_cbs_rate_year"),)
id: Mapped[int] = mapped_column(primary_key=True)
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
year: Mapped[int] = mapped_column(Integer, index=True)
rate_text: Mapped[str] = mapped_column(Text)
rate: Mapped[float | None] = mapped_column(nullable=True)
class FiscalContextEntry(Base):
"""Operator-supplied fiscal fact (product role, origin, ST retention, treatment, ...). Hierarchy: PRODUCT > GROUP > COMPANY."""
__tablename__ = "fiscal_context_entries"
__table_args__ = (UniqueConstraint("tenant_id", "company_key", "scope", "scope_key", "key", name="uq_fiscal_context_entry"),)
id: Mapped[int] = mapped_column(primary_key=True)
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
company_key: Mapped[str] = mapped_column(String(64), index=True)
scope: Mapped[str] = mapped_column(String(8))
scope_key: Mapped[str] = mapped_column(String(64), default="", server_default="")
key: Mapped[str] = mapped_column(String(48))
value: Mapped[str] = mapped_column(Text)
source: Mapped[str] = mapped_column(String(16), default="OPERATOR", server_default="OPERATOR")
reason: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
# ================= AI semantic layer V1 =================
# The AI layer is NOT a fiscal authority: it only selects among official candidates and
# never writes to Firebird. Knowledge is versioned, provenance-tracked and never deleted.
class KbProduct(Base):
"""GLOBAL product knowledge: only facts inherent to the product (never tenant context)."""
__tablename__ = "kb_products"
id: Mapped[int] = mapped_column(primary_key=True)
fingerprint: Mapped[str] = mapped_column(String(64), unique=True, index=True)
gtin: Mapped[str | None] = mapped_column(String(16), unique=True, nullable=True, index=True)
identity_kind: Mapped[str] = mapped_column(String(16), default="FINGERPRINT") # GTIN | FINGERPRINT | SEMANTIC | UNKNOWN
normalized_description: Mapped[str] = mapped_column(Text, default="")
brand: Mapped[str | None] = mapped_column(Text, nullable=True)
unit: Mapped[str | None] = mapped_column(String(32), nullable=True)
package_quantity: Mapped[str | None] = mapped_column(String(32), nullable=True)
category: Mapped[str | None] = mapped_column(Text, nullable=True)
semantic_attributes: Mapped[dict] = mapped_column(JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class KbKnowledge(Base):
"""Versioned knowledge rows. Unique per (product, key, value): conflicting values for the
same key are kept side by side and surfaced as KNOWLEDGE_CONFLICT, never majority-picked."""
__tablename__ = "kb_knowledge"
__table_args__ = (UniqueConstraint("kb_product_id", "knowledge_key", "value", name="uq_kb_knowledge_value"),)
id: Mapped[int] = mapped_column(primary_key=True)
kb_product_id: Mapped[int] = mapped_column(ForeignKey("kb_products.id"), index=True)
knowledge_key: Mapped[str] = mapped_column(String(48), index=True) # candidate_ncm | validated_ncm | candidate_cest | candidate_ex_tipi | ...
value: Mapped[str] = mapped_column(Text)
source: Mapped[str] = mapped_column(String(32)) # OFFICIAL_DATASET | ERP_OBSERVATION | OPERATOR | AI | AI_PLUS_OFFICIAL_VALIDATION
confidence: Mapped[float] = mapped_column(default=0.0)
evidence: Mapped[dict] = mapped_column(JSON, default=dict)
model: Mapped[str | None] = mapped_column(String(64), nullable=True)
prompt_version: Mapped[str | None] = mapped_column(String(32), nullable=True)
dataset_versions: Mapped[dict] = mapped_column(JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
validated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="OBSERVED", server_default="OBSERVED", index=True) # OBSERVED | CORROBORATED | EXTERNAL_CORROBORATED | VERIFIED_KB | CONFLICT | STALE
class KbObservation(Base):
"""Raw per-tenant ERP observation of a KB product (evidence aggregation only)."""
__tablename__ = "kb_observations"
id: Mapped[int] = mapped_column(primary_key=True)
kb_product_id: Mapped[int] = mapped_column(ForeignKey("kb_products.id"), index=True)
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
company_key: Mapped[str] = mapped_column(String(64), index=True)
erp_code: Mapped[str] = mapped_column(String(32))
database_label: Mapped[str] = mapped_column(String(64))
observed_fields: Mapped[dict] = mapped_column(JSON, default=dict)
source: Mapped[str] = mapped_column(String(32), default="ERP_OBSERVATION")
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class TenantProductContext(Base):
"""TENANT-specific product context: role, ST-withheld, operation profile, operator decisions.
Never promoted to global knowledge."""
__tablename__ = "tenant_product_context"
__table_args__ = (UniqueConstraint("tenant_id", "company_key", "erp_code", name="uq_tenant_product_context"),)
id: Mapped[int] = mapped_column(primary_key=True)
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
company_key: Mapped[str] = mapped_column(String(64), index=True)
erp_code: Mapped[str] = mapped_column(String(32))
kb_product_id: Mapped[int | None] = mapped_column(ForeignKey("kb_products.id"), nullable=True)
product_role: Mapped[str | None] = mapped_column(String(16), nullable=True) # RESALE | OWN_PRODUCTION | INPUT | SERVICE | UNKNOWN
st_withheld: Mapped[str | None] = mapped_column(String(4), nullable=True) # Y | N — never AI-decided
operation_profile: Mapped[dict] = mapped_column(JSON, default=dict)
source: Mapped[str] = mapped_column(String(32), default="ERP_OBSERVATION")
confidence: Mapped[float] = mapped_column(default=0.0)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class AiWorkItem(Base):
"""Deduplicated semantic work item (one knowledge need, many pending cells)."""
__tablename__ = "ai_work_items"
__table_args__ = (UniqueConstraint("dedup_key", name="uq_ai_work_item_dedup"),)
id: Mapped[int] = mapped_column(primary_key=True)
task_type: Mapped[str] = mapped_column(String(24), index=True) # INVALID_NCM | CEST | EX_TIPI | PRODUCT_ROLE
dedup_key: Mapped[str] = mapped_column(String(160))
question_refs: Mapped[list] = mapped_column(JSON, default=list)
payload: Mapped[dict] = mapped_column(JSON, default=dict)
status: Mapped[str] = mapped_column(String(32), default="PENDING", index=True) # PENDING | KB_EXACT_HIT | KB_FINGERPRINT_HIT | HIGH_CONFIDENCE_RETRIEVAL | SINGLE_OFFICIAL_CANDIDATE | LLM_SELECTION_REQUIRED | UNRESOLVED | UNRESOLVED_NO_OFFICIAL_MAPPING | CONFLICT | DONE | SKIPPED
kb_product_id: Mapped[int | None] = mapped_column(ForeignKey("kb_products.id"), nullable=True)
cells_covered: Mapped[int] = mapped_column(Integer, default=0)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class AiRun(Base):
"""One provider execution over a batch of work items, with budget-guard evidence."""
__tablename__ = "ai_runs"
id: Mapped[int] = mapped_column(primary_key=True)
provider: Mapped[str] = mapped_column(String(64))
model: Mapped[str] = mapped_column(String(64))
prompt_version: Mapped[str] = mapped_column(String(32))
work_item_ids: Mapped[list] = mapped_column(JSON, default=list)
input_tokens: Mapped[int] = mapped_column(Integer, default=0)
output_tokens: Mapped[int] = mapped_column(Integer, default=0)
cost_usd: Mapped[float] = mapped_column(default=0.0)
status: Mapped[str] = mapped_column(String(16), default="RUN")
results: Mapped[list] = mapped_column(JSON, default=list)
budget_guard: Mapped[dict] = mapped_column(JSON, default=dict)
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
class AiEvalCache(Base):
"""Persistent LLM result cache: identical (input_fingerprint, candidate_set_hash,
prompt_version, model, dataset_versions) re-execution makes ZERO new calls."""
__tablename__ = "ai_eval_cache"
__table_args__ = (UniqueConstraint("input_fingerprint", "candidate_set_hash", "prompt_version", "model", "dataset_version_ncm", name="uq_ai_eval_cache"),)
id: Mapped[int] = mapped_column(primary_key=True)
input_fingerprint: Mapped[str] = mapped_column(String(64), index=True)
candidate_set_hash: Mapped[str] = mapped_column(String(64))
prompt_version: Mapped[str] = mapped_column(String(32))
model: Mapped[str] = mapped_column(String(64))
dataset_version_ncm: Mapped[str] = mapped_column(String(64))
phase: Mapped[str] = mapped_column(String(16))
answer: Mapped[dict] = mapped_column(JSON)
gold_ncm: Mapped[str | None] = mapped_column(String(8), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class AiUsageLedger(Base):
"""Central persistent usage ledger reserved before any future provider call."""
__tablename__ = "ai_usage_ledger"
id: Mapped[int] = mapped_column(primary_key=True)
provider: Mapped[str] = mapped_column(String(64))
model: Mapped[str] = mapped_column(String(64))
run_id: Mapped[str] = mapped_column(String(128), index=True)
tenant: Mapped[str] = mapped_column(String(128), index=True)
items: Mapped[int] = mapped_column(Integer, default=0)
input_tokens: Mapped[int] = mapped_column(Integer, default=0)
output_tokens: Mapped[int] = mapped_column(Integer, default=0)
cost: Mapped[float] = mapped_column(default=0.0)
timestamp: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), index=True)
class ExternalProductCache(Base):
"""Versioned GTIN cache; legal/provider flags remain explicit metadata."""
__tablename__ = "external_product_cache"
__table_args__ = (UniqueConstraint("gtin", "provider", "provider_record_version", name="uq_external_product_cache_record"),)
id: Mapped[int] = mapped_column(primary_key=True)
gtin: Mapped[str] = mapped_column(String(16), index=True)
provider: Mapped[str] = mapped_column(String(64))
provider_record_version: Mapped[str] = mapped_column(String(64))
retrieved_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
raw_hash: Mapped[str] = mapped_column(String(64))
normalized_data: Mapped[dict] = mapped_column(JSON, default=dict)
cache_allowed: Mapped[bool] = mapped_column(Boolean, default=False)
redistribution_allowed: Mapped[bool] = mapped_column(Boolean, default=False)
class RtcWriteAudit(Base):
"""Append-only record of every attempted C000025.CCLASSTRIB write (applied or aborted). Holds the old value, so each change is revertible by hand."""
__tablename__ = "rtc_write_audits"
id: Mapped[int] = mapped_column(primary_key=True)
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
installation_id: Mapped[int] = mapped_column(ForeignKey("installations.id"), index=True)
agent_id: Mapped[int] = mapped_column(ForeignKey("agents.id"), index=True)
job_id: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
client: Mapped[str | None] = mapped_column(String(255), nullable=True)
product_code: Mapped[str] = mapped_column(String(64), index=True)
field: Mapped[str] = mapped_column(String(32))
outcome: Mapped[str] = mapped_column(String(16), index=True)
reason: Mapped[str | None] = mapped_column(String(96), nullable=True)
old_value: Mapped[str | None] = mapped_column(String(64), nullable=True)
new_value: Mapped[str | None] = mapped_column(String(64), nullable=True)
rule_id: Mapped[str | None] = mapped_column(String(32), nullable=True)
evidence: Mapped[dict] = mapped_column(JSON, default=dict)
official_source_version: Mapped[dict] = mapped_column(JSON, default=dict)
local_catalog_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
written_at: Mapped[str | None] = mapped_column(String(40), nullable=True)
rule_version: Mapped[str | None] = mapped_column(String(32), nullable=True)
company_crt: Mapped[str | None] = mapped_column(String(8), nullable=True)
authorization_origin: Mapped[str | None] = mapped_column(String(48), nullable=True)
description_snapshot: Mapped[str | None] = mapped_column(String(255), nullable=True)
installation_uuid: Mapped[str | None] = mapped_column(String(64), nullable=True)
rtc_status: Mapped[str | None] = mapped_column(String(24), nullable=True)
local_catalog_state: Mapped[str | None] = mapped_column(String(32), nullable=True)
action: Mapped[str | None] = mapped_column(String(16), nullable=True) # APPLY | ROLLBACK
rollback_state: Mapped[str | None] = mapped_column(String(16), nullable=True) # REQUESTED | DONE | BLOCKED (on the original APPLY record)
rollback_of: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
recorded_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), index=True)
class CfopIpptWriteAudit(Base):
"""Append-only record of every attempted C000025.CFOP/IPPT write (applied or aborted) under the
CFOP_IPPT_WRITE_V1 capability. Holds the old value, so each change is revertible by hand."""
__tablename__ = "cfop_ippt_write_audits"
id: Mapped[int] = mapped_column(primary_key=True)
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
installation_id: Mapped[int] = mapped_column(ForeignKey("installations.id"), index=True)
agent_id: Mapped[int] = mapped_column(ForeignKey("agents.id"), index=True)
job_id: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
product_code: Mapped[str] = mapped_column(String(64), index=True)
field: Mapped[str] = mapped_column(String(16)) # CFOP | IPPT only
outcome: Mapped[str] = mapped_column(String(16), index=True) # APPLIED | ABORTED | FAILED | ROLLED_BACK
reason: Mapped[str | None] = mapped_column(String(96), nullable=True)
old_value: Mapped[str | None] = mapped_column(String(16), nullable=True)
new_value: Mapped[str | None] = mapped_column(String(16), nullable=True)
action: Mapped[str | None] = mapped_column(String(16), nullable=True) # APPLY | ROLLBACK
rollback_of: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
written_at: Mapped[str | None] = mapped_column(String(40), nullable=True)
recorded_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), index=True)
class RtcCandidate(Base):
"""Server-side preview of the products the Agent could fill (R3 only). Rows are proposals; only the Agent's re-derivation at write time decides."""
__tablename__ = "rtc_candidates"
__table_args__ = (UniqueConstraint("installation_id", "product_code", name="uq_rtc_candidate_product"),)
id: Mapped[int] = mapped_column(primary_key=True)
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
installation_id: Mapped[int] = mapped_column(ForeignKey("installations.id"), index=True)
product_code: Mapped[str] = mapped_column(String(64))
description: Mapped[str | None] = mapped_column(String(255), nullable=True)
ncm: Mapped[str | None] = mapped_column(String(16), nullable=True)
current_value: Mapped[str | None] = mapped_column(String(16), nullable=True)
proposed: Mapped[str] = mapped_column(String(16))
cst_ibscbs: Mapped[str | None] = mapped_column(String(8), nullable=True)
rule_id: Mapped[str] = mapped_column(String(32))
basis: Mapped[str | None] = mapped_column(String(255), nullable=True)
status: Mapped[str] = mapped_column(String(16), default="PENDING", server_default="PENDING", index=True) # PENDING | APPROVED | REJECTED | APPLIED | STALE
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class ProductEnrichment(Base):
"""Per (installation, product) state of the new-product enrichment. Historical products never get a row (no mass provider use)."""
__tablename__ = "product_enrichments"
__table_args__ = (UniqueConstraint("installation_id", "product_code", name="uq_product_enrichment"),)
id: Mapped[int] = mapped_column(primary_key=True)
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
installation_id: Mapped[int] = mapped_column(ForeignKey("installations.id"), index=True)
product_code: Mapped[str] = mapped_column(String(64))
gtin: Mapped[str | None] = mapped_column(String(16), nullable=True, index=True)
description: Mapped[str | None] = mapped_column(String(255), nullable=True)
status: Mapped[str] = mapped_column(String(32), default="PENDING", server_default="PENDING", index=True)
# PENDING | RESOLVED | APPLIED | PARTIAL | NOT_RESOLVED | CONFLICT | PENDING_PROVIDER_QUOTA | PENDING_PROVIDERS_OFFLINE | INVALID_GTIN
decisions: Mapped[dict] = mapped_column(JSON, default=dict)
fields_applied: Mapped[dict] = mapped_column(JSON, default=dict)
human_overridden: Mapped[dict] = mapped_column(JSON, default=dict)
attempts: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
latency_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
reevaluate: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false", index=True) # set by panel / REPROCESS / SYNC / KB change; consumed by the Agent
last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
ui_status: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True) # human-facing state (never a silent product)
reason_code: Mapped[str | None] = mapped_column(String(48), nullable=True)
detected_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class ProductFieldAudit(Base):
"""Append-only audit of every field the enrichment wrote (or refused to). Never holds secrets."""
__tablename__ = "product_field_audits"
id: Mapped[int] = mapped_column(primary_key=True)
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
installation_id: Mapped[int] = mapped_column(ForeignKey("installations.id"), index=True)
agent_id: Mapped[int] = mapped_column(ForeignKey("agents.id"), index=True)
product_code: Mapped[str] = mapped_column(String(64), index=True)
gtin: Mapped[str | None] = mapped_column(String(16), nullable=True)
description_snapshot: Mapped[str | None] = mapped_column(String(255), nullable=True)
field: Mapped[str] = mapped_column(String(48))
old_value: Mapped[str | None] = mapped_column(String(64), nullable=True)
new_value: Mapped[str | None] = mapped_column(String(64), nullable=True)
rule: Mapped[str | None] = mapped_column(String(64), nullable=True)
kb_status: Mapped[str | None] = mapped_column(String(16), nullable=True)
evidence: Mapped[dict] = mapped_column(JSON, default=dict)
dataset_versions: Mapped[dict] = mapped_column(JSON, default=dict)
authorization_origin: Mapped[str | None] = mapped_column(String(48), nullable=True)
job_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
outcome: Mapped[str] = mapped_column(String(24), default="APPLIED", server_default="APPLIED") # APPLIED | NO_WRITE_REQUIRED | ABORTED | HUMAN_OVERRIDDEN
reason: Mapped[str | None] = mapped_column(String(96), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class ProviderState(Base):
"""Live state of each external product provider (owned by the Hub: quota belongs to the Hub, never to Agents)."""
__tablename__ = "provider_states"
provider: Mapped[str] = mapped_column(String(32), primary_key=True)
configured: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
status: Mapped[str] = mapped_column(String(16), default="NOT_CONFIGURED", server_default="NOT_CONFIGURED") # ONLINE | DEGRADED | RATE_LIMITED | AUTH_ERROR | OFFLINE | NOT_CONFIGURED
quota_kind: Mapped[str] = mapped_column(String(16), default="DOCUMENTED_RATE", server_default="DOCUMENTED_RATE") # REMOTE_QUOTA | LOCAL_ESTIMATE | DOCUMENTED_RATE
daily_limit: Mapped[int | None] = mapped_column(Integer, nullable=True)
remaining: Mapped[int | None] = mapped_column(Integer, nullable=True)
reset_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
rate_per_minute: Mapped[int | None] = mapped_column(Integer, nullable=True)
circuit_open_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
consecutive_failures: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
last_request_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_success_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_error: Mapped[str | None] = mapped_column(String(160), nullable=True)
last_error_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_latency_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class ProviderUsageDaily(Base):
__tablename__ = "provider_usage_daily"
__table_args__ = (UniqueConstraint("provider", "day", name="uq_provider_usage_day"),)
id: Mapped[int] = mapped_column(primary_key=True)
provider: Mapped[str] = mapped_column(String(32), index=True)
day: Mapped[str] = mapped_column(String(10), index=True)
requests: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
success: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
not_found: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
errors: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
rate_limited: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
timeouts: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
cache_hits: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
cache_misses: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
found: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
ncm_found: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
kb_hits: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
class GtinLookupCache(Base):
"""Positive and negative (NOT_FOUND) provider results per GTIN, with TTL. Single place that spares quota across all installations."""
__tablename__ = "gtin_lookup_cache"
__table_args__ = (UniqueConstraint("gtin", "provider", name="uq_gtin_lookup_cache"),)
id: Mapped[int] = mapped_column(primary_key=True)
gtin: Mapped[str] = mapped_column(String(16), index=True)
provider: Mapped[str] = mapped_column(String(32))
found: Mapped[bool] = mapped_column(Boolean, default=False)
data: Mapped[dict] = mapped_column(JSON, default=dict)
fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
class SourceRefreshRun(Base):
"""Observability of official fiscal sources (NCM Siscomex, ...): one row per refresh attempt."""
__tablename__ = "source_refresh_runs"
id: Mapped[int] = mapped_column(primary_key=True)
source: Mapped[str] = mapped_column(String(32), index=True)
source_url: Mapped[str] = mapped_column(Text)
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
status: Mapped[str] = mapped_column(String(16), default="RUNNING", server_default="RUNNING") # RUNNING | UPDATED | UNCHANGED | FAILED
sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
version_label: Mapped[str | None] = mapped_column(String(64), nullable=True)
record_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
error: Mapped[str | None] = mapped_column(String(300), nullable=True)
class KbPromotionAudit(Base):
"""Why a GTIN/NCM became (or stopped being) trustworthy: statuses, evidence families, source references and the official dataset hash at that moment."""
__tablename__ = "kb_promotion_audits"
id: Mapped[int] = mapped_column(primary_key=True)
kb_product_id: Mapped[int] = mapped_column(ForeignKey("kb_products.id"), index=True)
gtin: Mapped[str | None] = mapped_column(String(16), nullable=True, index=True)
field: Mapped[str] = mapped_column(String(24), default="ncm", server_default="ncm")
value: Mapped[str | None] = mapped_column(String(64), nullable=True)
old_status: Mapped[str | None] = mapped_column(String(24), nullable=True)
new_status: Mapped[str] = mapped_column(String(24))
families: Mapped[list] = mapped_column(JSON, default=list)
sources: Mapped[dict] = mapped_column(JSON, default=dict)
official_dataset: Mapped[dict] = mapped_column(JSON, default=dict)
rule_version: Mapped[str] = mapped_column(String(32))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+59
View File
@@ -0,0 +1,59 @@
"""Per-product operational state and safe partial-processing metrics."""
from collections import Counter
import hashlib
import json
STATES = {
"COMPLETE", "CORRECTED", "BLOCKED_MISSING_EVIDENCE", "BLOCKED_CONFLICT",
"BLOCKED_COMPANY_CONTEXT", "BLOCKED_SCHEMA", "BLOCKED_FISCAL_DATA", "FAILED_TECHNICAL", "NEEDS_REVIEW",
}
def blocker_fingerprint(tenant: str, company: str, product_code: str, field: str, reason_code: str, input_fingerprint: str, installation_id: str | int | None = None) -> str:
raw = {"tenant": tenant, "company": company, "product_code": product_code, "field": field, "reason_code": reason_code, "input_fingerprint": input_fingerprint}
if installation_id is not None:
raw["installation_id"] = str(installation_id)
return hashlib.sha256(json.dumps(raw, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
def classify_product_state(product: dict, decision: dict, *, write_status: str | None = None, fiscal: dict | None = None) -> dict:
"""`fiscal` = unified_fiscal_coverage.evaluate_product(...): PIS/COFINS + (only when the ERP has RTC on) cClassTrib. One definition of COMPLETE."""
status = str(decision.get("status") or "").upper()
if write_status == "FAILED":
state = "FAILED_TECHNICAL"
elif status in {"AMBIGUOUS", "CONFLICT", "NCM_CONFLICT"}:
state = "BLOCKED_CONFLICT"
elif status in {"INVALID_NCM", "LEGACY_NCM", "NOT_FOUND"}:
state = "BLOCKED_MISSING_EVIDENCE"
elif status in {"COMPANY_CONTEXT_REQUIRED", "COMPANY_CONTEXT_FAIL"}:
state = "BLOCKED_COMPANY_CONTEXT"
elif status in {"ERP_SCHEMA_LIMITATION", "UNSUPPORTED_SCHEMA"}:
state = "BLOCKED_SCHEMA"
elif write_status == "SUCCEEDED":
state = "CORRECTED"
else:
state = "COMPLETE"
fiscal = fiscal or {}
if state in ("COMPLETE", "CORRECTED"):
if fiscal.get("blockers"):
state = "BLOCKED_FISCAL_DATA"
elif fiscal.get("needs_review"):
state = "NEEDS_REVIEW"
return {"product_code": str(product.get("CODIGO") or ""), "state": state, "decision_status": status, "fiscal_blockers": list(fiscal.get("blockers") or [])}
def should_retry_blocker(previous_input_fingerprint: str | None, current_input_fingerprint: str) -> bool:
return previous_input_fingerprint != current_input_fingerprint
def summarize_operational_states(states: list[dict]) -> dict:
counts = Counter(item.get("state") for item in states)
result = {state: counts.get(state, 0) for state in sorted(STATES)}
result["TOTAL_PRODUCTS"] = len(states)
result["UNCLASSIFIED"] = result["NEEDS_REVIEW"]
result["COMPLETE_PRODUCTS"] = result["COMPLETE"] + result["CORRECTED"]
result["SAFE_BLOCKED"] = sum(result[state] for state in STATES if state.startswith("BLOCKED_"))
result["AUTONOMOUSLY_PROCESSABLE_PRODUCTS"] = result["TOTAL_PRODUCTS"] - result["FAILED_TECHNICAL"]
result["AGENT_HEALTH"] = "HEALTHY" if result["FAILED_TECHNICAL"] == 0 else "DEGRADED"
return result
+17
View File
@@ -0,0 +1,17 @@
"""Evidence-gated origin resolution; CST digits are never an origin source."""
def resolve_origin(product_id: str, aggregate: dict, kb_facts: dict) -> dict:
if aggregate.get("conflict_count", 0):
status = "ORIGIN_CONFLICT"
elif aggregate.get("strong_evidence") and aggregate.get("value") in {str(i) for i in range(9)}:
status = "ORIGIN_CONFIRMED"
else:
status = "ORIGIN_NEEDS_EVIDENCE"
return {
"product_id": product_id,
"status": status,
"value": aggregate.get("value") if status == "ORIGIN_CONFIRMED" else None,
"provenance": aggregate.get("provenance", []),
"reason": "explicit supplier observations only; CST digit ignored",
}
+254
View File
@@ -0,0 +1,254 @@
"""Human-readable values used by the operational control panel."""
from __future__ import annotations
from datetime import datetime, timezone
import os
from zoneinfo import ZoneInfo
from .config import settings
def _as_utc(value: datetime | None) -> datetime | None:
if value is None:
return None
if value.tzinfo is None:
value = value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
def _ui_timezone() -> ZoneInfo:
return ZoneInfo(os.getenv("UI_TIMEZONE", settings.ui_timezone))
def format_ui_datetime(value: datetime | None) -> str:
"""Format a stored timestamp in the configured UI timezone."""
value = _as_utc(value)
if value is None:
return "-"
return value.astimezone(_ui_timezone()).strftime("%d/%m/%Y %H:%M")
def format_age(value: datetime | None, now: datetime | None = None) -> str:
value = _as_utc(value)
if value is None:
return "-"
now = _as_utc(now or datetime.now(timezone.utc))
seconds = int((now - value).total_seconds())
if seconds <= 0:
return "agora"
minutes = seconds // 60
if minutes < 1:
return "agora"
if minutes < 60:
return f"há {minutes} min"
return f"há {minutes // 60} h"
def format_duration(started_at: datetime | None, finished_at: datetime | None, now: datetime | None = None) -> str:
del now
started_at = _as_utc(started_at)
if started_at is None:
return "-"
if finished_at is None:
return "em andamento"
elapsed = max(0, int((_as_utc(finished_at) - started_at).total_seconds()))
hours, remainder = divmod(elapsed, 3600)
minutes, seconds = divmod(remainder, 60)
parts = []
if hours:
parts.append(f"{hours} h")
if minutes:
parts.append(f"{minutes} min")
if seconds or not parts:
parts.append(f"{seconds} s")
return " ".join(parts)
def _blocker_value(blocker, name: str) -> str:
if isinstance(blocker, dict):
return str(blocker.get(name) or "").upper()
return str(getattr(blocker, name, "") or "").upper()
def operational_priority(blocker) -> str:
blocker_type = _blocker_value(blocker, "blocker_type")
field = _blocker_value(blocker, "field")
reason = _blocker_value(blocker, "reason_code")
operational_text = f"{blocker_type} {reason}"
if blocker_type in {"TECHNICAL_FAILURE", "CONNECTION_ERROR", "AGENT_OFFLINE", "JOB_FAILED"} or any(token in operational_text for token in ("TECHNICAL", "CONNECTION", "TIMEOUT")):
return "CRITICAL"
if blocker_type in {"BLOCKED_MISSING_EVIDENCE", "MISSING_EVIDENCE"}:
return "HIGH"
if field in {"CFOP", "ST", "ICMS", "CSOSN", "NCM", "CEST", "PIS", "COFINS", "RTC"} or reason in {"INSUFFICIENT_EVIDENCE", "NO_EVIDENCE"}:
return "NORMAL"
return "LOW"
_HEALTH_LABELS = {
"ONLINE": "Online",
"DEGRADED": "Degradado",
"OFFLINE": "Offline",
"FAILED": "Falha",
"UNKNOWN": "Desconhecido",
}
def health_label(value) -> str:
if value is None or value == "":
return "-"
text = str(value)
return _HEALTH_LABELS.get(text.upper(), text)
def _human_label(value, labels):
if not value:
return "Não informado"
text = str(value).upper()
return labels.get(text, str(value).replace("_", " ").lower().capitalize())
def fiscal_status_label(value):
return _human_label(value, {"BLOCKED": "Bloqueado", "COMPLETE": "Completo", "INCOMPLETE": "Incompleto", "UNKNOWN": "Não informado"})
def job_status_label(value):
return _human_label(value, {"PENDING": "Pendente", "RUNNING": "Executando", "COMPLETED": "Concluído", "FAILED": "Falhou"})
def job_kind_label(value):
return _human_label(value, {"SYNC_INCREMENTAL": "Sincronização incremental", "INITIAL_SYNC": "Sincronização inicial", "REPROCESS_ALL": "Reprocessar tudo", "REPROCESS_PRODUCTS": "Reprocessar produtos"})
_FISCAL_LABELS = {
"CCLASSTRIB_MISSING": "cClassTrib ausente", "CCLASSTRIB_MISSING_UNRESOLVED": "cClassTrib não determinado", "CCLASSTRIB_AMBIGUOUS": "cClassTrib ambíguo", "CCLASSTRIB_CONFLICT": "cClassTrib existente diverge da classificação proposta",
"CCLASSTRIB_OFFICIAL_MISMATCH": "cClassTrib diverge da tabela oficial", "RTC_LOCAL_CATALOG_INCOMPATIBLE": "Catálogo RTC incompatível", "RTC_RULE_EVIDENCE_MISSING": "Regra fiscal insuficiente", "CCLASSTRIB_INVALID_FORMAT": "cClassTrib com formato inválido", "CCLASSTRIB_NOT_IN_CATALOG": "cClassTrib fora do catálogo local",
"RTC_CST_MISSING": "CST IBS/CBS ausente no catálogo", "RTC_CATALOG_INVALID": "Catálogo RTC local inválido",
"PIS_CST_MISSING": "CST PIS ausente", "PIS_CST_INVALID": "CST PIS inválido", "PIS_RATE_MISSING": "Alíquota PIS ausente", "PIS_RATE_INVALID": "Alíquota PIS inválida", "PIS_CST_INCIDENCE_MISMATCH": "CST PIS incompatível com a incidência",
"COFINS_CST_MISSING": "CST COFINS ausente", "COFINS_CST_INVALID": "CST COFINS inválido", "COFINS_RATE_MISSING": "Alíquota COFINS ausente", "COFINS_RATE_INVALID": "Alíquota COFINS inválida", "COFINS_CST_INCIDENCE_MISMATCH": "CST COFINS incompatível com a incidência",
"PIS_COFINS_FLAG_INCONSISTENT": "Flag PIS/COFINS inconsistente", "BLOCKED_FISCAL_DATA": "Dado fiscal do cadastro",
}
def blocker_type_label(value):
return _human_label(value, {"BLOCKED_MISSING_EVIDENCE": "Evidência fiscal ausente", "MISSING_EVIDENCE": "Evidência fiscal ausente", "TECHNICAL_FAILURE": "Falha técnica", "CONNECTION_ERROR": "Erro de conexão", "OTHER": "Outro", **_FISCAL_LABELS})
_REASON_LABELS = {
"INVALID_NCM": "NCM inválido", "LEGACY_NCM": "NCM legado", "LEGACY_NCM_NO_AUTOMATIC_SUCCESSOR": "NCM legado sem sucessor automático",
"LEGACY_NCM_REVIEW": "NCM legado em revisão", "LEGACY_NCM_REVIEW_REQUIRED": "NCM legado exige revisão", "LEGACY_OR_INVALID_NCM": "NCM legado ou inválido",
"INSUFFICIENT_EVIDENCE": "Evidência fiscal insuficiente", "MISSING_EVIDENCE": "Evidência fiscal ausente", "BLOCKED_MISSING_EVIDENCE": "Evidência fiscal ausente",
"MISSING_CONTEXT": "Contexto fiscal ausente", "NO_EVIDENCE": "Sem evidência", "MISSING_PRODUCT_ORIGIN": "Origem do produto ausente",
"MISSING_PROFILE": "Perfil fiscal ausente", "MISSING_ST_DETERMINATION": "Determinação de ST ausente", "NO_IPI_DECISION": "Sem decisão de IPI",
"NO_NUMERIC_VALUE": "Sem valor numérico", "CHANGE_REQUIRED": "Alteração necessária", "SAME_VALUE": "Valor já correto", "AMBIGUOUS": "Ambíguo",
}
def blocker_reason_label(value):
return _human_label(value, {**_REASON_LABELS, **_FISCAL_LABELS})
def health_reason_label(value):
"""Operator wording for the technical health reason strings produced by compute_agent_health."""
text = str(value or "")
if text.startswith("last heartbeat exceeds"):
return "Sem heartbeat recente"
return {"no heartbeat received": "Nenhum heartbeat recebido", "persistent technical failure": "Falha técnica persistente",
"recent technical or job failure": "Falha técnica ou de job recente",
"recent heartbeat without critical technical failure": "Operando normalmente"}.get(text, text or "Não informado")
def priority_label(value):
return _human_label(value, {"CRITICAL": "Crítica", "HIGH": "Alta", "NORMAL": "Média", "MEDIUM": "Média", "LOW": "Baixa"})
def rtc_status_view(summary, mode="DISABLED"):
"""Discreet Reforma Tributaria status from the last completed job's `fiscal_context`, exactly as the Agent OBSERVED it in that installation's own database.
None when the Agent did not report it (older Agents)."""
context = (summary or {}).get("fiscal_context") if isinstance(summary, dict) else None
if not isinstance(context, dict) or "rtc_enabled" not in context:
return None
observed = None
if context.get("rtc_observed_at"):
try:
observed = format_ui_datetime(datetime.fromisoformat(str(context["rtc_observed_at"]).replace("Z", "+00:00")))
except ValueError:
observed = None
source = context.get("rtc_source") or "PARAMETROS.USAR_REFORMA_TRIBUTARIA"
products = _rtc_products_view(context.get("rtc_resolution"), bool(context["rtc_enabled"]), context.get("cclasstrib_applicability"))
return {"products": products, "apply": _rtc_apply_view(summary.get("rtc_auto_apply"), mode), "enabled": bool(context["rtc_enabled"]), "label": "Ativa" if context["rtc_enabled"] else "Desativada", "supported": bool(context.get("rtc_supported")),
"catalog_valid": bool(context.get("rtc_catalog_valid")), "observed_at": observed, "conflict": bool(context.get("rtc_param_conflict")),
"title": f"Lido no banco do cliente ({source})" + (f" em {observed}" if observed else "") + (" - linhas de PARAMETROS divergentes: RTC tratada como desativada" if context.get("rtc_param_conflict") else "")}
def _rtc_products_view(resolution, enabled, cclasstrib_applicability=None):
"""Compact 'Produtos RTC' summary. None when RTC is off (RTC never shows up as a problem) or the Agent did not report a resolution."""
if not enabled:
return None
if isinstance(cclasstrib_applicability, dict) and cclasstrib_applicability.get("state") == "NOT_APPLICABLE":
label = "cClassTrib: Aguardando confirmação do regime da empresa" if cclasstrib_applicability.get("basis") == "COMPANY_CRT_UNRESOLVED" else "cClassTrib: Não utilizado neste cliente"
return {"applicable": False, "label": label}
if not isinstance(resolution, dict) or not resolution.get("total"):
return None
total = int(resolution["total"])
valid = int(resolution.get("valid_existing", 0))
ready = int(resolution.get("ready_to_apply", 0))
conflicts = int(resolution.get("conflicts", 0)) + int(resolution.get("official_mismatch", 0))
catalog = int(resolution.get("catalog_incompatible", 0))
reasons = resolution.get("by_reason") if isinstance(resolution.get("by_reason"), dict) else {}
simple_count = sum(int(count or 0) for reason, count in reasons.items() if str(reason).endswith("SIMPLES_ORIENTATION_PENDING_OFFICIAL_NT"))
other_reason_max = max((int(count or 0) for reason, count in reasons.items() if not str(reason).endswith("SIMPLES_ORIENTATION_PENDING_OFFICIAL_NT")), default=0)
blocked = total - valid - ready
pending = total - valid
return {"applicable": True, "total": total, "valid": valid, "ready": ready, "blocked": blocked, "pending": pending, "conflicts": conflicts, "catalog_incompatible": catalog, "suggestion_only": int(resolution.get("suggestion_only", 0)),
"simple_orientation_count": simple_count, "simple_orientation_dominant": simple_count > 0 and simple_count > other_reason_max,
"headline": f"Reforma Tributária ativa — cClassTrib pendente em {pending} produto{'s' if pending != 1 else ''}" if pending else "Reforma Tributária ativa — todos os produtos com cClassTrib válido",
"catalog": (resolution.get("catalog_compatibility") or {}).get("compatibility")}
ENRICHMENT_MODES = ("DISABLED", "DRY_RUN", "ENABLED")
ENRICHMENT_MODE_LABELS = {"DISABLED": "Desativado", "DRY_RUN": "Simulação (não grava)", "ENABLED": "Automático"}
RTC_MODE_LABELS = {"DISABLED": "Desativada", "DRY_RUN": "Dry-run", "APPROVAL_REQUIRED": "Aprovação manual", "ENABLED": "Automática"}
RTC_MODE_EXPLANATION = "Somente cClassTrib com regra fiscal determinística e catálogo validado pode ser preenchido automaticamente."
def _rtc_apply_view(report, mode):
"""State of the (default DISABLED) centrally controlled fill. Never claims a write happened unless the Agent reported it."""
mode = mode if mode in RTC_MODE_LABELS else "DISABLED"
report = report or {}
return {"mode": mode, "state": RTC_MODE_LABELS[mode], "explanation": RTC_MODE_EXPLANATION, "applied_last_job": len(report.get("applied") or []), "aborted_last_job": len(report.get("aborted") or []),
"ready_total": report.get("ready_total"), "capability": report.get("capability"), "authorized": mode in ("ENABLED", "APPROVAL_REQUIRED")}
# ---- cClassTrib control page (Agent -> cClassTrib): candidate status, audit outcome/reason, rule and catalog wording ----
_CANDIDATE_STATUS = {"PENDING": "Pendente", "APPROVED": "Aprovado", "REJECTED": "Rejeitado", "APPLIED": "Aplicado", "STALE": "Obsoleto"}
_AUDIT_OUTCOME = {"APPLIED": "Aplicado", "ROLLED_BACK": "Revertido", "ABORTED": "Interrompido sem escrever", "FAILED": "Falha técnica", "ROLLBACK_REQUESTED": "Reversão solicitada"}
_RULE_LABELS = {"RTC-R3": "Anexo com classe única (RTC-R3)", "RTC-R4": "Sugestão sem autorização de preenchimento (RTC-R4)", "RTC-A3": "Bloqueada por evidência (RTC-A3)", "RTC-G2": "Bloqueada por evidência (RTC-G2)"}
_CATALOG_STATE = {"MATCH_SAFE": "Compatível com a tabela oficial", "MATCH_WITH_NON_MATERIAL_DIFFERENCE": "Diferença sem impacto (não autoriza escrita)", "MISMATCH": "Diverge da tabela oficial",
"EXPIRED": "Classe oficial vencida", "MISSING_LOCAL": "Classe oficial ausente no catálogo local do SGV", "MISSING_OFFICIAL": "Classe local ausente na tabela oficial"}
_ABORT_REASONS = {"VALUE_ALREADY_SET_NEVER_OVERWRITTEN": "cClassTrib existente diverge da classificação proposta", "NO_WRITE_REQUIRED": "Já preenchido com o valor proposto",
"RTC_NO_LONGER_ENABLED": "Reforma tributária desligada no ERP", "RTC_CATALOG_INVALID": "Catálogo RTC local inválido", "LOCAL_CATALOG_CHANGED_SINCE_DECISION": "Catálogo local mudou desde a decisão",
"COMPANY_CONTEXT_CHANGED": "Contexto da empresa mudou", "REGIME_NOT_NORMAL": "Regime sem regra segura de preenchimento", "PRODUCT_CHANGED_NCM": "NCM do produto mudou",
"PRODUCT_CHANGED_SINCE_DECISION": "Produto mudou desde a decisão", "DECISION_CHANGED": "A classificação proposta mudou", "PRODUCT_NOT_FOUND": "Produto não encontrado",
"PRODUCT_ROW_NOT_UNIQUE": "Código de produto duplicado", "UPDATE_ROWCOUNT_NOT_ONE": "Escrita recusada (contagem inesperada)", "ROLLBACK_CONFLICT_VALUE_CHANGED": "Valor alterado depois da escrita: reversão recusada",
"ALREADY_EMPTY": "Já estava vazio", "ROLLBACK_REQUEST_INVALID": "Pedido de reversão inválido", "TECHNICAL_FAILURE": "Falha técnica", "AUTHORIZATION_CHANGED": "Autorização central mudou antes da escrita", "AUTHORIZATION_UNVERIFIABLE": "Autorização central não pôde ser confirmada"}
def candidate_status_label(value):
return _human_label(value, _CANDIDATE_STATUS)
def audit_outcome_label(value):
return _human_label(value, _AUDIT_OUTCOME)
def rtc_rule_label(value):
return _human_label(value, _RULE_LABELS)
def catalog_state_label(value):
return _human_label(value, _CATALOG_STATE)
def abort_reason_label(value):
return _human_label(value, _ABORT_REASONS) if value else ""
@@ -0,0 +1,49 @@
"""Offline-safe product data provider boundary and permanent cache record."""
from dataclasses import dataclass
from datetime import datetime, timezone
import hashlib
import json
from typing import Any, Protocol
@dataclass(frozen=True)
class ProviderRecord:
provider: str
provider_record_version: str
gtin: str
retrieved_at: str
raw_hash: str
normalized_data: dict[str, Any]
fiscal_authority: str = "OBSERVATION_ONLY"
cache_allowed: bool = False
redistribution_allowed: bool = False
class ProductDataProvider(Protocol):
def lookup_gtin(self, gtin: str) -> ProviderRecord | None: ...
def normalize_record(provider: str, version: str, gtin: str, data: dict[str, Any]) -> ProviderRecord:
normalized = {str(key): value for key, value in sorted(data.items()) if value not in (None, "")}
raw_hash = hashlib.sha256(json.dumps(normalized, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8")).hexdigest()
return ProviderRecord(provider, version, str(gtin), datetime.now(timezone.utc).isoformat(), raw_hash, normalized)
def cache_key(gtin: str, provider: str, provider_record_version: str) -> str:
return f"{gtin}:{provider}:{provider_record_version}"
def result_cache_key(product: dict[str, Any], candidates: list[str], datasets: dict[str, str], prompt: str, model: str) -> str:
payload = {"product": product, "candidates": candidates, "datasets": datasets, "prompt": prompt, "model": model}
return hashlib.sha256(json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8")).hexdigest()
class LocalBulkDatasetProvider:
"""Offline provider backed by explicitly supplied records; never performs I/O."""
def __init__(self, records: dict[str, ProviderRecord] | None = None):
self.records = records or {}
def lookup_gtin(self, gtin: str) -> ProviderRecord | None:
return self.records.get(str(gtin))
+95
View File
@@ -0,0 +1,95 @@
"""Product Fiscal Defaults Engine V1 — C000025.CFOP / CSOSN / CST default decisions.
Pure functions. Scope: intra-UF (BA) NFC-e-style sale of merchandise, company CRT=1 (Simples).
Historical documents are NEVER an input here; they are used by the runner only to validate.
CFOP rule chain (all must hold, else CANDIDATE/CONTEXT/UNRESOLVED):
ERP CFOP mapping HIGH (product default, saída) + NCM EXACT + current CFOP in official table + first digit 5
(saída, intra-UF) + ERP declares resale family (5102/5405: 'adquirida ou recebida de terceiros')
+ ST applicability known from BA Anexo 1 (MVA_RESOLVED -> subject, NO_MVA_MAPPING_FOUND -> not subject)
=> derived CFOP = 5405 if subject else 5102. Equal to current => RESOLVED (already correct).
Different => CANDIDATE only: one of the inputs is wrong and the engine cannot tell which
(ST retention by the supplier is not available), so it is never write-eligible.
The chain never yields a write: filling a blank CFOP would need the resale-vs-production declaration that
only the current CFOP carries, so blank => OPERATION_CONTEXT_REQUIRED.
CSOSN: no dataset states whether a product is taxed (102), exempt/not taxed (400/300), ST-charged (500)...
=> never RESOLVED. Company default C000004.ICMS_CSOSN is only a CANDIDATE anchor for non-ST products.
"""
PRODUCT_DEFAULTS_ENGINE_VERSION = "product-defaults-v1"
RESALE_CFOPS = {"5102", "5405"}
def _blank(v):
return v is None or str(v).strip() == ""
def _out(field, current, proposed, state, reason, action, **extra):
return {"field": field, "current_value": current, "proposed_value": proposed, "status": state, "reason_code": reason,
"action_class": action, "write_eligible": False, "engine": PRODUCT_DEFAULTS_ENGINE_VERSION, **extra}
def st_applicability(ncm_status, mva_status):
if ncm_status != "EXACT_NCM":
return "UNKNOWN_NCM_NOT_VALID"
if mva_status == "MVA_RESOLVED":
return "SUBJECT"
if mva_status == "NO_MVA_MAPPING_FOUND":
return "NOT_SUBJECT"
return "UNKNOWN_UNCONFIRMED"
def cfop_default(product, ncm_status, mva_status, cfop_valid):
cur = None if _blank(product.get("CFOP")) else str(product["CFOP"]).strip()
st = st_applicability(ncm_status, mva_status)
base = {"st_applicability": st}
if st == "UNKNOWN_NCM_NOT_VALID":
return _out("CFOP", cur, None, "CFOP_UNRESOLVED", "NCM_NOT_VALID_CANNOT_ASSESS_ST", "NO_RULE", **base)
if cur is None:
return _out("CFOP", cur, None, "CFOP_OPERATION_CONTEXT_REQUIRED", "NO_DECLARED_RESALE_OR_PRODUCTION_FAMILY", "OPERATION_CONTEXT_REQUIRED", **base)
if cur not in cfop_valid:
return _out("CFOP", cur, None, "CFOP_UNRESOLVED", "CURRENT_CFOP_NOT_IN_OFFICIAL_TABLE", "REVIEW_REQUIRED", **base)
if cur[0] != "5":
return _out("CFOP", cur, None, "CFOP_OPERATION_CONTEXT_REQUIRED", "DIRECTION_OR_UF_SCOPE_NOT_INTRA_UF_SAIDA", "OPERATION_CONTEXT_REQUIRED", **base)
if cur not in RESALE_CFOPS:
return _out("CFOP", cur, None, "CFOP_OPERATION_CONTEXT_REQUIRED", "NO_RULE_FOR_DECLARED_CFOP_FAMILY", "OPERATION_CONTEXT_REQUIRED", **base)
if st == "UNKNOWN_UNCONFIRMED":
return _out("CFOP", cur, None, "CFOP_OPERATION_CONTEXT_REQUIRED", "ST_APPLICABILITY_UNCONFIRMED_NEEDS_CEST_CONFIRMATION", "OPERATION_CONTEXT_REQUIRED", **base)
derived = "5405" if st == "SUBJECT" else "5102"
if derived == cur:
return _out("CFOP", cur, derived, "CFOP_DEFAULT_RESOLVED", "CONSISTENT_WITH_ST_STATUS_SAME_VALUE", "ALREADY_CORRECT", **base)
return _out("CFOP", cur, derived, "CFOP_DEFAULT_CANDIDATE", "ST_STATUS_CONFLICTS_WITH_ERP_CFOP_RETENTION_CONTEXT_MISSING", "REVIEW_REQUIRED", **base)
def csosn_default(product, ncm_status, mva_status, csosn_valid, company_default):
cur = None if _blank(product.get("CSOSN")) else str(product["CSOSN"]).strip()
st = st_applicability(ncm_status, mva_status)
base = {"st_applicability": st, "company_default": company_default}
if st == "UNKNOWN_NCM_NOT_VALID":
return _out("CSOSN", cur, None, "CSOSN_UNRESOLVED", "NCM_NOT_VALID_CANNOT_ASSESS_ST", "NO_RULE", **base)
if st == "UNKNOWN_UNCONFIRMED":
return _out("CSOSN", cur, None, "CSOSN_OPERATION_CONTEXT_REQUIRED", "ST_APPLICABILITY_UNCONFIRMED", "OPERATION_CONTEXT_REQUIRED", **base)
if st == "SUBJECT":
flag = "ST_SUBJECT_BUT_CSOSN_NOT_500" if cur != "500" else "ST_CSOSN_500_RETENTION_UNVERIFIED"
return _out("CSOSN", cur, None, "CSOSN_OPERATION_CONTEXT_REQUIRED", "ST_RETENTION_BY_SUPPLIER_UNKNOWN", "OPERATION_CONTEXT_REQUIRED", consistency_flag=flag, **base)
# NOT_SUBJECT: 102 / 400 / 300 / 103 depend on the product's ICMS treatment, for which no source exists
if cur is not None and cur == company_default and cur in csosn_valid:
return _out("CSOSN", cur, cur, "CSOSN_DEFAULT_CANDIDATE", "MATCHES_COMPANY_DEFAULT_PRODUCT_ICMS_TREATMENT_UNPROVEN", "REVIEW_REQUIRED", **base)
if cur is None:
return _out("CSOSN", cur, company_default, "CSOSN_DEFAULT_CANDIDATE", "BLANK_COMPANY_DEFAULT_ONLY_ANCHOR_TREATMENT_UNPROVEN", "REVIEW_REQUIRED", **base)
return _out("CSOSN", cur, None, "CSOSN_DEFAULT_CANDIDATE", "DIFFERS_FROM_COMPANY_DEFAULT_NO_TREATMENT_SOURCE", "REVIEW_REQUIRED", **base)
def cst_status(product, cst_icms_codes, origem_codes):
cur = None if _blank(product.get("CST")) else str(product["CST"]).strip()
valid = cur is not None and len(cur) == 3 and cur[1:] in cst_icms_codes and cur[0] in origem_codes
return {"field": "CST", "current_value": cur, "status": "CST_REFERENCE_ONLY" if valid else "CST_INVALID_REVIEW",
"reason_code": "CRT1_EMISSION_USES_CSOSN_CST_IS_ORIGIN_PLUS_CST" if valid else "NOT_ORIGIN_DIGIT_PLUS_VALID_CST",
"origin_digit": cur[0] if valid else None, "write_eligible": False, "engine": PRODUCT_DEFAULTS_ENGINE_VERSION}
def validate_against_history(proposed, emitted_values):
"""History validates only; returns MATCH / MISMATCH / UNVALIDATED."""
if not emitted_values or proposed is None:
return "UNVALIDATED"
return "MATCH" if set(emitted_values) == {proposed} else "MISMATCH"
+36
View File
@@ -0,0 +1,36 @@
"""In-memory per-IP rate limiting for pre-authentication routes.
Same spirit as painel-clientes-web's `app/rate_limit.py`: a single-process
in-memory counter, fine because the Hub runs as one container behind a
trusted reverse proxy (FORWARDED_ALLOW_IPS), never multiple replicas behind
a shared load balancer. A process restart resets the history — accepted,
same tradeoff as that panel's SESSION_SECRET.
Used for the two public, pre-authentication Agent-distribution routes
(GET /dl/agent/{token}, POST /v1/agents/enroll): invalid attempts count
against the caller's IP; a legitimate Agent's own successful traffic never
gets counted, so it can never push other installations over the limit.
"""
import time
MAX_ATTEMPTS = 10
WINDOW_SECONDS = 5 * 60
_attempts: dict[tuple[str, str], list[float]] = {}
def _recent(bucket: str, origin: str, now: float) -> list[float]:
cutoff = now - WINDOW_SECONDS
return [t for t in _attempts.get((bucket, origin), []) if t > cutoff]
def blocked(bucket: str, origin: str, now: float | None = None) -> bool:
moment = now if now is not None else time.time()
return len(_recent(bucket, origin, moment)) >= MAX_ATTEMPTS
def record_attempt(bucket: str, origin: str, now: float | None = None) -> None:
moment = now if now is not None else time.time()
recent = _recent(bucket, origin, moment)
recent.append(moment)
_attempts[(bucket, origin)] = recent
+174
View File
@@ -0,0 +1,174 @@
"""Reference Data Registry (Missão 5, Fase 1): the NEW admin-approved ingestion workflow, separate
from central/app/datasets/service.py's existing auto-activate methods (untouched, still used by the
9 pre-existing dataset kinds). A dataset registered here lands as DRAFT and only becomes VIGENTE
(superseding the previous VIGENTE of the same kind) after an explicit ADMIN approval of the diff -
"Nenhuma versão de dataset vira 'vigente' sem aprovação ADMIN do diff" (mission rule).
"""
from __future__ import annotations
from datetime import datetime, timezone
from sqlalchemy import select, update
from .models import Dataset
STATUSES = ("DRAFT", "APPROVED", "VIGENTE", "ACTIVE", "OBSOLETO", "SUPERSEDED", "REJECTED")
VIGENTE_STATUSES = ("VIGENTE", "ACTIVE")
class RegistryError(Exception):
pass
def register_draft(session, *, kind: str, provider: str, source_url: str, sha256: str, version_label: str,
act: str = "", valid_from: str | None = None, valid_to: str | None = None, record_count: int = 0,
parse_errors: dict | None = None, licence_terms: str | None = None, raw_file_path: str | None = None,
file_size_bytes: int | None = None, submitted_by: str | None = None) -> Dataset:
"""Registers a new dataset submission as DRAFT. If this exact (provider, sha256) already exists
under ANY status, returns it unchanged (idempotent re-submission of an unchanged official file -
e.g. re-downloading the same official NCM table produces byte-identical content) instead of
creating a duplicate row (the (provider, sha256) constraint would refuse it anyway)."""
existing = session.scalar(select(Dataset).where(Dataset.provider == provider, Dataset.sha256 == sha256))
if existing:
return existing
dataset = Dataset(kind=kind, provider=provider, sha256=sha256, version_label=version_label, act=act, source_url=source_url,
downloaded_at=datetime.now(timezone.utc), status="DRAFT", record_count=record_count, licence_terms=licence_terms,
valid_from=valid_from, valid_to=valid_to, file_size_bytes=file_size_bytes, raw_file_path=raw_file_path,
parse_errors=parse_errors or {}, submitted_by=submitted_by)
session.add(dataset)
session.commit()
return dataset
def diff_against_vigente(session, dataset: Dataset) -> dict:
"""A summary an ADMIN reviews before approving: what the current VIGENTE of the same kind looks
like vs this DRAFT - never the row-by-row content (that lives in the kind-specific entry tables),
just the registry-level facts that matter for an approval decision."""
current = session.scalar(select(Dataset).where(Dataset.kind == dataset.kind, Dataset.status.in_(VIGENTE_STATUSES)).order_by(Dataset.id.desc()))
return {
"kind": dataset.kind,
"draft": {"id": dataset.id, "version_label": dataset.version_label, "sha256": dataset.sha256, "record_count": dataset.record_count,
"valid_from": dataset.valid_from, "valid_to": dataset.valid_to, "parse_errors": dataset.parse_errors},
"current_vigente": None if current is None else {"id": current.id, "version_label": current.version_label, "sha256": current.sha256,
"record_count": current.record_count, "valid_from": current.valid_from, "valid_to": current.valid_to},
"record_count_delta": None if current is None else dataset.record_count - current.record_count,
"is_first_version": current is None,
"sha256_unchanged": current is not None and current.sha256 == dataset.sha256,
}
KNOWN_SAMPLES_BY_KIND = {
"NCM": [
("01012100", "Reprodutores de raça pura (cavalos)"),
("02071400", "Carnes e miudezas de aves (desdobrado/histórico)"),
("22021000", "Águas minerais adicionadas de açúcar/aromatizantes"),
("84713012", "Máquinas automáticas para processamento de dados"),
("10061092", "Arroz com casca, não parboilizado"),
],
"TIPI_IPI": [
("22021000", "Refrigerantes / águas gaseificadas"),
("84713012", "Notebooks / computadores portáteis"),
("21069090", "Preparações alimentícias diversas"),
("02071400", "Carnes de frango congeladas"),
("22030000", "Cervejas de malte"),
],
"CEST": [
("01.001.00", "Autopeças - Motores e peças"),
("03.021.03", "Cervejas e chopes"),
("17.001.00", "Produtos alimentícios - Carnes"),
("28.001.00", "Venda porta a porta"),
("10.001.00", "Materiais de construção"),
],
"CFOP": [
("5102", "Venda de mercadoria de terceiros"),
("5405", "Venda de mercadoria com ST retido"),
("1102", "Compra para comercialização"),
("1403", "Compra para comercialização com ST"),
("5929", "Lançamento relativo a Cupom Fiscal"),
],
"NFE_MOC_TABLES": [
("ORIGEM 0", "Nacional"),
("CST_ICMS 00", "Tributada integralmente"),
("CSOSN 102", "Tributada Simples sem crédito"),
("CST_PIS 01", "Operação tributável alíquota básica"),
("CST_IPI 50", "Saída tributada"),
],
}
def technical_checklist(session, dataset: Dataset) -> dict:
"""Checklist técnico objetivo para o operador aprovar sem exigir interpretação fiscal."""
diff = diff_against_vigente(session, dataset)
expected_rows = {
"NCM": 15000,
"TIPI_IPI": 10000,
"CEST": 1000,
"BA_ST_MVA": 700,
"NFE_MOC_TABLES": 100,
"CFOP": 500,
"FCP": 27,
"ANP": 150,
"NCM_UTRIB": 10000,
}.get(dataset.kind, 1)
lines_ok = dataset.record_count >= expected_rows
samples = KNOWN_SAMPLES_BY_KIND.get(dataset.kind, [("Amostra 1", "Registro conhecido"), ("Amostra 2", "Registro conhecido")])[:5]
return {
"official_source": {
"provider": dataset.provider,
"url": dataset.source_url,
"act": dataset.act or "Ato normativo oficial",
"passed": bool(dataset.provider and dataset.source_url),
},
"integrity": {
"sha256": dataset.sha256,
"file_size_bytes": dataset.file_size_bytes,
"passed": len(dataset.sha256 or "") == 64,
},
"volume": {
"record_count": dataset.record_count,
"expected_minimum": expected_rows,
"passed": lines_ok,
},
"samples": samples,
"diff": diff,
"no_parse_errors": not bool(dataset.parse_errors),
"ready_to_approve": lines_ok and len(dataset.sha256 or "") == 64 and not bool(dataset.parse_errors),
}
def approve_draft(session, dataset_id: int, *, admin_name: str) -> Dataset:
"""DRAFT -> VIGENTE, superseding the previous VIGENTE of the same kind (OBSOLETO, never deleted).
Refuses a dataset that isn't DRAFT, and refuses a dataset with recorded parse errors (must be
re-submitted clean - an ADMIN cannot approve past a known parsing problem)."""
dataset = session.get(Dataset, dataset_id)
if dataset is None:
raise RegistryError("dataset not found")
if dataset.status != "DRAFT":
raise RegistryError(f"only a DRAFT dataset can be approved (current status: {dataset.status})")
if dataset.parse_errors:
raise RegistryError("dataset has recorded parse errors - re-submit a clean version")
session.execute(update(Dataset).where(Dataset.kind == dataset.kind, Dataset.status.in_(VIGENTE_STATUSES)).values(status="OBSOLETO"))
dataset.status = "VIGENTE"
dataset.approved_by = admin_name
dataset.approved_at = datetime.now(timezone.utc)
session.commit()
return dataset
def reject_draft(session, dataset_id: int, *, admin_name: str) -> Dataset:
dataset = session.get(Dataset, dataset_id)
if dataset is None:
raise RegistryError("dataset not found")
if dataset.status != "DRAFT":
raise RegistryError(f"only a DRAFT dataset can be rejected (current status: {dataset.status})")
dataset.status = "REJECTED"
dataset.approved_by = admin_name
dataset.approved_at = datetime.now(timezone.utc)
session.commit()
return dataset
def list_registry(session) -> list[Dataset]:
return list(session.scalars(select(Dataset).where(Dataset.status.in_(STATUSES)).order_by(Dataset.kind, Dataset.downloaded_at.desc())))
+39
View File
@@ -0,0 +1,39 @@
"""Local, non-web-served storage for published Agent release zips.
Binaries never live in PostgreSQL — only their metadata (AgentRelease rows)
do. The storage root is a plain directory outside `central/static`, so it is
never directly reachable by a URL; every download goes through the
authenticated or token-gated routes in agent_distribution.py.
"""
from pathlib import Path
from . import config
_SAFE_COMPONENT = __import__("re").compile(r"^[A-Za-z0-9._-]+$")
def storage_root() -> Path:
root = Path(config.settings.agent_release_storage_path)
root.mkdir(parents=True, exist_ok=True)
return root
def release_filename(version: str, platform: str, architecture: str, extension: str = ".zip") -> str:
return f"GTIFiscalAgent-{version}-{platform}-{architecture}{extension}"
def setup_exe_filename(version: str) -> str:
return f"GTIFiscalAgent-Setup-{version}.exe"
def safe_path_for(storage_path: str) -> Path:
"""Resolve a release's storage_path against the root, refusing anything
that could escape it (path traversal). storage_path is DB-controlled
(never derived from an HTTP request), but this stays fail-closed anyway."""
if not storage_path or not _SAFE_COMPONENT.match(storage_path):
raise ValueError(f"unsafe release storage_path: {storage_path!r}")
root = storage_root().resolve()
candidate = (root / storage_path).resolve()
if candidate.parent != root:
raise ValueError(f"release storage_path escapes storage root: {storage_path!r}")
return candidate
+104
View File
@@ -0,0 +1,104 @@
"""Persists the Agent's RTC write audit records (append-only), the candidate preview and the rollback bookkeeping; mirrors a summary event for the panel."""
from __future__ import annotations
from sqlalchemy import select, update
from .events import record_event
from .models import RtcCandidate, RtcWriteAudit
MAX_AUDITS_PER_JOB = 2000
MAX_CANDIDATES = 25000
WRITE_MODES = ("DISABLED", "DRY_RUN", "APPROVAL_REQUIRED", "ENABLED")
MAX_APPROVED_PER_JOB = 500
MAX_ROLLBACKS_PER_JOB = 500
def _clip(value, size):
return None if value is None else str(value)[:size]
def job_authorizations(session, installation) -> dict:
"""What the authenticated job carries to the Agent. Decided centrally per installation; the Agent has no local switch."""
mode = installation.rtc_write_mode if installation.rtc_write_mode in WRITE_MODES else "DISABLED"
out = {"rtc_write_mode": mode, "rtc_approved_codes": [], "rtc_rollbacks": [],
"features": {"corrector_enabled": bool(installation.corrector_enabled), "enrichment_mode": installation.enrichment_mode, "features_configured": bool(installation.features_configured)}}
if mode == "APPROVAL_REQUIRED":
out["rtc_approved_codes"] = list(session.scalars(select(RtcCandidate.product_code).where(RtcCandidate.installation_id == installation.id, RtcCandidate.status == "APPROVED").limit(MAX_APPROVED_PER_JOB)))
for audit in session.scalars(select(RtcWriteAudit).where(RtcWriteAudit.installation_id == installation.id, RtcWriteAudit.action == "APPLY", RtcWriteAudit.outcome == "APPLIED",
RtcWriteAudit.rollback_state == "REQUESTED").limit(MAX_ROLLBACKS_PER_JOB)):
out["rtc_rollbacks"].append({"audit_id": audit.id, "product_code": audit.product_code, "written_value": audit.new_value, "rule_id": audit.rule_id})
return out
def _store_preview(session, tenant_id, installation, summary, evaluated_all):
preview = summary.get("preview")
if not isinstance(preview, list) or installation.rtc_write_mode == "DISABLED":
return
existing = {c.product_code: c for c in session.scalars(select(RtcCandidate).where(RtcCandidate.installation_id == installation.id))}
seen = set()
for item in preview[:MAX_CANDIDATES]:
code = _clip(item.get("product_code"), 64)
if not code or not item.get("proposed"):
continue
seen.add(code)
row = existing.get(code)
if row is None:
session.add(RtcCandidate(tenant_id=tenant_id, installation_id=installation.id, product_code=code, description=_clip(item.get("description"), 255), ncm=_clip(item.get("ncm"), 16),
current_value=_clip(item.get("current") or None, 16), proposed=_clip(item["proposed"], 16), cst_ibscbs=_clip(item.get("cst"), 8), rule_id=_clip(item.get("rule_id"), 32),
basis=_clip(item.get("basis"), 255), status="PENDING"))
else:
if row.status in ("APPLIED", "STALE"):
row.status = "PENDING" # empty again (e.g. after a rollback) or newly eligible
if row.proposed != item["proposed"]:
row.proposed = _clip(item["proposed"], 16)
if row.status == "APPROVED":
row.status = "PENDING" # the proposal changed: the earlier approval does not carry over
row.description, row.ncm, row.cst_ibscbs = _clip(item.get("description"), 255), _clip(item.get("ncm"), 16), _clip(item.get("cst"), 8)
row.rule_id, row.basis = _clip(item.get("rule_id"), 32), _clip(item.get("basis"), 255)
if evaluated_all and not summary.get("preview_truncated"):
for code, row in existing.items():
if code not in seen and row.status in ("PENDING", "APPROVED"):
row.status = "STALE"
def record_rtc_write_audits(session, *, tenant_id: int, installation, agent, job_id: int | None, rtc_auto_apply: dict | None, evaluated_all: bool = False) -> int:
"""Stores every applied/aborted/rolled-back write of the job. Never trusts the Agent for identity: tenant/installation/agent come from the authenticated context."""
if not isinstance(rtc_auto_apply, dict):
return 0
_store_preview(session, tenant_id, installation, rtc_auto_apply, evaluated_all)
records = list(rtc_auto_apply.get("applied") or []) + list(rtc_auto_apply.get("aborted") or []) + list(rtc_auto_apply.get("rollbacks") or [])
stored = applied = aborted = rolled = 0
for item in records[:MAX_AUDITS_PER_JOB]:
if not isinstance(item, dict) or item.get("field") != "CCLASSTRIB":
continue
action = "ROLLBACK" if item.get("action") == "ROLLBACK" else "APPLY"
outcome = str(item.get("outcome", ""))[:16]
rollback_of = item.get("rollback_of") if action == "ROLLBACK" else None
code = str(item.get("product_code", ""))[:64]
session.add(RtcWriteAudit(
tenant_id=tenant_id, installation_id=installation.id, agent_id=agent.id, job_id=job_id, client=getattr(installation, "display_name", None),
product_code=code, field="CCLASSTRIB", outcome=outcome, reason=(item.get("reason") or None) and str(item["reason"])[:96],
old_value=_clip(item.get("old_value"), 64), new_value=_clip(item.get("new_value"), 64),
rule_id=(item.get("rule_id") or None) and str(item["rule_id"])[:32], evidence=item.get("evidence") or {}, official_source_version=item.get("official_source_version") or {},
local_catalog_hash=item.get("local_catalog_hash"), written_at=item.get("timestamp"), rule_version=_clip(item.get("rule_version"), 32), company_crt=_clip(item.get("company_crt"), 8),
authorization_origin=_clip(item.get("authorization_origin"), 48), description_snapshot=_clip(item.get("description_snapshot"), 255), installation_uuid=_clip(getattr(installation, "installation_id", None), 64),
rtc_status=_clip(item.get("rtc_status"), 24), local_catalog_state=_clip(item.get("local_catalog_state"), 32), action=action, rollback_of=rollback_of if isinstance(rollback_of, int) else None))
stored += 1
candidate_of = (RtcCandidate.installation_id == installation.id) & (RtcCandidate.product_code == code)
if action == "ROLLBACK":
if isinstance(rollback_of, int):
session.execute(update(RtcWriteAudit).where(RtcWriteAudit.id == rollback_of, RtcWriteAudit.installation_id == installation.id).values(rollback_state="DONE" if outcome == "ROLLED_BACK" else "BLOCKED"))
if outcome == "ROLLED_BACK":
rolled += 1
session.execute(update(RtcCandidate).where(candidate_of).values(status="PENDING"))
elif outcome == "APPLIED":
applied += 1
session.execute(update(RtcCandidate).where(candidate_of).values(status="APPLIED"))
else:
aborted += 1
if stored:
record_event(session, tenant_id=tenant_id, installation_id=installation.id, agent_id=agent.id, severity="WARNING" if (applied or rolled) else "INFO", event_type="RTC_AUTO_APPLY", job_id=job_id,
message=f"cClassTrib: {applied} preenchido(s), {rolled} revertido(s), {aborted} abortado(s)",
context={"applied": applied, "rolled_back": rolled, "aborted": aborted, "reasons": rtc_auto_apply.get("aborted_reasons") or {}, "mode": rtc_auto_apply.get("mode")})
session.commit()
return stored
+102
View File
@@ -0,0 +1,102 @@
"""Turns the deterministic cClassTrib resolution into the ONE root-cause RTC finding per product (pure).
`evaluate_product` (unified_fiscal_coverage) says WHETHER a product violates the RTC rule; this module says WHY and whether the gap is closable:
the single RTC blocker of a product is refined to the cause that matters, never several blockers for the same root cause.
"""
from __future__ import annotations
from .cclasstrib_resolver import HUMAN_REASONS, Resolution
RTC_CATALOG_REASONS = ("LOCAL_OFFICIAL_", "CLASS_NOT_IN_LOCAL_CATALOG", "CLASS_NOT_IN_OFFICIAL_TABLE", "CLASS_NOT_YET_IN_FORCE", "CLASS_NOT_USABLE_IN_NFE_NFCE", "LOCAL_RTC_CATALOG_INVALID")
EVIDENCE_REASONS = ("NO_VALID_NCM", "COMPANY_REGIME_UNKNOWN")
RTC_TYPES = ("CCLASSTRIB_MISSING", "CCLASSTRIB_MISSING_UNRESOLVED", "CCLASSTRIB_AMBIGUOUS", "CCLASSTRIB_CONFLICT", "CCLASSTRIB_OFFICIAL_MISMATCH", "RTC_LOCAL_CATALOG_INCOMPATIBLE",
"RTC_RULE_EVIDENCE_MISSING", "CCLASSTRIB_INVALID_FORMAT", "CCLASSTRIB_NOT_IN_CATALOG", "RTC_CST_MISSING", "RTC_CATALOG_INVALID")
BAD_LOCAL_STATES = ("MISMATCH", "EXPIRED", "MISSING_OFFICIAL")
def _text(value) -> str:
return "" if value is None else str(value).strip()
def slim(resolution: Resolution) -> dict:
"""Small, audit-grade view kept in the job result (full evidence stays in the resolver output)."""
return {"status": resolution.status, "rule_id": resolution.rule_id, "rule_class": resolution.rule_class, "reason": resolution.reason, "resolved_cclasstrib": resolution.resolved_cclasstrib,
"resolved_cst_ibscbs": resolution.resolved_cst_ibscbs, "confidence_type": resolution.confidence_type, "auto_apply_candidate": resolution.auto_apply_candidate,
"suggested_cclasstrib": resolution.suggested_cclasstrib}
def refine_rtc_finding(found: dict, product: dict, resolution: Resolution | None, compat_state: dict | None) -> dict:
"""Mutates and returns `found` (an `evaluate_product` result). Only acts when RTC is REQUIRED for the product."""
if found.get("applicability", {}).get("RTC") != "REQUIRED" or resolution is None:
return found
value = _text(product.get("CCLASSTRIB"))
rtc_blockers = [b for b in found["blockers"] if b["category"] == "RTC"]
kinds = {b["type"] for b in rtc_blockers}
found["rtc_resolution"] = slim(resolution)
if "RTC_CATALOG_INVALID" in kinds:
return found # nothing can be proposed or compared against an invalid catalog
others = [b for b in found["blockers"] if b["category"] != "RTC"]
if "CCLASSTRIB_MISSING" in kinds:
if resolution.auto_apply_candidate:
new = ("CCLASSTRIB_MISSING", f"READY_TO_APPLY:{resolution.resolved_cclasstrib}:{resolution.rule_id}")
elif resolution.status == "SUGGESTION_ONLY":
new = ("CCLASSTRIB_MISSING_UNRESOLVED", f"{resolution.rule_id}:SUGGESTION_ONLY:{resolution.resolved_cclasstrib}")
elif resolution.status == "AMBIGUOUS":
new = ("CCLASSTRIB_AMBIGUOUS", f"{resolution.rule_id}:{resolution.reason}")
elif resolution.reason.startswith(RTC_CATALOG_REASONS):
new = ("RTC_LOCAL_CATALOG_INCOMPATIBLE", resolution.reason)
elif resolution.reason in EVIDENCE_REASONS:
new = ("RTC_RULE_EVIDENCE_MISSING", resolution.reason)
else:
new = ("CCLASSTRIB_MISSING_UNRESOLVED", f"{resolution.rule_id}:{resolution.reason}")
blocker = {"type": new[0], "category": "RTC", "reason": new[1]}
if resolution.reason in HUMAN_REASONS:
blocker["human"] = HUMAN_REASONS[resolution.reason]
found["blockers"] = others + [blocker]
return found
if not rtc_blockers and value: # a valid existing class: compare, never overwrite
state = (compat_state or {}).get(value)
if state in BAD_LOCAL_STATES:
found["blockers"] = others + [{"type": "CCLASSTRIB_OFFICIAL_MISMATCH", "category": "RTC", "reason": f"LOCAL_VS_OFFICIAL:{state}"}]
elif resolution.auto_apply_candidate and resolution.resolved_cclasstrib != value:
found["blockers"] = others + [{"type": "CCLASSTRIB_CONFLICT", "category": "RTC", "reason": f"HAS:{value};RULE:{resolution.rule_id}:{resolution.resolved_cclasstrib}"}]
else:
found["rtc_resolution"]["no_write_required"] = resolution.auto_apply_candidate and resolution.resolved_cclasstrib == value
return found
def summarize_rtc(findings: dict, compat: dict | None = None) -> dict:
"""Counts for the panel and the audit: valid, auto-resolvable (safe), suggested (diagnostic only), blocked and conflicting, catalog-incompatible."""
out = {"total": 0, "valid_existing": 0, "ready_to_apply": 0, "suggestion_only": 0, "ambiguous": 0, "insufficient": 0, "conflicts": 0, "official_mismatch": 0, "catalog_incompatible": 0,
"not_applicable": 0, "other_blocked": 0, "by_rule": {}, "by_reason": {}}
for found in findings.values():
out["total"] += 1
if found.get("applicability", {}).get("RTC") != "REQUIRED":
out["not_applicable"] += 1
continue
types = {b["type"] for b in found["blockers"] if b["category"] == "RTC"}
res = found.get("rtc_resolution") or {}
if res:
out["by_rule"][res["rule_id"]] = out["by_rule"].get(res["rule_id"], 0) + 1
out["by_reason"][res["reason"]] = out["by_reason"].get(res["reason"], 0) + 1
if not types:
out["valid_existing"] += 1
elif "CCLASSTRIB_CONFLICT" in types:
out["conflicts"] += 1
elif "CCLASSTRIB_OFFICIAL_MISMATCH" in types:
out["official_mismatch"] += 1
elif "RTC_LOCAL_CATALOG_INCOMPATIBLE" in types or "RTC_CATALOG_INVALID" in types:
out["catalog_incompatible"] += 1
elif res.get("auto_apply_candidate") and "CCLASSTRIB_MISSING" in types:
out["ready_to_apply"] += 1
elif res.get("status") == "SUGGESTION_ONLY":
out["suggestion_only"] += 1
elif "CCLASSTRIB_AMBIGUOUS" in types:
out["ambiguous"] += 1
elif types & {"CCLASSTRIB_MISSING_UNRESOLVED", "RTC_RULE_EVIDENCE_MISSING", "CCLASSTRIB_MISSING"}:
out["insufficient"] += 1
else:
out["other_blocked"] += 1
if compat:
out["catalog_compatibility"] = {"compatibility": compat["compatibility"], "counts": compat["counts"], "expired_present_locally": compat.get("expired_present_locally", 0)}
return out
File diff suppressed because it is too large Load Diff
+173
View File
@@ -0,0 +1,173 @@
"""SSO from GTI Gerenciador: the panel is the identity authority.
Flow (docs/PILOT_DEPLOYMENT.md, "Human access"):
GET /login -> Hub sets a one-time state cookie and redirects to the
panel's /sso/fiscal?state=...&next=...
panel -> for a logged-in, active employee signs a JWT (Ed25519,
60 s, unique jti, nonce=state) and auto-POSTs it to
POST /auth/sso/callback -> verify_assertion + check state + consume_jti +
provision -> Hub session of its own.
The Hub only holds the panel's *public* key, never the panel's cookies,
passwords or database. The state cookie binds the assertion to the browser
that started the flow, so nobody can push their own assertion into someone
else's browser (login CSRF).
"""
from __future__ import annotations
import base64
import hmac
import re
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from urllib.parse import urlsplit
import jwt
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
from sqlalchemy import delete, select
from sqlalchemy.exc import IntegrityError
from . import config
from .models import AdminUser, SsoUsedAssertion
ISSUER = "gti-gerenciador"
AUDIENCE = "gti-fiscal"
AUTH_SOURCE = "GTI_GERENCIADOR"
REQUIRED_CLAIMS = ["iss", "aud", "sub", "iat", "nbf", "exp", "jti", "nonce"]
MAX_LIFETIME_SECONDS = 120 # the panel issues 60 s; anything longer is refused
CLOCK_LEEWAY_SECONDS = 10
STATE_COOKIE = "gti_sso_state"
STATE_COOKIE_PATH = "/auth/sso"
STATE_SECONDS = 300
MAX_NEXT_LENGTH = 512
_SUBJECT = re.compile(r"funcionario:[1-9][0-9]{0,18}")
_TOKEN_CHARS = re.compile(r"[A-Za-z0-9_\-]{16,128}")
class SsoRejected(Exception):
"""Reason codes are stable: they go to the audit log (never the token)."""
def __init__(self, reason: str, subject: str | None = None):
super().__init__(reason)
self.reason = reason
self.subject = subject
@dataclass(frozen=True)
class Identity:
subject: str
login: str
name: str
role: str | None
jti: str
expires_at: datetime
def enabled() -> bool:
return bool(config.settings.sso_public_key) and panel_home() is not None
def panel_home() -> str | None:
"""https origin of GTI Gerenciador, from SSO_START_URL (None if unset/not https)."""
parts = urlsplit(config.settings.sso_start_url or "")
if parts.scheme != "https" or not parts.hostname or parts.username or parts.password:
return None
return f"https://{parts.netloc}/"
def safe_next(value: str | None) -> str:
"""Only a relative path inside the Hub; anything else becomes "/"."""
if (
not value
or len(value) > MAX_NEXT_LENGTH
or not value.startswith("/")
or value.startswith("//")
or "\\" in value
or any(ord(char) < 32 or ord(char) == 127 for char in value)
):
return "/"
return value
def valid_state(value: str | None) -> bool:
return bool(value) and bool(_TOKEN_CHARS.fullmatch(value))
def _public_key() -> Ed25519PublicKey:
raw = config.settings.sso_public_key.strip()
return Ed25519PublicKey.from_public_bytes(base64.urlsafe_b64decode(raw + "=" * (-len(raw) % 4)))
def verify_assertion(token: str | None, expected_state: str | None) -> Identity:
if not token or len(token) > 4096:
raise SsoRejected("MISSING_ASSERTION")
try:
claims = jwt.decode(
token, _public_key(), algorithms=["EdDSA"], audience=AUDIENCE, issuer=ISSUER,
leeway=CLOCK_LEEWAY_SECONDS, options={"require": REQUIRED_CLAIMS},
)
except jwt.ExpiredSignatureError as error:
raise SsoRejected("EXPIRED") from error
except jwt.ImmatureSignatureError as error:
raise SsoRejected("NOT_YET_VALID") from error
except jwt.InvalidAudienceError as error:
raise SsoRejected("WRONG_AUDIENCE") from error
except jwt.InvalidIssuerError as error:
raise SsoRejected("WRONG_ISSUER") from error
except jwt.InvalidAlgorithmError as error:
raise SsoRejected("INVALID_ALGORITHM") from error
except jwt.InvalidSignatureError as error:
raise SsoRejected("INVALID_SIGNATURE") from error
except jwt.MissingRequiredClaimError as error:
raise SsoRejected("MISSING_CLAIM") from error
except jwt.InvalidTokenError as error:
raise SsoRejected("MALFORMED") from error
subject = claims["sub"]
if not isinstance(subject, str) or not _SUBJECT.fullmatch(subject):
raise SsoRejected("INVALID_SUBJECT")
if not isinstance(claims["iat"], int) or not isinstance(claims["exp"], int) or claims["exp"] - claims["iat"] > MAX_LIFETIME_SECONDS:
raise SsoRejected("LIFETIME_TOO_LONG", subject)
if not isinstance(claims["jti"], str) or not _TOKEN_CHARS.fullmatch(claims["jti"]) or len(claims["jti"]) > 64:
raise SsoRejected("INVALID_JTI", subject)
nonce = claims["nonce"]
if not valid_state(expected_state) or not isinstance(nonce, str) or not hmac.compare_digest(nonce, expected_state):
raise SsoRejected("STATE_MISMATCH", subject)
login, name, role = claims.get("login"), claims.get("name"), claims.get("perfil")
if not isinstance(login, str) or not login.strip() or not isinstance(name, str):
raise SsoRejected("MISSING_PROFILE", subject)
return Identity(
subject=subject, login=login.strip()[:128], name=name.strip()[:160] or login.strip()[:160],
role=role[:32] if isinstance(role, str) else None, jti=claims["jti"],
expires_at=datetime.fromtimestamp(claims["exp"], tz=timezone.utc),
)
def consume_jti(session, identity: Identity) -> None:
"""Records the jti; a second use of the same assertion is a replay."""
now = datetime.now(timezone.utc)
session.execute(delete(SsoUsedAssertion).where(SsoUsedAssertion.expires_at < now - timedelta(days=1)))
session.add(SsoUsedAssertion(jti=identity.jti, subject=identity.subject, expires_at=identity.expires_at))
try:
session.flush()
except IntegrityError as error:
session.rollback()
raise SsoRejected("REPLAY", identity.subject) from error
def provision(session, identity: Identity) -> AdminUser:
"""Creates the collaborator on first access, refreshes name/role afterwards."""
user = session.scalar(select(AdminUser).where(AdminUser.external_subject == identity.subject))
if user is None:
user = AdminUser(
username=f"gerenciador:{identity.subject.split(':', 1)[1]}", password_hash=None, auth_source=AUTH_SOURCE,
external_subject=identity.subject, display_name=identity.name, external_role=identity.role, is_active=True,
)
session.add(user)
session.flush()
return user
if user.auth_source != AUTH_SOURCE:
raise SsoRejected("SUBJECT_BOUND_TO_LOCAL_ACCOUNT", identity.subject)
if not user.is_active:
raise SsoRejected("DISABLED_IN_FISCAL", identity.subject)
user.display_name, user.external_role = identity.name, identity.role
return user
@@ -0,0 +1,273 @@
"""UNIFIED fiscal completeness: ONE definition of "complete" for a product (pure, no I/O).
mandatory(product, company, rtc) = base fiscal fields (decided elsewhere: NCM/CEST/CFOP/ICMS/IPI ...)
+ PIS/COFINS when they APPLY to the product
+ RTC (cClassTrib -> CST IBS/CBS) ONLY when the ERP has the tax reform switched on.
There is no "legacy scope" and no "expanded scope": every finding here becomes a normal blocker in the single queue and
changes the single coverage number. Nothing here writes to the ERP or invents a fiscal value.
Sources (see docs/pis_cofins_applicability.md): SPED EFD-Contribuicoes table 4.3.3/4.3.4 (CST domain), 4.3.10-4.3.15 (special regimes),
LC 123/2006 art. 13/18 (PIS/Cofins inside the Simples DAS), Leis 9.718/98, 10.637/2002, 10.833/2003 (cumulative / non-cumulative),
NT 2025.002-RTC v1.51 (IBS/CBS fields), LC 214/2025 art. 348.
"""
from __future__ import annotations
import re
import unicodedata
from datetime import date
# Official CST PIS / CST COFINS domain (SPED tables 4.3.3 / 4.3.4). Membership only - never used to pick a value.
CST_DOMAIN = frozenset([f"{n:02d}" for n in (*range(1, 10), 49, *range(50, 57), *range(60, 68), *range(70, 76), 98, 99)])
RATE_REQUIRED_CST = frozenset({"01", "02"}) # taxable by percentage: a rate is needed
RATE_NOT_APPLICABLE_CST = frozenset({"03", "04", "05", "06", "07", "08", "09"}) # per-unit amount / monophasic / ST / zero / exempt / no incidence / suspended
INCIDENCE_CST = {"MONOFASICA": frozenset({"02", "03", "04"}), "SUBSTITUICAO": frozenset({"05"})} # 4.3.10/4.3.11 (02,03,04) and 4.3.12 (05)
SIMPLES_FAMILY = frozenset({"1", "2", "4"}) # CRT 1 Simples, 2 Simples excess sublimit, 4 MEI (LC 123/2006): PIS/Cofins are collected inside the DAS
PIS_COFINS_FIELDS = ("COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS")
FIELD_BY_BLOCKER_PREFIX = (("PIS", "PIS"), ("COFINS", "COFINS"), ("CCLASSTRIB", "RTC"), ("RTC", "RTC"))
def _text(value) -> str:
return "" if value is None else str(value).strip()
def blocker_category(blocker_type: str) -> str:
"""Dashboard category of a blocker type: PIS | COFINS | RTC | OTHER."""
for prefix, category in FIELD_BY_BLOCKER_PREFIX:
if blocker_type.startswith(prefix):
return category
return "OTHER"
def incidence_kind(value) -> str:
"""NONE | MONOFASICA | SUBSTITUICAO | UNKNOWN (accent/case-insensitive, encoding tolerant). Anything the SGV does not define is UNKNOWN, never NONE."""
raw = _text(value)
text = re.sub(r"[^A-Z]", "", unicodedata.normalize("NFKD", raw).encode("ascii", "ignore").decode().upper())
if not raw or text == "NENHUMA":
return "NONE"
if text.startswith("MONOF"):
return "MONOFASICA"
if text.startswith("SUBST"):
return "SUBSTITUICAO"
return "UNKNOWN"
def canonical_cst(value) -> str | None:
"""The official 2-digit CST this text stands for, or None. Exactly two official digits, or ONE digit whose zero-padded form is an official code
("4" -> "04"). Nothing else is ever normalised ("049", "000", "2x" stay invalid)."""
code = _text(value)
if code in CST_DOMAIN:
return code
if len(code) == 1 and code.isdigit() and code.zfill(2) in CST_DOMAIN:
return code.zfill(2)
return None
def validate_cst(value) -> str:
"""EMPTY | VALID | VALID_AFTER_NORMALIZATION | INVALID."""
code = _text(value)
if not code:
return "EMPTY"
if code in CST_DOMAIN:
return "VALID"
return "VALID_AFTER_NORMALIZATION" if canonical_cst(code) else "INVALID"
def validate_rate(value) -> str:
text = _text(value)
if not text:
return "EMPTY"
if not re.fullmatch(r"\d{1,3}([.,]\d{1,4})?", text):
return "INVALID"
return "VALID" if float(text.replace(",", ".")) <= 100 else "INVALID"
# ------------------------------------------------------------------------------------------------- PIS / COFINS
def pis_cofins_applicability(product: dict, crt) -> dict:
"""{"state": NOT_APPLICABLE | REQUIRED | OPTIONAL | NEEDS_REVIEW, "basis": ...}.
Decided ONLY from signals that are independent of the six CST/rate fields - an empty field is a data state, never proof of "not applicable":
* the company regime (CRT);
* the product's own declaration: PISCOFINS flag and INCIDENCIA_PISCOFINS;
* official rules (LC 123/2006 art. 13 IV/V and art. 18 par. 4-A I; Leis 9.718/98, 10.637/02, 10.833/03; SPED 4.3.x for the CST domain/semantics).
REQUIRED the product declares a special treatment (flag S, or incidence MONOFASICA / SUBSTITUICAO): the exit CST must be valid, whatever the fields hold.
NOT_APPLICABLE Simples family (CRT 1/2/4) with no special treatment declared: PIS/Cofins are collected inside the DAS (LC 123 art. 13), so the product carries no
obligation. The same verdict holds whether the fields are empty or typed.
OPTIONAL taxed regime (CRT 3 or other) AND the product explicitly declares "no special treatment" (flag N + incidence NENHUMA): the tax applies but the
company default covers it, so the product default is not mandatory; anything typed must still be valid.
NEEDS_REVIEW the declaration cannot be read: incidence text outside the SGV vocabulary, or a taxed regime with a blank flag/incidence.
"""
flag = _text(product.get("PISCOFINS")).upper()
raw_incidence = _text(product.get("INCIDENCIA_PISCOFINS"))
kind = incidence_kind(raw_incidence)
if kind == "UNKNOWN":
return {"state": "NEEDS_REVIEW", "basis": "INCIDENCE_NOT_DEFINED_BY_SGV"}
if flag == "S" or kind in ("MONOFASICA", "SUBSTITUICAO"):
return {"state": "REQUIRED", "basis": "PRODUCT_DECLARES_SPECIAL_TREATMENT"}
if _text(crt) in SIMPLES_FAMILY:
return {"state": "NOT_APPLICABLE", "basis": "SIMPLES_DAS_NO_SPECIAL_TREATMENT"}
if flag == "N" and raw_incidence:
return {"state": "OPTIONAL", "basis": "TAXED_REGIME_COMPANY_DEFAULT_COVERS"}
return {"state": "NEEDS_REVIEW", "basis": "NO_DECLARATION_IN_TAXED_REGIME"}
def evaluate_pis_cofins(product: dict, crt) -> dict:
"""{"applicability": {...}, "blockers": [(type, reason)], "warnings": [(type, reason)]}. Entry and exit are separate; PIS and COFINS independent.
Blockers exist only where the field applies (REQUIRED / OPTIONAL). Non-blocking warnings report data-quality facts (non-canonical or ignored values)."""
app = pis_cofins_applicability(product, crt)
blockers: list[tuple[str, str]] = []
warnings: list[tuple[str, str]] = []
kind = incidence_kind(product.get("INCIDENCIA_PISCOFINS"))
flag = _text(product.get("PISCOFINS")).upper()
applies = app["state"] in ("REQUIRED", "OPTIONAL")
for tax in ("PIS", "COFINS"):
entry, exit_, rate = product.get(f"COD_ENTRADA_{tax}"), product.get(f"COD_SAIDA_{tax}"), product.get(f"ALIQ_{tax}")
entry_state, exit_state, rate_state = validate_cst(entry), validate_cst(exit_), validate_rate(rate)
for slot, value, state in (("ENTRY", entry, entry_state), ("EXIT", exit_, exit_state)):
if state == "VALID_AFTER_NORMALIZATION":
warnings.append((f"CST_{tax}_NON_CANONICAL_FORMAT", f"{slot}:{_text(value)}->{canonical_cst(value)}"))
elif state == "INVALID":
if applies:
blockers.append((f"{tax}_CST_INVALID", slot))
else:
warnings.append((f"CST_{tax}_INVALID_IGNORED", f"{slot}:{_text(value)}"))
if not applies:
continue
if app["state"] == "REQUIRED" and exit_state == "EMPTY":
blockers.append((f"{tax}_CST_MISSING", "EXIT"))
if rate_state == "INVALID":
blockers.append((f"{tax}_RATE_INVALID", "RATE"))
code = canonical_cst(exit_)
if code:
if code in RATE_REQUIRED_CST and rate_state == "EMPTY":
blockers.append((f"{tax}_RATE_MISSING", f"CST_{code}_NEEDS_RATE"))
if kind in INCIDENCE_CST and code not in INCIDENCE_CST[kind]:
blockers.append((f"{tax}_CST_INCIDENCE_MISMATCH", f"{kind}:CST_{code}"))
if kind == "NONE" and app["state"] == "REQUIRED" and code in RATE_NOT_APPLICABLE_CST - {"03"} and flag == "S":
blockers.append((f"{tax}_CST_INCIDENCE_MISMATCH", f"NENHUMA:CST_{code}"))
if applies and flag == "N" and kind != "NONE":
blockers.append(("PIS_COFINS_FLAG_INCONSISTENT", "FLAG_N_WITH_SPECIAL_INCIDENCE"))
if applies and flag == "S" and kind == "NONE" and not any(_text(product.get(c)) for c in PIS_COFINS_FIELDS):
blockers.append(("PIS_COFINS_FLAG_INCONSISTENT", "FLAG_S_WITHOUT_INCIDENCE_OR_CODES"))
return {"applicability": app, "blockers": _dedupe(blockers), "warnings": _dedupe(warnings)}
def _dedupe(items):
seen, out = set(), []
for item in items:
if item not in seen:
seen.add(item)
out.append(item)
return out
# ------------------------------------------------------------------------------------------------- RTC
CCLASSTRIB_EFFECTIVE_FROM = {
"3": date(2026, 8, 3),
"1": date(2027, 1, 1),
"2": date(2027, 1, 1),
"4": date(2027, 1, 1),
}
def rtc_enabled(caps: dict) -> bool:
"""RTC is in scope ONLY when the ERP is switched to the reform (PARAMETROS.USAR_REFORMA_TRIBUTARIA truthy) AND its schema can hold it."""
return bool(caps.get("HAS_PRODUCT_CCLASSTRIB") and caps.get("HAS_RTC_RELATIONSHIP") and caps.get("RTC_ENABLED_BY_PARAM"))
def cclasstrib_applicability(crt, rtc_enabled: bool, today: date | None = None) -> dict:
"""Company-level cClassTrib requiredness from live CRT and its effective date.
The ERP RTC switch scopes the feature; it does not make cClassTrib mandatory
for every company/product. Unknown regimes fail closed to NOT_APPLICABLE.
"""
today = today or date.today()
regime = _text(crt)
if not rtc_enabled:
return {"state": "NOT_APPLICABLE", "basis": "RTC_DISABLED_IN_ERP", "effective_from": None, "source": "PARAMETROS.USAR_REFORMA_TRIBUTARIA"}
effective_from = CCLASSTRIB_EFFECTIVE_FROM.get(regime)
if effective_from is None:
return {"state": "NOT_APPLICABLE", "basis": "COMPANY_CRT_UNRESOLVED", "effective_from": None, "source": "C000004.CRT"}
if today < effective_from:
basis = "SIMPLES_CCLASSTRIB_NOT_EFFECTIVE" if regime in SIMPLES_FAMILY else "CCLASSTRIB_NOT_YET_EFFECTIVE"
return {"state": "NOT_APPLICABLE", "basis": basis, "effective_from": effective_from.isoformat(), "source": "C000004.CRT + RTC effective-date rules"}
return {"state": "REQUIRED", "basis": "CCLASSTRIB_EFFECTIVE_FOR_CRT", "effective_from": effective_from.isoformat(), "source": "C000004.CRT + RTC effective-date rules"}
def catalog_integrity(catalog: dict) -> dict:
"""catalog: {"classes": {cClassTrib: cst_or_None}, "cst": {code, ...}, "duplicates": int}."""
classes, cst = catalog.get("classes", {}), set(catalog.get("cst", ()))
bad_format = [c for c in classes if not re.fullmatch(r"\d{6}", str(c))]
bad_cst_format = [c for c in cst if not re.fullmatch(r"\d{3}", str(c))]
orphans = [c for c, s in classes.items() if s is None or str(s) not in cst]
return {"classes": len(classes), "cst": len(cst), "orphan_classes": len(orphans), "bad_class_format": len(bad_format), "bad_cst_format": len(bad_cst_format),
"duplicates": int(catalog.get("duplicates", 0)), "valid": bool(classes and cst and not orphans and not bad_format and not bad_cst_format and not catalog.get("duplicates"))}
def validate_cclasstrib(value, catalog: dict | None, today: date | None = None) -> str:
"""EMPTY | INVALID_FORMAT | NOT_IN_CATALOG | EXPIRED | CST_MISSING | VALID (CST IBS/CBS derived through the catalog, never stored on the product)."""
code = _text(value)
if not code:
return "EMPTY"
if not re.fullmatch(r"\d{6}", code):
return "INVALID_FORMAT"
if not catalog or code not in catalog.get("classes", {}):
return "NOT_IN_CATALOG"
validity = (catalog.get("validity") or {}).get(code)
if validity and today is not None:
start, end = validity
if (start and start > today) or (end and end < today):
return "EXPIRED"
cst = catalog["classes"][code]
return "VALID" if cst is not None and str(cst) in set(catalog.get("cst", ())) else "CST_MISSING"
def evaluate_rtc(product: dict, rtc: dict | None, today: date | None = None, crt=None) -> dict:
"""Validate cClassTrib only when RTC is on and the company's regime/date gate applies."""
app = rtc.get("cclasstrib_applicability") if isinstance(rtc, dict) else None
if not isinstance(app, dict):
app = cclasstrib_applicability(crt, bool(rtc and rtc.get("enabled")), today)
if app["state"] != "REQUIRED":
return {"applicability": app, "blockers": []}
if not rtc.get("catalog_valid"):
return {"applicability": app, "blockers": [("RTC_CATALOG_INVALID", "LOCAL_CATALOG_FAILED_INTEGRITY")]}
state = validate_cclasstrib(product.get("CCLASSTRIB"), rtc.get("catalog"), today)
mapping = {"EMPTY": "CCLASSTRIB_MISSING", "INVALID_FORMAT": "CCLASSTRIB_INVALID_FORMAT", "NOT_IN_CATALOG": "CCLASSTRIB_NOT_IN_CATALOG", "EXPIRED": "CCLASSTRIB_NOT_IN_CATALOG", "CST_MISSING": "RTC_CST_MISSING"}
return {"applicability": app, "blockers": [(mapping[state], state)] if state in mapping else []}
# ------------------------------------------------------------------------------------------------- unified
def evaluate_product(product: dict, company: dict | None, rtc: dict | None, today: date | None = None) -> dict:
"""The single fiscal-completeness verdict for the fields this module owns.
Returns {"blockers": [{"type","category","reason"}], "applicability": {"PIS_COFINS": state, "RTC": state}, "mandatory_fields": [...], "needs_review": bool}."""
crt = (company or {}).get("crt")
pis = evaluate_pis_cofins(product, crt)
rtc_result = evaluate_rtc(product, rtc, today, crt)
blockers = [{"type": kind, "category": blocker_category(kind), "reason": reason} for kind, reason in pis["blockers"] + rtc_result["blockers"]]
warnings = [{"type": kind, "reason": reason} for kind, reason in pis["warnings"]]
mandatory = []
if pis["applicability"]["state"] == "REQUIRED":
mandatory += ["COD_SAIDA_PIS", "COD_SAIDA_COFINS"]
if rtc_result["applicability"]["state"] == "REQUIRED":
mandatory.append("CCLASSTRIB")
return {"blockers": blockers, "applicability": {"PIS_COFINS": pis["applicability"]["state"], "RTC": rtc_result["applicability"]["state"]},
"basis": {"PIS_COFINS": pis["applicability"]["basis"], "RTC": rtc_result["applicability"]["basis"]}, "mandatory_fields": mandatory, "warnings": warnings,
"needs_review": pis["applicability"]["state"] == "NEEDS_REVIEW"}
def legal_effective_context(crt) -> dict:
"""Summarize the broad IBS/CBS document timeline; cClassTrib requiredness has its own CRT/date gate above.
Source: Ato Conjunto RFB/CGIBS n 4/2026 (DOU 31/07/2026) art. 1: NF-e (55) and NFC-e (65) from 2026-08-03; par. 1: Simples optantes from 2027-01-01."""
simples = _text(crt) in SIMPLES_FAMILY
return {"crt": _text(crt) or None, "informational_only": True,
"ibs_cbs_in_documents": "MANDATORY_FROM_2027-01-01" if simples else ("MANDATORY_FROM_2026-08-03" if _text(crt) == "3" else "UNKNOWN_REGIME"),
"cclasstrib_orientation": "PENDING_OFFICIAL_NT" if simples else ("TABLE_AND_LC214_ANNEXES" if _text(crt) == "3" else "UNKNOWN_REGIME"),
"sources": ["Ato Conjunto RFB/CGIBS n 4/2026 (DOU 31/07/2026) art. 1 I, II and par. 1", "NT 2025.002-RTC v1.51 (Portal NF-e, 04/08/2026): CRT 1/2/4 orientations in a future NT", "LC 214/2025 art. 348"]}
+8
View File
@@ -0,0 +1,8 @@
"""Single source of truth for the release version. Also what the Agent Windows
package is versioned as (scripts/build_agent_package.py reads release/VERSION,
kept identical to this) and what an AgentRelease row's `version` column holds
once published — never a second, independent version number. Fiscal semantics
are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py,
DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump."""
GTI_FISCAL_VERSION = "1.3.15"
API_COMPAT_MAJOR_MINOR = "1.0" # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x
+882
View File
@@ -0,0 +1,882 @@
"""Server-rendered operator pages for the GTI Fiscal control panel."""
import secrets
import uuid
from datetime import datetime, timedelta, timezone
from urllib.parse import parse_qs, urlencode
from fastapi import APIRouter, Request
from fastapi.responses import HTMLResponse, RedirectResponse
from fastapi.templating import Jinja2Templates
from sqlalchemy import func, select, update
from . import agent_distribution, client_sync, config, enrollment, sso
from .config import settings
from .blocker_resolution import BlockerResolutionUnavailable, _company_key, resolve_blocker_with_context
from .context_keys import ALLOWED_KEYS, ENUMS
from .installation_identity import reference_agent_version
from .panel_presentation import ENRICHMENT_MODES, ENRICHMENT_MODE_LABELS, RTC_MODE_EXPLANATION, RTC_MODE_LABELS, abort_reason_label, audit_outcome_label, candidate_status_label, catalog_state_label, job_kind_label, job_status_label, rtc_rule_label, rtc_status_view
from .control_plane import blocker_detail, blocker_filter_summary, dashboard_summary, installation_view, jobs_status_summary, list_blockers, list_installations, list_jobs, list_panel_events, rtc_blocker_summary
from .human_auth import create_session, current_session, require_csrf, verify_password
from .panel_presentation import format_ui_datetime
from .models import AdminUser, Agent, AgentEnrollment, AgentRelease, AgentSnapshot, BlockedProduct, FiscalContextEntry, Installation, Job, ProductEnrichment, ProductFieldAudit, RtcCandidate, RtcWriteAudit, Tenant
from .jobs import create_job
from .audit import record_audit
from .enrichment import kb as kb_mod, status as enrichment_status
from .fiscal_context_ui import build_fiscal_context_view, set_fiscal_context_decision
from .enrichment.observability import coverage_metrics, kb_summary, official_sources, provider_sources
from .reference_registry import RegistryError, approve_draft, diff_against_vigente, list_registry, reject_draft, technical_checklist
def router(session_factory):
routes = APIRouter()
templates = Jinja2Templates(directory="central/templates")
def auth(request, session):
return current_session(request, session)
def render(request, template, context=None, status_code=200):
values = {"request": request, "csrf_token": request.cookies.get("gti_csrf", ""), "panel_home": sso.panel_home(), **(context or {})}
return templates.TemplateResponse(request=request, name=template, context=values, status_code=status_code)
def client_ip(request):
return request.client.host if request.client else None
def _aware(value: datetime | None) -> datetime | None:
"""SQLite (used by the test suite) drops tzinfo on round-trip even for
a DateTime(timezone=True) column; Postgres never does. Normalize to
UTC before comparing against datetime.now(timezone.utc)."""
if value is None or value.tzinfo is not None:
return value
return value.replace(tzinfo=timezone.utc)
def set_session_cookies(response, token, csrf, seconds):
cfg = config.settings
response.set_cookie(cfg.web_cookie_name, token, max_age=seconds, httponly=True, secure=cfg.web_cookie_secure, samesite="lax")
response.set_cookie("gti_csrf", csrf, max_age=seconds, httponly=False, secure=cfg.web_cookie_secure, samesite="lax")
def query_base(**values):
return urlencode({key: value for key, value in values.items() if value not in (None, "")})
def date_filter(value, *, end_of_day=False):
if not value:
return None
try:
parsed = datetime.fromisoformat(value)
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.replace(hour=23, minute=59, second=59, microsecond=999999) if end_of_day and len(value) == 10 else parsed
except ValueError:
return None
@routes.get("/login", response_class=HTMLResponse)
def login_page(request: Request, next: str | None = None):
"""Collaborators never see a Hub login: with SSO configured this starts
the flow at GTI Gerenciador (which shows its own login if needed)."""
if not sso.enabled():
return render(request, "login.html", {"error": None, "emergency": False})
cfg = config.settings
state = secrets.token_urlsafe(24)
response = RedirectResponse(f"{cfg.sso_start_url}?{urlencode({'state': state, 'next': sso.safe_next(next)})}", status_code=303)
# Sent back on the cross-site POST from the panel, hence SameSite=None (+Secure).
response.set_cookie(sso.STATE_COOKIE, state, max_age=sso.STATE_SECONDS, path=sso.STATE_COOKIE_PATH, httponly=True,
secure=cfg.web_cookie_secure, samesite="none" if cfg.web_cookie_secure else "lax")
return response
@routes.get("/login/local", response_class=HTMLResponse)
def emergency_login_page(request: Request):
"""Break-glass only (local AdminUser with a password). Not linked from the UI."""
return render(request, "login.html", {"error": None, "emergency": True})
@routes.post("/login")
async def login(request: Request):
values = await form_values(request)
username = values.get("username", "").strip().lower()
with session_factory() as session:
admin = session.scalar(select(AdminUser).where(AdminUser.username == username))
if not admin or not admin.is_active or admin.auth_source != "LOCAL" or not verify_password(values.get("password", ""), admin.password_hash):
record_audit(session, admin_user_id=None, action="LOGIN_LOCAL_FAILURE", target_type="admin_user", target_id=username[:128] or "-", context={"ip": client_ip(request)}, severity="WARNING")
session.commit()
return render(request, "login.html", {"error": "Credenciais inválidas", "emergency": True}, 401)
token, csrf = create_session(session, admin, request)
record_audit(session, admin_user_id=admin.id, action="LOGIN_LOCAL_SUCCESS", target_type="admin_user", target_id=admin.username, context={"ip": client_ip(request)}, severity="WARNING")
session.commit()
response = RedirectResponse("/", status_code=303)
set_session_cookies(response, token, csrf, config.settings.web_session_seconds)
return response
@routes.post("/auth/sso/callback")
async def sso_callback(request: Request):
values = await form_values(request)
with session_factory() as session:
try:
if not sso.enabled():
raise sso.SsoRejected("SSO_DISABLED")
identity = sso.verify_assertion(values.get("assertion"), request.cookies.get(sso.STATE_COOKIE))
sso.consume_jti(session, identity)
user = sso.provision(session, identity)
seconds = config.settings.federated_session_seconds
token, csrf = create_session(session, user, request, seconds=seconds)
record_audit(session, admin_user_id=user.id, action="SSO_LOGIN_SUCCESS", target_type="admin_user", target_id=identity.subject,
context={"ip": client_ip(request), "login": identity.login, "role": identity.role})
session.commit()
except sso.SsoRejected as rejected:
session.rollback()
record_audit(session, admin_user_id=None, action="SSO_LOGIN_REJECTED", target_type="sso", target_id=rejected.subject or "-",
context={"ip": client_ip(request), "reason": rejected.reason}, severity="WARNING")
session.commit()
response = render(request, "sso_error.html", {"reason": rejected.reason}, 401)
response.delete_cookie(sso.STATE_COOKIE, path=sso.STATE_COOKIE_PATH)
return response
response = RedirectResponse(sso.safe_next(values.get("next")), status_code=303)
set_session_cookies(response, token, csrf, seconds)
response.delete_cookie(sso.STATE_COOKIE, path=sso.STATE_COOKIE_PATH)
return response
@routes.post("/logout")
async def logout(request: Request):
"""Ends only the Hub session. Coming back through GTI Gerenciador signs in
again without a password (documented in docs/PILOT_DEPLOYMENT.md)."""
with session_factory() as session:
web_session, admin = current_session(request, session)
if web_session:
values = await form_values(request)
require_csrf(web_session, values.get("csrf_token"))
record_audit(session, admin_user_id=admin.id, action="LOGOUT", target_type="admin_user", target_id=admin.external_subject or admin.username, context={"ip": client_ip(request)})
session.delete(web_session); session.commit()
response = render(request, "logged_out.html", {}) if sso.enabled() else RedirectResponse("/login", status_code=303)
response.delete_cookie(config.settings.web_cookie_name); response.delete_cookie("gti_csrf")
return response
def page_auth(request):
session = session_factory()
web_session, admin = current_session(request, session)
if not admin:
session.close()
wanted = request.url.path + (f"?{request.url.query}" if request.url.query else "")
return None, None, RedirectResponse(f"/login?{urlencode({'next': wanted})}", status_code=303)
return session, (web_session, admin), None
async def form_values(request):
return {key: items[0] for key, items in parse_qs((await request.body()).decode()).items() if items}
@routes.get("/", response_class=HTMLResponse)
def dashboard(request: Request, kb_q: str | None = None, kb_page: int = 1):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try: return render(request, "dashboard.html", {"summary": dashboard_summary(session), "auth": auth_data, "kb_q": kb_q or "", "is_admin": _is_privileged(auth_data[1]), "csrf_token": request.cookies.get("gti_csrf", ""), "sources": provider_sources(session), "official": official_sources(session), "enrich_metrics": coverage_metrics(session), "kb": kb_summary(session, kb_q, kb_page)})
finally: session.close()
@routes.get("/admin/reference-datasets", response_class=HTMLResponse)
def reference_datasets_page(request: Request):
"""Missão 5 / Fase 1.2: 'Tabelas oficiais' - lista o Reference Data Registry (DRAFT/VIGENTE/
OBSOLETO/REJECTED), com diff contra o VIGENTE atual para cada DRAFT. ADMIN only."""
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
if not _is_privileged(auth_data[1]):
return HTMLResponse("Acesso restrito a administradores.", status_code=403)
rows = list_registry(session)
view = []
for r in rows:
item = {"id": r.id, "kind": r.kind, "provider": r.provider, "version_label": r.version_label, "status": r.status,
"record_count": r.record_count, "downloaded_at": format_ui_datetime(r.downloaded_at), "source_url": r.source_url,
"licence_terms": r.licence_terms, "parse_errors": r.parse_errors, "submitted_by": r.submitted_by,
"approved_by": r.approved_by, "diff": None, "checklist": technical_checklist(session, r)}
if r.status == "DRAFT":
item["diff"] = diff_against_vigente(session, r)
view.append(item)
return render(request, "reference_datasets.html", {"auth": auth_data, "csrf_token": request.cookies.get("gti_csrf", ""), "is_admin": True, "rows": view})
finally: session.close()
@routes.post("/admin/reference-datasets/{dataset_id}/decide")
async def decide_reference_dataset(request: Request, dataset_id: int):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
values = await form_values(request); web_session, admin = auth_data
require_csrf(web_session, values.get("csrf_token"))
if not _is_privileged(admin):
return _deny(session, admin, None, "reference_dataset:decide", str(dataset_id))
action = (values.get("action") or "").strip()
try:
if action == "approve":
dataset = approve_draft(session, dataset_id, admin_name=admin.display_name or admin.username)
elif action == "reject":
dataset = reject_draft(session, dataset_id, admin_name=admin.display_name or admin.username)
else:
return HTMLResponse("ação inválida", status_code=422)
except RegistryError as error:
return HTMLResponse(str(error), status_code=422)
record_audit(session, admin_user_id=admin.id, tenant_id=None, action=f"REFERENCE_DATASET_{action.upper()}", target_type="dataset", target_id=str(dataset_id),
context={"kind": dataset.kind, "version_label": dataset.version_label, "operator": admin.display_name or admin.username}, severity="WARNING")
session.commit()
return RedirectResponse("/admin/reference-datasets", status_code=303)
finally: session.close()
@routes.get("/agents", response_class=HTMLResponse)
def agents(request: Request, page: int = 1, search: str | None = None, health: str | None = None, sort: str = "display_name", scope: str = "active", version: str | None = None):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
scope = scope if scope in ("active", "all") else "active"
health = health if health in ("ONLINE", "DEGRADED", "OFFLINE", "FAILED", "UNKNOWN") else None
sort = sort if sort in ("display_name", "worst_coverage") else "display_name"
version = version if version == "outdated" else None
return render(request, "agents.html", {"result": list_installations(session, page=page, page_size=50, search=search, health=health, sort=sort, scope=scope, version=version), "auth": auth_data, "search": search or "", "health_filter": health or "", "sort": sort, "scope_filter": scope, "version_filter": version or "", "query_base": query_base(search=search, health=health, sort=sort, scope=scope, version=version)})
finally: session.close()
# --- Static /agents/* routes MUST be declared before /agents/{agent_id}:
# Starlette matches in registration order, so a dynamic route declared
# first would capture "/agents/new" and "/agents/releases" and fail the
# int conversion with a 422 (the production bug this ordering prevents).
@routes.get("/agents/releases", response_class=HTMLResponse)
def agent_releases(request: Request):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
releases = session.scalars(select(AgentRelease).order_by(AgentRelease.created_at.desc())).all()
counts = {}
for release_id, in session.execute(select(AgentEnrollment.release_id)).all():
counts[release_id] = counts.get(release_id, 0) + 1
return render(request, "agent_releases.html", {"releases": releases, "installation_counts": counts, "auth": auth_data})
finally: session.close()
@routes.post("/agents/releases/{release_id}/revoke")
async def revoke_release(request: Request, release_id: int):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
values = await form_values(request); web_session, admin = auth_data
require_csrf(web_session, values.get("csrf_token"))
release = session.get(AgentRelease, release_id)
if not release: return HTMLResponse("Release not found", status_code=404)
release.status = "REVOKED"
record_audit(session, admin_user_id=admin.id, action="AGENT_RELEASE_REVOKED", target_type="agent_release", target_id=release.version, context={"platform": release.platform, "architecture": release.architecture})
session.commit()
return RedirectResponse("/agents/releases", status_code=303)
finally: session.close()
def _active_gerenciador_tenants(session, search: str | None = None):
# GTI Gerenciador is the single source of truth for clients (central/app/
# client_sync.py) — a new installation never offers a LOCAL (dev/test) or
# inactive tenant, so it can never pick a client that doesn't exist there.
query = select(Tenant).where(Tenant.source == client_sync.SOURCE, Tenant.active.is_(True))
if search: query = query.where(Tenant.name.ilike(f"%{search}%"))
return session.scalars(query.order_by(Tenant.name)).all()
def _clients_json(tenants):
import json
# "<" escaped so the payload can never close the <script> tag it is embedded in.
return json.dumps([{"id": t.external_client_id, "name": t.name, "code": t.external_code or "", "cnpj": t.external_cnpj or ""} for t in tenants], ensure_ascii=False).replace("<", "\u003c")
@routes.get("/agents/new", response_class=HTMLResponse)
def new_installation_form(request: Request, search: str | None = None):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
tenants = _active_gerenciador_tenants(session, search)
releases = session.scalars(select(AgentRelease).where(AgentRelease.status == "PUBLISHED").order_by(AgentRelease.created_at.desc())).all()
return render(request, "agent_new_installation.html", {"tenants": tenants, "clients_json": _clients_json(tenants), "releases": releases, "auth": auth_data, "error": None, "search": search or ""})
finally: session.close()
@routes.post("/agents/new")
async def create_pending_installation(request: Request):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
values = await form_values(request); web_session, admin = auth_data
require_csrf(web_session, values.get("csrf_token"))
client_id = values.get("client_id") # Gerenciador's real CLIENTE.ID, never the text shown in the combobox
display_name = (values.get("display_name") or "").strip()[:160]
expected_hostname = (values.get("expected_hostname") or "").strip()[:255] or None
release_id = values.get("release_id")
errors = []
tenant = session.scalar(select(Tenant).where(Tenant.source == client_sync.SOURCE, Tenant.external_client_id == int(client_id))) if client_id and client_id.isdigit() else None
release = session.get(AgentRelease, int(release_id)) if release_id and release_id.isdigit() else None
# Server-side, not just a hidden option in the form: a LOCAL or
# inactive tenant can never receive a new installation, matching
# GTI Gerenciador's own active/inactive rule.
if tenant and not tenant.active: tenant = None
if not tenant: errors.append("Selecione um cliente da lista.")
if not display_name: errors.append("Informe um nome para a instalação.")
if not release or release.status != "PUBLISHED": errors.append("Selecione uma versão publicada do Agent.")
if errors:
tenants = _active_gerenciador_tenants(session)
releases = session.scalars(select(AgentRelease).where(AgentRelease.status == "PUBLISHED").order_by(AgentRelease.created_at.desc())).all()
return render(request, "agent_new_installation.html", {"tenants": tenants, "clients_json": _clients_json(tenants), "releases": releases, "auth": auth_data, "error": " ".join(errors), "search": ""}, 400)
enrollment_uuid = str(uuid.uuid4())
enr = AgentEnrollment(enrollment_uuid=enrollment_uuid, tenant_id=tenant.id, display_name=display_name, expected_hostname=expected_hostname, release_id=release.id, status="PENDING", created_by=admin.id)
if values.get("features_selected") == "1": # chosen explicitly at creation; otherwise the installation waits for an admin (WAITING_FEATURE_CONFIGURATION)
mode = (values.get("feature_enrichment_mode") or "DISABLED").strip().upper()
enr.features_selected, enr.feature_corrector = True, values.get("feature_corrector") == "1"
enr.feature_enrichment_mode = mode if mode in ENRICHMENT_MODES else "DISABLED"
session.add(enr)
record_audit(session, admin_user_id=admin.id, tenant_id=tenant.id, action="INSTALLATION_CREATED", target_type="agent_enrollment", target_id=enrollment_uuid, context={"display_name": display_name, "release_version": release.version, "features_selected": bool(enr.features_selected), "corrector": bool(enr.feature_corrector), "enrichment_mode": enr.feature_enrichment_mode})
session.commit()
return RedirectResponse(f"/agents/pending/{enrollment_uuid}", status_code=303)
finally: session.close()
@routes.get("/agents/{agent_id}", response_class=HTMLResponse)
def agent_detail(request: Request, agent_id: str, tab: str = "overview", event_page: int = 1, severity: str | None = None, event_type: str | None = None, job_id: int | None = None, product_code: str | None = None, since: str | None = None, until: str | None = None, enrich_code: str | None = None):
# agent_id is parsed here (not typed int in the path) so a non-numeric
# slug like /agents/abc gets a controlled HTML 404 instead of a raw
# 422 — and can never shadow the static routes above.
try:
agent_id = int(agent_id)
except ValueError:
return HTMLResponse("Agent não encontrado (identificador inválido)", status_code=404)
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
row = session.execute(select(Installation, Agent, Tenant, AgentSnapshot).join(Agent, Agent.id == Installation.agent_id).join(Tenant, Tenant.id == Installation.tenant_id).outerjoin(AgentSnapshot, AgentSnapshot.installation_id == Installation.id).where(Agent.id == agent_id)).first()
if row is None: return HTMLResponse("Agent not found", status_code=404)
installation, agent, tenant, snapshot = row
detail = installation_view(installation, agent, tenant, snapshot, reference=reference_agent_version(session))
event_page = max(1, event_page)
events = list_panel_events(session, installation.id, page=event_page, page_size=25, include_more=True, severity=severity, event_type=event_type, job_id=job_id, product_code=product_code, since=date_filter(since), until=date_filter(until, end_of_day=True))
has_more = len(events) > 25
events = events[:25]
for event in events:
event["occurred_display"] = format_ui_datetime(datetime.fromisoformat(event["occurred_at"]) if event["occurred_at"] else None)
agent_blockers = list_blockers(session, tenant_id=installation.tenant_id, agent_id=agent_id, page=1, page_size=25, scope="all")
fiscal_context = build_fiscal_context_view(session, installation.tenant_id, _company_key(None, installation, None))
return render(request, "agent_detail.html", {"detail": detail, "tab": tab, "blockers": agent_blockers, "blocker_breakdown": blocker_filter_summary(session, tenant_id=installation.tenant_id, agent_id=agent_id, status="OPEN", scope="all")["blocker_breakdown"], "jobs": list_jobs(session, agent_id=agent_id, page=1, page_size=25, scope="all"), "events": events, "event_page": max(1, event_page), "has_more": has_more, "severity": severity or "", "event_type": event_type or "", "job_id": job_id or "", "product_code": product_code or "", "since": since or "", "until": until or "", "event_query_base": query_base(tab=tab, severity=severity, event_type=event_type, job_id=job_id, product_code=product_code, since=since, until=until), "auth": auth_data, "csrf_token": request.cookies.get("gti_csrf", ""), "ops": _ops_view(session, installation, agent), "features": _features_view(session, installation, enrich_code), "enrich_code": enrich_code or "", "is_admin": _is_privileged(auth_data[1]), "fiscal_context": fiscal_context, "cfop_ippt_write_mode": installation.cfop_ippt_write_mode})
finally: session.close()
def _features_view(session, installation, enrich_code=None):
"""FUNCIONALIDADES + small ENRIQUECIMENTO summary of the Agent detail (no separate dashboard)."""
counts = dict(session.execute(select(ProductEnrichment.status, func.count()).where(ProductEnrichment.installation_id == installation.id).group_by(ProductEnrichment.status)).all())
applied = session.scalar(select(func.count()).select_from(ProductFieldAudit).where(ProductFieldAudit.installation_id == installation.id, ProductFieldAudit.outcome == "APPLIED")) or 0
lookup = None
if enrich_code:
row = session.scalar(select(ProductEnrichment).where(ProductEnrichment.installation_id == installation.id, ProductEnrichment.product_code == enrich_code.strip()[:64]))
audits = session.scalars(select(ProductFieldAudit).where(ProductFieldAudit.installation_id == installation.id, ProductFieldAudit.product_code == enrich_code.strip()[:64]).order_by(ProductFieldAudit.id.desc()).limit(20)).all() if row else []
lookup = {"found": row is not None, "row": row, "audits": audits}
tracked = []
for r in session.scalars(select(ProductEnrichment).where(ProductEnrichment.installation_id == installation.id).order_by(ProductEnrichment.id.desc()).limit(15)):
fields = ((r.decisions or {}).get("fields") or {})
ncm = fields.get("CLASSIFICACAO_FISCAL") or {}
tracked.append({"code": r.product_code, "description": r.description, "gtin": r.gtin, "ui_status": r.ui_status or "AGUARDANDO_EVIDENCIA", "label": enrichment_status.LABELS.get(r.ui_status or "AGUARDANDO_EVIDENCIA", ""),
"reason": r.reason_code, "detected": r.detected_at, "last_attempt": r.last_attempt_at, "next_retry": r.next_attempt_at, "kb_status": ncm.get("kb_status"), "ncm_candidate": ncm.get("value"),
"cest_candidate": (fields.get("CEST") or {}).get("value"), "ipi_candidate": (fields.get("IPI") or {}).get("value"), "applied": r.fields_applied or {}, "placeholder": ncm.get("technical_placeholder"),
"reevaluate": bool(r.reevaluate), "mode": installation.enrichment_mode})
mode = installation.enrichment_mode
waiting = not installation.features_configured
return {"configured": bool(installation.features_configured), "waiting": waiting, "corrector_enabled": bool(installation.corrector_enabled), "enrichment_mode": mode,
"mode_label": ENRICHMENT_MODE_LABELS.get(mode, mode), "counts": counts, "total": sum(counts.values()), "applied": applied, "lookup": lookup, "tracked": tracked,
"state_label": "Aguardando configuração de funcionalidades" if waiting else ENRICHMENT_MODE_LABELS.get(mode, mode)}
@routes.post("/agents/{agent_id}/features")
async def set_features(request: Request, agent_id: int):
"""Per-installation functions (ADMIN only, CSRF, audited). Central decision: the Agent has no local switch."""
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
values = await form_values(request); web_session, admin = auth_data
require_csrf(web_session, values.get("csrf_token"))
agent, installation = _rtc_target(session, agent_id)
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
if not _is_privileged(admin):
return _deny(session, admin, installation.tenant_id, "features:set", installation.installation_id)
mode = (values.get("enrichment_mode") or "").strip().upper()
if mode not in ENRICHMENT_MODES:
return HTMLResponse("modo inválido", status_code=422)
if mode == "ENABLED" and (values.get("confirm") or "").strip().upper() != "PREENCHER":
return HTMLResponse("Digite PREENCHER para confirmar o enriquecimento automático", status_code=422)
before = {"corrector": bool(installation.corrector_enabled), "enrichment_mode": installation.enrichment_mode, "configured": bool(installation.features_configured)}
installation.corrector_enabled = values.get("corrector_enabled") == "1"
installation.enrichment_mode = mode
installation.features_configured = True
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="INSTALLATION_FEATURES", target_type="installation", target_id=installation.installation_id,
context={"from": before, "to": {"corrector": installation.corrector_enabled, "enrichment_mode": mode}, "operator": admin.display_name or admin.username, "role": admin.external_role or admin.auth_source}, severity="WARNING")
session.commit()
return RedirectResponse(f"/agents/{agent_id}", status_code=303)
finally: session.close()
@routes.post("/agents/{agent_id}/fiscal-context")
async def set_fiscal_context(request: Request, agent_id: int):
"""Contexto Fiscal (ADMIN only, CSRF, audited): confirm/alter/clear one of the handful of real fiscal decisions field_contract
cannot derive from the ERP alone. Writes through FiscalContextEntry - the same table blocker_resolution already writes to."""
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
values = await form_values(request); web_session, admin = auth_data
require_csrf(web_session, values.get("csrf_token"))
agent, installation = _rtc_target(session, agent_id)
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
if not _is_privileged(admin):
return _deny(session, admin, installation.tenant_id, "fiscal_context:set", installation.installation_id)
key = (values.get("key") or "").strip()
action = (values.get("fc_action") or "confirm").strip()
value = None if action == "clear" else (values.get("value") or "").strip() or None
reason = (values.get("reason") or "").strip() or f"{action} via painel"
company_key = _company_key(None, installation, None)
try:
set_fiscal_context_decision(session, tenant_id=installation.tenant_id, company_key=company_key, key=key, value=value, reason=reason, admin=admin, installation_id=installation.id)
except ValueError as error:
return HTMLResponse(str(error), status_code=422)
session.commit()
return RedirectResponse(f"/agents/{agent_id}#contexto-fiscal", status_code=303)
finally: session.close()
@routes.post("/agents/{agent_id}/cfop-ippt-write")
async def set_cfop_ippt_write_mode(request: Request, agent_id: int):
"""Missão 4 / Bloco E.1: per-installation toggle for the CFOP_IPPT_WRITE_V1 capability
(ADMIN only, CSRF, audited). DISABLED by default; the Agent has no local switch."""
from .cfop_ippt_audit import WRITE_MODES as CFOP_IPPT_MODES
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
values = await form_values(request); web_session, admin = auth_data
require_csrf(web_session, values.get("csrf_token"))
agent, installation = _rtc_target(session, agent_id)
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
if not _is_privileged(admin):
return _deny(session, admin, installation.tenant_id, "cfop_ippt_write:set", installation.installation_id)
mode = (values.get("mode") or "").strip().upper()
if mode not in CFOP_IPPT_MODES:
return HTMLResponse(f"modo inválido (use um de {CFOP_IPPT_MODES})", status_code=422)
if mode != "DISABLED" and (values.get("confirm") or "").strip().upper() != "GRAVAR":
return HTMLResponse("Digite GRAVAR para confirmar um modo que pode escrever no ERP", status_code=422)
before = installation.cfop_ippt_write_mode
installation.cfop_ippt_write_mode = mode
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="CFOP_IPPT_WRITE_MODE", target_type="installation", target_id=installation.installation_id,
context={"from": before, "to": mode, "operator": admin.display_name or admin.username, "role": admin.external_role or admin.auth_source}, severity="WARNING")
session.commit()
return RedirectResponse(f"/agents/{agent_id}#contexto-fiscal", status_code=303)
finally: session.close()
@routes.post("/agents/{agent_id}/enrichment/reevaluate")
async def reevaluate_enrichment(request: Request, agent_id: int):
"""Explicit, idempotent 'evaluate again' for one product (code=...) or every tracked product that is not enriched yet. KB/cache first; providers only when really needed."""
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
values = await form_values(request); web_session, admin = auth_data
require_csrf(web_session, values.get("csrf_token"))
agent, installation = _rtc_target(session, agent_id)
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
if not installation.features_configured or installation.enrichment_mode == "DISABLED":
return HTMLResponse("O enriquecimento está desativado nesta instalação (Funcionalidades).", status_code=409)
code = (values.get("code") or "").strip()[:64]
n = enrichment_status.mark_for_reevaluation(session, installation.id, codes=[code] if code else None)
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="ENRICHMENT_REEVALUATE", target_type="installation", target_id=installation.installation_id,
context={"product": code or "ALL_UNRESOLVED", "flagged": n, "operator": admin.display_name or admin.username})
session.commit()
return RedirectResponse(f"/agents/{agent_id}#enriquecimento", status_code=303)
finally: session.close()
@routes.post("/kb/attest")
async def kb_attest(request: Request):
"""A human fiscal decision (ADMIN only): choose which candidate NCM is right for a GTIN. Only an existing, vigente candidate can be chosen; the others are recorded as rejected."""
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
values = await form_values(request); web_session, admin = auth_data
require_csrf(web_session, values.get("csrf_token"))
if not _is_privileged(admin):
record_audit(session, admin_user_id=admin.id, tenant_id=None, action="KB_ATTEST_DENIED", target_type="kb_product", target_id=values.get("gtin") or "", context={"role": admin.external_role or admin.auth_source}, severity="WARNING")
session.commit()
return HTMLResponse("Ação restrita a administradores do GTI Gerenciador (perfil ADMIN).", status_code=403)
gtin, ncm = (values.get("gtin") or "").strip(), (values.get("ncm") or "").strip()
from .enrichment import history
current = history.official_sets(session)
try:
result = kb_mod.attest(session, gtin, ncm, admin.display_name or admin.username, lambda v: v in current, note=(values.get("note") or "")[:200])
except ValueError as error:
return HTMLResponse(f"Decisão recusada: {error}", status_code=422)
record_audit(session, admin_user_id=admin.id, tenant_id=None, action="KB_NCM_ATTESTED", target_type="kb_product", target_id=gtin,
context={"ncm": ncm, "status": result["status"], "families": result["families"], "operator": admin.display_name or admin.username, "role": admin.external_role or admin.auth_source}, severity="WARNING")
for inst in session.scalars(select(Installation).where(Installation.enrichment_mode != "DISABLED", Installation.features_configured.is_(True))):
enrichment_status.mark_for_reevaluation(session, inst.id, gtin=gtin)
enrichment_status.mark_for_reevaluation(session, inst.id) # products identified by name are found through the KB aliases
session.commit()
return RedirectResponse("/#kb-resumo", status_code=303)
finally: session.close()
def _ops_view(session, installation, agent):
"""Processing status shown on the Agent detail (no clutter: 5 facts)."""
last_job = session.scalar(select(Job).where(Job.agent_id == agent.id).order_by(Job.id.desc()).limit(1))
last_sync = _aware(installation.last_sync_at)
next_sync = None
if installation.processing_enabled and last_sync is not None:
next_sync = last_sync + timedelta(seconds=installation.sync_interval_seconds)
last_summary = session.scalar(select(Job.result_summary).where(Job.agent_id == agent.id, Job.status == "COMPLETED", Job.kind.in_(("INITIAL_SYNC", "SYNC_INCREMENTAL", "REPROCESS_ALL"))).order_by(Job.id.desc()).limit(1))
return {"rtc": rtc_status_view(last_summary, installation.rtc_write_mode), "rtc_mode": installation.rtc_write_mode, "enabled": bool(installation.processing_enabled), "interval_minutes": installation.sync_interval_seconds // 60,
"last_sync": format_ui_datetime(last_sync) if last_sync else None, "next_sync": format_ui_datetime(next_sync) if next_sync else None,
"last_changed": installation.last_sync_changed, "last_job": {"kind": last_job.kind, "status": last_job.status, "id": last_job.id, "kind_label": job_kind_label(last_job.kind), "status_label": job_status_label(last_job.status)} if last_job else None}
async def _installation_action(request, agent_id, action):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
values = await form_values(request); web_session, admin = auth_data
require_csrf(web_session, values.get("csrf_token"))
agent = session.get(Agent, agent_id)
installation = session.scalar(select(Installation).where(Installation.agent_id == agent_id)) if agent else None
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
if action == "pause": installation.processing_enabled = False
elif action == "resume": installation.processing_enabled = True
elif action == "sync-now":
job = create_job(session, agent, "SYNC_INCREMENTAL", {"reason": "MANUAL_SYNC_NOW", "requested_by": admin.id})
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="SYNC_NOW", target_type="installation", target_id=installation.installation_id, context={"job_id": job.id, "kind": job.kind})
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action=f"PROCESSING_{action.upper().replace('-', '_')}", target_type="installation", target_id=installation.installation_id, context={"processing_enabled": installation.processing_enabled}) if action in ("pause", "resume") else None
session.commit()
return RedirectResponse(f"/agents/{agent_id}?tab=jobs", status_code=303)
finally: session.close()
def _is_privileged(admin) -> bool:
"""Who may change what the Agent is allowed to WRITE. The identity comes from the Gerenciador SSO (`perfil`): only ADMIN. LOCAL break-glass accounts are administrators by construction.
A normal (PADRAO) collaborator can look at the control page and use DISABLED/DRY_RUN (read-only) but cannot approve, enable, apply or roll back."""
return admin.auth_source == "LOCAL" or (admin.external_role or "").upper() == "ADMIN"
def _deny(session, admin, tenant_id, action, target):
record_audit(session, admin_user_id=admin.id, tenant_id=tenant_id, action="RTC_DENIED", target_type="installation", target_id=target, context={"attempted": action, "role": admin.external_role or admin.auth_source}, severity="WARNING")
session.commit()
return HTMLResponse("Ação restrita a administradores do GTI Gerenciador (perfil ADMIN).", status_code=403)
def _rtc_target(session, agent_id):
agent = session.get(Agent, agent_id)
installation = session.scalar(select(Installation).where(Installation.agent_id == agent_id)) if agent else None
return agent, installation
@routes.get("/agents/{agent_id}/rtc")
async def rtc_control_page(request: Request, agent_id: int):
"""Central cClassTrib control: write mode, candidate preview (server-side pagination), approvals and audit/rollback."""
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
agent, installation = _rtc_target(session, agent_id)
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
q = request.query_params
status = q.get("status") or "PENDING"
page = max(1, int(q.get("page", "1") or 1)) if str(q.get("page", "1")).isdigit() else 1
size = 50
base = select(RtcCandidate).where(RtcCandidate.installation_id == installation.id, RtcCandidate.rule_id == "RTC-R3")
counts = dict(session.execute(select(RtcCandidate.status, func.count()).where(RtcCandidate.installation_id == installation.id, RtcCandidate.rule_id == "RTC-R3").group_by(RtcCandidate.status)).all())
shown = base.where(RtcCandidate.status == status) if status in ("PENDING", "APPROVED", "REJECTED", "APPLIED", "STALE") else base
total = int(session.scalar(select(func.count()).select_from(shown.subquery())) or 0)
rows = list(session.scalars(shown.order_by(RtcCandidate.product_code).offset((page - 1) * size).limit(size)))
audits = list(session.scalars(select(RtcWriteAudit).where(RtcWriteAudit.installation_id == installation.id, RtcWriteAudit.outcome.in_(("APPLIED", "ROLLED_BACK"))).order_by(RtcWriteAudit.id.desc()).limit(25)))
ops = _ops_view(session, installation, agent)
rtc_blockers = rtc_blocker_summary(session, agent_id)
rtc_products = ((ops.get("rtc") or {}).get("products") or {})
r3_candidates = int(rtc_products.get("ready", counts.get("PENDING", 0) + counts.get("APPROVED", 0)))
blocked_products = max(int(rtc_products.get("blocked", 0)), rtc_blockers["products_affected"])
reasons = rtc_blockers["reason_counts"]
simple_reason_count = sum(count for reason, count in reasons.items() if reason.endswith("SIMPLES_ORIENTATION_PENDING_OFFICIAL_NT"))
simple_reason_count = max(simple_reason_count, int(rtc_products.get("simple_orientation_count", 0)))
other_reason_max = max((count for reason, count in reasons.items() if not reason.endswith("SIMPLES_ORIENTATION_PENDING_OFFICIAL_NT")), default=0)
simple_orientation_dominant = bool(rtc_products.get("simple_orientation_dominant")) or simple_reason_count > other_reason_max
rtc_empty_state = {"show": r3_candidates == 0 and blocked_products > 0, "r3_candidates": r3_candidates,
"blocked_products": blocked_products, "simple_orientation_dominant": simple_orientation_dominant}
return render(request, "rtc_control.html", {"agent_id": agent_id, "installation": installation, "mode": installation.rtc_write_mode, "modes": RTC_MODE_LABELS, "explanation": RTC_MODE_EXPLANATION, "ops": ops,
"privileged": _is_privileged(auth_data[1]), "rows": rows, "counts": counts, "status": status, "page": page, "pages": max(1, -(-total // size)), "total": total, "audits": audits, "rtc_empty_state": rtc_empty_state,
"L": {"status": candidate_status_label, "outcome": audit_outcome_label, "reason": abort_reason_label, "rule": rtc_rule_label, "catalog": catalog_state_label}})
finally: session.close()
@routes.post("/agents/{agent_id}/rtc-mode")
async def rtc_write_mode(request: Request, agent_id: int):
"""Central authorization of the cClassTrib fill, per installation. Default DISABLED; ENABLED needs a typed confirmation; every change is audited."""
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
values = await form_values(request); web_session, admin = auth_data
require_csrf(web_session, values.get("csrf_token"))
agent, installation = _rtc_target(session, agent_id)
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
mode = (values.get("mode") or "").strip().upper()
if mode not in RTC_MODE_LABELS:
return HTMLResponse("modo inválido", status_code=422)
if mode not in ("DISABLED", "DRY_RUN") and not _is_privileged(admin):
return _deny(session, admin, installation.tenant_id, f"rtc-mode:{mode}", installation.installation_id)
if installation.rtc_write_mode not in ("DISABLED", "DRY_RUN") and not _is_privileged(admin):
return _deny(session, admin, installation.tenant_id, f"rtc-mode:leave:{installation.rtc_write_mode}", installation.installation_id)
if mode == "ENABLED" and (values.get("confirm") or "").strip().upper() != "PREENCHER":
return HTMLResponse("Digite PREENCHER para confirmar o modo automático", status_code=422)
previous = installation.rtc_write_mode
installation.rtc_write_mode = mode
installation.rtc_auto_apply_authorized = mode == "ENABLED"
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="RTC_WRITE_MODE", target_type="installation", target_id=installation.installation_id,
context={"from": previous, "to": mode, "operator": admin.display_name or admin.username, "role": admin.external_role or admin.auth_source}, severity="WARNING")
session.commit()
return RedirectResponse(f"/agents/{agent_id}/rtc", status_code=303)
finally: session.close()
@routes.post("/agents/{agent_id}/rtc-candidates")
async def rtc_candidate_action(request: Request, agent_id: int):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
values = await form_values(request); web_session, admin = auth_data
require_csrf(web_session, values.get("csrf_token"))
agent, installation = _rtc_target(session, agent_id)
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
action = values.get("action")
if not _is_privileged(admin):
return _deny(session, admin, installation.tenant_id, f"rtc-candidates:{action}", installation.installation_id)
scope = RtcCandidate.installation_id == installation.id
ids = [int(x) for x in parse_qs((await request.body()).decode()).get("ids", []) if x.isdigit()]
select_all = values.get("all") == "1" or request.query_params.get("all") == "1"
changed = 0
if action in ("approve", "reject"):
new, old = ("APPROVED", ("PENDING",)) if action == "approve" else ("REJECTED", ("PENDING", "APPROVED"))
cond = scope & RtcCandidate.status.in_(old) & (RtcCandidate.id.in_(ids) if not select_all else RtcCandidate.status.in_(old))
changed = session.execute(update(RtcCandidate).where(cond).values(status=new)).rowcount
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action=f"RTC_CANDIDATES_{action.upper()}", target_type="installation", target_id=installation.installation_id, context={"changed": changed, "all": select_all, "operator": admin.display_name or admin.username}, severity="WARNING")
elif action == "apply_approved":
if installation.rtc_write_mode != "APPROVAL_REQUIRED":
return HTMLResponse("Disponível apenas no modo Aprovação manual", status_code=409)
codes = list(session.scalars(select(RtcCandidate.product_code).where(scope, RtcCandidate.status == "APPROVED").order_by(RtcCandidate.product_code).limit(500)))
if codes:
job = create_job(session, agent, "REPROCESS_PRODUCTS", {"codigos": codes, "reason": "RTC_APPLY_APPROVED", "requested_by": admin.id})
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="RTC_APPLY_APPROVED", target_type="installation", target_id=installation.installation_id, context={"job_id": job.id, "products": len(codes), "operator": admin.display_name or admin.username}, severity="WARNING")
else:
return HTMLResponse("ação inválida", status_code=422)
session.commit()
return RedirectResponse(f"/agents/{agent_id}/rtc", status_code=303)
finally: session.close()
@routes.post("/agents/{agent_id}/rtc-rollback")
async def rtc_rollback_request(request: Request, agent_id: int):
"""Requests the rollback of ONE audit record. The Agent executes it only if the value in the ERP still equals what was written; no free SQL exists."""
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
values = await form_values(request); web_session, admin = auth_data
require_csrf(web_session, values.get("csrf_token"))
agent, installation = _rtc_target(session, agent_id)
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
if not _is_privileged(admin):
return _deny(session, admin, installation.tenant_id, "rtc-rollback", installation.installation_id)
if (values.get("confirm") or "").strip().upper() != "REVERTER":
return HTMLResponse("Digite REVERTER para confirmar", status_code=422)
audit = session.get(RtcWriteAudit, int(values.get("audit_id") or 0))
if audit is None or audit.installation_id != installation.id or audit.action != "APPLY" or audit.outcome != "APPLIED" or audit.rollback_state in ("REQUESTED", "DONE"):
return HTMLResponse("registro não reversível", status_code=409)
audit.rollback_state = "REQUESTED"
job = create_job(session, agent, "REPROCESS_PRODUCTS", {"codigos": [audit.product_code], "reason": "RTC_ROLLBACK", "requested_by": admin.id})
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="RTC_ROLLBACK_REQUESTED", target_type="installation", target_id=installation.installation_id,
context={"audit_id": audit.id, "product_code": audit.product_code, "job_id": job.id, "operator": admin.display_name or admin.username}, severity="WARNING")
session.commit()
return RedirectResponse(f"/agents/{agent_id}/rtc", status_code=303)
finally: session.close()
@routes.post("/agents/{agent_id}/sync-now")
async def sync_now(request: Request, agent_id: int):
return await _installation_action(request, agent_id, "sync-now")
@routes.post("/agents/{agent_id}/pause")
async def pause_processing(request: Request, agent_id: int):
return await _installation_action(request, agent_id, "pause")
@routes.post("/agents/{agent_id}/resume")
async def resume_processing(request: Request, agent_id: int):
return await _installation_action(request, agent_id, "resume")
@routes.get("/agents/{agent_id}/reprocess-confirmation", response_class=HTMLResponse)
def reprocess_confirmation(request: Request, agent_id: int):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
if not session.scalar(select(Agent.id).where(Agent.id == agent_id)):
return HTMLResponse("Agent not found", status_code=404)
return render(request, "reprocess_confirmation.html", {"agent_id": agent_id, "auth": auth_data})
finally: session.close()
@routes.post("/agents/{agent_id}/reprocess")
async def reprocess_agent(request: Request, agent_id: int):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
values = await form_values(request); web_session, admin = auth_data
require_csrf(web_session, values.get("csrf_token"))
if values.get("confirm") != "yes": return render(request, "reprocess_confirmation.html", {"agent_id": agent_id, "auth": auth_data, "error": "Confirme explicitamente o reprocessamento."}, 400)
agent = session.get(Agent, agent_id)
installation = session.scalar(select(Installation).where(Installation.agent_id == agent_id))
if not agent or not installation: return HTMLResponse("Agent not found", status_code=404)
existing = next((candidate for candidate in session.scalars(select(Job).where(Job.agent_id == agent.id, Job.kind == "REPROCESS_ALL", Job.status == "PENDING")).all() if candidate.payload == {}), None)
job = existing or create_job(session, agent, "REPROCESS_ALL", {})
record_audit(session, admin_user_id=admin.id, tenant_id=installation.tenant_id, action="REPROCESS_ALL", target_type="installation", target_id=installation.installation_id, context={"job_id": job.id})
session.commit()
return RedirectResponse(f"/agents/{agent_id}?tab=jobs", status_code=303)
finally: session.close()
def _pending_or_404(session, enrollment_uuid):
return session.scalar(select(AgentEnrollment).where(AgentEnrollment.enrollment_uuid == enrollment_uuid))
@routes.get("/agents/pending/{enrollment_uuid}", response_class=HTMLResponse)
def pending_installation_detail(request: Request, enrollment_uuid: str, link: str | None = None, code: str | None = None):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
enr = _pending_or_404(session, enrollment_uuid)
if not enr: return HTMLResponse("Installation not found", status_code=404)
release = session.get(AgentRelease, enr.release_id)
tenant = session.get(Tenant, enr.tenant_id)
installed_agent = session.get(Agent, enr.agent_id) if enr.agent_id else None
active_code = bool(enr.install_code_hash and enr.install_code_expires_at and _aware(enr.install_code_expires_at) > datetime.now(timezone.utc))
code_expires_at = enr.install_code_expires_at if active_code else None
return render(request, "agent_pending_detail.html", {"enr": enr, "release": release, "tenant": tenant, "installed_agent": installed_agent,
"download_link": link, "install_code": code, "code_active": active_code,
"code_expires_at": code_expires_at, "auth": auth_data})
finally: session.close()
@routes.post("/agents/pending/{enrollment_uuid}/link")
async def create_download_link(request: Request, enrollment_uuid: str):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
values = await form_values(request); web_session, admin = auth_data
require_csrf(web_session, values.get("csrf_token"))
enr = _pending_or_404(session, enrollment_uuid)
if not enr: return HTMLResponse("Installation not found", status_code=404)
if enr.status in ("ENROLLED", "REVOKED"): return RedirectResponse(f"/agents/pending/{enrollment_uuid}", status_code=303)
token = enrollment.new_token()
enr.download_token_hash = enrollment.token_digest(token)
enr.download_token_expires_at = datetime.now(timezone.utc) + timedelta(seconds=config.settings.agent_download_link_ttl_seconds)
enr.download_max_uses = config.settings.agent_download_max_uses_default
enr.download_uses = 0
enr.status = "LINK_CREATED"
record_audit(session, admin_user_id=admin.id, tenant_id=enr.tenant_id, action="DOWNLOAD_LINK_CREATED", target_type="agent_enrollment", target_id=enrollment_uuid, context={"ttl_seconds": config.settings.agent_download_link_ttl_seconds, "ip": client_ip(request)})
session.commit()
url = f"{config.settings.base_url}/dl/agent/{token}"
return RedirectResponse(f"/agents/pending/{enrollment_uuid}?link={url}", status_code=303)
finally: session.close()
@routes.get("/agents/pending/{enrollment_uuid}/download")
def direct_download(request: Request, enrollment_uuid: str):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
web_session, admin = auth_data
enr = _pending_or_404(session, enrollment_uuid)
if not enr: return HTMLResponse("Installation not found", status_code=404)
if enr.status == "REVOKED": return HTMLResponse("Installation revoked", status_code=409)
try:
return agent_distribution.build_and_serve(session, enr, request, actor_admin_id=admin.id)
except enrollment.EnrollmentRejected as rejected:
return HTMLResponse(f"Download indisponível: {rejected.reason}", status_code=409)
finally: session.close()
@routes.post("/agents/pending/{enrollment_uuid}/revoke")
async def revoke_pending_installation(request: Request, enrollment_uuid: str):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
values = await form_values(request); web_session, admin = auth_data
require_csrf(web_session, values.get("csrf_token"))
enr = _pending_or_404(session, enrollment_uuid)
if not enr: return HTMLResponse("Installation not found", status_code=404)
was_code_active = bool(enr.install_code_hash)
enr.status = "REVOKED"
enr.download_token_hash = None
enr.enrollment_token_hash = None
enr.install_code_hash = None
enr.install_code_expires_at = None
if enr.agent_id:
agent = session.get(Agent, enr.agent_id)
if agent: agent.enabled = False
record_audit(session, admin_user_id=admin.id, tenant_id=enr.tenant_id, action="INSTALLATION_REVOKED", target_type="agent_enrollment", target_id=enrollment_uuid,
context={"had_agent": bool(enr.agent_id), "had_install_code": was_code_active})
if was_code_active:
record_audit(session, admin_user_id=admin.id, tenant_id=enr.tenant_id, action="INSTALLATION_CODE_REVOKED", target_type="agent_enrollment", target_id=enrollment_uuid, context={})
session.commit()
return RedirectResponse(f"/agents/pending/{enrollment_uuid}", status_code=303)
finally: session.close()
@routes.post("/agents/pending/{enrollment_uuid}/code")
async def generate_install_code(request: Request, enrollment_uuid: str):
"""Generate (or rotate) the installation code shown to the operator and
typed into the universal EXE installer. Only the SHA-256 of the
normalized code is ever stored; the plaintext is returned in the HTML
once, to the SSO-authenticated operator who created the installation."""
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
values = await form_values(request); web_session, admin = auth_data
require_csrf(web_session, values.get("csrf_token"))
enr = _pending_or_404(session, enrollment_uuid)
if not enr: return HTMLResponse("Installation not found", status_code=404)
if enr.status in ("ENROLLED", "REVOKED"):
return RedirectResponse(f"/agents/pending/{enrollment_uuid}", status_code=303)
code = enrollment.generate_install_code()
now = datetime.now(timezone.utc)
enr.install_code_hash = enrollment.install_code_digest(code)
enr.install_code_expires_at = now + timedelta(seconds=config.settings.agent_enrollment_ttl_seconds)
enr.install_code_attempts = 0
enr.install_code_created_at = now
record_audit(session, admin_user_id=admin.id, tenant_id=enr.tenant_id, action="INSTALLATION_CODE_CREATED", target_type="agent_enrollment",
target_id=enrollment_uuid, context={"ttl_seconds": config.settings.agent_enrollment_ttl_seconds, "ip": client_ip(request)})
session.commit()
return RedirectResponse(f"/agents/pending/{enrollment_uuid}?code={code}", status_code=303)
finally: session.close()
@routes.get("/blockers", response_class=HTMLResponse)
def blockers(request: Request, page: int = 1, client: str | None = None, installation: str | None = None, field: str | None = None, blocker_type: str | None = None, reason_code: str | None = None, status: str = "OPEN", age: str | None = None, search: str | None = None, agent_id: int | None = None, product_code: str | None = None, scope: str = "active"):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
scope = scope if scope in ("active", "all") else "active"
status = status if status in ("OPEN", "RESOLVED", "ALL") else "OPEN"
filters = {"agent_id": agent_id, "product_code": product_code, "client": client, "installation": installation, "field": field, "blocker_type": blocker_type, "reason_code": reason_code, "status": status, "age": age, "search": search, "scope": scope}
result = list_blockers(session, page=page, page_size=50, **filters)
summary = blocker_filter_summary(session, **filters)
return render(request, "blockers.html", {"result": result, "summary": summary, "auth": auth_data, "csrf_token": request.cookies.get("gti_csrf", ""), "client_filter": client or "", "installation_filter": installation or "", "field_filter": field or "", "blocker_type_filter": blocker_type or "", "reason_code_filter": reason_code or "", "status_filter": status, "scope_filter": scope, "age_filter": age or "", "search": search or "", "query_base": query_base(**filters), "field_query_base": query_base(**{key: value for key, value in filters.items() if key != "field"})})
finally: session.close()
@routes.get("/blockers/{blocker_id}", response_class=HTMLResponse)
def blocker_page(request: Request, blocker_id: int):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
detail = blocker_detail(session, blocker_id)
if detail is None: return HTMLResponse("Blocker not found", status_code=404)
blocker = session.get(BlockedProduct, blocker_id)
installation = session.get(Installation, blocker.installation_id) if blocker.installation_id else None
tenant = session.get(Tenant, blocker.tenant_id)
company_key = _company_key(blocker, installation, tenant)
current_context = session.scalars(select(FiscalContextEntry).where(FiscalContextEntry.tenant_id == blocker.tenant_id, FiscalContextEntry.company_key == company_key)).all() if installation else []
value_choices = sorted({value for values in ENUMS.values() for value in values})
return render(request, "blocker_detail.html", {"detail": detail, "auth": auth_data, "context_entries": current_context, "context_keys": sorted(ALLOWED_KEYS), "value_choices": value_choices, "error": None, "resolution_available": installation is not None, "resolution_values": {}})
finally: session.close()
@routes.post("/blockers/{blocker_id}/resolve-context")
async def resolve_blocker_with_context_route(request: Request, blocker_id: int):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
values = await form_values(request); web_session, admin = auth_data
require_csrf(web_session, values.get("csrf_token"))
payload = {key: values.get(key) for key in ("scope", "scope_key", "key", "value", "reason")}
try:
resolve_blocker_with_context(session, blocker_id, admin, payload, request_ip=request.client.host if request.client else None)
except BlockerResolutionUnavailable as error:
return HTMLResponse(str(error), status_code=409)
except LookupError:
return HTMLResponse("Blocker not found", status_code=404)
except ValueError as error:
detail = blocker_detail(session, blocker_id)
blocker = session.get(BlockedProduct, blocker_id)
installation = session.get(Installation, blocker.installation_id) if blocker and blocker.installation_id else None
tenant = session.get(Tenant, blocker.tenant_id) if blocker else None
company_key = _company_key(blocker, installation, tenant) if blocker else ""
current_context = session.scalars(select(FiscalContextEntry).where(FiscalContextEntry.tenant_id == blocker.tenant_id, FiscalContextEntry.company_key == company_key)).all() if blocker and installation else []
return render(request, "blocker_detail.html", {"detail": detail, "auth": auth_data, "context_entries": current_context, "context_keys": sorted(ALLOWED_KEYS), "value_choices": sorted({value for values in ENUMS.values() for value in values}), "error": str(error), "resolution_available": installation is not None, "resolution_values": payload}, 422)
return RedirectResponse(f"/blockers/{blocker_id}", status_code=303)
finally:
session.close()
@routes.get("/jobs", response_class=HTMLResponse)
def jobs(request: Request, page: int = 1, status: str | None = None, search: str | None = None, kind: str | None = None, job_id: int | None = None, scope: str = "active"):
session, auth_data, redirect = page_auth(request)
if redirect: return redirect
try:
scope = scope if scope in ("active", "all") else "active"
status = status if status in ("PENDING", "RUNNING", "COMPLETED", "FAILED") else None
result = list_jobs(session, page=page, page_size=50, status=status, search=search, kind=kind, job_id=job_id, scope=scope)
return render(request, "jobs.html", {"result": result, "status_counts": jobs_status_summary(session, scope=scope), "auth": auth_data, "status_filter": status or "", "scope_filter": scope, "search": search or "", "kind_filter": kind or "", "job_id_filter": job_id, "query_base": query_base(status=status, search=search, kind=kind, job_id=job_id, scope=scope)})
finally: session.close()
return routes