feat: add autonomous fiscal completion v1

2026-09-22 13:08:38 -03:00
parent eada0df37e
commit e3ac68c70e
28 changed files with 44630 additions and 0 deletions
@@ -0,0 +1,59 @@
+"""Persistent-boundary-compatible budget guard; provider invocation happens after this check."""
+
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from threading import Lock
+
+
+@dataclass(frozen=True)
+class BudgetLimits:
+    max_run_cost_usd: float = 0.54
+    max_daily_cost_usd: float = 1.00
+    max_monthly_cost_usd: float = 10.00
+    max_requests_per_day: int = 100
+    max_new_products_per_run: int = 750
+
+
+@dataclass(frozen=True)
+class BudgetDecision:
+    allowed: bool
+    reason: str
+    reserved_cost_usd: float
+
+
+@dataclass(frozen=True)
+class UsageEntry:
+    run_id: str
+    tenant: str
+    cost_usd: float
+    requests: int
+    new_products: int
+    timestamp: str
+
+
+class BudgetLedger:
+    """Atomic in-process ledger with an explicit persistence boundary for Hub storage."""
+
+    def __init__(self, limits: BudgetLimits):
+        self.limits = limits
+        self.entries: list[UsageEntry] = []
+        self._lock = Lock()
+
+    def check_and_reserve(self, run_id: str, tenant: str, cost_usd: float, requests: int, new_products: int) -> BudgetDecision:
+        with self._lock:
+            run_cost = sum(e.cost_usd for e in self.entries if e.run_id == run_id)
+            day = datetime.now(timezone.utc).date()
+            day_entries = [e for e in self.entries if e.timestamp[:10] == day.isoformat()]
+            month_entries = [e for e in self.entries if e.timestamp[:7] == day.strftime("%Y-%m")]
+            if run_cost + cost_usd > self.limits.max_run_cost_usd:
+                return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
+            if sum(e.cost_usd for e in day_entries) + cost_usd > self.limits.max_daily_cost_usd:
+                return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
+            if sum(e.cost_usd for e in month_entries) + cost_usd > self.limits.max_monthly_cost_usd:
+                return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
+            if sum(e.requests for e in day_entries) + requests > self.limits.max_requests_per_day:
+                return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
+            if sum(e.new_products for e in self.entries if e.run_id == run_id) + new_products > self.limits.max_new_products_per_run:
+                return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
+            self.entries.append(UsageEntry(run_id, tenant, cost_usd, requests, new_products, datetime.now(timezone.utc).isoformat()))
+            return BudgetDecision(True, "RESERVED", cost_usd)
@@ -0,0 +1,36 @@
+"""Conservative deterministic role, ST, and inbound scope resolution."""
+
+
+def resolve_product_role(signals: dict) -> dict:
+    if signals.get("recipe") and signals.get("production"):
+        return {"value": "OWN_PRODUCTION", "status": "CONFIRMED", "evidence": ["recipe", "production"]}
+    if signals.get("purchase") and signals.get("sale") and not signals.get("recipe"):
+        return {"value": "RESALE", "status": "CONFIRMED", "evidence": ["purchase", "sale"]}
+    if signals.get("input_consumed") and signals.get("recipe"):
+        return {"value": "INPUT", "status": "CONFIRMED", "evidence": ["input_consumed", "recipe"]}
+    if signals.get("service_flag") and signals.get("service_transactions"):
+        return {"value": "SERVICE", "status": "CONFIRMED", "evidence": ["service_flag", "service_transactions"]}
+    return {"value": None, "status": "PENDING_PRODUCT_ROLE", "evidence": []}
+
+
+def resolve_st_status(signals: dict) -> dict:
+    if not signals.get("official_cest"):
+        return {"value": "NOT_SUBJECT_TO_ST", "status": "CONFIRMED", "evidence": []}
+    if signals.get("purchase_retained") is True:
+        return {"value": "ST_ALREADY_WITHHELD_IN_PURCHASE", "status": "CONFIRMED", "evidence": ["official_cest", "purchase_retained"]}
+    if signals.get("purchase_retained") is False:
+        return {"value": "PRODUCT_SUBJECT_TO_ST", "status": "CONFIRMED", "evidence": ["official_cest", "purchase_not_retained"]}
+    return {"value": None, "status": "PENDING_ST_WITHHELD", "evidence": ["official_cest"]}
+
+
+def resolve_entry_scope(supplier_ufs: list[str], company_uf: str) -> dict:
+    observed = {str(uf).strip().upper() for uf in supplier_ufs if uf}
+    if not observed:
+        return {"value": None, "status": "PENDING_ENTRY_SCOPE", "evidence": []}
+    if len(observed - {company_uf.upper()}) and company_uf.upper() in observed:
+        value = "MIXED_OR_DYNAMIC"
+    elif observed - {company_uf.upper()}:
+        value = "INTER"
+    else:
+        value = "INTRA"
+    return {"value": value, "status": "CONFIRMED", "evidence": sorted(observed)}
@@ -0,0 +1,47 @@
+"""Conservative aggregation of supplier/document observations."""
+
+from dataclasses import dataclass
+from collections import defaultdict
+from typing import Any, Iterable
+
+
+@dataclass(frozen=True)
+class Observation:
+    product_id: str
+    attribute: str
+    value: str
+    supplier_id: str
+    document_id: str
+    observed_at: str
+    provenance: str = "SUPPLIER_OBSERVATION"
+
+
+def aggregate_observations(observations: Iterable[Observation]) -> dict[tuple[str, str], dict[str, Any]]:
+    grouped: dict[tuple[str, str], list[Observation]] = defaultdict(list)
+    seen: set[tuple[str, str, str, str]] = set()
+    for observation in observations:
+        key = (observation.product_id, observation.attribute)
+        dedupe = (observation.product_id, observation.attribute, observation.supplier_id, observation.document_id)
+        if dedupe not in seen and observation.value not in (None, ""):
+            grouped[key].append(observation)
+            seen.add(dedupe)
+
+    output = {}
+    for key, rows in grouped.items():
+        values = {row.value for row in rows}
+        counts = {value: sum(row.value == value for row in rows) for value in values}
+        supplier_count = len({row.supplier_id for row in rows})
+        agreement_rate = max(counts.values()) / len(rows) if rows else 0.0
+        conflict_count = max(0, len(values) - 1)
+        strong = len(rows) >= 2 and supplier_count >= 2 and len(values) == 1
+        output[key] = {
+            "value": next(iter(values)) if strong else None,
+            "observation_count": len(rows),
+            "supplier_count": supplier_count,
+            "agreement_rate": agreement_rate,
+            "latest_observation": max(row.observed_at for row in rows),
+            "conflict_count": conflict_count,
+            "strong_evidence": strong,
+            "provenance": sorted({row.provenance for row in rows}),
+        }
+    return output
@@ -0,0 +1,83 @@
+"""Pure requiredness and fiscal completeness evaluation."""
+
+import json
+from pathlib import Path
+from typing import Any
+
+
+VALID_CLASSIFICATIONS = {
+    "REQUIRED", "CONDITIONAL", "OPTIONAL_REFERENCE", "NOT_APPLICABLE", "ERP_NOT_USED"
+}
+
+
+def load_requiredness_matrix(path: str | Path) -> dict[str, Any]:
+    matrix = json.loads(Path(path).read_text(encoding="utf-8"))
+    fields = matrix.get("fields", {})
+    if set(matrix.get("classifications", [])) != VALID_CLASSIFICATIONS:
+        raise ValueError("requiredness matrix classifications are incomplete")
+    if len(fields) != 23 or any(rule.get("classification") not in VALID_CLASSIFICATIONS for rule in fields.values()):
+        raise ValueError("requiredness matrix must define 23 valid field rules")
+    return matrix
+
+
+def _has_value(value: Any) -> bool:
+    return value is not None and (not isinstance(value, str) or bool(value.strip()))
+
+
+def _conditional_state(field: str, context: dict[str, Any]) -> str:
+    if field == "IPI":
+        return "REQUIRED" if context.get("contribuinte_ipi") == "S" and context.get("has_numeric_tipi") else "NOT_APPLICABLE"
+    if field == "EX_TIPI":
+        return "REQUIRED" if context.get("has_supported_ex") and context.get("product_matches_ex") else "NOT_APPLICABLE"
+    if field == "CEST":
+        return "REQUIRED" if context.get("has_official_cest") or context.get("st_subject") else "NOT_APPLICABLE"
+    if field in {"CFOP", "SITUACAO_TRIBUTARIA"}:
+        return "REQUIRED" if context.get("operation", "SALE") == "SALE" else "NOT_APPLICABLE"
+    if field in {"CFOP_ENTRADA", "CST_ENTRADA", "COD_ENTRADA_PIS", "COD_ENTRADA_COFINS"}:
+        return "REQUIRED" if context.get("operation") == "PURCHASE" and context.get("erp_consumes_entry_fields") else "NOT_APPLICABLE"
+    if field == "CST":
+        return "REQUIRED" if context.get("regime") == "NORMAL" and context.get("operation", "SALE") == "SALE" else "NOT_APPLICABLE"
+    if field == "CSOSN":
+        return "REQUIRED" if context.get("regime") == "SIMPLES" and context.get("operation", "SALE") == "SALE" else "NOT_APPLICABLE"
+    if field == "ALIQUOTA":
+        return "REQUIRED" if context.get("icms_taxed") else "NOT_APPLICABLE"
+    if field == "COD_ORIGEM_MERCADORIA":
+        return "REQUIRED" if context.get("origin_proven") else "NOT_APPLICABLE"
+    if field == "CST_IPI":
+        return "REQUIRED" if context.get("contribuinte_ipi") == "S" and context.get("erp_consumes_ipi_cst") else "NOT_APPLICABLE"
+    if field in {"COD_SAIDA_PIS", "COD_SAIDA_COFINS"}:
+        return "REQUIRED" if context.get("operation", "SALE") == "SALE" and context.get("piscofins_profile_proven") else "NOT_APPLICABLE"
+    if field in {"ALIQ_PIS", "ALIQ_COFINS"}:
+        return "REQUIRED" if context.get("piscofins_rate_applies") else "NOT_APPLICABLE"
+    if field in {"INCIDENCIA_PISCOFINS", "PISCOFINS"}:
+        return "REQUIRED" if context.get("piscofins_profile_proven") else "NOT_APPLICABLE"
+    return "NOT_APPLICABLE"
+
+
+def evaluate_field(field: str, value: Any, context: dict[str, Any], matrix: dict[str, Any]) -> dict[str, Any]:
+    rule = matrix["fields"][field]
+    classification = rule["classification"]
+    if classification == "ERP_NOT_USED":
+        status = "ERP_NOT_SUPPORTED"
+        valid = True
+    elif classification == "NOT_APPLICABLE":
+        status = "NOT_APPLICABLE"
+        valid = True
+    elif classification == "OPTIONAL_REFERENCE":
+        status = "OPTIONAL_VALID" if _has_value(value) else "OPTIONAL_EMPTY"
+        valid = True
+    elif classification == "REQUIRED":
+        status = "REQUIRED_AND_VALID" if _has_value(value) else "REQUIRED_MISSING"
+        valid = _has_value(value)
+    else:
+        conditional = _conditional_state(field, context)
+        if conditional == "NOT_APPLICABLE":
+            status, valid = "CONDITIONAL_NOT_TRIGGERED", True
+        else:
+            status, valid = ("CONDITIONAL_AND_VALID", True) if _has_value(value) else ("REQUIRED_MISSING", False)
+    return {"field": field, "classification": classification, "status": status, "valid": valid, "value_present": _has_value(value), "source": rule["source"]}
+
+
+def evaluate_product(product: dict[str, Any], context: dict[str, Any], matrix: dict[str, Any]) -> dict[str, Any]:
+    fields = {field: evaluate_field(field, product.get(field), context, matrix) for field in matrix["fields"]}
+    return {"FISCALLY_COMPLETE": all(result["valid"] for result in fields.values()), "fields": fields}
@@ -464,3 +464,33 @@ class AiEvalCache(Base):
    answer: Mapped[dict] = mapped_column(JSON)
    gold_ncm: Mapped[str | None] = mapped_column(String(8), nullable=True)
    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+
+
+class AiUsageLedger(Base):
+    """Central persistent usage ledger reserved before any future provider call."""
+    __tablename__ = "ai_usage_ledger"
+    id: Mapped[int] = mapped_column(primary_key=True)
+    provider: Mapped[str] = mapped_column(String(64))
+    model: Mapped[str] = mapped_column(String(64))
+    run_id: Mapped[str] = mapped_column(String(128), index=True)
+    tenant: Mapped[str] = mapped_column(String(128), index=True)
+    items: Mapped[int] = mapped_column(Integer, default=0)
+    input_tokens: Mapped[int] = mapped_column(Integer, default=0)
+    output_tokens: Mapped[int] = mapped_column(Integer, default=0)
+    cost: Mapped[float] = mapped_column(default=0.0)
+    timestamp: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), index=True)
+
+
+class ExternalProductCache(Base):
+    """Versioned GTIN cache; legal/provider flags remain explicit metadata."""
+    __tablename__ = "external_product_cache"
+    __table_args__ = (UniqueConstraint("gtin", "provider", "provider_record_version", name="uq_external_product_cache_record"),)
+    id: Mapped[int] = mapped_column(primary_key=True)
+    gtin: Mapped[str] = mapped_column(String(16), index=True)
+    provider: Mapped[str] = mapped_column(String(64))
+    provider_record_version: Mapped[str] = mapped_column(String(64))
+    retrieved_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    raw_hash: Mapped[str] = mapped_column(String(64))
+    normalized_data: Mapped[dict] = mapped_column(JSON, default=dict)
+    cache_allowed: Mapped[bool] = mapped_column(Boolean, default=False)
+    redistribution_allowed: Mapped[bool] = mapped_column(Boolean, default=False)
@@ -0,0 +1,17 @@
+"""Evidence-gated origin resolution; CST digits are never an origin source."""
+
+
+def resolve_origin(product_id: str, aggregate: dict, kb_facts: dict) -> dict:
+    if aggregate.get("conflict_count", 0):
+        status = "ORIGIN_CONFLICT"
+    elif aggregate.get("strong_evidence") and aggregate.get("value") in {str(i) for i in range(9)}:
+        status = "ORIGIN_CONFIRMED"
+    else:
+        status = "ORIGIN_NEEDS_EVIDENCE"
+    return {
+        "product_id": product_id,
+        "status": status,
+        "value": aggregate.get("value") if status == "ORIGIN_CONFIRMED" else None,
+        "provenance": aggregate.get("provenance", []),
+        "reason": "explicit supplier observations only; CST digit ignored",
+    }
@@ -0,0 +1,49 @@
+"""Offline-safe product data provider boundary and permanent cache record."""
+
+from dataclasses import dataclass
+from datetime import datetime, timezone
+import hashlib
+import json
+from typing import Any, Protocol
+
+
+@dataclass(frozen=True)
+class ProviderRecord:
+    provider: str
+    provider_record_version: str
+    gtin: str
+    retrieved_at: str
+    raw_hash: str
+    normalized_data: dict[str, Any]
+    fiscal_authority: str = "OBSERVATION_ONLY"
+    cache_allowed: bool = False
+    redistribution_allowed: bool = False
+
+
+class ProductDataProvider(Protocol):
+    def lookup_gtin(self, gtin: str) -> ProviderRecord | None: ...
+
+
+def normalize_record(provider: str, version: str, gtin: str, data: dict[str, Any]) -> ProviderRecord:
+    normalized = {str(key): value for key, value in sorted(data.items()) if value not in (None, "")}
+    raw_hash = hashlib.sha256(json.dumps(normalized, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8")).hexdigest()
+    return ProviderRecord(provider, version, str(gtin), datetime.now(timezone.utc).isoformat(), raw_hash, normalized)
+
+
+def cache_key(gtin: str, provider: str, provider_record_version: str) -> str:
+    return f"{gtin}:{provider}:{provider_record_version}"
+
+
+def result_cache_key(product: dict[str, Any], candidates: list[str], datasets: dict[str, str], prompt: str, model: str) -> str:
+    payload = {"product": product, "candidates": candidates, "datasets": datasets, "prompt": prompt, "model": model}
+    return hashlib.sha256(json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8")).hexdigest()
+
+
+class LocalBulkDatasetProvider:
+    """Offline provider backed by explicitly supplied records; never performs I/O."""
+
+    def __init__(self, records: dict[str, ProviderRecord] | None = None):
+        self.records = records or {}
+
+    def lookup_gtin(self, gtin: str) -> ProviderRecord | None:
+        return self.records.get(str(gtin))