gustavosantos/GTI_GERENCIADOR_FISCAL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add autonomous fiscal completion v1

Browse Source
This commit is contained in:
gti
2026-09-22 13:08:38 -03:00
parent eada0df37e
commit e3ac68c70e
28 changed files with 44630 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
central/alembic/versions/0015_ai_budget_and_product_cache.py
+52
View File
@@ -0,0 +1,52 @@
"""Add central AI usage ledger and external GTIN cache."""
from alembic import op
import sqlalchemy as sa
revision = "0015_ai_budget_and_product_cache"
down_revision = "0014_ai_eval_cache"
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
"ai_usage_ledger",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("provider", sa.String(length=64), nullable=False),
sa.Column("model", sa.String(length=64), nullable=False),
sa.Column("run_id", sa.String(length=128), nullable=False),
sa.Column("tenant", sa.String(length=128), nullable=False),
sa.Column("items", sa.Integer(), nullable=False, server_default="0"),
sa.Column("input_tokens", sa.Integer(), nullable=False, server_default="0"),
sa.Column("output_tokens", sa.Integer(), nullable=False, server_default="0"),
sa.Column("cost", sa.Float(), nullable=False, server_default="0"),
sa.Column("timestamp", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
)
op.create_index("ix_ai_usage_ledger_run_id", "ai_usage_ledger", ["run_id"])
op.create_index("ix_ai_usage_ledger_tenant", "ai_usage_ledger", ["tenant"])
op.create_index("ix_ai_usage_ledger_timestamp", "ai_usage_ledger", ["timestamp"])
op.create_table(
"external_product_cache",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("gtin", sa.String(length=16), nullable=False),
sa.Column("provider", sa.String(length=64), nullable=False),
sa.Column("provider_record_version", sa.String(length=64), nullable=False),
sa.Column("retrieved_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.Column("raw_hash", sa.String(length=64), nullable=False),
sa.Column("normalized_data", sa.JSON(), nullable=False),
sa.Column("cache_allowed", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.Column("redistribution_allowed", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.UniqueConstraint("gtin", "provider", "provider_record_version", name="uq_external_product_cache_record"),
)
op.create_index("ix_external_product_cache_gtin", "external_product_cache", ["gtin"])
def downgrade():
op.drop_index("ix_external_product_cache_gtin", table_name="external_product_cache")
op.drop_table("external_product_cache")
op.drop_index("ix_ai_usage_ledger_timestamp", table_name="ai_usage_ledger")
op.drop_index("ix_ai_usage_ledger_tenant", table_name="ai_usage_ledger")
op.drop_index("ix_ai_usage_ledger_run_id", table_name="ai_usage_ledger")
op.drop_table("ai_usage_ledger")
central/app/ai/budget_ledger.py
+59
View File
@@ -0,0 +1,59 @@
"""Persistent-boundary-compatible budget guard; provider invocation happens after this check."""
from dataclasses import dataclass
from datetime import datetime, timezone
from threading import Lock
@dataclass(frozen=True)
class BudgetLimits:
max_run_cost_usd: float = 0.54
max_daily_cost_usd: float = 1.00
max_monthly_cost_usd: float = 10.00
max_requests_per_day: int = 100
max_new_products_per_run: int = 750
@dataclass(frozen=True)
class BudgetDecision:
allowed: bool
reason: str
reserved_cost_usd: float
@dataclass(frozen=True)
class UsageEntry:
run_id: str
tenant: str
cost_usd: float
requests: int
new_products: int
timestamp: str
class BudgetLedger:
"""Atomic in-process ledger with an explicit persistence boundary for Hub storage."""
def __init__(self, limits: BudgetLimits):
self.limits = limits
self.entries: list[UsageEntry] = []
self._lock = Lock()
def check_and_reserve(self, run_id: str, tenant: str, cost_usd: float, requests: int, new_products: int) -> BudgetDecision:
with self._lock:
run_cost = sum(e.cost_usd for e in self.entries if e.run_id == run_id)
day = datetime.now(timezone.utc).date()
day_entries = [e for e in self.entries if e.timestamp[:10] == day.isoformat()]
month_entries = [e for e in self.entries if e.timestamp[:7] == day.strftime("%Y-%m")]
if run_cost + cost_usd > self.limits.max_run_cost_usd:
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
if sum(e.cost_usd for e in day_entries) + cost_usd > self.limits.max_daily_cost_usd:
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
if sum(e.cost_usd for e in month_entries) + cost_usd > self.limits.max_monthly_cost_usd:
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
if sum(e.requests for e in day_entries) + requests > self.limits.max_requests_per_day:
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
if sum(e.new_products for e in self.entries if e.run_id == run_id) + new_products > self.limits.max_new_products_per_run:
return BudgetDecision(False, "AI_BUDGET_EXHAUSTED", 0.0)
self.entries.append(UsageEntry(run_id, tenant, cost_usd, requests, new_products, datetime.now(timezone.utc).isoformat()))
return BudgetDecision(True, "RESERVED", cost_usd)
central/app/deterministic_resolution.py
+36
View File
@@ -0,0 +1,36 @@
"""Conservative deterministic role, ST, and inbound scope resolution."""
def resolve_product_role(signals: dict) -> dict:
if signals.get("recipe") and signals.get("production"):
return {"value": "OWN_PRODUCTION", "status": "CONFIRMED", "evidence": ["recipe", "production"]}
if signals.get("purchase") and signals.get("sale") and not signals.get("recipe"):
return {"value": "RESALE", "status": "CONFIRMED", "evidence": ["purchase", "sale"]}
if signals.get("input_consumed") and signals.get("recipe"):
return {"value": "INPUT", "status": "CONFIRMED", "evidence": ["input_consumed", "recipe"]}
if signals.get("service_flag") and signals.get("service_transactions"):
return {"value": "SERVICE", "status": "CONFIRMED", "evidence": ["service_flag", "service_transactions"]}
return {"value": None, "status": "PENDING_PRODUCT_ROLE", "evidence": []}
def resolve_st_status(signals: dict) -> dict:
if not signals.get("official_cest"):
return {"value": "NOT_SUBJECT_TO_ST", "status": "CONFIRMED", "evidence": []}
if signals.get("purchase_retained") is True:
return {"value": "ST_ALREADY_WITHHELD_IN_PURCHASE", "status": "CONFIRMED", "evidence": ["official_cest", "purchase_retained"]}
if signals.get("purchase_retained") is False:
return {"value": "PRODUCT_SUBJECT_TO_ST", "status": "CONFIRMED", "evidence": ["official_cest", "purchase_not_retained"]}
return {"value": None, "status": "PENDING_ST_WITHHELD", "evidence": ["official_cest"]}
def resolve_entry_scope(supplier_ufs: list[str], company_uf: str) -> dict:
observed = {str(uf).strip().upper() for uf in supplier_ufs if uf}
if not observed:
return {"value": None, "status": "PENDING_ENTRY_SCOPE", "evidence": []}
if len(observed - {company_uf.upper()}) and company_uf.upper() in observed:
value = "MIXED_OR_DYNAMIC"
elif observed - {company_uf.upper()}:
value = "INTER"
else:
value = "INTRA"
return {"value": value, "status": "CONFIRMED", "evidence": sorted(observed)}
central/app/evidence_aggregation.py
+47
View File
@@ -0,0 +1,47 @@
"""Conservative aggregation of supplier/document observations."""
from dataclasses import dataclass
from collections import defaultdict
from typing import Any, Iterable
@dataclass(frozen=True)
class Observation:
product_id: str
attribute: str
value: str
supplier_id: str
document_id: str
observed_at: str
provenance: str = "SUPPLIER_OBSERVATION"
def aggregate_observations(observations: Iterable[Observation]) -> dict[tuple[str, str], dict[str, Any]]:
grouped: dict[tuple[str, str], list[Observation]] = defaultdict(list)
seen: set[tuple[str, str, str, str]] = set()
for observation in observations:
key = (observation.product_id, observation.attribute)
dedupe = (observation.product_id, observation.attribute, observation.supplier_id, observation.document_id)
if dedupe not in seen and observation.value not in (None, ""):
grouped[key].append(observation)
seen.add(dedupe)
output = {}
for key, rows in grouped.items():
values = {row.value for row in rows}
counts = {value: sum(row.value == value for row in rows) for value in values}
supplier_count = len({row.supplier_id for row in rows})
agreement_rate = max(counts.values()) / len(rows) if rows else 0.0
conflict_count = max(0, len(values) - 1)
strong = len(rows) >= 2 and supplier_count >= 2 and len(values) == 1
output[key] = {
"value": next(iter(values)) if strong else None,
"observation_count": len(rows),
"supplier_count": supplier_count,
"agreement_rate": agreement_rate,
"latest_observation": max(row.observed_at for row in rows),
"conflict_count": conflict_count,
"strong_evidence": strong,
"provenance": sorted({row.provenance for row in rows}),
}
return output
central/app/fiscal_completeness.py
+83
View File
@@ -0,0 +1,83 @@
"""Pure requiredness and fiscal completeness evaluation."""
import json
from pathlib import Path
from typing import Any
VALID_CLASSIFICATIONS = {
"REQUIRED", "CONDITIONAL", "OPTIONAL_REFERENCE", "NOT_APPLICABLE", "ERP_NOT_USED"
}
def load_requiredness_matrix(path: str | Path) -> dict[str, Any]:
matrix = json.loads(Path(path).read_text(encoding="utf-8"))
fields = matrix.get("fields", {})
if set(matrix.get("classifications", [])) != VALID_CLASSIFICATIONS:
raise ValueError("requiredness matrix classifications are incomplete")
if len(fields) != 23 or any(rule.get("classification") not in VALID_CLASSIFICATIONS for rule in fields.values()):
raise ValueError("requiredness matrix must define 23 valid field rules")
return matrix
def _has_value(value: Any) -> bool:
return value is not None and (not isinstance(value, str) or bool(value.strip()))
def _conditional_state(field: str, context: dict[str, Any]) -> str:
if field == "IPI":
return "REQUIRED" if context.get("contribuinte_ipi") == "S" and context.get("has_numeric_tipi") else "NOT_APPLICABLE"
if field == "EX_TIPI":
return "REQUIRED" if context.get("has_supported_ex") and context.get("product_matches_ex") else "NOT_APPLICABLE"
if field == "CEST":
return "REQUIRED" if context.get("has_official_cest") or context.get("st_subject") else "NOT_APPLICABLE"
if field in {"CFOP", "SITUACAO_TRIBUTARIA"}:
return "REQUIRED" if context.get("operation", "SALE") == "SALE" else "NOT_APPLICABLE"
if field in {"CFOP_ENTRADA", "CST_ENTRADA", "COD_ENTRADA_PIS", "COD_ENTRADA_COFINS"}:
return "REQUIRED" if context.get("operation") == "PURCHASE" and context.get("erp_consumes_entry_fields") else "NOT_APPLICABLE"
if field == "CST":
return "REQUIRED" if context.get("regime") == "NORMAL" and context.get("operation", "SALE") == "SALE" else "NOT_APPLICABLE"
if field == "CSOSN":
return "REQUIRED" if context.get("regime") == "SIMPLES" and context.get("operation", "SALE") == "SALE" else "NOT_APPLICABLE"
if field == "ALIQUOTA":
return "REQUIRED" if context.get("icms_taxed") else "NOT_APPLICABLE"
if field == "COD_ORIGEM_MERCADORIA":
return "REQUIRED" if context.get("origin_proven") else "NOT_APPLICABLE"
if field == "CST_IPI":
return "REQUIRED" if context.get("contribuinte_ipi") == "S" and context.get("erp_consumes_ipi_cst") else "NOT_APPLICABLE"
if field in {"COD_SAIDA_PIS", "COD_SAIDA_COFINS"}:
return "REQUIRED" if context.get("operation", "SALE") == "SALE" and context.get("piscofins_profile_proven") else "NOT_APPLICABLE"
if field in {"ALIQ_PIS", "ALIQ_COFINS"}:
return "REQUIRED" if context.get("piscofins_rate_applies") else "NOT_APPLICABLE"
if field in {"INCIDENCIA_PISCOFINS", "PISCOFINS"}:
return "REQUIRED" if context.get("piscofins_profile_proven") else "NOT_APPLICABLE"
return "NOT_APPLICABLE"
def evaluate_field(field: str, value: Any, context: dict[str, Any], matrix: dict[str, Any]) -> dict[str, Any]:
rule = matrix["fields"][field]
classification = rule["classification"]
if classification == "ERP_NOT_USED":
status = "ERP_NOT_SUPPORTED"
valid = True
elif classification == "NOT_APPLICABLE":
status = "NOT_APPLICABLE"
valid = True
elif classification == "OPTIONAL_REFERENCE":
status = "OPTIONAL_VALID" if _has_value(value) else "OPTIONAL_EMPTY"
valid = True
elif classification == "REQUIRED":
status = "REQUIRED_AND_VALID" if _has_value(value) else "REQUIRED_MISSING"
valid = _has_value(value)
else:
conditional = _conditional_state(field, context)
if conditional == "NOT_APPLICABLE":
status, valid = "CONDITIONAL_NOT_TRIGGERED", True
else:
status, valid = ("CONDITIONAL_AND_VALID", True) if _has_value(value) else ("REQUIRED_MISSING", False)
return {"field": field, "classification": classification, "status": status, "valid": valid, "value_present": _has_value(value), "source": rule["source"]}
def evaluate_product(product: dict[str, Any], context: dict[str, Any], matrix: dict[str, Any]) -> dict[str, Any]:
fields = {field: evaluate_field(field, product.get(field), context, matrix) for field in matrix["fields"]}
return {"FISCALLY_COMPLETE": all(result["valid"] for result in fields.values()), "fields": fields}
central/app/models.py
+30
View File
@@ -464,3 +464,33 @@ class AiEvalCache(Base):
answer: Mapped[dict] = mapped_column(JSON)
gold_ncm: Mapped[str | None] = mapped_column(String(8), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class AiUsageLedger(Base):
"""Central persistent usage ledger reserved before any future provider call."""
__tablename__ = "ai_usage_ledger"
id: Mapped[int] = mapped_column(primary_key=True)
provider: Mapped[str] = mapped_column(String(64))
model: Mapped[str] = mapped_column(String(64))
run_id: Mapped[str] = mapped_column(String(128), index=True)
tenant: Mapped[str] = mapped_column(String(128), index=True)
items: Mapped[int] = mapped_column(Integer, default=0)
input_tokens: Mapped[int] = mapped_column(Integer, default=0)
output_tokens: Mapped[int] = mapped_column(Integer, default=0)
cost: Mapped[float] = mapped_column(default=0.0)
timestamp: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), index=True)
class ExternalProductCache(Base):
"""Versioned GTIN cache; legal/provider flags remain explicit metadata."""
__tablename__ = "external_product_cache"
__table_args__ = (UniqueConstraint("gtin", "provider", "provider_record_version", name="uq_external_product_cache_record"),)
id: Mapped[int] = mapped_column(primary_key=True)
gtin: Mapped[str] = mapped_column(String(16), index=True)
provider: Mapped[str] = mapped_column(String(64))
provider_record_version: Mapped[str] = mapped_column(String(64))
retrieved_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
raw_hash: Mapped[str] = mapped_column(String(64))
normalized_data: Mapped[dict] = mapped_column(JSON, default=dict)
cache_allowed: Mapped[bool] = mapped_column(Boolean, default=False)
redistribution_allowed: Mapped[bool] = mapped_column(Boolean, default=False)
central/app/origin_resolution.py
+17
View File
@@ -0,0 +1,17 @@
"""Evidence-gated origin resolution; CST digits are never an origin source."""
def resolve_origin(product_id: str, aggregate: dict, kb_facts: dict) -> dict:
if aggregate.get("conflict_count", 0):
status = "ORIGIN_CONFLICT"
elif aggregate.get("strong_evidence") and aggregate.get("value") in {str(i) for i in range(9)}:
status = "ORIGIN_CONFIRMED"
else:
status = "ORIGIN_NEEDS_EVIDENCE"
return {
"product_id": product_id,
"status": status,
"value": aggregate.get("value") if status == "ORIGIN_CONFIRMED" else None,
"provenance": aggregate.get("provenance", []),
"reason": "explicit supplier observations only; CST digit ignored",
}
central/app/product_data_provider.py
+49
View File
@@ -0,0 +1,49 @@
"""Offline-safe product data provider boundary and permanent cache record."""
from dataclasses import dataclass
from datetime import datetime, timezone
import hashlib
import json
from typing import Any, Protocol
@dataclass(frozen=True)
class ProviderRecord:
provider: str
provider_record_version: str
gtin: str
retrieved_at: str
raw_hash: str
normalized_data: dict[str, Any]
fiscal_authority: str = "OBSERVATION_ONLY"
cache_allowed: bool = False
redistribution_allowed: bool = False
class ProductDataProvider(Protocol):
def lookup_gtin(self, gtin: str) -> ProviderRecord | None: ...
def normalize_record(provider: str, version: str, gtin: str, data: dict[str, Any]) -> ProviderRecord:
normalized = {str(key): value for key, value in sorted(data.items()) if value not in (None, "")}
raw_hash = hashlib.sha256(json.dumps(normalized, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8")).hexdigest()
return ProviderRecord(provider, version, str(gtin), datetime.now(timezone.utc).isoformat(), raw_hash, normalized)
def cache_key(gtin: str, provider: str, provider_record_version: str) -> str:
return f"{gtin}:{provider}:{provider_record_version}"
def result_cache_key(product: dict[str, Any], candidates: list[str], datasets: dict[str, str], prompt: str, model: str) -> str:
payload = {"product": product, "candidates": candidates, "datasets": datasets, "prompt": prompt, "model": model}
return hashlib.sha256(json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8")).hexdigest()
class LocalBulkDatasetProvider:
"""Offline provider backed by explicitly supplied records; never performs I/O."""
def __init__(self, records: dict[str, ProviderRecord] | None = None):
self.records = records or {}
def lookup_gtin(self, gtin: str) -> ProviderRecord | None:
return self.records.get(str(gtin))
central/tests/test_ai_budget_ledger.py
+16
View File
@@ -0,0 +1,16 @@
from central.app.ai.budget_ledger import BudgetLedger, BudgetLimits
def test_budget_rejects_before_provider_when_daily_limit_is_exceeded():
ledger = BudgetLedger(BudgetLimits(max_run_cost_usd=2.0, max_daily_cost_usd=1.0))
assert ledger.check_and_reserve("r1", "t1", 0.75, 1, 0).allowed is True
decision = ledger.check_and_reserve("r2", "t1", 0.30, 1, 0)
assert decision.allowed is False
assert decision.reason == "AI_BUDGET_EXHAUSTED"
def test_budget_allows_exact_limit_and_counts_new_products():
ledger = BudgetLedger(BudgetLimits(max_run_cost_usd=1.0, max_new_products_per_run=2))
assert ledger.check_and_reserve("r1", "t1", 1.0, 1, 2).allowed is True
assert ledger.check_and_reserve("r1", "t1", 0.0, 0, 1).allowed is False
central/tests/test_ai_idempotency.py
+15
View File
@@ -0,0 +1,15 @@
from central.app.product_data_provider import result_cache_key
def test_same_classification_inputs_have_same_cache_key():
first = result_cache_key({"gtin": "789", "description": "X"}, ["12345678"], {"ncm": "v1"}, "prompt-v1", "model-v1")
second = result_cache_key({"description": "X", "gtin": "789"}, ["12345678"], {"ncm": "v1"}, "prompt-v1", "model-v1")
assert first == second
def test_changed_dataset_changes_cache_key():
first = result_cache_key({"gtin": "789"}, ["12345678"], {"ncm": "v1"}, "prompt-v1", "model-v1")
second = result_cache_key({"gtin": "789"}, ["12345678"], {"ncm": "v2"}, "prompt-v1", "model-v1")
assert first != second
central/tests/test_deterministic_resolution.py
+16
View File
@@ -0,0 +1,16 @@
from central.app.deterministic_resolution import resolve_entry_scope, resolve_product_role, resolve_st_status
def test_product_role_requires_two_independent_strong_signals():
assert resolve_product_role({"recipe": True, "production": True})["value"] == "OWN_PRODUCTION"
assert resolve_product_role({"single_flag": True})["value"] is None
def test_st_distinguishes_subject_from_withheld():
assert resolve_st_status({"official_cest": True, "purchase_retained": True})["value"] == "ST_ALREADY_WITHHELD_IN_PURCHASE"
assert resolve_st_status({"official_cest": True, "purchase_retained": None})["status"] == "PENDING_ST_WITHHELD"
def test_entry_scope_is_dynamic_when_both_ufs_are_observed():
assert resolve_entry_scope(["BA", "SP"], "BA")["value"] == "MIXED_OR_DYNAMIC"
assert resolve_entry_scope(["BA"], "BA")["value"] == "INTRA"
central/tests/test_evidence_aggregation.py
+38
View File
@@ -0,0 +1,38 @@
from central.app.evidence_aggregation import Observation, aggregate_observations
from scripts.analyze_inbound_observations import summarize_rows
def test_two_suppliers_agree_without_conflict():
rows = [
Observation("P1", "origin", "0", "S1", "D1", "2026-01-01"),
Observation("P1", "origin", "0", "S2", "D2", "2026-02-01"),
]
result = aggregate_observations(rows)[("P1", "origin")]
assert result["observation_count"] == 2
assert result["supplier_count"] == 2
assert result["agreement_rate"] == 1.0
assert result["strong_evidence"] is True
assert result["conflict_count"] == 0
def test_supplier_disagreement_is_conflict_not_majority():
rows = [
Observation("P1", "origin", "0", "S1", "D1", "2026-01-01"),
Observation("P1", "origin", "0", "S2", "D2", "2026-02-01"),
Observation("P1", "origin", "1", "S3", "D3", "2026-03-01"),
]
result = aggregate_observations(rows)[("P1", "origin")]
assert result["conflict_count"] == 1
assert result["strong_evidence"] is False
assert result["value"] is None
def test_inbound_rows_become_supplier_observations_without_raw_identity():
result = summarize_rows([("P1", "S1", "2026-01-01", "12345678", "1102", "060")])
assert result["observation_count"] == 3
assert result["sources"] == ["SUPPLIER_OBSERVATION"]
central/tests/test_fiscal_completeness.py
+49
View File
@@ -0,0 +1,49 @@
import json
from pathlib import Path
from central.app.fiscal_completeness import evaluate_field, evaluate_product, load_requiredness_matrix
ROOT = Path(__file__).resolve().parents[2]
def test_requiredness_matrix_covers_contract_fields_once():
contract = json.loads((ROOT / "docs/FISCAL_FIELD_CONTRACT_V1.json").read_text(encoding="utf-8"))
matrix = load_requiredness_matrix(ROOT / "docs/FISCAL_REQUIREDNESS_MATRIX_V1.json")
assert set(matrix["fields"]) == set(contract["fields"])
assert len(matrix["fields"]) == 23
assert set(matrix["classifications"]) == {
"REQUIRED", "CONDITIONAL", "OPTIONAL_REFERENCE", "NOT_APPLICABLE", "ERP_NOT_USED"
}
def test_conditional_empty_when_not_triggered_is_not_missing():
matrix = load_requiredness_matrix(ROOT / "docs/FISCAL_REQUIREDNESS_MATRIX_V1.json")
result = evaluate_field("IPI", None, {"contribuinte_ipi": "N", "has_numeric_tipi": False}, matrix)
assert result["status"] == "CONDITIONAL_NOT_TRIGGERED"
assert result["valid"] is True
def test_required_missing_makes_product_incomplete():
matrix = load_requiredness_matrix(ROOT / "docs/FISCAL_REQUIREDNESS_MATRIX_V1.json")
result = evaluate_product(
{"CLASSIFICACAO_FISCAL": None, "CFOP": "5102"},
{"product_role": "RESALE", "operation": "SALE"},
matrix,
)
assert result["FISCALLY_COMPLETE"] is False
assert result["fields"]["CLASSIFICACAO_FISCAL"]["status"] == "REQUIRED_MISSING"
def test_erp_not_used_is_not_missing():
matrix = load_requiredness_matrix(ROOT / "docs/FISCAL_REQUIREDNESS_MATRIX_V1.json")
result = evaluate_field("IAT", None, {"erp_uses_iat": False}, matrix)
assert result["status"] == "ERP_NOT_SUPPORTED"
assert result["valid"] is True
central/tests/test_origin_resolution.py
+21
View File
@@ -0,0 +1,21 @@
from central.app.origin_resolution import resolve_origin
def test_origin_requires_strong_supplier_evidence():
result = resolve_origin("P1", {"value": "0", "strong_evidence": True, "conflict_count": 0}, {})
assert result["status"] == "ORIGIN_CONFIRMED"
assert result["value"] == "0"
def test_origin_conflict_stays_pending():
result = resolve_origin("P1", {"value": None, "strong_evidence": False, "conflict_count": 1}, {})
assert result["status"] == "ORIGIN_CONFLICT"
assert result["value"] is None
def test_origin_does_not_use_cst_digit():
result = resolve_origin("P1", {}, {"cst": "040"})
assert result["status"] == "ORIGIN_NEEDS_EVIDENCE"
central/tests/test_product_data_provider.py
+16
View File
@@ -0,0 +1,16 @@
from central.app.product_data_provider import ProviderRecord, cache_key, normalize_record
def test_provider_record_has_stable_gtin_cache_key_and_hash():
record = normalize_record("GS1", "v1", "789", {"brand": "Acme", "description": "Produto"})
assert isinstance(record, ProviderRecord)
assert record.raw_hash
assert cache_key(record.gtin, record.provider, record.provider_record_version).startswith("789:")
def test_provider_normalization_does_not_claim_fiscal_authority():
record = normalize_record("COMMERCIAL", "v1", "789", {"ncm": "12345678"})
assert record.normalized_data["ncm"] == "12345678"
assert record.fiscal_authority == "OBSERVATION_ONLY"
docs/AI_PROJECT_STATE.md
+14
View File
@@ -1,5 +1,19 @@
# AI Project State
## Autonomous fiscal completion V1 (2026-09-22)
`AUTONOMOUS_FISCAL_COMPLETION_V1_MEASURED` — requiredness matrix and pure
`FiscalCompletenessEvaluator` are active; read-only inbound discovery covered
all five allowlisted databases and produced `ERP_INBOUND_FISCAL_EVIDENCE_MAP_V1`
plus supplier observation metrics. Origin, role, ST/withheld, and entry scope
remain evidence-gated. No human decisions were answered, no NCM conflicts were
resolved, no external product provider was queried, and no Firebird write was
executed. The central `ai_usage_ledger` and `external_product_cache` schema
objects are defined by migration 0015, with offline budget and idempotency
guards. Measured matrix completeness is 34,872/35,066 products; AI residual is
unchanged at 511. `AI_WRITE_ALLOWED=false`, `FIREBIRD_AI_WRITES=0`, retrieval
V2 remains active, and retrieval V3 remains unpromoted.
## Current status (real LLM evaluation V1 — 2026-09-21)
`BLOCKED_EXTERNAL_AI_CREDENTIAL` (implementation complete). The entire real-LLM
docs/AUTONOMOUS_FISCAL_COMPLETION_V1.json
+86
View File
@@ -0,0 +1,86 @@
{
"artifact": "AUTONOMOUS_FISCAL_COMPLETION_V1",
"requiredness_matrix": "FISCAL_REQUIREDNESS_MATRIX_V1",
"firebird_writes": 0,
"external_api_calls": 0,
"databases": [
{
"database": "TESTESOFICIAIS.FDB",
"TOTAL_PRODUCTS": 521,
"FISCALLY_COMPLETE": 495,
"FISCALLY_INCOMPLETE": 26,
"required_valid": 521,
"required_missing": 46,
"missing_reasons": {
"CFOP": 21,
"CSOSN": 24,
"SITUACAO_TRIBUTARIA": 1
},
"company_context_observed": true,
"manual_context_not_answered": true,
"mode": "READ_ONLY"
},
{
"database": "CONSTRUCAO.FDB",
"TOTAL_PRODUCTS": 3712,
"FISCALLY_COMPLETE": 3685,
"FISCALLY_INCOMPLETE": 27,
"required_valid": 3712,
"required_missing": 30,
"missing_reasons": {
"SITUACAO_TRIBUTARIA": 25,
"CFOP": 2,
"CSOSN": 3
},
"company_context_observed": true,
"manual_context_not_answered": true,
"mode": "READ_ONLY"
},
{
"database": "MMMODAS.FDB",
"TOTAL_PRODUCTS": 20997,
"FISCALLY_COMPLETE": 20889,
"FISCALLY_INCOMPLETE": 108,
"required_valid": 20997,
"required_missing": 108,
"missing_reasons": {
"CFOP": 55,
"ALIQUOTA": 50,
"SITUACAO_TRIBUTARIA": 3
},
"company_context_observed": true,
"manual_context_not_answered": true,
"mode": "READ_ONLY"
},
{
"database": "RINCAO.FDB",
"TOTAL_PRODUCTS": 894,
"FISCALLY_COMPLETE": 890,
"FISCALLY_INCOMPLETE": 4,
"required_valid": 894,
"required_missing": 4,
"missing_reasons": {
"SITUACAO_TRIBUTARIA": 1,
"ALIQUOTA": 3
},
"company_context_observed": false,
"manual_context_not_answered": true,
"mode": "READ_ONLY"
},
{
"database": "UNIAO.FDB",
"TOTAL_PRODUCTS": 8942,
"FISCALLY_COMPLETE": 8913,
"FISCALLY_INCOMPLETE": 29,
"required_valid": 8942,
"required_missing": 29,
"missing_reasons": {
"SITUACAO_TRIBUTARIA": 28,
"CFOP": 1
},
"company_context_observed": true,
"manual_context_not_answered": true,
"mode": "READ_ONLY"
}
]
}
docs/AUTONOMOUS_FISCAL_COMPLETION_V1.md
+145
View File
@@ -0,0 +1,145 @@
# Autonomous Fiscal Completion V1
## Mission Result
`STATUS=COMPLETED_OFFLINE_WITH_MEASURED_RESIDUALS`
This mission ran without Z.ai, OpenAI, external provider queries, human-answer
inference, or Firebird writes. Discovery used read-only transactions on the five
explicitly allowlisted local copies.
## Consolidated Report
```text
DATABASES=5
TOTAL_PRODUCTS=35066
REQUIREDNESS_FIELDS=23
REQUIREDNESS_CONTEXTS=10
INBOUND_TABLES_DISCOVERED=1690 (schema map; external table row counts recorded as unavailable)
INBOUND_ITEMS_ANALYZED=118519
XML_DOCUMENTS_AVAILABLE=true (BLOB/XML-capable tables detected; payloads not exported)
ORIGIN_PENDING_BEFORE=35056
ORIGIN_RESOLVED_FROM_INBOUND=0
ORIGIN_CONFLICTS=0
ORIGIN_PENDING_AFTER=35056
PRODUCT_ROLE_PENDING_BEFORE=43
PRODUCT_ROLE_RESOLVED_DETERMINISTICALLY=0 (new mission resolutions)
PRODUCT_ROLE_PENDING_AFTER=43
ST_PENDING_BEFORE=86
ST_RESOLVED_FROM_INBOUND=0
ST_PENDING_AFTER=86
PIS_COFINS_PENDING_BEFORE=3 company profiles / product rules unavailable
PIS_COFINS_PENDING_AFTER=3 company profiles / product rules unavailable
IPI_CLASSIFICATION=OPTIONAL_REFERENCE for observed non-contributing companies; numeric TIPI remains reference data unless contribution/use is proven
IPI_HUMAN_DECISION_REQUIRED=false for the reference-only classification; no STORE_IPI_PROFILE applied
KB_PRODUCTS=30801
KB_CROSS_TENANT_REUSE=251 valid GTIN cross-tenant matches observed; tenant policy was not copied
EXTERNAL_PRODUCT_PROVIDERS_EVALUATED=5 candidate categories
RECOMMENDED_EXTERNAL_PROVIDER=authorization_required (licensed GS1/commercial bulk source)
RECOMMENDED_PROVIDER_COST=NOT_VERIFIED
EXPECTED_PROVIDER_COVERAGE=NOT_VERIFIED
EXPECTED_PROVIDER_AI_REDUCTION=NOT_VERIFIED
AI_BUDGET_PER_RUN=0.54 USD default guard
AI_BUDGET_DAILY=1.00 USD default guard
AI_BUDGET_MONTHLY=10.00 USD default guard
AI_USAGE_LEDGER_READY=true (model and migration 0015; no usage recorded)
TOTAL_REQUIRED_FIELD_INSTANCES=35283
REQUIRED_VALID=35066
REQUIRED_MISSING=217
FISCALLY_COMPLETE_PRODUCTS=34872
FISCALLY_INCOMPLETE_PRODUCTS=194
AUTONOMOUS_RATE=0.9944675754 (matrix-measured; human context remains incomplete)
AI_READY_BEFORE=511
AI_READY_AFTER=511
MANUAL_COMPANY_DECISIONS_REMAINING=19
MANUAL_GROUP_DECISIONS_REMAINING=738
MANUAL_PRODUCT_FACTS_REMAINING=46036
NEW_DETERMINISTIC_WRITES=0
NEW_DETERMINISTIC_WRITES_VERIFIED=0
SECOND_PASS_WRITES=0
FULL_TESTS=tests 324 passed, 1 skipped; central/tests 219 passed, 1 warning
FIREBIRD_AI_WRITES=0
AI_WRITE_ALLOWED=false
LOCAL_PRODUCT_COMPLETE=MEASURED_FOR_REQUIREDNESS_MATRIX; HUMAN_CONTEXT_INCOMPLETE
READY_FOR_ZAI_GOLD_EVAL=true
```
## What The ERP Provided
The read-only evidence map discovered 5 databases, 1,680 user tables/relations,
foreign-key candidates, fiscal item/header/supplier/key categories, and
XML/BLOB-capable structures. Populated inbound item evidence was concentrated
in `C000088` and `SINTEGRA_ENTRADA_ITEM`; 118,519 inbound rows were analyzed.
Supplier observations produced 284 strong non-origin aggregates and 406
conflicts in MMMODAS, but no inbound origin field was available in the
analyzed item shape. Consequently origin remained `ORIGIN_NEEDS_EVIDENCE`.
`C000062` and `C000032` contain substantial historical fiscal activity, but
their observed CFOP distributions and existing analysis do not prove that they
are inbound supplier evidence. They were not promoted as such.
## Requiredness And Completeness
The evaluator distinguishes required, conditional, reference-only, not
applicable, and ERP-not-used. Empty IPI, IAT, IPPT, PIS/COFINS, and other
conditional/reference fields are not automatically counted as incomplete when
their trigger is not proven. The 194 incomplete products are measured required
field gaps under the current known sale/company context, primarily CFOP,
CSOSN, ICMS treatment, and ICMS rate. This is not a claim that every company
or product context is complete.
## Writes And Safety
No new deterministic candidate passed the complete evidence/provenance and
write-guard contract without human context or unresolved conflict. Therefore
the correct result was zero writes, not forced completion. The write audit
records `NEW_DETERMINISTIC_WRITES=0`, `NEW_DETERMINISTIC_WRITES_VERIFIED=0`,
and `SECOND_PASS_WRITES=0`.
Current `NCM_CONFLICT=90` rows and all 73 historical
`INSUFFICIENT_EVIDENCE` conflicts were preserved. No origin default, ST
withholding, company decision, group decision, or product fact was invented.
## External Data And Future AI
The provider adapter accepts GTIN records with provider version, retrieval time,
raw hash, normalized data, and cache/redistribution flags. The central schema
now has the `ai_usage_ledger` and `external_product_cache` tables via migration
0015. Budget checks reserve atomically before a future provider invocation and
return `AI_BUDGET_EXHAUSTED` without sending a request. Identical future AI
inputs have a stable cache key; no provider or AI call occurred in this mission.
## Remaining Gaps
- Human COMPANY/GROUP/PRODUCT facts remain unresolved.
- Origin has no sufficiently strong inbound evidence for global promotion.
- ST withholding is not proven merely by ST subject/CEST evidence.
- PIS/COFINS NCM-linked Tabela 4.3.x rules are still unavailable locally.
- External provider price, license, and coverage require operator authorization and source contract review.
- The real Z.ai GOLD evaluation remains blocked by `EXTERNAL_AI_CREDENTIAL_OR_BALANCE`.
## Verdict
`VERDICT=MAXIMUM_MEASURED_OFFLINE_COMPLETION_WITHOUT_UNSAFE_INFERENCE`
The system now has the requiredness contract, read-only inbound evidence map,
conservative evidence aggregation, origin/role/ST/entry-scope gates, provider
cache boundary, central budget schema, idempotency key, and measured autonomy
artifacts. Remaining decisions are genuine evidence or authorization gaps.
docs/AUTONOMOUS_WRITE_AUDIT_V1.json
+11
View File
@@ -0,0 +1,11 @@
{
"artifact": "AUTONOMOUS_WRITE_AUDIT_V1",
"mode": "DRY_RUN_ONLY",
"firebird_writes": 0,
"databases": ["TESTESOFICIAIS", "CONSTRUCAO", "MMMODAS", "RINCAO", "UNIAO"],
"new_deterministic_writes": 0,
"new_deterministic_writes_verified": 0,
"second_pass_writes": 0,
"reason": "No new candidate passed the complete evidence, provenance, path, company, schema, type, compare-before-write, rowcount, and post-write gates without human context or unresolved conflict.",
"ai_write_allowed": false
}
docs/ERP_INBOUND_FISCAL_EVIDENCE_MAP_V1.json
+43292
View File
File diff suppressed because it is too large Load Diff
docs/EXTERNAL_PRODUCT_DATA_PROVIDER_EVALUATION_V1.md
+33
View File
@@ -0,0 +1,33 @@
# External Product Data Provider Evaluation V1
## Scope
This is a comparison only. No service was purchased, queried, or enabled.
External data remains `OBSERVATION_ONLY` until official validation and product
identity checks succeed.
| Candidate | Type | GTIN lookup | NCM/CEST | Bulk/cache | Cost | License status |
|---|---|---:|---:|---:|---|---|
| GS1 Brasil / official GTIN services | Official/commercial | available subject to account | not guaranteed as fiscal authority | contract-dependent | NOT_VERIFIED | authorization required |
| Licensed commercial catalog API | Commercial | usually available | vendor-dependent | contract-dependent | NOT_VERIFIED | license review required |
| Licensed commercial bulk dataset | Commercial bulk | expected | vendor-dependent | potentially best for cache | NOT_VERIFIED | redistribution/cache review required |
| Local bulk dataset | Local/offline | only if supplied | only if sourced/validated | yes | acquisition-dependent | must prove license |
| Official fiscal datasets already in Hub | Official fiscal | no GTIN lookup | NCM/CEST available | yes | existing | permitted for fiscal validation |
## Scenario Economics
For 35k products, 70 clients, 100k products, and 500k products, a permanent
licensed bulk dataset can save repeated lookups, but no price or query limit was
verified locally. Therefore `initial_cost`, `recurring_cost`, and exact
`queries_saved` remain `NOT_VERIFIED`; no purchase recommendation is made.
## Recommendation
`RECOMMENDED_SOURCE=authorization_required`
The adapter boundary is implemented offline in
`central/app/product_data_provider.py`. A future provider must supply provider
version, raw hash, normalized data, cache permission, and redistribution terms.
No commercial source is authoritative for fiscal fields without official
validation. Expected AI reduction cannot be measured before an authorized
dataset is available, so `EXPECTED_PROVIDER_AI_REDUCTION=NOT_VERIFIED`.
docs/FISCAL_REQUIREDNESS_MATRIX_V1.json
+48
View File
@@ -0,0 +1,48 @@
{
"artifact": "FISCAL_REQUIREDNESS_MATRIX_V1",
"version": "requiredness-v1",
"source_contract": "docs/FISCAL_FIELD_CONTRACT_V1.json",
"classifications": [
"REQUIRED",
"CONDITIONAL",
"OPTIONAL_REFERENCE",
"NOT_APPLICABLE",
"ERP_NOT_USED"
],
"contexts": [
"CRT/regime",
"UF",
"entry_or_sale",
"intra_or_interstate",
"consumer_final",
"resale_or_production_or_input_or_service",
"ST",
"monophase",
"IPI_contributor"
],
"fields": {
"CLASSIFICACAO_FISCAL": {"classification": "REQUIRED", "when": "all_products", "validity": "8-digit active NCM", "source": "active NCM dataset", "human_decision_required": true},
"IPI": {"classification": "CONDITIONAL", "when": "IPI_contributor_and_numeric_TIPI_rate", "validity": "numeric TIPI rate", "source": "active TIPI dataset", "human_decision_required": false},
"EX_TIPI": {"classification": "CONDITIONAL", "when": "supported_EX_row_and_product_evidence", "validity": "active TIPI EX code", "source": "active TIPI dataset plus description evidence", "human_decision_required": true},
"CEST": {"classification": "CONDITIONAL", "when": "official_CEST_candidate_or_ST_context", "validity": "official CEST for NCM", "source": "active CEST dataset", "human_decision_required": true},
"SITUACAO_TRIBUTARIA": {"classification": "CONDITIONAL", "when": "sale_item_and_ICMS_context_applies", "validity": "declared T/I/N/S not contradicted", "source": "ERP plus ST evidence", "human_decision_required": true},
"CFOP": {"classification": "CONDITIONAL", "when": "sale_item", "validity": "official operation-compatible CFOP", "source": "CFOP dataset plus role/ST context", "human_decision_required": true},
"CFOP_ENTRADA": {"classification": "CONDITIONAL", "when": "purchase_default_is_used_by_ERP", "validity": "operation-specific 1xxx/2xxx CFOP", "source": "inbound evidence and ERP behavior", "human_decision_required": true},
"CST": {"classification": "CONDITIONAL", "when": "normal_regime_ICMS_context", "validity": "valid ICMS CST for operation", "source": "ERP and official CST dataset", "human_decision_required": true},
"CST_ENTRADA": {"classification": "CONDITIONAL", "when": "inbound_CST_is_consumed_by_ERP", "validity": "valid inbound CST", "source": "inbound document evidence", "human_decision_required": true},
"CSOSN": {"classification": "CONDITIONAL", "when": "Simples_Nacional_context", "validity": "valid CSOSN for operation", "source": "ERP and official CSOSN dataset", "human_decision_required": true},
"ALIQUOTA": {"classification": "CONDITIONAL", "when": "taxed_ICMS_operation", "validity": "rate proven by company/UF context", "source": "ERP and official rate evidence", "human_decision_required": true},
"COD_ORIGEM_MERCADORIA": {"classification": "CONDITIONAL", "when": "origin_is_proven_for_product_or_purchase", "validity": "0-8 with explicit evidence", "source": "inbound evidence or validated product fact", "human_decision_required": true},
"CST_IPI": {"classification": "CONDITIONAL", "when": "IPI_contributor_and_CST_consumed", "validity": "valid IPI CST", "source": "ERP and document evidence", "human_decision_required": true},
"COD_ENTRADA_PIS": {"classification": "CONDITIONAL", "when": "PIS_COFINS_profile_and_inbound_field_consumed", "validity": "valid PIS CST", "source": "SPED/ERP evidence", "human_decision_required": true},
"COD_SAIDA_PIS": {"classification": "CONDITIONAL", "when": "PIS_COFINS_profile_and_sale_field_consumed", "validity": "valid PIS CST", "source": "SPED/ERP evidence", "human_decision_required": true},
"ALIQ_PIS": {"classification": "CONDITIONAL", "when": "PIS_rate_applies", "validity": "proven PIS rate", "source": "SPED/official table/ERP evidence", "human_decision_required": true},
"COD_ENTRADA_COFINS": {"classification": "CONDITIONAL", "when": "PIS_COFINS_profile_and_inbound_field_consumed", "validity": "valid COFINS CST", "source": "SPED/ERP evidence", "human_decision_required": true},
"COD_SAIDA_COFINS": {"classification": "CONDITIONAL", "when": "PIS_COFINS_profile_and_sale_field_consumed", "validity": "valid COFINS CST", "source": "SPED/ERP evidence", "human_decision_required": true},
"ALIQ_COFINS": {"classification": "CONDITIONAL", "when": "COFINS_rate_applies", "validity": "proven COFINS rate", "source": "SPED/official table/ERP evidence", "human_decision_required": true},
"INCIDENCIA_PISCOFINS": {"classification": "CONDITIONAL", "when": "PIS_COFINS_profile_is_proven", "validity": "proven incidence", "source": "official monophase data or ERP evidence", "human_decision_required": true},
"PISCOFINS": {"classification": "CONDITIONAL", "when": "PIS_COFINS_profile_is_proven", "validity": "valid PIS/COFINS profile", "source": "accounting/ERP evidence", "human_decision_required": true},
"IAT": {"classification": "ERP_NOT_USED", "when": "no_proven_SGV_consumption", "validity": "not evaluated as operational input", "source": "ERP binding audit", "human_decision_required": false},
"IPPT": {"classification": "ERP_NOT_USED", "when": "no_proven_SGV_consumption", "validity": "not evaluated as operational input", "source": "ERP binding audit", "human_decision_required": false}
}
}
docs/INBOUND_SUPPLIER_OBSERVATIONS_V1.json
+71
View File
@@ -0,0 +1,71 @@
{
"artifact": "INBOUND_SUPPLIER_OBSERVATIONS_V1",
"mode": "READ_ONLY",
"firebird_writes": 0,
"databases": [
{
"database": "TESTESOFICIAIS.FDB",
"status": "ANALYZED",
"items_analyzed": 12,
"observation_count": 14,
"aggregate_count": 13,
"sources": [
"SUPPLIER_OBSERVATION"
],
"strong_evidence_count": 0,
"conflict_count": 1,
"origin_field_available": false,
"origin_resolution": "ORIGIN_NEEDS_EVIDENCE"
},
{
"database": "CONSTRUCAO.FDB",
"status": "ANALYZED",
"items_analyzed": 0,
"observation_count": 0,
"aggregate_count": 0,
"sources": [],
"strong_evidence_count": 0,
"conflict_count": 0,
"origin_field_available": false,
"origin_resolution": "ORIGIN_NEEDS_EVIDENCE"
},
{
"database": "MMMODAS.FDB",
"status": "ANALYZED",
"items_analyzed": 117657,
"observation_count": 274597,
"aggregate_count": 30110,
"sources": [
"SUPPLIER_OBSERVATION"
],
"strong_evidence_count": 284,
"conflict_count": 406,
"origin_field_available": false,
"origin_resolution": "ORIGIN_NEEDS_EVIDENCE"
},
{
"database": "RINCAO.FDB",
"status": "ANALYZED",
"items_analyzed": 0,
"observation_count": 0,
"aggregate_count": 0,
"sources": [],
"strong_evidence_count": 0,
"conflict_count": 0,
"origin_field_available": false,
"origin_resolution": "ORIGIN_NEEDS_EVIDENCE"
},
{
"database": "UNIAO.FDB",
"status": "ANALYZED",
"items_analyzed": 0,
"observation_count": 0,
"aggregate_count": 0,
"sources": [],
"strong_evidence_count": 0,
"conflict_count": 0,
"origin_field_available": false,
"origin_resolution": "ORIGIN_NEEDS_EVIDENCE"
}
]
}
docs/LOCAL_PRODUCT_HEALTH_V2.json
+49
View File
@@ -0,0 +1,49 @@
{
"artifact": "LOCAL_PRODUCT_HEALTH_V2",
"status": "PASS_WITH_MEASURED_RESIDUALS",
"databases": 5,
"products": 35066,
"fiscally_complete_products": 34872,
"fiscally_incomplete_products": 194,
"autonomous_rate": 0.9944675754,
"required_field_instances_valid": 35066,
"required_field_instances_missing": 217,
"incompleteness_reason_instances": {
"MISSING_COMPANY_FACT": 0,
"MISSING_PRODUCT_FACT": 0,
"MISSING_EXTERNAL_PRODUCT_DATA": 0,
"AI_SEMANTIC_CANDIDATE": 0,
"DATA_CONFLICT": 0,
"ERP_SCHEMA_LIMITATION": 0,
"REQUIRED_FIELD_MISSING_MEASURED": 217
},
"company_context_complete": false,
"manual_company_decisions_remaining": 19,
"manual_group_decisions_remaining": 738,
"manual_product_facts_remaining": 46036,
"kb_products": 30801,
"kb_cross_tenant_reuse": 251,
"inbound_items_analyzed": 118519,
"xml_documents_available": true,
"origin_pending_before": 35056,
"origin_resolved_from_inbound": 0,
"origin_conflicts": 0,
"origin_pending_after": 35056,
"ai_ready_before": 511,
"ai_ready_after": 511,
"firebird_writes": 0,
"semantic_writes": 0,
"external_api_calls": 0,
"ai_write_allowed": false,
"retrieval_active": "V2",
"retrieval_v3": "NOT_ACTIVATED_WITHOUT_HOLDOUT_GAIN",
"current_ncm_conflicts": 90,
"historical_conflicts": 73,
"historical_conflicts_resolved": 0,
"second_pass_writes": 0,
"notes": [
"Completeness is evaluated using the requiredness matrix, not 23-field population.",
"Inbound observations are supplier observations and do not authorize origin or ST_WITHHELD decisions.",
"No deterministic write passed the complete evidence and guard contract in this run."
]
}
scripts/analyze_inbound_observations.py
+82
View File
@@ -0,0 +1,82 @@
"""Read-only extraction of populated supplier-item fiscal observations."""
from __future__ import annotations
import hashlib
import json
import os
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from central.app.evidence_aggregation import Observation, aggregate_observations # noqa: E402
def _hash(value: object) -> str:
return hashlib.sha256(str(value).encode("utf-8")).hexdigest()[:16]
def summarize_rows(rows: list[tuple[object, ...]]) -> dict:
observations = []
for product, supplier, observed_at, ncm, cfop, cst in rows:
identity = _hash(product)
supplier_id = _hash(supplier)
document = _hash(f"{supplier}:{product}:{observed_at}")
for attribute, value in (("ncm", ncm), ("cfop", cfop), ("cst", cst)):
if value not in (None, ""):
observations.append(Observation(identity, attribute, str(value).strip(), supplier_id, document, str(observed_at), "SUPPLIER_OBSERVATION"))
aggregate = aggregate_observations(observations)
return {
"observation_count": len(observations),
"aggregate_count": len(aggregate),
"sources": ["SUPPLIER_OBSERVATION"] if observations else [],
"strong_evidence_count": sum(1 for row in aggregate.values() if row["strong_evidence"]),
"conflict_count": sum(row["conflict_count"] for row in aggregate.values()),
}
def _env() -> None:
path = ROOT / ".env"
if path.exists():
for line in path.read_text(encoding="utf-8").splitlines():
if "=" in line and not line.lstrip().startswith("#"):
key, _, value = line.partition("=")
os.environ.setdefault(key.strip(), value.strip())
def analyze_database(path: Path) -> dict:
import fdb
conn = fdb.connect(database=str(path), user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"], fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"), isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
try:
cur = conn.cursor()
cur.execute("SELECT TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME = 'C000088'")
fields = {row[0] for row in cur.fetchall()}
required = {"CODPRODUTO", "CODFORNECEDOR", "DATA", "CLASSIFICACAO_FISCAL", "CFOP", "CST"}
if not required <= fields:
return {"database": path.name, "status": "NO_COMPLETE_INBOUND_ITEM_SHAPE", "fields_available": sorted(fields)}
cur.execute("SELECT CODPRODUTO, CODFORNECEDOR, DATA, CLASSIFICACAO_FISCAL, CFOP, CST FROM C000088")
rows = cur.fetchall()
summary = summarize_rows(rows)
return {"database": path.name, "status": "ANALYZED", "items_analyzed": len(rows), **summary, "origin_field_available": False, "origin_resolution": "ORIGIN_NEEDS_EVIDENCE"}
finally:
conn.close()
def main() -> None:
_env()
paths = [ROOT / "TESTESOFICIAIS.FDB"] + sorted((ROOT / "Banco").glob("*.FDB"))
result = {"artifact": "INBOUND_SUPPLIER_OBSERVATIONS_V1", "mode": "READ_ONLY", "firebird_writes": 0, "databases": []}
for path in paths:
try:
result["databases"].append(analyze_database(path))
except Exception as exc:
result["databases"].append({"database": path.name, "status": "UNAVAILABLE", "error_type": type(exc).__name__})
(ROOT / "docs/INBOUND_SUPPLIER_OBSERVATIONS_V1.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"artifact": result["artifact"], "databases": len(result["databases"]), "firebird_writes": 0}))
if __name__ == "__main__":
main()
scripts/discover_inbound_fiscal_evidence.py
+114
View File
@@ -0,0 +1,114 @@
"""Discover inbound fiscal evidence from SGV databases using SELECT-only access."""
from __future__ import annotations
import json
import os
import re
from pathlib import Path
from typing import Any, Iterable
ROOT = Path(__file__).resolve().parents[1]
SENSITIVE = re.compile(r"CNPJ|CPF|XML|CHAVE|RAZAO|NOME|ENDERECO|FONE|EMAIL", re.I)
TOKEN_GROUPS = {
"HEADER": {"NOTA", "NF", "DOCUMENTO", "CABECALHO", "EMISSAO", "SERIE"},
"ITEM": {"ITEM", "CODPRODUTO", "CODIGO", "QTD", "QUANTIDADE", "VALOR"},
"SUPPLIER": {"FORNEC", "EMITENTE", "CNPJ", "CPF", "RAZAO"},
"KEY": {"CHAVE", "NFE", "NFCE", "ACCESS", "PROTOCOLO"},
"FISCAL_ITEM": {"NCM", "CFOP", "CST", "CSOSN", "CEST", "ORIG", "ICMS", "PIS", "COFINS", "IPI", "MVA"},
}
def classify_evidence_source(table: str, columns: Iterable[str]) -> set[str]:
text = f"{table} {' '.join(columns)}".upper()
result = {category for category, tokens in TOKEN_GROUPS.items() if any(token in text for token in tokens)}
return result or {"UNKNOWN"}
def discover_from_rows(column_rows: Iterable[tuple[Any, ...]], relationship_rows: Iterable[tuple[Any, ...]]) -> dict[str, Any]:
tables: dict[str, dict[str, Any]] = {}
for table, field, field_type, field_length in column_rows:
table = str(table).strip()
field = str(field).strip()
record = tables.setdefault(table, {"columns": [], "categories": [], "xml_or_blob": False})
if not SENSITIVE.search(field):
record["columns"].append(field)
if field_type == 261 or re.search(r"XML|BLOB|PAYLOAD|DOCUMENT", field, re.I):
record["xml_or_blob"] = True
for table, record in tables.items():
categories = classify_evidence_source(table, record["columns"])
record["categories"] = sorted(categories)
relationships = []
for from_table, from_field, to_table, to_field in relationship_rows:
relationships.append({"from_table": str(from_table).strip(), "from_field": str(from_field).strip(), "to_table": str(to_table).strip(), "to_field": str(to_field).strip()})
return {"tables": tables, "relationships": relationships}
def safe_row_count(cursor: Any, table: str) -> dict[str, Any] | int:
try:
cursor.execute(f'SELECT COUNT(*) FROM "{table}"')
return int(cursor.fetchone()[0])
except Exception as exc: # external tables can be blocked by server policy
return {"status": "ROW_COUNT_UNAVAILABLE", "error_type": type(exc).__name__}
def _load_env() -> None:
path = ROOT / ".env"
if path.exists():
for line in path.read_text(encoding="utf-8").splitlines():
if "=" in line and not line.lstrip().startswith("#"):
key, _, value = line.partition("=")
os.environ.setdefault(key.strip(), value.strip())
def discover_database(path: Path) -> dict[str, Any]:
import fdb
conn = fdb.connect(
database=str(path),
user=os.environ["FB_USER"],
password=os.environ["FB_PASSWORD"],
fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO,
)
try:
cur = conn.cursor()
cur.execute("""SELECT TRIM(rf.RDB$RELATION_NAME), TRIM(rf.RDB$FIELD_NAME), f.RDB$FIELD_TYPE, COALESCE(f.RDB$CHARACTER_LENGTH, f.RDB$FIELD_LENGTH)
FROM RDB$RELATION_FIELDS rf JOIN RDB$FIELDS f ON f.RDB$FIELD_NAME = rf.RDB$FIELD_SOURCE
JOIN RDB$RELATIONS r ON r.RDB$RELATION_NAME = rf.RDB$RELATION_NAME
WHERE COALESCE(r.RDB$SYSTEM_FLAG, 0) = 0""")
columns = cur.fetchall()
cur.execute("""SELECT TRIM(rc.RDB$RELATION_NAME), TRIM(s.RDB$FIELD_NAME), TRIM(rc2.RDB$RELATION_NAME), TRIM(s2.RDB$FIELD_NAME)
FROM RDB$RELATION_CONSTRAINTS rc
JOIN RDB$INDEX_SEGMENTS s ON s.RDB$INDEX_NAME = rc.RDB$INDEX_NAME
JOIN RDB$REF_CONSTRAINTS ref ON ref.RDB$CONSTRAINT_NAME = rc.RDB$CONSTRAINT_NAME
JOIN RDB$RELATION_CONSTRAINTS rc2 ON rc2.RDB$CONSTRAINT_NAME = ref.RDB$CONST_NAME_UQ
JOIN RDB$INDEX_SEGMENTS s2 ON s2.RDB$INDEX_NAME = rc2.RDB$INDEX_NAME
WHERE rc.RDB$CONSTRAINT_TYPE = 'FOREIGN KEY'""")
relationships = cur.fetchall()
result = discover_from_rows(columns, relationships)
for table, record in result["tables"].items():
record["row_count"] = safe_row_count(cur, table)
result.update({"database": path.name, "mode": "READ_ONLY", "firebird_writes": 0, "xml_documents_available": any(r["xml_or_blob"] for r in result["tables"].values())})
return result
finally:
conn.close()
def main() -> None:
_load_env()
paths = [ROOT / "TESTESOFICIAIS.FDB"] + sorted((ROOT / "Banco").glob("*.FDB"))
output = {"artifact": "ERP_INBOUND_FISCAL_EVIDENCE_MAP_V1", "mode": "READ_ONLY", "databases": [], "firebird_writes": 0}
for path in paths:
try:
output["databases"].append(discover_database(path))
except Exception as exc: # record unavailable evidence without guessing
output["databases"].append({"database": path.name, "mode": "READ_ONLY", "status": "UNAVAILABLE", "error_type": type(exc).__name__, "firebird_writes": 0})
destination = ROOT / "docs" / "ERP_INBOUND_FISCAL_EVIDENCE_MAP_V1.json"
destination.write_text(json.dumps(output, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"artifact": output["artifact"], "databases": len(output["databases"]), "firebird_writes": 0}))
if __name__ == "__main__":
main()
scripts/generate_autonomous_completion.py
+106
View File
@@ -0,0 +1,106 @@
"""Measure required-field completeness from live read-only product masters."""
from __future__ import annotations
import json
import os
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from central.app.fiscal_completeness import evaluate_product, load_requiredness_matrix # noqa: E402
def _env() -> None:
path = ROOT / ".env"
if path.exists():
for line in path.read_text(encoding="utf-8").splitlines():
if "=" in line and not line.lstrip().startswith("#"):
key, _, value = line.partition("=")
os.environ.setdefault(key.strip(), value.strip())
def measure_database(path: Path, matrix: dict) -> dict:
import fdb
conn = fdb.connect(database=str(path), user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"], fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"), isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
try:
cur = conn.cursor()
cur.execute("SELECT TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME = 'C000025'")
available = {row[0] for row in cur.fetchall()}
fields = [field for field in matrix["fields"] if field in available]
cur.execute("SELECT UF, CRT FROM C000004")
company_rows = {(str(uf or '').strip().upper(), str(crt or '').strip()) for uf, crt in cur.fetchall()}
company = next(iter(company_rows)) if len(company_rows) == 1 else (None, None)
select_fields = ",".join(["CODIGO"] + fields)
cur.execute(f"SELECT {select_fields} FROM C000025")
complete = incomplete = 0
reason_counts: dict[str, int] = {}
required_valid = required_missing = 0
for row in cur.fetchall():
product = dict(zip(["CODIGO"] + fields, row))
context = {
"uf": company[0],
"crt": company[1],
"regime": "SIMPLES" if company[1] == "1" else "NORMAL" if company[1] in {"2", "3"} else None,
"operation": "SALE",
"contribuinte_ipi": "S" if str(product.get("IPI") or "").strip() else "N",
"has_numeric_tipi": False,
"has_supported_ex": False,
"product_matches_ex": False,
"has_official_cest": bool(str(product.get("CEST") or "").strip()),
"st_subject": str(product.get("SITUACAO_TRIBUTARIA") or "").strip() == "S",
"erp_consumes_entry_fields": False,
"icms_taxed": str(product.get("SITUACAO_TRIBUTARIA") or "").strip() == "T",
"origin_proven": False,
"erp_consumes_ipi_cst": False,
"piscofins_profile_proven": False,
"piscofins_rate_applies": False,
}
result = evaluate_product(product, context, matrix)
for field_result in result["fields"].values():
if field_result["status"] == "REQUIRED_AND_VALID":
required_valid += 1
elif field_result["status"] == "REQUIRED_MISSING":
required_missing += 1
if result["FISCALLY_COMPLETE"]:
complete += 1
else:
incomplete += 1
for field_result in result["fields"].values():
if field_result["status"] == "REQUIRED_MISSING":
reason_counts[field_result["field"]] = reason_counts.get(field_result["field"], 0) + 1
return {
"database": path.name,
"TOTAL_PRODUCTS": complete + incomplete,
"FISCALLY_COMPLETE": complete,
"FISCALLY_INCOMPLETE": incomplete,
"required_valid": required_valid,
"required_missing": required_missing,
"missing_reasons": reason_counts,
"company_context_observed": company[0] is not None,
"manual_context_not_answered": True,
"mode": "READ_ONLY",
}
finally:
conn.close()
def main() -> None:
_env()
matrix = load_requiredness_matrix(ROOT / "docs/FISCAL_REQUIREDNESS_MATRIX_V1.json")
paths = [ROOT / "TESTESOFICIAIS.FDB"] + sorted((ROOT / "Banco").glob("*.FDB"))
databases = []
for path in paths:
try:
databases.append(measure_database(path, matrix))
except Exception as exc:
databases.append({"database": path.name, "status": "UNAVAILABLE", "error_type": type(exc).__name__, "mode": "READ_ONLY"})
artifact = {"artifact": "AUTONOMOUS_FISCAL_COMPLETION_V1", "requiredness_matrix": "FISCAL_REQUIREDNESS_MATRIX_V1", "firebird_writes": 0, "external_api_calls": 0, "databases": databases}
(ROOT / "docs/AUTONOMOUS_FISCAL_COMPLETION_V1.json").write_text(json.dumps(artifact, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"artifact": artifact["artifact"], "databases": len(databases), "firebird_writes": 0}))
if __name__ == "__main__":
main()
tests/test_inbound_evidence_discovery.py
+35
View File
@@ -0,0 +1,35 @@
from scripts.discover_inbound_fiscal_evidence import classify_evidence_source, discover_from_rows, safe_row_count
def test_classifies_inbound_table_from_columns_not_name():
categories = classify_evidence_source("RANDOM_TABLE", {"CODPRODUTO", "CFOP", "NCM", "CHAVE"})
assert "ITEM" in categories
assert "FISCAL_ITEM" in categories
assert "KEY" in categories
def test_discovery_output_is_sanitized_and_capability_based():
result = discover_from_rows(
[
("RANDOM_TABLE", "CODPRODUTO", 37, 40),
("RANDOM_TABLE", "XML_PAYLOAD", 261, None),
("SUPPLIER_TABLE", "CNPJ", 37, 14),
],
[("RANDOM_TABLE", "CODPRODUTO", "PRODUCTS", "CODIGO")],
)
assert result["tables"]["RANDOM_TABLE"]["xml_or_blob"] is True
assert "CNPJ" not in result["tables"]["SUPPLIER_TABLE"]["columns"]
assert result["relationships"] == [{"from_table": "RANDOM_TABLE", "from_field": "CODPRODUTO", "to_table": "PRODUCTS", "to_field": "CODIGO"}]
def test_external_table_count_is_recorded_without_aborting_database():
class Cursor:
def execute(self, sql):
raise RuntimeError("external file is not allowed")
def fetchone(self):
return None
assert safe_row_count(Cursor(), "EXTERNAL_TABLE") == {"status": "ROW_COUNT_UNAVAILABLE", "error_type": "RuntimeError"}