feat: add autonomous fiscal completion v1

2026-09-22 13:08:38 -03:00
parent eada0df37e
commit e3ac68c70e
28 changed files with 44630 additions and 0 deletions
@@ -0,0 +1,82 @@
+"""Read-only extraction of populated supplier-item fiscal observations."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from central.app.evidence_aggregation import Observation, aggregate_observations  # noqa: E402
+
+
+def _hash(value: object) -> str:
+    return hashlib.sha256(str(value).encode("utf-8")).hexdigest()[:16]
+
+
+def summarize_rows(rows: list[tuple[object, ...]]) -> dict:
+    observations = []
+    for product, supplier, observed_at, ncm, cfop, cst in rows:
+        identity = _hash(product)
+        supplier_id = _hash(supplier)
+        document = _hash(f"{supplier}:{product}:{observed_at}")
+        for attribute, value in (("ncm", ncm), ("cfop", cfop), ("cst", cst)):
+            if value not in (None, ""):
+                observations.append(Observation(identity, attribute, str(value).strip(), supplier_id, document, str(observed_at), "SUPPLIER_OBSERVATION"))
+    aggregate = aggregate_observations(observations)
+    return {
+        "observation_count": len(observations),
+        "aggregate_count": len(aggregate),
+        "sources": ["SUPPLIER_OBSERVATION"] if observations else [],
+        "strong_evidence_count": sum(1 for row in aggregate.values() if row["strong_evidence"]),
+        "conflict_count": sum(row["conflict_count"] for row in aggregate.values()),
+    }
+
+
+def _env() -> None:
+    path = ROOT / ".env"
+    if path.exists():
+        for line in path.read_text(encoding="utf-8").splitlines():
+            if "=" in line and not line.lstrip().startswith("#"):
+                key, _, value = line.partition("=")
+                os.environ.setdefault(key.strip(), value.strip())
+
+
+def analyze_database(path: Path) -> dict:
+    import fdb
+
+    conn = fdb.connect(database=str(path), user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"], fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"), isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
+    try:
+        cur = conn.cursor()
+        cur.execute("SELECT TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME = 'C000088'")
+        fields = {row[0] for row in cur.fetchall()}
+        required = {"CODPRODUTO", "CODFORNECEDOR", "DATA", "CLASSIFICACAO_FISCAL", "CFOP", "CST"}
+        if not required <= fields:
+            return {"database": path.name, "status": "NO_COMPLETE_INBOUND_ITEM_SHAPE", "fields_available": sorted(fields)}
+        cur.execute("SELECT CODPRODUTO, CODFORNECEDOR, DATA, CLASSIFICACAO_FISCAL, CFOP, CST FROM C000088")
+        rows = cur.fetchall()
+        summary = summarize_rows(rows)
+        return {"database": path.name, "status": "ANALYZED", "items_analyzed": len(rows), **summary, "origin_field_available": False, "origin_resolution": "ORIGIN_NEEDS_EVIDENCE"}
+    finally:
+        conn.close()
+
+
+def main() -> None:
+    _env()
+    paths = [ROOT / "TESTESOFICIAIS.FDB"] + sorted((ROOT / "Banco").glob("*.FDB"))
+    result = {"artifact": "INBOUND_SUPPLIER_OBSERVATIONS_V1", "mode": "READ_ONLY", "firebird_writes": 0, "databases": []}
+    for path in paths:
+        try:
+            result["databases"].append(analyze_database(path))
+        except Exception as exc:
+            result["databases"].append({"database": path.name, "status": "UNAVAILABLE", "error_type": type(exc).__name__})
+    (ROOT / "docs/INBOUND_SUPPLIER_OBSERVATIONS_V1.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps({"artifact": result["artifact"], "databases": len(result["databases"]), "firebird_writes": 0}))
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,114 @@
+"""Discover inbound fiscal evidence from SGV databases using SELECT-only access."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from pathlib import Path
+from typing import Any, Iterable
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SENSITIVE = re.compile(r"CNPJ|CPF|XML|CHAVE|RAZAO|NOME|ENDERECO|FONE|EMAIL", re.I)
+TOKEN_GROUPS = {
+    "HEADER": {"NOTA", "NF", "DOCUMENTO", "CABECALHO", "EMISSAO", "SERIE"},
+    "ITEM": {"ITEM", "CODPRODUTO", "CODIGO", "QTD", "QUANTIDADE", "VALOR"},
+    "SUPPLIER": {"FORNEC", "EMITENTE", "CNPJ", "CPF", "RAZAO"},
+    "KEY": {"CHAVE", "NFE", "NFCE", "ACCESS", "PROTOCOLO"},
+    "FISCAL_ITEM": {"NCM", "CFOP", "CST", "CSOSN", "CEST", "ORIG", "ICMS", "PIS", "COFINS", "IPI", "MVA"},
+}
+
+
+def classify_evidence_source(table: str, columns: Iterable[str]) -> set[str]:
+    text = f"{table} {' '.join(columns)}".upper()
+    result = {category for category, tokens in TOKEN_GROUPS.items() if any(token in text for token in tokens)}
+    return result or {"UNKNOWN"}
+
+
+def discover_from_rows(column_rows: Iterable[tuple[Any, ...]], relationship_rows: Iterable[tuple[Any, ...]]) -> dict[str, Any]:
+    tables: dict[str, dict[str, Any]] = {}
+    for table, field, field_type, field_length in column_rows:
+        table = str(table).strip()
+        field = str(field).strip()
+        record = tables.setdefault(table, {"columns": [], "categories": [], "xml_or_blob": False})
+        if not SENSITIVE.search(field):
+            record["columns"].append(field)
+        if field_type == 261 or re.search(r"XML|BLOB|PAYLOAD|DOCUMENT", field, re.I):
+            record["xml_or_blob"] = True
+    for table, record in tables.items():
+        categories = classify_evidence_source(table, record["columns"])
+        record["categories"] = sorted(categories)
+    relationships = []
+    for from_table, from_field, to_table, to_field in relationship_rows:
+        relationships.append({"from_table": str(from_table).strip(), "from_field": str(from_field).strip(), "to_table": str(to_table).strip(), "to_field": str(to_field).strip()})
+    return {"tables": tables, "relationships": relationships}
+
+
+def safe_row_count(cursor: Any, table: str) -> dict[str, Any] | int:
+    try:
+        cursor.execute(f'SELECT COUNT(*) FROM "{table}"')
+        return int(cursor.fetchone()[0])
+    except Exception as exc:  # external tables can be blocked by server policy
+        return {"status": "ROW_COUNT_UNAVAILABLE", "error_type": type(exc).__name__}
+
+
+def _load_env() -> None:
+    path = ROOT / ".env"
+    if path.exists():
+        for line in path.read_text(encoding="utf-8").splitlines():
+            if "=" in line and not line.lstrip().startswith("#"):
+                key, _, value = line.partition("=")
+                os.environ.setdefault(key.strip(), value.strip())
+
+
+def discover_database(path: Path) -> dict[str, Any]:
+    import fdb
+
+    conn = fdb.connect(
+        database=str(path),
+        user=os.environ["FB_USER"],
+        password=os.environ["FB_PASSWORD"],
+        fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
+        isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO,
+    )
+    try:
+        cur = conn.cursor()
+        cur.execute("""SELECT TRIM(rf.RDB$RELATION_NAME), TRIM(rf.RDB$FIELD_NAME), f.RDB$FIELD_TYPE, COALESCE(f.RDB$CHARACTER_LENGTH, f.RDB$FIELD_LENGTH)
+                       FROM RDB$RELATION_FIELDS rf JOIN RDB$FIELDS f ON f.RDB$FIELD_NAME = rf.RDB$FIELD_SOURCE
+                       JOIN RDB$RELATIONS r ON r.RDB$RELATION_NAME = rf.RDB$RELATION_NAME
+                       WHERE COALESCE(r.RDB$SYSTEM_FLAG, 0) = 0""")
+        columns = cur.fetchall()
+        cur.execute("""SELECT TRIM(rc.RDB$RELATION_NAME), TRIM(s.RDB$FIELD_NAME), TRIM(rc2.RDB$RELATION_NAME), TRIM(s2.RDB$FIELD_NAME)
+                       FROM RDB$RELATION_CONSTRAINTS rc
+                       JOIN RDB$INDEX_SEGMENTS s ON s.RDB$INDEX_NAME = rc.RDB$INDEX_NAME
+                       JOIN RDB$REF_CONSTRAINTS ref ON ref.RDB$CONSTRAINT_NAME = rc.RDB$CONSTRAINT_NAME
+                       JOIN RDB$RELATION_CONSTRAINTS rc2 ON rc2.RDB$CONSTRAINT_NAME = ref.RDB$CONST_NAME_UQ
+                       JOIN RDB$INDEX_SEGMENTS s2 ON s2.RDB$INDEX_NAME = rc2.RDB$INDEX_NAME
+                       WHERE rc.RDB$CONSTRAINT_TYPE = 'FOREIGN KEY'""")
+        relationships = cur.fetchall()
+        result = discover_from_rows(columns, relationships)
+        for table, record in result["tables"].items():
+            record["row_count"] = safe_row_count(cur, table)
+        result.update({"database": path.name, "mode": "READ_ONLY", "firebird_writes": 0, "xml_documents_available": any(r["xml_or_blob"] for r in result["tables"].values())})
+        return result
+    finally:
+        conn.close()
+
+
+def main() -> None:
+    _load_env()
+    paths = [ROOT / "TESTESOFICIAIS.FDB"] + sorted((ROOT / "Banco").glob("*.FDB"))
+    output = {"artifact": "ERP_INBOUND_FISCAL_EVIDENCE_MAP_V1", "mode": "READ_ONLY", "databases": [], "firebird_writes": 0}
+    for path in paths:
+        try:
+            output["databases"].append(discover_database(path))
+        except Exception as exc:  # record unavailable evidence without guessing
+            output["databases"].append({"database": path.name, "mode": "READ_ONLY", "status": "UNAVAILABLE", "error_type": type(exc).__name__, "firebird_writes": 0})
+    destination = ROOT / "docs" / "ERP_INBOUND_FISCAL_EVIDENCE_MAP_V1.json"
+    destination.write_text(json.dumps(output, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps({"artifact": output["artifact"], "databases": len(output["databases"]), "firebird_writes": 0}))
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,106 @@
+"""Measure required-field completeness from live read-only product masters."""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+from central.app.fiscal_completeness import evaluate_product, load_requiredness_matrix  # noqa: E402
+
+
+def _env() -> None:
+    path = ROOT / ".env"
+    if path.exists():
+        for line in path.read_text(encoding="utf-8").splitlines():
+            if "=" in line and not line.lstrip().startswith("#"):
+                key, _, value = line.partition("=")
+                os.environ.setdefault(key.strip(), value.strip())
+
+
+def measure_database(path: Path, matrix: dict) -> dict:
+    import fdb
+
+    conn = fdb.connect(database=str(path), user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"], fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"), isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
+    try:
+        cur = conn.cursor()
+        cur.execute("SELECT TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME = 'C000025'")
+        available = {row[0] for row in cur.fetchall()}
+        fields = [field for field in matrix["fields"] if field in available]
+        cur.execute("SELECT UF, CRT FROM C000004")
+        company_rows = {(str(uf or '').strip().upper(), str(crt or '').strip()) for uf, crt in cur.fetchall()}
+        company = next(iter(company_rows)) if len(company_rows) == 1 else (None, None)
+        select_fields = ",".join(["CODIGO"] + fields)
+        cur.execute(f"SELECT {select_fields} FROM C000025")
+        complete = incomplete = 0
+        reason_counts: dict[str, int] = {}
+        required_valid = required_missing = 0
+        for row in cur.fetchall():
+            product = dict(zip(["CODIGO"] + fields, row))
+            context = {
+                "uf": company[0],
+                "crt": company[1],
+                "regime": "SIMPLES" if company[1] == "1" else "NORMAL" if company[1] in {"2", "3"} else None,
+                "operation": "SALE",
+                "contribuinte_ipi": "S" if str(product.get("IPI") or "").strip() else "N",
+                "has_numeric_tipi": False,
+                "has_supported_ex": False,
+                "product_matches_ex": False,
+                "has_official_cest": bool(str(product.get("CEST") or "").strip()),
+                "st_subject": str(product.get("SITUACAO_TRIBUTARIA") or "").strip() == "S",
+                "erp_consumes_entry_fields": False,
+                "icms_taxed": str(product.get("SITUACAO_TRIBUTARIA") or "").strip() == "T",
+                "origin_proven": False,
+                "erp_consumes_ipi_cst": False,
+                "piscofins_profile_proven": False,
+                "piscofins_rate_applies": False,
+            }
+            result = evaluate_product(product, context, matrix)
+            for field_result in result["fields"].values():
+                if field_result["status"] == "REQUIRED_AND_VALID":
+                    required_valid += 1
+                elif field_result["status"] == "REQUIRED_MISSING":
+                    required_missing += 1
+            if result["FISCALLY_COMPLETE"]:
+                complete += 1
+            else:
+                incomplete += 1
+                for field_result in result["fields"].values():
+                    if field_result["status"] == "REQUIRED_MISSING":
+                        reason_counts[field_result["field"]] = reason_counts.get(field_result["field"], 0) + 1
+        return {
+            "database": path.name,
+            "TOTAL_PRODUCTS": complete + incomplete,
+            "FISCALLY_COMPLETE": complete,
+            "FISCALLY_INCOMPLETE": incomplete,
+            "required_valid": required_valid,
+            "required_missing": required_missing,
+            "missing_reasons": reason_counts,
+            "company_context_observed": company[0] is not None,
+            "manual_context_not_answered": True,
+            "mode": "READ_ONLY",
+        }
+    finally:
+        conn.close()
+
+
+def main() -> None:
+    _env()
+    matrix = load_requiredness_matrix(ROOT / "docs/FISCAL_REQUIREDNESS_MATRIX_V1.json")
+    paths = [ROOT / "TESTESOFICIAIS.FDB"] + sorted((ROOT / "Banco").glob("*.FDB"))
+    databases = []
+    for path in paths:
+        try:
+            databases.append(measure_database(path, matrix))
+        except Exception as exc:
+            databases.append({"database": path.name, "status": "UNAVAILABLE", "error_type": type(exc).__name__, "mode": "READ_ONLY"})
+    artifact = {"artifact": "AUTONOMOUS_FISCAL_COMPLETION_V1", "requiredness_matrix": "FISCAL_REQUIREDNESS_MATRIX_V1", "firebird_writes": 0, "external_api_calls": 0, "databases": databases}
+    (ROOT / "docs/AUTONOMOUS_FISCAL_COMPLETION_V1.json").write_text(json.dumps(artifact, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps({"artifact": artifact["artifact"], "databases": len(databases), "firebird_writes": 0}))
+
+
+if __name__ == "__main__":
+    main()