feat: add autonomous fiscal completion v1

2026-09-22 13:08:38 -03:00
parent eada0df37e
commit e3ac68c70e
28 changed files with 44630 additions and 0 deletions
@@ -1,5 +1,19 @@
 # AI Project State

+## Autonomous fiscal completion V1 (2026-09-22)
+
+`AUTONOMOUS_FISCAL_COMPLETION_V1_MEASURED` — requiredness matrix and pure
+`FiscalCompletenessEvaluator` are active; read-only inbound discovery covered
+all five allowlisted databases and produced `ERP_INBOUND_FISCAL_EVIDENCE_MAP_V1`
+plus supplier observation metrics. Origin, role, ST/withheld, and entry scope
+remain evidence-gated. No human decisions were answered, no NCM conflicts were
+resolved, no external product provider was queried, and no Firebird write was
+executed. The central `ai_usage_ledger` and `external_product_cache` schema
+objects are defined by migration 0015, with offline budget and idempotency
+guards. Measured matrix completeness is 34,872/35,066 products; AI residual is
+unchanged at 511. `AI_WRITE_ALLOWED=false`, `FIREBIRD_AI_WRITES=0`, retrieval
+V2 remains active, and retrieval V3 remains unpromoted.
+
 ## Current status (real LLM evaluation V1 — 2026-09-21)

 `BLOCKED_EXTERNAL_AI_CREDENTIAL` (implementation complete). The entire real-LLM
@@ -0,0 +1,86 @@
+{
+  "artifact": "AUTONOMOUS_FISCAL_COMPLETION_V1",
+  "requiredness_matrix": "FISCAL_REQUIREDNESS_MATRIX_V1",
+  "firebird_writes": 0,
+  "external_api_calls": 0,
+  "databases": [
+    {
+      "database": "TESTESOFICIAIS.FDB",
+      "TOTAL_PRODUCTS": 521,
+      "FISCALLY_COMPLETE": 495,
+      "FISCALLY_INCOMPLETE": 26,
+      "required_valid": 521,
+      "required_missing": 46,
+      "missing_reasons": {
+        "CFOP": 21,
+        "CSOSN": 24,
+        "SITUACAO_TRIBUTARIA": 1
+      },
+      "company_context_observed": true,
+      "manual_context_not_answered": true,
+      "mode": "READ_ONLY"
+    },
+    {
+      "database": "CONSTRUCAO.FDB",
+      "TOTAL_PRODUCTS": 3712,
+      "FISCALLY_COMPLETE": 3685,
+      "FISCALLY_INCOMPLETE": 27,
+      "required_valid": 3712,
+      "required_missing": 30,
+      "missing_reasons": {
+        "SITUACAO_TRIBUTARIA": 25,
+        "CFOP": 2,
+        "CSOSN": 3
+      },
+      "company_context_observed": true,
+      "manual_context_not_answered": true,
+      "mode": "READ_ONLY"
+    },
+    {
+      "database": "MMMODAS.FDB",
+      "TOTAL_PRODUCTS": 20997,
+      "FISCALLY_COMPLETE": 20889,
+      "FISCALLY_INCOMPLETE": 108,
+      "required_valid": 20997,
+      "required_missing": 108,
+      "missing_reasons": {
+        "CFOP": 55,
+        "ALIQUOTA": 50,
+        "SITUACAO_TRIBUTARIA": 3
+      },
+      "company_context_observed": true,
+      "manual_context_not_answered": true,
+      "mode": "READ_ONLY"
+    },
+    {
+      "database": "RINCAO.FDB",
+      "TOTAL_PRODUCTS": 894,
+      "FISCALLY_COMPLETE": 890,
+      "FISCALLY_INCOMPLETE": 4,
+      "required_valid": 894,
+      "required_missing": 4,
+      "missing_reasons": {
+        "SITUACAO_TRIBUTARIA": 1,
+        "ALIQUOTA": 3
+      },
+      "company_context_observed": false,
+      "manual_context_not_answered": true,
+      "mode": "READ_ONLY"
+    },
+    {
+      "database": "UNIAO.FDB",
+      "TOTAL_PRODUCTS": 8942,
+      "FISCALLY_COMPLETE": 8913,
+      "FISCALLY_INCOMPLETE": 29,
+      "required_valid": 8942,
+      "required_missing": 29,
+      "missing_reasons": {
+        "SITUACAO_TRIBUTARIA": 28,
+        "CFOP": 1
+      },
+      "company_context_observed": true,
+      "manual_context_not_answered": true,
+      "mode": "READ_ONLY"
+    }
+  ]
+}
@@ -0,0 +1,145 @@
+# Autonomous Fiscal Completion V1
+
+## Mission Result
+
+`STATUS=COMPLETED_OFFLINE_WITH_MEASURED_RESIDUALS`
+
+This mission ran without Z.ai, OpenAI, external provider queries, human-answer
+inference, or Firebird writes. Discovery used read-only transactions on the five
+explicitly allowlisted local copies.
+
+## Consolidated Report
+
+```text
+DATABASES=5
+TOTAL_PRODUCTS=35066
+
+REQUIREDNESS_FIELDS=23
+REQUIREDNESS_CONTEXTS=10
+
+INBOUND_TABLES_DISCOVERED=1690 (schema map; external table row counts recorded as unavailable)
+INBOUND_ITEMS_ANALYZED=118519
+XML_DOCUMENTS_AVAILABLE=true (BLOB/XML-capable tables detected; payloads not exported)
+
+ORIGIN_PENDING_BEFORE=35056
+ORIGIN_RESOLVED_FROM_INBOUND=0
+ORIGIN_CONFLICTS=0
+ORIGIN_PENDING_AFTER=35056
+
+PRODUCT_ROLE_PENDING_BEFORE=43
+PRODUCT_ROLE_RESOLVED_DETERMINISTICALLY=0 (new mission resolutions)
+PRODUCT_ROLE_PENDING_AFTER=43
+
+ST_PENDING_BEFORE=86
+ST_RESOLVED_FROM_INBOUND=0
+ST_PENDING_AFTER=86
+
+PIS_COFINS_PENDING_BEFORE=3 company profiles / product rules unavailable
+PIS_COFINS_PENDING_AFTER=3 company profiles / product rules unavailable
+
+IPI_CLASSIFICATION=OPTIONAL_REFERENCE for observed non-contributing companies; numeric TIPI remains reference data unless contribution/use is proven
+IPI_HUMAN_DECISION_REQUIRED=false for the reference-only classification; no STORE_IPI_PROFILE applied
+
+KB_PRODUCTS=30801
+KB_CROSS_TENANT_REUSE=251 valid GTIN cross-tenant matches observed; tenant policy was not copied
+
+EXTERNAL_PRODUCT_PROVIDERS_EVALUATED=5 candidate categories
+RECOMMENDED_EXTERNAL_PROVIDER=authorization_required (licensed GS1/commercial bulk source)
+RECOMMENDED_PROVIDER_COST=NOT_VERIFIED
+EXPECTED_PROVIDER_COVERAGE=NOT_VERIFIED
+EXPECTED_PROVIDER_AI_REDUCTION=NOT_VERIFIED
+
+AI_BUDGET_PER_RUN=0.54 USD default guard
+AI_BUDGET_DAILY=1.00 USD default guard
+AI_BUDGET_MONTHLY=10.00 USD default guard
+AI_USAGE_LEDGER_READY=true (model and migration 0015; no usage recorded)
+
+TOTAL_REQUIRED_FIELD_INSTANCES=35283
+REQUIRED_VALID=35066
+REQUIRED_MISSING=217
+
+FISCALLY_COMPLETE_PRODUCTS=34872
+FISCALLY_INCOMPLETE_PRODUCTS=194
+AUTONOMOUS_RATE=0.9944675754 (matrix-measured; human context remains incomplete)
+
+AI_READY_BEFORE=511
+AI_READY_AFTER=511
+
+MANUAL_COMPANY_DECISIONS_REMAINING=19
+MANUAL_GROUP_DECISIONS_REMAINING=738
+MANUAL_PRODUCT_FACTS_REMAINING=46036
+
+NEW_DETERMINISTIC_WRITES=0
+NEW_DETERMINISTIC_WRITES_VERIFIED=0
+SECOND_PASS_WRITES=0
+
+FULL_TESTS=tests 324 passed, 1 skipped; central/tests 219 passed, 1 warning
+FIREBIRD_AI_WRITES=0
+AI_WRITE_ALLOWED=false
+
+LOCAL_PRODUCT_COMPLETE=MEASURED_FOR_REQUIREDNESS_MATRIX; HUMAN_CONTEXT_INCOMPLETE
+READY_FOR_ZAI_GOLD_EVAL=true
+```
+
+## What The ERP Provided
+
+The read-only evidence map discovered 5 databases, 1,680 user tables/relations,
+foreign-key candidates, fiscal item/header/supplier/key categories, and
+XML/BLOB-capable structures. Populated inbound item evidence was concentrated
+in `C000088` and `SINTEGRA_ENTRADA_ITEM`; 118,519 inbound rows were analyzed.
+Supplier observations produced 284 strong non-origin aggregates and 406
+conflicts in MMMODAS, but no inbound origin field was available in the
+analyzed item shape. Consequently origin remained `ORIGIN_NEEDS_EVIDENCE`.
+
+`C000062` and `C000032` contain substantial historical fiscal activity, but
+their observed CFOP distributions and existing analysis do not prove that they
+are inbound supplier evidence. They were not promoted as such.
+
+## Requiredness And Completeness
+
+The evaluator distinguishes required, conditional, reference-only, not
+applicable, and ERP-not-used. Empty IPI, IAT, IPPT, PIS/COFINS, and other
+conditional/reference fields are not automatically counted as incomplete when
+their trigger is not proven. The 194 incomplete products are measured required
+field gaps under the current known sale/company context, primarily CFOP,
+CSOSN, ICMS treatment, and ICMS rate. This is not a claim that every company
+or product context is complete.
+
+## Writes And Safety
+
+No new deterministic candidate passed the complete evidence/provenance and
+write-guard contract without human context or unresolved conflict. Therefore
+the correct result was zero writes, not forced completion. The write audit
+records `NEW_DETERMINISTIC_WRITES=0`, `NEW_DETERMINISTIC_WRITES_VERIFIED=0`,
+and `SECOND_PASS_WRITES=0`.
+
+Current `NCM_CONFLICT=90` rows and all 73 historical
+`INSUFFICIENT_EVIDENCE` conflicts were preserved. No origin default, ST
+withholding, company decision, group decision, or product fact was invented.
+
+## External Data And Future AI
+
+The provider adapter accepts GTIN records with provider version, retrieval time,
+raw hash, normalized data, and cache/redistribution flags. The central schema
+now has the `ai_usage_ledger` and `external_product_cache` tables via migration
+0015. Budget checks reserve atomically before a future provider invocation and
+return `AI_BUDGET_EXHAUSTED` without sending a request. Identical future AI
+inputs have a stable cache key; no provider or AI call occurred in this mission.
+
+## Remaining Gaps
+
+- Human COMPANY/GROUP/PRODUCT facts remain unresolved.
+- Origin has no sufficiently strong inbound evidence for global promotion.
+- ST withholding is not proven merely by ST subject/CEST evidence.
+- PIS/COFINS NCM-linked Tabela 4.3.x rules are still unavailable locally.
+- External provider price, license, and coverage require operator authorization and source contract review.
+- The real Z.ai GOLD evaluation remains blocked by `EXTERNAL_AI_CREDENTIAL_OR_BALANCE`.
+
+## Verdict
+
+`VERDICT=MAXIMUM_MEASURED_OFFLINE_COMPLETION_WITHOUT_UNSAFE_INFERENCE`
+
+The system now has the requiredness contract, read-only inbound evidence map,
+conservative evidence aggregation, origin/role/ST/entry-scope gates, provider
+cache boundary, central budget schema, idempotency key, and measured autonomy
+artifacts. Remaining decisions are genuine evidence or authorization gaps.
@@ -0,0 +1,11 @@
+{
+  "artifact": "AUTONOMOUS_WRITE_AUDIT_V1",
+  "mode": "DRY_RUN_ONLY",
+  "firebird_writes": 0,
+  "databases": ["TESTESOFICIAIS", "CONSTRUCAO", "MMMODAS", "RINCAO", "UNIAO"],
+  "new_deterministic_writes": 0,
+  "new_deterministic_writes_verified": 0,
+  "second_pass_writes": 0,
+  "reason": "No new candidate passed the complete evidence, provenance, path, company, schema, type, compare-before-write, rowcount, and post-write gates without human context or unresolved conflict.",
+  "ai_write_allowed": false
+}
@@ -0,0 +1,33 @@
+# External Product Data Provider Evaluation V1
+
+## Scope
+
+This is a comparison only. No service was purchased, queried, or enabled.
+External data remains `OBSERVATION_ONLY` until official validation and product
+identity checks succeed.
+
+| Candidate | Type | GTIN lookup | NCM/CEST | Bulk/cache | Cost | License status |
+|---|---|---:|---:|---:|---|---|
+| GS1 Brasil / official GTIN services | Official/commercial | available subject to account | not guaranteed as fiscal authority | contract-dependent | NOT_VERIFIED | authorization required |
+| Licensed commercial catalog API | Commercial | usually available | vendor-dependent | contract-dependent | NOT_VERIFIED | license review required |
+| Licensed commercial bulk dataset | Commercial bulk | expected | vendor-dependent | potentially best for cache | NOT_VERIFIED | redistribution/cache review required |
+| Local bulk dataset | Local/offline | only if supplied | only if sourced/validated | yes | acquisition-dependent | must prove license |
+| Official fiscal datasets already in Hub | Official fiscal | no GTIN lookup | NCM/CEST available | yes | existing | permitted for fiscal validation |
+
+## Scenario Economics
+
+For 35k products, 70 clients, 100k products, and 500k products, a permanent
+licensed bulk dataset can save repeated lookups, but no price or query limit was
+verified locally. Therefore `initial_cost`, `recurring_cost`, and exact
+`queries_saved` remain `NOT_VERIFIED`; no purchase recommendation is made.
+
+## Recommendation
+
+`RECOMMENDED_SOURCE=authorization_required`
+
+The adapter boundary is implemented offline in
+`central/app/product_data_provider.py`. A future provider must supply provider
+version, raw hash, normalized data, cache permission, and redistribution terms.
+No commercial source is authoritative for fiscal fields without official
+validation. Expected AI reduction cannot be measured before an authorized
+dataset is available, so `EXPECTED_PROVIDER_AI_REDUCTION=NOT_VERIFIED`.
@@ -0,0 +1,48 @@
+{
+  "artifact": "FISCAL_REQUIREDNESS_MATRIX_V1",
+  "version": "requiredness-v1",
+  "source_contract": "docs/FISCAL_FIELD_CONTRACT_V1.json",
+  "classifications": [
+    "REQUIRED",
+    "CONDITIONAL",
+    "OPTIONAL_REFERENCE",
+    "NOT_APPLICABLE",
+    "ERP_NOT_USED"
+  ],
+  "contexts": [
+    "CRT/regime",
+    "UF",
+    "entry_or_sale",
+    "intra_or_interstate",
+    "consumer_final",
+    "resale_or_production_or_input_or_service",
+    "ST",
+    "monophase",
+    "IPI_contributor"
+  ],
+  "fields": {
+    "CLASSIFICACAO_FISCAL": {"classification": "REQUIRED", "when": "all_products", "validity": "8-digit active NCM", "source": "active NCM dataset", "human_decision_required": true},
+    "IPI": {"classification": "CONDITIONAL", "when": "IPI_contributor_and_numeric_TIPI_rate", "validity": "numeric TIPI rate", "source": "active TIPI dataset", "human_decision_required": false},
+    "EX_TIPI": {"classification": "CONDITIONAL", "when": "supported_EX_row_and_product_evidence", "validity": "active TIPI EX code", "source": "active TIPI dataset plus description evidence", "human_decision_required": true},
+    "CEST": {"classification": "CONDITIONAL", "when": "official_CEST_candidate_or_ST_context", "validity": "official CEST for NCM", "source": "active CEST dataset", "human_decision_required": true},
+    "SITUACAO_TRIBUTARIA": {"classification": "CONDITIONAL", "when": "sale_item_and_ICMS_context_applies", "validity": "declared T/I/N/S not contradicted", "source": "ERP plus ST evidence", "human_decision_required": true},
+    "CFOP": {"classification": "CONDITIONAL", "when": "sale_item", "validity": "official operation-compatible CFOP", "source": "CFOP dataset plus role/ST context", "human_decision_required": true},
+    "CFOP_ENTRADA": {"classification": "CONDITIONAL", "when": "purchase_default_is_used_by_ERP", "validity": "operation-specific 1xxx/2xxx CFOP", "source": "inbound evidence and ERP behavior", "human_decision_required": true},
+    "CST": {"classification": "CONDITIONAL", "when": "normal_regime_ICMS_context", "validity": "valid ICMS CST for operation", "source": "ERP and official CST dataset", "human_decision_required": true},
+    "CST_ENTRADA": {"classification": "CONDITIONAL", "when": "inbound_CST_is_consumed_by_ERP", "validity": "valid inbound CST", "source": "inbound document evidence", "human_decision_required": true},
+    "CSOSN": {"classification": "CONDITIONAL", "when": "Simples_Nacional_context", "validity": "valid CSOSN for operation", "source": "ERP and official CSOSN dataset", "human_decision_required": true},
+    "ALIQUOTA": {"classification": "CONDITIONAL", "when": "taxed_ICMS_operation", "validity": "rate proven by company/UF context", "source": "ERP and official rate evidence", "human_decision_required": true},
+    "COD_ORIGEM_MERCADORIA": {"classification": "CONDITIONAL", "when": "origin_is_proven_for_product_or_purchase", "validity": "0-8 with explicit evidence", "source": "inbound evidence or validated product fact", "human_decision_required": true},
+    "CST_IPI": {"classification": "CONDITIONAL", "when": "IPI_contributor_and_CST_consumed", "validity": "valid IPI CST", "source": "ERP and document evidence", "human_decision_required": true},
+    "COD_ENTRADA_PIS": {"classification": "CONDITIONAL", "when": "PIS_COFINS_profile_and_inbound_field_consumed", "validity": "valid PIS CST", "source": "SPED/ERP evidence", "human_decision_required": true},
+    "COD_SAIDA_PIS": {"classification": "CONDITIONAL", "when": "PIS_COFINS_profile_and_sale_field_consumed", "validity": "valid PIS CST", "source": "SPED/ERP evidence", "human_decision_required": true},
+    "ALIQ_PIS": {"classification": "CONDITIONAL", "when": "PIS_rate_applies", "validity": "proven PIS rate", "source": "SPED/official table/ERP evidence", "human_decision_required": true},
+    "COD_ENTRADA_COFINS": {"classification": "CONDITIONAL", "when": "PIS_COFINS_profile_and_inbound_field_consumed", "validity": "valid COFINS CST", "source": "SPED/ERP evidence", "human_decision_required": true},
+    "COD_SAIDA_COFINS": {"classification": "CONDITIONAL", "when": "PIS_COFINS_profile_and_sale_field_consumed", "validity": "valid COFINS CST", "source": "SPED/ERP evidence", "human_decision_required": true},
+    "ALIQ_COFINS": {"classification": "CONDITIONAL", "when": "COFINS_rate_applies", "validity": "proven COFINS rate", "source": "SPED/official table/ERP evidence", "human_decision_required": true},
+    "INCIDENCIA_PISCOFINS": {"classification": "CONDITIONAL", "when": "PIS_COFINS_profile_is_proven", "validity": "proven incidence", "source": "official monophase data or ERP evidence", "human_decision_required": true},
+    "PISCOFINS": {"classification": "CONDITIONAL", "when": "PIS_COFINS_profile_is_proven", "validity": "valid PIS/COFINS profile", "source": "accounting/ERP evidence", "human_decision_required": true},
+    "IAT": {"classification": "ERP_NOT_USED", "when": "no_proven_SGV_consumption", "validity": "not evaluated as operational input", "source": "ERP binding audit", "human_decision_required": false},
+    "IPPT": {"classification": "ERP_NOT_USED", "when": "no_proven_SGV_consumption", "validity": "not evaluated as operational input", "source": "ERP binding audit", "human_decision_required": false}
+  }
+}
@@ -0,0 +1,71 @@
+{
+  "artifact": "INBOUND_SUPPLIER_OBSERVATIONS_V1",
+  "mode": "READ_ONLY",
+  "firebird_writes": 0,
+  "databases": [
+    {
+      "database": "TESTESOFICIAIS.FDB",
+      "status": "ANALYZED",
+      "items_analyzed": 12,
+      "observation_count": 14,
+      "aggregate_count": 13,
+      "sources": [
+        "SUPPLIER_OBSERVATION"
+      ],
+      "strong_evidence_count": 0,
+      "conflict_count": 1,
+      "origin_field_available": false,
+      "origin_resolution": "ORIGIN_NEEDS_EVIDENCE"
+    },
+    {
+      "database": "CONSTRUCAO.FDB",
+      "status": "ANALYZED",
+      "items_analyzed": 0,
+      "observation_count": 0,
+      "aggregate_count": 0,
+      "sources": [],
+      "strong_evidence_count": 0,
+      "conflict_count": 0,
+      "origin_field_available": false,
+      "origin_resolution": "ORIGIN_NEEDS_EVIDENCE"
+    },
+    {
+      "database": "MMMODAS.FDB",
+      "status": "ANALYZED",
+      "items_analyzed": 117657,
+      "observation_count": 274597,
+      "aggregate_count": 30110,
+      "sources": [
+        "SUPPLIER_OBSERVATION"
+      ],
+      "strong_evidence_count": 284,
+      "conflict_count": 406,
+      "origin_field_available": false,
+      "origin_resolution": "ORIGIN_NEEDS_EVIDENCE"
+    },
+    {
+      "database": "RINCAO.FDB",
+      "status": "ANALYZED",
+      "items_analyzed": 0,
+      "observation_count": 0,
+      "aggregate_count": 0,
+      "sources": [],
+      "strong_evidence_count": 0,
+      "conflict_count": 0,
+      "origin_field_available": false,
+      "origin_resolution": "ORIGIN_NEEDS_EVIDENCE"
+    },
+    {
+      "database": "UNIAO.FDB",
+      "status": "ANALYZED",
+      "items_analyzed": 0,
+      "observation_count": 0,
+      "aggregate_count": 0,
+      "sources": [],
+      "strong_evidence_count": 0,
+      "conflict_count": 0,
+      "origin_field_available": false,
+      "origin_resolution": "ORIGIN_NEEDS_EVIDENCE"
+    }
+  ]
+}
@@ -0,0 +1,49 @@
+{
+  "artifact": "LOCAL_PRODUCT_HEALTH_V2",
+  "status": "PASS_WITH_MEASURED_RESIDUALS",
+  "databases": 5,
+  "products": 35066,
+  "fiscally_complete_products": 34872,
+  "fiscally_incomplete_products": 194,
+  "autonomous_rate": 0.9944675754,
+  "required_field_instances_valid": 35066,
+  "required_field_instances_missing": 217,
+  "incompleteness_reason_instances": {
+    "MISSING_COMPANY_FACT": 0,
+    "MISSING_PRODUCT_FACT": 0,
+    "MISSING_EXTERNAL_PRODUCT_DATA": 0,
+    "AI_SEMANTIC_CANDIDATE": 0,
+    "DATA_CONFLICT": 0,
+    "ERP_SCHEMA_LIMITATION": 0,
+    "REQUIRED_FIELD_MISSING_MEASURED": 217
+  },
+  "company_context_complete": false,
+  "manual_company_decisions_remaining": 19,
+  "manual_group_decisions_remaining": 738,
+  "manual_product_facts_remaining": 46036,
+  "kb_products": 30801,
+  "kb_cross_tenant_reuse": 251,
+  "inbound_items_analyzed": 118519,
+  "xml_documents_available": true,
+  "origin_pending_before": 35056,
+  "origin_resolved_from_inbound": 0,
+  "origin_conflicts": 0,
+  "origin_pending_after": 35056,
+  "ai_ready_before": 511,
+  "ai_ready_after": 511,
+  "firebird_writes": 0,
+  "semantic_writes": 0,
+  "external_api_calls": 0,
+  "ai_write_allowed": false,
+  "retrieval_active": "V2",
+  "retrieval_v3": "NOT_ACTIVATED_WITHOUT_HOLDOUT_GAIN",
+  "current_ncm_conflicts": 90,
+  "historical_conflicts": 73,
+  "historical_conflicts_resolved": 0,
+  "second_pass_writes": 0,
+  "notes": [
+    "Completeness is evaluated using the requiredness matrix, not 23-field population.",
+    "Inbound observations are supplier observations and do not authorize origin or ST_WITHHELD decisions.",
+    "No deterministic write passed the complete evidence and guard contract in this run."
+  ]
+}