feat: unified fiscal coverage - PIS/COFINS applicability + conditional RTC in one definition, one queue, one panel

- central/app/unified_fiscal_coverage.py: pure rules (PIS/COFINS NOT_APPLICABLE/REQUIRED/OPTIONAL/NEEDS_REVIEW, RTC only when the ERP switch is on)
- agent: read-only loader (unified_fiscal.py) replaces the legacy/expanded module; findings feed the single operational state, coverage and blocker queue
- hub: BLOCKED_FISCAL_DATA blockers (PIS/COFINS/RTC) share the queue; stale ones close on the next full pass; dashboard breakdown gains CEST/PIS/COFINS/RTC
- panel: legacy/expanded coverage block removed; discreet 'Reforma Tributaria: Ativa/Desativada'
- docs: unified_fiscal_completeness, pis_cofins_applicability, rtc_pis_cofins_model + official source manifests; read-only E2E script

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
gti
2026-09-24 16:39:53 -03:00
co-authored by Claude Sonnet 5
parent d897cceebb
commit b15dbea213
32 changed files with 3979 additions and 517 deletions
+21
View File
@@ -1,3 +1,24 @@
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parents[1] / "src"))
import pytest
@pytest.fixture(autouse=True)
def _no_real_firebird_for_unified_fiscal(monkeypatch, request):
"""The unified fiscal assessment opens its own READ ONLY connection; unit tests have no Firebird, so it is neutral here unless a test opts in."""
if request.node.get_closest_marker("real_unified_fiscal"):
return
from gti_fiscal_agent import firebird_reader, unified_fiscal
class _Conn:
def close(self):
pass
monkeypatch.setattr(firebird_reader, "_connect", lambda env: _Conn())
monkeypatch.setattr(unified_fiscal, "assess_from_connection", lambda conn, company, **kw: {"context": {"rtc_enabled": False, "rtc_supported": False, "rtc_catalog_valid": False, "products_evaluated": 0}, "findings": {}})
def pytest_configure(config):
config.addinivalue_line("markers", "real_unified_fiscal: use the real unified fiscal assessment (with a fake cursor) instead of the neutral stub")
-134
View File
@@ -1,134 +0,0 @@
"""RTC (cClassTrib / CST IBS-CBS) and PIS/COFINS: capabilities, validators, applicability, legacy vs expanded coverage. Read-only."""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parents[1] / "src"))
from gti_fiscal_agent import rtc_pis_cofins as r # noqa: E402
FULL_SCHEMA = {"C000025": {"CODIGO", "CCLASSTRIB", *r.PIS_COFINS_COLUMNS}, "RTC_CCLASSTRIB": {"CCLASSTRIB", "CSTIBSCBS"}, "RTC_CST": {"CSTIBSCBS"}}
CATALOG = {"classes": {"000001": "000", "200003": "200", "999999": None}, "cst": {"000", "200"}, "duplicates": 0}
def caps(schema=FULL_SCHEMA, param=0, catalog=CATALOG):
c = r.detect_rtc_capabilities(schema, param_value=param, catalog=catalog, physical_fk=True)
c["_PIS_COFINS_SCHEMA"] = all(col in schema.get("C000025", ()) for col in r.PIS_COFINS_COLUMNS[:6])
return c
def test_database_without_rtc_fails_closed_and_is_not_blocked():
c = caps({"C000025": {"CODIGO"}}, param=None, catalog=None)
assert not any(c[k] for k in ("HAS_PRODUCT_CCLASSTRIB", "HAS_RTC_CCLASSTRIB_TABLE", "HAS_RTC_CST_TABLE", "HAS_RTC_RELATIONSHIP", "RTC_CATALOG_VALID"))
assert r.rtc_applicability(c) == "NOT_SUPPORTED"
found = r.expanded_findings({"CODIGO": 1}, c, None)
assert found["blockers"] == [] and found["unknown"] == []
def test_partial_rtc_column_without_catalog_tables_is_not_a_relationship():
c = caps({"C000025": {"CODIGO", "CCLASSTRIB"}}, param=1, catalog=None)
assert c["HAS_PRODUCT_CCLASSTRIB"] and not c["HAS_RTC_RELATIONSHIP"] and not c["RTC_CATALOG_VALID"]
assert r.rtc_applicability(c) == "NOT_SUPPORTED"
def test_full_model_and_valid_catalog_capabilities():
good = {"classes": {"000001": "000"}, "cst": {"000"}, "duplicates": 0}
c = caps(catalog=good)
assert c["HAS_RTC_RELATIONSHIP"] and c["RTC_CATALOG_VALID"] and c["HAS_PHYSICAL_RTC_FK"] and c["RTC_ENABLED_BY_PARAM"] is False and c["LOCAL_CATALOG_ONLY"]
assert not caps(catalog=CATALOG)["RTC_CATALOG_VALID"] # a class pointing to a missing CST makes the catalog invalid
def test_cclasstrib_validator_states():
assert r.validate_cclasstrib(None, CATALOG) == "EMPTY" and r.validate_cclasstrib(" ", CATALOG) == "EMPTY"
assert r.validate_cclasstrib("12A456", CATALOG) == "INVALID_FORMAT" and r.validate_cclasstrib("12345", CATALOG) == "INVALID_FORMAT"
assert r.validate_cclasstrib("123456", CATALOG) == "NOT_IN_LOCAL_CATALOG"
assert r.validate_cclasstrib("200003", CATALOG) == "VALID_WITH_CST"
assert r.validate_cclasstrib("999999", CATALOG) == "CST_MISSING"
assert r.validate_cclasstrib("000001", {"classes": {"000001": None}, "cst": set()}) == "VALID_LOCAL"
assert r.validate_cclasstrib("000001", None) == "NOT_IN_LOCAL_CATALOG"
def test_pis_cofins_validators():
assert r.validate_pis_cofins_cst("") == "EMPTY" and r.validate_pis_cofins_cst(None) == "EMPTY"
assert r.validate_pis_cofins_cst("01") == "VALID" and r.validate_pis_cofins_cst("73") == "VALID"
assert r.validate_pis_cofins_cst("2") == "INVALID_FORMAT" and r.validate_pis_cofins_cst("10") == "INVALID_FORMAT"
assert r.validate_pis_cofins_rate("0,65") == "VALID" and r.validate_pis_cofins_rate("3") == "VALID" and r.validate_pis_cofins_rate("") == "EMPTY"
assert r.validate_pis_cofins_rate("abc") == "INVALID_RATE" and r.validate_pis_cofins_rate("150") == "INVALID_RATE"
ok = r.validate_pis_cofins({"PISCOFINS": "S", "INCIDENCIA_PISCOFINS": "MONOFASICA", "COD_SAIDA_PIS": "04", "COD_SAIDA_COFINS": "04"})
assert ok["FLAG_CONSISTENCY"] == "CONSISTENT"
bad = r.validate_pis_cofins({"PISCOFINS": "N", "INCIDENCIA_PISCOFINS": "NENHUMA", "COD_SAIDA_PIS": "01"})
assert bad["FLAG_CONSISTENCY"] == "INCONSISTENT"
def test_empty_pis_cofins_is_unknown_applicability_not_a_blocker():
found = r.expanded_findings({"CODIGO": 1, "PISCOFINS": "N", "INCIDENCIA_PISCOFINS": "NENHUMA"}, caps(), CATALOG)
assert found["blockers"] == [] and found["unknown"] == ["PIS_COFINS"] and found["rtc_applicability"] == "NOT_ENABLED_BY_ERP"
def test_populated_pis_cofins_is_applicable_and_valid_data_completes():
row = {"CODIGO": 1, "PISCOFINS": "S", "INCIDENCIA_PISCOFINS": "SUBST.TRIBUTARIA", "COD_ENTRADA_PIS": "73", "COD_SAIDA_PIS": "01", "ALIQ_PIS": "0,65",
"COD_ENTRADA_COFINS": "73", "COD_SAIDA_COFINS": "01", "ALIQ_COFINS": "3"}
found = r.expanded_findings(row, caps(), CATALOG)
assert found["blockers"] == [] and found["unknown"] == [] and found["pis_cofins_applicability"] == "APPLICABLE"
def test_invalid_values_where_applicable_create_specific_blockers():
row = {"CODIGO": 2, "PISCOFINS": "N", "INCIDENCIA_PISCOFINS": "NENHUMA", "COD_ENTRADA_PIS": "2", "COD_SAIDA_PIS": "2", "ALIQ_PIS": "2",
"COD_ENTRADA_COFINS": "2", "COD_SAIDA_COFINS": "2", "ALIQ_COFINS": "x"}
kinds = {k for k, _ in r.expanded_findings(row, caps(), CATALOG)["blockers"]}
assert {"PIS_CST_INVALID", "COFINS_CST_INVALID", "COFINS_RATE_INVALID", "PIS_COFINS_FLAG_INCONSISTENT"} <= kinds
def test_rtc_becomes_applicable_only_when_the_erp_enables_it():
on = caps(param=1)
assert r.rtc_applicability(on) == "APPLICABLE"
assert {k for k, _ in r.expanded_findings({"CODIGO": 1}, on, CATALOG)["blockers"]} == {"CCLASSTRIB_MISSING"}
assert {k for k, _ in r.expanded_findings({"CODIGO": 1, "CCLASSTRIB": "123456"}, on, CATALOG)["blockers"]} == {"CCLASSTRIB_NOT_IN_CATALOG"}
assert {k for k, _ in r.expanded_findings({"CODIGO": 1, "CCLASSTRIB": "999999"}, on, CATALOG)["blockers"]} == {"RTC_CST_INVALID"}
assert {k for k, _ in r.expanded_findings({"CODIGO": 1, "CCLASSTRIB": "12x"}, on, CATALOG)["blockers"]} == {"CCLASSTRIB_INVALID"}
assert not [k for k, _ in r.expanded_findings({"CODIGO": 1, "CCLASSTRIB": "200003"}, on, CATALOG)["blockers"] if k.startswith(("CCLASS", "RTC"))]
assert r.rtc_applicability(caps(param=None)) == "UNKNOWN_APPLICABILITY"
def test_legacy_and_expanded_coverage_are_reported_side_by_side_and_close():
products = [{"CODIGO": 1}, {"CODIGO": 2}, {"CODIGO": 3},
{"CODIGO": 4, "PISCOFINS": "S", "INCIDENCIA_PISCOFINS": "MONOFASICA", "COD_SAIDA_PIS": "01", "COD_SAIDA_COFINS": "01", "COD_ENTRADA_PIS": "73", "COD_ENTRADA_COFINS": "73"},
{"CODIGO": 5, "COD_SAIDA_PIS": "2", "PISCOFINS": "S"}]
states = {"1": "COMPLETE", "2": "BLOCKED_MISSING_EVIDENCE", "3": "FAILED_TECHNICAL", "4": "COMPLETE", "5": "CORRECTED"}
out = r.expanded_scope_summary(products, states, caps(), CATALOG)
assert out["legacy_scope"] == {"complete": 3, "safe_blocked": 1, "failed": 1}
# 1 unknown (no PIS/COFINS evidence), 1 complete (#4 valid), blocked = legacy-blocked #2 + #5 (invalid CST), failed #3
assert out["expanded_scope"] == {"complete": 1, "safe_blocked": 2, "failed": 1, "unknown_applicability": 1}
assert out["invariant_pass"] and out["blockers_by_type"].get("PIS_CST_INVALID") == 1 and out["local_catalog_only"]
class FakeCursor:
def __init__(self, tables, rows):
self.tables, self.rows, self.result = tables, rows, []
def execute(self, sql):
assert sql.lstrip().upper().startswith("SELECT") # read-only by construction
if "RDB$RELATION_FIELDS" in sql:
self.result = [(t, c) for t, cols in self.tables.items() for c in cols]
elif "USAR_REFORMA_TRIBUTARIA" in sql:
self.result = [(0,)]
elif "RDB$RELATION_CONSTRAINTS" in sql:
self.result = [(1,)]
elif "FROM RTC_CCLASSTRIB" in sql:
self.result = [("000001", "000")]
elif "FROM RTC_CST" in sql:
self.result = [("000",)]
elif "FROM C000025" in sql:
self.result = self.rows
def fetchall(self):
return self.result
def fetchone(self):
return self.result[0] if self.result else None
def test_loader_reads_schema_catalog_and_param_with_select_only():
cur = FakeCursor({"C000025": ["CODIGO", "CCLASSTRIB"], "RTC_CCLASSTRIB": ["CCLASSTRIB", "CSTIBSCBS"], "RTC_CST": ["CSTIBSCBS"], "PARAMETROS": ["USAR_REFORMA_TRIBUTARIA"]}, [(1, None)])
schema, catalog, extras = r.load_rtc_context(cur)
assert catalog == {"classes": {"000001": "000"}, "cst": {"000"}, "duplicates": 0} and extras == {"param_value": 0, "physical_fk": True}
assert r.read_products_rtc_pis_cofins(cur, schema) == [{"CODIGO": 1, "CCLASSTRIB": None}]
+110
View File
@@ -0,0 +1,110 @@
import re
from datetime import date
import pytest
from gti_fiscal_agent import unified_fiscal as uf
pytestmark = pytest.mark.real_unified_fiscal
SCHEMA_ROWS = ([("C000025", c) for c in ("CODIGO", "CCLASSTRIB", "COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS", "PISCOFINS", "INCIDENCIA_PISCOFINS")]
+ [("RTC_CCLASSTRIB", c) for c in ("CCLASSTRIB", "CSTIBSCBS", "INICIO_VIGENCIA", "FIM_VIGENCIA")] + [("RTC_CST", "CSTIBSCBS")] + [("PARAMETROS", "USAR_REFORMA_TRIBUTARIA")])
PRODUCT_COLS = ("CODIGO", "CCLASSTRIB", "COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS", "PISCOFINS", "INCIDENCIA_PISCOFINS")
PRODUCTS = [("1", None, None, None, None, None, None, None, "N", "NENHUMA"), ("2", "200003", None, "04", None, None, "04", None, "S", "MONOFÁSICA"), ("3", None, "2", "2", "2", "2", "2", "2", "N", "NENHUMA")]
CLASSES = [("000001", "000", date(2025, 5, 5), None), ("200003", "200", date(2025, 5, 5), None)]
class FakeCursor:
"""READ ONLY fake: answers only SELECTs and records every statement so tests can prove nothing else ran."""
def __init__(self, param=0, schema=SCHEMA_ROWS, products=PRODUCTS, classes=CLASSES, cst=(("000",), ("200",))):
self.param, self.schema, self.products, self.classes, self.cst, self.statements, self._rows = param, schema, products, classes, cst, [], []
def execute(self, sql, *args):
self.statements.append(sql)
text = " ".join(sql.split()).upper()
assert text.startswith("SELECT"), f"non-SELECT statement: {sql}"
if "FROM RDB$RELATION_FIELDS" in text:
self._rows = list(self.schema)
elif "FROM PARAMETROS" in text:
self._rows = [(self.param,)]
elif "RDB$RELATION_CONSTRAINTS" in text:
self._rows = [(1,)]
elif "FROM RTC_CCLASSTRIB" in text:
self._rows = list(self.classes)
elif "FROM RTC_CST" in text:
self._rows = list(self.cst)
elif "FROM C000025" in text:
self._rows = list(self.products)
else:
raise AssertionError(sql)
def fetchall(self):
return self._rows
def fetchone(self):
return self._rows[0] if self._rows else None
class FakeConn:
def __init__(self, **kw):
self.cur = FakeCursor(**kw)
def cursor(self):
return self.cur
def kinds(found):
return sorted({b["type"] for b in found["blockers"]})
def test_rtc_off_in_the_erp_is_out_of_scope_and_never_blocks():
conn = FakeConn(param=0)
out = uf.assess_from_connection(conn, {"crt": "1"}, today=date(2026, 9, 24))
assert out["context"]["rtc_enabled"] is False and out["context"]["rtc_supported"] is True and out["context"]["rtc_catalog_valid"] is True
assert not [b for f in out["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
assert kinds(out["findings"]["1"]) == [] and kinds(out["findings"]["2"]) == [] and kinds(out["findings"]["3"]) == ["COFINS_CST_INVALID", "PIS_CST_INVALID"]
assert all(s.strip().upper().startswith("SELECT") for s in conn.cur.statements) # read-only: SELECT statements only
def test_rtc_on_in_the_erp_brings_cclasstrib_into_the_same_findings():
out = uf.assess_from_connection(FakeConn(param=1), {"crt": "1"}, today=date(2026, 9, 24))
assert out["context"]["rtc_enabled"] is True
assert kinds(out["findings"]["1"]) == ["CCLASSTRIB_MISSING"] and kinds(out["findings"]["2"]) == []
assert kinds(out["findings"]["3"]) == ["CCLASSTRIB_MISSING", "COFINS_CST_INVALID", "PIS_CST_INVALID"] # one queue, one verdict per product
def test_simulation_flag_never_touches_the_erp_parameter():
conn = FakeConn(param=0)
on = uf.assess_from_connection(conn, {"crt": "1"}, rtc_override=True)
assert on["context"]["rtc_enabled"] is True and kinds(on["findings"]["1"]) == ["CCLASSTRIB_MISSING"]
assert on["context"]["rtc_param"] == 0 and not any("UPDATE" in s.upper() for s in conn.cur.statements)
def test_old_schema_without_rtc_columns_reports_unsupported_and_still_evaluates_pis_cofins():
old = [("C000025", c) for c in ("CODIGO", "COD_SAIDA_PIS", "PISCOFINS", "INCIDENCIA_PISCOFINS")]
rows = [("1", None, "N", "NENHUMA"), ("2", "9", "N", "NENHUMA")]
out = uf.assess_from_connection(FakeConn(schema=old, products=rows, classes=[], cst=()), {"crt": "3"})
assert out["context"]["rtc_supported"] is False and out["context"]["rtc_enabled"] is False
assert kinds(out["findings"]["1"]) == [] and kinds(out["findings"]["2"]) == ["PIS_CST_INVALID"]
@pytest.mark.parametrize("value,expected", [(None, False), (0, False), ("0", False), (1, True), ("1", True), ("S", True), ("N", False)])
def test_parameter_values_are_read_fail_closed(value, expected):
caps = uf.detect_rtc_capabilities({"C000025": {"CCLASSTRIB"}, "RTC_CCLASSTRIB": {"CCLASSTRIB", "CSTIBSCBS"}, "RTC_CST": {"CSTIBSCBS"}}, param_value=value)
assert caps["RTC_ENABLED_BY_PARAM"] is expected
def test_catalog_with_orphan_or_duplicate_is_invalid_and_flagged_only_when_rtc_is_on():
bad = FakeConn(param=1, classes=CLASSES + [("200003", "200", date(2025, 5, 5), None)]) # duplicate class
on = uf.assess_from_connection(bad, {"crt": "1"})
assert on["context"]["rtc_catalog_valid"] is False and kinds(on["findings"]["2"]) == ["RTC_CATALOG_INVALID"]
off = uf.assess_from_connection(FakeConn(param=0, classes=CLASSES + [("200003", "200", date(2025, 5, 5), None)]), {"crt": "1"})
assert not [b for f in off["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
def test_input_fingerprint_changes_with_the_fiscal_values_and_the_rtc_switch():
a = uf.assess_from_connection(FakeConn(param=0), {"crt": "1"})["findings"]["1"]["input_fingerprint"]
b = uf.assess_from_connection(FakeConn(param=1), {"crt": "1"})["findings"]["1"]["input_fingerprint"]
changed = [("1", None, None, "99", None, None, None, None, "N", "NENHUMA")] + PRODUCTS[1:]
c = uf.assess_from_connection(FakeConn(param=0, products=changed), {"crt": "1"})["findings"]["1"]["input_fingerprint"]
assert len({a, b, c}) == 3 and re.fullmatch(r"[0-9a-f]{64}", a)
+39
View File
@@ -146,3 +146,42 @@ def test_heartbeats_keep_flowing_while_a_long_job_runs(monkeypatch, tmp_path):
monkeypatch.setattr(service, "execute_job", lambda *a, **k: (time.sleep(0.4), ({"ok": 1}, None))[1])
service.run(_config(agent_id="2", poll_interval_seconds=0, state_db=str(tmp_path / "a.db")), {"AGENT_ID": "2"}, stopper=stopper)
assert beats.count("BUSY") >= 3 # heartbeats during the job, not only before it
def _findings(**by_code):
out = {}
for code, kinds in by_code.items():
out[code] = {"blockers": [{"type": t, "category": c, "reason": "R"} for t, c in kinds], "applicability": {"PIS_COFINS": "REQUIRED", "RTC": "NOT_APPLICABLE"}, "input_fingerprint": "fp" + code, "needs_review": False}
return out
def test_unified_fiscal_blockers_change_the_single_coverage_and_the_single_queue(env_setup, monkeypatch):
from gti_fiscal_agent import unified_fiscal
findings = _findings(**{"2": [("PIS_CST_INVALID", "PIS"), ("COFINS_CST_INVALID", "COFINS")]})
monkeypatch.setattr(unified_fiscal, "assess_from_connection", lambda conn, company, **kw: {"context": {"rtc_enabled": False, "rtc_supported": True, "rtc_catalog_valid": True}, "findings": findings})
hub, state = env_setup
result, error = run_job("INITIAL_SYNC", [product("1"), product("2"), product("3", ncm="")], state, monkeypatch)
assert error is None
op = result["operational"]
assert op["TOTAL_PRODUCTS"] == 3 and op["COMPLETE_PRODUCTS"] == 1 and op["SAFE_BLOCKED"] == 2 and op["BLOCKED_FISCAL_DATA"] == 1 # one number, one definition
assert op["COMPLETE_PRODUCTS"] + op["SAFE_BLOCKED"] + op["FAILED_TECHNICAL"] + op["UNCLASSIFIED"] == 3
queue = sorted((b["product_code"], b["field"], b["reason_code"]) for b in result["blocked_products"])
assert queue == [("2", "COFINS", "COFINS_CST_INVALID"), ("2", "PIS", "PIS_CST_INVALID"), ("3", "NCM", "INVALID_NCM")] # ONE queue: NCM and fiscal blockers together
assert result["resolved_products"] == ["1"] # a fiscal blocker keeps the product out of resolved
assert result["fiscal_context"]["rtc_enabled"] is False and result["fiscal_context"]["evaluated_all"] is True
assert "expanded_scope" not in result
def test_rtc_context_is_reported_but_blockers_only_exist_when_the_evaluator_produced_them(env_setup, monkeypatch):
from gti_fiscal_agent import unified_fiscal
monkeypatch.setattr(unified_fiscal, "assess_from_connection", lambda conn, company, **kw: {"context": {"rtc_enabled": True, "rtc_supported": True, "rtc_catalog_valid": True}, "findings": _findings(**{"1": [("CCLASSTRIB_MISSING", "RTC")]})})
hub, state = env_setup
result, _ = run_job("REPROCESS_ALL", [product("1"), product("2")], state, monkeypatch)
assert [b["reason_code"] for b in result["blocked_products"]] == ["CCLASSTRIB_MISSING"] and result["blocked_products"][0]["field"] == "RTC"
assert result["fiscal_context"]["rtc_enabled"] is True and result["operational"]["BLOCKED_FISCAL_DATA"] == 1
def test_partial_reprocess_does_not_claim_a_full_fiscal_evaluation(env_setup, monkeypatch):
hub, state = env_setup
result, _ = run_job("REPROCESS_PRODUCTS", [product("1")], state, monkeypatch)
assert result["fiscal_context"] is None and result["operational"] is None