feat: read-only RTC + PIS/COFINS assessment (capabilities, validators, applicability, legacy vs expanded coverage) with panel labels and docs
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
gti
Claude Sonnet 5
@@ -0,0 +1,277 @@
|
||||
"""READ-ONLY RTC (cClassTrib / CST IBS-CBS) and PIS/COFINS assessment of SGV products.
|
||||
|
||||
Everything here is observation: capability detection, catalog integrity, deterministic validators, applicability and an
|
||||
EXPANDED coverage view that sits next to (never replaces) the legacy scope coverage. Nothing writes to Firebird, nothing
|
||||
fills a value, and no fiscal rule is invented: where local data cannot prove that a field applies, the answer is
|
||||
UNKNOWN_APPLICABILITY, not a blocker.
|
||||
|
||||
Vocabulary kept apart on purpose: SCHEMA_SUPPORT (the column exists) / DATA_PRESENT (it has a value) /
|
||||
APPLICABILITY (the field applies to this product) / MANDATORYNESS (a missing value is a defect). Only the last one
|
||||
may block, and only when applicability is proven from local, reliable data.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
LOCAL_CATALOG_ONLY = True # the RTC catalog used is the one stored in the client's own SGV database; no official dataset is integrated here
|
||||
|
||||
PRODUCT_TABLE = "C000025"
|
||||
RTC_CCLASSTRIB_TABLE = "RTC_CCLASSTRIB"
|
||||
RTC_CST_TABLE = "RTC_CST"
|
||||
PIS_COFINS_COLUMNS = ("COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS", "PISCOFINS", "INCIDENCIA_PISCOFINS")
|
||||
PRODUCT_COLUMNS = ("CODIGO", "CCLASSTRIB") + PIS_COFINS_COLUMNS
|
||||
|
||||
# Official CST PIS/COFINS codes (2 digits). Used for FORMAT/membership only, never to pick a value.
|
||||
OFFICIAL_PIS_COFINS_CST = frozenset([f"{n:02d}" for n in (*range(1, 10), 49, *range(50, 57), *range(60, 68), *range(70, 76), 98, 99)])
|
||||
|
||||
# ---------------------------------------------------------------- capabilities -------------------------------------------------
|
||||
|
||||
|
||||
def detect_rtc_capabilities(schema: dict, *, param_value=None, catalog: dict | None = None, physical_fk: bool | None = None) -> dict:
|
||||
"""schema: {TABLE: {COLUMN, ...}}. Fail-closed: anything not proven is False. Old databases without RTC simply report False."""
|
||||
product_cols = set(schema.get(PRODUCT_TABLE, ()))
|
||||
class_cols = set(schema.get(RTC_CCLASSTRIB_TABLE, ()))
|
||||
cst_cols = set(schema.get(RTC_CST_TABLE, ()))
|
||||
has_product = "CCLASSTRIB" in product_cols
|
||||
has_class = {"CCLASSTRIB", "CSTIBSCBS"} <= class_cols
|
||||
has_cst = "CSTIBSCBS" in cst_cols
|
||||
has_relationship = bool(has_product and has_class and has_cst) # column-level relationship: product -> class -> cst
|
||||
caps = {
|
||||
"HAS_PRODUCT_CCLASSTRIB": has_product,
|
||||
"HAS_RTC_CCLASSTRIB_TABLE": has_class,
|
||||
"HAS_RTC_CST_TABLE": has_cst,
|
||||
"HAS_RTC_RELATIONSHIP": has_relationship,
|
||||
"HAS_PHYSICAL_RTC_FK": bool(physical_fk),
|
||||
"RTC_ENABLED_BY_PARAM": None if param_value is None else str(param_value).strip() in {"1", "S", "T", "TRUE"},
|
||||
"RTC_CATALOG_VALID": False,
|
||||
"LOCAL_CATALOG_ONLY": LOCAL_CATALOG_ONLY,
|
||||
}
|
||||
if catalog is not None and has_class and has_cst:
|
||||
caps["RTC_CATALOG_VALID"] = catalog_integrity(catalog)["valid"]
|
||||
return caps
|
||||
|
||||
|
||||
def catalog_integrity(catalog: dict) -> dict:
|
||||
"""catalog: {"classes": {cClassTrib: cst_or_None}, "cst": {code, ...}, "duplicates": int}. Pure."""
|
||||
classes, cst = catalog.get("classes", {}), set(catalog.get("cst", ()))
|
||||
bad_format = [c for c in classes if not re.fullmatch(r"\d{6}", str(c))]
|
||||
bad_cst_format = [c for c in cst if not re.fullmatch(r"\d{3}", str(c))]
|
||||
orphans = [c for c, s in classes.items() if s is None or str(s) not in cst]
|
||||
return {"classes": len(classes), "cst": len(cst), "orphan_classes": len(orphans), "bad_class_format": len(bad_format),
|
||||
"bad_cst_format": len(bad_cst_format), "duplicates": int(catalog.get("duplicates", 0)),
|
||||
"valid": bool(classes and cst and not orphans and not bad_format and not bad_cst_format and not catalog.get("duplicates"))}
|
||||
|
||||
# ---------------------------------------------------------------- validators ---------------------------------------------------
|
||||
|
||||
|
||||
def _text(value) -> str:
|
||||
return "" if value is None else str(value).strip()
|
||||
|
||||
|
||||
def validate_cclasstrib(value, catalog: dict | None) -> str:
|
||||
"""EMPTY | INVALID_FORMAT | NOT_IN_LOCAL_CATALOG | VALID_LOCAL | CST_MISSING | VALID_WITH_CST (deterministic, no inference)."""
|
||||
code = _text(value)
|
||||
if not code:
|
||||
return "EMPTY"
|
||||
if not re.fullmatch(r"\d{6}", code):
|
||||
return "INVALID_FORMAT"
|
||||
if not catalog or code not in catalog.get("classes", {}):
|
||||
return "NOT_IN_LOCAL_CATALOG"
|
||||
if not catalog.get("cst"):
|
||||
return "VALID_LOCAL" # code exists, but there is no CST table to verify its CST
|
||||
cst = catalog["classes"][code]
|
||||
return "VALID_WITH_CST" if cst is not None and str(cst) in catalog["cst"] else "CST_MISSING"
|
||||
|
||||
|
||||
def validate_pis_cofins_cst(value) -> str:
|
||||
code = _text(value)
|
||||
if not code:
|
||||
return "EMPTY"
|
||||
return "VALID" if code in OFFICIAL_PIS_COFINS_CST else "INVALID_FORMAT"
|
||||
|
||||
|
||||
def validate_pis_cofins_rate(value) -> str:
|
||||
text = _text(value)
|
||||
if not text:
|
||||
return "EMPTY"
|
||||
if not re.fullmatch(r"\d{1,3}([.,]\d{1,4})?", text):
|
||||
return "INVALID_RATE"
|
||||
return "VALID" if float(text.replace(",", ".")) <= 100 else "INVALID_RATE"
|
||||
|
||||
|
||||
def validate_pis_cofins(product: dict) -> dict:
|
||||
"""Field-level validation for one product row. Flag consistency compares the ERP's own flag/incidence with its own codes."""
|
||||
out = {}
|
||||
for tax in ("PIS", "COFINS"):
|
||||
out[f"{tax}_ENTRY_CST"] = validate_pis_cofins_cst(product.get(f"COD_ENTRADA_{tax}"))
|
||||
out[f"{tax}_EXIT_CST"] = validate_pis_cofins_cst(product.get(f"COD_SAIDA_{tax}"))
|
||||
out[f"{tax}_RATE"] = validate_pis_cofins_rate(product.get(f"ALIQ_{tax}"))
|
||||
flag = _text(product.get("PISCOFINS")).upper()
|
||||
incidence = _text(product.get("INCIDENCIA_PISCOFINS")).upper()
|
||||
has_codes = any(_text(product.get(c)) for c in PIS_COFINS_COLUMNS[:6])
|
||||
special = incidence not in ("", "NENHUMA")
|
||||
inconsistent = (flag == "N" and (special or has_codes)) or (flag == "S" and incidence in ("", "NENHUMA"))
|
||||
out["FLAG_CONSISTENCY"] = "INCONSISTENT" if inconsistent else ("EMPTY" if not flag else "CONSISTENT")
|
||||
return out
|
||||
|
||||
# ---------------------------------------------------------------- applicability -----------------------------------------------
|
||||
|
||||
|
||||
def rtc_applicability(caps: dict) -> str:
|
||||
"""APPLICABLE only when the ERP itself is switched to the reform (PARAMETROS.USAR_REFORMA_TRIBUTARIA) AND the model exists.
|
||||
NOT_ENABLED_BY_ERP when the ERP says it does not use RTC; NOT_SUPPORTED when the schema cannot hold it; else UNKNOWN."""
|
||||
if not (caps.get("HAS_PRODUCT_CCLASSTRIB") and caps.get("HAS_RTC_RELATIONSHIP")):
|
||||
return "NOT_SUPPORTED"
|
||||
enabled = caps.get("RTC_ENABLED_BY_PARAM")
|
||||
if enabled is None:
|
||||
return "UNKNOWN_APPLICABILITY"
|
||||
return "APPLICABLE" if enabled else "NOT_ENABLED_BY_ERP"
|
||||
|
||||
|
||||
def pis_cofins_applicability(product: dict, schema_ok: bool) -> str:
|
||||
"""Product-level PIS/COFINS defaults are treated as applicable only when the product itself carries evidence of them
|
||||
(a code, or a special incidence/flag). A product with no data and flag N / NENHUMA gives NO proof either way (documents
|
||||
take their CST from elsewhere), so it is UNKNOWN_APPLICABILITY - not complete and not blocked."""
|
||||
if not schema_ok:
|
||||
return "NOT_SUPPORTED"
|
||||
evidence = any(_text(product.get(c)) for c in PIS_COFINS_COLUMNS[:6]) or _text(product.get("PISCOFINS")).upper() == "S" \
|
||||
or _text(product.get("INCIDENCIA_PISCOFINS")).upper() not in ("", "NENHUMA")
|
||||
return "APPLICABLE" if evidence else "UNKNOWN_APPLICABILITY"
|
||||
|
||||
# ---------------------------------------------------------------- blockers + expanded coverage ------------------------------
|
||||
|
||||
|
||||
def expanded_findings(product: dict, caps: dict, catalog: dict | None) -> dict:
|
||||
"""{"blockers": [(TYPE, reason)], "unknown": [area], "notes": [...]}: specific blockers only where applicability is proven."""
|
||||
blockers, unknown = [], []
|
||||
rtc = rtc_applicability(caps)
|
||||
if rtc == "APPLICABLE":
|
||||
state = validate_cclasstrib(product.get("CCLASSTRIB"), catalog)
|
||||
if state == "EMPTY":
|
||||
blockers.append(("CCLASSTRIB_MISSING", "EMPTY"))
|
||||
elif state == "INVALID_FORMAT":
|
||||
blockers.append(("CCLASSTRIB_INVALID", state))
|
||||
elif state == "NOT_IN_LOCAL_CATALOG":
|
||||
blockers.append(("CCLASSTRIB_NOT_IN_CATALOG", state))
|
||||
elif state == "CST_MISSING":
|
||||
blockers.append(("RTC_CST_INVALID", state))
|
||||
elif rtc == "UNKNOWN_APPLICABILITY":
|
||||
unknown.append("RTC")
|
||||
schema_ok = caps.get("_PIS_COFINS_SCHEMA", False)
|
||||
pis_cofins = pis_cofins_applicability(product, schema_ok)
|
||||
if pis_cofins == "APPLICABLE":
|
||||
checks = validate_pis_cofins(product)
|
||||
for tax in ("PIS", "COFINS"):
|
||||
for slot, label in (("ENTRY_CST", "ENTRY"), ("EXIT_CST", "EXIT")):
|
||||
result = checks[f"{tax}_{slot}"]
|
||||
if result == "INVALID_FORMAT":
|
||||
blockers.append((f"{tax}_CST_INVALID", f"{label}:{result}"))
|
||||
if checks[f"{tax}_RATE"] == "INVALID_RATE":
|
||||
blockers.append((f"{tax}_RATE_INVALID", "INVALID_RATE"))
|
||||
exit_state = checks[f"{tax}_EXIT_CST"]
|
||||
if exit_state == "EMPTY" and checks[f"{tax}_RATE"] != "EMPTY":
|
||||
blockers.append((f"{tax}_CST_MISSING", "EXIT:EMPTY_WITH_RATE"))
|
||||
if checks["FLAG_CONSISTENCY"] == "INCONSISTENT":
|
||||
blockers.append(("PIS_COFINS_FLAG_INCONSISTENT", "FLAG_VS_DATA"))
|
||||
elif pis_cofins == "UNKNOWN_APPLICABILITY":
|
||||
unknown.append("PIS_COFINS")
|
||||
return {"blockers": blockers, "unknown": unknown, "rtc_applicability": rtc, "pis_cofins_applicability": pis_cofins}
|
||||
|
||||
|
||||
def expanded_scope_summary(products: list[dict], legacy_states: dict, caps: dict, catalog: dict | None) -> dict:
|
||||
"""Legacy coverage is reported unchanged (LEGACY_SCOPE_COVERAGE); EXPANDED_FISCAL_SCOPE_COVERAGE adds RTC + PIS/COFINS.
|
||||
legacy_states: {product_code: state} from classify_product_state (COMPLETE/CORRECTED/BLOCKED_*/FAILED_TECHNICAL)."""
|
||||
legacy = {"complete": 0, "safe_blocked": 0, "failed": 0}
|
||||
expanded = {"complete": 0, "safe_blocked": 0, "failed": 0, "unknown_applicability": 0}
|
||||
blockers: dict[str, int] = {}
|
||||
blocked_rows = []
|
||||
caps = dict(caps)
|
||||
for product in products:
|
||||
code = str(product.get("CODIGO") or "")
|
||||
state = legacy_states.get(code, "COMPLETE")
|
||||
if state == "FAILED_TECHNICAL":
|
||||
legacy["failed"] += 1
|
||||
expanded["failed"] += 1
|
||||
continue
|
||||
if state.startswith("BLOCKED_"):
|
||||
legacy["safe_blocked"] += 1
|
||||
expanded["safe_blocked"] += 1 # blocked in the legacy scope stays blocked: the expanded scope only ADDS conditions
|
||||
continue
|
||||
legacy["complete"] += 1
|
||||
found = expanded_findings(product, caps, catalog)
|
||||
if found["blockers"]:
|
||||
expanded["safe_blocked"] += 1
|
||||
for kind, reason in found["blockers"]:
|
||||
blockers[kind] = blockers.get(kind, 0) + 1
|
||||
if len(blocked_rows) < 200:
|
||||
blocked_rows.append({"product_code": code, "type": kind, "reason": reason})
|
||||
elif found["unknown"]:
|
||||
expanded["unknown_applicability"] += 1
|
||||
else:
|
||||
expanded["complete"] += 1
|
||||
total = len(products)
|
||||
return {"scope_version": "expanded-v1", "local_catalog_only": LOCAL_CATALOG_ONLY, "total_products": total,
|
||||
"legacy_scope": legacy, "expanded_scope": expanded, "blockers_by_type": dict(sorted(blockers.items())),
|
||||
"blocker_samples": blocked_rows, "invariant_pass": sum(expanded.values()) == total and sum(legacy.values()) == total,
|
||||
"capabilities": {k: v for k, v in caps.items() if not k.startswith("_")}}
|
||||
|
||||
# ---------------------------------------------------------------- read-only loader -------------------------------------------
|
||||
|
||||
|
||||
def load_rtc_context(cursor) -> tuple[dict, dict | None, dict]:
|
||||
"""Read schema + catalog + ERP parameter through an already open READ ONLY cursor (SELECT only).
|
||||
Returns (schema, catalog|None, extras). Never raises for a database that lacks RTC."""
|
||||
schema: dict[str, set] = {}
|
||||
cursor.execute("SELECT TRIM(rf.RDB$RELATION_NAME), TRIM(rf.RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS rf WHERE rf.RDB$RELATION_NAME IN ('C000025','RTC_CCLASSTRIB','RTC_CST','PARAMETROS')")
|
||||
for table, column in cursor.fetchall():
|
||||
schema.setdefault(table, set()).add(column)
|
||||
extras: dict = {"param_value": None, "physical_fk": False}
|
||||
if "USAR_REFORMA_TRIBUTARIA" in schema.get("PARAMETROS", ()):
|
||||
try:
|
||||
cursor.execute("SELECT FIRST 1 USAR_REFORMA_TRIBUTARIA FROM PARAMETROS")
|
||||
row = cursor.fetchone()
|
||||
extras["param_value"] = None if row is None or row[0] is None else row[0]
|
||||
except Exception: # noqa: BLE001 - unreadable parameter => applicability stays unknown
|
||||
extras["param_value"] = None
|
||||
try:
|
||||
cursor.execute("SELECT COUNT(*) FROM RDB$RELATION_CONSTRAINTS WHERE RDB$RELATION_NAME = 'C000025' AND RDB$CONSTRAINT_TYPE = 'FOREIGN KEY' AND RDB$CONSTRAINT_NAME CONTAINING 'CCLASSTRIB'")
|
||||
extras["physical_fk"] = cursor.fetchone()[0] > 0
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
catalog = None
|
||||
if {"CCLASSTRIB", "CSTIBSCBS"} <= schema.get(RTC_CCLASSTRIB_TABLE, set()) and "CSTIBSCBS" in schema.get(RTC_CST_TABLE, set()):
|
||||
cursor.execute("SELECT TRIM(CCLASSTRIB), TRIM(CSTIBSCBS) FROM RTC_CCLASSTRIB")
|
||||
rows = cursor.fetchall()
|
||||
classes = {c: (s or None) for c, s in rows}
|
||||
cursor.execute("SELECT TRIM(CSTIBSCBS) FROM RTC_CST")
|
||||
cst = {r[0] for r in cursor.fetchall()}
|
||||
catalog = {"classes": classes, "cst": cst, "duplicates": len(rows) - len(classes)}
|
||||
return schema, catalog, extras
|
||||
|
||||
|
||||
def read_products_rtc_pis_cofins(cursor, schema: dict) -> list[dict]:
|
||||
"""SELECT of only the columns that exist (old schemas keep working)."""
|
||||
have = schema.get(PRODUCT_TABLE, set())
|
||||
columns = [c for c in PRODUCT_COLUMNS if c in have]
|
||||
if "CODIGO" not in columns:
|
||||
return []
|
||||
cursor.execute(f"SELECT {', '.join(columns)} FROM {PRODUCT_TABLE} ORDER BY CODIGO")
|
||||
return [dict(zip(columns, row)) for row in cursor.fetchall()]
|
||||
|
||||
|
||||
def assess_from_connection(conn, legacy_states: dict) -> dict:
|
||||
"""One read-only pass: capabilities -> catalog integrity -> expanded scope. Any failure degrades to an explicit unavailable
|
||||
marker; it never raises into the job and never touches the database beyond SELECT."""
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
schema, catalog, extras = load_rtc_context(cursor)
|
||||
caps = detect_rtc_capabilities(schema, param_value=extras["param_value"], catalog=catalog, physical_fk=extras["physical_fk"])
|
||||
caps["_PIS_COFINS_SCHEMA"] = all(c in schema.get(PRODUCT_TABLE, ()) for c in PIS_COFINS_COLUMNS[:6])
|
||||
products = read_products_rtc_pis_cofins(cursor, schema)
|
||||
summary = expanded_scope_summary(products, legacy_states, caps, catalog)
|
||||
summary["catalog_integrity"] = catalog_integrity(catalog) if catalog else None
|
||||
summary["capabilities"]["PIS_COFINS_SCHEMA_VALID"] = caps["_PIS_COFINS_SCHEMA"]
|
||||
summary["data_present"] = {c: sum(1 for p in products if _text(p.get(c))) for c in ("CCLASSTRIB",) + PIS_COFINS_COLUMNS if any(c in p for p in products[:1])}
|
||||
return summary
|
||||
except Exception as error: # noqa: BLE001
|
||||
return {"scope_version": "expanded-v1", "unavailable": type(error).__name__}
|
||||
Reference in New Issue
Block a user