docs: read-only fiscal/RTC/PIS-COFINS discovery of TESTESOFICIAIS.FDB
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
gti
Claude Sonnet 5
@@ -0,0 +1,168 @@
|
||||
"""READ-ONLY discovery of fiscal/RTC/PIS/COFINS fields in TESTESOFICIAIS.FDB (and only that database).
|
||||
|
||||
python scripts/discover_testesoficiais_fiscal.py [--json out.json]
|
||||
|
||||
Credentials come from the process environment (FB_USER / FB_PASSWORD, optional FIREBIRD_CLIENT_LIBRARY) or the
|
||||
local ignored .env - never from the command line, never printed. The connection uses READ COMMITTED READ ONLY
|
||||
isolation and every statement goes through a guard that only lets SELECT reach the driver.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DATABASE = ROOT / "TESTESOFICIAIS.FDB" # fixed on purpose: this script audits no other database
|
||||
|
||||
KEYWORDS = ("TRIB", "IBS", "CBS", "PIS", "COFINS", "CRED", "REFORMA", "RTC", "CLASS")
|
||||
FISCAL_HINTS = ("TRIB", "CST", "CFOP", "NCM", "PIS", "COF", "IBS", "CBS", "CRED", "ALIQ", "ICMS", "IPI", "CEST", "CSOSN", "CLASS", "ORIG", "REFORMA", "RTC", "SIT_T")
|
||||
_TYPES = {7: "SMALLINT", 8: "INTEGER", 16: "BIGINT", 10: "FLOAT", 27: "DOUBLE", 12: "DATE", 13: "TIME", 35: "TIMESTAMP", 14: "CHAR", 37: "VARCHAR", 261: "BLOB"}
|
||||
|
||||
|
||||
class ReadOnlyCursor:
|
||||
"""Only SELECT statements are ever handed to the driver; anything else is counted and refused."""
|
||||
write_attempts = 0
|
||||
write_successes = 0
|
||||
|
||||
def __init__(self, cursor):
|
||||
self._cursor = cursor
|
||||
|
||||
def execute(self, sql, params=()):
|
||||
if not re.match(r"^\s*select\b", sql, re.IGNORECASE):
|
||||
ReadOnlyCursor.write_attempts += 1
|
||||
raise RuntimeError("non-SELECT statement refused")
|
||||
return self._cursor.execute(sql, params) if params else self._cursor.execute(sql)
|
||||
|
||||
def fetchall(self):
|
||||
return self._cursor.fetchall()
|
||||
|
||||
def fetchone(self):
|
||||
return self._cursor.fetchone()
|
||||
|
||||
|
||||
def load_env():
|
||||
env_file = ROOT / ".env"
|
||||
if env_file.exists():
|
||||
for line in env_file.read_text(encoding="utf-8").splitlines():
|
||||
if "=" in line and not line.startswith("#"):
|
||||
key, _, value = line.partition("=")
|
||||
os.environ.setdefault(key.strip(), value.strip())
|
||||
|
||||
|
||||
def classify(table, column):
|
||||
"""Name-based first pass only. It is never taken as proof of meaning: the report lists evidence and a human/agent decides."""
|
||||
name = column.upper()
|
||||
if re.search(r"CCLASSTRIB|CLASS_?TRIB|CLASSIFICACAO_TRIBUTARIA", name):
|
||||
return "CONFIRMED_MATCH", "cClassTrib"
|
||||
if re.search(r"CST_?(IBS|CBS)|(IBS|CBS)_?CST|CSTIBSCBS|CST_IBSCBS", name):
|
||||
return "CONFIRMED_MATCH", "CST IBS/CBS"
|
||||
if re.search(r"(^|_)(IBS|CBS)($|_)|IBS|CBS", name) and not re.search(r"CBSA", name):
|
||||
return "CONFIRMED_MATCH", "IBS/CBS"
|
||||
if re.search(r"CCREDPRES|CRED_?PRES", name):
|
||||
return "CONFIRMED_MATCH", "crédito presumido"
|
||||
if re.search(r"(^|_)PIS($|_)|_PIS|PIS_|COD_(ENTRADA|SAIDA)_PIS", name):
|
||||
return "CONFIRMED_MATCH", "PIS"
|
||||
if re.search(r"COFINS", name):
|
||||
return "CONFIRMED_MATCH", "COFINS"
|
||||
if re.search(r"REFORMA|(^|_)RTC($|_)", name):
|
||||
return "POSSIBLE_MATCH", "reforma/RTC (nome)"
|
||||
if re.search(r"TRIB", name):
|
||||
return "POSSIBLE_MATCH", "tributação (nome genérico)"
|
||||
if re.search(r"CRED", name):
|
||||
return "POSSIBLE_MATCH", "crédito (nome genérico)"
|
||||
if re.search(r"CLASS", name):
|
||||
return "POSSIBLE_MATCH", "classificação (nome genérico)"
|
||||
return "UNRELATED", ""
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--json", default=None)
|
||||
args = parser.parse_args()
|
||||
load_env()
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
import fdb
|
||||
|
||||
conn = fdb.connect(database=str(DATABASE), user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"],
|
||||
fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"), isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
|
||||
cur = ReadOnlyCursor(conn.cursor())
|
||||
result = {"database": DATABASE.name, "mode": "READ_ONLY", "firebird_server_version": getattr(conn, "server_version", None),
|
||||
"ods": f"{getattr(conn, 'ods_version', '?')}.{getattr(conn, 'ods_minor_version', '?')}"}
|
||||
cur.execute("""SELECT TRIM(rf.RDB$RELATION_NAME), TRIM(rf.RDB$FIELD_NAME), f.RDB$FIELD_TYPE, COALESCE(f.RDB$CHARACTER_LENGTH, f.RDB$FIELD_LENGTH),
|
||||
f.RDB$FIELD_SCALE, rf.RDB$NULL_FLAG, TRIM(rf.RDB$FIELD_SOURCE), rf.RDB$DEFAULT_SOURCE, f.RDB$DEFAULT_SOURCE, rf.RDB$FIELD_POSITION
|
||||
FROM RDB$RELATION_FIELDS rf JOIN RDB$FIELDS f ON f.RDB$FIELD_NAME = rf.RDB$FIELD_SOURCE
|
||||
JOIN RDB$RELATIONS r ON r.RDB$RELATION_NAME = rf.RDB$RELATION_NAME
|
||||
WHERE COALESCE(r.RDB$SYSTEM_FLAG, 0) = 0 AND r.RDB$VIEW_BLR IS NULL""")
|
||||
columns = []
|
||||
for table, column, ftype, length, scale, null_flag, source, rf_default, f_default, position in cur.fetchall():
|
||||
default = rf_default or f_default
|
||||
default = str(default.read() if hasattr(default, "read") else default).replace("DEFAULT", "").strip() if default else None
|
||||
columns.append({"table": table, "column": column, "type": _TYPES.get(ftype, f"type{ftype}") + (f"(scale {scale})" if scale else ""), "length": length,
|
||||
"nullable": not bool(null_flag), "default": default, "domain": None if source.startswith("RDB$") else source, "position": position})
|
||||
result["tables"] = len({c["table"] for c in columns})
|
||||
result["total_columns"] = len(columns)
|
||||
|
||||
c25 = [c for c in columns if c["table"] == "C000025"]
|
||||
result["c000025_columns"] = [c["column"] for c in sorted(c25, key=lambda c: c["position"])]
|
||||
result["c000025_fiscal_columns"] = [c for c in sorted(c25, key=lambda c: c["position"]) if any(h in c["column"].upper() for h in FISCAL_HINTS)]
|
||||
|
||||
matches = []
|
||||
for column in columns:
|
||||
verdict, concept = classify(column["table"], column["column"])
|
||||
if verdict != "UNRELATED":
|
||||
matches.append({**column, "classification": verdict, "concept": concept})
|
||||
result["matches"] = matches
|
||||
|
||||
def stats(table, column):
|
||||
cur.execute(f'SELECT COUNT(*), COUNT("{column}"), COUNT(DISTINCT "{column}") FROM "{table}"')
|
||||
total, non_null, distinct = cur.fetchone()
|
||||
info = {"rows": total, "non_null": non_null, "distinct": distinct}
|
||||
cur.execute(f'SELECT FIRST 6 "{column}", COUNT(*) FROM "{table}" WHERE "{column}" IS NOT NULL GROUP BY "{column}" ORDER BY 2 DESC')
|
||||
info["top_values"] = [[str(v).strip()[:24] if v is not None else None, n] for v, n in cur.fetchall()]
|
||||
return info
|
||||
|
||||
for match in matches:
|
||||
if match["type"].startswith(("BLOB",)):
|
||||
continue
|
||||
try:
|
||||
match["usage"] = stats(match["table"], match["column"])
|
||||
except Exception as error: # noqa: BLE001 - reported, never fatal
|
||||
match["usage"] = {"error": type(error).__name__}
|
||||
for column in result["c000025_fiscal_columns"]:
|
||||
column["classification"], column["concept"] = classify("C000025", column["column"])
|
||||
try:
|
||||
column["usage"] = stats("C000025", column["column"])
|
||||
cur.execute(f"""SELECT COUNT(*) FROM "C000025" WHERE TRIM(COALESCE(CAST("{column["column"]}" AS VARCHAR(40)), '')) NOT IN ('', '0', '0.00', '0,00')""")
|
||||
column["usage"]["meaningful_non_empty"] = cur.fetchone()[0]
|
||||
except Exception as error: # noqa: BLE001
|
||||
column["usage"] = {"error": type(error).__name__}
|
||||
# tables that are entirely about the reform (name based), with their full column list and a few rows' worth of values
|
||||
rtc_tables = sorted({c["table"] for c in columns if re.search(r"RTC|IBS|CBS|REFORMA|CLASSTRIB", c["table"].upper())})
|
||||
result["rtc_tables"] = {}
|
||||
for table in rtc_tables:
|
||||
cols = [c for c in columns if c["table"] == table]
|
||||
cur.execute(f'SELECT COUNT(*) FROM "{table}"')
|
||||
result["rtc_tables"][table] = {"rows": cur.fetchone()[0], "columns": [(c["column"], c["type"], c["length"]) for c in sorted(cols, key=lambda c: c["position"])]}
|
||||
if table == "RTC_CST":
|
||||
cur.execute('SELECT FIRST 20 * FROM "RTC_CST"')
|
||||
result["rtc_tables"][table]["sample"] = [[str(v).strip()[:60] for v in row] for row in cur.fetchall()]
|
||||
cur.execute('SELECT USAR_REFORMA_TRIBUTARIA FROM PARAMETROS')
|
||||
result["parametros_usar_reforma_tributaria"] = [str(v[0]) for v in cur.fetchall()]
|
||||
cur.execute('SELECT COUNT(*) FROM C000025')
|
||||
result["c000025_rows"] = cur.fetchone()[0]
|
||||
# how many products carry a CCLASSTRIB that exists in the RTC_CCLASSTRIB catalog (orphan check, no inference)
|
||||
cur.execute("""SELECT COUNT(*) FROM C000025 p WHERE TRIM(COALESCE(p.CCLASSTRIB, '')) <> '' AND EXISTS (SELECT 1 FROM RTC_CCLASSTRIB c WHERE c.CCLASSTRIB = p.CCLASSTRIB)""")
|
||||
result["c000025_cclasstrib_in_catalog"] = cur.fetchone()[0]
|
||||
result["write_attempts"] = ReadOnlyCursor.write_attempts
|
||||
result["write_successes"] = ReadOnlyCursor.write_successes
|
||||
conn.close()
|
||||
text = json.dumps(result, ensure_ascii=False, indent=1, default=str)
|
||||
if args.json:
|
||||
Path(args.json).write_text(text, encoding="utf-8")
|
||||
print(f"tables={result['tables']} columns={result['total_columns']} matches={len(matches)} writes_attempted={result['write_attempts']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user