169 lines
9.1 KiB
Python
169 lines
9.1 KiB
Python
"""READ-ONLY discovery of fiscal/RTC/PIS/COFINS fields in TESTESOFICIAIS.FDB (and only that database).
|
|
|
|
python scripts/discover_testesoficiais_fiscal.py [--json out.json]
|
|
|
|
Credentials come from the process environment (FB_USER / FB_PASSWORD, optional FIREBIRD_CLIENT_LIBRARY) or the
|
|
local ignored .env - never from the command line, never printed. The connection uses READ COMMITTED READ ONLY
|
|
isolation and every statement goes through a guard that only lets SELECT reach the driver.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DATABASE = ROOT / "TESTESOFICIAIS.FDB" # fixed on purpose: this script audits no other database
|
|
|
|
KEYWORDS = ("TRIB", "IBS", "CBS", "PIS", "COFINS", "CRED", "REFORMA", "RTC", "CLASS")
|
|
FISCAL_HINTS = ("TRIB", "CST", "CFOP", "NCM", "PIS", "COF", "IBS", "CBS", "CRED", "ALIQ", "ICMS", "IPI", "CEST", "CSOSN", "CLASS", "ORIG", "REFORMA", "RTC", "SIT_T")
|
|
_TYPES = {7: "SMALLINT", 8: "INTEGER", 16: "BIGINT", 10: "FLOAT", 27: "DOUBLE", 12: "DATE", 13: "TIME", 35: "TIMESTAMP", 14: "CHAR", 37: "VARCHAR", 261: "BLOB"}
|
|
|
|
|
|
class ReadOnlyCursor:
|
|
"""Only SELECT statements are ever handed to the driver; anything else is counted and refused."""
|
|
write_attempts = 0
|
|
write_successes = 0
|
|
|
|
def __init__(self, cursor):
|
|
self._cursor = cursor
|
|
|
|
def execute(self, sql, params=()):
|
|
if not re.match(r"^\s*select\b", sql, re.IGNORECASE):
|
|
ReadOnlyCursor.write_attempts += 1
|
|
raise RuntimeError("non-SELECT statement refused")
|
|
return self._cursor.execute(sql, params) if params else self._cursor.execute(sql)
|
|
|
|
def fetchall(self):
|
|
return self._cursor.fetchall()
|
|
|
|
def fetchone(self):
|
|
return self._cursor.fetchone()
|
|
|
|
|
|
def load_env():
|
|
env_file = ROOT / ".env"
|
|
if env_file.exists():
|
|
for line in env_file.read_text(encoding="utf-8").splitlines():
|
|
if "=" in line and not line.startswith("#"):
|
|
key, _, value = line.partition("=")
|
|
os.environ.setdefault(key.strip(), value.strip())
|
|
|
|
|
|
def classify(table, column):
|
|
"""Name-based first pass only. It is never taken as proof of meaning: the report lists evidence and a human/agent decides."""
|
|
name = column.upper()
|
|
if re.search(r"CCLASSTRIB|CLASS_?TRIB|CLASSIFICACAO_TRIBUTARIA", name):
|
|
return "CONFIRMED_MATCH", "cClassTrib"
|
|
if re.search(r"CST_?(IBS|CBS)|(IBS|CBS)_?CST|CSTIBSCBS|CST_IBSCBS", name):
|
|
return "CONFIRMED_MATCH", "CST IBS/CBS"
|
|
if re.search(r"(^|_)(IBS|CBS)($|_)|IBS|CBS", name) and not re.search(r"CBSA", name):
|
|
return "CONFIRMED_MATCH", "IBS/CBS"
|
|
if re.search(r"CCREDPRES|CRED_?PRES", name):
|
|
return "CONFIRMED_MATCH", "crédito presumido"
|
|
if re.search(r"(^|_)PIS($|_)|_PIS|PIS_|COD_(ENTRADA|SAIDA)_PIS", name):
|
|
return "CONFIRMED_MATCH", "PIS"
|
|
if re.search(r"COFINS", name):
|
|
return "CONFIRMED_MATCH", "COFINS"
|
|
if re.search(r"REFORMA|(^|_)RTC($|_)", name):
|
|
return "POSSIBLE_MATCH", "reforma/RTC (nome)"
|
|
if re.search(r"TRIB", name):
|
|
return "POSSIBLE_MATCH", "tributação (nome genérico)"
|
|
if re.search(r"CRED", name):
|
|
return "POSSIBLE_MATCH", "crédito (nome genérico)"
|
|
if re.search(r"CLASS", name):
|
|
return "POSSIBLE_MATCH", "classificação (nome genérico)"
|
|
return "UNRELATED", ""
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--json", default=None)
|
|
args = parser.parse_args()
|
|
load_env()
|
|
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
|
import fdb
|
|
|
|
conn = fdb.connect(database=str(DATABASE), user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"],
|
|
fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"), isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
|
|
cur = ReadOnlyCursor(conn.cursor())
|
|
result = {"database": DATABASE.name, "mode": "READ_ONLY", "firebird_server_version": getattr(conn, "server_version", None),
|
|
"ods": f"{getattr(conn, 'ods_version', '?')}.{getattr(conn, 'ods_minor_version', '?')}"}
|
|
cur.execute("""SELECT TRIM(rf.RDB$RELATION_NAME), TRIM(rf.RDB$FIELD_NAME), f.RDB$FIELD_TYPE, COALESCE(f.RDB$CHARACTER_LENGTH, f.RDB$FIELD_LENGTH),
|
|
f.RDB$FIELD_SCALE, rf.RDB$NULL_FLAG, TRIM(rf.RDB$FIELD_SOURCE), rf.RDB$DEFAULT_SOURCE, f.RDB$DEFAULT_SOURCE, rf.RDB$FIELD_POSITION
|
|
FROM RDB$RELATION_FIELDS rf JOIN RDB$FIELDS f ON f.RDB$FIELD_NAME = rf.RDB$FIELD_SOURCE
|
|
JOIN RDB$RELATIONS r ON r.RDB$RELATION_NAME = rf.RDB$RELATION_NAME
|
|
WHERE COALESCE(r.RDB$SYSTEM_FLAG, 0) = 0 AND r.RDB$VIEW_BLR IS NULL""")
|
|
columns = []
|
|
for table, column, ftype, length, scale, null_flag, source, rf_default, f_default, position in cur.fetchall():
|
|
default = rf_default or f_default
|
|
default = str(default.read() if hasattr(default, "read") else default).replace("DEFAULT", "").strip() if default else None
|
|
columns.append({"table": table, "column": column, "type": _TYPES.get(ftype, f"type{ftype}") + (f"(scale {scale})" if scale else ""), "length": length,
|
|
"nullable": not bool(null_flag), "default": default, "domain": None if source.startswith("RDB$") else source, "position": position})
|
|
result["tables"] = len({c["table"] for c in columns})
|
|
result["total_columns"] = len(columns)
|
|
|
|
c25 = [c for c in columns if c["table"] == "C000025"]
|
|
result["c000025_columns"] = [c["column"] for c in sorted(c25, key=lambda c: c["position"])]
|
|
result["c000025_fiscal_columns"] = [c for c in sorted(c25, key=lambda c: c["position"]) if any(h in c["column"].upper() for h in FISCAL_HINTS)]
|
|
|
|
matches = []
|
|
for column in columns:
|
|
verdict, concept = classify(column["table"], column["column"])
|
|
if verdict != "UNRELATED":
|
|
matches.append({**column, "classification": verdict, "concept": concept})
|
|
result["matches"] = matches
|
|
|
|
def stats(table, column):
|
|
cur.execute(f'SELECT COUNT(*), COUNT("{column}"), COUNT(DISTINCT "{column}") FROM "{table}"')
|
|
total, non_null, distinct = cur.fetchone()
|
|
info = {"rows": total, "non_null": non_null, "distinct": distinct}
|
|
cur.execute(f'SELECT FIRST 6 "{column}", COUNT(*) FROM "{table}" WHERE "{column}" IS NOT NULL GROUP BY "{column}" ORDER BY 2 DESC')
|
|
info["top_values"] = [[str(v).strip()[:24] if v is not None else None, n] for v, n in cur.fetchall()]
|
|
return info
|
|
|
|
for match in matches:
|
|
if match["type"].startswith(("BLOB",)):
|
|
continue
|
|
try:
|
|
match["usage"] = stats(match["table"], match["column"])
|
|
except Exception as error: # noqa: BLE001 - reported, never fatal
|
|
match["usage"] = {"error": type(error).__name__}
|
|
for column in result["c000025_fiscal_columns"]:
|
|
column["classification"], column["concept"] = classify("C000025", column["column"])
|
|
try:
|
|
column["usage"] = stats("C000025", column["column"])
|
|
cur.execute(f"""SELECT COUNT(*) FROM "C000025" WHERE TRIM(COALESCE(CAST("{column["column"]}" AS VARCHAR(40)), '')) NOT IN ('', '0', '0.00', '0,00')""")
|
|
column["usage"]["meaningful_non_empty"] = cur.fetchone()[0]
|
|
except Exception as error: # noqa: BLE001
|
|
column["usage"] = {"error": type(error).__name__}
|
|
# tables that are entirely about the reform (name based), with their full column list and a few rows' worth of values
|
|
rtc_tables = sorted({c["table"] for c in columns if re.search(r"RTC|IBS|CBS|REFORMA|CLASSTRIB", c["table"].upper())})
|
|
result["rtc_tables"] = {}
|
|
for table in rtc_tables:
|
|
cols = [c for c in columns if c["table"] == table]
|
|
cur.execute(f'SELECT COUNT(*) FROM "{table}"')
|
|
result["rtc_tables"][table] = {"rows": cur.fetchone()[0], "columns": [(c["column"], c["type"], c["length"]) for c in sorted(cols, key=lambda c: c["position"])]}
|
|
if table == "RTC_CST":
|
|
cur.execute('SELECT FIRST 20 * FROM "RTC_CST"')
|
|
result["rtc_tables"][table]["sample"] = [[str(v).strip()[:60] for v in row] for row in cur.fetchall()]
|
|
cur.execute('SELECT USAR_REFORMA_TRIBUTARIA FROM PARAMETROS')
|
|
result["parametros_usar_reforma_tributaria"] = [str(v[0]) for v in cur.fetchall()]
|
|
cur.execute('SELECT COUNT(*) FROM C000025')
|
|
result["c000025_rows"] = cur.fetchone()[0]
|
|
# how many products carry a CCLASSTRIB that exists in the RTC_CCLASSTRIB catalog (orphan check, no inference)
|
|
cur.execute("""SELECT COUNT(*) FROM C000025 p WHERE TRIM(COALESCE(p.CCLASSTRIB, '')) <> '' AND EXISTS (SELECT 1 FROM RTC_CCLASSTRIB c WHERE c.CCLASSTRIB = p.CCLASSTRIB)""")
|
|
result["c000025_cclasstrib_in_catalog"] = cur.fetchone()[0]
|
|
result["write_attempts"] = ReadOnlyCursor.write_attempts
|
|
result["write_successes"] = ReadOnlyCursor.write_successes
|
|
conn.close()
|
|
text = json.dumps(result, ensure_ascii=False, indent=1, default=str)
|
|
if args.json:
|
|
Path(args.json).write_text(text, encoding="utf-8")
|
|
print(f"tables={result['tables']} columns={result['total_columns']} matches={len(matches)} writes_attempted={result['write_attempts']}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|