GTI_GERENCIADOR_FISCAL/fiscal/scripts/discover_testesoficiais_fiscal.py

"""READ-ONLY discovery of fiscal/RTC/PIS/COFINS fields in TESTESOFICIAIS.FDB (and only that database).

    python scripts/discover_testesoficiais_fiscal.py [--json out.json]

Credentials come from the process environment (FB_USER / FB_PASSWORD, optional FIREBIRD_CLIENT_LIBRARY) or the
local ignored .env - never from the command line, never printed. The connection uses READ COMMITTED READ ONLY
isolation and every statement goes through a guard that only lets SELECT reach the driver.
"""
import argparse
import json
import os
import re
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
DATABASE = ROOT / "TESTESOFICIAIS.FDB"  # fixed on purpose: this script audits no other database

KEYWORDS = ("TRIB", "IBS", "CBS", "PIS", "COFINS", "CRED", "REFORMA", "RTC", "CLASS")
FISCAL_HINTS = ("TRIB", "CST", "CFOP", "NCM", "PIS", "COF", "IBS", "CBS", "CRED", "ALIQ", "ICMS", "IPI", "CEST", "CSOSN", "CLASS", "ORIG", "REFORMA", "RTC", "SIT_T")
_TYPES = {7: "SMALLINT", 8: "INTEGER", 16: "BIGINT", 10: "FLOAT", 27: "DOUBLE", 12: "DATE", 13: "TIME", 35: "TIMESTAMP", 14: "CHAR", 37: "VARCHAR", 261: "BLOB"}


class ReadOnlyCursor:
    """Only SELECT statements are ever handed to the driver; anything else is counted and refused."""
    write_attempts = 0
    write_successes = 0

    def __init__(self, cursor):
        self._cursor = cursor

    def execute(self, sql, params=()):
        if not re.match(r"^\s*select\b", sql, re.IGNORECASE):
            ReadOnlyCursor.write_attempts += 1
            raise RuntimeError("non-SELECT statement refused")
        return self._cursor.execute(sql, params) if params else self._cursor.execute(sql)

    def fetchall(self):
        return self._cursor.fetchall()

    def fetchone(self):
        return self._cursor.fetchone()


def load_env():
    env_file = ROOT / ".env"
    if env_file.exists():
        for line in env_file.read_text(encoding="utf-8").splitlines():
            if "=" in line and not line.startswith("#"):
                key, _, value = line.partition("=")
                os.environ.setdefault(key.strip(), value.strip())


def classify(table, column):
    """Name-based first pass only. It is never taken as proof of meaning: the report lists evidence and a human/agent decides."""
    name = column.upper()
    if re.search(r"CCLASSTRIB|CLASS_?TRIB|CLASSIFICACAO_TRIBUTARIA", name):
        return "CONFIRMED_MATCH", "cClassTrib"
    if re.search(r"CST_?(IBS|CBS)|(IBS|CBS)_?CST|CSTIBSCBS|CST_IBSCBS", name):
        return "CONFIRMED_MATCH", "CST IBS/CBS"
    if re.search(r"(^|_)(IBS|CBS)($|_)|IBS|CBS", name) and not re.search(r"CBSA", name):
        return "CONFIRMED_MATCH", "IBS/CBS"
    if re.search(r"CCREDPRES|CRED_?PRES", name):
        return "CONFIRMED_MATCH", "crédito presumido"
    if re.search(r"(^|_)PIS($|_)|_PIS|PIS_|COD_(ENTRADA|SAIDA)_PIS", name):
        return "CONFIRMED_MATCH", "PIS"
    if re.search(r"COFINS", name):
        return "CONFIRMED_MATCH", "COFINS"
    if re.search(r"REFORMA|(^|_)RTC($|_)", name):
        return "POSSIBLE_MATCH", "reforma/RTC (nome)"
    if re.search(r"TRIB", name):
        return "POSSIBLE_MATCH", "tributação (nome genérico)"
    if re.search(r"CRED", name):
        return "POSSIBLE_MATCH", "crédito (nome genérico)"
    if re.search(r"CLASS", name):
        return "POSSIBLE_MATCH", "classificação (nome genérico)"
    return "UNRELATED", ""


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--json", default=None)
    args = parser.parse_args()
    load_env()
    sys.path.insert(0, str(ROOT / "agent" / "src"))
    import fdb

    conn = fdb.connect(database=str(DATABASE), user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"],
                       fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"), isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
    cur = ReadOnlyCursor(conn.cursor())
    result = {"database": DATABASE.name, "mode": "READ_ONLY", "firebird_server_version": getattr(conn, "server_version", None),
              "ods": f"{getattr(conn, 'ods_version', '?')}.{getattr(conn, 'ods_minor_version', '?')}"}
    cur.execute("""SELECT TRIM(rf.RDB$RELATION_NAME), TRIM(rf.RDB$FIELD_NAME), f.RDB$FIELD_TYPE, COALESCE(f.RDB$CHARACTER_LENGTH, f.RDB$FIELD_LENGTH),
                          f.RDB$FIELD_SCALE, rf.RDB$NULL_FLAG, TRIM(rf.RDB$FIELD_SOURCE), rf.RDB$DEFAULT_SOURCE, f.RDB$DEFAULT_SOURCE, rf.RDB$FIELD_POSITION
                   FROM RDB$RELATION_FIELDS rf JOIN RDB$FIELDS f ON f.RDB$FIELD_NAME = rf.RDB$FIELD_SOURCE
                   JOIN RDB$RELATIONS r ON r.RDB$RELATION_NAME = rf.RDB$RELATION_NAME
                   WHERE COALESCE(r.RDB$SYSTEM_FLAG, 0) = 0 AND r.RDB$VIEW_BLR IS NULL""")
    columns = []
    for table, column, ftype, length, scale, null_flag, source, rf_default, f_default, position in cur.fetchall():
        default = rf_default or f_default
        default = str(default.read() if hasattr(default, "read") else default).replace("DEFAULT", "").strip() if default else None
        columns.append({"table": table, "column": column, "type": _TYPES.get(ftype, f"type{ftype}") + (f"(scale {scale})" if scale else ""), "length": length,
                        "nullable": not bool(null_flag), "default": default, "domain": None if source.startswith("RDB$") else source, "position": position})
    result["tables"] = len({c["table"] for c in columns})
    result["total_columns"] = len(columns)

    c25 = [c for c in columns if c["table"] == "C000025"]
    result["c000025_columns"] = [c["column"] for c in sorted(c25, key=lambda c: c["position"])]
    result["c000025_fiscal_columns"] = [c for c in sorted(c25, key=lambda c: c["position"]) if any(h in c["column"].upper() for h in FISCAL_HINTS)]

    matches = []
    for column in columns:
        verdict, concept = classify(column["table"], column["column"])
        if verdict != "UNRELATED":
            matches.append({**column, "classification": verdict, "concept": concept})
    result["matches"] = matches

    def stats(table, column):
        cur.execute(f'SELECT COUNT(*), COUNT("{column}"), COUNT(DISTINCT "{column}") FROM "{table}"')
        total, non_null, distinct = cur.fetchone()
        info = {"rows": total, "non_null": non_null, "distinct": distinct}
        cur.execute(f'SELECT FIRST 6 "{column}", COUNT(*) FROM "{table}" WHERE "{column}" IS NOT NULL GROUP BY "{column}" ORDER BY 2 DESC')
        info["top_values"] = [[str(v).strip()[:24] if v is not None else None, n] for v, n in cur.fetchall()]
        return info

    for match in matches:
        if match["type"].startswith(("BLOB",)):
            continue
        try:
            match["usage"] = stats(match["table"], match["column"])
        except Exception as error:  # noqa: BLE001 - reported, never fatal
            match["usage"] = {"error": type(error).__name__}
    for column in result["c000025_fiscal_columns"]:
        column["classification"], column["concept"] = classify("C000025", column["column"])
        try:
            column["usage"] = stats("C000025", column["column"])
            cur.execute(f"""SELECT COUNT(*) FROM "C000025" WHERE TRIM(COALESCE(CAST("{column["column"]}" AS VARCHAR(40)), '')) NOT IN ('', '0', '0.00', '0,00')""")
            column["usage"]["meaningful_non_empty"] = cur.fetchone()[0]
        except Exception as error:  # noqa: BLE001
            column["usage"] = {"error": type(error).__name__}
    # tables that are entirely about the reform (name based), with their full column list and a few rows' worth of values
    rtc_tables = sorted({c["table"] for c in columns if re.search(r"RTC|IBS|CBS|REFORMA|CLASSTRIB", c["table"].upper())})
    result["rtc_tables"] = {}
    for table in rtc_tables:
        cols = [c for c in columns if c["table"] == table]
        cur.execute(f'SELECT COUNT(*) FROM "{table}"')
        result["rtc_tables"][table] = {"rows": cur.fetchone()[0], "columns": [(c["column"], c["type"], c["length"]) for c in sorted(cols, key=lambda c: c["position"])]}
        if table == "RTC_CST":
            cur.execute('SELECT FIRST 20 * FROM "RTC_CST"')
            result["rtc_tables"][table]["sample"] = [[str(v).strip()[:60] for v in row] for row in cur.fetchall()]
    cur.execute('SELECT USAR_REFORMA_TRIBUTARIA FROM PARAMETROS')
    result["parametros_usar_reforma_tributaria"] = [str(v[0]) for v in cur.fetchall()]
    cur.execute('SELECT COUNT(*) FROM C000025')
    result["c000025_rows"] = cur.fetchone()[0]
    # how many products carry a CCLASSTRIB that exists in the RTC_CCLASSTRIB catalog (orphan check, no inference)
    cur.execute("""SELECT COUNT(*) FROM C000025 p WHERE TRIM(COALESCE(p.CCLASSTRIB, '')) <> '' AND EXISTS (SELECT 1 FROM RTC_CCLASSTRIB c WHERE c.CCLASSTRIB = p.CCLASSTRIB)""")
    result["c000025_cclasstrib_in_catalog"] = cur.fetchone()[0]
    result["write_attempts"] = ReadOnlyCursor.write_attempts
    result["write_successes"] = ReadOnlyCursor.write_successes
    conn.close()
    text = json.dumps(result, ensure_ascii=False, indent=1, default=str)
    if args.json:
        Path(args.json).write_text(text, encoding="utf-8")
    print(f"tables={result['tables']} columns={result['total_columns']} matches={len(matches)} writes_attempted={result['write_attempts']}")


if __name__ == "__main__":
    main()