GTI_GERENCIADOR_FISCAL/fiscal/scripts/audit_multi_databases.py

"""READ-ONLY schema/capability/population audit of every .FDB in a directory (default: ./Banco).

    python scripts/audit_multi_databases.py [DIR]

Uses read-only isolation and SELECT only; credentials from FB_USER/FB_PASSWORD (env or local ignored .env).
Writes docs/runs/multi_db/<DB>.json (raw) and docs/MULTI_DATABASE_SCHEMA_AUDIT_V1.json,
docs/MULTI_DATABASE_FIELD_COMPATIBILITY_V1.json. Never modifies a database.
"""
import json
import os
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts")); sys.path.insert(0, str(ROOT / "agent" / "src")); sys.path.insert(0, str(ROOT / "src"))
import sgv_capabilities as cap  # noqa: E402

REFERENCE = json.loads((ROOT / "docs" / "runs" / "audit_TESTESOFICIAIS.json").read_text(encoding="utf-8"))["schema_fingerprint"]
SIGNAL_TABLES = ["MESAS", "R000002", "RECEITA_INGREDIENTES", "DELIVERY_M", "C000051", "NFCE", "NFCE_ITENS", "C000032", "C000062", "C000088", "C000100", "C000030", "C000061"]
CONTEXT_TABLES = ["C000030", "C000032", "C000062", "C000088", "C000100", "NFCE_ITENS", "RECEITA_INGREDIENTES"]


def load_env():
    f = ROOT / ".env"
    if f.exists():
        for line in f.read_text(encoding="utf-8").splitlines():
            if "=" in line and not line.startswith("#"):
                k, _, v = line.partition("=")
                os.environ.setdefault(k.strip(), v.strip())


def audit_one(path):
    import fdb
    from gti_discovery.fingerprint import build_fingerprint
    from gti_fiscal_agent.firebird_reader import parse_company_context
    conn = fdb.connect(database=str(path), user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"], fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
                       isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
    try:
        cur = conn.cursor()
        res = {"file": path.name, "size_mb": round(path.stat().st_size / 1e6, 1), "mode": "READ_ONLY", "firebird_server_version": getattr(conn, "server_version", None),
               "ods": f"{getattr(conn, 'ods_version', '?')}.{getattr(conn, 'ods_minor_version', '?')}"}
        cur.execute("""SELECT TRIM(rf.RDB$RELATION_NAME), TRIM(rf.RDB$FIELD_NAME), f.RDB$FIELD_TYPE, COALESCE(f.RDB$CHARACTER_LENGTH, f.RDB$FIELD_LENGTH), f.RDB$FIELD_SCALE
                       FROM RDB$RELATION_FIELDS rf JOIN RDB$FIELDS f ON f.RDB$FIELD_NAME = rf.RDB$FIELD_SOURCE JOIN RDB$RELATIONS r ON r.RDB$RELATION_NAME = rf.RDB$RELATION_NAME
                       WHERE COALESCE(r.RDB$SYSTEM_FLAG, 0) = 0""")
        meta = {}
        for t, c, ty, ln, sc in cur.fetchall():
            meta.setdefault(t, []).append({"name": c, "type": ty, "length": ln, "scale": sc})
        res["tables"] = len(meta)
        res["schema_fingerprint"] = build_fingerprint({"tables": [{"name": t, "columns": cols} for t, cols in sorted(meta.items())]})
        res["C000004_present"] = "C000004" in meta; res["C000025_present"] = "C000025" in meta
        company = {"status": "C000004_ABSENT"}
        if res["C000004_present"]:
            names = {c["name"] for c in meta["C000004"]}
            try:
                cur.execute("SELECT UF, CRT FROM C000004"); company = {"status": "OK", **parse_company_context(cur.fetchall())}
            except ValueError as e:
                company = {"status": "UNAVAILABLE", "error": str(e)}
            for col in ("OPTANTE_SIMPLES", "OPTANTE_SUPER_SIMPLES", "SUBSTITUTO_TRIBUTARIO", "CONTRIBUINTE_IPI", "ICMS_CSOSN"):
                if col in names:
                    cur.execute(f"SELECT FIRST 1 {col} FROM C000004"); row = cur.fetchone(); company[col] = None if row is None or row[0] is None else str(row[0]).strip()
            for col in ("FANTASIA", "NOME_FANTASIA", "RAZAO_SOCIAL", "EMPRESA", "NOME"):  # short display name only, never CNPJ
                if col in names:
                    cur.execute(f"SELECT FIRST 1 {col} FROM C000004"); row = cur.fetchone(); company["name"] = None if row is None or row[0] is None else str(row[0]).strip()[:40]; break
            cur.execute("SELECT CODIGO, UF, CRT FROM C000004")
            rows = [[None if v is None else str(v).strip() for v in r] for r in cur.fetchall()]
            company["rows"] = len(rows); company["rows_summary"] = rows
            if "FILIAL" in names:
                cur.execute("SELECT FIRST 1 FILIAL FROM C000004 ORDER BY CODIGO"); r0 = cur.fetchone()
                company["filial_name_first_row"] = None if r0 is None or r0[0] is None else str(r0[0]).strip()[:50]
        if "EMISSOR" in meta and any(c["name"] == "EMITENTE_CRT" for c in meta["EMISSOR"]):
            cur.execute("SELECT EMITENTE_CRT FROM EMISSOR"); company["emissor_crt"] = sorted({str(r[0]).strip() for r in cur.fetchall() if r[0] is not None})
        res["company"] = company
        counts = {}
        for t in SIGNAL_TABLES:
            if t in meta:
                cur.execute(f'SELECT COUNT(*) FROM "{t}"'); counts[t] = cur.fetchone()[0]
            else:
                counts[t] = None
        res["table_rows"] = counts
        if res["C000025_present"]:
            cols = {c["name"]: c["type"] for c in meta["C000025"]}
            res["C000025_columns"] = len(cols)
            full = {c["name"]: c for c in meta["C000025"]}
            res["field_types"] = {f: [full[f]["type"], full[f]["length"], full[f]["scale"]] for f in cap.WRITE_TARGETS + [cap.ORIGIN_FIELD] if f in full}
            res["capabilities"] = cap.detect_capabilities(cols)
            res["field_compatibility"] = cap.field_compatibility(cols)
            cur.execute("SELECT COUNT(*) FROM C000025"); res["product_count"] = cur.fetchone()[0]
            dist, pop = {}, {}
            for f in cap.WRITE_TARGETS + [cap.ORIGIN_FIELD]:
                if f in cols:
                    cur.execute(f'SELECT "{f}", COUNT(*) FROM C000025 GROUP BY "{f}" ORDER BY 2 DESC')
                    rows = [(None if v is None else str(v).strip(), n) for v, n in cur.fetchall()]
                    pop[f] = sum(n for v, n in rows if v not in (None, ""))
                    dist[f] = {"distinct": len(rows), "top": rows[:8]}
            res["fiscal_field_population"] = pop; res["fiscal_field_distribution"] = dist
            for extra, sql in (("kitchen_products", "SELECT COUNT(*) FROM C000025 WHERE PROD_COZINHA = 1"), ("vehicle_products", "SELECT COUNT(*) FROM C000025 WHERE VEIC_TIPO IS NOT NULL AND TRIM(VEIC_TIPO) <> ''")):
                col = "PROD_COZINHA" if "COZINHA" in sql else "VEIC_TIPO"
                if col in cols:
                    cur.execute(sql); res[extra] = cur.fetchone()[0]
            grade = [t for t in meta if "GRADE" in t]
            res["grade_tables"] = {t: None for t in grade}
            gi = 0
            for t in grade:
                cur.execute(f'SELECT COUNT(*) FROM "{t}"'); n = cur.fetchone()[0]; res["grade_tables"][t] = n; gi += n
            sig = {"restaurant_tables_rows": (counts.get("MESAS") or 0) + (counts.get("R000002") or 0), "kitchen_products": res.get("kitchen_products", 0), "recipes": counts.get("RECEITA_INGREDIENTES") or 0,
                   "service_orders": counts.get("C000051") or 0, "vehicle_products": res.get("vehicle_products", 0), "grade_items": gi, "nfce_items": counts.get("NFCE_ITENS") or 0}
            seg, score, ev = cap.infer_segment(sig, " ".join(str(company.get(k) or "") for k in ("name", "filial_name_first_row")))
            res["segment_inferred"] = {"segment": seg, "score": score, "evidence": ev, "signals": sig, "note": "inferred from table population, not from the file name"}
        return res
    finally:
        conn.close()


def main():
    load_env()
    directory = Path(sys.argv[1]) if len(sys.argv) > 1 else ROOT / "Banco"
    dbs = sorted(directory.glob("*.FDB")) + sorted(p for p in directory.glob("*.fdb") if p.suffix != ".FDB")
    out_dir = ROOT / "docs" / "runs" / "multi_db"; out_dir.mkdir(parents=True, exist_ok=True)
    audits = []
    ref_types = audit_one(ROOT / "TESTESOFICIAIS.FDB").get("field_types", {})  # reference database, read-only
    for p in dbs:
        try:
            a = audit_one(p)
            a["type_variations_vs_reference"] = {f: {"reference": ref_types.get(f), "this": v} for f, v in a.get("field_types", {}).items() if ref_types.get(f) != v}
            a["schema_class"] = cap.classify_schema(a["C000004_present"], a["C000025_present"], a["schema_fingerprint"], REFERENCE, a.get("field_compatibility", {}))
        except Exception as e:  # noqa: BLE001 — record, never guess
            a = {"file": p.name, "size_mb": round(p.stat().st_size / 1e6, 1), "error": f"{type(e).__name__}: {str(e)[:200]}", "schema_class": "INCOMPATIBLE"}
        (out_dir / f"{p.stem}.json").write_text(json.dumps(a, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
        audits.append(a)
        print(p.name, a.get("schema_class"), a.get("product_count"), a.get("company"), a.get("error", ""))
    ok = [a for a in audits if "error" not in a]
    schema_doc = {"artifact": "MULTI_DATABASE_SCHEMA_AUDIT_V1", "firebird_writes": 0, "directory": directory.name, "reference_fingerprint_TESTESOFICIAIS": REFERENCE, "databases_found": len(dbs), "databases_audited": len(ok),
                  "classification_counts": {k: sum(1 for a in audits if a["schema_class"] == k) for k in ("REFERENCE_SCHEMA_MATCH", "COMPATIBLE_WITH_VARIATIONS", "INCOMPATIBLE")},
                  "databases": [{k: a.get(k) for k in ("file", "size_mb", "firebird_server_version", "ods", "schema_fingerprint", "tables", "product_count", "company", "C000004_present", "C000025_present", "schema_class",
                                                        "capabilities", "segment_inferred", "table_rows", "error")} for a in audits]}
    (ROOT / "docs" / "MULTI_DATABASE_SCHEMA_AUDIT_V1.json").write_text(json.dumps(schema_doc, ensure_ascii=False, indent=1, default=str), encoding="utf-8")
    fc = {}
    for f in cap.WRITE_TARGETS + [cap.ORIGIN_FIELD]:
        fc[f] = {"status_per_db": {a["file"]: a.get("field_compatibility", {}).get(f, "N/A") for a in audits},
                 "population_per_db": {a["file"]: a.get("fiscal_field_population", {}).get(f) for a in ok},
                 "distribution_top_per_db": {a["file"]: a.get("fiscal_field_distribution", {}).get(f, {}).get("top", [])[:5] for a in ok}}
    all_present = [f for f in cap.WRITE_TARGETS if all(a.get("field_compatibility", {}).get(f) in ("PRESENT_OK", "PRESENT_TYPE_VARIATION") for a in ok)]
    variations = {}
    for a in ok:
        for f, v in a.get("type_variations_vs_reference", {}).items():
            variations.setdefault(f, {})[a["file"]] = v
        for f, v in a.get("field_compatibility", {}).items():
            if v not in ("PRESENT_OK",):
                variations.setdefault(f, {})[a["file"]] = v
    (ROOT / "docs" / "MULTI_DATABASE_FIELD_COMPATIBILITY_V1.json").write_text(json.dumps({"artifact": "MULTI_DATABASE_FIELD_COMPATIBILITY_V1", "write_targets_present_in_all": all_present, "write_targets_count": len(cap.WRITE_TARGETS),
                                                                                       "variations": variations, "fields": fc}, ensure_ascii=False, indent=1, default=str), encoding="utf-8")


main()