"""Compare the cell states before/after a source refresh: gate_before/*_second.ndjson.gz vs local_completion/<stage>.ndjson.gz."""
import gzip, json, sys
from collections import Counter
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]; stage = sys.argv[1] if len(sys.argv) > 1 else "dryrun"
DBS = ["CONSTRUCAO", "MMMODAS", "RINCAO", "UNIAO", "TESTESOFICIAIS"]
trans = Counter(); ex = {}; man_b = man_a = 0
for n in DBS:
    old = {}
    for l in gzip.open(ROOT / "docs/runs/gate_before" / f"{n}_second.ndjson.gz", "rt", encoding="utf-8"):
        r = json.loads(l); old[(r["row_key"], r["field"])] = r
    for l in gzip.open(ROOT / "docs/runs/local_completion" / f"{n}_{stage}.ndjson.gz", "rt", encoding="utf-8"):
        r = json.loads(l); o = old[(r["row_key"], r["field"])]
        man_b += o["status"] == "MANUAL_INPUT_REQUIRED"; man_a += r["status"] == "MANUAL_INPUT_REQUIRED"
        if o["status"] != r["status"] or o["proposed"] != r["proposed"]:
            k = (n, r["field"], o["status"], o["reason"], "->", r["status"], r["reason"][:40]); trans[k] += 1
            ex.setdefault(k, (r["codigo"], o["before"], r["proposed"]))
print("MANUAL before", man_b, "after", man_a)
for k, v in trans.most_common(30): print(v, k, ex[k])