20 lines
1.2 KiB
Python
20 lines
1.2 KiB
Python
"""Compare the cell states before/after a source refresh: gate_before/*_second.ndjson.gz vs local_completion/<stage>.ndjson.gz."""
|
|
import gzip, json, sys
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
ROOT = Path(__file__).resolve().parents[1]; stage = sys.argv[1] if len(sys.argv) > 1 else "dryrun"
|
|
DBS = ["CONSTRUCAO", "MMMODAS", "RINCAO", "UNIAO", "TESTESOFICIAIS"]
|
|
trans = Counter(); ex = {}; man_b = man_a = 0
|
|
for n in DBS:
|
|
old = {}
|
|
for l in gzip.open(ROOT / "docs/runs/gate_before" / f"{n}_second.ndjson.gz", "rt", encoding="utf-8"):
|
|
r = json.loads(l); old[(r["row_key"], r["field"])] = r
|
|
for l in gzip.open(ROOT / "docs/runs/local_completion" / f"{n}_{stage}.ndjson.gz", "rt", encoding="utf-8"):
|
|
r = json.loads(l); o = old[(r["row_key"], r["field"])]
|
|
man_b += o["status"] == "MANUAL_INPUT_REQUIRED"; man_a += r["status"] == "MANUAL_INPUT_REQUIRED"
|
|
if o["status"] != r["status"] or o["proposed"] != r["proposed"]:
|
|
k = (n, r["field"], o["status"], o["reason"], "->", r["status"], r["reason"][:40]); trans[k] += 1
|
|
ex.setdefault(k, (r["codigo"], o["before"], r["proposed"]))
|
|
print("MANUAL before", man_b, "after", man_a)
|
|
for k, v in trans.most_common(30): print(v, k, ex[k])
|