Add 'fiscal/' from commit '102ba84500dbd9655a0471288c3e278652593ca9'

git-subtree-dir: fiscal
git-subtree-mainline: d95165fc8f
git-subtree-split: 102ba84500

fiscal/scripts/gate_diff.py

@@ -0,0 +1,19 @@
+"""Compare the cell states before/after a source refresh: gate_before/*_second.ndjson.gz vs local_completion/<stage>.ndjson.gz."""
+import gzip, json, sys
+from collections import Counter
+from pathlib import Path
+ROOT = Path(__file__).resolve().parents[1]; stage = sys.argv[1] if len(sys.argv) > 1 else "dryrun"
+DBS = ["CONSTRUCAO", "MMMODAS", "RINCAO", "UNIAO", "TESTESOFICIAIS"]
+trans = Counter(); ex = {}; man_b = man_a = 0
+for n in DBS:
+    old = {}
+    for l in gzip.open(ROOT / "docs/runs/gate_before" / f"{n}_second.ndjson.gz", "rt", encoding="utf-8"):
+        r = json.loads(l); old[(r["row_key"], r["field"])] = r
+    for l in gzip.open(ROOT / "docs/runs/local_completion" / f"{n}_{stage}.ndjson.gz", "rt", encoding="utf-8"):
+        r = json.loads(l); o = old[(r["row_key"], r["field"])]
+        man_b += o["status"] == "MANUAL_INPUT_REQUIRED"; man_a += r["status"] == "MANUAL_INPUT_REQUIRED"
+        if o["status"] != r["status"] or o["proposed"] != r["proposed"]:
+            k = (n, r["field"], o["status"], o["reason"], "->", r["status"], r["reason"][:40]); trans[k] += 1
+            ex.setdefault(k, (r["codigo"], o["before"], r["proposed"]))
+print("MANUAL before", man_b, "after", man_a)
+for k, v in trans.most_common(30): print(v, k, ex[k])