29 lines
1.0 KiB
Python
29 lines
1.0 KiB
Python
"""Deterministic prompt compaction that preserves candidate evidence."""
|
|
|
|
import json
|
|
|
|
|
|
def estimate_tokens(payload) -> int:
|
|
return max(1, len(json.dumps(payload, ensure_ascii=False, sort_keys=True)) // 4)
|
|
|
|
|
|
def compact_payload(payload):
|
|
"""Remove repeated metadata only; candidate codes/descriptions are untouched."""
|
|
compact = dict(payload)
|
|
compact.pop("debug", None)
|
|
compact.pop("duplicate_examples", None)
|
|
candidates = compact.get("official_candidates")
|
|
if isinstance(candidates, list):
|
|
seen = set()
|
|
compact["official_candidates"] = [
|
|
row for row in candidates
|
|
if not (row.get("ncm") in seen or seen.add(row.get("ncm")))
|
|
]
|
|
return compact
|
|
|
|
|
|
def measure_payloads(payloads):
|
|
before = sum(estimate_tokens(payload) for payload in payloads)
|
|
after = sum(estimate_tokens(compact_payload(payload)) for payload in payloads)
|
|
return {"items": len(payloads), "tokens_before": before, "tokens_after": after, "reduction_percent": round((1 - after / before) * 100, 2) if before else 0.0}
|