Add 'fiscal/' from commit '102ba84500dbd9655a0471288c3e278652593ca9'

git-subtree-dir: fiscal
git-subtree-mainline: d95165fc8f
git-subtree-split: 102ba84500

fiscal/central/app/enrichment/textnorm.py

@@ -0,0 +1,51 @@
+"""Deterministic product-description normalization and fingerprint (no ML). Used to FIND candidates; never authorizes a fiscal value on its own."""
+import hashlib
+import re
+import unicodedata
+
+UNIT_MAP = {"LT": "L", "LTS": "L", "LITRO": "L", "LITROS": "L", "L": "L", "ML": "ML", "KG": "KG", "KGS": "KG", "QUILO": "KG", "G": "G", "GR": "G", "GRS": "G", "GRAMAS": "G", "MG": "MG", "UN": "UN", "UND": "UN", "UNID": "UN", "UNIDADE": "UN", "PCT": "PCT", "CX": "CX"}
+ABBREVIATIONS = {"REFRIG": "REFRIGERANTE", "REFRI": "REFRIGERANTE", "CHOC": "CHOCOLATE", "BISC": "BISCOITO", "SAB": "SABONETE", "DET": "DETERGENTE", "SH": "SHAMPOO", "COND": "CONDICIONADOR",
+                 "MACARR": "MACARRAO", "MAC": "MACARRAO", "AMAC": "AMACIANTE", "DESOD": "DESODORANTE", "PAP": "PAPEL", "HIG": "HIGIENICO", "ORIG": "ORIGINAL", "TRAD": "TRADICIONAL"}
+STOP = {"DE", "DA", "DO", "DAS", "DOS", "E", "COM", "C", "PARA", "P", "EM", "A", "O"}
+MARKETING = {"ORIGINAL", "TRADICIONAL", "NOVO", "NOVA", "PROMO", "PROMOCAO", "OFERTA"}      # ignored ONLY by the fingerprint; a different pack/flavour still differs
+_QTY = re.compile(r"^(\d+(?:[.,]\d+)?)(ML|L|KG|G|MG|UN|PCT|CX|LT|LTS|GR)$")
+
+
+def _ascii_upper(text) -> str:
+    return unicodedata.normalize("NFKD", str(text or "")).encode("ascii", "ignore").decode().upper()
+
+
+def tokens(description) -> list[str]:
+    text = _ascii_upper(description)
+    text = re.sub(r"(\d)([A-Z])", r"\1 \2", text)                # 350ML -> 350 ML
+    text = re.sub(r"([A-Z])(\d)", r"\1 \2", text)
+    text = re.sub(r"(\d)[,.](\d)", r"\1.\2", text)               # decimal comma -> dot (kept inside the number)
+    text = re.sub(r"[^A-Z0-9.]+", " ", text)
+    out = []
+    for tok in text.split():
+        tok = tok.strip(".")
+        if not tok or tok in STOP:
+            continue
+        tok = UNIT_MAP.get(tok, ABBREVIATIONS.get(tok, tok))
+        out.append(tok)
+    return out
+
+
+def normalized(description) -> str:
+    return " ".join(tokens(description))
+
+
+def fingerprint(description) -> str | None:
+    toks = [t for t in tokens(description) if t not in MARKETING]
+    if not toks:
+        return None
+    return hashlib.sha256(" ".join(sorted(toks)).encode("utf-8")).hexdigest()
+
+
+def is_specific(description) -> bool:
+    """A description strong enough to be used as an identity key: at least 3 informative tokens AND a quantity (number + unit), or >= 4 tokens.
+    Generic names (PIPOCA, OLEO, REFRIGERANTE, SABONETE) are NEVER specific."""
+    toks = tokens(description)
+    has_qty = any(re.fullmatch(r"\d+(?:\.\d+)?", t) and i + 1 < len(toks) and toks[i + 1] in UNIT_MAP.values() for i, t in enumerate(toks))
+    informative = [t for t in toks if not re.fullmatch(r"\d+(?:\.\d+)?", t) and t not in UNIT_MAP.values()]
+    return (len(informative) >= 2 and has_qty and len(toks) >= 3) or len(informative) >= 4