51 lines
2.6 KiB
Python
51 lines
2.6 KiB
Python
"""Deterministic product-description normalization and fingerprint (no ML). Used to FIND candidates; never authorizes a fiscal value on its own."""
|
|
import hashlib
|
|
import re
|
|
import unicodedata
|
|
|
|
UNIT_MAP = {"LT": "L", "LTS": "L", "LITRO": "L", "LITROS": "L", "L": "L", "ML": "ML", "KG": "KG", "KGS": "KG", "QUILO": "KG", "G": "G", "GR": "G", "GRS": "G", "GRAMAS": "G", "MG": "MG", "UN": "UN", "UND": "UN", "UNID": "UN", "UNIDADE": "UN", "PCT": "PCT", "CX": "CX"}
|
|
ABBREVIATIONS = {"REFRIG": "REFRIGERANTE", "REFRI": "REFRIGERANTE", "CHOC": "CHOCOLATE", "BISC": "BISCOITO", "SAB": "SABONETE", "DET": "DETERGENTE", "SH": "SHAMPOO", "COND": "CONDICIONADOR",
|
|
"MACARR": "MACARRAO", "MAC": "MACARRAO", "AMAC": "AMACIANTE", "DESOD": "DESODORANTE", "PAP": "PAPEL", "HIG": "HIGIENICO", "ORIG": "ORIGINAL", "TRAD": "TRADICIONAL"}
|
|
STOP = {"DE", "DA", "DO", "DAS", "DOS", "E", "COM", "C", "PARA", "P", "EM", "A", "O"}
|
|
_QTY = re.compile(r"^(\d+(?:[.,]\d+)?)(ML|L|KG|G|MG|UN|PCT|CX|LT|LTS|GR)$")
|
|
|
|
|
|
def _ascii_upper(text) -> str:
|
|
return unicodedata.normalize("NFKD", str(text or "")).encode("ascii", "ignore").decode().upper()
|
|
|
|
|
|
def tokens(description) -> list[str]:
|
|
text = _ascii_upper(description)
|
|
text = re.sub(r"(\d)([A-Z])", r"\1 \2", text) # 350ML -> 350 ML
|
|
text = re.sub(r"([A-Z])(\d)", r"\1 \2", text)
|
|
text = re.sub(r"(\d)[,.](\d)", r"\1.\2", text) # decimal comma -> dot (kept inside the number)
|
|
text = re.sub(r"[^A-Z0-9.]+", " ", text)
|
|
out = []
|
|
for tok in text.split():
|
|
tok = tok.strip(".")
|
|
if not tok or tok in STOP:
|
|
continue
|
|
tok = UNIT_MAP.get(tok, ABBREVIATIONS.get(tok, tok))
|
|
out.append(tok)
|
|
return out
|
|
|
|
|
|
def normalized(description) -> str:
|
|
return " ".join(tokens(description))
|
|
|
|
|
|
def fingerprint(description) -> str | None:
|
|
toks = tokens(description)
|
|
if not toks:
|
|
return None
|
|
return hashlib.sha256(" ".join(sorted(toks)).encode("utf-8")).hexdigest()
|
|
|
|
|
|
def is_specific(description) -> bool:
|
|
"""A description strong enough to be used as an identity key: at least 3 informative tokens AND a quantity (number + unit), or >= 4 tokens.
|
|
Generic names (PIPOCA, OLEO, REFRIGERANTE, SABONETE) are NEVER specific."""
|
|
toks = tokens(description)
|
|
has_qty = any(re.fullmatch(r"\d+(?:\.\d+)?", t) and i + 1 < len(toks) and toks[i + 1] in UNIT_MAP.values() for i, t in enumerate(toks))
|
|
informative = [t for t in toks if not re.fullmatch(r"\d+(?:\.\d+)?", t) and t not in UNIT_MAP.values()]
|
|
return (len(informative) >= 2 and has_qty and len(toks) >= 3) or len(informative) >= 4
|