From c68c6c67ed378d0bdf16e1320078cf1665d21979 Mon Sep 17 00:00:00 2001 From: gti Date: Sat, 26 Sep 2026 10:50:12 -0300 Subject: [PATCH] feat: TECHNICAL_PLACEHOLDER (NCM 00000000) is replaceable by the product enrichment; real values preserved; Agent 1.3.4 Co-Authored-By: Claude Sonnet 5 --- agent/pyproject.toml | 2 +- agent/scripts/run_agent_service.py | 6 +- agent/src/gti_fiscal_agent/enrichment.py | 28 +++++- agent/src/gti_fiscal_agent/preflight.py | 2 +- agent/tests/test_enrichment.py | 67 ++++++++++++- central/app/enrichment/api.py | 4 + central/app/enrichment/placeholders.py | 27 +++++ central/app/enrichment/resolver.py | 6 +- central/app/version.py | 2 +- central/pyproject.toml | 2 +- central/tests/test_health.py | 2 +- .../tests/test_product_enrichment_resolver.py | 12 +++ deploy/agent-windows/agent.env.template | 2 +- deploy/agent-windows/gti-agent.ps1 | 2 +- .../pilot-construcao.env.example | 2 +- .../profiles/mmmodas.env.example | 2 +- .../agent-windows/profiles/rincao.env.example | 2 +- .../agent-windows/profiles/uniao.env.example | 2 +- deploy/hub/.env.example | 2 +- deploy/hub/docker-compose.prod.yml | 2 +- docs/product_enrichment.md | 10 ++ docs/runs/enrichment_placeholder_e2e.json | 22 +++++ release/RELEASE_MANIFEST.json | 8 +- release/VERSION | 2 +- scripts/enrichment_placeholder_e2e.py | 99 +++++++++++++++++++ tests/test_release_candidate.py | 4 +- 26 files changed, 290 insertions(+), 31 deletions(-) create mode 100644 central/app/enrichment/placeholders.py create mode 100644 docs/runs/enrichment_placeholder_e2e.json create mode 100644 scripts/enrichment_placeholder_e2e.py diff --git a/agent/pyproject.toml b/agent/pyproject.toml index 9de85cb..829cc92 100644 --- a/agent/pyproject.toml +++ b/agent/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "gti-fiscal-agent" -version = "1.3.3" +version = "1.3.4" requires-python = ">=3.10" dependencies = ["httpx>=0.27,<1"] diff --git a/agent/scripts/run_agent_service.py b/agent/scripts/run_agent_service.py index 051de43..03c215a 100644 --- a/agent/scripts/run_agent_service.py +++ b/agent/scripts/run_agent_service.py @@ -107,7 +107,7 @@ class Config: self.poll_interval_seconds = float(env.get("POLL_INTERVAL_SECONDS", "5")) # RTC auto-apply is a SEPARATE switch (default OFF): FIREBIRD_WRITES alone never lets the Agent fill cClassTrib. self.rtc_auto_apply_max_per_job = int(env.get("RTC_AUTO_APPLY_MAX_PER_JOB", str(_rtc_apply.DEFAULT_MAX_PER_JOB))) - self.agent_version = env.get("AGENT_VERSION", "1.3.3") + self.agent_version = env.get("AGENT_VERSION", "1.3.4") self.schema_fingerprint = env.get("SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1) self.state_db = env.get("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite")) # Pilot write authorization: both must be explicitly set by the installer for @@ -349,7 +349,7 @@ def execute_job(job, config, executor, state, env): "fiscal_context": fiscal_context, "rtc_auto_apply": rtc_apply, "fiscal_mode": getattr(config, "fiscal_mode", None), - "release_version": env.get("AGENT_VERSION") or "1.3.3", + "release_version": env.get("AGENT_VERSION") or "1.3.4", "products_total": len(products), "products_changed": len(to_process), "products_processed": summary["processed"], @@ -520,7 +520,7 @@ def run(config, env, *, max_iterations=None, stopper=None): def enforce_startup_preflight(env): """Full read-only preflight at startup (Hub, auth, Firebird, capabilities, company context, guards). Fail closed.""" from gti_fiscal_agent.preflight import run_preflight - report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.3") + report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.4") logger.info("startup preflight: %s", json.dumps({k: report[k] for k in ("PREFLIGHT", "blockers", "fiscal_mode", "effective_firebird_writes")}, ensure_ascii=False)) print(json.dumps({"PREFLIGHT": report["PREFLIGHT"], "blockers": report["blockers"]}, ensure_ascii=False)) if report["PREFLIGHT"] != "PASS": diff --git a/agent/src/gti_fiscal_agent/enrichment.py b/agent/src/gti_fiscal_agent/enrichment.py index a124e8d..a3034f0 100644 --- a/agent/src/gti_fiscal_agent/enrichment.py +++ b/agent/src/gti_fiscal_agent/enrichment.py @@ -21,7 +21,10 @@ from datetime import datetime, timezone logger = logging.getLogger("gti_fiscal_agent.enrichment") CAPABILITY = "PRODUCT_ENRICHMENT_WRITE_V1" -ALLOWED_FIELDS = ("CLASSIFICACAO_FISCAL", "CEST", "IPI") # explicit allowlist; anything else raises before touching SQL +ALLOWED_FIELDS = ("CLASSIFICACAO_FISCAL", "CEST", "IPI") +# TECHNICAL_PLACEHOLDER: the SGV refuses a product without NCM, so operators type a filler. It means "not classified yet" (not a human decision) and may be replaced. +# Explicit per-field allowlist, identical to central/app/enrichment/placeholders.py (parity test). Nothing else is ever overwritten. +PLACEHOLDERS = {"CLASSIFICACAO_FISCAL": frozenset({"00000000"})} # explicit allowlist; anything else raises before touching SQL READ_FIELDS = ("CODIGO", "PRODUTO", "GTIN", "CODBARRA") + ALLOWED_FIELDS # the SGV operator types the EAN in CODBARRA; GTIN (fiscal) is mostly empty POLL_SECONDS = 2.0 SAFETY_DIFF_SECONDS = 60.0 @@ -108,6 +111,16 @@ def is_human_overridden(store_path, code, field) -> bool: # ---- Firebird access (injected: `read_conn()` returns a READ-ONLY connection, `write_conn()` a write-capable one, only ever opened by ENABLED mode) ---------------------------------------- +def is_placeholder(field, value) -> bool: + text = _text(value) + text = "".join(ch for ch in text if ch.isdigit()) if field == "CLASSIFICACAO_FISCAL" else text + return text in PLACEHOLDERS.get(field, frozenset()) + + +def writable_now(field, value) -> bool: + return not _text(value) or is_placeholder(field, value) + + def _digits(value) -> str: return "".join(ch for ch in _text(value) if ch.isdigit()) @@ -174,10 +187,14 @@ def write_field(conn, code: str, field: str, value) -> dict: if len(rows) != 1: conn.rollback() return {"outcome": "ABORTED", "reason": "PRODUCT_ROW_NOT_UNIQUE" if rows else "PRODUCT_NOT_FOUND"} - if _text(rows[0][0]): + raw = rows[0][0] + if not writable_now(field, raw): conn.rollback() return {"outcome": "ABORTED", "reason": "VALUE_ALREADY_SET_NEVER_OVERWRITTEN"} - cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND ({field} IS NULL OR TRIM({field}) = '')", (value, code)) + if _text(raw): # technical placeholder: compare-and-swap on the exact value just read + cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND TRIM({field}) = ?", (value, code, _text(raw))) + else: + cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND ({field} IS NULL OR TRIM({field}) = '')", (value, code)) if cur.rowcount != 1: conn.rollback() return {"outcome": "ABORTED", "reason": "UPDATE_ROWCOUNT_NOT_ONE"} @@ -189,7 +206,7 @@ def write_field(conn, code: str, field: str, value) -> dict: same = (float(after) == value) if field == "IPI" and after is not None else _text(after) == value if not same: return {"outcome": "FAILED", "reason": "POST_WRITE_VERIFICATION_FAILED"} - return {"outcome": "APPLIED", "value": value, "rowcount": 1} + return {"outcome": "APPLIED", "value": value, "rowcount": 1, "replaced_placeholder": _text(raw) if _text(raw) else None} except Exception as error: # noqa: BLE001 try: conn.rollback() @@ -345,7 +362,8 @@ class EnrichmentWorker: for field in ALLOWED_FIELDS: decision = fields.get(field) or {} status, value = decision.get("status"), decision.get("value") - base = {"field": field, "rule": decision.get("rule"), "kb_status": decision.get("kb_status"), "evidence": decision.get("evidence") or [], "old": None, "new": value} + placeholder = _text(current.get(field)) if is_placeholder(field, current.get(field)) else None + base = {"field": field, "rule": decision.get("rule"), "kb_status": decision.get("kb_status"), "evidence": decision.get("evidence") or [], "old": placeholder, "new": value} if status == "HUMAN_OVERRIDDEN" or overridden.get(field): reports.append({**base, "outcome": "HUMAN_OVERRIDDEN", "reason": "HUMAN_OVERRIDDEN", "new": None}) continue diff --git a/agent/src/gti_fiscal_agent/preflight.py b/agent/src/gti_fiscal_agent/preflight.py index 2b77cc9..e5e6eaf 100644 --- a/agent/src/gti_fiscal_agent/preflight.py +++ b/agent/src/gti_fiscal_agent/preflight.py @@ -9,7 +9,7 @@ import os import sqlite3 from pathlib import Path -AGENT_VERSION_DEFAULT = "1.3.3" +AGENT_VERSION_DEFAULT = "1.3.4" REQUIRED_ENV = ("GTI_HUB_URL", "AGENT_TOKEN", "FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD") REQUIRED_CAPABILITIES = ("supports_ncm", "supports_ipi", "supports_cest") FISCAL_MODES = ("DRY_RUN", "AUTO_APPLY") diff --git a/agent/tests/test_enrichment.py b/agent/tests/test_enrichment.py index aa12d21..d4054a5 100644 --- a/agent/tests/test_enrichment.py +++ b/agent/tests/test_enrichment.py @@ -45,7 +45,7 @@ class Hub: for f in list(fields): if f in self.overridden.get(payload["product_code"], set()): fields[f] = {"status": "HUMAN_OVERRIDDEN", "value": None} - elif payload["current"].get(f): + elif payload["current"].get(f) and not E.is_placeholder(f, payload["current"].get(f)): fields[f] = {"status": "ALREADY_SET", "value": None} return {"state": self.state, "fields": fields, "dataset_versions": {"NCM": {"sha256": "abc"}}} @@ -230,3 +230,68 @@ def test_products_created_while_inactive_are_not_new_when_reactivated(env): add(db, "W2") now[0] += 5 assert worker.step() == ["W2"] + + +# ---- TECHNICAL_PLACEHOLDER (the SGV refuses a product without NCM: operators type 00000000) --------------------------------------------------- +def add_with_ncm(db, code, ncm, name="COCA-COLA ORIGINAL LATA 350 ML"): + db.execute("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,CLASSIFICACAO_FISCAL,OTHER) VALUES(?,?,?,?,?)", (code, name, "7891000100103", ncm, "keep")) + db.commit() + + +def test_placeholder_parity_with_the_hub(): + sys.path.insert(0, str(ROOT)) + from central.app.enrichment import placeholders as hub + assert {k: set(v) for k, v in E.PLACEHOLDERS.items()} == {k: set(v) for k, v in hub.PRODUCT_ENRICHMENT_PLACEHOLDERS.items()} + for value in ("00000000", "0000.00.00", " 00000000 ", "22021000", "99999999", "0", "SEM NCM", "", None): + assert E.is_placeholder("CLASSIFICACAO_FISCAL", value) == hub.is_placeholder("CLASSIFICACAO_FISCAL", value), value + assert not E.is_placeholder("CEST", "0000000") and not E.is_placeholder("IPI", 0) + + +def test_placeholder_ncm_is_replaced_dry_run_first_then_enabled_and_second_run_is_zero(env): + db, hub, store, worker, now, _ = env + hub.features = {"features_configured": True, "enrichment_mode": "DRY_RUN"} + worker._features_at = -999 + worker.step() # baseline + add_with_ncm(db, "P1", "00000000") + now[0] += 5 + worker.step() + assert row(db, "P1")[0] == "00000000" and hub.reports[-1]["fields"][0]["outcome"] == "DRY_RUN" # DRY_RUN: detected, zero writes + assert hub.resolve_calls[-1]["current"]["CLASSIFICACAO_FISCAL"] == "00000000" + hub.features = dict(ENABLED) + worker._features_at = -999 + worker._auth_cache = (None, 0.0) + add_with_ncm(db, "P2", "00000000") + now[0] += 5 + worker.step() + assert row(db, "P2") == ("22021000", "0300700", 0.0, "keep") + first = [f for f in hub.reports[-1]["fields"] if f["field"] == "CLASSIFICACAO_FISCAL"][0] + assert first["outcome"] == "APPLIED" and first["old"] == "00000000" + before = db.total_changes + now[0] += 5 + worker.step() + now[0] += 60 + worker.step() + assert db.total_changes == before # second run: 0 writes + + +def test_real_or_unlisted_ncm_is_never_replaced(env): + db, hub, store, worker, now, _ = env + worker.step() + for code, ncm in (("R1", "22021000"), ("R2", "99999999"), ("R3", "0"), ("R4", "SEM NCM"), ("R5", "19059090")): + add_with_ncm(db, code, ncm) + now[0] += 5 + worker.step() + for code, ncm in (("R1", "22021000"), ("R2", "99999999"), ("R3", "0"), ("R4", "SEM NCM"), ("R5", "19059090")): + assert row(db, code)[0] == ncm, code # EXISTING_REAL_VALUE_PRESERVED, even if the KB says otherwise + + +def test_write_field_guards_the_placeholder_swap(): + db = sqlite3.connect(":memory:") + db.execute("CREATE TABLE C000025(CODIGO TEXT, CLASSIFICACAO_FISCAL TEXT)") + db.executemany("INSERT INTO C000025 VALUES(?,?)", [("a", "00000000"), ("b", "22021000"), ("c", None), ("d", "0000.00.00")]) + db.commit() + assert E.write_field(Conn(db), "a", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED" + assert E.write_field(Conn(db), "b", "CLASSIFICACAO_FISCAL", "19059090")["reason"] == "VALUE_ALREADY_SET_NEVER_OVERWRITTEN" + assert E.write_field(Conn(db), "c", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED" + assert E.write_field(Conn(db), "d", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED" + assert [r[0] for r in db.execute("SELECT CLASSIFICACAO_FISCAL FROM C000025 ORDER BY CODIGO")] == ["19059090", "22021000", "19059090", "19059090"] diff --git a/central/app/enrichment/api.py b/central/app/enrichment/api.py index 3c23e65..ccdbc51 100644 --- a/central/app/enrichment/api.py +++ b/central/app/enrichment/api.py @@ -10,6 +10,7 @@ from sqlalchemy import select from ..blocked_products import upsert_blocker from ..config import settings from ..models import Installation, ProductEnrichment, ProductFieldAudit +from .placeholders import is_placeholder from .providers import build_providers from .quota import ProviderGateway from .resolver import FIELDS, resolve @@ -74,6 +75,9 @@ def router(principal, providers_factory=None): if field in result["fields"]: result["fields"][field] = {"status": "HUMAN_OVERRIDDEN", "value": None, "rule": None, "kb_status": None, "evidence": [], "reason": "valor alterado por uma pessoa; nunca mais preenchido automaticamente"} pending = result["state"] in PENDING_STATES + for name, decision in result["fields"].items(): # explain in the audit trail that a technical placeholder was treated as "not classified yet" + if is_placeholder(name, current.get(name)): + decision["technical_placeholder"] = str(current.get(name)) row.gtin, row.description, row.latency_ms = result.get("gtin"), description, latency row.attempts = (row.attempts or 0) + 1 row.status = "PENDING" if pending else "RESOLVED" diff --git a/central/app/enrichment/placeholders.py b/central/app/enrichment/placeholders.py new file mode 100644 index 0000000..9c6be77 --- /dev/null +++ b/central/app/enrichment/placeholders.py @@ -0,0 +1,27 @@ +"""TECHNICAL_PLACEHOLDER semantics for the product enrichment. + +The SGV refuses to save a product without an NCM, so operators type a technical filler to get past the form. That filler is NOT a human fiscal decision: it means "not classified yet". +Explicit allowlist PER FIELD (never generalised: "0", "99999999", "SEM NCM" ... are NOT placeholders until the real SGV behaviour is confirmed). The Agent carries the same table +(agent/src/gti_fiscal_agent/enrichment.py PLACEHOLDERS); a test asserts both are identical. +""" +import re + +PRODUCT_ENRICHMENT_PLACEHOLDERS = {"CLASSIFICACAO_FISCAL": frozenset({"00000000"})} + + +def _norm(field: str, value) -> str: + text = "" if value is None else str(value).strip() + return re.sub(r"\D", "", text) if field == "CLASSIFICACAO_FISCAL" else text # 0000.00.00 and 00000000 are the same NCM filler + + +def is_placeholder(field: str, value) -> bool: + return _norm(field, value) in PRODUCT_ENRICHMENT_PLACEHOLDERS.get(field, frozenset()) + + +def is_blank(value) -> bool: + return not ("" if value is None else str(value)).strip() + + +def is_writable_empty(field: str, value) -> bool: + """True when the field is empty OR holds the approved technical placeholder. A real value (even a wrong one) is never writable here.""" + return is_blank(value) or is_placeholder(field, value) diff --git a/central/app/enrichment/resolver.py b/central/app/enrichment/resolver.py index de0ad4d..ab5a9a6 100644 --- a/central/app/enrichment/resolver.py +++ b/central/app/enrichment/resolver.py @@ -17,6 +17,7 @@ from ..datasets.service import DatasetService from ..models import Dataset, NcmEntry from . import kb, textnorm from .gtin import normalize_gtin +from .placeholders import is_placeholder, is_writable_empty from .quota import ProviderGateway AUTO, SUGGEST, CONFLICT, INSUFFICIENT, ALREADY, NA = "AUTO_FILL_SAFE", "SUGGESTION_ONLY", "CONFLICT", "INSUFFICIENT_EVIDENCE", "ALREADY_SET", "NOT_APPLICABLE" @@ -75,7 +76,7 @@ def _field(status, value=None, rule=None, kb_status=None, evidence=None, reason= def _empty(current: dict, name: str) -> bool: - return not str(current.get(name) or "").strip() + return is_writable_empty(name, current.get(name)) def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current: dict, description: str, *, name_verified=False) -> dict: @@ -92,7 +93,8 @@ def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current: else: out["CLASSIFICACAO_FISCAL"] = _field(INSUFFICIENT, reason="nenhuma fonte com NCM válido no dataset oficial") ncm = out["CLASSIFICACAO_FISCAL"] - ncm_for_engine = ncm["value"] if ncm["status"] == AUTO else (normalize_code(current.get("CLASSIFICACAO_FISCAL")) or None) + own = current.get("CLASSIFICACAO_FISCAL") + ncm_for_engine = ncm["value"] if ncm["status"] == AUTO else (None if is_placeholder("CLASSIFICACAO_FISCAL", own) else (normalize_code(own) or None)) if not ncm_for_engine or (ncm["status"] == ALREADY and not ncm_is_official(session, ncm_for_engine)): for name in ("CEST", "IPI"): out[name] = _field(ALREADY if not _empty(current, name) else INSUFFICIENT, reason="depende de um NCM oficial válido") diff --git a/central/app/version.py b/central/app/version.py index c508121..18c7a88 100644 --- a/central/app/version.py +++ b/central/app/version.py @@ -4,5 +4,5 @@ kept identical to this) and what an AgentRelease row's `version` column holds once published — never a second, independent version number. Fiscal semantics are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py, DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump.""" -GTI_FISCAL_VERSION = "1.3.3" +GTI_FISCAL_VERSION = "1.3.4" API_COMPAT_MAJOR_MINOR = "1.0" # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x diff --git a/central/pyproject.toml b/central/pyproject.toml index 2222aff..193eff8 100644 --- a/central/pyproject.toml +++ b/central/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "gti-fiscal-hub" -version = "1.3.3" +version = "1.3.4" requires-python = ">=3.10" dependencies = [ "fastapi>=0.115,<1", diff --git a/central/tests/test_health.py b/central/tests/test_health.py index 4eb4d63..9c97b10 100644 --- a/central/tests/test_health.py +++ b/central/tests/test_health.py @@ -17,7 +17,7 @@ def test_health_is_unavailable_when_database_is_down(): def test_version_endpoint_reports_release_and_fiscal_versions(): body = TestClient(create_app(database_probe=lambda: True)).get("/version").json() - assert body["version"] == "1.3.3" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"] + assert body["version"] == "1.3.4" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"] def test_production_refuses_the_dev_token(monkeypatch): diff --git a/central/tests/test_product_enrichment_resolver.py b/central/tests/test_product_enrichment_resolver.py index b9febbc..953d3f5 100644 --- a/central/tests/test_product_enrichment_resolver.py +++ b/central/tests/test_product_enrichment_resolver.py @@ -313,3 +313,15 @@ def test_backfill_respects_operational_reserve(env): outs = [gw.lookup(fake, gtin(f"78970000{i:04d}"), "BACKFILL", 20).state for i in range(10)] assert outs.count("SKIPPED") == 2 and fake.calls == 8 # 20% of 10 kept for new products assert gw.lookup(fake, gtin("789700009999"), "OPERATIONAL").state in ("FOUND", "NOT_FOUND") + + +def test_technical_placeholder_ncm_is_treated_as_not_classified_but_real_values_are_not(env): + p = kb.upsert_product(env, "0" + G1, "COCA-COLA ORIGINAL LATA 350 ML") + for fam in ("PROVIDER:a", "PROVIDER:b"): + kb.add_evidence(env, p, "ncm", "22021000", "x", family=fam) + env.commit() + for value, expected in (("00000000", "AUTO_FILL_SAFE"), ("0000.00.00", "AUTO_FILL_SAFE"), ("22021000", "ALREADY_SET"), ("99999999", "ALREADY_SET"), ("0", "ALREADY_SET"), ("", "AUTO_FILL_SAFE")): + r = run(env, [], G1, "COCA-COLA ORIGINAL LATA 350 ML", current={"CLASSIFICACAO_FISCAL": value}) + assert r["fields"]["CLASSIFICACAO_FISCAL"]["status"] == expected, value + r = run(env, [], G1, "COCA-COLA ORIGINAL LATA 350 ML", current={"CLASSIFICACAO_FISCAL": "00000000"}) + assert r["fields"]["IPI"]["value"] == "0.0" # CEST/IPI follow the NCM that is about to replace the filler diff --git a/deploy/agent-windows/agent.env.template b/deploy/agent-windows/agent.env.template index fcde8a4..d507aec 100644 --- a/deploy/agent-windows/agent.env.template +++ b/deploy/agent-windows/agent.env.template @@ -19,4 +19,4 @@ FB_PASSWORD= FISCAL_MODE=DRY_RUN FIREBIRD_WRITES=0 POLL_INTERVAL_SECONDS=5 -AGENT_VERSION=1.3.3 +AGENT_VERSION=1.3.4 diff --git a/deploy/agent-windows/gti-agent.ps1 b/deploy/agent-windows/gti-agent.ps1 index d375c0d..9f4acd6 100644 --- a/deploy/agent-windows/gti-agent.ps1 +++ b/deploy/agent-windows/gti-agent.ps1 @@ -164,7 +164,7 @@ switch ($Command) { "GTI_HUB_URL=$hub", "AGENT_TOKEN=$token", "AGENT_ID=$agentId", "COMPANY_CODE=$code", "COMPANY_CNPJ=$cnpj", "COMPANY_CRT_SOURCE=$crtSource", "FIREBIRD_DATABASE_PATH=$db", "FB_USER=$fbUser", "FB_PASSWORD=$fbPass", - 'FISCAL_MODE=DRY_RUN', 'FIREBIRD_WRITES=0', 'POLL_INTERVAL_SECONDS=5', 'AGENT_VERSION=1.3.3' + 'FISCAL_MODE=DRY_RUN', 'FIREBIRD_WRITES=0', 'POLL_INTERVAL_SECONDS=5', 'AGENT_VERSION=1.3.4' ) Set-Content -Path $EnvFile -Value $lines -Encoding UTF8 } diff --git a/deploy/agent-windows/pilot-construcao.env.example b/deploy/agent-windows/pilot-construcao.env.example index 2e73d53..595fab1 100644 --- a/deploy/agent-windows/pilot-construcao.env.example +++ b/deploy/agent-windows/pilot-construcao.env.example @@ -19,4 +19,4 @@ FB_PASSWORD= FISCAL_MODE=DRY_RUN FIREBIRD_WRITES=0 POLL_INTERVAL_SECONDS=5 -AGENT_VERSION=1.3.3 +AGENT_VERSION=1.3.4 diff --git a/deploy/agent-windows/profiles/mmmodas.env.example b/deploy/agent-windows/profiles/mmmodas.env.example index 204d4c1..ff37fbe 100644 --- a/deploy/agent-windows/profiles/mmmodas.env.example +++ b/deploy/agent-windows/profiles/mmmodas.env.example @@ -19,4 +19,4 @@ FB_PASSWORD= FISCAL_MODE=DRY_RUN FIREBIRD_WRITES=0 POLL_INTERVAL_SECONDS=5 -AGENT_VERSION=1.3.3 +AGENT_VERSION=1.3.4 diff --git a/deploy/agent-windows/profiles/rincao.env.example b/deploy/agent-windows/profiles/rincao.env.example index 17b4a2b..8cfacbe 100644 --- a/deploy/agent-windows/profiles/rincao.env.example +++ b/deploy/agent-windows/profiles/rincao.env.example @@ -19,4 +19,4 @@ FB_PASSWORD= FISCAL_MODE=DRY_RUN FIREBIRD_WRITES=0 POLL_INTERVAL_SECONDS=5 -AGENT_VERSION=1.3.3 +AGENT_VERSION=1.3.4 diff --git a/deploy/agent-windows/profiles/uniao.env.example b/deploy/agent-windows/profiles/uniao.env.example index 16c91ac..1afb70f 100644 --- a/deploy/agent-windows/profiles/uniao.env.example +++ b/deploy/agent-windows/profiles/uniao.env.example @@ -19,4 +19,4 @@ FB_PASSWORD= FISCAL_MODE=DRY_RUN FIREBIRD_WRITES=0 POLL_INTERVAL_SECONDS=5 -AGENT_VERSION=1.3.3 +AGENT_VERSION=1.3.4 diff --git a/deploy/hub/.env.example b/deploy/hub/.env.example index f098b4a..3c117b9 100644 --- a/deploy/hub/.env.example +++ b/deploy/hub/.env.example @@ -1,5 +1,5 @@ # GTI Fiscal Hub — production environment (copy to .env, never commit .env) -GTI_FISCAL_VERSION=1.3.3 +GTI_FISCAL_VERSION=1.3.4 # REQUIRED. URL-safe secret (used inside a connection URL): openssl rand -hex 24 POSTGRES_PASSWORD= diff --git a/deploy/hub/docker-compose.prod.yml b/deploy/hub/docker-compose.prod.yml index 5ce9759..401587e 100644 --- a/deploy/hub/docker-compose.prod.yml +++ b/deploy/hub/docker-compose.prod.yml @@ -36,7 +36,7 @@ services: build: context: ../.. dockerfile: central/Dockerfile - image: gti-fiscal-hub:${GTI_FISCAL_VERSION:-1.3.3} + image: gti-fiscal-hub:${GTI_FISCAL_VERSION:-1.3.4} restart: unless-stopped depends_on: postgres: diff --git a/docs/product_enrichment.md b/docs/product_enrichment.md index b8e205b..994207e 100644 --- a/docs/product_enrichment.md +++ b/docs/product_enrichment.md @@ -107,3 +107,13 @@ Production KB starts empty (0 products, head 0028 before deploy). The lab databa ### Measured (local data only; `docs/runs/enrichment_history_analysis.json`) See the V2 report in the mission summary; key findings: cross-client NCM agreement is poor (a candidate taken from the other clients matches the held-out client's own NCM only 62 % of the time, and only 42 % even when >= 2 clients corroborate it), which is exactly why client data can never verify an NCM on its own. + + +## TECHNICAL_PLACEHOLDER (1.3.4) +The SGV refuses to save a product without NCM, so operators type a filler. An empty-only rule therefore never fires for that product. Contract: +- Per-field allowlist `PRODUCT_ENRICHMENT_PLACEHOLDERS = {CLASSIFICACAO_FISCAL: {"00000000"}}` (dots ignored: `0000.00.00` is the same filler). `0`, `99999999`, `SEM NCM`... are NOT placeholders (no evidence of SGV behaviour); nothing else is ever overwritten. +- A placeholder is "not classified yet", never `HUMAN_OVERRIDDEN`. A real NCM (even a wrong one) is preserved, whatever the KB says. cClassTrib typed by the operator is never touched (RTC gates own it). +- Where it is decided: the Hub treats a placeholder like an empty field when it builds the per-field decisions (`resolver._empty`, audit note `technical_placeholder`); the Agent must ALSO accept it in the typed write (`write_field`: compare-and-swap `UPDATE ... WHERE TRIM(f)=`), so this needs Agent >= 1.3.4. + Both tables are identical (parity test). The audit records `old = 00000000`. +- Scope reminders: products created while the function was DISABLED / WAITING stay history (re-baseline); REPROCESS_ALL/SYNC do not push products through the enrichment (only NEW products after the baseline do); + a product WITHOUT EAN is resolved by name only from a unique VERIFIED_KB match. diff --git a/docs/runs/enrichment_placeholder_e2e.json b/docs/runs/enrichment_placeholder_e2e.json new file mode 100644 index 0000000..3e1254e --- /dev/null +++ b/docs/runs/enrichment_placeholder_e2e.json @@ -0,0 +1,22 @@ +{ + "copy_only": true, + "checks": { + "PLACEHOLDER_PRODUCT_DETECTED_DRY_RUN": true, + "DRY_RUN_ZERO_WRITES": true, + "DRY_RUN_DECISION_GENERATED_FOR_NCM": true, + "ENABLED_PLACEHOLDER_REPLACED": true, + "CEST_IPI_ONLY_WHEN_SAFE": true, + "CCLASSTRIB_OPERATOR_VALUE_UNTOUCHED": true, + "AUDIT_RECORDS_THE_REPLACED_PLACEHOLDER": true, + "EXISTING_REAL_VALUE_PRESERVED": true, + "UNLISTED_FILLER_NOT_OVERWRITTEN": true, + "SECOND_RUN_WRITES_0": true + }, + "ALL_PASS": true, + "PH002": [ + "22021000", + "0300700", + 0.0, + "200009" + ] +} \ No newline at end of file diff --git a/release/RELEASE_MANIFEST.json b/release/RELEASE_MANIFEST.json index 8c70b45..52107d9 100644 --- a/release/RELEASE_MANIFEST.json +++ b/release/RELEASE_MANIFEST.json @@ -1,14 +1,14 @@ { "artifact": "RELEASE_MANIFEST", - "GTI_FISCAL_VERSION": "1.3.3", + "GTI_FISCAL_VERSION": "1.3.4", "generated_at": "2026-09-25T15:43:40+00:00", "hub": { - "version": "1.3.3", + "version": "1.3.4", "api_compat": "1.0", "engine_version": "0.3.0", "decision_version": "mission-04c-ipi-ex-split" }, - "agent_version": "1.3.3", + "agent_version": "1.3.4", "fiscal_engine": { "ENGINE_VERSION": "0.3.0", "DECISION_VERSION": "mission-04c-ipi-ex-split", @@ -89,5 +89,5 @@ "record_count": 11103 } ], - "note": "Hub-only release 1.3.3: the Windows Agent installer distributed remains GTIFiscalAgent-Setup-1.3.1.exe (no functional Agent change)." + "note": "1.3.4: Agent + Hub. Agent 1.3.4 replaces only an approved TECHNICAL_PLACEHOLDER (NCM 00000000) besides empty fields; real values are never overwritten." } \ No newline at end of file diff --git a/release/VERSION b/release/VERSION index 31e5c84..d0149fe 100644 --- a/release/VERSION +++ b/release/VERSION @@ -1 +1 @@ -1.3.3 +1.3.4 diff --git a/scripts/enrichment_placeholder_e2e.py b/scripts/enrichment_placeholder_e2e.py new file mode 100644 index 0000000..d52a7a6 --- /dev/null +++ b/scripts/enrichment_placeholder_e2e.py @@ -0,0 +1,99 @@ +"""COPY-ONLY E2E of the TECHNICAL_PLACEHOLDER case found by the operator: the SGV refuses a product without NCM, so it was saved with CLASSIFICACAO_FISCAL = '00000000'. + + python scripts/enrichment_placeholder_e2e.py + +Real Firebird COPY + the real Agent worker + the real Hub resolver (in-process, fake providers, no internet). Proves: placeholder detected and classified (DRY_RUN = zero writes), replaced under ENABLED, +CEST/IPI filled only when safe, cClassTrib typed by the operator untouched, a real NCM (or an unlisted filler such as 99999999) preserved even when the KB disagrees, second run = 0 writes. +""" +import json +import sys +import threading +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "scripts")); sys.path.insert(0, str(ROOT / "agent" / "src")) +from dotenv import load_dotenv # noqa: E402 + +load_dotenv(ROOT / ".env") +import enrichment_copy_e2e as ce # noqa: E402 +from gti_fiscal_agent import enrichment as E # noqa: E402 + +base = ce.base + + +def insert(code, ncm, ean, cclasstrib=None, name="COCA-COLA ORIGINAL LATA 350 ML"): + conn = base.connect(True); cur = conn.cursor() + cur.execute("INSERT INTO C000025 (CODIGO, PRODUTO, CODBARRA, CLASSIFICACAO_FISCAL, CCLASSTRIB) VALUES (?, ?, ?, ?, ?)", (code, name, ean, ncm, cclasstrib)) + conn.commit(); conn.close() + + +def row(code): + conn = base.connect(False); cur = conn.cursor() + cur.execute("SELECT CLASSIFICACAO_FISCAL, CEST, IPI, CCLASSTRIB FROM C000025 WHERE CODIGO=?", (code,)) + r = cur.fetchone(); conn.close() + return tuple(None if v is None else (v if isinstance(v, float) else str(v).strip()) for v in r) if r else None + + +def main(): + checks = {} + base.restore_copy() + hub = ce.InProcessHub() + from central.app.models import CestEntry + for entry in hub.session.query(CestEntry).all(): + entry.description = "COCA-COLA ORIGINAL LATA 350 ML" # official CEST text close enough to the product for the (conservative) description match + hub.session.commit() + for suffix in ("", "-wal", "-shm"): + try: + (ROOT / "scratch_enrich" / f"enrich_ph_e2e.db{suffix}").unlink() + except FileNotFoundError: + pass + store = E.EnrichmentStore(ROOT / "scratch_enrich" / "enrich_ph_e2e.db") + worker = E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: base.connect(False), write_conn=lambda: base.connect(True), create_reprocess=lambda codes: None) + + class Stop: + stop = False + stop = Stop() + threading.Thread(target=worker.run_forever, args=(stop,), daemon=True).start() + ean = ce.gtin("789700000001") + hub.features = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "DRY_RUN"} + worker._features_at = -999 + time.sleep(6) # baseline taken while the function is active + insert("PH001", "00000000", ean, cclasstrib="200009") + time.sleep(9) + checks["PLACEHOLDER_PRODUCT_DETECTED_DRY_RUN"] = store.get("PH001") is not None and hub.calls >= 1 + checks["DRY_RUN_ZERO_WRITES"] = row("PH001") == ("00000000", None, None, "200009") + decisions = [it for rep in hub.reports for it in rep["fields"] if rep["product_code"] == "PH001"] + checks["DRY_RUN_DECISION_GENERATED_FOR_NCM"] = any(d["field"] == "CLASSIFICACAO_FISCAL" and d["outcome"] == "DRY_RUN" and d["old"] == "00000000" and d["new"] == "22021000" for d in decisions) + hub.features = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "ENABLED"} + worker._features_at = -999 + worker._auth_cache = (None, 0.0) + insert("PH002", "00000000", ean, cclasstrib="200009") # same real scenario, function already ENABLED + t0 = time.monotonic() + while time.monotonic() - t0 < 25 and row("PH002")[0] == "00000000": + time.sleep(0.2) + time.sleep(4) # remaining fields + audit report follow the NCM write + r2 = row("PH002") + checks["ENABLED_PLACEHOLDER_REPLACED"] = r2[0] == "22021000" + checks["CEST_IPI_ONLY_WHEN_SAFE"] = r2[2] == 0.0 and r2[1] == "0300700" + checks["CCLASSTRIB_OPERATOR_VALUE_UNTOUCHED"] = r2[3] == "200009" + applied = [it for rep in hub.reports for it in rep["fields"] if rep["product_code"] == "PH002" and it["outcome"] == "APPLIED"] + checks["AUDIT_RECORDS_THE_REPLACED_PLACEHOLDER"] = any(a["field"] == "CLASSIFICACAO_FISCAL" and a["old"] == "00000000" for a in applied) + # negative: real NCM / unlisted filler are preserved even though the KB says 22021000 + insert("PH003", "19059090", ean); insert("PH004", "99999999", ean); insert("PH005", "22021000", ean) + time.sleep(12) + checks["EXISTING_REAL_VALUE_PRESERVED"] = row("PH003")[0] == "19059090" and row("PH005")[0] == "22021000" + checks["UNLISTED_FILLER_NOT_OVERWRITTEN"] = row("PH004")[0] == "99999999" + calls_before, writes_before = hub.calls, sum(1 for rep in hub.reports for it in rep["fields"] if it["outcome"] == "APPLIED") + time.sleep(14) + writes_after = sum(1 for rep in hub.reports for it in rep["fields"] if it["outcome"] == "APPLIED") + checks["SECOND_RUN_WRITES_0"] = writes_after == writes_before and row("PH002")[0] == "22021000" + stop.stop = True + result = {"copy_only": True, "checks": checks, "ALL_PASS": all(checks.values()), "PH002": row("PH002")} + (ROOT / "docs" / "runs" / "enrichment_placeholder_e2e.json").write_text(json.dumps(result, indent=1), encoding="utf-8") + print(json.dumps(result, indent=1)) + sys.exit(0 if result["ALL_PASS"] else 1) + + +if __name__ == "__main__": + main() diff --git a/tests/test_release_candidate.py b/tests/test_release_candidate.py index feed52a..c6fcaf1 100644 --- a/tests/test_release_candidate.py +++ b/tests/test_release_candidate.py @@ -14,11 +14,11 @@ sys.path.insert(0, str(ROOT)) def test_versions_are_coherent_across_hub_agent_and_packages(): from central.app.version import GTI_FISCAL_VERSION v = (ROOT / "release" / "VERSION").read_text(encoding="utf-8").strip() - assert v == GTI_FISCAL_VERSION == "1.3.3" + assert v == GTI_FISCAL_VERSION == "1.3.4" pep = v.replace("-rc", "rc") for pp in ("central/pyproject.toml", "agent/pyproject.toml"): assert f'version = "{pep}"' in (ROOT / pp).read_text(encoding="utf-8") - assert '"AGENT_VERSION", "1.3.3"' in (ROOT / "agent" / "scripts" / "run_agent_service.py").read_text(encoding="utf-8") + assert '"AGENT_VERSION", "1.3.4"' in (ROOT / "agent" / "scripts" / "run_agent_service.py").read_text(encoding="utf-8") manifest = ROOT / "release" / "RELEASE_MANIFEST.json" if manifest.exists(): m = json.loads(manifest.read_text(encoding="utf-8"))