feat: TECHNICAL_PLACEHOLDER (NCM 00000000) is replaceable by the product enrichment; real values preserved; Agent 1.3.4

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
gti
2026-09-26 10:50:12 -03:00
co-authored by Claude Sonnet 5
parent 5281c1d79b
commit c68c6c67ed
26 changed files with 290 additions and 31 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
[project]
name = "gti-fiscal-agent"
version = "1.3.3"
version = "1.3.4"
requires-python = ">=3.10"
dependencies = ["httpx>=0.27,<1"]
+3 -3
View File
@@ -107,7 +107,7 @@ class Config:
self.poll_interval_seconds = float(env.get("POLL_INTERVAL_SECONDS", "5"))
# RTC auto-apply is a SEPARATE switch (default OFF): FIREBIRD_WRITES alone never lets the Agent fill cClassTrib.
self.rtc_auto_apply_max_per_job = int(env.get("RTC_AUTO_APPLY_MAX_PER_JOB", str(_rtc_apply.DEFAULT_MAX_PER_JOB)))
self.agent_version = env.get("AGENT_VERSION", "1.3.3")
self.agent_version = env.get("AGENT_VERSION", "1.3.4")
self.schema_fingerprint = env.get("SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1)
self.state_db = env.get("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite"))
# Pilot write authorization: both must be explicitly set by the installer for
@@ -349,7 +349,7 @@ def execute_job(job, config, executor, state, env):
"fiscal_context": fiscal_context,
"rtc_auto_apply": rtc_apply,
"fiscal_mode": getattr(config, "fiscal_mode", None),
"release_version": env.get("AGENT_VERSION") or "1.3.3",
"release_version": env.get("AGENT_VERSION") or "1.3.4",
"products_total": len(products),
"products_changed": len(to_process),
"products_processed": summary["processed"],
@@ -520,7 +520,7 @@ def run(config, env, *, max_iterations=None, stopper=None):
def enforce_startup_preflight(env):
"""Full read-only preflight at startup (Hub, auth, Firebird, capabilities, company context, guards). Fail closed."""
from gti_fiscal_agent.preflight import run_preflight
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.3")
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.4")
logger.info("startup preflight: %s", json.dumps({k: report[k] for k in ("PREFLIGHT", "blockers", "fiscal_mode", "effective_firebird_writes")}, ensure_ascii=False))
print(json.dumps({"PREFLIGHT": report["PREFLIGHT"], "blockers": report["blockers"]}, ensure_ascii=False))
if report["PREFLIGHT"] != "PASS":
+23 -5
View File
@@ -21,7 +21,10 @@ from datetime import datetime, timezone
logger = logging.getLogger("gti_fiscal_agent.enrichment")
CAPABILITY = "PRODUCT_ENRICHMENT_WRITE_V1"
ALLOWED_FIELDS = ("CLASSIFICACAO_FISCAL", "CEST", "IPI") # explicit allowlist; anything else raises before touching SQL
ALLOWED_FIELDS = ("CLASSIFICACAO_FISCAL", "CEST", "IPI")
# TECHNICAL_PLACEHOLDER: the SGV refuses a product without NCM, so operators type a filler. It means "not classified yet" (not a human decision) and may be replaced.
# Explicit per-field allowlist, identical to central/app/enrichment/placeholders.py (parity test). Nothing else is ever overwritten.
PLACEHOLDERS = {"CLASSIFICACAO_FISCAL": frozenset({"00000000"})} # explicit allowlist; anything else raises before touching SQL
READ_FIELDS = ("CODIGO", "PRODUTO", "GTIN", "CODBARRA") + ALLOWED_FIELDS # the SGV operator types the EAN in CODBARRA; GTIN (fiscal) is mostly empty
POLL_SECONDS = 2.0
SAFETY_DIFF_SECONDS = 60.0
@@ -108,6 +111,16 @@ def is_human_overridden(store_path, code, field) -> bool:
# ---- Firebird access (injected: `read_conn()` returns a READ-ONLY connection, `write_conn()` a write-capable one, only ever opened by ENABLED mode) ----------------------------------------
def is_placeholder(field, value) -> bool:
text = _text(value)
text = "".join(ch for ch in text if ch.isdigit()) if field == "CLASSIFICACAO_FISCAL" else text
return text in PLACEHOLDERS.get(field, frozenset())
def writable_now(field, value) -> bool:
return not _text(value) or is_placeholder(field, value)
def _digits(value) -> str:
return "".join(ch for ch in _text(value) if ch.isdigit())
@@ -174,10 +187,14 @@ def write_field(conn, code: str, field: str, value) -> dict:
if len(rows) != 1:
conn.rollback()
return {"outcome": "ABORTED", "reason": "PRODUCT_ROW_NOT_UNIQUE" if rows else "PRODUCT_NOT_FOUND"}
if _text(rows[0][0]):
raw = rows[0][0]
if not writable_now(field, raw):
conn.rollback()
return {"outcome": "ABORTED", "reason": "VALUE_ALREADY_SET_NEVER_OVERWRITTEN"}
cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND ({field} IS NULL OR TRIM({field}) = '')", (value, code))
if _text(raw): # technical placeholder: compare-and-swap on the exact value just read
cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND TRIM({field}) = ?", (value, code, _text(raw)))
else:
cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND ({field} IS NULL OR TRIM({field}) = '')", (value, code))
if cur.rowcount != 1:
conn.rollback()
return {"outcome": "ABORTED", "reason": "UPDATE_ROWCOUNT_NOT_ONE"}
@@ -189,7 +206,7 @@ def write_field(conn, code: str, field: str, value) -> dict:
same = (float(after) == value) if field == "IPI" and after is not None else _text(after) == value
if not same:
return {"outcome": "FAILED", "reason": "POST_WRITE_VERIFICATION_FAILED"}
return {"outcome": "APPLIED", "value": value, "rowcount": 1}
return {"outcome": "APPLIED", "value": value, "rowcount": 1, "replaced_placeholder": _text(raw) if _text(raw) else None}
except Exception as error: # noqa: BLE001
try:
conn.rollback()
@@ -345,7 +362,8 @@ class EnrichmentWorker:
for field in ALLOWED_FIELDS:
decision = fields.get(field) or {}
status, value = decision.get("status"), decision.get("value")
base = {"field": field, "rule": decision.get("rule"), "kb_status": decision.get("kb_status"), "evidence": decision.get("evidence") or [], "old": None, "new": value}
placeholder = _text(current.get(field)) if is_placeholder(field, current.get(field)) else None
base = {"field": field, "rule": decision.get("rule"), "kb_status": decision.get("kb_status"), "evidence": decision.get("evidence") or [], "old": placeholder, "new": value}
if status == "HUMAN_OVERRIDDEN" or overridden.get(field):
reports.append({**base, "outcome": "HUMAN_OVERRIDDEN", "reason": "HUMAN_OVERRIDDEN", "new": None})
continue
+1 -1
View File
@@ -9,7 +9,7 @@ import os
import sqlite3
from pathlib import Path
AGENT_VERSION_DEFAULT = "1.3.3"
AGENT_VERSION_DEFAULT = "1.3.4"
REQUIRED_ENV = ("GTI_HUB_URL", "AGENT_TOKEN", "FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")
REQUIRED_CAPABILITIES = ("supports_ncm", "supports_ipi", "supports_cest")
FISCAL_MODES = ("DRY_RUN", "AUTO_APPLY")
+66 -1
View File
@@ -45,7 +45,7 @@ class Hub:
for f in list(fields):
if f in self.overridden.get(payload["product_code"], set()):
fields[f] = {"status": "HUMAN_OVERRIDDEN", "value": None}
elif payload["current"].get(f):
elif payload["current"].get(f) and not E.is_placeholder(f, payload["current"].get(f)):
fields[f] = {"status": "ALREADY_SET", "value": None}
return {"state": self.state, "fields": fields, "dataset_versions": {"NCM": {"sha256": "abc"}}}
@@ -230,3 +230,68 @@ def test_products_created_while_inactive_are_not_new_when_reactivated(env):
add(db, "W2")
now[0] += 5
assert worker.step() == ["W2"]
# ---- TECHNICAL_PLACEHOLDER (the SGV refuses a product without NCM: operators type 00000000) ---------------------------------------------------
def add_with_ncm(db, code, ncm, name="COCA-COLA ORIGINAL LATA 350 ML"):
db.execute("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,CLASSIFICACAO_FISCAL,OTHER) VALUES(?,?,?,?,?)", (code, name, "7891000100103", ncm, "keep"))
db.commit()
def test_placeholder_parity_with_the_hub():
sys.path.insert(0, str(ROOT))
from central.app.enrichment import placeholders as hub
assert {k: set(v) for k, v in E.PLACEHOLDERS.items()} == {k: set(v) for k, v in hub.PRODUCT_ENRICHMENT_PLACEHOLDERS.items()}
for value in ("00000000", "0000.00.00", " 00000000 ", "22021000", "99999999", "0", "SEM NCM", "", None):
assert E.is_placeholder("CLASSIFICACAO_FISCAL", value) == hub.is_placeholder("CLASSIFICACAO_FISCAL", value), value
assert not E.is_placeholder("CEST", "0000000") and not E.is_placeholder("IPI", 0)
def test_placeholder_ncm_is_replaced_dry_run_first_then_enabled_and_second_run_is_zero(env):
db, hub, store, worker, now, _ = env
hub.features = {"features_configured": True, "enrichment_mode": "DRY_RUN"}
worker._features_at = -999
worker.step() # baseline
add_with_ncm(db, "P1", "00000000")
now[0] += 5
worker.step()
assert row(db, "P1")[0] == "00000000" and hub.reports[-1]["fields"][0]["outcome"] == "DRY_RUN" # DRY_RUN: detected, zero writes
assert hub.resolve_calls[-1]["current"]["CLASSIFICACAO_FISCAL"] == "00000000"
hub.features = dict(ENABLED)
worker._features_at = -999
worker._auth_cache = (None, 0.0)
add_with_ncm(db, "P2", "00000000")
now[0] += 5
worker.step()
assert row(db, "P2") == ("22021000", "0300700", 0.0, "keep")
first = [f for f in hub.reports[-1]["fields"] if f["field"] == "CLASSIFICACAO_FISCAL"][0]
assert first["outcome"] == "APPLIED" and first["old"] == "00000000"
before = db.total_changes
now[0] += 5
worker.step()
now[0] += 60
worker.step()
assert db.total_changes == before # second run: 0 writes
def test_real_or_unlisted_ncm_is_never_replaced(env):
db, hub, store, worker, now, _ = env
worker.step()
for code, ncm in (("R1", "22021000"), ("R2", "99999999"), ("R3", "0"), ("R4", "SEM NCM"), ("R5", "19059090")):
add_with_ncm(db, code, ncm)
now[0] += 5
worker.step()
for code, ncm in (("R1", "22021000"), ("R2", "99999999"), ("R3", "0"), ("R4", "SEM NCM"), ("R5", "19059090")):
assert row(db, code)[0] == ncm, code # EXISTING_REAL_VALUE_PRESERVED, even if the KB says otherwise
def test_write_field_guards_the_placeholder_swap():
db = sqlite3.connect(":memory:")
db.execute("CREATE TABLE C000025(CODIGO TEXT, CLASSIFICACAO_FISCAL TEXT)")
db.executemany("INSERT INTO C000025 VALUES(?,?)", [("a", "00000000"), ("b", "22021000"), ("c", None), ("d", "0000.00.00")])
db.commit()
assert E.write_field(Conn(db), "a", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
assert E.write_field(Conn(db), "b", "CLASSIFICACAO_FISCAL", "19059090")["reason"] == "VALUE_ALREADY_SET_NEVER_OVERWRITTEN"
assert E.write_field(Conn(db), "c", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
assert E.write_field(Conn(db), "d", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
assert [r[0] for r in db.execute("SELECT CLASSIFICACAO_FISCAL FROM C000025 ORDER BY CODIGO")] == ["19059090", "22021000", "19059090", "19059090"]
+4
View File
@@ -10,6 +10,7 @@ from sqlalchemy import select
from ..blocked_products import upsert_blocker
from ..config import settings
from ..models import Installation, ProductEnrichment, ProductFieldAudit
from .placeholders import is_placeholder
from .providers import build_providers
from .quota import ProviderGateway
from .resolver import FIELDS, resolve
@@ -74,6 +75,9 @@ def router(principal, providers_factory=None):
if field in result["fields"]:
result["fields"][field] = {"status": "HUMAN_OVERRIDDEN", "value": None, "rule": None, "kb_status": None, "evidence": [], "reason": "valor alterado por uma pessoa; nunca mais preenchido automaticamente"}
pending = result["state"] in PENDING_STATES
for name, decision in result["fields"].items(): # explain in the audit trail that a technical placeholder was treated as "not classified yet"
if is_placeholder(name, current.get(name)):
decision["technical_placeholder"] = str(current.get(name))
row.gtin, row.description, row.latency_ms = result.get("gtin"), description, latency
row.attempts = (row.attempts or 0) + 1
row.status = "PENDING" if pending else "RESOLVED"
+27
View File
@@ -0,0 +1,27 @@
"""TECHNICAL_PLACEHOLDER semantics for the product enrichment.
The SGV refuses to save a product without an NCM, so operators type a technical filler to get past the form. That filler is NOT a human fiscal decision: it means "not classified yet".
Explicit allowlist PER FIELD (never generalised: "0", "99999999", "SEM NCM" ... are NOT placeholders until the real SGV behaviour is confirmed). The Agent carries the same table
(agent/src/gti_fiscal_agent/enrichment.py PLACEHOLDERS); a test asserts both are identical.
"""
import re
PRODUCT_ENRICHMENT_PLACEHOLDERS = {"CLASSIFICACAO_FISCAL": frozenset({"00000000"})}
def _norm(field: str, value) -> str:
text = "" if value is None else str(value).strip()
return re.sub(r"\D", "", text) if field == "CLASSIFICACAO_FISCAL" else text # 0000.00.00 and 00000000 are the same NCM filler
def is_placeholder(field: str, value) -> bool:
return _norm(field, value) in PRODUCT_ENRICHMENT_PLACEHOLDERS.get(field, frozenset())
def is_blank(value) -> bool:
return not ("" if value is None else str(value)).strip()
def is_writable_empty(field: str, value) -> bool:
"""True when the field is empty OR holds the approved technical placeholder. A real value (even a wrong one) is never writable here."""
return is_blank(value) or is_placeholder(field, value)
+4 -2
View File
@@ -17,6 +17,7 @@ from ..datasets.service import DatasetService
from ..models import Dataset, NcmEntry
from . import kb, textnorm
from .gtin import normalize_gtin
from .placeholders import is_placeholder, is_writable_empty
from .quota import ProviderGateway
AUTO, SUGGEST, CONFLICT, INSUFFICIENT, ALREADY, NA = "AUTO_FILL_SAFE", "SUGGESTION_ONLY", "CONFLICT", "INSUFFICIENT_EVIDENCE", "ALREADY_SET", "NOT_APPLICABLE"
@@ -75,7 +76,7 @@ def _field(status, value=None, rule=None, kb_status=None, evidence=None, reason=
def _empty(current: dict, name: str) -> bool:
return not str(current.get(name) or "").strip()
return is_writable_empty(name, current.get(name))
def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current: dict, description: str, *, name_verified=False) -> dict:
@@ -92,7 +93,8 @@ def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current:
else:
out["CLASSIFICACAO_FISCAL"] = _field(INSUFFICIENT, reason="nenhuma fonte com NCM válido no dataset oficial")
ncm = out["CLASSIFICACAO_FISCAL"]
ncm_for_engine = ncm["value"] if ncm["status"] == AUTO else (normalize_code(current.get("CLASSIFICACAO_FISCAL")) or None)
own = current.get("CLASSIFICACAO_FISCAL")
ncm_for_engine = ncm["value"] if ncm["status"] == AUTO else (None if is_placeholder("CLASSIFICACAO_FISCAL", own) else (normalize_code(own) or None))
if not ncm_for_engine or (ncm["status"] == ALREADY and not ncm_is_official(session, ncm_for_engine)):
for name in ("CEST", "IPI"):
out[name] = _field(ALREADY if not _empty(current, name) else INSUFFICIENT, reason="depende de um NCM oficial válido")
+1 -1
View File
@@ -4,5 +4,5 @@ kept identical to this) and what an AgentRelease row's `version` column holds
once published — never a second, independent version number. Fiscal semantics
are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py,
DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump."""
GTI_FISCAL_VERSION = "1.3.3"
GTI_FISCAL_VERSION = "1.3.4"
API_COMPAT_MAJOR_MINOR = "1.0" # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x
+1 -1
View File
@@ -1,6 +1,6 @@
[project]
name = "gti-fiscal-hub"
version = "1.3.3"
version = "1.3.4"
requires-python = ">=3.10"
dependencies = [
"fastapi>=0.115,<1",
+1 -1
View File
@@ -17,7 +17,7 @@ def test_health_is_unavailable_when_database_is_down():
def test_version_endpoint_reports_release_and_fiscal_versions():
body = TestClient(create_app(database_probe=lambda: True)).get("/version").json()
assert body["version"] == "1.3.3" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
assert body["version"] == "1.3.4" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
def test_production_refuses_the_dev_token(monkeypatch):
@@ -313,3 +313,15 @@ def test_backfill_respects_operational_reserve(env):
outs = [gw.lookup(fake, gtin(f"78970000{i:04d}"), "BACKFILL", 20).state for i in range(10)]
assert outs.count("SKIPPED") == 2 and fake.calls == 8 # 20% of 10 kept for new products
assert gw.lookup(fake, gtin("789700009999"), "OPERATIONAL").state in ("FOUND", "NOT_FOUND")
def test_technical_placeholder_ncm_is_treated_as_not_classified_but_real_values_are_not(env):
p = kb.upsert_product(env, "0" + G1, "COCA-COLA ORIGINAL LATA 350 ML")
for fam in ("PROVIDER:a", "PROVIDER:b"):
kb.add_evidence(env, p, "ncm", "22021000", "x", family=fam)
env.commit()
for value, expected in (("00000000", "AUTO_FILL_SAFE"), ("0000.00.00", "AUTO_FILL_SAFE"), ("22021000", "ALREADY_SET"), ("99999999", "ALREADY_SET"), ("0", "ALREADY_SET"), ("", "AUTO_FILL_SAFE")):
r = run(env, [], G1, "COCA-COLA ORIGINAL LATA 350 ML", current={"CLASSIFICACAO_FISCAL": value})
assert r["fields"]["CLASSIFICACAO_FISCAL"]["status"] == expected, value
r = run(env, [], G1, "COCA-COLA ORIGINAL LATA 350 ML", current={"CLASSIFICACAO_FISCAL": "00000000"})
assert r["fields"]["IPI"]["value"] == "0.0" # CEST/IPI follow the NCM that is about to replace the filler
+1 -1
View File
@@ -19,4 +19,4 @@ FB_PASSWORD=
FISCAL_MODE=DRY_RUN
FIREBIRD_WRITES=0
POLL_INTERVAL_SECONDS=5
AGENT_VERSION=1.3.3
AGENT_VERSION=1.3.4
+1 -1
View File
@@ -164,7 +164,7 @@ switch ($Command) {
"GTI_HUB_URL=$hub", "AGENT_TOKEN=$token", "AGENT_ID=$agentId",
"COMPANY_CODE=$code", "COMPANY_CNPJ=$cnpj", "COMPANY_CRT_SOURCE=$crtSource",
"FIREBIRD_DATABASE_PATH=$db", "FB_USER=$fbUser", "FB_PASSWORD=$fbPass",
'FISCAL_MODE=DRY_RUN', 'FIREBIRD_WRITES=0', 'POLL_INTERVAL_SECONDS=5', 'AGENT_VERSION=1.3.3'
'FISCAL_MODE=DRY_RUN', 'FIREBIRD_WRITES=0', 'POLL_INTERVAL_SECONDS=5', 'AGENT_VERSION=1.3.4'
)
Set-Content -Path $EnvFile -Value $lines -Encoding UTF8
}
@@ -19,4 +19,4 @@ FB_PASSWORD=
FISCAL_MODE=DRY_RUN
FIREBIRD_WRITES=0
POLL_INTERVAL_SECONDS=5
AGENT_VERSION=1.3.3
AGENT_VERSION=1.3.4
@@ -19,4 +19,4 @@ FB_PASSWORD=
FISCAL_MODE=DRY_RUN
FIREBIRD_WRITES=0
POLL_INTERVAL_SECONDS=5
AGENT_VERSION=1.3.3
AGENT_VERSION=1.3.4
@@ -19,4 +19,4 @@ FB_PASSWORD=
FISCAL_MODE=DRY_RUN
FIREBIRD_WRITES=0
POLL_INTERVAL_SECONDS=5
AGENT_VERSION=1.3.3
AGENT_VERSION=1.3.4
@@ -19,4 +19,4 @@ FB_PASSWORD=
FISCAL_MODE=DRY_RUN
FIREBIRD_WRITES=0
POLL_INTERVAL_SECONDS=5
AGENT_VERSION=1.3.3
AGENT_VERSION=1.3.4
+1 -1
View File
@@ -1,5 +1,5 @@
# GTI Fiscal Hub — production environment (copy to .env, never commit .env)
GTI_FISCAL_VERSION=1.3.3
GTI_FISCAL_VERSION=1.3.4
# REQUIRED. URL-safe secret (used inside a connection URL): openssl rand -hex 24
POSTGRES_PASSWORD=
+1 -1
View File
@@ -36,7 +36,7 @@ services:
build:
context: ../..
dockerfile: central/Dockerfile
image: gti-fiscal-hub:${GTI_FISCAL_VERSION:-1.3.3}
image: gti-fiscal-hub:${GTI_FISCAL_VERSION:-1.3.4}
restart: unless-stopped
depends_on:
postgres:
+10
View File
@@ -107,3 +107,13 @@ Production KB starts empty (0 products, head 0028 before deploy). The lab databa
### Measured (local data only; `docs/runs/enrichment_history_analysis.json`)
See the V2 report in the mission summary; key findings: cross-client NCM agreement is poor (a candidate taken from the other clients matches the held-out client's own NCM only 62 % of the time, and only 42 % even when >= 2 clients corroborate it), which is exactly why client data can never verify an NCM on its own.
## TECHNICAL_PLACEHOLDER (1.3.4)
The SGV refuses to save a product without NCM, so operators type a filler. An empty-only rule therefore never fires for that product. Contract:
- Per-field allowlist `PRODUCT_ENRICHMENT_PLACEHOLDERS = {CLASSIFICACAO_FISCAL: {"00000000"}}` (dots ignored: `0000.00.00` is the same filler). `0`, `99999999`, `SEM NCM`... are NOT placeholders (no evidence of SGV behaviour); nothing else is ever overwritten.
- A placeholder is "not classified yet", never `HUMAN_OVERRIDDEN`. A real NCM (even a wrong one) is preserved, whatever the KB says. cClassTrib typed by the operator is never touched (RTC gates own it).
- Where it is decided: the Hub treats a placeholder like an empty field when it builds the per-field decisions (`resolver._empty`, audit note `technical_placeholder`); the Agent must ALSO accept it in the typed write (`write_field`: compare-and-swap `UPDATE ... WHERE TRIM(f)=<exact value just read>`), so this needs Agent >= 1.3.4.
Both tables are identical (parity test). The audit records `old = 00000000`.
- Scope reminders: products created while the function was DISABLED / WAITING stay history (re-baseline); REPROCESS_ALL/SYNC do not push products through the enrichment (only NEW products after the baseline do);
a product WITHOUT EAN is resolved by name only from a unique VERIFIED_KB match.
+22
View File
@@ -0,0 +1,22 @@
{
"copy_only": true,
"checks": {
"PLACEHOLDER_PRODUCT_DETECTED_DRY_RUN": true,
"DRY_RUN_ZERO_WRITES": true,
"DRY_RUN_DECISION_GENERATED_FOR_NCM": true,
"ENABLED_PLACEHOLDER_REPLACED": true,
"CEST_IPI_ONLY_WHEN_SAFE": true,
"CCLASSTRIB_OPERATOR_VALUE_UNTOUCHED": true,
"AUDIT_RECORDS_THE_REPLACED_PLACEHOLDER": true,
"EXISTING_REAL_VALUE_PRESERVED": true,
"UNLISTED_FILLER_NOT_OVERWRITTEN": true,
"SECOND_RUN_WRITES_0": true
},
"ALL_PASS": true,
"PH002": [
"22021000",
"0300700",
0.0,
"200009"
]
}
+4 -4
View File
@@ -1,14 +1,14 @@
{
"artifact": "RELEASE_MANIFEST",
"GTI_FISCAL_VERSION": "1.3.3",
"GTI_FISCAL_VERSION": "1.3.4",
"generated_at": "2026-09-25T15:43:40+00:00",
"hub": {
"version": "1.3.3",
"version": "1.3.4",
"api_compat": "1.0",
"engine_version": "0.3.0",
"decision_version": "mission-04c-ipi-ex-split"
},
"agent_version": "1.3.3",
"agent_version": "1.3.4",
"fiscal_engine": {
"ENGINE_VERSION": "0.3.0",
"DECISION_VERSION": "mission-04c-ipi-ex-split",
@@ -89,5 +89,5 @@
"record_count": 11103
}
],
"note": "Hub-only release 1.3.3: the Windows Agent installer distributed remains GTIFiscalAgent-Setup-1.3.1.exe (no functional Agent change)."
"note": "1.3.4: Agent + Hub. Agent 1.3.4 replaces only an approved TECHNICAL_PLACEHOLDER (NCM 00000000) besides empty fields; real values are never overwritten."
}
+1 -1
View File
@@ -1 +1 @@
1.3.3
1.3.4
+99
View File
@@ -0,0 +1,99 @@
"""COPY-ONLY E2E of the TECHNICAL_PLACEHOLDER case found by the operator: the SGV refuses a product without NCM, so it was saved with CLASSIFICACAO_FISCAL = '00000000'.
python scripts/enrichment_placeholder_e2e.py
Real Firebird COPY + the real Agent worker + the real Hub resolver (in-process, fake providers, no internet). Proves: placeholder detected and classified (DRY_RUN = zero writes), replaced under ENABLED,
CEST/IPI filled only when safe, cClassTrib typed by the operator untouched, a real NCM (or an unlisted filler such as 99999999) preserved even when the KB disagrees, second run = 0 writes.
"""
import json
import sys
import threading
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "scripts")); sys.path.insert(0, str(ROOT / "agent" / "src"))
from dotenv import load_dotenv # noqa: E402
load_dotenv(ROOT / ".env")
import enrichment_copy_e2e as ce # noqa: E402
from gti_fiscal_agent import enrichment as E # noqa: E402
base = ce.base
def insert(code, ncm, ean, cclasstrib=None, name="COCA-COLA ORIGINAL LATA 350 ML"):
conn = base.connect(True); cur = conn.cursor()
cur.execute("INSERT INTO C000025 (CODIGO, PRODUTO, CODBARRA, CLASSIFICACAO_FISCAL, CCLASSTRIB) VALUES (?, ?, ?, ?, ?)", (code, name, ean, ncm, cclasstrib))
conn.commit(); conn.close()
def row(code):
conn = base.connect(False); cur = conn.cursor()
cur.execute("SELECT CLASSIFICACAO_FISCAL, CEST, IPI, CCLASSTRIB FROM C000025 WHERE CODIGO=?", (code,))
r = cur.fetchone(); conn.close()
return tuple(None if v is None else (v if isinstance(v, float) else str(v).strip()) for v in r) if r else None
def main():
checks = {}
base.restore_copy()
hub = ce.InProcessHub()
from central.app.models import CestEntry
for entry in hub.session.query(CestEntry).all():
entry.description = "COCA-COLA ORIGINAL LATA 350 ML" # official CEST text close enough to the product for the (conservative) description match
hub.session.commit()
for suffix in ("", "-wal", "-shm"):
try:
(ROOT / "scratch_enrich" / f"enrich_ph_e2e.db{suffix}").unlink()
except FileNotFoundError:
pass
store = E.EnrichmentStore(ROOT / "scratch_enrich" / "enrich_ph_e2e.db")
worker = E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: base.connect(False), write_conn=lambda: base.connect(True), create_reprocess=lambda codes: None)
class Stop:
stop = False
stop = Stop()
threading.Thread(target=worker.run_forever, args=(stop,), daemon=True).start()
ean = ce.gtin("789700000001")
hub.features = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "DRY_RUN"}
worker._features_at = -999
time.sleep(6) # baseline taken while the function is active
insert("PH001", "00000000", ean, cclasstrib="200009")
time.sleep(9)
checks["PLACEHOLDER_PRODUCT_DETECTED_DRY_RUN"] = store.get("PH001") is not None and hub.calls >= 1
checks["DRY_RUN_ZERO_WRITES"] = row("PH001") == ("00000000", None, None, "200009")
decisions = [it for rep in hub.reports for it in rep["fields"] if rep["product_code"] == "PH001"]
checks["DRY_RUN_DECISION_GENERATED_FOR_NCM"] = any(d["field"] == "CLASSIFICACAO_FISCAL" and d["outcome"] == "DRY_RUN" and d["old"] == "00000000" and d["new"] == "22021000" for d in decisions)
hub.features = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "ENABLED"}
worker._features_at = -999
worker._auth_cache = (None, 0.0)
insert("PH002", "00000000", ean, cclasstrib="200009") # same real scenario, function already ENABLED
t0 = time.monotonic()
while time.monotonic() - t0 < 25 and row("PH002")[0] == "00000000":
time.sleep(0.2)
time.sleep(4) # remaining fields + audit report follow the NCM write
r2 = row("PH002")
checks["ENABLED_PLACEHOLDER_REPLACED"] = r2[0] == "22021000"
checks["CEST_IPI_ONLY_WHEN_SAFE"] = r2[2] == 0.0 and r2[1] == "0300700"
checks["CCLASSTRIB_OPERATOR_VALUE_UNTOUCHED"] = r2[3] == "200009"
applied = [it for rep in hub.reports for it in rep["fields"] if rep["product_code"] == "PH002" and it["outcome"] == "APPLIED"]
checks["AUDIT_RECORDS_THE_REPLACED_PLACEHOLDER"] = any(a["field"] == "CLASSIFICACAO_FISCAL" and a["old"] == "00000000" for a in applied)
# negative: real NCM / unlisted filler are preserved even though the KB says 22021000
insert("PH003", "19059090", ean); insert("PH004", "99999999", ean); insert("PH005", "22021000", ean)
time.sleep(12)
checks["EXISTING_REAL_VALUE_PRESERVED"] = row("PH003")[0] == "19059090" and row("PH005")[0] == "22021000"
checks["UNLISTED_FILLER_NOT_OVERWRITTEN"] = row("PH004")[0] == "99999999"
calls_before, writes_before = hub.calls, sum(1 for rep in hub.reports for it in rep["fields"] if it["outcome"] == "APPLIED")
time.sleep(14)
writes_after = sum(1 for rep in hub.reports for it in rep["fields"] if it["outcome"] == "APPLIED")
checks["SECOND_RUN_WRITES_0"] = writes_after == writes_before and row("PH002")[0] == "22021000"
stop.stop = True
result = {"copy_only": True, "checks": checks, "ALL_PASS": all(checks.values()), "PH002": row("PH002")}
(ROOT / "docs" / "runs" / "enrichment_placeholder_e2e.json").write_text(json.dumps(result, indent=1), encoding="utf-8")
print(json.dumps(result, indent=1))
sys.exit(0 if result["ALL_PASS"] else 1)
if __name__ == "__main__":
main()
+2 -2
View File
@@ -14,11 +14,11 @@ sys.path.insert(0, str(ROOT))
def test_versions_are_coherent_across_hub_agent_and_packages():
from central.app.version import GTI_FISCAL_VERSION
v = (ROOT / "release" / "VERSION").read_text(encoding="utf-8").strip()
assert v == GTI_FISCAL_VERSION == "1.3.3"
assert v == GTI_FISCAL_VERSION == "1.3.4"
pep = v.replace("-rc", "rc")
for pp in ("central/pyproject.toml", "agent/pyproject.toml"):
assert f'version = "{pep}"' in (ROOT / pp).read_text(encoding="utf-8")
assert '"AGENT_VERSION", "1.3.3"' in (ROOT / "agent" / "scripts" / "run_agent_service.py").read_text(encoding="utf-8")
assert '"AGENT_VERSION", "1.3.4"' in (ROOT / "agent" / "scripts" / "run_agent_service.py").read_text(encoding="utf-8")
manifest = ROOT / "release" / "RELEASE_MANIFEST.json"
if manifest.exists():
m = json.loads(manifest.read_text(encoding="utf-8"))