feat: TECHNICAL_PLACEHOLDER (NCM 00000000) is replaceable by the product enrichment; real values preserved; Agent 1.3.4
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "gti-fiscal-agent"
|
||||
version = "1.3.3"
|
||||
version = "1.3.4"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = ["httpx>=0.27,<1"]
|
||||
|
||||
|
||||
@@ -107,7 +107,7 @@ class Config:
|
||||
self.poll_interval_seconds = float(env.get("POLL_INTERVAL_SECONDS", "5"))
|
||||
# RTC auto-apply is a SEPARATE switch (default OFF): FIREBIRD_WRITES alone never lets the Agent fill cClassTrib.
|
||||
self.rtc_auto_apply_max_per_job = int(env.get("RTC_AUTO_APPLY_MAX_PER_JOB", str(_rtc_apply.DEFAULT_MAX_PER_JOB)))
|
||||
self.agent_version = env.get("AGENT_VERSION", "1.3.3")
|
||||
self.agent_version = env.get("AGENT_VERSION", "1.3.4")
|
||||
self.schema_fingerprint = env.get("SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1)
|
||||
self.state_db = env.get("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite"))
|
||||
# Pilot write authorization: both must be explicitly set by the installer for
|
||||
@@ -349,7 +349,7 @@ def execute_job(job, config, executor, state, env):
|
||||
"fiscal_context": fiscal_context,
|
||||
"rtc_auto_apply": rtc_apply,
|
||||
"fiscal_mode": getattr(config, "fiscal_mode", None),
|
||||
"release_version": env.get("AGENT_VERSION") or "1.3.3",
|
||||
"release_version": env.get("AGENT_VERSION") or "1.3.4",
|
||||
"products_total": len(products),
|
||||
"products_changed": len(to_process),
|
||||
"products_processed": summary["processed"],
|
||||
@@ -520,7 +520,7 @@ def run(config, env, *, max_iterations=None, stopper=None):
|
||||
def enforce_startup_preflight(env):
|
||||
"""Full read-only preflight at startup (Hub, auth, Firebird, capabilities, company context, guards). Fail closed."""
|
||||
from gti_fiscal_agent.preflight import run_preflight
|
||||
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.3")
|
||||
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.4")
|
||||
logger.info("startup preflight: %s", json.dumps({k: report[k] for k in ("PREFLIGHT", "blockers", "fiscal_mode", "effective_firebird_writes")}, ensure_ascii=False))
|
||||
print(json.dumps({"PREFLIGHT": report["PREFLIGHT"], "blockers": report["blockers"]}, ensure_ascii=False))
|
||||
if report["PREFLIGHT"] != "PASS":
|
||||
|
||||
@@ -21,7 +21,10 @@ from datetime import datetime, timezone
|
||||
logger = logging.getLogger("gti_fiscal_agent.enrichment")
|
||||
|
||||
CAPABILITY = "PRODUCT_ENRICHMENT_WRITE_V1"
|
||||
ALLOWED_FIELDS = ("CLASSIFICACAO_FISCAL", "CEST", "IPI") # explicit allowlist; anything else raises before touching SQL
|
||||
ALLOWED_FIELDS = ("CLASSIFICACAO_FISCAL", "CEST", "IPI")
|
||||
# TECHNICAL_PLACEHOLDER: the SGV refuses a product without NCM, so operators type a filler. It means "not classified yet" (not a human decision) and may be replaced.
|
||||
# Explicit per-field allowlist, identical to central/app/enrichment/placeholders.py (parity test). Nothing else is ever overwritten.
|
||||
PLACEHOLDERS = {"CLASSIFICACAO_FISCAL": frozenset({"00000000"})} # explicit allowlist; anything else raises before touching SQL
|
||||
READ_FIELDS = ("CODIGO", "PRODUTO", "GTIN", "CODBARRA") + ALLOWED_FIELDS # the SGV operator types the EAN in CODBARRA; GTIN (fiscal) is mostly empty
|
||||
POLL_SECONDS = 2.0
|
||||
SAFETY_DIFF_SECONDS = 60.0
|
||||
@@ -108,6 +111,16 @@ def is_human_overridden(store_path, code, field) -> bool:
|
||||
|
||||
|
||||
# ---- Firebird access (injected: `read_conn()` returns a READ-ONLY connection, `write_conn()` a write-capable one, only ever opened by ENABLED mode) ----------------------------------------
|
||||
def is_placeholder(field, value) -> bool:
|
||||
text = _text(value)
|
||||
text = "".join(ch for ch in text if ch.isdigit()) if field == "CLASSIFICACAO_FISCAL" else text
|
||||
return text in PLACEHOLDERS.get(field, frozenset())
|
||||
|
||||
|
||||
def writable_now(field, value) -> bool:
|
||||
return not _text(value) or is_placeholder(field, value)
|
||||
|
||||
|
||||
def _digits(value) -> str:
|
||||
return "".join(ch for ch in _text(value) if ch.isdigit())
|
||||
|
||||
@@ -174,10 +187,14 @@ def write_field(conn, code: str, field: str, value) -> dict:
|
||||
if len(rows) != 1:
|
||||
conn.rollback()
|
||||
return {"outcome": "ABORTED", "reason": "PRODUCT_ROW_NOT_UNIQUE" if rows else "PRODUCT_NOT_FOUND"}
|
||||
if _text(rows[0][0]):
|
||||
raw = rows[0][0]
|
||||
if not writable_now(field, raw):
|
||||
conn.rollback()
|
||||
return {"outcome": "ABORTED", "reason": "VALUE_ALREADY_SET_NEVER_OVERWRITTEN"}
|
||||
cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND ({field} IS NULL OR TRIM({field}) = '')", (value, code))
|
||||
if _text(raw): # technical placeholder: compare-and-swap on the exact value just read
|
||||
cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND TRIM({field}) = ?", (value, code, _text(raw)))
|
||||
else:
|
||||
cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND ({field} IS NULL OR TRIM({field}) = '')", (value, code))
|
||||
if cur.rowcount != 1:
|
||||
conn.rollback()
|
||||
return {"outcome": "ABORTED", "reason": "UPDATE_ROWCOUNT_NOT_ONE"}
|
||||
@@ -189,7 +206,7 @@ def write_field(conn, code: str, field: str, value) -> dict:
|
||||
same = (float(after) == value) if field == "IPI" and after is not None else _text(after) == value
|
||||
if not same:
|
||||
return {"outcome": "FAILED", "reason": "POST_WRITE_VERIFICATION_FAILED"}
|
||||
return {"outcome": "APPLIED", "value": value, "rowcount": 1}
|
||||
return {"outcome": "APPLIED", "value": value, "rowcount": 1, "replaced_placeholder": _text(raw) if _text(raw) else None}
|
||||
except Exception as error: # noqa: BLE001
|
||||
try:
|
||||
conn.rollback()
|
||||
@@ -345,7 +362,8 @@ class EnrichmentWorker:
|
||||
for field in ALLOWED_FIELDS:
|
||||
decision = fields.get(field) or {}
|
||||
status, value = decision.get("status"), decision.get("value")
|
||||
base = {"field": field, "rule": decision.get("rule"), "kb_status": decision.get("kb_status"), "evidence": decision.get("evidence") or [], "old": None, "new": value}
|
||||
placeholder = _text(current.get(field)) if is_placeholder(field, current.get(field)) else None
|
||||
base = {"field": field, "rule": decision.get("rule"), "kb_status": decision.get("kb_status"), "evidence": decision.get("evidence") or [], "old": placeholder, "new": value}
|
||||
if status == "HUMAN_OVERRIDDEN" or overridden.get(field):
|
||||
reports.append({**base, "outcome": "HUMAN_OVERRIDDEN", "reason": "HUMAN_OVERRIDDEN", "new": None})
|
||||
continue
|
||||
|
||||
@@ -9,7 +9,7 @@ import os
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_VERSION_DEFAULT = "1.3.3"
|
||||
AGENT_VERSION_DEFAULT = "1.3.4"
|
||||
REQUIRED_ENV = ("GTI_HUB_URL", "AGENT_TOKEN", "FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")
|
||||
REQUIRED_CAPABILITIES = ("supports_ncm", "supports_ipi", "supports_cest")
|
||||
FISCAL_MODES = ("DRY_RUN", "AUTO_APPLY")
|
||||
|
||||
@@ -45,7 +45,7 @@ class Hub:
|
||||
for f in list(fields):
|
||||
if f in self.overridden.get(payload["product_code"], set()):
|
||||
fields[f] = {"status": "HUMAN_OVERRIDDEN", "value": None}
|
||||
elif payload["current"].get(f):
|
||||
elif payload["current"].get(f) and not E.is_placeholder(f, payload["current"].get(f)):
|
||||
fields[f] = {"status": "ALREADY_SET", "value": None}
|
||||
return {"state": self.state, "fields": fields, "dataset_versions": {"NCM": {"sha256": "abc"}}}
|
||||
|
||||
@@ -230,3 +230,68 @@ def test_products_created_while_inactive_are_not_new_when_reactivated(env):
|
||||
add(db, "W2")
|
||||
now[0] += 5
|
||||
assert worker.step() == ["W2"]
|
||||
|
||||
|
||||
# ---- TECHNICAL_PLACEHOLDER (the SGV refuses a product without NCM: operators type 00000000) ---------------------------------------------------
|
||||
def add_with_ncm(db, code, ncm, name="COCA-COLA ORIGINAL LATA 350 ML"):
|
||||
db.execute("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,CLASSIFICACAO_FISCAL,OTHER) VALUES(?,?,?,?,?)", (code, name, "7891000100103", ncm, "keep"))
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_placeholder_parity_with_the_hub():
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from central.app.enrichment import placeholders as hub
|
||||
assert {k: set(v) for k, v in E.PLACEHOLDERS.items()} == {k: set(v) for k, v in hub.PRODUCT_ENRICHMENT_PLACEHOLDERS.items()}
|
||||
for value in ("00000000", "0000.00.00", " 00000000 ", "22021000", "99999999", "0", "SEM NCM", "", None):
|
||||
assert E.is_placeholder("CLASSIFICACAO_FISCAL", value) == hub.is_placeholder("CLASSIFICACAO_FISCAL", value), value
|
||||
assert not E.is_placeholder("CEST", "0000000") and not E.is_placeholder("IPI", 0)
|
||||
|
||||
|
||||
def test_placeholder_ncm_is_replaced_dry_run_first_then_enabled_and_second_run_is_zero(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
hub.features = {"features_configured": True, "enrichment_mode": "DRY_RUN"}
|
||||
worker._features_at = -999
|
||||
worker.step() # baseline
|
||||
add_with_ncm(db, "P1", "00000000")
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert row(db, "P1")[0] == "00000000" and hub.reports[-1]["fields"][0]["outcome"] == "DRY_RUN" # DRY_RUN: detected, zero writes
|
||||
assert hub.resolve_calls[-1]["current"]["CLASSIFICACAO_FISCAL"] == "00000000"
|
||||
hub.features = dict(ENABLED)
|
||||
worker._features_at = -999
|
||||
worker._auth_cache = (None, 0.0)
|
||||
add_with_ncm(db, "P2", "00000000")
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert row(db, "P2") == ("22021000", "0300700", 0.0, "keep")
|
||||
first = [f for f in hub.reports[-1]["fields"] if f["field"] == "CLASSIFICACAO_FISCAL"][0]
|
||||
assert first["outcome"] == "APPLIED" and first["old"] == "00000000"
|
||||
before = db.total_changes
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
now[0] += 60
|
||||
worker.step()
|
||||
assert db.total_changes == before # second run: 0 writes
|
||||
|
||||
|
||||
def test_real_or_unlisted_ncm_is_never_replaced(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
for code, ncm in (("R1", "22021000"), ("R2", "99999999"), ("R3", "0"), ("R4", "SEM NCM"), ("R5", "19059090")):
|
||||
add_with_ncm(db, code, ncm)
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
for code, ncm in (("R1", "22021000"), ("R2", "99999999"), ("R3", "0"), ("R4", "SEM NCM"), ("R5", "19059090")):
|
||||
assert row(db, code)[0] == ncm, code # EXISTING_REAL_VALUE_PRESERVED, even if the KB says otherwise
|
||||
|
||||
|
||||
def test_write_field_guards_the_placeholder_swap():
|
||||
db = sqlite3.connect(":memory:")
|
||||
db.execute("CREATE TABLE C000025(CODIGO TEXT, CLASSIFICACAO_FISCAL TEXT)")
|
||||
db.executemany("INSERT INTO C000025 VALUES(?,?)", [("a", "00000000"), ("b", "22021000"), ("c", None), ("d", "0000.00.00")])
|
||||
db.commit()
|
||||
assert E.write_field(Conn(db), "a", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
|
||||
assert E.write_field(Conn(db), "b", "CLASSIFICACAO_FISCAL", "19059090")["reason"] == "VALUE_ALREADY_SET_NEVER_OVERWRITTEN"
|
||||
assert E.write_field(Conn(db), "c", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
|
||||
assert E.write_field(Conn(db), "d", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
|
||||
assert [r[0] for r in db.execute("SELECT CLASSIFICACAO_FISCAL FROM C000025 ORDER BY CODIGO")] == ["19059090", "22021000", "19059090", "19059090"]
|
||||
|
||||
@@ -10,6 +10,7 @@ from sqlalchemy import select
|
||||
from ..blocked_products import upsert_blocker
|
||||
from ..config import settings
|
||||
from ..models import Installation, ProductEnrichment, ProductFieldAudit
|
||||
from .placeholders import is_placeholder
|
||||
from .providers import build_providers
|
||||
from .quota import ProviderGateway
|
||||
from .resolver import FIELDS, resolve
|
||||
@@ -74,6 +75,9 @@ def router(principal, providers_factory=None):
|
||||
if field in result["fields"]:
|
||||
result["fields"][field] = {"status": "HUMAN_OVERRIDDEN", "value": None, "rule": None, "kb_status": None, "evidence": [], "reason": "valor alterado por uma pessoa; nunca mais preenchido automaticamente"}
|
||||
pending = result["state"] in PENDING_STATES
|
||||
for name, decision in result["fields"].items(): # explain in the audit trail that a technical placeholder was treated as "not classified yet"
|
||||
if is_placeholder(name, current.get(name)):
|
||||
decision["technical_placeholder"] = str(current.get(name))
|
||||
row.gtin, row.description, row.latency_ms = result.get("gtin"), description, latency
|
||||
row.attempts = (row.attempts or 0) + 1
|
||||
row.status = "PENDING" if pending else "RESOLVED"
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
"""TECHNICAL_PLACEHOLDER semantics for the product enrichment.
|
||||
|
||||
The SGV refuses to save a product without an NCM, so operators type a technical filler to get past the form. That filler is NOT a human fiscal decision: it means "not classified yet".
|
||||
Explicit allowlist PER FIELD (never generalised: "0", "99999999", "SEM NCM" ... are NOT placeholders until the real SGV behaviour is confirmed). The Agent carries the same table
|
||||
(agent/src/gti_fiscal_agent/enrichment.py PLACEHOLDERS); a test asserts both are identical.
|
||||
"""
|
||||
import re
|
||||
|
||||
PRODUCT_ENRICHMENT_PLACEHOLDERS = {"CLASSIFICACAO_FISCAL": frozenset({"00000000"})}
|
||||
|
||||
|
||||
def _norm(field: str, value) -> str:
|
||||
text = "" if value is None else str(value).strip()
|
||||
return re.sub(r"\D", "", text) if field == "CLASSIFICACAO_FISCAL" else text # 0000.00.00 and 00000000 are the same NCM filler
|
||||
|
||||
|
||||
def is_placeholder(field: str, value) -> bool:
|
||||
return _norm(field, value) in PRODUCT_ENRICHMENT_PLACEHOLDERS.get(field, frozenset())
|
||||
|
||||
|
||||
def is_blank(value) -> bool:
|
||||
return not ("" if value is None else str(value)).strip()
|
||||
|
||||
|
||||
def is_writable_empty(field: str, value) -> bool:
|
||||
"""True when the field is empty OR holds the approved technical placeholder. A real value (even a wrong one) is never writable here."""
|
||||
return is_blank(value) or is_placeholder(field, value)
|
||||
@@ -17,6 +17,7 @@ from ..datasets.service import DatasetService
|
||||
from ..models import Dataset, NcmEntry
|
||||
from . import kb, textnorm
|
||||
from .gtin import normalize_gtin
|
||||
from .placeholders import is_placeholder, is_writable_empty
|
||||
from .quota import ProviderGateway
|
||||
|
||||
AUTO, SUGGEST, CONFLICT, INSUFFICIENT, ALREADY, NA = "AUTO_FILL_SAFE", "SUGGESTION_ONLY", "CONFLICT", "INSUFFICIENT_EVIDENCE", "ALREADY_SET", "NOT_APPLICABLE"
|
||||
@@ -75,7 +76,7 @@ def _field(status, value=None, rule=None, kb_status=None, evidence=None, reason=
|
||||
|
||||
|
||||
def _empty(current: dict, name: str) -> bool:
|
||||
return not str(current.get(name) or "").strip()
|
||||
return is_writable_empty(name, current.get(name))
|
||||
|
||||
|
||||
def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current: dict, description: str, *, name_verified=False) -> dict:
|
||||
@@ -92,7 +93,8 @@ def _fiscal_decisions(session, ncm_eval: dict, gtin_kb_verified: bool, current:
|
||||
else:
|
||||
out["CLASSIFICACAO_FISCAL"] = _field(INSUFFICIENT, reason="nenhuma fonte com NCM válido no dataset oficial")
|
||||
ncm = out["CLASSIFICACAO_FISCAL"]
|
||||
ncm_for_engine = ncm["value"] if ncm["status"] == AUTO else (normalize_code(current.get("CLASSIFICACAO_FISCAL")) or None)
|
||||
own = current.get("CLASSIFICACAO_FISCAL")
|
||||
ncm_for_engine = ncm["value"] if ncm["status"] == AUTO else (None if is_placeholder("CLASSIFICACAO_FISCAL", own) else (normalize_code(own) or None))
|
||||
if not ncm_for_engine or (ncm["status"] == ALREADY and not ncm_is_official(session, ncm_for_engine)):
|
||||
for name in ("CEST", "IPI"):
|
||||
out[name] = _field(ALREADY if not _empty(current, name) else INSUFFICIENT, reason="depende de um NCM oficial válido")
|
||||
|
||||
@@ -4,5 +4,5 @@ kept identical to this) and what an AgentRelease row's `version` column holds
|
||||
once published — never a second, independent version number. Fiscal semantics
|
||||
are versioned separately (ENGINE_VERSION/DECISION_VERSION in classification.py,
|
||||
DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump."""
|
||||
GTI_FISCAL_VERSION = "1.3.3"
|
||||
GTI_FISCAL_VERSION = "1.3.4"
|
||||
API_COMPAT_MAJOR_MINOR = "1.0" # an Agent 1.0.x/1.1.x talks to any Hub 1.0.x
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "gti-fiscal-hub"
|
||||
version = "1.3.3"
|
||||
version = "1.3.4"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"fastapi>=0.115,<1",
|
||||
|
||||
@@ -17,7 +17,7 @@ def test_health_is_unavailable_when_database_is_down():
|
||||
|
||||
def test_version_endpoint_reports_release_and_fiscal_versions():
|
||||
body = TestClient(create_app(database_probe=lambda: True)).get("/version").json()
|
||||
assert body["version"] == "1.3.3" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
|
||||
assert body["version"] == "1.3.4" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
|
||||
|
||||
|
||||
def test_production_refuses_the_dev_token(monkeypatch):
|
||||
|
||||
@@ -313,3 +313,15 @@ def test_backfill_respects_operational_reserve(env):
|
||||
outs = [gw.lookup(fake, gtin(f"78970000{i:04d}"), "BACKFILL", 20).state for i in range(10)]
|
||||
assert outs.count("SKIPPED") == 2 and fake.calls == 8 # 20% of 10 kept for new products
|
||||
assert gw.lookup(fake, gtin("789700009999"), "OPERATIONAL").state in ("FOUND", "NOT_FOUND")
|
||||
|
||||
|
||||
def test_technical_placeholder_ncm_is_treated_as_not_classified_but_real_values_are_not(env):
|
||||
p = kb.upsert_product(env, "0" + G1, "COCA-COLA ORIGINAL LATA 350 ML")
|
||||
for fam in ("PROVIDER:a", "PROVIDER:b"):
|
||||
kb.add_evidence(env, p, "ncm", "22021000", "x", family=fam)
|
||||
env.commit()
|
||||
for value, expected in (("00000000", "AUTO_FILL_SAFE"), ("0000.00.00", "AUTO_FILL_SAFE"), ("22021000", "ALREADY_SET"), ("99999999", "ALREADY_SET"), ("0", "ALREADY_SET"), ("", "AUTO_FILL_SAFE")):
|
||||
r = run(env, [], G1, "COCA-COLA ORIGINAL LATA 350 ML", current={"CLASSIFICACAO_FISCAL": value})
|
||||
assert r["fields"]["CLASSIFICACAO_FISCAL"]["status"] == expected, value
|
||||
r = run(env, [], G1, "COCA-COLA ORIGINAL LATA 350 ML", current={"CLASSIFICACAO_FISCAL": "00000000"})
|
||||
assert r["fields"]["IPI"]["value"] == "0.0" # CEST/IPI follow the NCM that is about to replace the filler
|
||||
|
||||
@@ -19,4 +19,4 @@ FB_PASSWORD=
|
||||
FISCAL_MODE=DRY_RUN
|
||||
FIREBIRD_WRITES=0
|
||||
POLL_INTERVAL_SECONDS=5
|
||||
AGENT_VERSION=1.3.3
|
||||
AGENT_VERSION=1.3.4
|
||||
|
||||
@@ -164,7 +164,7 @@ switch ($Command) {
|
||||
"GTI_HUB_URL=$hub", "AGENT_TOKEN=$token", "AGENT_ID=$agentId",
|
||||
"COMPANY_CODE=$code", "COMPANY_CNPJ=$cnpj", "COMPANY_CRT_SOURCE=$crtSource",
|
||||
"FIREBIRD_DATABASE_PATH=$db", "FB_USER=$fbUser", "FB_PASSWORD=$fbPass",
|
||||
'FISCAL_MODE=DRY_RUN', 'FIREBIRD_WRITES=0', 'POLL_INTERVAL_SECONDS=5', 'AGENT_VERSION=1.3.3'
|
||||
'FISCAL_MODE=DRY_RUN', 'FIREBIRD_WRITES=0', 'POLL_INTERVAL_SECONDS=5', 'AGENT_VERSION=1.3.4'
|
||||
)
|
||||
Set-Content -Path $EnvFile -Value $lines -Encoding UTF8
|
||||
}
|
||||
|
||||
@@ -19,4 +19,4 @@ FB_PASSWORD=
|
||||
FISCAL_MODE=DRY_RUN
|
||||
FIREBIRD_WRITES=0
|
||||
POLL_INTERVAL_SECONDS=5
|
||||
AGENT_VERSION=1.3.3
|
||||
AGENT_VERSION=1.3.4
|
||||
|
||||
@@ -19,4 +19,4 @@ FB_PASSWORD=
|
||||
FISCAL_MODE=DRY_RUN
|
||||
FIREBIRD_WRITES=0
|
||||
POLL_INTERVAL_SECONDS=5
|
||||
AGENT_VERSION=1.3.3
|
||||
AGENT_VERSION=1.3.4
|
||||
|
||||
@@ -19,4 +19,4 @@ FB_PASSWORD=
|
||||
FISCAL_MODE=DRY_RUN
|
||||
FIREBIRD_WRITES=0
|
||||
POLL_INTERVAL_SECONDS=5
|
||||
AGENT_VERSION=1.3.3
|
||||
AGENT_VERSION=1.3.4
|
||||
|
||||
@@ -19,4 +19,4 @@ FB_PASSWORD=
|
||||
FISCAL_MODE=DRY_RUN
|
||||
FIREBIRD_WRITES=0
|
||||
POLL_INTERVAL_SECONDS=5
|
||||
AGENT_VERSION=1.3.3
|
||||
AGENT_VERSION=1.3.4
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# GTI Fiscal Hub — production environment (copy to .env, never commit .env)
|
||||
GTI_FISCAL_VERSION=1.3.3
|
||||
GTI_FISCAL_VERSION=1.3.4
|
||||
|
||||
# REQUIRED. URL-safe secret (used inside a connection URL): openssl rand -hex 24
|
||||
POSTGRES_PASSWORD=
|
||||
|
||||
@@ -36,7 +36,7 @@ services:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: central/Dockerfile
|
||||
image: gti-fiscal-hub:${GTI_FISCAL_VERSION:-1.3.3}
|
||||
image: gti-fiscal-hub:${GTI_FISCAL_VERSION:-1.3.4}
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
postgres:
|
||||
|
||||
@@ -107,3 +107,13 @@ Production KB starts empty (0 products, head 0028 before deploy). The lab databa
|
||||
|
||||
### Measured (local data only; `docs/runs/enrichment_history_analysis.json`)
|
||||
See the V2 report in the mission summary; key findings: cross-client NCM agreement is poor (a candidate taken from the other clients matches the held-out client's own NCM only 62 % of the time, and only 42 % even when >= 2 clients corroborate it), which is exactly why client data can never verify an NCM on its own.
|
||||
|
||||
|
||||
## TECHNICAL_PLACEHOLDER (1.3.4)
|
||||
The SGV refuses to save a product without NCM, so operators type a filler. An empty-only rule therefore never fires for that product. Contract:
|
||||
- Per-field allowlist `PRODUCT_ENRICHMENT_PLACEHOLDERS = {CLASSIFICACAO_FISCAL: {"00000000"}}` (dots ignored: `0000.00.00` is the same filler). `0`, `99999999`, `SEM NCM`... are NOT placeholders (no evidence of SGV behaviour); nothing else is ever overwritten.
|
||||
- A placeholder is "not classified yet", never `HUMAN_OVERRIDDEN`. A real NCM (even a wrong one) is preserved, whatever the KB says. cClassTrib typed by the operator is never touched (RTC gates own it).
|
||||
- Where it is decided: the Hub treats a placeholder like an empty field when it builds the per-field decisions (`resolver._empty`, audit note `technical_placeholder`); the Agent must ALSO accept it in the typed write (`write_field`: compare-and-swap `UPDATE ... WHERE TRIM(f)=<exact value just read>`), so this needs Agent >= 1.3.4.
|
||||
Both tables are identical (parity test). The audit records `old = 00000000`.
|
||||
- Scope reminders: products created while the function was DISABLED / WAITING stay history (re-baseline); REPROCESS_ALL/SYNC do not push products through the enrichment (only NEW products after the baseline do);
|
||||
a product WITHOUT EAN is resolved by name only from a unique VERIFIED_KB match.
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"copy_only": true,
|
||||
"checks": {
|
||||
"PLACEHOLDER_PRODUCT_DETECTED_DRY_RUN": true,
|
||||
"DRY_RUN_ZERO_WRITES": true,
|
||||
"DRY_RUN_DECISION_GENERATED_FOR_NCM": true,
|
||||
"ENABLED_PLACEHOLDER_REPLACED": true,
|
||||
"CEST_IPI_ONLY_WHEN_SAFE": true,
|
||||
"CCLASSTRIB_OPERATOR_VALUE_UNTOUCHED": true,
|
||||
"AUDIT_RECORDS_THE_REPLACED_PLACEHOLDER": true,
|
||||
"EXISTING_REAL_VALUE_PRESERVED": true,
|
||||
"UNLISTED_FILLER_NOT_OVERWRITTEN": true,
|
||||
"SECOND_RUN_WRITES_0": true
|
||||
},
|
||||
"ALL_PASS": true,
|
||||
"PH002": [
|
||||
"22021000",
|
||||
"0300700",
|
||||
0.0,
|
||||
"200009"
|
||||
]
|
||||
}
|
||||
@@ -1,14 +1,14 @@
|
||||
{
|
||||
"artifact": "RELEASE_MANIFEST",
|
||||
"GTI_FISCAL_VERSION": "1.3.3",
|
||||
"GTI_FISCAL_VERSION": "1.3.4",
|
||||
"generated_at": "2026-09-25T15:43:40+00:00",
|
||||
"hub": {
|
||||
"version": "1.3.3",
|
||||
"version": "1.3.4",
|
||||
"api_compat": "1.0",
|
||||
"engine_version": "0.3.0",
|
||||
"decision_version": "mission-04c-ipi-ex-split"
|
||||
},
|
||||
"agent_version": "1.3.3",
|
||||
"agent_version": "1.3.4",
|
||||
"fiscal_engine": {
|
||||
"ENGINE_VERSION": "0.3.0",
|
||||
"DECISION_VERSION": "mission-04c-ipi-ex-split",
|
||||
@@ -89,5 +89,5 @@
|
||||
"record_count": 11103
|
||||
}
|
||||
],
|
||||
"note": "Hub-only release 1.3.3: the Windows Agent installer distributed remains GTIFiscalAgent-Setup-1.3.1.exe (no functional Agent change)."
|
||||
"note": "1.3.4: Agent + Hub. Agent 1.3.4 replaces only an approved TECHNICAL_PLACEHOLDER (NCM 00000000) besides empty fields; real values are never overwritten."
|
||||
}
|
||||
+1
-1
@@ -1 +1 @@
|
||||
1.3.3
|
||||
1.3.4
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
"""COPY-ONLY E2E of the TECHNICAL_PLACEHOLDER case found by the operator: the SGV refuses a product without NCM, so it was saved with CLASSIFICACAO_FISCAL = '00000000'.
|
||||
|
||||
python scripts/enrichment_placeholder_e2e.py
|
||||
|
||||
Real Firebird COPY + the real Agent worker + the real Hub resolver (in-process, fake providers, no internet). Proves: placeholder detected and classified (DRY_RUN = zero writes), replaced under ENABLED,
|
||||
CEST/IPI filled only when safe, cClassTrib typed by the operator untouched, a real NCM (or an unlisted filler such as 99999999) preserved even when the KB disagrees, second run = 0 writes.
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "scripts")); sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
from dotenv import load_dotenv # noqa: E402
|
||||
|
||||
load_dotenv(ROOT / ".env")
|
||||
import enrichment_copy_e2e as ce # noqa: E402
|
||||
from gti_fiscal_agent import enrichment as E # noqa: E402
|
||||
|
||||
base = ce.base
|
||||
|
||||
|
||||
def insert(code, ncm, ean, cclasstrib=None, name="COCA-COLA ORIGINAL LATA 350 ML"):
|
||||
conn = base.connect(True); cur = conn.cursor()
|
||||
cur.execute("INSERT INTO C000025 (CODIGO, PRODUTO, CODBARRA, CLASSIFICACAO_FISCAL, CCLASSTRIB) VALUES (?, ?, ?, ?, ?)", (code, name, ean, ncm, cclasstrib))
|
||||
conn.commit(); conn.close()
|
||||
|
||||
|
||||
def row(code):
|
||||
conn = base.connect(False); cur = conn.cursor()
|
||||
cur.execute("SELECT CLASSIFICACAO_FISCAL, CEST, IPI, CCLASSTRIB FROM C000025 WHERE CODIGO=?", (code,))
|
||||
r = cur.fetchone(); conn.close()
|
||||
return tuple(None if v is None else (v if isinstance(v, float) else str(v).strip()) for v in r) if r else None
|
||||
|
||||
|
||||
def main():
|
||||
checks = {}
|
||||
base.restore_copy()
|
||||
hub = ce.InProcessHub()
|
||||
from central.app.models import CestEntry
|
||||
for entry in hub.session.query(CestEntry).all():
|
||||
entry.description = "COCA-COLA ORIGINAL LATA 350 ML" # official CEST text close enough to the product for the (conservative) description match
|
||||
hub.session.commit()
|
||||
for suffix in ("", "-wal", "-shm"):
|
||||
try:
|
||||
(ROOT / "scratch_enrich" / f"enrich_ph_e2e.db{suffix}").unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
store = E.EnrichmentStore(ROOT / "scratch_enrich" / "enrich_ph_e2e.db")
|
||||
worker = E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: base.connect(False), write_conn=lambda: base.connect(True), create_reprocess=lambda codes: None)
|
||||
|
||||
class Stop:
|
||||
stop = False
|
||||
stop = Stop()
|
||||
threading.Thread(target=worker.run_forever, args=(stop,), daemon=True).start()
|
||||
ean = ce.gtin("789700000001")
|
||||
hub.features = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "DRY_RUN"}
|
||||
worker._features_at = -999
|
||||
time.sleep(6) # baseline taken while the function is active
|
||||
insert("PH001", "00000000", ean, cclasstrib="200009")
|
||||
time.sleep(9)
|
||||
checks["PLACEHOLDER_PRODUCT_DETECTED_DRY_RUN"] = store.get("PH001") is not None and hub.calls >= 1
|
||||
checks["DRY_RUN_ZERO_WRITES"] = row("PH001") == ("00000000", None, None, "200009")
|
||||
decisions = [it for rep in hub.reports for it in rep["fields"] if rep["product_code"] == "PH001"]
|
||||
checks["DRY_RUN_DECISION_GENERATED_FOR_NCM"] = any(d["field"] == "CLASSIFICACAO_FISCAL" and d["outcome"] == "DRY_RUN" and d["old"] == "00000000" and d["new"] == "22021000" for d in decisions)
|
||||
hub.features = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "ENABLED"}
|
||||
worker._features_at = -999
|
||||
worker._auth_cache = (None, 0.0)
|
||||
insert("PH002", "00000000", ean, cclasstrib="200009") # same real scenario, function already ENABLED
|
||||
t0 = time.monotonic()
|
||||
while time.monotonic() - t0 < 25 and row("PH002")[0] == "00000000":
|
||||
time.sleep(0.2)
|
||||
time.sleep(4) # remaining fields + audit report follow the NCM write
|
||||
r2 = row("PH002")
|
||||
checks["ENABLED_PLACEHOLDER_REPLACED"] = r2[0] == "22021000"
|
||||
checks["CEST_IPI_ONLY_WHEN_SAFE"] = r2[2] == 0.0 and r2[1] == "0300700"
|
||||
checks["CCLASSTRIB_OPERATOR_VALUE_UNTOUCHED"] = r2[3] == "200009"
|
||||
applied = [it for rep in hub.reports for it in rep["fields"] if rep["product_code"] == "PH002" and it["outcome"] == "APPLIED"]
|
||||
checks["AUDIT_RECORDS_THE_REPLACED_PLACEHOLDER"] = any(a["field"] == "CLASSIFICACAO_FISCAL" and a["old"] == "00000000" for a in applied)
|
||||
# negative: real NCM / unlisted filler are preserved even though the KB says 22021000
|
||||
insert("PH003", "19059090", ean); insert("PH004", "99999999", ean); insert("PH005", "22021000", ean)
|
||||
time.sleep(12)
|
||||
checks["EXISTING_REAL_VALUE_PRESERVED"] = row("PH003")[0] == "19059090" and row("PH005")[0] == "22021000"
|
||||
checks["UNLISTED_FILLER_NOT_OVERWRITTEN"] = row("PH004")[0] == "99999999"
|
||||
calls_before, writes_before = hub.calls, sum(1 for rep in hub.reports for it in rep["fields"] if it["outcome"] == "APPLIED")
|
||||
time.sleep(14)
|
||||
writes_after = sum(1 for rep in hub.reports for it in rep["fields"] if it["outcome"] == "APPLIED")
|
||||
checks["SECOND_RUN_WRITES_0"] = writes_after == writes_before and row("PH002")[0] == "22021000"
|
||||
stop.stop = True
|
||||
result = {"copy_only": True, "checks": checks, "ALL_PASS": all(checks.values()), "PH002": row("PH002")}
|
||||
(ROOT / "docs" / "runs" / "enrichment_placeholder_e2e.json").write_text(json.dumps(result, indent=1), encoding="utf-8")
|
||||
print(json.dumps(result, indent=1))
|
||||
sys.exit(0 if result["ALL_PASS"] else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -14,11 +14,11 @@ sys.path.insert(0, str(ROOT))
|
||||
def test_versions_are_coherent_across_hub_agent_and_packages():
|
||||
from central.app.version import GTI_FISCAL_VERSION
|
||||
v = (ROOT / "release" / "VERSION").read_text(encoding="utf-8").strip()
|
||||
assert v == GTI_FISCAL_VERSION == "1.3.3"
|
||||
assert v == GTI_FISCAL_VERSION == "1.3.4"
|
||||
pep = v.replace("-rc", "rc")
|
||||
for pp in ("central/pyproject.toml", "agent/pyproject.toml"):
|
||||
assert f'version = "{pep}"' in (ROOT / pp).read_text(encoding="utf-8")
|
||||
assert '"AGENT_VERSION", "1.3.3"' in (ROOT / "agent" / "scripts" / "run_agent_service.py").read_text(encoding="utf-8")
|
||||
assert '"AGENT_VERSION", "1.3.4"' in (ROOT / "agent" / "scripts" / "run_agent_service.py").read_text(encoding="utf-8")
|
||||
manifest = ROOT / "release" / "RELEASE_MANIFEST.json"
|
||||
if manifest.exists():
|
||||
m = json.loads(manifest.read_text(encoding="utf-8"))
|
||||
|
||||
Reference in New Issue
Block a user