Add 'fiscal/' from commit '102ba84500dbd9655a0471288c3e278652593ca9'
git-subtree-dir: fiscal git-subtree-mainline:d95165fc8fgit-subtree-split:102ba84500
This commit is contained in:
gti
@@ -0,0 +1,41 @@
|
||||
# One of these per Agent installation (one per client machine). Copy to
|
||||
# agent/.env on that machine; never commit it. Get AGENT_ID/AGENT_TOKEN/
|
||||
# COMPANY_ID from central/scripts/register_agent.py, run once per installation.
|
||||
|
||||
GTI_HUB_URL=https://hub.example.com
|
||||
AGENT_ID=
|
||||
AGENT_TOKEN=
|
||||
COMPANY_ID=
|
||||
COMPANY_UF=
|
||||
|
||||
# The client's Firebird ERP database.
|
||||
FIREBIRD_DATABASE_PATH=
|
||||
FB_USER=
|
||||
FB_PASSWORD=
|
||||
FIREBIRD_CLIENT_LIBRARY=
|
||||
|
||||
# Default MUST be 0. Writes only ever happen against exactly the database
|
||||
# named by AUTHORIZED_WRITE_DATABASE_PATH below (never inferred from
|
||||
# FIREBIRD_DATABASE_PATH alone, never from the ".FDB" extension).
|
||||
FIREBIRD_WRITES=0
|
||||
|
||||
# Required to enable writes for THIS installation. Must be set to the exact
|
||||
# same absolute path as FIREBIRD_DATABASE_PATH above — this is the operator's
|
||||
# explicit, per-installation authorization. Leaving it unset means
|
||||
# FIREBIRD_WRITES=1 has no effect against a real client database (it falls
|
||||
# back to the developer-only disposable test database path and therefore
|
||||
# fails closed for anything else).
|
||||
AUTHORIZED_WRITE_DATABASE_PATH=
|
||||
|
||||
# Required to enable writes for THIS installation. Must equal the schema
|
||||
# fingerprint this installation was verified/registered against; if the live
|
||||
# schema fingerprint (reported in heartbeats) ever differs, writes are
|
||||
# refused until an operator re-verifies and updates this value.
|
||||
EXPECTED_SCHEMA_FINGERPRINT=
|
||||
|
||||
POLL_INTERVAL_SECONDS=10
|
||||
AGENT_VERSION=0.1.0
|
||||
|
||||
# Optional overrides:
|
||||
# AGENT_STATE_DB=C:\GTI\Agent\agent_state.sqlite
|
||||
# AGENT_LOG_PATH=C:\GTI\Agent\logs\agent.log
|
||||
@@ -0,0 +1,8 @@
|
||||
[project]
|
||||
name = "gti-fiscal-agent"
|
||||
version = "1.3.9"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = ["httpx>=0.27,<1"]
|
||||
|
||||
[project.optional-dependencies]
|
||||
windows-service = ["pywin32>=306; sys_platform == 'win32'"]
|
||||
@@ -0,0 +1,46 @@
|
||||
"""GTI Fiscal Agent installation preflight (READ-ONLY): python agent/scripts/preflight.py [--env-file FILE] [--json]
|
||||
|
||||
Prints PREFLIGHT=PASS|FAIL and the blockers; exit code 0 on PASS, 2 on FAIL. Secrets are never printed.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(AGENT_ROOT / "src"))
|
||||
from gti_fiscal_agent.preflight import run_preflight # noqa: E402
|
||||
|
||||
|
||||
def load_env(path):
|
||||
if path and Path(path).exists():
|
||||
for line in Path(path).read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
k, _, v = line.partition("=")
|
||||
os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--env-file")
|
||||
ap.add_argument("--json", action="store_true")
|
||||
a = ap.parse_args(argv)
|
||||
load_env(a.env_file or os.environ.get("GTI_AGENT_ENV_FILE") or AGENT_ROOT.parent / ".env")
|
||||
os.environ.setdefault("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite"))
|
||||
os.environ.setdefault("AGENT_LOG_PATH", str(AGENT_ROOT / "logs" / "agent.log"))
|
||||
res = run_preflight(os.environ)
|
||||
if a.json:
|
||||
print(json.dumps(res, ensure_ascii=False, indent=1, default=str))
|
||||
else:
|
||||
for c in res["checks"]:
|
||||
print(f" [{c['status']:<4}] {c['name']:<24} {c['detail']}")
|
||||
print(f"PREFLIGHT={res['PREFLIGHT']}")
|
||||
for b in res["blockers"]:
|
||||
print(f"BLOCKER: {b}")
|
||||
raise SystemExit(0 if res["PREFLIGHT"] == "PASS" else 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Agent preflight: COMPANY_CONTEXT=PASS/FAIL for one Firebird database (READ-ONLY).
|
||||
|
||||
python agent/scripts/preflight_company_context.py [--database <FDB>]
|
||||
|
||||
Configuration (environment or the local ignored .env): COMPANY_CODE / COMPANY_CNPJ (company selection when C000004 is ambiguous),
|
||||
COMPANY_CRT_SOURCE = EMISSOR | C000004 (explicit choice when the two CRT sources disagree). The full CNPJ is never printed.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
from gti_fiscal_agent.company_context import preflight_company_context, preflight_report # noqa: E402
|
||||
|
||||
|
||||
def load_dotenv(path):
|
||||
if path.exists():
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if "=" in line and not line.startswith("#"):
|
||||
k, _, v = line.partition("=")
|
||||
os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--database")
|
||||
a = ap.parse_args(argv)
|
||||
load_dotenv(ROOT / ".env")
|
||||
if a.database:
|
||||
os.environ["FIREBIRD_DATABASE_PATH"] = a.database
|
||||
report = preflight_report(preflight_company_context(os.environ))
|
||||
print(json.dumps(report, ensure_ascii=False, indent=1))
|
||||
raise SystemExit(0 if report["COMPANY_CONTEXT"] == "PASS" else 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Run the agent against a sanitized sample only after an explicit fingerprint gate."""
|
||||
import argparse, json, os, sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).parents[1] / "src"))
|
||||
from gti_fiscal_agent.client import HubClient
|
||||
from gti_fiscal_agent.runner import run_sample
|
||||
from gti_fiscal_agent.state import AgentState
|
||||
|
||||
FIELDS = "CODIGO,PRODUTO,NCM,CLASSIFICACAO_FISCAL,CFOP,CFOP_ENTRADA,CST,CST_ENTRADA,ALIQUOTA,CEST,CSOSN,COD_ENTRADA_COFINS,COD_SAIDA_COFINS,ALIQ_COFINS,COD_ENTRADA_PIS,COD_SAIDA_PIS,ALIQ_PIS,ALIQ_IPI,CST_IPI,ORIGEM,IAT,INCIDENCIA_PISCOFINS,IPPT,ULTIMA_ALTERACAO"
|
||||
|
||||
def read_firebird_products(codes):
|
||||
import fdb
|
||||
connection = fdb.connect(database=os.environ["FIREBIRD_DATABASE_PATH"], user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"], fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"), isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
|
||||
try:
|
||||
cursor = connection.cursor(); cursor.execute(f"SELECT {FIELDS} FROM C000025 WHERE CODIGO IN ({','.join('?' for _ in codes)}) ORDER BY CODIGO", tuple(codes))
|
||||
names = [column[0].strip() for column in cursor.description]
|
||||
return [dict(zip(names, row)) for row in cursor.fetchall()]
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(); parser.add_argument("--sample", required=True); parser.add_argument("--output", required=True); parser.add_argument("--schema-fingerprint", default="")
|
||||
args = parser.parse_args(); sample = json.loads(Path(args.sample).read_text(encoding="utf-8"))
|
||||
codes = [item["CODIGO"] for item in sample.get("sample_without_ncm", [])[:20]]
|
||||
products = read_firebird_products(codes)
|
||||
result = run_sample(products, HubClient(os.getenv("HUB_URL", "http://localhost:8000"), os.getenv("AGENT_DEV_TOKEN", "")), AgentState(Path(args.output).with_suffix(".sqlite")), args.schema_fingerprint)
|
||||
result["products_requested"] = len(products); result["firebird_products_read"] = len(products); result["firebird_access"] = "READ_ONLY_SELECT"
|
||||
output = Path(args.output)
|
||||
if result["results"]:
|
||||
output.write_text(json.dumps(result, indent=2, default=str), encoding="utf-8")
|
||||
elif not output.exists():
|
||||
output.write_text(json.dumps(result, indent=2, default=str), encoding="utf-8")
|
||||
print(json.dumps(result, indent=2))
|
||||
if __name__ == "__main__": main()
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Integrated Agent flow: Firebird read -> Hub /v1/classify/batch -> IPI dry-run
|
||||
-> guarded auto-apply write.
|
||||
|
||||
FIREBIRD_WRITES=0 (default): classifies every product and logs the decision
|
||||
(and what write *would* happen) to the local SQLite audit log; never writes.
|
||||
|
||||
FIREBIRD_WRITES=1: also auto-applies every write_eligible C000025.IPI proposal
|
||||
where rate_kind=NUMERIC, reusing the same schema/path/allowlist/compare-before-write
|
||||
guards as the manual executor. Fails closed before touching the connection unless
|
||||
the configured database is exactly the authorized disposable test database.
|
||||
|
||||
Auto-apply is scoped to table=C000025, field=IPI, rate_kind=NUMERIC only. NT/
|
||||
non-numeric markers, blank rates, ambiguous EX candidates, legacy/invalid NCM,
|
||||
CEST and EX_TIPI are never auto-applied here.
|
||||
"""
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_ROOT = Path(__file__).resolve().parents[1]
|
||||
PROJECT_ROOT = AGENT_ROOT.parent
|
||||
sys.path.insert(0, str(AGENT_ROOT / "src"))
|
||||
|
||||
from gti_fiscal_agent.client import HubClient # noqa: E402
|
||||
from gti_fiscal_agent.firebird_reader import read_products # noqa: E402
|
||||
from gti_fiscal_agent.pipeline import run_pipeline # noqa: E402
|
||||
from gti_fiscal_agent.runner import REFERENCE_SCHEMA_V1, to_hub_product # noqa: E402
|
||||
from gti_fiscal_agent.state import AgentState # noqa: E402
|
||||
|
||||
|
||||
def load_dotenv(path: Path) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
os.environ.setdefault(key.strip(), value.strip())
|
||||
|
||||
|
||||
def load_executor():
|
||||
spec = importlib.util.spec_from_file_location("firebird_write_executor", PROJECT_ROOT / "scripts" / "firebird_write_executor.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--hub-url", default=os.getenv("HUB_URL", "http://localhost:8000"))
|
||||
parser.add_argument("--token", default=os.getenv("AGENT_DEV_TOKEN", ""))
|
||||
parser.add_argument("--state-db", default=str(AGENT_ROOT / "agent_state.sqlite"))
|
||||
parser.add_argument("--schema-fingerprint", default=REFERENCE_SCHEMA_V1)
|
||||
args = parser.parse_args()
|
||||
|
||||
load_dotenv(PROJECT_ROOT / ".env")
|
||||
if not args.token:
|
||||
args.token = os.environ.get("AGENT_DEV_TOKEN", "")
|
||||
|
||||
firebird_writes = os.environ.get("FIREBIRD_WRITES", "0") == "1"
|
||||
executor = load_executor()
|
||||
database_path = os.environ["FIREBIRD_DATABASE_PATH"]
|
||||
|
||||
if firebird_writes:
|
||||
# Fail closed before any product is read or any connection is opened for writing.
|
||||
executor.assert_authorized_write_path(database_path)
|
||||
|
||||
if args.schema_fingerprint != REFERENCE_SCHEMA_V1:
|
||||
print(json.dumps({"status": "SCHEMA_UNKNOWN", "processed": 0}))
|
||||
return
|
||||
|
||||
products = read_products()
|
||||
client = HubClient(args.hub_url, args.token)
|
||||
decisions = client.classify_batch([to_hub_product(p) for p in products])
|
||||
|
||||
state = AgentState(args.state_db)
|
||||
|
||||
write_connection = None
|
||||
if firebird_writes:
|
||||
import fdb
|
||||
write_connection = fdb.connect(
|
||||
database=database_path,
|
||||
user=os.environ["FB_USER"],
|
||||
password=os.environ["FB_PASSWORD"],
|
||||
fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
)
|
||||
|
||||
try:
|
||||
summary = run_pipeline(products, decisions, state, firebird_writes, write_connection, executor, database_path)
|
||||
finally:
|
||||
if write_connection is not None:
|
||||
write_connection.close()
|
||||
|
||||
summary = {"status": "OK", "firebird_writes_enabled": firebird_writes, **summary}
|
||||
print(json.dumps(summary, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,582 @@
|
||||
"""Long-running Agent service: heartbeat + poll Hub jobs + execute pipeline.
|
||||
|
||||
startup -> load config -> validate Hub -> validate Firebird -> heartbeat
|
||||
-> poll job -> claim -> execute -> report result -> back to polling.
|
||||
|
||||
No job available: sleep POLL_INTERVAL_SECONDS.
|
||||
Hub unavailable: retry with simple capped backoff; the Agent stays alive.
|
||||
Job fails internally (bad Firebird read, classify error, write refused):
|
||||
the job is reported FAILED with a sanitized error and the Agent keeps polling.
|
||||
SIGINT/SIGTERM: finish the current job (if any) and stop the loop cleanly.
|
||||
|
||||
Written to be runnable as-is today, and easy to later wrap as a Windows
|
||||
Service (a service wrapper would just call main()); no installer here.
|
||||
"""
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_ROOT = Path(__file__).resolve().parents[1]
|
||||
PROJECT_ROOT = AGENT_ROOT.parent
|
||||
sys.path.insert(0, str(AGENT_ROOT / "src"))
|
||||
sys.path.insert(0, str(PROJECT_ROOT)) # ships central/app engine modules read-only (central.app.*)
|
||||
|
||||
from gti_fiscal_agent.client import HubClient # noqa: E402
|
||||
from gti_fiscal_agent import firebird_reader as _firebird_reader, rtc_auto_apply as _rtc_apply, unified_fiscal as _unified # noqa: E402
|
||||
from gti_fiscal_agent.firebird_reader import check_connection, read_products # noqa: E402
|
||||
from gti_fiscal_agent.pipeline import run_pipeline # noqa: E402
|
||||
from gti_fiscal_agent.runner import REFERENCE_SCHEMA_V1, product_fingerprint, to_hub_product # noqa: E402
|
||||
from central.app.operational_state import classify_product_state, summarize_operational_states # noqa: E402
|
||||
from gti_fiscal_agent.state import AgentState # noqa: E402
|
||||
from gti_fiscal_agent import enrichment as _enrichment # noqa: E402
|
||||
|
||||
_MAX_BACKOFF_SECONDS = 60
|
||||
SYNC_JOB_KINDS = ("INITIAL_SYNC", "SYNC_INCREMENTAL", "REPROCESS_ALL") # read-only ERP jobs that refresh coverage
|
||||
|
||||
logger = logging.getLogger("gti_fiscal_agent")
|
||||
|
||||
|
||||
def setup_logging(log_path):
|
||||
"""Rotating local log file. Never logs FB_PASSWORD or AGENT_TOKEN — every
|
||||
log call in this module only references non-secret identifiers/counts."""
|
||||
logger.setLevel(logging.INFO)
|
||||
if logger.handlers:
|
||||
return logger
|
||||
Path(log_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
handler = logging.handlers.RotatingFileHandler(log_path, maxBytes=5_000_000, backupCount=5, encoding="utf-8")
|
||||
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
|
||||
logger.addHandler(handler)
|
||||
logger.addHandler(logging.StreamHandler())
|
||||
return logger
|
||||
|
||||
|
||||
def load_dotenv(path: Path) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
os.environ.setdefault(key.strip(), value.strip())
|
||||
|
||||
|
||||
def load_executor():
|
||||
# The frozen service host has no repo tree; the installer writes
|
||||
# GTI_WRITE_EXECUTOR_PATH pointing at the extracted executor next to
|
||||
# agent.env. Falls back to the dev/repo path for source runs.
|
||||
executor_path = os.environ.get("GTI_WRITE_EXECUTOR_PATH") or (PROJECT_ROOT / "scripts" / "firebird_write_executor.py")
|
||||
spec = importlib.util.spec_from_file_location("firebird_write_executor", executor_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class Config:
|
||||
def __init__(self, env):
|
||||
self.hub_url = env.get("GTI_HUB_URL") or env.get("HUB_URL") or "http://localhost:8000"
|
||||
self.agent_id = env.get("AGENT_ID", "")
|
||||
self.credential_file = env.get("GTI_CREDENTIAL_FILE") or str(Path(os.environ.get("PROGRAMFILES", r"C:\Program Files")) / "GTI" / "FiscalAgent" / "credentials.bin")
|
||||
# The permanent token NEVER lives in agent.env: it is decrypted from
|
||||
# credentials.bin (DPAPI LocalMachine + SYSTEM/Administrators ACL). An
|
||||
# explicit AGENT_TOKEN/AGENT_DEV_TOKEN in env wins (dev override only).
|
||||
self.agent_token = self._resolve_agent_token(env)
|
||||
self.company_id = env.get("COMPANY_ID", "")
|
||||
self.company_uf = env.get("COMPANY_UF", "")
|
||||
_resolve_firebird_credentials(env)
|
||||
self.database_path = env["FIREBIRD_DATABASE_PATH"]
|
||||
# FISCAL_MODE gates writes technically: DRY_RUN can never write, whatever FIREBIRD_WRITES says. Unset = legacy
|
||||
# behaviour (FIREBIRD_WRITES decides). `firebird_writes` below is the EFFECTIVE value used by every write path.
|
||||
requested = env.get("FIREBIRD_WRITES", "0") == "1"
|
||||
self.fiscal_mode = (env.get("FISCAL_MODE") or ("AUTO_APPLY" if requested else "DRY_RUN")).strip().upper()
|
||||
self.requested_writes = requested
|
||||
self.firebird_writes = requested and self.fiscal_mode == "AUTO_APPLY"
|
||||
# Job execution is controlled CENTRALLY (Hub heartbeat: processing_enabled) — there is no
|
||||
# local toggle, so no installation ever needs its agent.env edited to start/stop work.
|
||||
# (A legacy AGENT_JOBS_ENABLED line in an old agent.env is deliberately ignored.)
|
||||
self.agent_jobs_enabled = True
|
||||
self.poll_interval_seconds = float(env.get("POLL_INTERVAL_SECONDS", "5"))
|
||||
# RTC auto-apply is a SEPARATE switch (default OFF): FIREBIRD_WRITES alone never lets the Agent fill cClassTrib.
|
||||
self.rtc_auto_apply_max_per_job = int(env.get("RTC_AUTO_APPLY_MAX_PER_JOB", str(_rtc_apply.DEFAULT_MAX_PER_JOB)))
|
||||
self.agent_version = env.get("AGENT_VERSION", "1.3.9")
|
||||
self.schema_fingerprint = env.get("SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1)
|
||||
self.state_db = env.get("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite"))
|
||||
# Pilot write authorization: both must be explicitly set by the installer for
|
||||
# a real client database. Left unset, both default to the dev-only values,
|
||||
# so a real client path/schema is never silently authorized (see
|
||||
# scripts/firebird_write_executor.py's assert_authorized_write_path).
|
||||
self.authorized_write_database_path = env.get("AUTHORIZED_WRITE_DATABASE_PATH")
|
||||
self.expected_schema_fingerprint = env.get("EXPECTED_SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1)
|
||||
self.log_path = env.get("AGENT_LOG_PATH", str(AGENT_ROOT / "logs" / "agent.log"))
|
||||
|
||||
@staticmethod
|
||||
def _resolve_agent_token(env):
|
||||
"""DPAPI credential from credentials.bin; explicit AGENT_TOKEN env wins
|
||||
(dev override). Fail closed: never log or fabricate a token."""
|
||||
from gti_fiscal_agent.credential_store import install_env_from_credentials
|
||||
|
||||
explicit = env.get("AGENT_TOKEN") or env.get("AGENT_DEV_TOKEN") or ""
|
||||
if explicit:
|
||||
return explicit
|
||||
try:
|
||||
return install_env_from_credentials(env.get("GTI_CREDENTIAL_FILE") or str(
|
||||
Path(os.environ.get("PROGRAMFILES", r"C:\Program Files")) / "GTI" / "FiscalAgent" / "credentials.bin"), env)
|
||||
except Exception as exc: # noqa: BLE001 — fail closed: no token, no heartbeat
|
||||
logger.warning("no usable agent credential: %s", exc)
|
||||
return ""
|
||||
|
||||
|
||||
def _resolve_firebird_credentials(env):
|
||||
"""FB_USER/FB_PASSWORD come from the DPAPI-protected firebird.bin written by Setup
|
||||
(never from a hand-edited agent.env, never sent to the Hub). An explicit FB_PASSWORD in
|
||||
the environment still wins (dev override)."""
|
||||
if env.get("FB_PASSWORD"):
|
||||
return
|
||||
from gti_fiscal_agent.credential_store import read_credential
|
||||
|
||||
path = env.get("GTI_FIREBIRD_CREDENTIAL_FILE") or str(Path(os.environ.get("PROGRAMFILES", "C:/Program Files")) / "GTI" / "FiscalAgent" / "firebird.bin")
|
||||
try:
|
||||
data = json.loads(read_credential(path, env=env))
|
||||
except Exception as exc: # noqa: BLE001 — absent/unreadable: preflight will fail closed with a clear blocker
|
||||
logger.info("no Firebird credential file usable: %s", type(exc).__name__)
|
||||
return
|
||||
env["FB_USER"], env["FB_PASSWORD"] = data.get("user", ""), data.get("password", "")
|
||||
|
||||
|
||||
class GracefulStop:
|
||||
def __init__(self):
|
||||
self.stop = False
|
||||
signal.signal(signal.SIGINT, self._handle)
|
||||
signal.signal(signal.SIGTERM, self._handle)
|
||||
|
||||
def _handle(self, signum, frame):
|
||||
self.stop = True
|
||||
|
||||
|
||||
def build_job_coverage(env, client, products, decisions):
|
||||
"""Company-context-aware coverage over the job's products (read-only). Fails closed when the company context is unresolved."""
|
||||
from gti_fiscal_agent.company_context import mask_cnpj, preflight_company_context
|
||||
from gti_fiscal_agent.coverage import build_refs, compute_coverage
|
||||
from gti_fiscal_agent.firebird_reader import read_evidence
|
||||
ctx = preflight_company_context(env)
|
||||
if ctx["status"] != "PASS":
|
||||
raise RuntimeError("COMPANY_CONTEXT=FAIL: " + "; ".join(ctx.get("reasons") or ["unresolved"]))
|
||||
evidence = read_evidence(env, ctx["company_code"])
|
||||
refs = build_refs(client.get_json, products, ctx["company_uf"])
|
||||
report = compute_coverage(products, decisions, {"uf": ctx["company_uf"], "crt": ctx["company_crt"]}, evidence.get("company_default_csosn"), refs, evidence)
|
||||
report["company_context"] = {"COMPANY_CONTEXT_STATUS": "PASS", "classification": ctx["classification"], "company_code": ctx["company_code"], "company_cnpj": ctx["company_cnpj"], "company_uf": ctx["company_uf"],
|
||||
"company_crt": ctx["company_crt"], "source_of_truth": ctx["source_of_truth"], "confidence": ctx["confidence"], "warnings": ctx["warnings"]}
|
||||
return report
|
||||
|
||||
|
||||
def open_write_connection(env, database_path, firebird_writes=True):
|
||||
if not firebird_writes: # technical guarantee: no write-capable connection exists in DRY_RUN / FIREBIRD_WRITES=0
|
||||
raise RuntimeError("write connection refused: writes are disabled (FISCAL_MODE=DRY_RUN or FIREBIRD_WRITES=0)")
|
||||
import fdb
|
||||
return fdb.connect(
|
||||
database=database_path,
|
||||
user=env["FB_USER"],
|
||||
password=env["FB_PASSWORD"],
|
||||
fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
)
|
||||
|
||||
|
||||
def _unified_fiscal(env, company, ncm_valid=None):
|
||||
"""PIS/COFINS applicability + RTC (only when the ERP has the reform on), read-only (SELECT only). Fail-closed: if it cannot be assessed the job fails
|
||||
instead of reporting a coverage that ignores these fields."""
|
||||
connection = _firebird_reader._connect(env)
|
||||
try:
|
||||
return _unified.assess_from_connection(connection, company, ncm_valid=ncm_valid)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def _fiscal_blocked_products(products, findings):
|
||||
"""One queue entry per (product, blocker). Same shape as every other blocker; `field` is the dashboard category (PIS/COFINS/RTC)."""
|
||||
out = []
|
||||
for product in products:
|
||||
code = str(product.get("CODIGO") or "")
|
||||
found = findings.get(code) or {}
|
||||
for blocker in found.get("blockers") or []:
|
||||
out.append({"company": "", "product_code": code, "gtin": product.get("GTIN") or product.get("EAN_GTIN"), "description": product.get("PRODUTO"), "blocker_type": "BLOCKED_FISCAL_DATA",
|
||||
"field": blocker["category"], "reason_code": blocker["type"], "evidence_summary": {"reason": blocker["reason"], "applicability": found.get("applicability"), **({"human": blocker["human"]} if blocker.get("human") else {})},
|
||||
"input_fingerprint": found.get("input_fingerprint", "")})
|
||||
return out
|
||||
|
||||
|
||||
def _fetch_authorizations(config, job):
|
||||
"""Current central authorization for this installation, straight from the Hub (authenticated). Tests replace this; production never trusts the job's copy for the write itself."""
|
||||
return HubClient(config.hub_url, config.agent_token).authorizations()
|
||||
|
||||
|
||||
def _make_authorizer(config, job, mode, approved_at_claim):
|
||||
"""Re-checks the Hub right before writing (cached for a few seconds): if the mode changed, the product lost its approval, or the Hub cannot be reached, nothing is written (fail closed)."""
|
||||
state = {"at": None, "auth": None}
|
||||
|
||||
def authorize(code):
|
||||
now = time.monotonic()
|
||||
if state["auth"] is None or now - state["at"] > 3.0:
|
||||
try:
|
||||
state["auth"], state["at"] = _fetch_authorizations(config, job) or {}, now
|
||||
except Exception: # noqa: BLE001 - unverifiable authorization is not authorization
|
||||
return "AUTHORIZATION_UNVERIFIABLE"
|
||||
fresh = state["auth"]
|
||||
if fresh.get("rtc_write_mode") != mode:
|
||||
return "AUTHORIZATION_CHANGED"
|
||||
if mode == "APPROVAL_REQUIRED" and str(code) not in {str(c) for c in fresh.get("rtc_approved_codes") or []}:
|
||||
return "AUTHORIZATION_CHANGED"
|
||||
return None
|
||||
return authorize
|
||||
|
||||
|
||||
def _rtc_auto_apply(env, config, job, products, fiscal, company):
|
||||
"""Preview (always) and, only when the Hub-delivered mode permits it, fill CCLASSTRIB for products whose class is PROVEN by an AUTO_APPLY_SAFE rule; also executes
|
||||
Hub-requested rollbacks. Authorization arrives ONLY inside the authenticated job (`authorizations`); there is no local switch. Never raises for a single product."""
|
||||
from datetime import datetime, timezone
|
||||
auth = job.get("authorizations") or {}
|
||||
mode = auth.get("rtc_write_mode") if auth.get("rtc_write_mode") in _rtc_apply.MODES else "DISABLED"
|
||||
approved = {str(c) for c in (auth.get("rtc_approved_codes") or [])}
|
||||
limit = int(getattr(config, "rtc_auto_apply_max_per_job", _rtc_apply.DEFAULT_MAX_PER_JOB))
|
||||
rtc_on = bool(fiscal["context"].get("rtc_enabled"))
|
||||
all_candidates, ready_total = _rtc_apply.select_candidates(fiscal["findings"], products, limit=_rtc_apply.MAX_PREVIEW_CANDIDATES)
|
||||
gate = _rtc_apply.write_gate(mode=mode, capability=True, approved=bool(approved))
|
||||
summary = {"gate": gate, "mode": mode if mode != "DISABLED" else "REPORT_ONLY", "hub_mode": mode, "capability": _rtc_apply.CAPABILITY, "ready_total": ready_total, "max_per_job": limit, "write_attempts": 0, "write_successes": 0,
|
||||
"applied": [], "aborted": [], "aborted_reasons": {}, "rollbacks": [], "allowed_fields": list(_rtc_apply.ALLOWED_RTC_WRITE_FIELDS),
|
||||
"preview": [{k: c[k] for k in ("product_code", "description", "ncm", "current", "proposed", "cst", "rule_id", "basis")} for c in all_candidates] if mode != "DISABLED" and rtc_on else [],
|
||||
"preview_truncated": ready_total > len(all_candidates)}
|
||||
if not rtc_on:
|
||||
return summary
|
||||
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
rollbacks = [r for r in (auth.get("rtc_rollbacks") or []) if isinstance(r, dict)]
|
||||
if mode == "ENABLED":
|
||||
wanted = all_candidates[:limit]
|
||||
elif mode == "APPROVAL_REQUIRED":
|
||||
wanted = [c for c in all_candidates if c["product_code"] in approved][:limit]
|
||||
else:
|
||||
wanted = []
|
||||
if not (rollbacks or (gate["allowed"] and wanted)):
|
||||
return summary
|
||||
connection = open_write_connection(env, config.database_path)
|
||||
try:
|
||||
if rollbacks:
|
||||
summary["rollbacks"] = _rtc_apply.rollback_records(connection, rollbacks, now=now, job_id=job.get("job_id"))["rollbacks"]
|
||||
if gate["allowed"] and wanted:
|
||||
outcome = _rtc_apply.apply_candidates(connection, wanted, company=company or {}, decision={"local_catalog_hash": fiscal["context"].get("local_catalog_hash"), "origin": f"HUB_MODE:{mode}"}, now=now, job_id=job.get("job_id"), authorize=_make_authorizer(config, job, mode, approved))
|
||||
summary.update({k: outcome[k] for k in ("applied", "aborted", "aborted_reasons", "write_attempts", "write_successes")})
|
||||
finally:
|
||||
connection.close()
|
||||
return summary
|
||||
|
||||
|
||||
def execute_job(job, config, executor, state, env):
|
||||
"""Run the pipeline for one claimed job. Never raises for job-domain
|
||||
failures (bad Firebird read, classify error) — returns (result_summary, error)."""
|
||||
kind = job["kind"]
|
||||
payload = job.get("payload") or {}
|
||||
try:
|
||||
if config.firebird_writes:
|
||||
# Layered, per-installation authorization: the path must match this
|
||||
# installation's own explicitly configured authorized_write_database_path
|
||||
# (defaults to the dev-only constant, so a real client path is refused
|
||||
# unless the installer deliberately set it), AND the live schema
|
||||
# fingerprint must match what this installation was configured to expect.
|
||||
executor.assert_authorized_write_path(config.database_path, authorized_path=config.authorized_write_database_path)
|
||||
executor.assert_schema_fingerprint_authorized(config.schema_fingerprint, config.expected_schema_fingerprint)
|
||||
|
||||
codigos = payload.get("codigos") if kind == "REPROCESS_PRODUCTS" else None
|
||||
products = read_products(env, codigos=codigos)
|
||||
logger.info("job %s: read %d product(s) from Firebird", job.get("job_id"), len(products))
|
||||
|
||||
client = HubClient(config.hub_url, config.agent_token)
|
||||
sync_kind = kind in SYNC_JOB_KINDS
|
||||
cache = sync_kind and hasattr(state, "remember_decision")
|
||||
# SYNC_INCREMENTAL = delta: only products whose source fingerprint changed (or are new) are
|
||||
# re-classified by the Hub; INITIAL_SYNC / REPROCESS_ALL / REPROCESS_PRODUCTS process everything they read.
|
||||
if kind == "SYNC_INCREMENTAL" and cache:
|
||||
to_process = [p for p in products if state.cached_fingerprint(p["CODIGO"]) != product_fingerprint(p)]
|
||||
else:
|
||||
to_process = products
|
||||
decisions = client.classify_batch([to_hub_product(p) for p in to_process])
|
||||
|
||||
write_connection = None
|
||||
if config.firebird_writes:
|
||||
write_connection = open_write_connection(env, config.database_path)
|
||||
try:
|
||||
summary = run_pipeline(to_process, decisions, state, config.firebird_writes, write_connection, executor, config.database_path, authorized_path=config.authorized_write_database_path)
|
||||
finally:
|
||||
if write_connection is not None:
|
||||
write_connection.close()
|
||||
|
||||
operational = coverage = fiscal_context = rtc_apply = None
|
||||
fiscal = None
|
||||
if sync_kind:
|
||||
if cache:
|
||||
for product, decision in zip(to_process, decisions):
|
||||
state.remember_decision(product["CODIGO"], product_fingerprint(product), decision)
|
||||
state.forget_missing([p["CODIGO"] for p in products])
|
||||
all_decisions = [state.cached_decision(p["CODIGO"]) or {} for p in products]
|
||||
else:
|
||||
all_decisions = decisions
|
||||
coverage = build_job_coverage(env, client, products, all_decisions)
|
||||
company = {"crt": ((coverage or {}).get("company_context") or {}).get("company_crt")}
|
||||
ncm_ok = {str(p["CODIGO"]): (d or {}).get("status") == "EXACT_NCM" for p, d in zip(products, all_decisions)}
|
||||
fiscal = _unified_fiscal(env, company, ncm_ok)
|
||||
rtc_apply = _rtc_auto_apply(env, config, job, products, fiscal, company)
|
||||
if rtc_apply["write_successes"] or any(r.get("outcome") == "ROLLED_BACK" for r in rtc_apply["rollbacks"]):
|
||||
fiscal = _unified_fiscal(env, company, ncm_ok) # coverage must reflect the fields that were just filled
|
||||
findings = fiscal["findings"]
|
||||
product_states = [classify_product_state(p, d, fiscal=findings.get(str(p["CODIGO"]))) for p, d in zip(products, all_decisions)]
|
||||
operational = summarize_operational_states(product_states)
|
||||
fiscal_context = {**fiscal["context"], "evaluated_all": kind != "REPROCESS_PRODUCTS"}
|
||||
else:
|
||||
# REPROCESS_PRODUCTS (and any other non-sync kind) used to hardcode company=None here - not a missing feature, a real gap: the SAME
|
||||
# deterministic, fail-closed company_context.resolve() every sync job already uses is cheap (no full-table scan) and applies just as
|
||||
# well to a one-off reprocess job. `company` stays None only when the company genuinely cannot be resolved yet - never a silent guess.
|
||||
try:
|
||||
company = {"crt": _firebird_reader.read_company_context(env).get("crt")}
|
||||
except Exception:
|
||||
company = None
|
||||
fiscal = _unified_fiscal(env, company, {str(p["CODIGO"]): (d or {}).get("status") == "EXACT_NCM" for p, d in zip(to_process, decisions)})
|
||||
fiscal_blocked = _fiscal_blocked_products(products, fiscal["findings"])
|
||||
fiscal_codes = {b["product_code"] for b in fiscal_blocked}
|
||||
summary["blocked_products"] = summary["blocked_products"] + fiscal_blocked
|
||||
summary["resolved_products"] = [c for c in summary["resolved_products"] if str(c) not in fiscal_codes]
|
||||
result_summary = {
|
||||
"coverage": coverage,
|
||||
"operational": operational,
|
||||
"fiscal_context": fiscal_context,
|
||||
"rtc_auto_apply": rtc_apply,
|
||||
"fiscal_mode": getattr(config, "fiscal_mode", None),
|
||||
"release_version": env.get("AGENT_VERSION") or "1.3.9",
|
||||
"products_total": len(products),
|
||||
"products_changed": len(to_process),
|
||||
"products_processed": summary["processed"],
|
||||
"blocked_products": summary["blocked_products"],
|
||||
"resolved_products": summary["resolved_products"],
|
||||
"writes_attempted": summary["writes_attempted"],
|
||||
"writes_succeeded": summary["writes_succeeded"],
|
||||
"writes_failed": summary["writes_failed"],
|
||||
"no_ops": summary["no_ops"],
|
||||
"firebird_writes_enabled": config.firebird_writes,
|
||||
"agent_id": job["agent_id"],
|
||||
"schema_fingerprint": config.schema_fingerprint,
|
||||
}
|
||||
logger.info(
|
||||
"job %s finished: processed=%d writes_attempted=%d writes_succeeded=%d writes_failed=%d no_ops=%d",
|
||||
job.get("job_id"), summary["processed"], summary["writes_attempted"], summary["writes_succeeded"], summary["writes_failed"], summary["no_ops"],
|
||||
)
|
||||
if summary["writes_failed"] > 0:
|
||||
return result_summary, f"{summary['writes_failed']} write(s) failed guard checks"
|
||||
return result_summary, None
|
||||
except executor.WriteRefused as error:
|
||||
logger.warning("job %s refused: %s", job.get("job_id"), error)
|
||||
return None, f"write refused: {error}"
|
||||
except Exception as error: # noqa: BLE001 — job-domain isolation boundary
|
||||
logger.warning("job %s failed internally: %s: %s", job.get("job_id"), type(error).__name__, error)
|
||||
return None, f"{type(error).__name__}: {error}"
|
||||
|
||||
|
||||
BUSY_HEARTBEAT_SECONDS = 20
|
||||
|
||||
|
||||
def _busy_heartbeats(client, installation_id, config, job_id, done):
|
||||
while not done.wait(BUSY_HEARTBEAT_SECONDS):
|
||||
try:
|
||||
client.heartbeat_metadata({"installation_id": installation_id, "status": "BUSY", "schema_fingerprint": config.schema_fingerprint, "agent_version": config.agent_version, "last_job_id": job_id})
|
||||
except Exception: # noqa: BLE001 — Hub hiccup mid-job must not affect the job
|
||||
pass
|
||||
|
||||
|
||||
def resolve_installation_id(config, env, state, client):
|
||||
"""Recover and persist the Hub-bound identity; never invent one for an Agent."""
|
||||
persisted = state.stored_installation_id()
|
||||
configured = (env.get("INSTALLATION_ID") or "").strip() or None
|
||||
if persisted is not None:
|
||||
return state.installation_id(configured)
|
||||
if configured is not None:
|
||||
return state.installation_id(configured)
|
||||
|
||||
configured_agent_id = str(env.get("AGENT_ID") or config.agent_id or "").strip()
|
||||
if not configured_agent_id:
|
||||
raise RuntimeError("AGENT_ID is required to recover installation identity")
|
||||
identity = client.identity()
|
||||
if str(identity.get("agent_id") or "") != configured_agent_id:
|
||||
raise RuntimeError("authenticated identity does not match configured Agent")
|
||||
installation_id = identity.get("installation_id")
|
||||
if not installation_id:
|
||||
raise RuntimeError("authenticated identity response has no installation_id")
|
||||
return state.installation_id(str(installation_id))
|
||||
|
||||
|
||||
def _company_context_for_heartbeat(env, cached):
|
||||
"""Resolves company UF/CRT via the SAME deterministic, fail-closed resolver already used for coverage/RTC (`company_context.resolve`,
|
||||
called through `preflight_company_context`) - never a second, separate guess. Previously the heartbeat only ever sent `COMPANY_UF` from an
|
||||
operator-set env var and never sent CRT at all, so `Installation.company_crt` stayed permanently empty in the Hub even though the Agent
|
||||
already proves CRT from C000004/EMISSOR for every coverage job. Resolves once and caches: never re-queries Firebird once solid, retries only
|
||||
while still unresolved (e.g. the database was not yet reachable when the service started)."""
|
||||
if cached is not None:
|
||||
return cached, None
|
||||
try:
|
||||
from gti_fiscal_agent.company_context import preflight_company_context
|
||||
result = preflight_company_context(env)
|
||||
except Exception as error:
|
||||
return None, f"company context not resolved yet: {error}"
|
||||
if result.get("status") != "PASS":
|
||||
return None, f"company context not resolved yet: {result.get('classification')} ({'; '.join(result.get('reasons', []))})"
|
||||
return result, None
|
||||
|
||||
|
||||
def _start_enrichment(config, env, client, stopper):
|
||||
"""New-product enrichment worker (own thread, own SQLite file). It gates ITSELF on the Hub's per-installation features: DISABLED / not configured = zero Firebird activity."""
|
||||
if not hasattr(client, "resolve_product"):
|
||||
return None
|
||||
store = _enrichment.EnrichmentStore(Path(config.state_db).with_name("enrichment_state.db"))
|
||||
worker = _enrichment.EnrichmentWorker(hub=client, store=store, read_conn=lambda: _firebird_reader._connect(env, True), write_conn=lambda: open_write_connection(env, config.database_path),
|
||||
create_reprocess=lambda codes: client.create_job("REPROCESS_PRODUCTS", {"codigos": list(codes), "reason": "ENRICHMENT"}))
|
||||
threading.Thread(target=worker.run_forever, args=(stopper,), name="product-enrichment", daemon=True).start()
|
||||
return worker
|
||||
|
||||
|
||||
def run(config, env, *, max_iterations=None, stopper=None):
|
||||
"""`stopper` defaults to a signal-based GracefulStop (SIGINT/SIGTERM), which
|
||||
only works from the main thread. A host that manages its own lifecycle
|
||||
(e.g. the Windows Service wrapper, which stops the loop via the Service
|
||||
Control Manager instead of OS signals) should pass its own stop object —
|
||||
anything with a `.stop` boolean attribute — and flip it from any thread."""
|
||||
executor = load_executor()
|
||||
state = AgentState(config.state_db)
|
||||
client = HubClient(config.hub_url, config.agent_token)
|
||||
installation_id = resolve_installation_id(config, env, state, client)
|
||||
stopper = stopper if stopper is not None else GracefulStop()
|
||||
if max_iterations is None:
|
||||
_start_enrichment(config, env, client, stopper)
|
||||
|
||||
backoff = 1
|
||||
was_down = False
|
||||
iterations = 0
|
||||
last_job_id = None
|
||||
company_ctx = None
|
||||
while not stopper.stop:
|
||||
if max_iterations is not None and iterations >= max_iterations:
|
||||
break
|
||||
iterations += 1
|
||||
|
||||
try:
|
||||
company_ctx, reason = _company_context_for_heartbeat(env, company_ctx)
|
||||
if reason:
|
||||
logger.debug(reason)
|
||||
heartbeat_payload = {
|
||||
"installation_id": installation_id, "status": "IDLE", "schema_fingerprint": config.schema_fingerprint,
|
||||
"agent_version": config.agent_version, "last_job_id": last_job_id, "display_name": env.get("AGENT_DISPLAY_NAME", ""),
|
||||
"customer_code": env.get("CUSTOMER_CODE"), "hostname": env.get("AGENT_HOSTNAME") or os.environ.get("COMPUTERNAME"),
|
||||
"windows_version": env.get("WINDOWS_VERSION"), "company_code": config.company_id, "company_uf": (company_ctx or {}).get("company_uf") or config.company_uf,
|
||||
"company_crt": (company_ctx or {}).get("company_crt"), "company_contribuinte_ipi": (company_ctx or {}).get("company_contribuinte_ipi"),
|
||||
"database_path": config.database_path, "capabilities": {"heartbeat": True, "jobs": True, "local_logs": True, "RTC_CCLASSTRIB_WRITE_V1": True, "PRODUCT_ENRICHMENT_WRITE_V1": True},
|
||||
"fiscal_engine_version": env.get("FISCAL_ENGINE_VERSION", config.agent_version), "dataset_versions": {},
|
||||
}
|
||||
if hasattr(client, "heartbeat_metadata"):
|
||||
hb_response = client.heartbeat_metadata(heartbeat_payload)
|
||||
else:
|
||||
hb_response = client.heartbeat("IDLE", schema_fingerprint=config.schema_fingerprint, agent_version=config.agent_version, last_job_id=last_job_id)
|
||||
processing_enabled = not (isinstance(hb_response, dict) and hb_response.get("processing_enabled") is False)
|
||||
if was_down:
|
||||
logger.info("reconnected to Hub after %ss backoff", backoff)
|
||||
was_down = False
|
||||
backoff = 1
|
||||
except Exception as error:
|
||||
logger.warning("heartbeat failed, Hub unreachable: %s (retrying in %ss)", error, backoff)
|
||||
was_down = True
|
||||
time.sleep(backoff)
|
||||
backoff = min(backoff * 2, _MAX_BACKOFF_SECONDS)
|
||||
continue
|
||||
|
||||
if not processing_enabled: # central pause: heartbeat keeps flowing, no job is claimed or executed
|
||||
time.sleep(config.poll_interval_seconds)
|
||||
continue
|
||||
|
||||
try:
|
||||
job = client.next_job()
|
||||
except Exception as error:
|
||||
logger.warning("poll failed, Hub unreachable: %s (retrying in %ss)", error, backoff)
|
||||
was_down = True
|
||||
time.sleep(backoff)
|
||||
backoff = min(backoff * 2, _MAX_BACKOFF_SECONDS)
|
||||
continue
|
||||
|
||||
if job is None:
|
||||
time.sleep(config.poll_interval_seconds)
|
||||
continue
|
||||
|
||||
last_job_id = job.get("job_id")
|
||||
logger.info("job %s received: kind=%s", job.get("job_id"), job["kind"])
|
||||
try:
|
||||
if hasattr(client, "heartbeat_metadata"):
|
||||
client.heartbeat_metadata({"installation_id": installation_id, "status": "BUSY", "schema_fingerprint": config.schema_fingerprint, "agent_version": config.agent_version, "last_job_id": last_job_id})
|
||||
else:
|
||||
client.heartbeat("BUSY", schema_fingerprint=config.schema_fingerprint, agent_version=config.agent_version, last_job_id=last_job_id)
|
||||
except Exception:
|
||||
pass # best-effort; the job itself still proceeds
|
||||
|
||||
# A full sync can take minutes: keep the heartbeat alive (BUSY) from a side thread so the
|
||||
# panel never shows a working Agent as OFFLINE. The side thread only talks to the Hub
|
||||
# (no SQLite/Firebird), so it cannot interfere with the job.
|
||||
done = threading.Event()
|
||||
beat = threading.Thread(target=_busy_heartbeats, args=(client, installation_id, config, last_job_id, done), daemon=True)
|
||||
beat.start()
|
||||
try:
|
||||
result_summary, error = execute_job(job, config, executor, state, env)
|
||||
finally:
|
||||
done.set()
|
||||
try:
|
||||
if error is None:
|
||||
client.complete_job(job.get("job_id"), result_summary)
|
||||
logger.info("job %s reported COMPLETED", job.get("job_id"))
|
||||
else:
|
||||
client.fail_job(job.get("job_id"), error)
|
||||
logger.warning("job %s reported FAILED: %s", job.get("job_id"), error)
|
||||
except Exception as report_error:
|
||||
logger.warning("could not report job %s result to Hub: %s", job.get("job_id"), report_error)
|
||||
|
||||
|
||||
def enforce_startup_preflight(env):
|
||||
"""Full read-only preflight at startup (Hub, auth, Firebird, capabilities, company context, guards). Fail closed."""
|
||||
from gti_fiscal_agent.preflight import run_preflight
|
||||
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.9")
|
||||
logger.info("startup preflight: %s", json.dumps({k: report[k] for k in ("PREFLIGHT", "blockers", "fiscal_mode", "effective_firebird_writes")}, ensure_ascii=False))
|
||||
print(json.dumps({"PREFLIGHT": report["PREFLIGHT"], "blockers": report["blockers"]}, ensure_ascii=False))
|
||||
if report["PREFLIGHT"] != "PASS":
|
||||
logger.error("startup aborted: preflight failed (fail closed): %s", report["blockers"])
|
||||
raise SystemExit(2)
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--max-iterations", type=int, default=None, help="stop after N poll cycles (used by tests); omit to run forever")
|
||||
args = parser.parse_args()
|
||||
|
||||
load_dotenv(Path(os.environ.get("GTI_AGENT_ENV_FILE") or PROJECT_ROOT / ".env"))
|
||||
config = Config(os.environ)
|
||||
setup_logging(config.log_path)
|
||||
|
||||
logger.info("startup: agent_id=%s company_id=%s agent_version=%s firebird_writes=%s", config.agent_id, config.company_id, config.agent_version, config.firebird_writes)
|
||||
hub_ok = HubClient(config.hub_url, config.agent_token).health()
|
||||
firebird_ok = check_connection(os.environ)
|
||||
logger.info("startup checks: hub_ok=%s firebird_ok=%s", hub_ok, firebird_ok)
|
||||
enforce_startup_preflight(os.environ)
|
||||
|
||||
run(config, os.environ, max_iterations=args.max_iterations)
|
||||
logger.info("shutdown: stopped cleanly")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Windows Service wrapper around run_agent_service.run() — reuses it as-is,
|
||||
does not reimplement any polling/pipeline logic.
|
||||
|
||||
Requires pywin32 (`pip install pywin32`), Windows-only.
|
||||
|
||||
Commands (run from an elevated/administrator terminal):
|
||||
python gti_agent_windows_service.py install # register the service, Startup=Automatic
|
||||
python gti_agent_windows_service.py start
|
||||
python gti_agent_windows_service.py stop
|
||||
python gti_agent_windows_service.py restart
|
||||
python gti_agent_windows_service.py remove # uninstall
|
||||
python gti_agent_windows_service.py --startup=auto --username . --password . install
|
||||
|
||||
Recovery (restart on failure) is not set by pywin32's `install` alone; run once
|
||||
after installing:
|
||||
sc.exe failure "GTIFiscalAgent" reset= 86400 actions= restart/60000/restart/60000/restart/60000
|
||||
|
||||
This module only wraps agent/scripts/run_agent_service.py's `run()` — the
|
||||
actual polling/heartbeat/pipeline logic lives there, unchanged.
|
||||
"""
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(AGENT_ROOT / "scripts"))
|
||||
sys.path.insert(0, str(AGENT_ROOT / "src"))
|
||||
|
||||
import run_agent_service as agent_service # noqa: E402
|
||||
|
||||
try:
|
||||
import servicemanager
|
||||
import win32event
|
||||
import win32service
|
||||
import win32serviceutil
|
||||
except ImportError: # pragma: no cover — only available on Windows with pywin32 installed
|
||||
servicemanager = win32event = win32service = win32serviceutil = None
|
||||
|
||||
|
||||
class _ServiceStopFlag:
|
||||
"""Plain stop flag (no signal.signal registration) — safe to flip from any
|
||||
thread. The Windows Service Control Manager stops the service, not a
|
||||
console SIGINT/SIGTERM, so this replaces GracefulStop for this host."""
|
||||
def __init__(self):
|
||||
self.stop = False
|
||||
|
||||
|
||||
if win32serviceutil is not None:
|
||||
|
||||
class GTIFiscalAgentService(win32serviceutil.ServiceFramework):
|
||||
_svc_name_ = "GTIFiscalAgent"
|
||||
_svc_display_name_ = "GTI Fiscal Agent"
|
||||
_svc_description_ = "Reads the local Firebird ERP, polls the GTI Fiscal Hub for jobs, and applies guarded fiscal updates."
|
||||
|
||||
def __init__(self, args):
|
||||
win32serviceutil.ServiceFramework.__init__(self, args)
|
||||
self.stop_event = win32event.CreateEvent(None, 0, 0, None)
|
||||
self.stop_flag = _ServiceStopFlag()
|
||||
self._thread = None
|
||||
|
||||
def SvcStop(self):
|
||||
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
|
||||
self.stop_flag.stop = True
|
||||
win32event.SetEvent(self.stop_event)
|
||||
|
||||
def SvcDoRun(self):
|
||||
servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE, servicemanager.PYS_SERVICE_STARTED, (self._svc_name_, ""))
|
||||
self._thread = threading.Thread(target=self._run_agent, daemon=True)
|
||||
self._thread.start()
|
||||
win32event.WaitForSingleObject(self.stop_event, win32event.INFINITE)
|
||||
|
||||
def _run_agent(self):
|
||||
import os
|
||||
agent_service.load_dotenv(Path(os.environ.get("GTI_AGENT_ENV_FILE") or agent_service.PROJECT_ROOT / ".env"))
|
||||
config = agent_service.Config(os.environ)
|
||||
agent_service.setup_logging(config.log_path)
|
||||
try:
|
||||
agent_service.enforce_startup_preflight(os.environ) # fail closed: a failing preflight stops the service
|
||||
except SystemExit:
|
||||
self.stop_flag.stop = True
|
||||
servicemanager.LogErrorMsg("GTIFiscalAgent: startup preflight failed; see the agent log")
|
||||
win32event.SetEvent(self.stop_event)
|
||||
return
|
||||
try:
|
||||
agent_service.run(config, os.environ, stopper=self.stop_flag)
|
||||
except BaseException as error: # noqa: BLE001 - a dead loop must never look like a healthy "Running" service
|
||||
import logging
|
||||
logging.getLogger("gti_fiscal_agent").exception("fatal error in the Agent loop; stopping the service")
|
||||
servicemanager.LogErrorMsg(f"GTIFiscalAgent: fatal error, stopping: {type(error).__name__}: {error}")
|
||||
self.stop_flag.stop = True
|
||||
win32event.SetEvent(self.stop_event)
|
||||
|
||||
|
||||
def main():
|
||||
if win32serviceutil is None:
|
||||
raise SystemExit("pywin32 is required: pip install pywin32")
|
||||
if len(sys.argv) == 1:
|
||||
# SCM launches a service executable without CLI arguments. HandleCommandLine
|
||||
# requires a command (install/start/debug) and exits with usage when none
|
||||
# is present, which makes SCM report error 1053 for a frozen service host.
|
||||
servicemanager.Initialize()
|
||||
servicemanager.PrepareToHostSingle(GTIFiscalAgentService)
|
||||
servicemanager.StartServiceCtrlDispatcher()
|
||||
else:
|
||||
win32serviceutil.HandleCommandLine(GTIFiscalAgentService)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
"""Read-only ERP to Fiscal Hub agent."""
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Pure capability detection / schema classification for SGV (C000025/C000004) databases. No I/O, no writes."""
|
||||
|
||||
WRITE_TARGETS = ["CLASSIFICACAO_FISCAL", "CFOP_ENTRADA", "CST_ENTRADA", "CFOP", "CST", "ALIQUOTA", "CEST", "SITUACAO_TRIBUTARIA", "COD_ENTRADA_COFINS",
|
||||
"COD_SAIDA_COFINS", "ALIQ_COFINS", "COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "CSOSN", "IPI", "CST_IPI", "IAT", "INCIDENCIA_PISCOFINS",
|
||||
"PISCOFINS", "IPPT"]
|
||||
ORIGIN_FIELD = "COD_ORIGEM_MERCADORIA"
|
||||
|
||||
# Firebird RDB$FIELD_TYPE -> family
|
||||
FAMILY = {7: "int", 8: "int", 16: "int", 10: "float", 11: "float", 27: "float", 12: "date", 13: "time", 35: "timestamp", 14: "text", 37: "text", 261: "blob"}
|
||||
# accepted families per field (UI edit controls take text or numbers; numeric columns may be int/float)
|
||||
ACCEPT = {"ALIQUOTA": {"float", "int"}, "IPI": {"float", "int"}, "COD_ORIGEM_MERCADORIA": {"int"}}
|
||||
DEFAULT_ACCEPT = {"text", "int"} # text columns; some ERPs keep codes such as CST_ENTRADA in SMALLINT
|
||||
|
||||
CAPABILITIES = {
|
||||
"supports_ncm": ["CLASSIFICACAO_FISCAL"],
|
||||
"supports_cfop_entry": ["CFOP_ENTRADA"],
|
||||
"supports_cfop_exit": ["CFOP"],
|
||||
"supports_cst_entry": ["CST_ENTRADA"],
|
||||
"supports_cst_exit": ["CST"],
|
||||
"supports_icms_rate": ["ALIQUOTA"],
|
||||
"supports_cest": ["CEST"],
|
||||
"supports_csosn": ["CSOSN"],
|
||||
"supports_ipi": ["IPI", "CST_IPI"],
|
||||
"supports_pis": ["COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS"],
|
||||
"supports_cofins": ["COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS"],
|
||||
"supports_origin": [ORIGIN_FIELD],
|
||||
"supports_iat": ["IAT"],
|
||||
"supports_incidence": ["INCIDENCIA_PISCOFINS"],
|
||||
"supports_ippt": ["IPPT"],
|
||||
}
|
||||
|
||||
|
||||
def type_ok(field, type_code):
|
||||
return FAMILY.get(type_code, "other") in ACCEPT.get(field, DEFAULT_ACCEPT)
|
||||
|
||||
|
||||
def detect_capabilities(columns):
|
||||
"""columns: {FIELD: rdb_type_code} of C000025. capability=true only if every required column exists with a usable type."""
|
||||
out = {}
|
||||
for cap, fields in CAPABILITIES.items():
|
||||
out[cap] = all(f in columns and type_ok(f, columns[f]) for f in fields)
|
||||
return out
|
||||
|
||||
|
||||
def field_compatibility(columns, reference_types=None):
|
||||
"""Per write target (+ origin): PRESENT_OK / PRESENT_TYPE_VARIATION / MISSING."""
|
||||
res = {}
|
||||
for f in WRITE_TARGETS + [ORIGIN_FIELD]:
|
||||
if f not in columns:
|
||||
res[f] = "MISSING"
|
||||
elif not type_ok(f, columns[f]):
|
||||
res[f] = "PRESENT_TYPE_INCOMPATIBLE"
|
||||
elif reference_types and reference_types.get(f) is not None and reference_types[f] != columns[f]:
|
||||
res[f] = "PRESENT_TYPE_VARIATION"
|
||||
else:
|
||||
res[f] = "PRESENT_OK"
|
||||
return res
|
||||
|
||||
|
||||
def classify_schema(has_c000004, has_c000025, fingerprint, reference_fingerprint, compat):
|
||||
"""REFERENCE_SCHEMA_MATCH | COMPATIBLE_WITH_VARIATIONS | INCOMPATIBLE. A different hash is not a failure
|
||||
when every required capability exists (origin is optional: it is only PROBABLE in the map)."""
|
||||
if not (has_c000004 and has_c000025):
|
||||
return "INCOMPATIBLE"
|
||||
required = {f: v for f, v in compat.items() if f in WRITE_TARGETS}
|
||||
if any(v in ("MISSING", "PRESENT_TYPE_INCOMPATIBLE") for v in required.values()):
|
||||
return "INCOMPATIBLE"
|
||||
if fingerprint == reference_fingerprint:
|
||||
return "REFERENCE_SCHEMA_MATCH"
|
||||
return "COMPATIBLE_WITH_VARIATIONS"
|
||||
|
||||
|
||||
NAME_KEYWORDS = {"RESTAURANT": ("RESTAURANTE", "PIZZARIA", "LANCHONETE", "CHURRASC"), "RETAIL_MARKET": ("MERCADO", "SUPERMERC", "MERCEARIA", "ATACAD"),
|
||||
"APPAREL": ("MODAS", "CONFEC", "BOUTIQUE", "ROUPAS"), "CONSTRUCTION": ("CONSTRU", "MATERIAIS"), "AUTO_REPAIR": ("OFICINA", "MECANICA", "AUTO PECAS", "AUTOPECAS")}
|
||||
|
||||
|
||||
def infer_segment(signals, company_name=""):
|
||||
"""Evidence-based segment guess from populated tables/flags (never from the file name).
|
||||
signals: {"restaurant_tables_rows": int, "kitchen_products": int, "recipes": int, "service_orders": int, "vehicle_products": int, "grade_items": int, "nfce_items": int}"""
|
||||
ev = []
|
||||
score = {"RESTAURANT": 0, "AUTO_REPAIR": 0, "APPAREL": 0, "RETAIL_MARKET": 0, "CONSTRUCTION": 0}
|
||||
import unicodedata
|
||||
up = unicodedata.normalize("NFKD", company_name or "").encode("ascii", "ignore").decode().upper()
|
||||
for seg_name, kws in NAME_KEYWORDS.items():
|
||||
hit = [k for k in kws if k in up]
|
||||
if hit:
|
||||
score[seg_name] += 1; ev.append(f"company name in C000004 contains {hit[0]} (weak, name is data, not the file name)")
|
||||
if signals.get("restaurant_tables_rows", 0) > 50: score["RESTAURANT"] += 2; ev.append(f"mesas/comandas rows={signals['restaurant_tables_rows']}")
|
||||
if signals.get("kitchen_products", 0) > 0: score["RESTAURANT"] += 1; ev.append(f"PROD_COZINHA products={signals['kitchen_products']}")
|
||||
if signals.get("recipes", 0) > 0: score["RESTAURANT"] += 1; ev.append(f"recipes={signals['recipes']}")
|
||||
if signals.get("service_orders", 0) > 20: score["AUTO_REPAIR"] += 2; ev.append(f"ordens de serviço={signals['service_orders']}")
|
||||
if signals.get("vehicle_products", 0) > 0: score["AUTO_REPAIR"] += 1; ev.append(f"veículos={signals['vehicle_products']}")
|
||||
if signals.get("grade_items", 0) > 50: score["APPAREL"] += 2; ev.append(f"grade (tamanho/cor) rows={signals['grade_items']}")
|
||||
if signals.get("nfce_items", 0) > 1000 and max(score.values()) == 0: score["RETAIL_MARKET"] += 1; ev.append("high-volume NFC-e retail without restaurant/OS/grade signals")
|
||||
best = max(score, key=score.get)
|
||||
return (best if score[best] > 0 else "UNKNOWN"), score, ev
|
||||
@@ -0,0 +1,185 @@
|
||||
"""CFOP/IPPT auto-apply: fills or corrects ONLY C000025.CFOP and C000025.IPPT, and nothing else,
|
||||
when central.app.field_contract.resolve_product already classified the cell AUTO_FILL_SAFE
|
||||
(state=CHANGE, write_eligible=True) for this product.
|
||||
|
||||
Safety model (defense in depth, default OFF), mirrors gti_fiscal_agent.rtc_auto_apply: the Hub sets
|
||||
a per-installation, per-capability mode DISABLED / DRY_RUN / APPROVAL_REQUIRED / ENABLED and delivers
|
||||
it inside an authenticated job (central.app.cfop_ippt_audit.job_authorizations). The Agent only
|
||||
exposes the restricted capability CFOP_IPPT_WRITE_V1; there is no local .env switch. Write allowlist:
|
||||
ALLOWED_FIELDS = ("CFOP", "IPPT") on C000025 - no free SQL, one short transaction per cell.
|
||||
|
||||
Compare-before-write ("human override sempre vence"): every candidate carries the exact value the
|
||||
field_contract resolver saw when it was selected (`expected_before`). Immediately before the UPDATE
|
||||
the cell is re-read; if it no longer equals `expected_before` - a human (or any other process) changed
|
||||
it since - the write is ABORTED with STALE_VALUE_CHANGED_SINCE_DECISION and nothing is written; if it
|
||||
already equals the proposed value the write is skipped as NO_WRITE_REQUIRED (idempotent - a second run
|
||||
over the same candidates always writes 0). Every applied change is read back and verified; a mismatch
|
||||
never returns APPLIED. `rollback_one` restores the previous value only if the current value still
|
||||
equals what was written (a later human edit blocks the rollback too).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
CAPABILITY = "CFOP_IPPT_WRITE_V1"
|
||||
ALLOWED_FIELDS = ("CFOP", "IPPT")
|
||||
MODES = ("DISABLED", "DRY_RUN", "APPROVAL_REQUIRED", "ENABLED")
|
||||
DEFAULT_MAX_PER_JOB = 500
|
||||
|
||||
|
||||
class CfopIpptApplyAborted(Exception):
|
||||
"""The cell is left untouched; `reason` is reported in the audit."""
|
||||
|
||||
def __init__(self, reason: str):
|
||||
super().__init__(reason)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
def _text(value) -> str:
|
||||
return "" if value is None else str(value).strip()
|
||||
|
||||
|
||||
def write_gate(*, mode: str, capability: bool = True, approved: bool = True) -> dict:
|
||||
"""Same shape/semantics as rtc_auto_apply.write_gate: a real write needs the capability AND an
|
||||
authenticated Hub mode that permits writing (ENABLED, or APPROVAL_REQUIRED for approved cells)."""
|
||||
mode = mode if mode in MODES else "DISABLED"
|
||||
missing = []
|
||||
if not capability:
|
||||
missing.append("CAPABILITY")
|
||||
if mode not in ("ENABLED", "APPROVAL_REQUIRED"):
|
||||
missing.append("HUB_MODE_" + mode)
|
||||
elif mode == "APPROVAL_REQUIRED" and not approved:
|
||||
missing.append("HUB_APPROVAL")
|
||||
return {"allowed": not missing, "missing": missing, "mode": mode, "capability": bool(capability)}
|
||||
|
||||
|
||||
def candidate_fingerprint(code: str, field: str, expected_before, proposed) -> str:
|
||||
import hashlib
|
||||
return hashlib.sha256("|".join((_text(code), field, _text(expected_before), _text(proposed))).encode("utf-8")).hexdigest()[:24]
|
||||
|
||||
|
||||
def select_candidates(results: dict, *, limit: int = DEFAULT_MAX_PER_JOB) -> tuple[list[dict], int]:
|
||||
"""`results` is {product_code: {field: {"state": ..., "current": ..., "proposed": ..., "write_eligible": ..., "reason": ...}}} -
|
||||
the SAME per-field dict central.app.field_contract.resolve_product already returns for every product in the job's dry-run pass.
|
||||
Only CFOP/IPPT cells with state=CHANGE and write_eligible=True are candidates - never a second, independent rule."""
|
||||
from central.app.field_contract import CHANGE
|
||||
ready = []
|
||||
for code, fields in results.items():
|
||||
for field in ALLOWED_FIELDS:
|
||||
r = (fields or {}).get(field) or {}
|
||||
if r.get("state") != CHANGE or not r.get("write_eligible"):
|
||||
continue
|
||||
expected_before, proposed = r.get("current"), r.get("proposed")
|
||||
ready.append({"product_code": code, "field": field, "expected_before": expected_before, "proposed": proposed, "reason": r.get("reason"),
|
||||
"fingerprint": candidate_fingerprint(code, field, expected_before, proposed)})
|
||||
return ready[:limit], len(ready)
|
||||
|
||||
|
||||
def apply_candidate(conn, candidate: dict, *, now: str, job_id=None) -> dict:
|
||||
"""One cell, one short transaction. Raises CfopIpptApplyAborted (nothing written)."""
|
||||
field = candidate["field"]
|
||||
assert field in ALLOWED_FIELDS
|
||||
code, expected_before, proposed = candidate["product_code"], candidate["expected_before"], candidate["proposed"]
|
||||
base = {"product_code": code, "field": field, "job_id": job_id, "timestamp": now, "action": "APPLY", "reason_code": candidate.get("reason")}
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
rows = cur.fetchall()
|
||||
if len(rows) != 1:
|
||||
raise CfopIpptApplyAborted("PRODUCT_ROW_NOT_UNIQUE" if rows else "PRODUCT_NOT_FOUND")
|
||||
current = _text(rows[0][0]) or None
|
||||
norm_expected = _text(expected_before) or None
|
||||
if current == (_text(proposed) or None):
|
||||
raise CfopIpptApplyAborted("NO_WRITE_REQUIRED")
|
||||
if current != norm_expected:
|
||||
raise CfopIpptApplyAborted("STALE_VALUE_CHANGED_SINCE_DECISION")
|
||||
cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND (({field} IS NULL) OR (TRIM({field}) = ?))", (proposed, code, norm_expected or ""))
|
||||
if cur.rowcount != 1:
|
||||
conn.rollback()
|
||||
raise CfopIpptApplyAborted("UPDATE_ROWCOUNT_NOT_ONE")
|
||||
conn.commit()
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
after = _text(cur.fetchone()[0]) or None
|
||||
if after != (_text(proposed) or None):
|
||||
raise RuntimeError(f"post-write verification failed for {code}.{field}: {after!r} != {proposed!r}")
|
||||
return {**base, "outcome": "APPLIED", "old_value": current, "new_value": proposed, "rowcount": 1, "verified_after": True}
|
||||
except CfopIpptApplyAborted:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
raise
|
||||
except Exception:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def apply_candidates(conn, candidates: list[dict], *, now: str, job_id=None, authorize=None) -> dict:
|
||||
"""Applies each candidate independently; a failure or an abort never stops the rest and never
|
||||
leaves a partial change. `authorize(product_code, field)` (optional) re-asks the Hub, right before
|
||||
each UPDATE, whether the central authorization still permits it."""
|
||||
applied, aborted, reasons = [], [], {}
|
||||
for candidate in candidates:
|
||||
code, field = candidate["product_code"], candidate["field"]
|
||||
if authorize is not None:
|
||||
denied = authorize(code, field)
|
||||
if denied:
|
||||
aborted.append({"outcome": "ABORTED", "product_code": code, "field": field, "reason": denied, "job_id": job_id, "timestamp": now, "action": "APPLY"})
|
||||
reasons[denied] = reasons.get(denied, 0) + 1
|
||||
continue
|
||||
try:
|
||||
applied.append(apply_candidate(conn, candidate, now=now, job_id=job_id))
|
||||
except CfopIpptApplyAborted as stop:
|
||||
aborted.append({"outcome": "ABORTED", "product_code": code, "field": field, "reason": stop.reason, "job_id": job_id, "timestamp": now, "action": "APPLY"})
|
||||
reasons[stop.reason] = reasons.get(stop.reason, 0) + 1
|
||||
except Exception: # noqa: BLE001 - a technical failure of ONE cell must not abort the batch
|
||||
aborted.append({"outcome": "FAILED", "product_code": code, "field": field, "reason": "TECHNICAL_FAILURE", "job_id": job_id, "timestamp": now})
|
||||
reasons["TECHNICAL_FAILURE"] = reasons.get("TECHNICAL_FAILURE", 0) + 1
|
||||
return {"applied": applied, "aborted": aborted, "aborted_reasons": reasons, "write_attempts": len(candidates), "write_successes": len(applied)}
|
||||
|
||||
|
||||
def rollback_one(conn, item: dict, *, now: str, job_id=None) -> dict:
|
||||
"""Restores C000025.<field> to `old_value` for ONE previously applied audit record, only if the
|
||||
current value still equals what was written (`new_value`) - a later human edit blocks the rollback."""
|
||||
field = item.get("field")
|
||||
if field not in ALLOWED_FIELDS:
|
||||
return {"outcome": "ABORTED", "reason": "UNKNOWN_FIELD", "field": field, "job_id": job_id, "timestamp": now}
|
||||
code, written, restore_to = _text(item.get("product_code")), _text(item.get("new_value")) or None, item.get("old_value")
|
||||
base = {"product_code": code, "field": field, "job_id": job_id, "timestamp": now, "action": "ROLLBACK", "rollback_of": item.get("audit_id"), "old_value": written, "new_value": restore_to}
|
||||
if not code:
|
||||
return {**base, "outcome": "ABORTED", "reason": "ROLLBACK_REQUEST_INVALID"}
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
rows = cur.fetchall()
|
||||
if len(rows) != 1:
|
||||
conn.rollback()
|
||||
return {**base, "outcome": "ABORTED", "reason": "PRODUCT_ROW_NOT_UNIQUE" if rows else "PRODUCT_NOT_FOUND"}
|
||||
current = _text(rows[0][0]) or None
|
||||
if current != written:
|
||||
conn.rollback()
|
||||
return {**base, "outcome": "ABORTED", "reason": "ROLLBACK_CONFLICT_VALUE_CHANGED"}
|
||||
cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND (({field} IS NULL) OR (TRIM({field}) = ?))", (restore_to, code, written or ""))
|
||||
if cur.rowcount != 1:
|
||||
conn.rollback()
|
||||
return {**base, "outcome": "ABORTED", "reason": "UPDATE_ROWCOUNT_NOT_ONE"}
|
||||
conn.commit()
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
after = _text(cur.fetchone()[0]) or None
|
||||
if after != (_text(restore_to) or None):
|
||||
raise RuntimeError("post-rollback verification failed")
|
||||
return {**base, "outcome": "ROLLED_BACK", "rowcount": 1, "verified_after": True}
|
||||
except Exception as error: # noqa: BLE001
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return {**base, "outcome": "FAILED", "reason": type(error).__name__}
|
||||
|
||||
|
||||
def rollback_records(conn, items: list[dict], *, now: str, job_id=None) -> dict:
|
||||
results = [rollback_one(conn, item, now=now, job_id=job_id) for item in items]
|
||||
return {"rollbacks": results, "rolled_back": sum(1 for r in results if r["outcome"] == "ROLLED_BACK")}
|
||||
@@ -0,0 +1,85 @@
|
||||
import time
|
||||
|
||||
import httpx
|
||||
|
||||
TRANSIENT_STATUS = (502, 503, 504) # reverse proxy / Hub restarting
|
||||
BACKOFF_SECONDS = (2, 5, 10, 20, 30, 30, 30, 30, 30) # 10 attempts, ~3 minutes of outage tolerated mid-job
|
||||
|
||||
|
||||
def _resilient(send):
|
||||
"""Runs one idempotent HTTP call, retrying transport failures (connect/TLS/read/protocol errors) and proxy 502/503/504 with backoff, so a network blip in
|
||||
the middle of a long job does not throw the whole job (and minutes of classification) away. Non-transient errors (4xx) are returned untouched."""
|
||||
last = None
|
||||
for attempt in range(len(BACKOFF_SECONDS) + 1):
|
||||
try:
|
||||
response = send()
|
||||
if response.status_code not in TRANSIENT_STATUS or attempt == len(BACKOFF_SECONDS):
|
||||
return response
|
||||
last = None
|
||||
except httpx.TransportError as error:
|
||||
last = error
|
||||
if attempt == len(BACKOFF_SECONDS):
|
||||
raise
|
||||
time.sleep(BACKOFF_SECONDS[attempt])
|
||||
raise last # pragma: no cover
|
||||
|
||||
|
||||
class HubClient:
|
||||
def __init__(self, url, token, chunk_size=25): self.url, self.token, self.chunk_size = url.rstrip("/"), token, chunk_size
|
||||
def _headers(self): return {"Authorization": f"Bearer {self.token}"}
|
||||
def classify_batch(self, products):
|
||||
results = []
|
||||
for start in range(0, len(products), self.chunk_size):
|
||||
chunk = products[start:start + self.chunk_size]
|
||||
# Small chunks + generous timeout + retry with backoff on any transport error or proxy 5xx: a large first sync must survive
|
||||
# a slow Hub, a reverse-proxy read timeout or a TLS/network blip instead of failing the whole job.
|
||||
response = _resilient(lambda: httpx.post(f"{self.url}/v1/classify/batch", json={"products": chunk}, headers=self._headers(), timeout=120))
|
||||
response.raise_for_status(); results.extend(response.json()["results"])
|
||||
return results
|
||||
def get_json(self, path):
|
||||
response = httpx.get(f"{self.url}{path}", headers=self._headers(), timeout=30)
|
||||
response.raise_for_status(); return response.json()
|
||||
def identity(self): return self.get_json("/v1/agents/me/identity")
|
||||
def authorizations(self): return self.get_json("/v1/agents/me/authorizations")
|
||||
def resolve_product(self, payload):
|
||||
"""Asks the Hub to resolve a NEW product (the Agent never contacts external providers). Bounded: the Hub answers within its own time budget."""
|
||||
response = httpx.post(f"{self.url}/v1/products/resolve", json=payload, headers=self._headers(), timeout=20)
|
||||
response.raise_for_status(); return response.json()
|
||||
def pending_enrichment(self):
|
||||
"""Codes the Hub flagged for a new evaluation (panel button, Reprocessar/Sincronizar, KB decision)."""
|
||||
response = httpx.get(f"{self.url}/v1/products/enrichment/pending", headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json().get("codes") or []
|
||||
def report_enrichment(self, payload):
|
||||
response = _resilient(lambda: httpx.post(f"{self.url}/v1/products/enrichment/applied", json=payload, headers=self._headers(), timeout=30))
|
||||
response.raise_for_status(); return response.json()
|
||||
def health(self):
|
||||
response = httpx.get(f"{self.url}/health", timeout=10)
|
||||
return response.status_code == 200
|
||||
def heartbeat(self, status, schema_fingerprint=None, agent_version=None, last_job_id=None):
|
||||
payload = {"status": status, "schema_fingerprint": schema_fingerprint, "agent_version": agent_version, "last_job_id": last_job_id}
|
||||
response = httpx.post(f"{self.url}/v1/agents/heartbeat", json=payload, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def register(self, payload):
|
||||
response = httpx.post(f"{self.url}/v1/agents/register", json=payload, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def heartbeat_metadata(self, payload):
|
||||
response = httpx.post(f"{self.url}/v1/agents/heartbeat", json=payload, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def event(self, payload):
|
||||
response = httpx.post(f"{self.url}/v1/agents/events", json=payload, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def next_job(self):
|
||||
response = httpx.get(f"{self.url}/v1/jobs/next", headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def create_job(self, kind, payload=None):
|
||||
response = httpx.post(f"{self.url}/v1/jobs", json={"kind": kind, "payload": payload or {}}, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def get_job(self, job_id):
|
||||
response = httpx.get(f"{self.url}/v1/jobs/{job_id}", headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def complete_job(self, job_id, result_summary):
|
||||
response = _resilient(lambda: httpx.post(f"{self.url}/v1/jobs/{job_id}/complete", json={"result_summary": result_summary}, headers=self._headers(), timeout=30))
|
||||
response.raise_for_status(); return response.json()
|
||||
def fail_job(self, job_id, error):
|
||||
response = _resilient(lambda: httpx.post(f"{self.url}/v1/jobs/{job_id}/fail", json={"error": str(error)}, headers=self._headers(), timeout=30))
|
||||
response.raise_for_status(); return response.json()
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Deterministic, auditable company-context resolution for an Agent installation (READ-ONLY).
|
||||
|
||||
Evidence (proved on 4 SGV databases, see docs/COMPANY_CONTEXT_RESOLUTION_V1.json):
|
||||
* EMISSOR.EMITENTE_CNPJ/UF/CRT is the emitter block read by the emission programs (GtiVendas_NFCe.exe and Envio NF-e.exe both bind
|
||||
QrEmissor.EMITENTE_*; neither reads C000004.CRT/OPTANTE_SIMPLES). 100% of emitted NFC-e access keys carry the CNPJ of the C000004 row
|
||||
whose CNPJ equals EMISSOR.EMITENTE_CNPJ, and cUF=29.
|
||||
* C000004.CRT / OPTANTE_SIMPLES belong to the ERP filial registry (SPED, filial screen) and can disagree with EMISSOR.
|
||||
Never guesses: first row / lowest code / latest row are NOT used. Ambiguity -> FAIL with the reason, unless the operator configures
|
||||
COMPANY_CODE or COMPANY_CNPJ (company) and COMPANY_CRT_SOURCE = EMISSOR | C000004 (explicit choice when the two CRT sources conflict).
|
||||
"""
|
||||
import hashlib
|
||||
import os
|
||||
|
||||
IBGE_UF = {"11": "RO", "12": "AC", "13": "AM", "14": "RR", "15": "PA", "16": "AP", "17": "TO", "21": "MA", "22": "PI", "23": "CE", "24": "RN", "25": "PB", "26": "PE",
|
||||
"27": "AL", "28": "SE", "29": "BA", "31": "MG", "32": "ES", "33": "RJ", "35": "SP", "41": "PR", "42": "SC", "43": "RS", "50": "MS", "51": "MT", "52": "GO",
|
||||
"53": "DF"}
|
||||
VALID_CRT = {"1", "2", "3"}
|
||||
|
||||
|
||||
class CompanyContextError(Exception):
|
||||
def __init__(self, result):
|
||||
super().__init__(f"COMPANY_CONTEXT=FAIL ({result.get('classification')}): {'; '.join(result.get('reasons', []))}")
|
||||
self.result = result
|
||||
|
||||
|
||||
def digits(value):
|
||||
return "".join(c for c in str(value or "") if c.isdigit())
|
||||
|
||||
|
||||
def mask_cnpj(value):
|
||||
d = digits(value)
|
||||
return f"{d[:2]}**********{d[-2:]}" if len(d) >= 4 else None
|
||||
|
||||
|
||||
def cnpj_fingerprint(value):
|
||||
d = digits(value)
|
||||
return hashlib.sha256(d.encode()).hexdigest()[:8] if d else None
|
||||
|
||||
|
||||
def _s(v):
|
||||
return None if v is None else str(v).strip()
|
||||
|
||||
|
||||
def config_from_env(env):
|
||||
crt_source = (env.get("COMPANY_CRT_SOURCE") or "").strip().upper() or None
|
||||
if crt_source == "AUTO": # AUTO = resolve automatically, only when there is no conflict
|
||||
crt_source = None
|
||||
invalid = False
|
||||
else:
|
||||
invalid = bool(crt_source and crt_source not in ("EMISSOR", "C000004"))
|
||||
uf = (env.get("COMPANY_UF") or "").strip().upper() or None
|
||||
return {"company_code": (env.get("COMPANY_CODE") or "").strip() or None, "company_cnpj": digits(env.get("COMPANY_CNPJ")) or None,
|
||||
"crt_source": crt_source if crt_source in ("EMISSOR", "C000004") else None, "crt_source_invalid": invalid,
|
||||
"uf_override": uf}
|
||||
|
||||
|
||||
def resolve(evidence, config=None):
|
||||
"""evidence: {"c000004": [{code, cnpj, ie, uf, crt, optante, name}], "emissor": [{cnpj, uf, crt}],
|
||||
"nfce_keys": [{"cuf", "cnpj", "n"}], "nf_by_filial": {code: n}}. Pure function."""
|
||||
cfg = config or {}
|
||||
out = {"status": "FAIL", "classification": "UNRESOLVED", "company_code": None, "company_cnpj": None, "company_cnpj_fingerprint": None, "company_uf": None, "company_crt": None,
|
||||
"company_contribuinte_ipi": None, "source_of_truth": {}, "confidence": "NONE", "evidence": [], "warnings": [], "reasons": [], "configured": False}
|
||||
|
||||
def fail(cls, reason):
|
||||
out["classification"] = cls; out["reasons"].append(reason); return out
|
||||
|
||||
if cfg.get("crt_source_invalid"):
|
||||
return fail("CONFIG_REQUIRED", "COMPANY_CRT_SOURCE must be EMISSOR or C000004")
|
||||
companies = [{**c, "cnpj": digits(c.get("cnpj"))} for c in evidence.get("c000004", [])]
|
||||
emissors = [{**e, "cnpj": digits(e.get("cnpj"))} for e in evidence.get("emissor", [])]
|
||||
docs = evidence.get("nfce_keys", [])
|
||||
if not companies:
|
||||
return fail("UNRESOLVED", "C000004 has no company rows")
|
||||
|
||||
# ---- company selection -------------------------------------------------------------------
|
||||
selected = None; basis = None
|
||||
by_code, by_cnpj = cfg.get("company_code"), cfg.get("company_cnpj")
|
||||
if by_code or by_cnpj:
|
||||
out["configured"] = True
|
||||
cand = [c for c in companies if (not by_code or c["code"] == by_code) and (not by_cnpj or c["cnpj"] == by_cnpj)]
|
||||
if len(cand) != 1:
|
||||
return fail("CONFIG_REQUIRED", "configured company (COMPANY_CODE/COMPANY_CNPJ) does not identify exactly one C000004 row")
|
||||
selected, basis = cand[0], "CONFIGURED"
|
||||
elif len(companies) == 1:
|
||||
selected, basis = companies[0], "SINGLE_C000004_ROW"
|
||||
elif len(emissors) == 1 and emissors[0]["cnpj"]:
|
||||
match = [c for c in companies if c["cnpj"] == emissors[0]["cnpj"]]
|
||||
if len(match) != 1:
|
||||
return fail("CONFIG_REQUIRED", f"{len(companies)} C000004 rows and the EMISSOR CNPJ matches {len(match)} of them: configure COMPANY_CODE or COMPANY_CNPJ")
|
||||
selected, basis = match[0], "EMISSOR_CNPJ_MATCH"
|
||||
else:
|
||||
return fail("CONFIG_REQUIRED", f"{len(companies)} C000004 rows and no unique EMISSOR row to disambiguate: configure COMPANY_CODE or COMPANY_CNPJ")
|
||||
out["evidence"].append(f"company selected by {basis}")
|
||||
out.update(company_code=selected["code"], company_cnpj=mask_cnpj(selected["cnpj"]), company_cnpj_fingerprint=cnpj_fingerprint(selected["cnpj"]))
|
||||
out["source_of_truth"]["company"] = basis
|
||||
contribuinte_ipi = _s(selected.get("contribuinte_ipi"))
|
||||
if contribuinte_ipi and contribuinte_ipi.upper() in ("S", "N"): # same row already proven to be THE company; no extra query, no extra trust needed
|
||||
out["company_contribuinte_ipi"] = contribuinte_ipi.upper()
|
||||
out["source_of_truth"]["contribuinte_ipi"] = "C000004"
|
||||
|
||||
# ---- emitter (EMISSOR) must describe the same company --------------------------------------
|
||||
emissor = None
|
||||
if emissors:
|
||||
same = [e for e in emissors if e["cnpj"] and e["cnpj"] == selected["cnpj"]]
|
||||
if same:
|
||||
emissor = same[0]; out["evidence"].append("EMISSOR.EMITENTE_CNPJ equals the selected company CNPJ")
|
||||
else:
|
||||
return fail("CONFLICT", "EMISSOR emitter CNPJ differs from the selected company (unexplained): refusing to guess")
|
||||
else:
|
||||
out["warnings"].append("NO_EMISSOR_ROW")
|
||||
|
||||
# ---- emitted documents must belong to the same CNPJ ------------------------------------------
|
||||
doc_ufs = set()
|
||||
if docs:
|
||||
total = sum(d["n"] for d in docs); own = sum(d["n"] for d in docs if digits(d["cnpj"]) == selected["cnpj"])
|
||||
out["evidence"].append(f"emitted NFC-e keys: {own}/{total} carry the selected CNPJ")
|
||||
if own == 0:
|
||||
return fail("CONFLICT", f"none of the {total} emitted NFC-e keys carries the selected company CNPJ")
|
||||
if own != total: # e.g. a database copy/migration that kept another emitter's history: evidence, not a reason to guess
|
||||
out["warnings"].append(f"HISTORICAL_NFCE_KEYS_FROM_OTHER_CNPJ {total - own}/{total}")
|
||||
doc_ufs = {IBGE_UF.get(d["cuf"]) for d in docs if digits(d["cnpj"]) == selected["cnpj"]} - {None}
|
||||
|
||||
# ---- UF ------------------------------------------------------------------------------------------
|
||||
uf_sources = {}
|
||||
if emissor and _s(emissor.get("uf")): uf_sources["EMISSOR"] = _s(emissor["uf"]).upper()
|
||||
if _s(selected.get("uf")): uf_sources["C000004"] = _s(selected["uf"]).upper()
|
||||
if len(doc_ufs) == 1: uf_sources["DOCUMENT_KEYS"] = next(iter(doc_ufs))
|
||||
if len(doc_ufs) > 1: return fail("CONFLICT", "emitted NFC-e keys carry several UFs")
|
||||
values = set(uf_sources.values())
|
||||
override = cfg.get("uf_override")
|
||||
if len(values) > 1:
|
||||
return fail("CONFLICT", f"UF_SOURCE_CONFLICT {uf_sources}")
|
||||
if override and values and override not in values:
|
||||
return fail("CONFLICT", f"COMPANY_UF={override} differs from the UF proved by the database {sorted(values)}: an override never replaces the resolver silently")
|
||||
if not values and override: # explicit override only when the database proves nothing
|
||||
uf_sources["CONFIG_OVERRIDE"] = override; values = {override}; out["warnings"].append("UF_FROM_EXPLICIT_OVERRIDE")
|
||||
if not values:
|
||||
return fail("UNRESOLVED", "UF_SOURCE_UNKNOWN")
|
||||
out["company_uf"] = next(iter(values)); out["source_of_truth"]["uf"] = "+".join(uf_sources)
|
||||
out["uf_status"] = "UF_SOURCE_CONFIRMED"
|
||||
|
||||
# ---- CRT ------------------------------------------------------------------------------------------
|
||||
e_crt = _s(emissor.get("crt")) if emissor else None
|
||||
c_crt = _s(selected.get("crt"))
|
||||
e_ok, c_ok = e_crt in VALID_CRT, c_crt in VALID_CRT
|
||||
if e_ok and c_ok and e_crt == c_crt:
|
||||
crt, src, status = e_crt, "EMISSOR+C000004", "CRT_SOURCE_CONFIRMED"
|
||||
elif e_ok and c_ok:
|
||||
choice = cfg.get("crt_source")
|
||||
if not choice:
|
||||
out["crt_candidates"] = {"EMISSOR": e_crt, "C000004": c_crt}
|
||||
return fail("CONFLICT", f"CRT_SOURCE_CONFLICT EMISSOR={e_crt} C000004={c_crt}: configure COMPANY_CRT_SOURCE (emission programs read EMISSOR)")
|
||||
crt, src, status = (e_crt if choice == "EMISSOR" else c_crt), f"{choice} (configured; conflict acknowledged)", "CRT_SOURCE_CONFLICT_RESOLVED_BY_CONFIG"
|
||||
out["warnings"].append(f"CRT_CONFLICT EMISSOR={e_crt} C000004={c_crt}")
|
||||
elif e_ok or c_ok:
|
||||
crt, src, status = (e_crt, "EMISSOR", "CRT_SOURCE_SINGLE") if e_ok else (c_crt, "C000004", "CRT_SOURCE_SINGLE")
|
||||
out["warnings"].append("CRT_HAS_A_SINGLE_SOURCE")
|
||||
else:
|
||||
return fail("UNRESOLVED", "CRT_SOURCE_UNKNOWN (no valid CRT in EMISSOR or C000004)")
|
||||
out.update(company_crt=crt, crt_status=status); out["source_of_truth"]["crt"] = src
|
||||
opt = _s(selected.get("optante"))
|
||||
if opt and ((crt in ("1", "2") and opt.upper() != "S") or (crt == "3" and opt.upper() == "S")):
|
||||
out["warnings"].append(f"OPTANTE_SIMPLES={opt} inconsistent with CRT={crt}")
|
||||
|
||||
agree = len(uf_sources) + (2 if status == "CRT_SOURCE_CONFIRMED" else 1)
|
||||
out["confidence"] = "HIGH" if (status == "CRT_SOURCE_CONFIRMED" and len(uf_sources) >= 2) else "MEDIUM"
|
||||
out["status"] = "PASS"
|
||||
out["classification"] = "CONFIGURED" if out["configured"] or status == "CRT_SOURCE_CONFLICT_RESOLVED_BY_CONFIG" else "AUTO_RESOLVED"
|
||||
out["reasons"] = []
|
||||
out["evidence_source_count"] = agree
|
||||
return out
|
||||
|
||||
|
||||
def collect_evidence(cursor):
|
||||
"""Read-only SELECTs. Missing optional tables/columns simply yield less evidence."""
|
||||
cursor.execute("SELECT TRIM(RDB$RELATION_NAME), TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME IN ('C000004','EMISSOR','NFCE','C000061')")
|
||||
cols = {}
|
||||
for t, f in cursor.fetchall():
|
||||
cols.setdefault(t, set()).add(f)
|
||||
ev = {"c000004": [], "emissor": [], "nfce_keys": [], "nf_by_filial": {}}
|
||||
c4 = cols.get("C000004", set())
|
||||
want = [("CODIGO", "code"), ("CNPJ", "cnpj"), ("IE", "ie"), ("UF", "uf"), ("CRT", "crt"), ("OPTANTE_SIMPLES", "optante"), ("FILIAL", "name"), ("CONTRIBUINTE_IPI", "contribuinte_ipi")]
|
||||
use = [(c, k) for c, k in want if c in c4]
|
||||
if use:
|
||||
cursor.execute(f"SELECT {','.join(c for c, _ in use)} FROM C000004")
|
||||
for row in cursor.fetchall():
|
||||
ev["c000004"].append({k: _s(v) for (_, k), v in zip(use, row)})
|
||||
em = cols.get("EMISSOR", set())
|
||||
want = [("EMITENTE_CNPJ", "cnpj"), ("EMITENTE_UF", "uf"), ("EMITENTE_CRT", "crt")]
|
||||
use = [(c, k) for c, k in want if c in em]
|
||||
if use:
|
||||
cursor.execute(f"SELECT {','.join(c for c, _ in use)} FROM EMISSOR")
|
||||
for row in cursor.fetchall():
|
||||
ev["emissor"].append({k: _s(v) for (_, k), v in zip(use, row)})
|
||||
if "CHAVE" in cols.get("NFCE", set()):
|
||||
cursor.execute("SELECT SUBSTRING(CHAVE FROM 1 FOR 2), SUBSTRING(CHAVE FROM 7 FOR 14), COUNT(*) FROM NFCE WHERE CHAVE IS NOT NULL AND CHAR_LENGTH(TRIM(CHAVE)) = 44 "
|
||||
"GROUP BY SUBSTRING(CHAVE FROM 1 FOR 2), SUBSTRING(CHAVE FROM 7 FOR 14)")
|
||||
ev["nfce_keys"] = [{"cuf": _s(a), "cnpj": _s(b), "n": n} for a, b, n in cursor.fetchall()]
|
||||
if "CODFILIAL" in cols.get("C000061", set()):
|
||||
cursor.execute("SELECT CODFILIAL, COUNT(*) FROM C000061 GROUP BY CODFILIAL")
|
||||
ev["nf_by_filial"] = {_s(a): n for a, n in cursor.fetchall()}
|
||||
return ev
|
||||
|
||||
|
||||
def resolve_from_cursor(cursor, env=None):
|
||||
return resolve(collect_evidence(cursor), config_from_env(env or os.environ))
|
||||
|
||||
|
||||
def preflight_company_context(env=None):
|
||||
"""Opens the configured Firebird database read-only and resolves the company. Never prints the full CNPJ."""
|
||||
from .firebird_reader import _connect
|
||||
env = env or os.environ
|
||||
conn = _connect(env)
|
||||
try:
|
||||
return resolve_from_cursor(conn.cursor(), env)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def preflight_report(result):
|
||||
keys = ("company_code", "company_cnpj", "company_uf", "company_crt", "company_contribuinte_ipi", "source_of_truth", "confidence", "classification", "warnings", "reasons")
|
||||
return {"COMPANY_CONTEXT": "PASS" if result["status"] == "PASS" else "FAIL", **{k: result.get(k) for k in keys}}
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Read-only fiscal coverage report for a set of products (used by the Agent's REPROCESS_ALL result and by the multi-database tooling).
|
||||
|
||||
Pure computation over Hub decisions + product rows + local evidence. It reuses the shipped pure engines in central.app
|
||||
(fiscal_decision, product_defaults, icms_context) — no fiscal rule lives here, only guards for the company context:
|
||||
the BA Anexo 1 ST/MVA dataset is applied to UF=BA only and CSOSN logic only under CRT=1.
|
||||
"""
|
||||
from collections import Counter
|
||||
|
||||
PRODUCT_COLUMNS = ("CODIGO", "PRODUTO", "CLASSIFICACAO_FISCAL", "CFOP", "CSOSN", "CST", "CEST", "IPI", "EX_TIPI", "TIPO", "PROD_COZINHA", "PRODUTO_SERVICO", "SITUACAO_TRIBUTARIA")
|
||||
|
||||
|
||||
def _s(v):
|
||||
return None if v is None else str(v).strip()
|
||||
|
||||
|
||||
def compute_coverage(products, decisions, company, company_default, refs, evidence):
|
||||
"""products/decisions aligned lists. company: {"uf","crt"} (resolved). evidence: {"recipes","composed","purchases","stock_entry"}."""
|
||||
from central.app.fiscal_decision import decide_product
|
||||
from central.app.icms_context import csosn_v2, product_role, st_status, withheld_status
|
||||
from central.app.product_defaults import cfop_default
|
||||
uf_ba = company.get("uf") == "BA"
|
||||
crt1 = company.get("crt") == "1"
|
||||
recipes, composed = evidence.get("recipes", set()), evidence.get("composed", set())
|
||||
purchases, stock_entry = evidence.get("purchases", {}), evidence.get("stock_entry", {})
|
||||
m, cfop_values, situ = Counter(), Counter(), Counter()
|
||||
for p, d in zip(products, decisions):
|
||||
p = {c: p.get(c) for c in PRODUCT_COLUMNS}
|
||||
code = _s(p["CODIGO"]); p["CODIGO"] = code
|
||||
cls = d.get("classification") or {}
|
||||
rec = decide_product(p, d, {"uf": company.get("uf") or "", "crt": company.get("crt") or ""}, refs)
|
||||
ncm_status = d.get("status"); raw_ncm = _s(p["CLASSIFICACAO_FISCAL"])
|
||||
m["NCM_MISSING" if not raw_ncm else ("NCM_VALID" if ncm_status == "EXACT_NCM" else ("NCM_LEGACY" if ncm_status == "LEGACY_NCM" else "NCM_INVALID"))] += 1
|
||||
ipi = rec["IPI"]; rk = (cls.get("ipi") or {}).get("rate_kind"); ist = d.get("ipi_status")
|
||||
if ist == "IPI_RESOLVED":
|
||||
m[{"NUMERIC": "IPI_NUMERIC", "NON_NUMERIC": "IPI_NON_NUMERIC", "BLANK": "IPI_SOURCE_BLANK"}.get(rk, "IPI_NO_MATCH")] += 1
|
||||
elif ist == "IPI_CANDIDATES":
|
||||
m["IPI_AMBIGUOUS"] += 1
|
||||
elif ist != "LEGACY_NCM" and raw_ncm:
|
||||
m["IPI_NO_MATCH"] += 1
|
||||
if ipi["action_class"] == "ALREADY_CORRECT":
|
||||
m["IPI_ALREADY_CORRECT"] += 1
|
||||
if ipi["write_eligible"]:
|
||||
m["IPI_CHANGE_REQUIRED"] += 1; m["IPI_WRITE_ELIGIBLE"] += 1
|
||||
cs_ = d.get("cest_status")
|
||||
if cs_ == "CEST_CONFIRMED":
|
||||
m["CEST_CONFIRMED"] += 1
|
||||
elif cs_ == "CEST_CANDIDATES":
|
||||
m["CEST_CANDIDATES"] += 1
|
||||
cf = cfop_default(p, ncm_status, d.get("ba_st_mva_status") if uf_ba else None, refs["cfop_valid"])
|
||||
cfop_values[_s(p["CFOP"])] += 1
|
||||
m["CFOP_RESOLVED" if cf["status"] == "CFOP_DEFAULT_RESOLVED" else "CFOP_CONTEXT_REQUIRED"] += 1
|
||||
ba = cls.get("ba_st_mva") or {}
|
||||
if uf_ba:
|
||||
st, _ = st_status(ncm_status, d.get("ba_st_mva_status"), cs_, p["CEST"], ba.get("cest"), None)
|
||||
else:
|
||||
st = "ST_UNKNOWN"
|
||||
m[st] += 1
|
||||
wh, _ = withheld_status(purchases.get(code, []))
|
||||
m[{"ST_ALREADY_WITHHELD_CONFIRMED": "ST_WITHHOLDING_CONFIRMED", "ST_ALREADY_WITHHELD_NOT_CONFIRMED": "ST_WITHHOLDING_NOT_CONFIRMED"}.get(wh, "ST_WITHHOLDING_UNKNOWN")] += 1
|
||||
role, _ = product_role(p["TIPO"], p["PROD_COZINHA"] == 1, code in recipes, code in composed, bool(_s(p["PRODUTO_SERVICO"])), bool(purchases.get(code)) or stock_entry.get(code, False))
|
||||
m["PRODUCT_ROLE_KNOWN" if role != "UNKNOWN" else "PRODUCT_ROLE_UNKNOWN"] += 1
|
||||
m["ROLE_" + role] += 1
|
||||
state = csosn_v2(p["CSOSN"], company_default, ncm_status, st if uf_ba else "ST_UNKNOWN", wh, role, refs["csosn"])["status"] if crt1 else "CSOSN_CONTEXT_REQUIRED"
|
||||
m["CSOSN_RESOLVED" if state == "CSOSN_RESOLVED" else ("CSOSN_CANDIDATES" if state == "CSOSN_CANDIDATE" else "CSOSN_CONTEXT_REQUIRED")] += 1
|
||||
situ[_s(p["SITUACAO_TRIBUTARIA"])] += 1
|
||||
n = len(products)
|
||||
fill = lambda f: sum(1 for p in products if _s(p.get(f)) not in (None, "")) # noqa: E731
|
||||
return {"TOTAL_PRODUCTS": n, "metrics": dict(m),
|
||||
"IPI_RESOLVED": m["IPI_NUMERIC"] + m["IPI_NON_NUMERIC"] + m["IPI_SOURCE_BLANK"],
|
||||
"quality": {"ncm_valid_share": round(m["NCM_VALID"] / n, 4) if n else None, "cest_filled": fill("CEST"), "csosn_filled": fill("CSOSN"), "cfop_filled": fill("CFOP"), "ipi_filled": fill("IPI"),
|
||||
"cfop_distinct": len(cfop_values), "cfop_top": cfop_values.most_common(6), "situacao_tributaria": dict(situ.most_common(6)),
|
||||
"purchase_items_C000088": sum(len(v) for v in purchases.values()), "products_with_purchase_evidence": len(purchases), "recipes": len(recipes), "composed_products": len(composed)},
|
||||
"guards_applied": {"st_dataset_ba_only": not uf_ba, "csosn_requires_crt1": not crt1}}
|
||||
|
||||
|
||||
def build_refs(get, products, company_uf):
|
||||
"""Reference-data snapshot from the Hub (`get(path) -> json`). Used by the engines for domain validity only."""
|
||||
refs = {}
|
||||
for key, domain in (("csosn", "CSOSN"), ("cst_icms", "CST_ICMS"), ("cst_pis", "CST_PIS"), ("cst_cofins", "CST_COFINS"), ("origem", "ORIGEM_MERCADORIA")):
|
||||
refs[key] = set(get(f"/v1/reference/fiscal_codes/{domain}")["codes"])
|
||||
valid = set()
|
||||
for code in {_s(p.get(f)) for p in products for f in ("CFOP", "CFOP_ENTRADA") if _s(p.get(f))}:
|
||||
if get(f"/v1/reference/cfop/{code}")["found"]:
|
||||
valid.add(code)
|
||||
refs["cfop_valid"] = valid
|
||||
refs["fcp_uf"] = {}
|
||||
fcp = get(f"/v1/reference/fcp/{company_uf}")
|
||||
if fcp.get("found"):
|
||||
refs["fcp_uf"][company_uf] = fcp
|
||||
refs["cbs_rate_2026"] = get("/v1/reference/rtc_cbs_rate/2026").get("rate")
|
||||
refs["cclasstrib_count"] = sum(d["record_count"] for d in get("/v1/datasets/status") if d["kind"] == "RTC_CCLASSTRIB" and d["status"] == "ACTIVE")
|
||||
return refs
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Windows DPAPI credential store for the GTI Fiscal Agent.
|
||||
|
||||
The permanent Agent credential (AGENT_TOKEN) must NEVER live in `agent.env`
|
||||
(which is world-readable config). It is stored in a dedicated
|
||||
`credentials.bin` file, encrypted with DPAPI in LocalMachine scope
|
||||
(`CRYPTPROTECT_LOCAL_MACHINE`), so any local Windows account on the host can
|
||||
decrypt it — and the file ACL is tightened to SYSTEM/Administrators only via
|
||||
`icacls`. The token is decrypted in-process at runtime by the service host and
|
||||
never logged.
|
||||
|
||||
`write_credential`/`read_credential` are shared by Setup.exe (writes at
|
||||
install time) and the Service host (reads at every start).
|
||||
|
||||
Fallback: a plaintext token file is accepted ONLY in development, guarded by
|
||||
`GTI_ALLOW_PLAINTEXT_CREDENTIAL=1`. Production Setup.exe never enables it.
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_PROGRAM_DIR = Path("C:/Program Files/GTI/FiscalAgent")
|
||||
_MAGIC = b"GTIDPAPI1"
|
||||
_DEFAULT_FLAGS = 0x1 | 0x4 # CRYPTPROTECT_UI_FORBIDDEN | CRYPTPROTECT_LOCAL_MACHINE
|
||||
|
||||
|
||||
class CredentialStoreError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class _DpapiLazy:
|
||||
"""Lazy wrapper around win32crypt so importing this module (and the whole
|
||||
agent package) works on non-Windows hosts without pywin32. Functions accept
|
||||
the same signatures pywin32 exposes on this machine's build, tolerating both
|
||||
the tuple and the bytes return shapes seen across pywin32 versions."""
|
||||
|
||||
def __init__(self):
|
||||
self._crypt = None
|
||||
|
||||
def _api(self):
|
||||
if self._crypt is None:
|
||||
try:
|
||||
import win32crypt
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise CredentialStoreError(f"win32crypt unavailable: {exc}") from exc
|
||||
self._crypt = win32crypt
|
||||
return self._crypt
|
||||
|
||||
def protect(self, data):
|
||||
crypt = self._api()
|
||||
returned = crypt.CryptProtectData(data, "gti-fiscal-agent-credential", None, None, None, _DEFAULT_FLAGS)
|
||||
if isinstance(returned, tuple):
|
||||
returned, _ = returned
|
||||
return bytes(returned)
|
||||
|
||||
def unprotect(self, blob):
|
||||
crypt = self._api()
|
||||
returned = crypt.CryptUnprotectData(bytes(blob), None, None, None, 0)
|
||||
if isinstance(returned, tuple):
|
||||
_, returned = returned
|
||||
return bytes(returned)
|
||||
|
||||
|
||||
_dpapi = _DpapiLazy()
|
||||
|
||||
|
||||
def _dpapi_available() -> bool:
|
||||
try:
|
||||
_dpapi._api()
|
||||
return True
|
||||
except CredentialStoreError:
|
||||
return False
|
||||
|
||||
|
||||
def default_credential_path() -> Path:
|
||||
return _PROGRAM_DIR / "credentials.bin"
|
||||
|
||||
|
||||
def _apply_acl(path: Path) -> None:
|
||||
"""Tighten ACL so only SYSTEM and Administrators can read/modify the blob.
|
||||
Well-known SIDs are used (`*S-1-5-18` = SYSTEM, `*S-1-5-32-544` = BUILTIN\
|
||||
Administrators) because account-name mapping is locale-dependent and fails
|
||||
on non-English Windows. Setup.exe runs elevated (`--uac-admin`), so icacls
|
||||
succeeds (the owner — the installing administrator — can change the DACL)."""
|
||||
cmd = ["icacls", str(path), "/inheritance:r", "/grant:r", "*S-1-5-18:(F)", "*S-1-5-32-544:(F)"]
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
except FileNotFoundError as exc: # non-Windows host without icacls
|
||||
raise CredentialStoreError(f"icacls unavailable: {exc}") from exc
|
||||
if result.returncode != 0:
|
||||
raise CredentialStoreError(f"icacls failed: {result.stdout} {result.stderr}")
|
||||
|
||||
|
||||
def write_credential(path, payload: str) -> None:
|
||||
"""Encrypt `payload` with DPAPI LocalMachine and write `path` with a
|
||||
SYSTEM/Administrators-only ACL. Fails closed: no plaintext file is ever
|
||||
created on this path."""
|
||||
if isinstance(payload, str):
|
||||
payload = payload.encode("utf-8")
|
||||
if not _dpapi_available():
|
||||
raise CredentialStoreError("DPAPI unavailable; refusing to store the credential as plaintext (fail closed)")
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
blob = _MAGIC + _dpapi.protect(payload)
|
||||
path.write_bytes(blob)
|
||||
try:
|
||||
_apply_acl(path)
|
||||
except CredentialStoreError:
|
||||
path.unlink(missing_ok=True) # do not leave a credential file without its ACL
|
||||
raise
|
||||
|
||||
|
||||
def read_credential(path, *, env=None) -> str:
|
||||
"""Decrypt `path`. A plaintext file is honored ONLY when
|
||||
`GTI_ALLOW_PLAINTEXT_CREDENTIAL=1` is set in `env`/environment (development)."""
|
||||
env = env if env is not None else os.environ
|
||||
path = Path(path)
|
||||
if not path.exists():
|
||||
raise CredentialStoreError(f"credential file missing: {path}")
|
||||
raw = path.read_bytes()
|
||||
if raw.startswith(_MAGIC):
|
||||
return _dpapi.unprotect(raw[len(_MAGIC):]).decode("utf-8")
|
||||
if (env.get("GTI_ALLOW_PLAINTEXT_CREDENTIAL") or "").strip() != "1":
|
||||
raise CredentialStoreError("credential file is plaintext and GTI_ALLOW_PLAINTEXT_CREDENTIAL is not set; refusing (fail closed)")
|
||||
return raw.decode("utf-8").strip()
|
||||
|
||||
|
||||
def install_env_from_credentials(path, env) -> str:
|
||||
"""Fill `env["AGENT_TOKEN"]` from `credentials.bin` unless the env already
|
||||
carries an explicit token (which wins). Returns the effective token —
|
||||
WITHOUT writing it into `agent.env` — and never logs it."""
|
||||
existing = env.get("AGENT_TOKEN") or env.get("AGENT_DEV_TOKEN") or ""
|
||||
if existing:
|
||||
return existing
|
||||
token = read_credential(path, env=env)
|
||||
env["AGENT_TOKEN"] = token
|
||||
return token
|
||||
@@ -0,0 +1,530 @@
|
||||
"""New-product enrichment on the Agent: detect NEW rows of C000025, ask the Hub, write ONLY safe empty fields.
|
||||
|
||||
Safety model (defense in depth; nothing here is switchable locally - the Hub owns the per-installation feature flags):
|
||||
- Feature gate: nothing touches Firebird unless the Hub says features_configured AND enrichment_mode != DISABLED. DRY_RUN never opens a write connection.
|
||||
- Scope: only products that appear AFTER the baseline (taken when the function is first active). The historical catalog is never sent anywhere.
|
||||
- No trigger, no table, no procedure in the SGV database: detection is one `SELECT COUNT(*)` per poll (+ a code diff only when the count moved, and every 60 s as a safety net).
|
||||
- Typed capability PRODUCT_ENRICHMENT_WRITE_V1: ALLOWED_FIELDS is an explicit allowlist on C000025; the only statement is a fixed UPDATE ... WHERE CODIGO=? AND (field IS NULL OR TRIM(field)=''),
|
||||
one short transaction per field, followed by a re-read. COD_ORIGEM_MERCADORIA and cClassTrib are NOT in the list (cClassTrib stays with the existing RTC gates).
|
||||
- Right before EVERY field write: re-read the row, re-check the Hub authorization (fail closed), skip when a human already touched the field.
|
||||
- Human overrides: after writing, the Agent watches the value; anything different from what it wrote (including a deleted value) marks the field HUMAN_OVERRIDDEN forever, is reported to the Hub
|
||||
and is never written again by the enrichment (nor re-suggested by the Hub).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
logger = logging.getLogger("gti_fiscal_agent.enrichment")
|
||||
|
||||
CAPABILITY = "PRODUCT_ENRICHMENT_WRITE_V1"
|
||||
ALLOWED_FIELDS = ("CLASSIFICACAO_FISCAL", "CEST", "IPI")
|
||||
# TECHNICAL_PLACEHOLDER: the SGV refuses a product without NCM, so operators type a filler. It means "not classified yet" (not a human decision) and may be replaced.
|
||||
# Explicit per-field allowlist, identical to central/app/enrichment/placeholders.py (parity test). Nothing else is ever overwritten.
|
||||
PLACEHOLDERS = {"CLASSIFICACAO_FISCAL": frozenset({"00000000"})} # explicit allowlist; anything else raises before touching SQL
|
||||
# READ-ONLY reporting to the Hub for field_contract.resolve_product's post-NCM fiscal-completion decisions (CFOP/CST/CSOSN/PIS-COFINS/...).
|
||||
# This is NOT a write allowlist - PRODUCT_ENRICHMENT_WRITE_V1 still only ever writes ALLOWED_FIELDS above. Sending the CURRENT value of these
|
||||
# fields (and the classification columns below) just lets the Hub's field_contract engine see what's already filled and never guess.
|
||||
FISCAL_COMPLETION_FIELDS = ("SITUACAO_TRIBUTARIA", "CFOP", "CFOP_ENTRADA", "CST", "CST_ENTRADA", "CSOSN", "ALIQUOTA", "CST_IPI",
|
||||
"COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS",
|
||||
"INCIDENCIA_PISCOFINS", "PISCOFINS", "IAT", "IPPT")
|
||||
# Classification metadata field_contract.infer_role reads to auto-discover product_role (RESALE/OWN_PRODUCTION/INPUT/SERVICE) from real ERP
|
||||
# facts instead of asking the operator. Read-only here too.
|
||||
CLASSIFICATION_CONTEXT_FIELDS = ("CODGRUPO", "TIPO", "PROD_COZINHA", "PRODUTO_SERVICO")
|
||||
CURRENT_REPORT_FIELDS = ALLOWED_FIELDS + FISCAL_COMPLETION_FIELDS + CLASSIFICATION_CONTEXT_FIELDS
|
||||
READ_FIELDS = ("CODIGO", "PRODUTO", "GTIN", "CODBARRA") + CURRENT_REPORT_FIELDS # the SGV operator types the EAN in CODBARRA; GTIN (fiscal) is mostly empty
|
||||
POLL_SECONDS = 2.0
|
||||
SAFETY_DIFF_SECONDS = 60.0
|
||||
OVERRIDE_CHECK_SECONDS = 20.0
|
||||
PENDING_POLL_SECONDS = 10.0 # cheap GET: products the Hub flagged for a new evaluation
|
||||
WATCH_WINDOW_SECONDS = 120.0 # a fresh product is re-read for a while: the operator may add the EAN seconds after the first save
|
||||
RETRY_BACKOFF = (30, 60, 120, 300, 600, 1800)
|
||||
|
||||
|
||||
def _now() -> float:
|
||||
return time.time()
|
||||
|
||||
|
||||
def _text(value) -> str:
|
||||
return "" if value is None else str(value).strip()
|
||||
|
||||
|
||||
class EnrichmentStore:
|
||||
"""Own SQLite connection (the worker runs in its own thread)."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.db = sqlite3.connect(str(path), timeout=10, check_same_thread=False)
|
||||
self.db.row_factory = sqlite3.Row
|
||||
self.db.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS enrich_meta(k TEXT PRIMARY KEY, v TEXT);
|
||||
CREATE TABLE IF NOT EXISTS enrich_known(code TEXT PRIMARY KEY);
|
||||
CREATE TABLE IF NOT EXISTS enrich_products(
|
||||
code TEXT PRIMARY KEY, state TEXT NOT NULL, detected_at REAL NOT NULL, next_attempt_at REAL, attempts INTEGER NOT NULL DEFAULT 0,
|
||||
snapshot TEXT, written TEXT NOT NULL DEFAULT '{}', overridden TEXT NOT NULL DEFAULT '{}', last_state TEXT);
|
||||
""")
|
||||
self.db.commit()
|
||||
|
||||
def meta(self, key, default=None):
|
||||
row = self.db.execute("SELECT v FROM enrich_meta WHERE k=?", (key,)).fetchone()
|
||||
return row["v"] if row else default
|
||||
|
||||
def set_meta(self, key, value):
|
||||
self.db.execute("INSERT INTO enrich_meta(k,v) VALUES(?,?) ON CONFLICT(k) DO UPDATE SET v=excluded.v", (key, str(value)))
|
||||
self.db.commit()
|
||||
|
||||
def known(self) -> set[str]:
|
||||
return {r["code"] for r in self.db.execute("SELECT code FROM enrich_known")}
|
||||
|
||||
def add_known(self, codes):
|
||||
self.db.executemany("INSERT OR IGNORE INTO enrich_known(code) VALUES(?)", [(c,) for c in codes])
|
||||
self.db.commit()
|
||||
|
||||
def enqueue(self, code, now=None):
|
||||
now = _now() if now is None else now
|
||||
self.db.execute("INSERT OR IGNORE INTO enrich_products(code,state,detected_at,next_attempt_at) VALUES(?,?,?,?)", (code, "PENDING", now, now))
|
||||
self.db.commit()
|
||||
|
||||
def get(self, code):
|
||||
row = self.db.execute("SELECT * FROM enrich_products WHERE code=?", (code,)).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def due(self, now):
|
||||
return [dict(r) for r in self.db.execute("SELECT * FROM enrich_products WHERE state IN ('PENDING','RETRY') AND next_attempt_at<=? ORDER BY detected_at LIMIT 20", (now,))]
|
||||
|
||||
def watching(self, now):
|
||||
return [dict(r) for r in self.db.execute("SELECT * FROM enrich_products WHERE state IN ('DONE','DRY_RUN_DONE') AND detected_at>=?", (now - WATCH_WINDOW_SECONDS,))]
|
||||
|
||||
def tracked_written(self):
|
||||
return [dict(r) for r in self.db.execute("SELECT * FROM enrich_products WHERE written != '{}'")]
|
||||
|
||||
def update(self, code, **values):
|
||||
for key in ("written", "overridden"):
|
||||
if key in values and not isinstance(values[key], str):
|
||||
values[key] = json.dumps(values[key])
|
||||
sets = ",".join(f"{k}=?" for k in values)
|
||||
self.db.execute(f"UPDATE enrich_products SET {sets} WHERE code=?", (*values.values(), code))
|
||||
self.db.commit()
|
||||
|
||||
def is_overridden(self, code, field) -> bool:
|
||||
row = self.get(code)
|
||||
return bool(row and json.loads(row["overridden"] or "{}").get(field))
|
||||
|
||||
|
||||
def is_human_overridden(store_path, code, field) -> bool:
|
||||
"""Used by every automatic writer of the Agent (the Fiscal Corrector included) before touching a field the enrichment tracks."""
|
||||
try:
|
||||
return EnrichmentStore(store_path).is_overridden(str(code), field)
|
||||
except Exception: # noqa: BLE001 - no state means nothing is protected yet
|
||||
return False
|
||||
|
||||
|
||||
# ---- Firebird access (injected: `read_conn()` returns a READ-ONLY connection, `write_conn()` a write-capable one, only ever opened by ENABLED mode) ----------------------------------------
|
||||
def is_placeholder(field, value) -> bool:
|
||||
text = _text(value)
|
||||
text = "".join(ch for ch in text if ch.isdigit()) if field == "CLASSIFICACAO_FISCAL" else text
|
||||
return text in PLACEHOLDERS.get(field, frozenset())
|
||||
|
||||
|
||||
def writable_now(field, value) -> bool:
|
||||
return not _text(value) or is_placeholder(field, value)
|
||||
|
||||
|
||||
def _digits(value) -> str:
|
||||
return "".join(ch for ch in _text(value) if ch.isdigit())
|
||||
|
||||
|
||||
def gtin_check_ok(value) -> bool:
|
||||
"""Same acceptance as central.app.enrichment.gtin (8/12/13/14 digits + check digit); parity is asserted by a test. Only used to pick WHICH column to send: the Hub decides."""
|
||||
d = _digits(value)
|
||||
if len(d) not in (8, 12, 13, 14) or len(set(d)) == 1:
|
||||
return False
|
||||
total = sum(int(x) * (3 if i % 2 == 0 else 1) for i, x in enumerate(reversed(d[:-1])))
|
||||
return (10 - total % 10) % 10 == int(d[-1])
|
||||
|
||||
|
||||
def pick_gtin(row) -> str:
|
||||
"""First VALID of GTIN / CODBARRA; otherwise whatever was typed (the Hub classifies it INVALID_GTIN/MISSING_GTIN without calling any provider)."""
|
||||
candidates = [_text(row.get("GTIN")), _text(row.get("CODBARRA"))]
|
||||
return next((c for c in candidates if gtin_check_ok(c)), next((c for c in candidates if c), ""))
|
||||
|
||||
|
||||
def read_codes(conn) -> set[str]:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT CODIGO FROM C000025")
|
||||
codes = {_text(r[0]) for r in cur.fetchall()}
|
||||
conn.commit()
|
||||
return codes
|
||||
|
||||
|
||||
def read_count(conn) -> int:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT COUNT(*) FROM C000025")
|
||||
n = int(cur.fetchone()[0])
|
||||
conn.commit()
|
||||
return n
|
||||
|
||||
|
||||
def read_rows(conn, codes) -> dict[str, dict]:
|
||||
out = {}
|
||||
codes = list(codes)
|
||||
for start in range(0, len(codes), 100):
|
||||
chunk = codes[start:start + 100]
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT {','.join(READ_FIELDS)} FROM C000025 WHERE CODIGO IN ({','.join('?' for _ in chunk)})", tuple(chunk))
|
||||
for row in cur.fetchall():
|
||||
rec = dict(zip(READ_FIELDS, row))
|
||||
out[_text(rec["CODIGO"])] = rec
|
||||
conn.commit()
|
||||
return out
|
||||
|
||||
|
||||
def write_field(conn, code: str, field: str, value, observed=None, guard_observed: bool = False) -> dict:
|
||||
"""PRODUCT_ENRICHMENT_WRITE_V1: one field, one short transaction, only if the field is still empty. Returns {outcome, reason?}."""
|
||||
if field not in ALLOWED_FIELDS:
|
||||
raise ValueError(f"field {field!r} is outside the PRODUCT_ENRICHMENT_WRITE_V1 allowlist")
|
||||
if field == "IPI":
|
||||
value = float(value)
|
||||
else:
|
||||
value = _digits(value)
|
||||
if not value or (field == "CLASSIFICACAO_FISCAL" and len(value) != 8) or (field == "CEST" and len(value) != 7):
|
||||
return {"outcome": "ABORTED", "reason": "VALUE_SHAPE_INVALID"}
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
rows = cur.fetchall()
|
||||
if len(rows) != 1:
|
||||
conn.rollback()
|
||||
return {"outcome": "ABORTED", "reason": "PRODUCT_ROW_NOT_UNIQUE" if rows else "PRODUCT_NOT_FOUND"}
|
||||
raw = rows[0][0]
|
||||
if guard_observed and _text(raw) != _text(observed): # STALE_CONFLICT: the value changed between the Hub decision and now (e.g. a person typed something) -> zero write
|
||||
conn.rollback()
|
||||
return {"outcome": "ABORTED", "reason": "STALE_CONFLICT"}
|
||||
if not writable_now(field, raw):
|
||||
conn.rollback()
|
||||
return {"outcome": "ABORTED", "reason": "VALUE_ALREADY_SET_NEVER_OVERWRITTEN"}
|
||||
if _text(raw): # technical placeholder: compare-and-swap on the exact value just read
|
||||
cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND TRIM({field}) = ?", (value, code, _text(raw)))
|
||||
else:
|
||||
cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND ({field} IS NULL OR TRIM({field}) = '')", (value, code))
|
||||
if cur.rowcount != 1:
|
||||
conn.rollback()
|
||||
return {"outcome": "ABORTED", "reason": "UPDATE_ROWCOUNT_NOT_ONE"}
|
||||
conn.commit()
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
after = cur.fetchone()[0]
|
||||
conn.commit()
|
||||
same = (float(after) == value) if field == "IPI" and after is not None else _text(after) == value
|
||||
if not same:
|
||||
return {"outcome": "FAILED", "reason": "POST_WRITE_VERIFICATION_FAILED"}
|
||||
return {"outcome": "APPLIED", "value": value, "rowcount": 1, "replaced_placeholder": _text(raw) if _text(raw) else None}
|
||||
except Exception as error: # noqa: BLE001
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return {"outcome": "FAILED", "reason": type(error).__name__}
|
||||
|
||||
|
||||
def _same_value(field, current, written) -> bool:
|
||||
if field == "IPI":
|
||||
try:
|
||||
return current is not None and _text(current) != "" and abs(float(current) - float(written)) < 1e-9
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return _text(current) == _text(written) and _text(current) != ""
|
||||
|
||||
|
||||
class EnrichmentWorker:
|
||||
def __init__(self, *, hub, store: EnrichmentStore, read_conn, write_conn, create_reprocess=None, clock=_now, sleep=time.sleep):
|
||||
self.hub, self.store, self.read_conn, self.write_conn = hub, store, read_conn, write_conn
|
||||
self.create_reprocess, self.clock, self.sleep = create_reprocess, clock, sleep
|
||||
self._features = None
|
||||
self._features_at = 0.0
|
||||
self._last_count = None
|
||||
self._last_diff = 0.0
|
||||
self._last_override = 0.0
|
||||
self._auth_cache = (None, 0.0)
|
||||
self._count_ms = 0.0
|
||||
self._last_pending = 0.0
|
||||
self._wconn = None
|
||||
self.metrics = {"detected": 0, "resolved": 0, "fields_written": 0, "detect_latency_ms": []}
|
||||
|
||||
def _open(self, factory, attempts=3):
|
||||
"""Firebird can drop a connection under load ('connection lost'): retry briefly instead of aborting the product."""
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
return factory()
|
||||
except Exception: # noqa: BLE001
|
||||
if attempt == attempts - 1:
|
||||
raise
|
||||
self.sleep(0.3 * (attempt + 1))
|
||||
|
||||
# ---- feature gate ------------------------------------------------------------------------------------------------------------------------------
|
||||
def features(self, max_age=10.0) -> dict:
|
||||
if self._features is None or self.clock() - self._features_at > max_age:
|
||||
try:
|
||||
auth = self.hub.authorizations()
|
||||
self._features = (auth or {}).get("features") or {}
|
||||
except Exception: # noqa: BLE001 - Hub offline: keep the last known gate, never invent one
|
||||
self._features = self._features or {}
|
||||
self._features_at = self.clock()
|
||||
return self._features
|
||||
|
||||
def mode(self) -> str:
|
||||
f = self.features()
|
||||
if not f.get("features_configured"):
|
||||
return "WAITING_FEATURE_CONFIGURATION"
|
||||
return f.get("enrichment_mode") or "DISABLED"
|
||||
|
||||
def _fresh_enabled(self) -> bool:
|
||||
"""Fail-closed authorization read right before a write (cached ~3 s)."""
|
||||
auth, at = self._auth_cache
|
||||
if auth is None or self.clock() - at > 3.0:
|
||||
try:
|
||||
auth = self.hub.authorizations().get("features") or {}
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
self._auth_cache = (auth, self.clock())
|
||||
return bool(auth.get("features_configured")) and auth.get("enrichment_mode") == "ENABLED"
|
||||
|
||||
# ---- detection ---------------------------------------------------------------------------------------------------------------------------------
|
||||
def detect(self) -> list[str]:
|
||||
conn = self._open(self.read_conn)
|
||||
try:
|
||||
if self.store.meta("baseline_done") != "1":
|
||||
codes = read_codes(conn)
|
||||
self.store.add_known(codes) # historical catalog: known, never sent
|
||||
self.store.set_meta("baseline_done", "1")
|
||||
self.store.set_meta("baseline_at", datetime.now(timezone.utc).isoformat())
|
||||
self.store.set_meta("baseline_count", len(codes))
|
||||
self._last_count, self._last_diff = len(codes), self.clock()
|
||||
return []
|
||||
if self.store.meta("rebaseline") == "1":
|
||||
# the function was inactive (DISABLED / WAITING_FEATURE_CONFIGURATION) for a while: whatever appeared meanwhile is NOT "new" for the enrichment
|
||||
# (a long-disabled installation must never send a burst of products to the providers when it is switched on again)
|
||||
codes = read_codes(conn)
|
||||
self.store.add_known(codes)
|
||||
self.store.set_meta("rebaseline", "0")
|
||||
self._last_count, self._last_diff = len(codes), self.clock()
|
||||
return []
|
||||
started = self.clock()
|
||||
t0 = time.monotonic()
|
||||
count = read_count(conn)
|
||||
self._count_ms = (time.monotonic() - t0) * 1000
|
||||
moved = self._last_count is None or count != self._last_count
|
||||
if not moved and self.clock() - self._last_diff < SAFETY_DIFF_SECONDS:
|
||||
return []
|
||||
codes = read_codes(conn)
|
||||
self._last_count, self._last_diff = len(codes), self.clock()
|
||||
new = sorted(codes - self.store.known())
|
||||
self.store.add_known(new)
|
||||
for code in new:
|
||||
self.store.enqueue(code, self.clock())
|
||||
if new:
|
||||
self.metrics["detected"] += len(new)
|
||||
self.metrics["detect_latency_ms"].append(int((self.clock() - started) * 1000))
|
||||
return new
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# ---- one product ---------------------------------------------------------------------------------------------------------------------------------
|
||||
def process(self, item: dict):
|
||||
code, mode = item["code"], self.mode()
|
||||
conn = self._open(self.read_conn)
|
||||
try:
|
||||
rows = read_rows(conn, [code])
|
||||
row = rows.get(code)
|
||||
if row is None or mode not in ("DRY_RUN", "ENABLED"):
|
||||
ev = None
|
||||
else:
|
||||
from .firebird_reader import read_product_evidence # targeted, indexed by product_code - never a full-table scan (see docstring)
|
||||
ev = read_product_evidence(conn, code)
|
||||
finally:
|
||||
conn.close()
|
||||
if row is None:
|
||||
self.store.update(code, state="GONE")
|
||||
return
|
||||
snapshot = f"{pick_gtin(row)}|{_text(row['PRODUTO'])}"
|
||||
if mode not in ("DRY_RUN", "ENABLED"):
|
||||
self.store.update(code, state="SKIPPED_FEATURE", snapshot=snapshot)
|
||||
return
|
||||
attempts = int(item["attempts"] or 0) + 1
|
||||
current = {f: row[f] for f in CURRENT_REPORT_FIELDS}
|
||||
try:
|
||||
result = self.hub.resolve_product({"product_code": code, "gtin": pick_gtin(row), "description": _text(row["PRODUTO"]), "ev": {k: (sorted(v) if isinstance(v, set) else v) for k, v in ev.items()},
|
||||
"current": {k: (None if v is None else str(v)) for k, v in current.items()}})
|
||||
except Exception as error: # noqa: BLE001 - Hub offline / slow: the product stays pending, the SGV is unaffected
|
||||
delay = RETRY_BACKOFF[min(attempts - 1, len(RETRY_BACKOFF) - 1)]
|
||||
self.store.update(code, state="RETRY", attempts=attempts, next_attempt_at=self.clock() + delay, last_state=f"HUB_ERROR:{type(error).__name__}")
|
||||
return
|
||||
self.metrics["resolved"] += 1
|
||||
state = result.get("state")
|
||||
if state == "FEATURE_DISABLED":
|
||||
self.store.update(code, state="SKIPPED_FEATURE", snapshot=snapshot, last_state=state)
|
||||
return
|
||||
if state in ("PENDING_PROVIDER_QUOTA", "PENDING_PROVIDERS_OFFLINE"):
|
||||
retry_at = result.get("retry_at")
|
||||
due = self.clock() + RETRY_BACKOFF[min(attempts - 1, len(RETRY_BACKOFF) - 1)]
|
||||
if retry_at:
|
||||
try:
|
||||
due = max(due, datetime.fromisoformat(retry_at).timestamp())
|
||||
except ValueError:
|
||||
pass
|
||||
self.store.update(code, state="RETRY", attempts=attempts, next_attempt_at=due, last_state=state, snapshot=snapshot)
|
||||
return
|
||||
fields = result.get("fields") or {}
|
||||
reports = []
|
||||
written = json.loads(item.get("written") or "{}")
|
||||
overridden = json.loads(item.get("overridden") or "{}")
|
||||
for field in ALLOWED_FIELDS:
|
||||
decision = fields.get(field) or {}
|
||||
status, value = decision.get("status"), decision.get("value")
|
||||
placeholder = _text(current.get(field)) if is_placeholder(field, current.get(field)) else None
|
||||
base = {"field": field, "rule": decision.get("rule"), "kb_status": decision.get("kb_status"), "evidence": decision.get("evidence") or [], "old": placeholder, "new": value}
|
||||
if status == "HUMAN_OVERRIDDEN" or overridden.get(field):
|
||||
reports.append({**base, "outcome": "HUMAN_OVERRIDDEN", "reason": "HUMAN_OVERRIDDEN", "new": None})
|
||||
continue
|
||||
if status != "AUTO_FILL_SAFE" or value in (None, ""):
|
||||
continue
|
||||
if mode == "DRY_RUN":
|
||||
reports.append({**base, "outcome": "DRY_RUN", "reason": "DRY_RUN_NO_WRITE"})
|
||||
continue
|
||||
outcome = self._write_safely(code, field, value, observed=current.get(field))
|
||||
reports.append({**base, "outcome": outcome["outcome"], "reason": outcome.get("reason")})
|
||||
if outcome["outcome"] == "APPLIED":
|
||||
written[field] = value
|
||||
self.metrics["fields_written"] += 1
|
||||
self.store.update(code, written=written) # remembered at once: a crash right after a write still leaves it protected/tracked
|
||||
self._close_write()
|
||||
self.store.update(code, state="DRY_RUN_DONE" if mode == "DRY_RUN" else "DONE", attempts=attempts, snapshot=snapshot, written=written, last_state=state, next_attempt_at=None)
|
||||
if reports:
|
||||
try:
|
||||
self.hub.report_enrichment({"product_code": code, "fields": reports, "dataset_versions": result.get("dataset_versions") or {}})
|
||||
except Exception: # noqa: BLE001 - audit delivery is retried on the next cycle via last_state
|
||||
self.store.update(code, last_state="AUDIT_PENDING")
|
||||
if written and self.create_reprocess is not None and mode == "ENABLED":
|
||||
try:
|
||||
self.create_reprocess([code]) # cClassTrib is decided by the existing RTC gates, never here
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("reprocess request for %s failed", code)
|
||||
|
||||
def _write_safely(self, code, field, value, observed=None) -> dict:
|
||||
"""Per field: fresh central authorization (fail closed) + guarded typed UPDATE. write_field re-reads the field INSIDE its own transaction and refuses when anything is there."""
|
||||
if not self._fresh_enabled():
|
||||
return {"outcome": "ABORTED", "reason": "AUTHORIZATION_CHANGED"}
|
||||
try:
|
||||
if self._wconn is None:
|
||||
self._wconn = self._open(self.write_conn)
|
||||
return write_field(self._wconn, code, field, value, observed=observed, guard_observed=True)
|
||||
except Exception as error: # noqa: BLE001 - one field failing never breaks the others or the SGV
|
||||
self._close_write()
|
||||
return {"outcome": "FAILED", "reason": type(error).__name__}
|
||||
|
||||
def _close_write(self):
|
||||
conn, self._wconn = self._wconn, None
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
def reevaluate_flagged(self) -> list[str]:
|
||||
"""'Reavaliar' in the panel, Reprocessar/Sincronizar and KB decisions: re-run the normal resolve for tracked, not-yet-enriched products (KB/cache first, the Hub decides about providers)."""
|
||||
try:
|
||||
codes = self.hub.pending_enrichment()
|
||||
except Exception: # noqa: BLE001 - Hub offline: try again next cycle
|
||||
return []
|
||||
done = []
|
||||
for code in codes[:25]:
|
||||
item = self.store.get(code)
|
||||
if item is None:
|
||||
self.store.enqueue(code, self.clock())
|
||||
item = self.store.get(code)
|
||||
try:
|
||||
self.process(item)
|
||||
except Exception as error: # noqa: BLE001
|
||||
self._close_write()
|
||||
logger.warning("re-evaluation of %s failed: %s", code, type(error).__name__)
|
||||
done.append(code)
|
||||
return done
|
||||
|
||||
# ---- human override watch --------------------------------------------------------------------------------------------------------------------------
|
||||
def check_overrides(self):
|
||||
tracked = self.store.tracked_written()
|
||||
if not tracked:
|
||||
return
|
||||
conn = self._open(self.read_conn)
|
||||
try:
|
||||
rows = read_rows(conn, [t["code"] for t in tracked])
|
||||
finally:
|
||||
conn.close()
|
||||
for item in tracked:
|
||||
row = rows.get(item["code"])
|
||||
written, overridden = json.loads(item["written"]), json.loads(item["overridden"] or "{}")
|
||||
new = []
|
||||
for field, value in written.items():
|
||||
if overridden.get(field) or row is None:
|
||||
continue
|
||||
if not _same_value(field, row[field], value): # different value OR blanked/deleted: a person decided, forever
|
||||
overridden[field] = True
|
||||
new.append({"field": field, "outcome": "HUMAN_OVERRIDDEN", "reason": "VALUE_CHANGED_AFTER_WRITE", "old": value, "new": None})
|
||||
if new:
|
||||
self.store.update(item["code"], overridden=overridden)
|
||||
try:
|
||||
self.hub.report_enrichment({"product_code": item["code"], "fields": new})
|
||||
except Exception: # noqa: BLE001 - local protection is already in place; the Hub copy follows on the next cycle
|
||||
logger.warning("override report for %s failed", item["code"])
|
||||
|
||||
# ---- loop ----------------------------------------------------------------------------------------------------------------------------------------
|
||||
def step(self):
|
||||
"""One iteration; returns the list of processed product codes."""
|
||||
mode = self.mode()
|
||||
if mode not in ("DRY_RUN", "ENABLED"):
|
||||
if self.store.meta("baseline_done") == "1":
|
||||
self.store.set_meta("rebaseline", "1")
|
||||
return [] # WAITING_FEATURE_CONFIGURATION / DISABLED: zero Firebird activity
|
||||
self.detect()
|
||||
done = []
|
||||
for item in self.store.due(self.clock()):
|
||||
try:
|
||||
self.process(item)
|
||||
except Exception as error: # noqa: BLE001 - keep the product retryable, never lose the batch
|
||||
self._close_write()
|
||||
attempts = int(item["attempts"] or 0) + 1
|
||||
self.store.update(item["code"], state="RETRY", attempts=attempts, next_attempt_at=self.clock() + RETRY_BACKOFF[min(attempts - 1, len(RETRY_BACKOFF) - 1)], last_state=f"ERROR:{type(error).__name__}")
|
||||
logger.warning("enrichment of %s failed: %s", item["code"], type(error).__name__)
|
||||
done.append(item["code"])
|
||||
for item in self.store.watching(self.clock()): # operator completes the record (EAN typed a moment later): re-resolve once per change
|
||||
rows = None
|
||||
conn = self._open(self.read_conn)
|
||||
try:
|
||||
rows = read_rows(conn, [item["code"]])
|
||||
finally:
|
||||
conn.close()
|
||||
row = (rows or {}).get(item["code"])
|
||||
if row and f"{pick_gtin(row)}|{_text(row['PRODUTO'])}" != item.get("snapshot"):
|
||||
self.process(item)
|
||||
done.append(item["code"])
|
||||
if self.clock() - self._last_pending >= PENDING_POLL_SECONDS:
|
||||
self._last_pending = self.clock()
|
||||
done += self.reevaluate_flagged()
|
||||
if self.clock() - self._last_override >= OVERRIDE_CHECK_SECONDS:
|
||||
self._last_override = self.clock()
|
||||
self.check_overrides()
|
||||
return done
|
||||
|
||||
def poll_seconds(self) -> float:
|
||||
"""Adaptive: the SGV is never loaded by more than ~2.5% of one query-time. 3 s on small catalogs, longer when COUNT(*) is slow (huge catalogs); never above 30 s."""
|
||||
return min(30.0, max(POLL_SECONDS, 40 * self._count_ms / 1000))
|
||||
|
||||
def run_forever(self, stopper):
|
||||
while not stopper.stop:
|
||||
try:
|
||||
self.step()
|
||||
except Exception: # noqa: BLE001 - the enrichment thread must never die or affect the SGV
|
||||
logger.exception("enrichment step failed")
|
||||
self.sleep(self.poll_seconds())
|
||||
@@ -0,0 +1,182 @@
|
||||
"""Shared read-only Firebird access for the Agent: full table or a CODIGO subset."""
|
||||
import os
|
||||
|
||||
FIELDS = "CODIGO,PRODUTO,GTIN AS EAN_GTIN,NCM,CLASSIFICACAO_FISCAL,CFOP,CST,ALIQUOTA,CEST,CSOSN,IPI,EX_TIPI"
|
||||
# Present in every audited SGV database but selected only when the column exists (capability-based, never assumed).
|
||||
OPTIONAL_FIELDS = ("TIPO", "PROD_COZINHA", "PRODUTO_SERVICO", "SITUACAO_TRIBUTARIA")
|
||||
|
||||
|
||||
def _select_fields(cursor):
|
||||
cursor.execute("SELECT TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME = 'C000025'")
|
||||
have = {r[0] for r in cursor.fetchall()}
|
||||
return FIELDS + "".join(f",{c}" for c in OPTIONAL_FIELDS if c in have)
|
||||
|
||||
|
||||
def _connect(env, read_only=True):
|
||||
import fdb
|
||||
kwargs = dict(
|
||||
database=env["FIREBIRD_DATABASE_PATH"],
|
||||
user=env["FB_USER"],
|
||||
password=env["FB_PASSWORD"],
|
||||
fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
)
|
||||
if read_only:
|
||||
kwargs["isolation_level"] = fdb.ISOLATION_LEVEL_READ_COMMITED_RO
|
||||
return fdb.connect(**kwargs)
|
||||
|
||||
|
||||
def check_connection(env=None):
|
||||
"""Best-effort read-only connectivity probe. Returns True/False, never raises."""
|
||||
env = env or os.environ
|
||||
try:
|
||||
conn = _connect(env)
|
||||
try:
|
||||
conn.cursor().execute("SELECT 1 FROM RDB$DATABASE")
|
||||
finally:
|
||||
conn.close()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def read_products(env=None, codigos=None):
|
||||
env = env or os.environ
|
||||
conn = _connect(env)
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
fields = _select_fields(cursor)
|
||||
if codigos:
|
||||
placeholders = ",".join("?" for _ in codigos)
|
||||
cursor.execute(f"SELECT {fields} FROM C000025 WHERE CODIGO IN ({placeholders}) ORDER BY CODIGO", tuple(codigos))
|
||||
else:
|
||||
cursor.execute(f"SELECT {fields} FROM C000025 ORDER BY CODIGO")
|
||||
names = [column[0].strip() for column in cursor.description]
|
||||
return [dict(zip(names, row)) for row in cursor.fetchall()]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def parse_company_context(rows):
|
||||
"""Pure: rows of (UF, CRT) from C000004 -> {"uf","crt"}. Fails closed (ValueError) if absent,
|
||||
blank or ambiguous — the engines never fall back to a hardcoded UF/CRT."""
|
||||
pairs = {(str(uf or "").strip().upper(), str(crt or "").strip()) for uf, crt in rows}
|
||||
if len(pairs) != 1:
|
||||
raise ValueError(f"company context unavailable or ambiguous ({len(pairs)} distinct UF/CRT rows in C000004)")
|
||||
uf, crt = next(iter(pairs))
|
||||
if len(uf) != 2 or crt not in ("1", "2", "3"):
|
||||
raise ValueError(f"company context invalid: UF={uf!r} CRT={crt!r}")
|
||||
return {"uf": uf, "crt": crt}
|
||||
|
||||
|
||||
def read_company_context(env=None):
|
||||
"""Resolve the ACTIVE company (UF/CRT) with the deterministic resolver (see company_context.py); fails closed with
|
||||
CompanyContextError when the context is ambiguous, conflicting or unresolved. No hardcoded company values, no first-row fallback."""
|
||||
from .company_context import CompanyContextError, preflight_company_context
|
||||
result = preflight_company_context(env or os.environ)
|
||||
if result["status"] != "PASS":
|
||||
raise CompanyContextError(result)
|
||||
return {"uf": result["company_uf"], "crt": result["company_crt"], "company_code": result["company_code"], "contribuinte_ipi": result.get("company_contribuinte_ipi"),
|
||||
"source_of_truth": result["source_of_truth"], "confidence": result["confidence"]}
|
||||
|
||||
|
||||
def read_product_evidence(conn, code):
|
||||
"""Targeted, single-product evidence for an ad-hoc /v1/products/resolve call (new-product enrichment) - the same facts `read_evidence`
|
||||
already gathers for a full sync job (recipes/composed/purchases/stock), but scoped to ONE product_code via an indexed WHERE, never a
|
||||
full-table scan. Measured on a real production-equivalent database: read_evidence's full scan took ~40ms total for the whole catalog,
|
||||
so this is not a correctness-driven optimization - it is what the mission explicitly asked for (never transport/scan the whole dataset
|
||||
for one ad-hoc product). Every table/column is optional: a missing one just means less evidence (capability-based), matching
|
||||
`read_evidence`. Takes an already-open connection (the SAME one the caller already opened for `read_rows`) - never opens its own,
|
||||
so it works with the EnrichmentWorker's injected `read_conn` (including test fakes; a non-Firebird connection has no RDB$RELATION_FIELDS,
|
||||
so the capability probe below degrades to "no evidence" rather than raising)."""
|
||||
from collections import defaultdict
|
||||
cur = conn.cursor()
|
||||
cols = defaultdict(set)
|
||||
try:
|
||||
cur.execute("SELECT TRIM(RDB$RELATION_NAME), TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME IN ('C000032','C000088','C000100','RECEITA_INGREDIENTES')")
|
||||
for t, f in cur.fetchall():
|
||||
cols[t].add(f)
|
||||
except Exception:
|
||||
return {"recipes": set(), "composed": set(), "composed_ratio": {}, "purchases": {}, "stock_entry": {}, "stock_in": {}}
|
||||
has = lambda t, *c: t in cols and all(x in cols[t] for x in c) # noqa: E731
|
||||
s = lambda v: None if v is None else str(v).strip() # noqa: E731
|
||||
ev = {"recipes": set(), "composed": set(), "composed_ratio": {}, "purchases": {}, "stock_entry": {}, "stock_in": {}}
|
||||
code_padded = str(code).zfill(6)
|
||||
if has("RECEITA_INGREDIENTES", "CODRECEITA"):
|
||||
cur.execute("SELECT FIRST 1 CODRECEITA FROM RECEITA_INGREDIENTES WHERE CODRECEITA = ?", (code_padded,))
|
||||
if cur.fetchone():
|
||||
ev["recipes"] = {code_padded}
|
||||
if has("C000032", "CODPRODUTO", "TIPO"):
|
||||
# Counted, never a bare "ever happened" check: a single COMPOSICAO row among many VEND rows
|
||||
# (e.g. a beer occasionally used as a recipe component) is not proof the product IS own
|
||||
# production - central.app.field_contract.infer_role needs the ratio to judge that.
|
||||
cur.execute("SELECT TRIM(TIPO), COUNT(*) FROM C000032 WHERE CODPRODUTO = ? GROUP BY 1", (code,))
|
||||
counts = {(t or ""): n for t, n in cur.fetchall()}
|
||||
composed_n = counts.get("COMPOSICAO", 0)
|
||||
other_n = sum(n for t, n in counts.items() if t != "COMPOSICAO")
|
||||
if composed_n or other_n:
|
||||
ev["composed_ratio"][s(code)] = {"composed": composed_n, "other": other_n}
|
||||
if composed_n:
|
||||
ev["composed"] = {s(code)}
|
||||
if has("C000088", "CODPRODUTO", "ICMS_RETIDO", "ICMS_VALORRETIDO", "SUB_VALOR"):
|
||||
cur.execute("SELECT ICMS_RETIDO, ICMS_VALORRETIDO, SUB_VALOR FROM C000088 WHERE CODPRODUTO = ?", (code,))
|
||||
rows = [{"icms_retido": s(ret), "valor_retido": vr, "sub_valor": sv} for ret, vr, sv in cur.fetchall()]
|
||||
if rows:
|
||||
ev["purchases"][s(code)] = rows
|
||||
if has("C000100", "CODPRODUTO", "ULTIMA_ENTRADA", "NOTA_FISCAL"):
|
||||
cur.execute("SELECT FIRST 1 ULTIMA_ENTRADA, NOTA_FISCAL FROM C000100 WHERE CODPRODUTO = ?", (code,))
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
ev["stock_entry"][s(code)] = row[0] is not None or s(row[1]) not in (None, "", "000000")
|
||||
if has("C000100", "CODPRODUTO", "ENTRADAS"):
|
||||
cur.execute("SELECT FIRST 1 ENTRADAS FROM C000100 WHERE CODPRODUTO = ?", (code,))
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
ev["stock_in"][s(code)] = bool(row[0] and float(row[0]) > 0)
|
||||
return ev
|
||||
|
||||
|
||||
def read_evidence(env=None, selected_company_code=None):
|
||||
"""Read-only evidence tables used by the coverage report (purchases, stock entries, recipes, compositions) and the company's
|
||||
default CSOSN. Every table/column is optional: a missing one just means less evidence (capability-based)."""
|
||||
from collections import defaultdict
|
||||
env = env or os.environ
|
||||
conn = _connect(env)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT TRIM(RDB$RELATION_NAME), TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME IN ('C000004','C000032','C000088','C000100','RECEITA_INGREDIENTES')")
|
||||
cols = defaultdict(set)
|
||||
for t, f in cur.fetchall():
|
||||
cols[t].add(f)
|
||||
has = lambda t, *c: t in cols and all(x in cols[t] for x in c) # noqa: E731
|
||||
s = lambda v: None if v is None else str(v).strip() # noqa: E731
|
||||
ev = {"recipes": set(), "composed": set(), "composed_ratio": {}, "purchases": defaultdict(list), "stock_entry": {}, "stock_in": {}, "company_default_csosn": None}
|
||||
if has("RECEITA_INGREDIENTES", "CODRECEITA"):
|
||||
cur.execute("SELECT CODRECEITA FROM RECEITA_INGREDIENTES WHERE CODRECEITA IS NOT NULL")
|
||||
ev["recipes"] = {str(r[0]).zfill(6) for r in cur.fetchall()}
|
||||
if has("C000032", "CODPRODUTO", "TIPO"):
|
||||
# Counted per product, never a bare "ever happened" check - see read_product_evidence for why
|
||||
# (a single COMPOSICAO row among many VEND rows is not proof of own production).
|
||||
cur.execute("SELECT CODPRODUTO, TRIM(TIPO), COUNT(*) FROM C000032 GROUP BY 1, 2")
|
||||
counts = defaultdict(lambda: {"composed": 0, "other": 0})
|
||||
for cod, tipo, n in cur.fetchall():
|
||||
counts[s(cod)]["composed" if tipo == "COMPOSICAO" else "other"] += n
|
||||
ev["composed_ratio"] = dict(counts)
|
||||
ev["composed"] = {code for code, c in counts.items() if c["composed"]}
|
||||
if has("C000088", "CODPRODUTO", "ICMS_RETIDO", "ICMS_VALORRETIDO", "SUB_VALOR"):
|
||||
cur.execute("SELECT CODPRODUTO, ICMS_RETIDO, ICMS_VALORRETIDO, SUB_VALOR FROM C000088")
|
||||
for cod, ret, vr, sv in cur.fetchall():
|
||||
ev["purchases"][s(cod)].append({"icms_retido": s(ret), "valor_retido": vr, "sub_valor": sv})
|
||||
if has("C000100", "CODPRODUTO", "ULTIMA_ENTRADA", "NOTA_FISCAL"):
|
||||
cur.execute("SELECT CODPRODUTO, ULTIMA_ENTRADA, NOTA_FISCAL FROM C000100")
|
||||
ev["stock_entry"] = {s(r[0]): (r[1] is not None or s(r[2]) not in (None, "", "000000")) for r in cur.fetchall()}
|
||||
if has("C000100", "CODPRODUTO", "ENTRADAS"):
|
||||
cur.execute("SELECT CODPRODUTO, ENTRADAS FROM C000100")
|
||||
ev["stock_in"] = {s(r[0]): bool(r[1] and float(r[1]) > 0) for r in cur.fetchall()}
|
||||
if has("C000004", "ICMS_CSOSN", "CODIGO"):
|
||||
cur.execute("SELECT CODIGO, ICMS_CSOSN FROM C000004")
|
||||
rows = {s(a): s(b) for a, b in cur.fetchall()}
|
||||
ev["company_default_csosn"] = rows.get(selected_company_code) if selected_company_code else (next(iter(rows.values())) if len(rows) == 1 else None)
|
||||
ev["purchases"] = dict(ev["purchases"])
|
||||
return ev
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Decide whether a classified product's IPI rate may be auto-applied.
|
||||
|
||||
Scope is deliberately narrow: C000025.IPI only, and only when the Hub's TIPI
|
||||
match is a single unambiguous numeric rate. Never auto-applies NT/non-numeric
|
||||
markers, blank source rates, ambiguous EX candidates, or legacy/invalid NCM.
|
||||
"""
|
||||
|
||||
|
||||
def ipi_write_decision(decision, current_ipi):
|
||||
"""current_ipi must be a value freshly read from Firebird immediately before
|
||||
this call — None here means the ERP field is genuinely NULL right now, not
|
||||
that it wasn't read."""
|
||||
if decision.get("status") == "LEGACY_NCM":
|
||||
return {"eligible": False, "reason_code": "LEGACY_NCM", "proposed_value": None}
|
||||
if decision.get("ipi_status") != "IPI_RESOLVED":
|
||||
return {"eligible": False, "reason_code": f"IPI_STATUS_{decision.get('ipi_status')}", "proposed_value": None}
|
||||
|
||||
ipi = (decision.get("classification") or {}).get("ipi")
|
||||
if not ipi:
|
||||
return {"eligible": False, "reason_code": "NO_IPI_DECISION", "proposed_value": None}
|
||||
|
||||
rate_kind = ipi.get("rate_kind")
|
||||
if rate_kind != "NUMERIC":
|
||||
return {"eligible": False, "reason_code": f"RATE_KIND_{rate_kind}", "proposed_value": None}
|
||||
|
||||
proposed_value = ipi.get("rate_numeric")
|
||||
if proposed_value is None:
|
||||
return {"eligible": False, "reason_code": "NO_NUMERIC_VALUE", "proposed_value": None}
|
||||
|
||||
if current_ipi is not None and str(current_ipi).strip() == str(proposed_value).strip():
|
||||
return {"eligible": False, "reason_code": "SAME_VALUE", "proposed_value": proposed_value}
|
||||
|
||||
return {"eligible": True, "reason_code": "CHANGE_REQUIRED", "proposed_value": proposed_value}
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Shared per-product IPI auto-apply pipeline: Hub decision -> dry-run ->
|
||||
guarded write -> local SQLite audit. Used by both the manual CLI
|
||||
(run_agent_auto_apply.py) and the long-running job-polling service
|
||||
(run_agent_service.py) so the processing logic exists in exactly one place.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from .ipi_auto_apply import ipi_write_decision
|
||||
from .runner import product_fingerprint
|
||||
from central.app.operational_state import classify_product_state, summarize_operational_states
|
||||
|
||||
|
||||
def now():
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def run_pipeline(products, decisions, state, firebird_writes, write_connection, executor, database_path, authorized_path=None):
|
||||
"""Process every (product, decision) pair. Returns a summary dict.
|
||||
|
||||
products come from a fresh Firebird read (current IPI is always "known").
|
||||
decisions come from the Hub's /v1/classify/batch, one per product, same order.
|
||||
"""
|
||||
summary = {
|
||||
"processed": 0,
|
||||
"auto_apply_eligible": 0,
|
||||
"writes_attempted": 0,
|
||||
"writes_succeeded": 0,
|
||||
"writes_failed": 0,
|
||||
"skipped_not_eligible": 0,
|
||||
"no_ops": 0,
|
||||
"product_states": [],
|
||||
"blocked_products": [],
|
||||
"resolved_products": [],
|
||||
}
|
||||
for product, decision in zip(products, decisions):
|
||||
code = product["CODIGO"]
|
||||
fingerprint = product_fingerprint(product)
|
||||
current_ipi = product.get("IPI")
|
||||
summary["processed"] += 1
|
||||
|
||||
wd = ipi_write_decision(decision, current_ipi)
|
||||
decision_status = decision.get("status")
|
||||
ipi_status = decision.get("ipi_status")
|
||||
operational = classify_product_state(product, decision)
|
||||
summary["product_states"].append(operational)
|
||||
if operational["state"].startswith("BLOCKED_"):
|
||||
summary["blocked_products"].append({
|
||||
"company": product.get("COMPANY") or product.get("company") or "",
|
||||
"product_code": code,
|
||||
"gtin": product.get("GTIN") or product.get("EAN_GTIN"),
|
||||
"description": product.get("PRODUTO") or product.get("description"),
|
||||
"blocker_type": operational["state"],
|
||||
"field": "NCM",
|
||||
"reason_code": decision_status or "MISSING_EVIDENCE",
|
||||
"evidence_summary": {"decision_status": decision_status, "ipi_status": ipi_status},
|
||||
"input_fingerprint": fingerprint,
|
||||
})
|
||||
else:
|
||||
summary["resolved_products"].append(code)
|
||||
|
||||
if not firebird_writes:
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "DRY_RUN_ONLY", current_ipi, wd["proposed_value"], None, wd["reason_code"], now())
|
||||
if wd["eligible"]:
|
||||
summary["auto_apply_eligible"] += 1
|
||||
elif wd["reason_code"] == "SAME_VALUE":
|
||||
summary["no_ops"] += 1
|
||||
continue
|
||||
|
||||
if not wd["eligible"]:
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "SKIPPED_NOT_ELIGIBLE", current_ipi, wd["proposed_value"], None, wd["reason_code"], now())
|
||||
summary["skipped_not_eligible"] += 1
|
||||
if wd["reason_code"] == "SAME_VALUE":
|
||||
summary["no_ops"] += 1
|
||||
continue
|
||||
|
||||
summary["auto_apply_eligible"] += 1
|
||||
summary["writes_attempted"] += 1
|
||||
proposal = {
|
||||
"product_code": code,
|
||||
"field": "IPI",
|
||||
"current_value": current_ipi,
|
||||
"proposed_value": wd["proposed_value"],
|
||||
"status": "CHANGE_REQUIRED",
|
||||
"write_eligible": True,
|
||||
"source_dataset": "TIPI_IPI",
|
||||
"source_version": decision.get("dataset_versions", {}).get("tipi_ipi"),
|
||||
}
|
||||
try:
|
||||
record = executor.execute_guarded_update(write_connection, database_path=database_path, proposal=proposal, now=now(), authorized_path=authorized_path)
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "SUCCEEDED", record.before_value, record.after_value, f"rowcount={record.rowcount}", wd["reason_code"], now())
|
||||
summary["writes_succeeded"] += 1
|
||||
summary["product_states"][-1] = classify_product_state(product, decision, write_status="SUCCEEDED")
|
||||
except executor.WriteRefused as error:
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "FAILED", current_ipi, wd["proposed_value"], str(error), wd["reason_code"], now())
|
||||
summary["writes_failed"] += 1
|
||||
summary["product_states"][-1] = classify_product_state(product, decision, write_status="FAILED")
|
||||
|
||||
summary["operational"] = summarize_operational_states(summary["product_states"])
|
||||
return summary
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Single installation preflight for the GTI Fiscal Agent. READ-ONLY: never writes to Firebird, never prints secrets.
|
||||
|
||||
PREFLIGHT=PASS | FAIL (+ blockers, one line each)
|
||||
|
||||
Checks: config, FISCAL_MODE/FIREBIRD_WRITES, Hub reachable/auth/version, Firebird reachable + version + ODS, C000004/C000025,
|
||||
capabilities (minimum: NCM, IPI, CEST), company context (UF/CRT, deterministic resolver), schema/type guards, SQLite state, log dir.
|
||||
"""
|
||||
import os
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_VERSION_DEFAULT = "1.3.9"
|
||||
REQUIRED_ENV = ("GTI_HUB_URL", "AGENT_TOKEN", "FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")
|
||||
REQUIRED_CAPABILITIES = ("supports_ncm", "supports_ipi", "supports_cest")
|
||||
FISCAL_MODES = ("DRY_RUN", "AUTO_APPLY")
|
||||
|
||||
|
||||
def _check(name, ok, detail="", warn=False):
|
||||
return {"name": name, "status": "PASS" if ok else ("WARN" if warn else "FAIL"), "detail": detail}
|
||||
|
||||
|
||||
def major_minor(version):
|
||||
parts = str(version or "").replace("-", ".").replace("rc", ".").split(".")
|
||||
return ".".join(parts[:2])
|
||||
|
||||
|
||||
def default_http_get(url, token=None, timeout=10):
|
||||
import httpx
|
||||
headers = {"Authorization": f"Bearer {token}"} if token else {}
|
||||
r = httpx.get(url, headers=headers, timeout=timeout)
|
||||
return r.status_code, (r.json() if r.headers.get("content-type", "").startswith("application/json") else {})
|
||||
|
||||
|
||||
def firebird_probe(env):
|
||||
"""Read-only introspection of the configured database."""
|
||||
import fdb
|
||||
from .company_context import collect_evidence, config_from_env, resolve
|
||||
conn = fdb.connect(database=env["FIREBIRD_DATABASE_PATH"], user=env["FB_USER"], password=env["FB_PASSWORD"], fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute("""SELECT TRIM(rf.RDB$RELATION_NAME), TRIM(rf.RDB$FIELD_NAME), f.RDB$FIELD_TYPE, COALESCE(f.RDB$CHARACTER_LENGTH, f.RDB$FIELD_LENGTH)
|
||||
FROM RDB$RELATION_FIELDS rf JOIN RDB$FIELDS f ON f.RDB$FIELD_NAME = rf.RDB$FIELD_SOURCE WHERE rf.RDB$RELATION_NAME IN ('C000004', 'C000025')""")
|
||||
tables = {}
|
||||
for t, c, ty, ln in cur.fetchall():
|
||||
tables.setdefault(t, {})[c] = (ty, ln)
|
||||
products = None
|
||||
if "C000025" in tables:
|
||||
cur.execute("SELECT COUNT(*) FROM C000025")
|
||||
products = cur.fetchone()[0]
|
||||
company = resolve(collect_evidence(cur), config_from_env(env))
|
||||
return {"server_version": getattr(conn, "server_version", None), "ods": f"{getattr(conn, 'ods_version', '?')}.{getattr(conn, 'ods_minor_version', '?')}",
|
||||
"tables": tables, "products": products, "company": company}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def run_preflight(env=None, *, http_get=default_http_get, probe=firebird_probe, agent_version=None):
|
||||
from .capabilities import detect_capabilities, type_ok
|
||||
from .company_context import preflight_report
|
||||
env = env if env is not None else os.environ
|
||||
checks, info = [], {}
|
||||
agent_version = agent_version or env.get("AGENT_VERSION") or AGENT_VERSION_DEFAULT
|
||||
|
||||
missing = [k for k in REQUIRED_ENV if not (env.get(k) or "").strip()]
|
||||
checks.append(_check("CONFIG", not missing, f"missing: {', '.join(missing)}" if missing else "required variables present"))
|
||||
|
||||
mode = (env.get("FISCAL_MODE") or "").strip().upper()
|
||||
writes = (env.get("FIREBIRD_WRITES") or "0").strip() == "1"
|
||||
checks.append(_check("FISCAL_MODE", mode in FISCAL_MODES, f"FISCAL_MODE={mode or 'unset'} (must be one of {FISCAL_MODES})"))
|
||||
dry_conflict = mode == "DRY_RUN" and writes
|
||||
checks.append(_check("FIREBIRD_WRITES", not dry_conflict, "FIREBIRD_WRITES=1 conflicts with FISCAL_MODE=DRY_RUN (writes stay disabled; fix the config)" if dry_conflict else f"requested={int(writes)}"))
|
||||
effective = writes and mode == "AUTO_APPLY"
|
||||
info["effective_firebird_writes"] = effective
|
||||
if effective:
|
||||
ok = bool((env.get("AUTHORIZED_WRITE_DATABASE_PATH") or "").strip() and (env.get("EXPECTED_SCHEMA_FINGERPRINT") or "").strip())
|
||||
checks.append(_check("WRITE_AUTHORIZATION", ok, "AUTO_APPLY needs AUTHORIZED_WRITE_DATABASE_PATH and EXPECTED_SCHEMA_FINGERPRINT (two independent layers)"))
|
||||
|
||||
hub = (env.get("GTI_HUB_URL") or "").rstrip("/")
|
||||
hub_ok = False
|
||||
if hub:
|
||||
try:
|
||||
code, _ = http_get(f"{hub}/health")
|
||||
hub_ok = code == 200
|
||||
checks.append(_check("HUB_REACHABLE", hub_ok, f"GET /health -> {code}"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("HUB_REACHABLE", False, type(e).__name__))
|
||||
if hub_ok:
|
||||
try:
|
||||
code, body = http_get(f"{hub}/version")
|
||||
hub_v = body.get("version")
|
||||
info["hub_version"] = hub_v
|
||||
checks.append(_check("HUB_VERSION_COMPATIBLE", code == 200 and major_minor(hub_v) == major_minor(agent_version), f"hub={hub_v} agent={agent_version} (major.minor must match)"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("HUB_VERSION_COMPATIBLE", False, type(e).__name__))
|
||||
try:
|
||||
code, _ = http_get(f"{hub}/v1/datasets/status", token=env.get("AGENT_TOKEN"))
|
||||
checks.append(_check("AUTH", code == 200, f"agent token -> HTTP {code}"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("AUTH", False, type(e).__name__))
|
||||
|
||||
fb = None
|
||||
if all((env.get(k) or "").strip() for k in ("FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")):
|
||||
try:
|
||||
fb = probe(env)
|
||||
checks.append(_check("FIREBIRD_REACHABLE", True, "read-only connection opened"))
|
||||
info.update(firebird_version=fb.get("server_version"), ods=fb.get("ods"), products=fb.get("products"))
|
||||
checks.append(_check("FIREBIRD_VERSION_ODS", bool(fb.get("server_version")), f"{fb.get('server_version')} / ODS {fb.get('ods')}"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("FIREBIRD_REACHABLE", False, f"{type(e).__name__}: {str(e)[:80]}"))
|
||||
if fb:
|
||||
t = fb["tables"]
|
||||
checks.append(_check("C000004", "C000004" in t, "company table present" if "C000004" in t else "missing"))
|
||||
checks.append(_check("C000025", "C000025" in t, "product table present" if "C000025" in t else "missing"))
|
||||
cols = {c: ty for c, (ty, _) in t.get("C000025", {}).items()}
|
||||
caps = detect_capabilities(cols)
|
||||
info["capabilities"] = caps
|
||||
lacking = [c for c in REQUIRED_CAPABILITIES if not caps.get(c)]
|
||||
checks.append(_check("CAPABILITIES", not lacking, f"missing required: {', '.join(lacking)}" if lacking else f"required present; optional true: {sum(1 for c, v in caps.items() if v and c not in REQUIRED_CAPABILITIES)}"))
|
||||
rep = preflight_report(fb["company"])
|
||||
info["company_context"] = rep
|
||||
checks.append(_check("COMPANY_CONTEXT", rep["COMPANY_CONTEXT"] == "PASS", "; ".join(rep.get("reasons") or []) or f"{rep['classification']} source={rep['source_of_truth']}"))
|
||||
checks.append(_check("UF", bool(rep.get("company_uf")), str(rep.get("company_uf"))))
|
||||
checks.append(_check("CRT", bool(rep.get("company_crt")), str(rep.get("company_crt"))))
|
||||
writable = [f for f in ("IPI", "CEST", "EX_TIPI", "CLASSIFICACAO_FISCAL") if f in t.get("C000025", {})]
|
||||
bad = [f for f in writable if not type_ok(f, t["C000025"][f][0])]
|
||||
lens = {f: t["C000025"][f][1] for f in ("CEST", "EX_TIPI") if f in t.get("C000025", {})}
|
||||
checks.append(_check("SCHEMA_TYPE_GUARDS", not bad and len(writable) >= 3, f"incompatible: {bad}" if bad else f"types ok for {writable}; text lengths {lens} (the executor re-checks length/type before any write)"))
|
||||
|
||||
state_db = env.get("AGENT_STATE_DB") or str(Path.cwd() / "agent_state.sqlite")
|
||||
try:
|
||||
Path(state_db).parent.mkdir(parents=True, exist_ok=True)
|
||||
c = sqlite3.connect(state_db)
|
||||
c.execute("SELECT 1")
|
||||
c.close()
|
||||
checks.append(_check("SQLITE", True, "state database opens"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("SQLITE", False, type(e).__name__))
|
||||
log_path = Path(env.get("AGENT_LOG_PATH") or Path.cwd() / "logs" / "agent.log")
|
||||
try:
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
checks.append(_check("LOG_DIRECTORY", os.access(log_path.parent, os.W_OK), "log directory writable"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("LOG_DIRECTORY", False, type(e).__name__))
|
||||
|
||||
blockers = [f"{c['name']}: {c['detail']}" for c in checks if c["status"] == "FAIL"]
|
||||
return {"PREFLIGHT": "FAIL" if blockers else "PASS", "blockers": blockers, "checks": checks, "agent_version": agent_version, "fiscal_mode": mode or None, **info}
|
||||
@@ -0,0 +1,214 @@
|
||||
"""RTC auto-apply: fills C000025.CCLASSTRIB, and NOTHING else, when a deterministic rule proves the class.
|
||||
|
||||
Safety model (defense in depth, every gate must hold, default is OFF): the Hub sets a per-installation mode DISABLED / DRY_RUN / APPROVAL_REQUIRED / ENABLED and
|
||||
delivers it inside an authenticated job. The Agent only exposes the restricted capability RTC_CCLASSTRIB_WRITE_V1; there is no local switch to edit (no agent.env, no FIREBIRD_WRITES).
|
||||
Write allowlist: ALLOWED_RTC_WRITE_FIELDS = ["CCLASSTRIB"] on C000025 (no free SQL exists). One short transaction per product. Immediately before the UPDATE the product,
|
||||
company, RTC parameter and local catalog are re-read and the whole decision is re-derived; if ANYTHING changed that product is ABORTED and nothing is written.
|
||||
An existing value is never overwritten. Every applied change returns an audit record with the old value; ROLLBACK restores it only if the value still equals what was written.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import date
|
||||
|
||||
from central.app.cclasstrib_resolver import RULE_VERSION, compare_catalogs, local_catalog_hash, resolve_cclasstrib
|
||||
from central.app.unified_fiscal_coverage import rtc_enabled
|
||||
|
||||
from .unified_fiscal import detect_rtc_capabilities, load_rtc_context
|
||||
|
||||
ALLOWED_RTC_WRITE_FIELDS = ("CCLASSTRIB",)
|
||||
DEFAULT_MAX_PER_JOB = 500
|
||||
CAPABILITY = "RTC_CCLASSTRIB_WRITE_V1"
|
||||
MODES = ("DISABLED", "DRY_RUN", "APPROVAL_REQUIRED", "ENABLED")
|
||||
MAX_PREVIEW_CANDIDATES = 5000
|
||||
|
||||
|
||||
class RtcApplyAborted(Exception):
|
||||
"""The product is left untouched; `reason` is reported in the audit."""
|
||||
|
||||
def __init__(self, reason: str):
|
||||
super().__init__(reason)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
def _text(value) -> str:
|
||||
return "" if value is None else str(value).strip()
|
||||
|
||||
|
||||
def _digits(value) -> str:
|
||||
return re.sub(r"\D", "", _text(value))
|
||||
|
||||
|
||||
def write_gate(*, mode: str, capability: bool = True, approved: bool = True) -> dict:
|
||||
"""{"allowed": bool, "missing": [...]}: a real write needs the capability, an authenticated Hub mode that permits writing (ENABLED, or APPROVAL_REQUIRED for approved products)."""
|
||||
mode = mode if mode in MODES else "DISABLED"
|
||||
missing = []
|
||||
if not capability:
|
||||
missing.append("CAPABILITY")
|
||||
if mode not in ("ENABLED", "APPROVAL_REQUIRED"):
|
||||
missing.append("HUB_MODE_" + mode)
|
||||
elif mode == "APPROVAL_REQUIRED" and not approved:
|
||||
missing.append("HUB_APPROVAL")
|
||||
return {"allowed": not missing, "missing": missing, "mode": mode, "capability": bool(capability)}
|
||||
|
||||
|
||||
def candidate_fingerprint(code, ncm, description, current) -> str:
|
||||
import hashlib
|
||||
return hashlib.sha256("|".join((_text(code), _digits(ncm), _text(description), _text(current))).encode("utf-8")).hexdigest()[:24]
|
||||
|
||||
|
||||
def select_candidates(findings: dict, products: list[dict], *, limit: int = DEFAULT_MAX_PER_JOB) -> tuple[list[dict], int]:
|
||||
"""Products READY_TO_APPLY: empty CCLASSTRIB + a deterministic resolution, no other RTC problem. Returns (candidates up to `limit`, total ready)."""
|
||||
by_code = {str(p.get("CODIGO") or ""): p for p in products}
|
||||
ready = []
|
||||
for code, found in findings.items():
|
||||
res = found.get("rtc_resolution") or {}
|
||||
if not res.get("auto_apply_candidate"):
|
||||
continue
|
||||
rtc_types = {b["type"] for b in found.get("blockers", []) if b["category"] == "RTC"}
|
||||
if rtc_types != {"CCLASSTRIB_MISSING"}:
|
||||
continue
|
||||
p = by_code.get(code, {})
|
||||
if _text(p.get("CCLASSTRIB")):
|
||||
continue
|
||||
ready.append({"product_code": code, "proposed": res["resolved_cclasstrib"], "cst": res["resolved_cst_ibscbs"], "rule_id": res["rule_id"], "ncm": _digits(p.get("CLASSIFICACAO_FISCAL")), "confidence_type": res["confidence_type"], "description": _text(p.get("PRODUTO"))[:255], "current": "", "basis": f'{res["rule_id"]}:{res["confidence_type"]}',
|
||||
"fingerprint": candidate_fingerprint(code, p.get("CLASSIFICACAO_FISCAL"), p.get("PRODUTO"), "")})
|
||||
return ready[:limit], len(ready)
|
||||
|
||||
|
||||
def apply_candidate(conn, candidate: dict, *, company: dict, decision: dict, now: str, job_id=None, today: date | None = None) -> dict:
|
||||
"""One product, one short transaction. `decision` = {"local_catalog_hash": ...} captured when the candidate was chosen. Raises RtcApplyAborted (nothing written)."""
|
||||
field = "CCLASSTRIB"
|
||||
assert field in ALLOWED_RTC_WRITE_FIELDS
|
||||
today = today or date.today()
|
||||
code, proposed = candidate["product_code"], candidate["proposed"]
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
schema, catalog, extras = load_rtc_context(cur) # fresh: the ERP is asked again, nothing cached
|
||||
caps = detect_rtc_capabilities(schema, param_value=extras["param_value"], catalog=catalog, physical_fk=extras["physical_fk"])
|
||||
if extras["param_conflict"] or not rtc_enabled(caps):
|
||||
raise RtcApplyAborted("RTC_NO_LONGER_ENABLED")
|
||||
if not caps["RTC_CATALOG_VALID"]:
|
||||
raise RtcApplyAborted("RTC_CATALOG_INVALID")
|
||||
local_rows = (catalog or {}).get("rows") or {}
|
||||
fresh_hash = local_catalog_hash(local_rows)
|
||||
if decision.get("local_catalog_hash") and fresh_hash != decision["local_catalog_hash"]:
|
||||
raise RtcApplyAborted("LOCAL_CATALOG_CHANGED_SINCE_DECISION")
|
||||
cur.execute("SELECT DISTINCT TRIM(CRT) FROM C000004")
|
||||
crts = {_text(r[0]) for r in cur.fetchall()}
|
||||
expected_crt = _text((company or {}).get("crt"))
|
||||
if not expected_crt or crts != {expected_crt}:
|
||||
raise RtcApplyAborted("COMPANY_CONTEXT_CHANGED")
|
||||
if expected_crt != "3":
|
||||
raise RtcApplyAborted("REGIME_NOT_NORMAL")
|
||||
cur.execute(f"SELECT {field}, CLASSIFICACAO_FISCAL, PRODUTO FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
rows = cur.fetchall()
|
||||
if len(rows) != 1:
|
||||
raise RtcApplyAborted("PRODUCT_ROW_NOT_UNIQUE" if rows else "PRODUCT_NOT_FOUND")
|
||||
current, ncm = _text(rows[0][0]), _digits(rows[0][1])
|
||||
if current:
|
||||
raise RtcApplyAborted("VALUE_ALREADY_SET_NEVER_OVERWRITTEN" if current != proposed else "NO_WRITE_REQUIRED")
|
||||
if ncm != candidate["ncm"]:
|
||||
raise RtcApplyAborted("PRODUCT_CHANGED_NCM")
|
||||
if candidate.get("fingerprint") and candidate_fingerprint(code, ncm, rows[0][2], current) != candidate["fingerprint"]:
|
||||
raise RtcApplyAborted("PRODUCT_CHANGED_SINCE_DECISION")
|
||||
compat = compare_catalogs(local_rows, today)
|
||||
resolution = resolve_cclasstrib({"CODIGO": code, "CLASSIFICACAO_FISCAL": ncm, "ncm_valid": True}, company, {"rtc_enabled": True, "local_catalog_valid": True}, local_rows, compat=compat, today=today)
|
||||
if not resolution.auto_apply_candidate or resolution.resolved_cclasstrib != proposed or resolution.rule_id != candidate["rule_id"]:
|
||||
raise RtcApplyAborted("DECISION_CHANGED")
|
||||
cur.execute(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND ({field} IS NULL OR TRIM({field}) = '')", (proposed, code))
|
||||
if cur.rowcount != 1:
|
||||
conn.rollback()
|
||||
raise RtcApplyAborted("UPDATE_ROWCOUNT_NOT_ONE")
|
||||
conn.commit()
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
after = _text(cur.fetchone()[0])
|
||||
conn.commit()
|
||||
if after != proposed:
|
||||
raise RuntimeError(f"post-write verification failed for {code}: {after!r} != {proposed!r}")
|
||||
official = compat["official_catalog"]
|
||||
return {"outcome": "APPLIED", "product_code": code, "field": field, "old_value": None, "new_value": proposed, "cst_ibscbs": resolution.resolved_cst_ibscbs, "rule_id": resolution.rule_id,
|
||||
"confidence_type": resolution.confidence_type, "evidence": {"inputs_used": resolution.inputs_used, "evidence": resolution.evidence[:5], "reason": resolution.reason},
|
||||
"official_source_version": {"cclasstrib_table": official["publication"], "cclasstrib_sha256": official["sha256"], "lc214_sha256": resolution.source_references[0]["sha256"]},
|
||||
"local_catalog_hash": fresh_hash, "timestamp": now, "job_id": job_id, "rowcount": 1, "verified_after": True, "action": "APPLY", "rule_version": RULE_VERSION,
|
||||
"company_crt": expected_crt, "description_snapshot": _text(rows[0][2])[:255], "rtc_status": "ENABLED", "local_catalog_state": (compat.get("per_code") or {}).get(proposed), "authorization_origin": decision.get("origin")}
|
||||
except RtcApplyAborted:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
raise
|
||||
except Exception:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def apply_candidates(conn, candidates: list[dict], *, company: dict, decision: dict, now: str, job_id=None, today: date | None = None, authorize=None) -> dict:
|
||||
"""Applies each candidate independently; a failure or an abort never stops the rest and never leaves a partial change.
|
||||
`authorize(product_code)` (optional) re-asks the Hub, right before each UPDATE, whether the central authorization still permits it: it returns None to proceed or a
|
||||
reason (AUTHORIZATION_CHANGED / AUTHORIZATION_UNVERIFIABLE) to skip the product with zero writes."""
|
||||
applied, aborted, reasons = [], [], {}
|
||||
for candidate in candidates:
|
||||
if authorize is not None:
|
||||
denied = authorize(candidate["product_code"])
|
||||
if denied:
|
||||
aborted.append({"outcome": "ABORTED", "product_code": candidate["product_code"], "field": "CCLASSTRIB", "reason": denied, "rule_id": candidate["rule_id"], "job_id": job_id, "timestamp": now, "action": "APPLY", "authorization_origin": decision.get("origin")})
|
||||
reasons[denied] = reasons.get(denied, 0) + 1
|
||||
continue
|
||||
try:
|
||||
applied.append(apply_candidate(conn, candidate, company=company, decision=decision, now=now, job_id=job_id, today=today))
|
||||
except RtcApplyAborted as stop:
|
||||
aborted.append({"outcome": "ABORTED", "product_code": candidate["product_code"], "field": "CCLASSTRIB", "reason": stop.reason, "rule_id": candidate["rule_id"], "job_id": job_id, "timestamp": now, "action": "APPLY", "authorization_origin": decision.get("origin")})
|
||||
reasons[stop.reason] = reasons.get(stop.reason, 0) + 1
|
||||
except Exception as error: # noqa: BLE001 - technical failure of ONE product must not abort the batch
|
||||
aborted.append({"outcome": "FAILED", "product_code": candidate["product_code"], "field": "CCLASSTRIB", "reason": f"{type(error).__name__}", "rule_id": candidate["rule_id"], "job_id": job_id, "timestamp": now})
|
||||
reasons["TECHNICAL_FAILURE"] = reasons.get("TECHNICAL_FAILURE", 0) + 1
|
||||
return {"applied": applied, "aborted": aborted, "aborted_reasons": reasons, "write_attempts": len(candidates), "write_successes": len(applied)}
|
||||
|
||||
|
||||
def rollback_one(conn, item: dict, *, now: str, job_id=None) -> dict:
|
||||
"""Restores C000025.CCLASSTRIB to empty for ONE previously applied audit record, only if the value still equals what was written. No free SQL: the only statement is fixed."""
|
||||
field = "CCLASSTRIB"
|
||||
assert field in ALLOWED_RTC_WRITE_FIELDS
|
||||
code, written = _text(item.get("product_code")), _text(item.get("written_value"))
|
||||
base = {"product_code": code, "field": field, "job_id": job_id, "timestamp": now, "action": "ROLLBACK", "rollback_of": item.get("audit_id"), "old_value": written, "new_value": None, "rule_id": item.get("rule_id")}
|
||||
if not code or not written or item.get("restore_value"):
|
||||
return {**base, "outcome": "ABORTED", "reason": "ROLLBACK_REQUEST_INVALID"}
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
rows = cur.fetchall()
|
||||
if len(rows) != 1:
|
||||
conn.rollback()
|
||||
return {**base, "outcome": "ABORTED", "reason": "PRODUCT_ROW_NOT_UNIQUE" if rows else "PRODUCT_NOT_FOUND"}
|
||||
current = _text(rows[0][0])
|
||||
if current != written:
|
||||
conn.rollback()
|
||||
return {**base, "outcome": "ABORTED", "reason": "ROLLBACK_CONFLICT_VALUE_CHANGED" if current else "ALREADY_EMPTY"}
|
||||
cur.execute(f"UPDATE C000025 SET {field} = NULL WHERE CODIGO = ? AND TRIM({field}) = ?", (code, written))
|
||||
if cur.rowcount != 1:
|
||||
conn.rollback()
|
||||
return {**base, "outcome": "ABORTED", "reason": "UPDATE_ROWCOUNT_NOT_ONE"}
|
||||
conn.commit()
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT {field} FROM C000025 WHERE CODIGO = ?", (code,))
|
||||
after = _text(cur.fetchone()[0])
|
||||
conn.commit()
|
||||
if after:
|
||||
raise RuntimeError("post-rollback verification failed")
|
||||
return {**base, "outcome": "ROLLED_BACK", "rowcount": 1, "verified_after": True}
|
||||
except Exception as error: # noqa: BLE001
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return {**base, "outcome": "FAILED", "reason": type(error).__name__}
|
||||
|
||||
|
||||
def rollback_records(conn, items: list[dict], *, now: str, job_id=None) -> dict:
|
||||
results = [rollback_one(conn, item, now=now, job_id=job_id) for item in items]
|
||||
return {"rollbacks": results, "rolled_back": sum(1 for r in results if r["outcome"] == "ROLLED_BACK")}
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Gate Firebird reads on the registered schema fingerprint."""
|
||||
import hashlib, json
|
||||
REFERENCE_SCHEMA_V1 = "1198f89897532b643439f06b729e93b189018f56e7d478e95095f8f56977005b"
|
||||
|
||||
def product_fingerprint(product):
|
||||
fields = ("PRODUTO", "EAN_GTIN", "NCM", "CLASSIFICACAO_FISCAL", "CFOP", "CST", "ALIQUOTA", "CEST", "CSOSN")
|
||||
return hashlib.sha256(json.dumps({key: product.get(key) for key in fields}, sort_keys=True, default=str).encode()).hexdigest()
|
||||
def to_hub_product(product):
|
||||
# C000025 has NCM empty and stores the confirmed eight-digit NCM in CLASSIFICACAO_FISCAL.
|
||||
return {"product_id": product["CODIGO"], "description": product.get("PRODUTO", ""), "gtin": product.get("EAN_GTIN"), "current_ncm": product.get("CLASSIFICACAO_FISCAL")}
|
||||
def run_sample(products, client, state, schema_fingerprint):
|
||||
if schema_fingerprint != REFERENCE_SCHEMA_V1: return {"status": "SCHEMA_UNKNOWN", "results": [], "firebird_writes": 0}
|
||||
outbound = []
|
||||
for product in products:
|
||||
fingerprint = product_fingerprint(product)
|
||||
if state.enqueue(product["CODIGO"], fingerprint, "pending", "0.1.0"): outbound.append((product, fingerprint))
|
||||
if not outbound: return {"status": "OK", "results": [], "firebird_writes": 0, "api_requests": 0}
|
||||
try: results = client.classify_batch([to_hub_product(product) for product, _ in outbound])
|
||||
except Exception:
|
||||
for product, _ in outbound: state.retry(product["CODIGO"])
|
||||
return {"status": "API_UNAVAILABLE", "results": [], "firebird_writes": 0}
|
||||
for (product, fingerprint), result in zip(outbound, results): state.result_received(product["CODIGO"], fingerprint, result["dataset_versions"].get("ncm") or "none", result["engine_version"])
|
||||
return {"status": "OK", "results": results, "firebird_writes": 0, "api_requests": 1}
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Local SQLite outbox; it has no Firebird dependency or write path."""
|
||||
import json
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
import uuid
|
||||
|
||||
class AgentState:
|
||||
def __init__(self, path):
|
||||
self.connection = sqlite3.connect(Path(path)); self.connection.row_factory = sqlite3.Row
|
||||
self.connection.executescript("CREATE TABLE IF NOT EXISTS product_state(product_id TEXT PRIMARY KEY, source_fingerprint TEXT NOT NULL, dataset_version TEXT NOT NULL, engine_version TEXT NOT NULL, status TEXT NOT NULL, last_processed_at TEXT, last_result_hash TEXT, reason_code TEXT); CREATE TABLE IF NOT EXISTS outbox(product_id TEXT PRIMARY KEY, source_fingerprint TEXT NOT NULL, status TEXT NOT NULL, retries INTEGER NOT NULL DEFAULT 0); CREATE TABLE IF NOT EXISTS ipi_write_log(product_id TEXT NOT NULL, source_fingerprint TEXT NOT NULL, decision_status TEXT, ipi_status TEXT, write_status TEXT NOT NULL, before_value TEXT, proposed_value TEXT, result TEXT, reason_code TEXT, timestamp TEXT NOT NULL);")
|
||||
self.connection.execute("CREATE TABLE IF NOT EXISTS installation_identity (id INTEGER PRIMARY KEY CHECK(id=1), installation_id TEXT NOT NULL)")
|
||||
# Last Hub decision per product + the source fingerprint it was computed for: what makes
|
||||
# SYNC_INCREMENTAL a real delta (only new/changed products are re-classified).
|
||||
self.connection.execute("CREATE TABLE IF NOT EXISTS decision_cache (product_id TEXT PRIMARY KEY, source_fingerprint TEXT NOT NULL, decision_json TEXT NOT NULL)")
|
||||
self.connection.commit()
|
||||
def stored_installation_id(self):
|
||||
row = self.connection.execute("SELECT installation_id FROM installation_identity WHERE id=1").fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
try:
|
||||
return str(uuid.UUID(row["installation_id"]))
|
||||
except (TypeError, ValueError, AttributeError) as error:
|
||||
raise ValueError("stored installation_id is not a valid UUID") from error
|
||||
def installation_id(self, candidate=None):
|
||||
existing = self.stored_installation_id()
|
||||
if existing:
|
||||
if candidate is not None and str(uuid.UUID(str(candidate))) != existing:
|
||||
raise ValueError("configured installation_id conflicts with persisted identity")
|
||||
return existing
|
||||
if candidate is None or not str(candidate).strip():
|
||||
raise ValueError("installation_id is required; refusing to generate a new identity")
|
||||
try:
|
||||
value = str(uuid.UUID(str(candidate)))
|
||||
except (TypeError, ValueError, AttributeError) as error:
|
||||
raise ValueError("installation_id must be a valid UUID") from error
|
||||
self.connection.execute("INSERT OR IGNORE INTO installation_identity(id, installation_id) VALUES(1, ?)", (value,))
|
||||
self.connection.commit()
|
||||
row = self.connection.execute("SELECT installation_id FROM installation_identity WHERE id=1").fetchone()
|
||||
if row is None or str(uuid.UUID(row["installation_id"])) != value:
|
||||
raise ValueError("persisted installation_id conflicts with candidate")
|
||||
return value
|
||||
def cached_fingerprint(self, product_id):
|
||||
row = self.connection.execute("SELECT source_fingerprint FROM decision_cache WHERE product_id=?", (product_id,)).fetchone()
|
||||
return row["source_fingerprint"] if row else None
|
||||
def cached_decision(self, product_id):
|
||||
row = self.connection.execute("SELECT decision_json FROM decision_cache WHERE product_id=?", (product_id,)).fetchone()
|
||||
return json.loads(row["decision_json"]) if row else None
|
||||
def remember_decision(self, product_id, source_fingerprint, decision):
|
||||
self.connection.execute("INSERT OR REPLACE INTO decision_cache(product_id,source_fingerprint,decision_json) VALUES(?,?,?)", (product_id, source_fingerprint, json.dumps(decision, default=str)))
|
||||
self.connection.commit()
|
||||
def forget_missing(self, present_product_ids):
|
||||
"""Products no longer in the ERP leave the LOCAL cache only; Hub-side history/blockers are never deleted by this."""
|
||||
present = set(present_product_ids)
|
||||
gone = [row["product_id"] for row in self.connection.execute("SELECT product_id FROM decision_cache") if row["product_id"] not in present]
|
||||
for product_id in gone:
|
||||
self.connection.execute("DELETE FROM decision_cache WHERE product_id=?", (product_id,))
|
||||
self.connection.commit()
|
||||
return len(gone)
|
||||
def enqueue(self, product_id, source_fingerprint, dataset_version, engine_version):
|
||||
row = self.connection.execute("SELECT * FROM product_state WHERE product_id=?", (product_id,)).fetchone()
|
||||
if row and tuple(row[key] for key in ("source_fingerprint", "engine_version", "status")) == (source_fingerprint, engine_version, "RESULT_RECEIVED"): return False
|
||||
self.connection.execute("INSERT OR REPLACE INTO product_state(product_id,source_fingerprint,dataset_version,engine_version,status) VALUES(?,?,?,?,?)", (product_id, source_fingerprint, dataset_version, engine_version, "PENDING_API")); self.connection.execute("INSERT OR REPLACE INTO outbox(product_id,source_fingerprint,status) VALUES(?,?,?)", (product_id, source_fingerprint, "PENDING_API")); self.connection.commit(); return True
|
||||
def pending(self): return [dict(row) for row in self.connection.execute("SELECT * FROM outbox WHERE status='PENDING_API'")]
|
||||
def retry(self, product_id): self.connection.execute("UPDATE outbox SET retries=retries+1,status='PENDING_API' WHERE product_id=?", (product_id,)); self.connection.commit()
|
||||
def result_received(self, product_id, source_fingerprint, dataset_version, engine_version):
|
||||
self.connection.execute("UPDATE product_state SET source_fingerprint=?,dataset_version=?,engine_version=?,status='RESULT_RECEIVED' WHERE product_id=?", (source_fingerprint, dataset_version, engine_version, product_id)); self.connection.execute("DELETE FROM outbox WHERE product_id=?", (product_id,)); self.connection.commit()
|
||||
def log_ipi_write(self, product_id, source_fingerprint, decision_status, ipi_status, write_status, before_value, proposed_value, result, reason_code, timestamp):
|
||||
self.connection.execute(
|
||||
"INSERT INTO ipi_write_log(product_id,source_fingerprint,decision_status,ipi_status,write_status,before_value,proposed_value,result,reason_code,timestamp) VALUES(?,?,?,?,?,?,?,?,?,?)",
|
||||
(product_id, source_fingerprint, decision_status, ipi_status, write_status, str(before_value) if before_value is not None else None, str(proposed_value) if proposed_value is not None else None, result, reason_code, timestamp),
|
||||
)
|
||||
self.connection.commit()
|
||||
def ipi_write_log_for(self, product_id):
|
||||
return [dict(row) for row in self.connection.execute("SELECT * FROM ipi_write_log WHERE product_id=? ORDER BY timestamp", (product_id,))]
|
||||
@@ -0,0 +1,148 @@
|
||||
"""READ-ONLY ERP loader for the unified fiscal completeness (PIS/COFINS applicability + RTC when the ERP has the reform on).
|
||||
|
||||
The rules live in central/app/unified_fiscal_coverage.py (pure). This module only observes the ERP through an already open
|
||||
READ ONLY connection (SELECT only): schema capabilities, RTC catalog, PARAMETROS.USAR_REFORMA_TRIBUTARIA and the product columns.
|
||||
Nothing is written and no value is ever filled in.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections import Counter
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
from central.app.cclasstrib_resolver import compare_catalogs, local_catalog_hash, resolve_cclasstrib
|
||||
from central.app.rtc_findings import refine_rtc_finding, summarize_rtc
|
||||
from central.app.unified_fiscal_coverage import PIS_COFINS_FIELDS, catalog_integrity, cclasstrib_applicability, evaluate_product, legal_effective_context, rtc_enabled
|
||||
|
||||
PRODUCT_TABLE = "C000025"
|
||||
RTC_CCLASSTRIB_TABLE = "RTC_CCLASSTRIB"
|
||||
RTC_CST_TABLE = "RTC_CST"
|
||||
PRODUCT_COLUMNS = ("CODIGO", "CCLASSTRIB", "CLASSIFICACAO_FISCAL") + PIS_COFINS_FIELDS + ("PISCOFINS", "INCIDENCIA_PISCOFINS")
|
||||
TRUTHY = {"1", "S", "T", "TRUE"}
|
||||
|
||||
|
||||
def detect_rtc_capabilities(schema: dict, *, param_value=None, catalog: dict | None = None, physical_fk: bool | None = None) -> dict:
|
||||
"""schema: {TABLE: {COLUMN, ...}}. Fail-closed: anything not proven is False. Old databases without RTC simply report False."""
|
||||
product_cols = set(schema.get(PRODUCT_TABLE, ()))
|
||||
class_cols = set(schema.get(RTC_CCLASSTRIB_TABLE, ()))
|
||||
cst_cols = set(schema.get(RTC_CST_TABLE, ()))
|
||||
has_product = "CCLASSTRIB" in product_cols
|
||||
has_class = {"CCLASSTRIB", "CSTIBSCBS"} <= class_cols
|
||||
has_cst = "CSTIBSCBS" in cst_cols
|
||||
caps = {
|
||||
"HAS_PRODUCT_CCLASSTRIB": has_product, "HAS_RTC_CCLASSTRIB_TABLE": has_class, "HAS_RTC_CST_TABLE": has_cst,
|
||||
"HAS_RTC_RELATIONSHIP": bool(has_product and has_class and has_cst), "HAS_PHYSICAL_RTC_FK": bool(physical_fk),
|
||||
"RTC_ENABLED_BY_PARAM": False if param_value is None else str(param_value).strip().upper() in TRUTHY,
|
||||
"RTC_CATALOG_VALID": False,
|
||||
}
|
||||
if catalog is not None and has_class and has_cst:
|
||||
caps["RTC_CATALOG_VALID"] = catalog_integrity(catalog)["valid"]
|
||||
return caps
|
||||
|
||||
|
||||
def load_rtc_context(cursor) -> tuple[dict, dict | None, dict]:
|
||||
"""Schema + catalog + ERP parameter through a READ ONLY cursor. Never raises for a database that lacks RTC."""
|
||||
schema: dict[str, set] = {}
|
||||
cursor.execute("SELECT TRIM(rf.RDB$RELATION_NAME), TRIM(rf.RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS rf WHERE rf.RDB$RELATION_NAME IN ('C000025','RTC_CCLASSTRIB','RTC_CST','PARAMETROS')")
|
||||
for table, column in cursor.fetchall():
|
||||
schema.setdefault(table, set()).add(column)
|
||||
extras: dict = {"param_value": None, "param_rows": 0, "param_conflict": False, "physical_fk": False}
|
||||
if "USAR_REFORMA_TRIBUTARIA" in schema.get("PARAMETROS", ()):
|
||||
try:
|
||||
# ALL rows are read: the source of truth is what THIS installation's database says NOW (never a cached or Hub-side value).
|
||||
cursor.execute("SELECT FIRST 20 USAR_REFORMA_TRIBUTARIA FROM PARAMETROS")
|
||||
values = [None if r[0] is None else str(r[0]).strip().upper() for r in cursor.fetchall()]
|
||||
extras["param_rows"] = len(values)
|
||||
if len(set(values)) > 1:
|
||||
extras["param_conflict"] = True # rows disagree: never guess "on" (RTC stays out of scope, and the conflict is reported)
|
||||
elif values:
|
||||
extras["param_value"] = values[0]
|
||||
except Exception: # noqa: BLE001 - unreadable parameter => RTC stays off (not enabled by the ERP)
|
||||
extras["param_value"] = None
|
||||
try:
|
||||
cursor.execute("SELECT COUNT(*) FROM RDB$RELATION_CONSTRAINTS WHERE RDB$RELATION_NAME = 'C000025' AND RDB$CONSTRAINT_TYPE = 'FOREIGN KEY' AND RDB$CONSTRAINT_NAME CONTAINING 'CCLASSTRIB'")
|
||||
extras["physical_fk"] = cursor.fetchone()[0] > 0
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
catalog = None
|
||||
class_cols = schema.get(RTC_CCLASSTRIB_TABLE, set())
|
||||
if {"CCLASSTRIB", "CSTIBSCBS"} <= class_cols and "CSTIBSCBS" in schema.get(RTC_CST_TABLE, set()):
|
||||
validity = "INICIO_VIGENCIA" in class_cols and "FIM_VIGENCIA" in class_cols
|
||||
extra = [c for c in ("TIPO_ALIQUOTA", "PRED_IBS", "PRED_CBS", "IND_NFE", "IND_NFCE") if c in class_cols]
|
||||
cursor.execute("SELECT TRIM(CCLASSTRIB), TRIM(CSTIBSCBS)" + (", INICIO_VIGENCIA, FIM_VIGENCIA" if validity else "") + "".join(f", {c}" for c in extra) + " FROM RTC_CCLASSTRIB")
|
||||
rows = cursor.fetchall()
|
||||
classes = {r[0]: (r[1] or None) for r in rows}
|
||||
cursor.execute("SELECT TRIM(CSTIBSCBS) FROM RTC_CST")
|
||||
catalog = {"classes": classes, "cst": {r[0] for r in cursor.fetchall()}, "duplicates": len(rows) - len(classes)}
|
||||
if validity:
|
||||
catalog["validity"] = {r[0]: (r[2], r[3]) for r in rows}
|
||||
offset = 4 if validity else 2
|
||||
catalog["rows"] = {r[0]: {"cst": r[1], "start": (r[2] if validity else None), "end": (r[3] if validity else None), **{c: r[offset + i] for i, c in enumerate(extra)}} for r in rows}
|
||||
for value in catalog["rows"].values():
|
||||
for key in ("TIPO_ALIQUOTA", "PRED_IBS", "PRED_CBS", "IND_NFE", "IND_NFCE"):
|
||||
if key in value:
|
||||
value[key.lower()] = value.pop(key)
|
||||
for flag in ("ind_nfe", "ind_nfce"):
|
||||
if flag in value:
|
||||
value[flag] = str(value[flag]).strip() in ("1", "True", "true")
|
||||
return schema, catalog, extras
|
||||
|
||||
|
||||
def read_products(cursor, schema: dict) -> list[dict]:
|
||||
"""SELECT of only the columns that exist (old schemas keep working)."""
|
||||
have = schema.get(PRODUCT_TABLE, set())
|
||||
columns = [c for c in PRODUCT_COLUMNS if c in have]
|
||||
if "CODIGO" not in columns:
|
||||
return []
|
||||
cursor.execute(f"SELECT {', '.join(columns)} FROM {PRODUCT_TABLE} ORDER BY CODIGO")
|
||||
return [dict(zip(columns, row)) for row in cursor.fetchall()]
|
||||
|
||||
|
||||
def assess_from_connection(conn, company: dict | None, *, today: date | None = None, rtc_override: bool | None = None, ncm_valid: dict | None = None) -> dict:
|
||||
"""One read-only pass. Returns {"context": {...}, "findings": {product_code: evaluate_product(...)}}.
|
||||
rtc_override forces RTC on/off for simulations WITHOUT touching the ERP parameter (tests / what-if only).
|
||||
ncm_valid: {product_code: bool} from the decision engine (EXACT_NCM); a product whose NCM is not confirmed never gets a cClassTrib proposal."""
|
||||
cursor = conn.cursor()
|
||||
schema, catalog, extras = load_rtc_context(cursor)
|
||||
caps = detect_rtc_capabilities(schema, param_value=extras["param_value"], catalog=catalog, physical_fk=extras["physical_fk"])
|
||||
enabled = rtc_enabled(caps) if rtc_override is None else bool(rtc_override and caps["HAS_RTC_RELATIONSHIP"])
|
||||
today = today or date.today()
|
||||
cclasstrib_app = cclasstrib_applicability((company or {}).get("crt"), enabled, today)
|
||||
rtc = {"enabled": enabled, "catalog": catalog, "catalog_valid": caps["RTC_CATALOG_VALID"], "cclasstrib_applicability": cclasstrib_app}
|
||||
products = read_products(cursor, schema)
|
||||
findings = {}
|
||||
warn_by_type: Counter = Counter()
|
||||
warn_products: set = set()
|
||||
samples: list = []
|
||||
applicability: Counter = Counter()
|
||||
local_rows = (catalog or {}).get("rows") or {}
|
||||
applicable = cclasstrib_app["state"] == "REQUIRED"
|
||||
compat = compare_catalogs(local_rows, today) if (enabled and applicable and local_rows) else None
|
||||
resolver_ctx = {"rtc_enabled": enabled, "cclasstrib_applicable": applicable, "local_catalog_valid": bool(caps["RTC_CATALOG_VALID"])}
|
||||
for p in products:
|
||||
found = evaluate_product(p, company, rtc, today)
|
||||
if enabled and applicable:
|
||||
code0 = str(p.get("CODIGO") or "")
|
||||
resolution = resolve_cclasstrib({**p, "ncm_valid": bool((ncm_valid or {}).get(code0, False))}, company, resolver_ctx, local_rows, compat=compat, today=today)
|
||||
refine_rtc_finding(found, p, resolution, compat["per_code"] if compat else None)
|
||||
found["input_fingerprint"] = hashlib.sha256(json.dumps({k: (None if v is None else str(v).strip()) for k, v in p.items()} | {"rtc": enabled, "cclasstrib_applicability": cclasstrib_app}, sort_keys=True).encode()).hexdigest()
|
||||
code = str(p.get("CODIGO") or "")
|
||||
findings[code] = found
|
||||
applicability[found["applicability"]["PIS_COFINS"]] += 1
|
||||
for w in found.get("warnings", []):
|
||||
warn_by_type[w["type"]] += 1
|
||||
warn_products.add(code)
|
||||
if len(samples) < 10:
|
||||
samples.append({"product_code": code, **w})
|
||||
integrity = catalog_integrity(catalog) if catalog else None
|
||||
return {"context": {"rtc_enabled": enabled, "erp_rtc_enabled": enabled, "rtc_supported": caps["HAS_RTC_RELATIONSHIP"], "rtc_catalog_valid": caps["RTC_CATALOG_VALID"], "rtc_param": extras["param_value"],
|
||||
"rtc_source": "SIMULATION" if rtc_override is not None else "PARAMETROS.USAR_REFORMA_TRIBUTARIA", "rtc_param_rows": extras["param_rows"], "rtc_param_conflict": extras["param_conflict"],
|
||||
"rtc_observed_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "cclasstrib_applicability": cclasstrib_app,
|
||||
"legal_effective_context": legal_effective_context((company or {}).get("crt")),
|
||||
"catalog_integrity": integrity, "crt": (company or {}).get("crt"), "capabilities": caps, "products_evaluated": len(products),
|
||||
"pis_cofins_applicability": dict(applicability),
|
||||
"rtc_resolution": summarize_rtc(findings, compat) if enabled else None, "local_catalog_hash": local_catalog_hash(local_rows) if local_rows else None,
|
||||
"official_catalog": (compat or {}).get("official_catalog"),
|
||||
"non_canonical_warnings": {"products": len(warn_products), "by_type": dict(sorted(warn_by_type.items())), "samples": samples}},
|
||||
"findings": findings}
|
||||
@@ -0,0 +1,24 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).parents[1] / "src"))
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_real_firebird_for_unified_fiscal(monkeypatch, request):
|
||||
"""The unified fiscal assessment opens its own READ ONLY connection; unit tests have no Firebird, so it is neutral here unless a test opts in."""
|
||||
if request.node.get_closest_marker("real_unified_fiscal"):
|
||||
return
|
||||
from gti_fiscal_agent import firebird_reader, unified_fiscal
|
||||
|
||||
class _Conn:
|
||||
def close(self):
|
||||
pass
|
||||
monkeypatch.setattr(firebird_reader, "_connect", lambda env: _Conn())
|
||||
monkeypatch.setattr(unified_fiscal, "assess_from_connection", lambda conn, company, **kw: {"context": {"rtc_enabled": False, "rtc_supported": False, "rtc_catalog_valid": False, "products_evaluated": 0}, "findings": {}})
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
config.addinivalue_line("markers", "real_unified_fiscal: use the real unified fiscal assessment (with a fake cursor) instead of the neutral stub")
|
||||
@@ -0,0 +1,239 @@
|
||||
from gti_fiscal_agent import cfop_ippt_auto_apply as apply_mod
|
||||
from central.app.field_contract import ALREADY, CHANGE, MANUAL, NA
|
||||
|
||||
|
||||
class FakeFirebird:
|
||||
"""A tiny C000025: every statement is recorded; only SELECT <field> and the one allowed UPDATE per field are understood."""
|
||||
|
||||
def __init__(self, products):
|
||||
self.products = {c: dict(v) for c, v in products.items()} # code -> {"CFOP": ..., "IPPT": ...}
|
||||
self.statements, self.commits, self.rollbacks = [], 0, 0
|
||||
self._pending = {}
|
||||
|
||||
def cursor(self):
|
||||
return FakeCursor(self)
|
||||
|
||||
def commit(self):
|
||||
for (code, field), value in self._pending.items():
|
||||
self.products[code][field] = value
|
||||
self._pending = {}
|
||||
self.commits += 1
|
||||
|
||||
def rollback(self):
|
||||
self._pending = {}
|
||||
self.rollbacks += 1
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
def __init__(self, db):
|
||||
self.db, self._rows, self.rowcount = db, [], -1
|
||||
|
||||
def fetchall(self):
|
||||
return self._rows
|
||||
|
||||
def fetchone(self):
|
||||
return self._rows[0] if self._rows else None
|
||||
|
||||
def execute(self, sql, params=()):
|
||||
db = self.db
|
||||
db.statements.append((sql, tuple(params)))
|
||||
text = " ".join(sql.split())
|
||||
up = text.upper()
|
||||
for field in ("CFOP", "IPPT"):
|
||||
if up == f"SELECT {field} FROM C000025 WHERE CODIGO = ?":
|
||||
code = params[0]
|
||||
self._rows = [(db.products[code][field],)] if code in db.products else []
|
||||
if code in db.products and db.products[code].get("DUP"):
|
||||
self._rows = self._rows * 2
|
||||
return
|
||||
if up.startswith(f"UPDATE C000025 SET {field} = ? WHERE CODIGO = ? AND"):
|
||||
proposed, code, expected = params
|
||||
product = db.products.get(code)
|
||||
current = (product.get(field) if product else None) or None
|
||||
hit = product is not None and current == (expected or None)
|
||||
self.rowcount = 1 if hit else 0
|
||||
if hit:
|
||||
db._pending[(code, field)] = proposed
|
||||
return
|
||||
raise AssertionError(f"unexpected SQL: {sql!r}")
|
||||
|
||||
|
||||
def candidate(code, field, expected_before, proposed, reason="X"):
|
||||
return {"product_code": code, "field": field, "expected_before": expected_before, "proposed": proposed, "reason": reason,
|
||||
"fingerprint": apply_mod.candidate_fingerprint(code, field, expected_before, proposed)}
|
||||
|
||||
|
||||
def results_for(code, field, *, state, current, proposed, write_eligible):
|
||||
return {code: {field: {"state": state, "current": current, "proposed": proposed, "write_eligible": write_eligible, "reason": "R"}}}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ write_gate (flag semantics)
|
||||
|
||||
def test_write_gate_default_off_blocks_every_mode_but_enabled_and_approval_required():
|
||||
assert apply_mod.write_gate(mode="DISABLED")["allowed"] is False
|
||||
assert apply_mod.write_gate(mode="DRY_RUN")["allowed"] is False
|
||||
assert apply_mod.write_gate(mode="APPROVAL_REQUIRED", approved=False)["allowed"] is False
|
||||
assert apply_mod.write_gate(mode="APPROVAL_REQUIRED", approved=True)["allowed"] is True
|
||||
assert apply_mod.write_gate(mode="ENABLED")["allowed"] is True
|
||||
assert apply_mod.write_gate(mode="ENABLED", capability=False)["allowed"] is False
|
||||
assert apply_mod.write_gate(mode="NOT_A_REAL_MODE")["allowed"] is False
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ select_candidates only picks AUTO_FILL_SAFE CFOP/IPPT
|
||||
|
||||
def test_select_candidates_only_change_and_write_eligible_cfop_or_ippt():
|
||||
results = {
|
||||
"000001": {"CFOP": {"state": CHANGE, "current": "5102", "proposed": "5101", "write_eligible": True, "reason": "R1"},
|
||||
"IPPT": {"state": MANUAL, "current": None, "proposed": None, "write_eligible": False, "reason": "R2"},
|
||||
"CSOSN": {"state": CHANGE, "current": None, "proposed": "102", "write_eligible": True, "reason": "R3"}}, # not CFOP/IPPT: never a candidate
|
||||
"000002": {"CFOP": {"state": ALREADY, "current": "5102", "proposed": None, "write_eligible": False, "reason": "R4"},
|
||||
"IPPT": {"state": CHANGE, "current": "T", "proposed": "P", "write_eligible": False, "reason": "R5"}}, # write_eligible=False: never a candidate
|
||||
"000003": {"IPPT": {"state": CHANGE, "current": "T", "proposed": "P", "write_eligible": True, "reason": "R6"}},
|
||||
}
|
||||
cands, total = apply_mod.select_candidates(results)
|
||||
assert total == 2
|
||||
keys = {(c["product_code"], c["field"]) for c in cands}
|
||||
assert keys == {("000001", "CFOP"), ("000003", "IPPT")}
|
||||
assert all(c["field"] in apply_mod.ALLOWED_FIELDS for c in cands)
|
||||
|
||||
|
||||
def test_weak_composed_evidence_never_becomes_a_candidate_regardless_of_the_write_flag():
|
||||
"""Missão 4 / Bloco A.3: a cell whose role evidence is weak (field_contract itself refuses to
|
||||
classify it OWN_PRODUCTION/RESALE with confidence) must never reach select_candidates as
|
||||
AUTO_FILL_SAFE - proven end-to-end from the real resolver, not just asserted in isolation.
|
||||
This holds independently of cfop_ippt_write_mode/write_gate, which only gates whether
|
||||
apply_candidates() is ever invoked - it never affects what counts as a candidate."""
|
||||
from central.app.field_contract import resolve_product, Context, WRITE_FIELDS
|
||||
|
||||
def product(**kw):
|
||||
p = {f: None for f in WRITE_FIELDS}
|
||||
p.update({"CODIGO": "000321", "PRODUTO": "CERVEJA LONG NECK NOVA SCHIN", "CODGRUPO": "G1", "TIPO": "VENDA", "PROD_COZINHA": None, "PRODUTO_SERVICO": None,
|
||||
"CLASSIFICACAO_FISCAL": "22030000", "CFOP": "5102", "CSOSN": "102", "CST": "000", "SITUACAO_TRIBUTARIA": "T", "IAT": "A", "IPPT": "T",
|
||||
"INCIDENCIA_PISCOFINS": "NENHUMA", "PISCOFINS": "N"})
|
||||
p.update(kw)
|
||||
return p
|
||||
|
||||
decision = {"status": "EXACT_NCM", "cest_status": "NO_CEST_MAPPING_FOUND", "ipi_status": "IPI_RESOLVED", "ba_st_mva_status": "NO_MVA_MAPPING_FOUND", "dataset_versions": {},
|
||||
"classification": {"ncm": "22030000", "ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.9, "rate": "3.9"}, "ipi_candidates": [], "cest": None, "cest_candidates": [], "ba_st_mva": None, "ba_st_mva_candidates": []}}
|
||||
refs = {"cfop_valid": {"5101", "5102"}, "csosn": {"102"}, "cst_icms": set(), "cst_pis": set(), "cst_cofins": set(), "origem": set("012345678"), "fcp_uf": {}, "cbs_rate_2026": None, "cclasstrib_count": 0}
|
||||
co = {"uf": "BA", "crt": "1", "contribuinte_ipi": "N", "company_code": "000001"}
|
||||
ev_weak = {"recipes": set(), "composed": set(), "composed_ratio": {"000321": {"composed": 1, "other": 19}}, "purchases": {}, "stock_entry": {"000321": True}, "stock_in": {"000321": True}}
|
||||
result = resolve_product(product(), decision, co, refs, ev_weak, Context([]))
|
||||
candidates, total = apply_mod.select_candidates({"000321": result})
|
||||
assert total == 0 and candidates == [] # the weak-evidence beer never becomes a write candidate
|
||||
|
||||
|
||||
def test_select_candidates_respects_limit():
|
||||
results = {f"{i:06d}": {"CFOP": {"state": CHANGE, "current": None, "proposed": "5102", "write_eligible": True, "reason": "R"}} for i in range(5)}
|
||||
cands, total = apply_mod.select_candidates(results, limit=2)
|
||||
assert total == 5 and len(cands) == 2
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ apply_candidate: happy path, idempotency, stale guard
|
||||
|
||||
def test_apply_candidate_writes_verifies_and_commits():
|
||||
db = FakeFirebird({"000001": {"CFOP": "5102"}})
|
||||
c = candidate("000001", "CFOP", "5102", "5101")
|
||||
rec = apply_mod.apply_candidate(db, c, now="t0", job_id=1)
|
||||
assert rec["outcome"] == "APPLIED" and rec["old_value"] == "5102" and rec["new_value"] == "5101" and rec["verified_after"] is True
|
||||
assert db.products["000001"]["CFOP"] == "5101" and db.commits >= 1 and db.rollbacks == 0
|
||||
|
||||
|
||||
def test_second_run_over_the_same_candidate_writes_zero_no_write_required():
|
||||
db = FakeFirebird({"000001": {"CFOP": "5101"}}) # already at the proposed value (as if a first pass already applied it)
|
||||
c = candidate("000001", "CFOP", "5102", "5101")
|
||||
try:
|
||||
apply_mod.apply_candidate(db, c, now="t0", job_id=1)
|
||||
assert False, "expected CfopIpptApplyAborted"
|
||||
except apply_mod.CfopIpptApplyAborted as e:
|
||||
assert e.reason == "NO_WRITE_REQUIRED"
|
||||
assert db.products["000001"]["CFOP"] == "5101" # unchanged, no write attempted
|
||||
|
||||
|
||||
def test_human_override_wins_stale_value_aborts_without_writing():
|
||||
db = FakeFirebird({"000001": {"CFOP": "9999"}}) # a human (or anything else) set a THIRD value since the candidate was selected
|
||||
c = candidate("000001", "CFOP", "5102", "5101") # candidate still expects the OLD "5102"
|
||||
try:
|
||||
apply_mod.apply_candidate(db, c, now="t0", job_id=1)
|
||||
assert False, "expected CfopIpptApplyAborted"
|
||||
except apply_mod.CfopIpptApplyAborted as e:
|
||||
assert e.reason == "STALE_VALUE_CHANGED_SINCE_DECISION"
|
||||
assert db.products["000001"]["CFOP"] == "9999" # the human's value is preserved, never overwritten
|
||||
|
||||
|
||||
def test_apply_candidate_blank_expected_before_and_fill():
|
||||
db = FakeFirebird({"000801": {"CFOP": None}})
|
||||
c = candidate("000801", "CFOP", None, "5102")
|
||||
rec = apply_mod.apply_candidate(db, c, now="t0", job_id=1)
|
||||
assert rec["outcome"] == "APPLIED" and rec["old_value"] is None and rec["new_value"] == "5102"
|
||||
|
||||
|
||||
def test_apply_candidate_product_not_found_or_duplicate_row_aborts():
|
||||
db = FakeFirebird({})
|
||||
try:
|
||||
apply_mod.apply_candidate(db, candidate("999999", "CFOP", None, "5102"), now="t0")
|
||||
assert False
|
||||
except apply_mod.CfopIpptApplyAborted as e:
|
||||
assert e.reason == "PRODUCT_NOT_FOUND"
|
||||
db2 = FakeFirebird({"000001": {"CFOP": "5102", "DUP": True}})
|
||||
try:
|
||||
apply_mod.apply_candidate(db2, candidate("000001", "CFOP", "5102", "5101"), now="t0")
|
||||
assert False
|
||||
except apply_mod.CfopIpptApplyAborted as e:
|
||||
assert e.reason == "PRODUCT_ROW_NOT_UNIQUE"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ apply_candidates: batch semantics, authorize() gate, flag OFF => zero writes
|
||||
|
||||
def test_apply_candidates_batch_never_stops_on_one_abort_and_reports_reasons():
|
||||
db = FakeFirebird({"000001": {"CFOP": "5102"}, "000002": {"CFOP": "9999"}})
|
||||
cands = [candidate("000001", "CFOP", "5102", "5101"), candidate("000002", "CFOP", "5102", "5101")]
|
||||
out = apply_mod.apply_candidates(db, cands, now="t0", job_id=1)
|
||||
assert out["write_successes"] == 1 and out["write_attempts"] == 2
|
||||
assert out["aborted_reasons"] == {"STALE_VALUE_CHANGED_SINCE_DECISION": 1}
|
||||
assert db.products["000001"]["CFOP"] == "5101" and db.products["000002"]["CFOP"] == "9999"
|
||||
|
||||
|
||||
def test_flag_off_means_caller_never_invokes_apply_zero_writes():
|
||||
"""The Agent must gate BEFORE calling apply_candidates at all when write_gate(...).allowed is False -
|
||||
this is the actual "flag OFF => zero writes" contract; apply_candidates itself is only ever reached
|
||||
once the gate already said yes."""
|
||||
gate = apply_mod.write_gate(mode="DISABLED")
|
||||
assert not gate["allowed"]
|
||||
db = FakeFirebird({"000001": {"CFOP": "5102"}})
|
||||
# execute_job's real code path: `if gate["allowed"]: apply_candidates(...)` - simulate the guard directly.
|
||||
if gate["allowed"]:
|
||||
apply_mod.apply_candidates(db, [candidate("000001", "CFOP", "5102", "5101")], now="t0")
|
||||
assert db.products["000001"]["CFOP"] == "5102" and db.commits == 0
|
||||
|
||||
|
||||
def test_authorize_callback_can_veto_a_candidate_right_before_the_write():
|
||||
db = FakeFirebird({"000001": {"CFOP": "5102"}})
|
||||
out = apply_mod.apply_candidates(db, [candidate("000001", "CFOP", "5102", "5101")], now="t0", authorize=lambda code, field: "AUTHORIZATION_CHANGED")
|
||||
assert out["write_successes"] == 0 and out["aborted_reasons"] == {"AUTHORIZATION_CHANGED": 1}
|
||||
assert db.products["000001"]["CFOP"] == "5102"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ rollback: only if the value still equals what was written
|
||||
|
||||
def test_rollback_restores_old_value_when_unchanged():
|
||||
db = FakeFirebird({"000001": {"CFOP": "5101"}})
|
||||
item = {"product_code": "000001", "field": "CFOP", "old_value": "5102", "new_value": "5101", "audit_id": 42}
|
||||
rec = apply_mod.rollback_one(db, item, now="t1", job_id=1)
|
||||
assert rec["outcome"] == "ROLLED_BACK" and db.products["000001"]["CFOP"] == "5102"
|
||||
|
||||
|
||||
def test_rollback_refuses_when_a_human_changed_the_value_since():
|
||||
db = FakeFirebird({"000001": {"CFOP": "7777"}}) # someone changed it after the write, before the rollback
|
||||
item = {"product_code": "000001", "field": "CFOP", "old_value": "5102", "new_value": "5101", "audit_id": 42}
|
||||
rec = apply_mod.rollback_one(db, item, now="t1", job_id=1)
|
||||
assert rec["outcome"] == "ABORTED" and rec["reason"] == "ROLLBACK_CONFLICT_VALUE_CHANGED"
|
||||
assert db.products["000001"]["CFOP"] == "7777" # untouched
|
||||
|
||||
|
||||
def test_rollback_records_batch():
|
||||
db = FakeFirebird({"000001": {"CFOP": "5101"}, "000002": {"IPPT": "P"}})
|
||||
items = [{"product_code": "000001", "field": "CFOP", "old_value": "5102", "new_value": "5101"},
|
||||
{"product_code": "000002", "field": "IPPT", "old_value": "T", "new_value": "P"}]
|
||||
out = apply_mod.rollback_records(db, items, now="t1", job_id=1)
|
||||
assert out["rolled_back"] == 2
|
||||
@@ -0,0 +1,63 @@
|
||||
import httpx
|
||||
|
||||
from gti_fiscal_agent.client import HubClient
|
||||
|
||||
|
||||
def test_identity_requests_authenticated_identity_only(monkeypatch):
|
||||
calls = {}
|
||||
payload = {
|
||||
"agent_id": 2,
|
||||
"installation_id": "00000000-0000-4000-8000-000000000222",
|
||||
"hostname": "DESKTOP-B0HL02I",
|
||||
"version": "1.1.1",
|
||||
"status": "OFFLINE",
|
||||
}
|
||||
|
||||
class Response:
|
||||
status_code = 200
|
||||
|
||||
def raise_for_status(self):
|
||||
return None
|
||||
|
||||
def json(self):
|
||||
return payload
|
||||
|
||||
def fake_get(url, *, headers, timeout):
|
||||
calls.update(url=url, headers=headers, timeout=timeout)
|
||||
return Response()
|
||||
|
||||
monkeypatch.setattr(httpx, "get", fake_get)
|
||||
client = HubClient("https://hub.example", "test-agent-token")
|
||||
assert client.identity() == payload
|
||||
|
||||
assert calls["url"] == "https://hub.example/v1/agents/me/identity"
|
||||
assert calls["headers"] == {"Authorization": "Bearer test-agent-token"}
|
||||
|
||||
|
||||
def test_transport_errors_and_proxy_5xx_are_retried_but_client_errors_are_not(monkeypatch):
|
||||
import httpx
|
||||
from gti_fiscal_agent import client as c
|
||||
monkeypatch.setattr(c.time, "sleep", lambda s: None)
|
||||
calls = {"n": 0}
|
||||
|
||||
def flaky():
|
||||
calls["n"] += 1
|
||||
if calls["n"] == 1:
|
||||
raise httpx.ConnectError("ssl handshake failure")
|
||||
if calls["n"] == 2:
|
||||
return httpx.Response(502)
|
||||
return httpx.Response(200, json={"ok": True})
|
||||
|
||||
assert c._resilient(flaky).status_code == 200 and calls["n"] == 3
|
||||
calls["n"] = 0
|
||||
assert c._resilient(lambda: (calls.__setitem__("n", calls["n"] + 1), httpx.Response(401))[1]).status_code == 401 and calls["n"] == 1 # 4xx: no retry
|
||||
calls["n"] = 0
|
||||
|
||||
def always_down():
|
||||
calls["n"] += 1
|
||||
raise httpx.ConnectError("down")
|
||||
|
||||
import pytest
|
||||
with pytest.raises(httpx.ConnectError):
|
||||
c._resilient(always_down)
|
||||
assert calls["n"] == 10 # bounded: 10 attempts (~3 min), then the job fails cleanly
|
||||
@@ -0,0 +1,134 @@
|
||||
import pytest
|
||||
|
||||
from gti_fiscal_agent.company_context import CompanyContextError, config_from_env, mask_cnpj, preflight_report, resolve
|
||||
from gti_fiscal_agent.firebird_reader import parse_company_context
|
||||
|
||||
CN_A, CN_B, CN_C = "11111111000111", "22222222000122", "33333333000133"
|
||||
|
||||
|
||||
def co(code, cnpj, uf="BA", crt="1", optante="S"):
|
||||
return {"code": code, "cnpj": cnpj, "uf": uf, "crt": crt, "optante": optante, "name": "X"}
|
||||
|
||||
|
||||
def ev(companies, emissor=None, docs=None):
|
||||
return {"c000004": companies, "emissor": emissor if emissor is not None else [], "nfce_keys": docs or [], "nf_by_filial": {}}
|
||||
|
||||
|
||||
def emis(cnpj=CN_A, uf="BA", crt="1"):
|
||||
return {"cnpj": cnpj, "uf": uf, "crt": crt}
|
||||
|
||||
|
||||
DOC_A = [{"cuf": "29", "cnpj": CN_A, "n": 100}]
|
||||
|
||||
|
||||
def test_single_company_all_sources_agree_is_auto_resolved():
|
||||
r = resolve(ev([co("000001", CN_A)], [emis()], DOC_A))
|
||||
assert r["status"] == "PASS" and r["classification"] == "AUTO_RESOLVED" and r["company_uf"] == "BA" and r["company_crt"] == "1" and r["confidence"] == "HIGH"
|
||||
assert "DOCUMENT_KEYS" in r["source_of_truth"]["uf"]
|
||||
|
||||
|
||||
def test_multiple_rows_resolved_only_through_emitter_identity_never_first_row():
|
||||
rows = [co("000001", CN_B, "SP", None, "N"), co("000005", CN_A), co("000007", CN_C, "BA", None, "N")]
|
||||
r = resolve(ev(rows, [emis(CN_A)], DOC_A))
|
||||
assert r["status"] == "PASS" and r["company_code"] == "000005" and r["source_of_truth"]["company"] == "EMISSOR_CNPJ_MATCH"
|
||||
|
||||
|
||||
def test_multiple_rows_without_disambiguation_requires_config():
|
||||
rows = [co("000001", CN_A), co("000002", CN_B)]
|
||||
r = resolve(ev(rows, []))
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFIG_REQUIRED" and r["company_code"] is None
|
||||
assert resolve(ev(rows, [emis(CN_C)]))["classification"] == "CONFIG_REQUIRED" # emitter matches none
|
||||
ok = resolve(ev(rows, [], []), {"company_code": "000002"})
|
||||
assert ok["status"] == "PASS" and ok["company_code"] == "000002" and ok["classification"] == "CONFIGURED"
|
||||
|
||||
|
||||
def test_configured_company_must_exist_and_agree_with_emitter():
|
||||
rows = [co("000001", CN_A), co("000002", CN_B)]
|
||||
assert resolve(ev(rows, [emis(CN_A)]), {"company_cnpj": "99999999999999"})["classification"] == "CONFIG_REQUIRED"
|
||||
r = resolve(ev(rows, [emis(CN_A)], DOC_A), {"company_code": "000002"})
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFLICT" # selected company differs from the emitter: unexplained
|
||||
|
||||
|
||||
def test_documents_only_from_another_cnpj_conflict_but_mixed_history_is_a_warning():
|
||||
r = resolve(ev([co("000001", CN_A)], [emis()], [{"cuf": "29", "cnpj": CN_B, "n": 10}]))
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFLICT"
|
||||
r = resolve(ev([co("000001", CN_A)], [emis()], [{"cuf": "29", "cnpj": CN_A, "n": 10}, {"cuf": "29", "cnpj": CN_B, "n": 1}]))
|
||||
assert r["status"] == "PASS" and any("HISTORICAL_NFCE_KEYS_FROM_OTHER_CNPJ" in w for w in r["warnings"])
|
||||
|
||||
|
||||
def test_uf_conflict_and_unknown():
|
||||
assert resolve(ev([co("1", CN_A, "BA")], [emis(uf="SP")]))["classification"] == "CONFLICT"
|
||||
assert resolve(ev([co("1", CN_A, "BA")], [emis()], [{"cuf": "35", "cnpj": CN_A, "n": 5}]))["classification"] == "CONFLICT" # key says SP
|
||||
assert resolve(ev([co("1", CN_A, "")], [emis(uf="")]))["classification"] == "UNRESOLVED"
|
||||
|
||||
|
||||
def test_crt_conflict_needs_explicit_source_and_records_it():
|
||||
base = ev([co("1", CN_A, crt="3")], [emis(crt="1")], DOC_A)
|
||||
r = resolve(base)
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFLICT" and r["crt_candidates"] == {"EMISSOR": "1", "C000004": "3"}
|
||||
ok = resolve(base, {"crt_source": "EMISSOR"})
|
||||
assert ok["status"] == "PASS" and ok["company_crt"] == "1" and ok["classification"] == "CONFIGURED" and ok["confidence"] == "MEDIUM"
|
||||
assert resolve(base, {"crt_source": "C000004"})["company_crt"] == "3"
|
||||
assert resolve(base, {"crt_source_invalid": True})["classification"] == "CONFIG_REQUIRED"
|
||||
|
||||
|
||||
def test_crt_unknown_fails_and_single_source_is_flagged():
|
||||
assert resolve(ev([co("1", CN_A, crt=None)], [emis(crt="")]))["classification"] == "UNRESOLVED"
|
||||
r = resolve(ev([co("1", CN_A, crt=None)], [emis(crt="1")]))
|
||||
assert r["status"] == "PASS" and "CRT_HAS_A_SINGLE_SOURCE" in r["warnings"]
|
||||
|
||||
|
||||
def test_optante_inconsistency_is_a_warning_not_a_guess():
|
||||
r = resolve(ev([co("1", CN_A, crt="3", optante="S")], [emis(crt="3")]))
|
||||
assert r["status"] == "PASS" and any("OPTANTE_SIMPLES" in w for w in r["warnings"])
|
||||
|
||||
|
||||
def test_contribuinte_ipi_is_auto_discovered_from_the_same_selected_c000004_row():
|
||||
"""C000004.CONTRIBUINTE_IPI (real column, same row already proven to be THE company by CRT/UF resolution) closes the fiscal-completion gap
|
||||
where CST_IPI could not be resolved without asking an operator for a fact the ERP already proves."""
|
||||
row = co("000001", CN_A)
|
||||
row["contribuinte_ipi"] = "N"
|
||||
r = resolve(ev([row], [emis()], DOC_A))
|
||||
assert r["company_contribuinte_ipi"] == "N" and r["source_of_truth"]["contribuinte_ipi"] == "C000004"
|
||||
|
||||
|
||||
def test_contribuinte_ipi_absent_or_invalid_stays_unresolved_never_guessed():
|
||||
r = resolve(ev([co("000001", CN_A)], [emis()], DOC_A)) # co() never sets contribuinte_ipi
|
||||
assert r["company_contribuinte_ipi"] is None
|
||||
row = co("000002", CN_B); row["contribuinte_ipi"] = "X" # garbage value: never trusted
|
||||
r2 = resolve(ev([row], [emis(cnpj=CN_B)], [{"cuf": "29", "cnpj": CN_B, "n": 1}]))
|
||||
assert r2["company_contribuinte_ipi"] is None
|
||||
|
||||
|
||||
def test_full_cnpj_is_never_exposed():
|
||||
r = resolve(ev([co("1", CN_A)], [emis()], DOC_A))
|
||||
dump = str(preflight_report(r)) + str(r)
|
||||
assert CN_A not in dump and mask_cnpj(CN_A) == "11**********11"
|
||||
|
||||
|
||||
def test_read_company_context_fails_closed(monkeypatch):
|
||||
from gti_fiscal_agent import company_context as cc, firebird_reader as fr
|
||||
monkeypatch.setattr(cc, "preflight_company_context", lambda env: resolve(ev([co("1", CN_A), co("2", CN_B)], [])))
|
||||
with pytest.raises(CompanyContextError):
|
||||
fr.read_company_context({})
|
||||
monkeypatch.setattr(cc, "preflight_company_context", lambda env: resolve(ev([co("1", CN_A)], [emis()], DOC_A)))
|
||||
assert fr.read_company_context({})["uf"] == "BA"
|
||||
|
||||
|
||||
def test_config_from_env_and_legacy_parser_unchanged():
|
||||
assert config_from_env({"COMPANY_CODE": " 000005 ", "COMPANY_CNPJ": "11.111.111/0001-11", "COMPANY_CRT_SOURCE": "emissor"}) == {"company_code": "000005", "company_cnpj": "11111111000111", "crt_source": "EMISSOR", "crt_source_invalid": False, "uf_override": None}
|
||||
assert config_from_env({"COMPANY_CRT_SOURCE": "x"})["crt_source_invalid"] is True
|
||||
assert parse_company_context([("ba", "1")]) == {"uf": "BA", "crt": "1"}
|
||||
|
||||
|
||||
def test_auto_crt_source_and_uf_override_semantics():
|
||||
assert config_from_env({"COMPANY_CRT_SOURCE": "AUTO"}) ["crt_source"] is None and config_from_env({"COMPANY_CRT_SOURCE": "AUTO"})["crt_source_invalid"] is False
|
||||
conflict = ev([co("1", CN_A, crt="3")], [emis(crt="1")], DOC_A)
|
||||
assert resolve(conflict, config_from_env({"COMPANY_CRT_SOURCE": "AUTO"}))["status"] == "FAIL" # AUTO never hides a CRT conflict
|
||||
good = ev([co("1", CN_A)], [emis()], DOC_A)
|
||||
assert resolve(good, {"uf_override": "BA"})["status"] == "PASS"
|
||||
r = resolve(good, {"uf_override": "SP"})
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFLICT" # override never replaces the resolver silently
|
||||
blank = ev([co("1", CN_A, uf="")], [emis(uf="")])
|
||||
r = resolve(blank, {"uf_override": "BA"})
|
||||
assert r["status"] == "PASS" and "UF_FROM_EXPLICIT_OVERRIDE" in r["warnings"]
|
||||
@@ -0,0 +1,179 @@
|
||||
"""DPAPI LocalMachine credential store: encrypt/decrypt round-trip, plaintext
|
||||
fallback is dev-only, ACL is SYSTEM/Administrators, and the token is never
|
||||
written into agent.env."""
|
||||
import importlib.util
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gti_fiscal_agent import credential_store
|
||||
|
||||
import win32api
|
||||
|
||||
AGENT_ROOT = Path(__file__).resolve().parents[1]
|
||||
RUN_SCRIPT = AGENT_ROOT / "scripts" / "run_agent_service.py"
|
||||
|
||||
|
||||
def load_run_service():
|
||||
spec = importlib.util.spec_from_file_location("run_agent_service_cred", RUN_SCRIPT)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def write_plaintext(path: Path, value: str = "TOKEN_PLAINTEXT_DEV"):
|
||||
path.write_text(value, encoding="utf-8")
|
||||
|
||||
|
||||
# ---------- DPAPI round-trip ----------
|
||||
|
||||
def _allow_current_user(path: Path) -> None:
|
||||
"""The strict SYSTEM/Administrators DACL blocks the pytest user (not an
|
||||
admin here). As the file OWNER the current user may grant itself read —
|
||||
exactly like the elevated installer/local administrator does on the real
|
||||
host before the credential is consumed by the SYSTEM service."""
|
||||
import subprocess
|
||||
|
||||
import win32security
|
||||
|
||||
acct_sid, _domain, _type = win32security.LookupAccountName(None, win32api.GetUserName())
|
||||
sid_str = win32security.ConvertSidToStringSid(acct_sid)
|
||||
subprocess.run(["icacls", str(path), "/grant", f"*{sid_str}:(R)"], check=True, capture_output=True)
|
||||
|
||||
|
||||
def test_write_then_read_roundtrip(tmp_path):
|
||||
path = tmp_path / "credentials.bin"
|
||||
credential_store.write_credential(path, "TOKEN_DPAPI_1")
|
||||
_allow_current_user(path)
|
||||
assert credential_store.read_credential(path) == "TOKEN_DPAPI_1"
|
||||
|
||||
|
||||
def test_stored_blob_never_contains_the_plaintext_secret(tmp_path):
|
||||
path = tmp_path / "credentials.bin"
|
||||
credential_store.write_credential(path, "TOKEN_VERY_SECRET_9")
|
||||
_allow_current_user(path)
|
||||
raw = path.read_bytes()
|
||||
assert b"TOKEN_VERY_SECRET_9" not in raw
|
||||
|
||||
|
||||
def test_read_missing_file_raises(tmp_path):
|
||||
with pytest.raises(credential_store.CredentialStoreError):
|
||||
credential_store.read_credential(tmp_path / "nope.bin")
|
||||
|
||||
|
||||
def test_write_detects_dpapi_unavailable_and_fails_closed(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(credential_store, "_dpapi_available", lambda: False)
|
||||
with pytest.raises(credential_store.CredentialStoreError):
|
||||
credential_store.write_credential(tmp_path / "credentials.bin", "never")
|
||||
|
||||
|
||||
# ---------- plaintext fallback (dev-only) ----------
|
||||
|
||||
def test_plaintext_rejected_by_default(tmp_path):
|
||||
path = tmp_path / "credentials.bin"
|
||||
write_plaintext(path)
|
||||
with pytest.raises(credential_store.CredentialStoreError):
|
||||
credential_store.read_credential(path)
|
||||
|
||||
|
||||
def test_plaintext_accepted_only_with_dev_flag(tmp_path, monkeypatch):
|
||||
path = tmp_path / "credentials.bin"
|
||||
write_plaintext(path, "TOKEN_DEV_PLAIN")
|
||||
assert credential_store.read_credential(path, env={"GTI_ALLOW_PLAINTEXT_CREDENTIAL": "1"}) == "TOKEN_DEV_PLAIN"
|
||||
monkeypatch.setenv("GTI_ALLOW_PLAINTEXT_CREDENTIAL", "1")
|
||||
assert credential_store.read_credential(path) == "TOKEN_DEV_PLAIN"
|
||||
monkeypatch.delenv("GTI_ALLOW_PLAINTEXT_CREDENTIAL", raising=False)
|
||||
with pytest.raises(credential_store.CredentialStoreError):
|
||||
credential_store.read_credential(path)
|
||||
|
||||
|
||||
# ---------- ACL (SYSTEM/Administrators only) ----------
|
||||
|
||||
def test_write_sets_system_and_administrators_acl(tmp_path):
|
||||
import win32security
|
||||
|
||||
path = tmp_path / "credentials.bin"
|
||||
credential_store.write_credential(path, "TOKEN_ACL_7")
|
||||
admin_sid = win32security.ConvertStringSidToSid("S-1-5-32-544")
|
||||
system_sid = win32security.ConvertStringSidToSid("S-1-5-18")
|
||||
sd = win32security.GetFileSecurity(str(path), win32security.DACL_SECURITY_INFORMATION)
|
||||
dacl = sd.GetSecurityDescriptorDacl()
|
||||
assert dacl.GetAceCount() == 2
|
||||
allow_trustees = []
|
||||
for i in range(dacl.GetAceCount()):
|
||||
ace = dacl.GetAce(i)
|
||||
assert ace[0][0] == win32security.ACCESS_ALLOWED_ACE_TYPE, ace # only allows, no deny
|
||||
allow_trustees.append(win32security.ConvertSidToStringSid(ace[2]))
|
||||
assert set(allow_trustees) == {"S-1-5-18", "S-1-5-32-544"}
|
||||
|
||||
|
||||
# ---------- install_env_from_credentials / Config integration ----------
|
||||
|
||||
def test_install_env_from_credentials_fills_agent_token(tmp_path):
|
||||
path = tmp_path / "credentials.bin"
|
||||
credential_store.write_credential(path, "TOKEN_FROM_FILE")
|
||||
_allow_current_user(path)
|
||||
env = {"AGENT_TOKEN": "", "HUB_URL": "https://x"}
|
||||
install_env_from_credentials = credential_store.install_env_from_credentials
|
||||
assert install_env_from_credentials(path, env) == "TOKEN_FROM_FILE"
|
||||
assert env["AGENT_TOKEN"] == "TOKEN_FROM_FILE"
|
||||
|
||||
|
||||
def test_install_env_prefers_explicit_agent_token(tmp_path):
|
||||
path = tmp_path / "credentials.bin"
|
||||
credential_store.write_credential(path, "TOKEN_FROM_FILE")
|
||||
env = {"AGENT_TOKEN": "TOKEN_FROM_ENV"}
|
||||
install_env_from_credentials = credential_store.install_env_from_credentials
|
||||
assert install_env_from_credentials(path, env) == "TOKEN_FROM_ENV"
|
||||
assert env["AGENT_TOKEN"] == "TOKEN_FROM_ENV"
|
||||
|
||||
|
||||
def test_config_reads_token_from_credential_file(tmp_path, monkeypatch):
|
||||
credential_path = tmp_path / "credentials.bin"
|
||||
credential_store.write_credential(credential_path, "TOKEN_CRED_FILE")
|
||||
_allow_current_user(credential_path)
|
||||
env = {
|
||||
"FIREBIRD_DATABASE_PATH": str(tmp_path / "nope.fdb"),
|
||||
"FIREBIRD_WRITES": "0",
|
||||
"FISCAL_MODE": "DRY_RUN",
|
||||
"GTI_CREDENTIAL_FILE": str(credential_path),
|
||||
}
|
||||
for key in list(env):
|
||||
monkeypatch.setenv(key, env[key])
|
||||
service = load_run_service()
|
||||
config = service.Config(os.environ)
|
||||
assert config.agent_token == "TOKEN_CRED_FILE"
|
||||
assert config.firebird_writes is False # FIREBIRD_WRITES stays 0 regardless of credential
|
||||
|
||||
|
||||
# ---------- no-write enforcement (drift) ----------
|
||||
|
||||
def test_firebird_writes_fail_closed_with_credential_file(tmp_path, monkeypatch):
|
||||
credential_path = tmp_path / "credentials.bin"
|
||||
credential_store.write_credential(credential_path, "TOKEN")
|
||||
_allow_current_user(credential_path)
|
||||
env = {
|
||||
"FIREBIRD_DATABASE_PATH": str(tmp_path / "x.fdb"),
|
||||
"FIREBIRD_WRITES": "0",
|
||||
"GTI_CREDENTIAL_FILE": str(credential_path),
|
||||
}
|
||||
for key in list(env):
|
||||
monkeypatch.setenv(key, env[key])
|
||||
service = load_run_service()
|
||||
|
||||
# Even a hostile FISCAL_MODE cannot enable writes when FIREBIRD_WRITES=0.
|
||||
monkeypatch.setenv("FISCAL_MODE", "AUTO_APPLY")
|
||||
config = service.Config(os.environ)
|
||||
assert config.firebird_writes is False
|
||||
|
||||
|
||||
def test_credential_file_default_lives_under_program_files():
|
||||
default = credential_store.default_credential_path()
|
||||
assert "Program Files" in str(default)
|
||||
assert default.name == "credentials.bin"
|
||||
@@ -0,0 +1,407 @@
|
||||
"""Agent enrichment: baseline/new-product detection, feature gating, typed write, second run = 0 writes, human override (incl. deleted value)."""
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
from gti_fiscal_agent import enrichment as E # noqa: E402
|
||||
|
||||
|
||||
class Conn:
|
||||
"""sqlite stand-in for the Firebird connection; close() is a no-op so the worker can 'open' it repeatedly."""
|
||||
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
def cursor(self):
|
||||
return self.db.cursor()
|
||||
|
||||
def commit(self):
|
||||
self.db.commit()
|
||||
|
||||
def rollback(self):
|
||||
self.db.rollback()
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
class Hub:
|
||||
def __init__(self, features, decisions=None, state="EXACT_GTIN_VERIFIED"):
|
||||
self.features = features
|
||||
self.decisions = decisions or {}
|
||||
self.state = state
|
||||
self.resolve_calls, self.reports, self.overridden = [], [], {}
|
||||
|
||||
def authorizations(self):
|
||||
return {"features": dict(self.features)}
|
||||
|
||||
def resolve_product(self, payload):
|
||||
self.resolve_calls.append(payload)
|
||||
fields = {k: dict(v) for k, v in self.decisions.items()}
|
||||
for f in list(fields):
|
||||
if f in self.overridden.get(payload["product_code"], set()):
|
||||
fields[f] = {"status": "HUMAN_OVERRIDDEN", "value": None}
|
||||
elif payload["current"].get(f) and not E.is_placeholder(f, payload["current"].get(f)):
|
||||
fields[f] = {"status": "ALREADY_SET", "value": None}
|
||||
return {"state": self.state, "fields": fields, "dataset_versions": {"NCM": {"sha256": "abc"}}}
|
||||
|
||||
def pending_enrichment(self):
|
||||
codes, self.flagged = list(getattr(self, "flagged", [])), []
|
||||
return codes
|
||||
|
||||
def report_enrichment(self, payload):
|
||||
self.reports.append(payload)
|
||||
for item in payload["fields"]:
|
||||
if item["outcome"] == "HUMAN_OVERRIDDEN":
|
||||
self.overridden.setdefault(payload["product_code"], set()).add(item["field"])
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
ENABLED = {"features_configured": True, "corrector_enabled": True, "enrichment_mode": "ENABLED"}
|
||||
DECISIONS = {"CLASSIFICACAO_FISCAL": {"status": "AUTO_FILL_SAFE", "value": "22021000", "rule": "NCM_KB_VERIFIED", "kb_status": "VERIFIED_KB"},
|
||||
"CEST": {"status": "AUTO_FILL_SAFE", "value": "0300700", "rule": "CEST_OFFICIAL_UNIQUE_MAPPING"},
|
||||
"IPI": {"status": "AUTO_FILL_SAFE", "value": "0.0", "rule": "IPI_TIPI_NUMERIC"}}
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def env(tmp_path):
|
||||
db = sqlite3.connect(":memory:", check_same_thread=False)
|
||||
db.execute("CREATE TABLE C000025(CODIGO TEXT PRIMARY KEY, PRODUTO TEXT, GTIN TEXT, CLASSIFICACAO_FISCAL TEXT, CEST TEXT, IPI REAL, OTHER TEXT, CODBARRA TEXT, "
|
||||
+ ", ".join(f"{f} TEXT" for f in E.FISCAL_COMPLETION_FIELDS + E.CLASSIFICATION_CONTEXT_FIELDS) + ")")
|
||||
db.executemany("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,OTHER) VALUES(?,?,?,?)", [(f"H{i}", f"HIST {i}", "", "keep") for i in range(50)])
|
||||
db.commit()
|
||||
hub = Hub(dict(ENABLED), DECISIONS)
|
||||
store = E.EnrichmentStore(tmp_path / "e.db")
|
||||
now = [1000.0]
|
||||
reproc = []
|
||||
worker = E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: Conn(db), write_conn=lambda: Conn(db), create_reprocess=reproc.append, clock=lambda: now[0], sleep=lambda s: None)
|
||||
return db, hub, store, worker, now, reproc
|
||||
|
||||
|
||||
def add(db, code, name="REFRIGERANTE COLA LATA 350ML", gtin="7891000100103"):
|
||||
db.execute("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,OTHER) VALUES(?,?,?,?)", (code, name, gtin, "keep"))
|
||||
db.commit()
|
||||
|
||||
|
||||
def row(db, code):
|
||||
return db.execute("SELECT CLASSIFICACAO_FISCAL,CEST,IPI,OTHER FROM C000025 WHERE CODIGO=?", (code,)).fetchone()
|
||||
|
||||
|
||||
def test_baseline_never_sends_historical_products(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
assert worker.step() == [] # baseline
|
||||
now[0] += 5
|
||||
assert worker.step() == [] and hub.resolve_calls == []
|
||||
assert len(store.known()) == 50
|
||||
|
||||
|
||||
def test_new_product_is_enriched_only_in_empty_fields_then_second_run_zero_writes(env):
|
||||
db, hub, store, worker, now, reproc = env
|
||||
worker.step()
|
||||
add(db, "N1")
|
||||
now[0] += 5
|
||||
assert worker.step() == ["N1"]
|
||||
assert row(db, "N1") == ("22021000", "0300700", 0.0, "keep")
|
||||
assert reproc == [["N1"]]
|
||||
assert row(db, "H1") == (None, None, None, "keep") # historical untouched
|
||||
assert [r["outcome"] for r in hub.reports[0]["fields"]] == ["APPLIED"] * 3
|
||||
now[0] += 5
|
||||
before = db.total_changes
|
||||
worker.step()
|
||||
assert db.total_changes == before and len(hub.resolve_calls) == 1 # nothing re-processed, 0 writes
|
||||
|
||||
|
||||
def test_current_report_includes_fiscal_completion_and_classification_columns(tmp_path):
|
||||
"""`current` sent to the Hub used to carry only CLASSIFICACAO_FISCAL/CEST/IPI. field_contract.resolve_product (now wired into
|
||||
/v1/products/resolve) also needs the current CFOP/CST/CSOSN/... values and the classification columns infer_role reads (TIPO/
|
||||
PROD_COZINHA/PRODUTO_SERVICO/CODGRUPO) - otherwise it can never see what's already filled or auto-discover product_role."""
|
||||
db = sqlite3.connect(":memory:", check_same_thread=False)
|
||||
db.execute("CREATE TABLE C000025(CODIGO TEXT PRIMARY KEY, PRODUTO TEXT, GTIN TEXT, CLASSIFICACAO_FISCAL TEXT, CEST TEXT, IPI REAL, OTHER TEXT, CODBARRA TEXT, "
|
||||
+ ", ".join(f"{f} TEXT" for f in E.FISCAL_COMPLETION_FIELDS + E.CLASSIFICATION_CONTEXT_FIELDS) + ")")
|
||||
db.commit()
|
||||
hub = Hub(dict(ENABLED), DECISIONS)
|
||||
store = E.EnrichmentStore(tmp_path / "e.db")
|
||||
now = [1000.0]
|
||||
worker = E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: Conn(db), write_conn=lambda: Conn(db), create_reprocess=lambda c: None, clock=lambda: now[0], sleep=lambda s: None)
|
||||
worker.step() # baseline
|
||||
db.execute("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,OTHER,CSOSN,TIPO,PRODUTO_SERVICO) VALUES(?,?,?,?,?,?,?)", ("N9", "PRODUTO X", "7891000100103", "keep", "102", "VENDA", None))
|
||||
db.commit()
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
sent = hub.resolve_calls[0]["current"]
|
||||
assert sent["CSOSN"] == "102" and sent["TIPO"] == "VENDA"
|
||||
for f in E.FISCAL_COMPLETION_FIELDS + E.CLASSIFICATION_CONTEXT_FIELDS:
|
||||
assert f in sent, f
|
||||
assert hub.resolve_calls[0]["ev"] == {"recipes": [], "composed": [], "composed_ratio": {}, "purchases": {}, "stock_entry": {}, "stock_in": {}} # targeted evidence rides along (empty here: sqlite has no RDB$RELATION_FIELDS)
|
||||
|
||||
|
||||
def test_existing_value_is_never_overwritten(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add(db, "N2")
|
||||
db.execute("UPDATE C000025 SET CLASSIFICACAO_FISCAL='19059090' WHERE CODIGO='N2'")
|
||||
db.commit()
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert row(db, "N2")[0] == "19059090" and row(db, "N2")[1] == "0300700"
|
||||
|
||||
|
||||
def test_dry_run_and_disabled_and_waiting_do_not_write_or_touch_firebird(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
for features in ({"features_configured": False}, {"features_configured": True, "enrichment_mode": "DISABLED"}):
|
||||
hub.features = features
|
||||
worker._features_at = -999
|
||||
assert worker.step() == [] and hub.resolve_calls == []
|
||||
hub.features = {"features_configured": True, "enrichment_mode": "DRY_RUN"}
|
||||
worker._features_at = -999
|
||||
now[0] += 5
|
||||
worker.step() # re-baseline after the inactive period
|
||||
add(db, "N3")
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert row(db, "N3") == (None, None, None, "keep") and hub.reports[0]["fields"][0]["outcome"] == "DRY_RUN"
|
||||
|
||||
|
||||
def test_authorization_revoked_right_before_write_fails_closed(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add(db, "N4")
|
||||
calls = {"n": 0}
|
||||
|
||||
def flip():
|
||||
calls["n"] += 1
|
||||
return {"features": dict(ENABLED if calls["n"] == 1 else {"features_configured": True, "enrichment_mode": "DISABLED"})}
|
||||
hub.authorizations = flip
|
||||
now[0] += 5
|
||||
worker._features_at = -999
|
||||
worker.step()
|
||||
assert row(db, "N4") == (None, None, None, "keep")
|
||||
assert all(f["outcome"] == "ABORTED" and f["reason"] == "AUTHORIZATION_CHANGED" for f in hub.reports[0]["fields"])
|
||||
|
||||
|
||||
def test_human_override_including_deleted_value_is_final(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add(db, "N5")
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
db.execute("UPDATE C000025 SET CEST='', CLASSIFICACAO_FISCAL='19059090' WHERE CODIGO='N5'") # a person deletes the CEST and changes the NCM
|
||||
db.commit()
|
||||
now[0] += 30
|
||||
worker.step()
|
||||
assert store.is_overridden("N5", "CEST") and store.is_overridden("N5", "CLASSIFICACAO_FISCAL") and not store.is_overridden("N5", "IPI")
|
||||
# the description changes a moment later: the worker re-resolves, but overridden fields must NOT be written again
|
||||
db.execute("UPDATE C000025 SET PRODUTO='REFRIGERANTE COLA LATA 350ML ZERO' WHERE CODIGO='N5'")
|
||||
db.commit()
|
||||
now[0] += 30
|
||||
worker.step()
|
||||
assert row(db, "N5")[:2] == ("19059090", "")
|
||||
|
||||
|
||||
def test_write_field_allowlist_and_shape():
|
||||
db = sqlite3.connect(":memory:")
|
||||
db.execute("CREATE TABLE C000025(CODIGO TEXT, COD_ORIGEM_MERCADORIA TEXT, CLASSIFICACAO_FISCAL TEXT)")
|
||||
with pytest.raises(ValueError):
|
||||
E.write_field(Conn(db), "1", "COD_ORIGEM_MERCADORIA", "1")
|
||||
with pytest.raises(ValueError):
|
||||
E.write_field(Conn(db), "1", "CCLASSTRIB", "000001")
|
||||
db.execute("INSERT INTO C000025(CODIGO) VALUES('1')")
|
||||
db.commit()
|
||||
assert E.write_field(Conn(db), "1", "CLASSIFICACAO_FISCAL", "123")["reason"] == "VALUE_SHAPE_INVALID"
|
||||
|
||||
|
||||
def test_hub_offline_keeps_product_pending_and_retries(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add(db, "N6")
|
||||
|
||||
def boom(payload):
|
||||
raise ConnectionError("down")
|
||||
good = hub.resolve_product
|
||||
hub.resolve_product = boom
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert store.get("N6")["state"] == "RETRY" and row(db, "N6")[0] is None
|
||||
hub.resolve_product = good
|
||||
now[0] += 40
|
||||
assert worker.step() == ["N6"] and row(db, "N6")[0] == "22021000"
|
||||
|
||||
|
||||
def test_gtin_check_parity_with_the_hub_and_codbarra_is_used():
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from central.app.enrichment.gtin import normalize_gtin
|
||||
for value in ("7891000100103", "7891000100104", "", "0", "12345678", "40170725", "036000291452", "11111111", "78910001001030", "abc", "2000000000008"):
|
||||
assert E.gtin_check_ok(value) == (normalize_gtin(value)["status"] in ("VALID_GTIN", "RESTRICTED_GTIN")), value
|
||||
assert E.pick_gtin({"GTIN": "", "CODBARRA": "7891000100103"}) == "7891000100103"
|
||||
assert E.pick_gtin({"GTIN": "7891000100103", "CODBARRA": "999"}) == "7891000100103"
|
||||
assert E.pick_gtin({"GTIN": "", "CODBARRA": "123"}) == "123" and E.pick_gtin({}) == ""
|
||||
|
||||
|
||||
def test_products_created_while_inactive_are_not_new_when_reactivated(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step() # baseline
|
||||
hub.features = {"features_configured": True, "enrichment_mode": "DISABLED"}
|
||||
worker._features_at = -999
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
add(db, "W1") # created while the function is off
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
hub.features = dict(ENABLED)
|
||||
worker._features_at = -999
|
||||
now[0] += 5
|
||||
assert worker.step() == [] and hub.resolve_calls == [] # re-baselined: W1 is history now
|
||||
add(db, "W2")
|
||||
now[0] += 5
|
||||
assert worker.step() == ["W2"]
|
||||
|
||||
|
||||
# ---- TECHNICAL_PLACEHOLDER (the SGV refuses a product without NCM: operators type 00000000) ---------------------------------------------------
|
||||
def add_with_ncm(db, code, ncm, name="COCA-COLA ORIGINAL LATA 350 ML"):
|
||||
db.execute("INSERT INTO C000025(CODIGO,PRODUTO,GTIN,CLASSIFICACAO_FISCAL,OTHER) VALUES(?,?,?,?,?)", (code, name, "7891000100103", ncm, "keep"))
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_placeholder_parity_with_the_hub():
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from central.app.enrichment import placeholders as hub
|
||||
assert {k: set(v) for k, v in E.PLACEHOLDERS.items()} == {k: set(v) for k, v in hub.PRODUCT_ENRICHMENT_PLACEHOLDERS.items()}
|
||||
for value in ("00000000", "0000.00.00", " 00000000 ", "22021000", "99999999", "0", "SEM NCM", "", None):
|
||||
assert E.is_placeholder("CLASSIFICACAO_FISCAL", value) == hub.is_placeholder("CLASSIFICACAO_FISCAL", value), value
|
||||
assert not E.is_placeholder("CEST", "0000000") and not E.is_placeholder("IPI", 0)
|
||||
|
||||
|
||||
def test_placeholder_ncm_is_replaced_dry_run_first_then_enabled_and_second_run_is_zero(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
hub.features = {"features_configured": True, "enrichment_mode": "DRY_RUN"}
|
||||
worker._features_at = -999
|
||||
worker.step() # baseline
|
||||
add_with_ncm(db, "P1", "00000000")
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert row(db, "P1")[0] == "00000000" and hub.reports[-1]["fields"][0]["outcome"] == "DRY_RUN" # DRY_RUN: detected, zero writes
|
||||
assert hub.resolve_calls[-1]["current"]["CLASSIFICACAO_FISCAL"] == "00000000"
|
||||
hub.features = dict(ENABLED)
|
||||
worker._features_at = -999
|
||||
worker._auth_cache = (None, 0.0)
|
||||
add_with_ncm(db, "P2", "00000000")
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert row(db, "P2") == ("22021000", "0300700", 0.0, "keep")
|
||||
first = [f for f in hub.reports[-1]["fields"] if f["field"] == "CLASSIFICACAO_FISCAL"][0]
|
||||
assert first["outcome"] == "APPLIED" and first["old"] == "00000000"
|
||||
before = db.total_changes
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
now[0] += 60
|
||||
worker.step()
|
||||
assert db.total_changes == before # second run: 0 writes
|
||||
|
||||
|
||||
def test_real_or_unlisted_ncm_is_never_replaced(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
for code, ncm in (("R1", "22021000"), ("R2", "99999999"), ("R3", "0"), ("R4", "SEM NCM"), ("R5", "19059090")):
|
||||
add_with_ncm(db, code, ncm)
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
for code, ncm in (("R1", "22021000"), ("R2", "99999999"), ("R3", "0"), ("R4", "SEM NCM"), ("R5", "19059090")):
|
||||
assert row(db, code)[0] == ncm, code # EXISTING_REAL_VALUE_PRESERVED, even if the KB says otherwise
|
||||
|
||||
|
||||
def test_write_field_guards_the_placeholder_swap():
|
||||
db = sqlite3.connect(":memory:")
|
||||
db.execute("CREATE TABLE C000025(CODIGO TEXT, CLASSIFICACAO_FISCAL TEXT)")
|
||||
db.executemany("INSERT INTO C000025 VALUES(?,?)", [("a", "00000000"), ("b", "22021000"), ("c", None), ("d", "0000.00.00")])
|
||||
db.commit()
|
||||
assert E.write_field(Conn(db), "a", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
|
||||
assert E.write_field(Conn(db), "b", "CLASSIFICACAO_FISCAL", "19059090")["reason"] == "VALUE_ALREADY_SET_NEVER_OVERWRITTEN"
|
||||
assert E.write_field(Conn(db), "c", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
|
||||
assert E.write_field(Conn(db), "d", "CLASSIFICACAO_FISCAL", "19059090")["outcome"] == "APPLIED"
|
||||
assert [r[0] for r in db.execute("SELECT CLASSIFICACAO_FISCAL FROM C000025 ORDER BY CODIGO")] == ["19059090", "22021000", "19059090", "19059090"]
|
||||
|
||||
|
||||
# ---- restart / race / multiple products / stale guard / reevaluation / provenance --------------------------------------------------------------
|
||||
def make_worker(db, hub, store, now):
|
||||
return E.EnrichmentWorker(hub=hub, store=store, read_conn=lambda: Conn(db), write_conn=lambda: Conn(db), create_reprocess=lambda codes: None, clock=lambda: now[0], sleep=lambda s: None)
|
||||
|
||||
|
||||
def test_product_created_while_agent_is_down_is_found_after_restart(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add(db, "D1") # created while the Agent is stopped / restarting
|
||||
worker2 = make_worker(db, hub, store, now) # new process, same persisted state
|
||||
now[0] += 5
|
||||
assert worker2.step() == ["D1"] and row(db, "D1")[0] == "22021000"
|
||||
|
||||
|
||||
def test_many_products_between_polls_and_a_deleted_one(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
for i in range(5):
|
||||
add(db, f"M{i}")
|
||||
db.execute("DELETE FROM C000025 WHERE CODIGO='M2'")
|
||||
db.commit()
|
||||
add(db, "M9")
|
||||
now[0] += 5
|
||||
done = worker.step()
|
||||
assert sorted(done) == ["M0", "M1", "M3", "M4", "M9"] and all(row(db, c)[0] == "22021000" for c in done)
|
||||
|
||||
|
||||
def test_reevaluation_flagged_by_the_hub_retries_an_unresolved_product(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
hub.decisions = {} # first attempt: nothing known yet
|
||||
add(db, "U1")
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert row(db, "U1")[0] is None and store.get("U1")["state"] == "DONE"
|
||||
hub.decisions = dict(DECISIONS) # new evidence (KB decision) + 'Reavaliar'/'Reprocessar'
|
||||
hub.flagged = ["U1"]
|
||||
now[0] += 30
|
||||
assert "U1" in worker.step() and row(db, "U1")[0] == "22021000"
|
||||
now[0] += 30
|
||||
before = db.total_changes
|
||||
worker.step()
|
||||
assert db.total_changes == before
|
||||
|
||||
|
||||
def test_stale_write_is_refused_when_a_person_changed_the_value_after_the_decision(env):
|
||||
db = sqlite3.connect(":memory:")
|
||||
db.execute("CREATE TABLE C000025(CODIGO TEXT, CLASSIFICACAO_FISCAL TEXT)")
|
||||
db.execute("INSERT INTO C000025 VALUES('a','19059090')")
|
||||
db.execute("INSERT INTO C000025 VALUES('b','00000000')")
|
||||
db.commit()
|
||||
assert E.write_field(Conn(db), "a", "CLASSIFICACAO_FISCAL", "22021000", observed="00000000", guard_observed=True)["reason"] == "STALE_CONFLICT"
|
||||
assert E.write_field(Conn(db), "b", "CLASSIFICACAO_FISCAL", "22021000", observed="00000000", guard_observed=True)["outcome"] == "APPLIED"
|
||||
|
||||
|
||||
def test_placeholder_after_a_human_override_is_not_filled_again(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add_with_ncm(db, "HP1", "00000000") # born with the placeholder: eligible
|
||||
now[0] += 5
|
||||
worker.step()
|
||||
assert row(db, "HP1")[0] == "22021000"
|
||||
db.execute("UPDATE C000025 SET CLASSIFICACAO_FISCAL='00000000' WHERE CODIGO='HP1'") # a person puts the SAME value back: a human decision now
|
||||
db.commit()
|
||||
now[0] += 30
|
||||
worker.step()
|
||||
assert store.is_overridden("HP1", "CLASSIFICACAO_FISCAL")
|
||||
hub.flagged = ["HP1"]
|
||||
now[0] += 30
|
||||
worker.step()
|
||||
assert row(db, "HP1")[0] == "00000000" # provenance matters: same value, different meaning
|
||||
|
||||
|
||||
def test_automatic_mode_needs_no_sync_button_or_restart(env):
|
||||
db, hub, store, worker, now, _ = env
|
||||
worker.step()
|
||||
add(db, "A1")
|
||||
now[0] += 3
|
||||
assert worker.step() == ["A1"] and row(db, "A1")[0] == "22021000" # save -> next poll -> written
|
||||
@@ -0,0 +1,85 @@
|
||||
"""read_product_evidence: targeted, single-product evidence for the ad-hoc /v1/products/resolve call - the same recipes/composed/purchases/
|
||||
stock facts read_evidence already gathers for a full sync job, scoped to ONE product_code, never a full-table scan. Measured against a real
|
||||
production-equivalent database (D:\\PROJETOS\\GTISISTEMAS\\Banco\\GTI.FDB, read-only): a full read_evidence() scan took ~40ms for the whole
|
||||
catalog, and a targeted per-product lookup correctly drove field_contract.infer_role to RESALE/OWN_PRODUCTION for 5 of 8 real products with
|
||||
zero operator questions - this test locks in the behavior with a portable (sqlite) fixture."""
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parents[1] / "src"))
|
||||
from gti_fiscal_agent.firebird_reader import read_product_evidence
|
||||
|
||||
|
||||
def _conn():
|
||||
db = sqlite3.connect(":memory:")
|
||||
db.execute("CREATE TABLE RECEITA_INGREDIENTES(CODRECEITA TEXT)")
|
||||
db.execute("CREATE TABLE C000032(CODPRODUTO TEXT, TIPO TEXT)")
|
||||
db.execute("CREATE TABLE C000088(CODPRODUTO TEXT, ICMS_RETIDO TEXT, ICMS_VALORRETIDO REAL, SUB_VALOR REAL)")
|
||||
db.execute("CREATE TABLE C000100(CODPRODUTO TEXT, ULTIMA_ENTRADA TEXT, NOTA_FISCAL TEXT, ENTRADAS REAL)")
|
||||
return db
|
||||
|
||||
|
||||
def test_no_firebird_system_table_degrades_to_no_evidence_never_raises():
|
||||
"""A non-Firebird connection (this project's own sqlite test fakes) has no RDB$RELATION_FIELDS: the capability probe must degrade
|
||||
gracefully to "no evidence", exactly like a missing table would, never raise."""
|
||||
db = sqlite3.connect(":memory:")
|
||||
ev = read_product_evidence(db, "000001")
|
||||
assert ev == {"recipes": set(), "composed": set(), "composed_ratio": {}, "purchases": {}, "stock_entry": {}, "stock_in": {}}
|
||||
|
||||
|
||||
def test_targeted_lookup_is_scoped_to_the_one_product_code():
|
||||
db = _conn()
|
||||
db.executemany("INSERT INTO RECEITA_INGREDIENTES VALUES (?)", [("000009",), ("000010",)])
|
||||
db.execute("INSERT INTO C000032 VALUES ('000011', 'COMPOSICAO')")
|
||||
db.execute("INSERT INTO C000088 VALUES ('000012', 'S', 0, 0)")
|
||||
db.execute("INSERT INTO C000100 VALUES ('000013', '2026-01-01', '000001', 5)")
|
||||
db.commit()
|
||||
|
||||
def has_relation_fields():
|
||||
cur = db.cursor()
|
||||
cur.execute("SELECT TRIM(RDB$RELATION_NAME), TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME IN ('C000032','C000088','C000100','RECEITA_INGREDIENTES')")
|
||||
return cur.fetchall()
|
||||
|
||||
# sqlite has no RDB$RELATION_FIELDS either; monkeypatch the cursor to answer the capability probe like a real Firebird database would
|
||||
class ProbedConn:
|
||||
def __init__(self, real):
|
||||
self.real = real
|
||||
|
||||
def cursor(self):
|
||||
return ProbedCursor(self.real.cursor())
|
||||
|
||||
class ProbedCursor:
|
||||
def __init__(self, real):
|
||||
self.real = real
|
||||
self._probe_result = None
|
||||
|
||||
def execute(self, sql, params=()):
|
||||
if "RDB$RELATION_FIELDS" in sql:
|
||||
self._probe_result = [
|
||||
("RECEITA_INGREDIENTES", "CODRECEITA"), ("C000032", "CODPRODUTO"), ("C000032", "TIPO"),
|
||||
("C000088", "CODPRODUTO"), ("C000088", "ICMS_RETIDO"), ("C000088", "ICMS_VALORRETIDO"), ("C000088", "SUB_VALOR"),
|
||||
("C000100", "CODPRODUTO"), ("C000100", "ULTIMA_ENTRADA"), ("C000100", "NOTA_FISCAL"), ("C000100", "ENTRADAS"),
|
||||
]
|
||||
return
|
||||
self._probe_result = None
|
||||
sqlite_sql = sql.replace("SELECT FIRST 1 ", "SELECT ") + (" LIMIT 1" if "FIRST 1" in sql else "")
|
||||
self.real.execute(sqlite_sql, params)
|
||||
|
||||
def fetchall(self):
|
||||
return self._probe_result if self._probe_result is not None else self.real.fetchall()
|
||||
|
||||
def fetchone(self):
|
||||
return self.real.fetchone()
|
||||
|
||||
conn = ProbedConn(db)
|
||||
ev = read_product_evidence(conn, "000009")
|
||||
assert ev["recipes"] == {"000009"} and ev["composed"] == set() and ev["purchases"] == {} and ev["stock_entry"] == {} and ev["stock_in"] == {}
|
||||
ev = read_product_evidence(conn, "000011")
|
||||
assert ev["composed"] == {"000011"} and ev["recipes"] == set()
|
||||
ev = read_product_evidence(conn, "000012")
|
||||
assert ev["purchases"] == {"000012": [{"icms_retido": "S", "valor_retido": 0, "sub_valor": 0}]}
|
||||
ev = read_product_evidence(conn, "000013")
|
||||
assert ev["stock_entry"] == {"000013": True} and ev["stock_in"] == {"000013": True}
|
||||
ev = read_product_evidence(conn, "999999")
|
||||
assert ev == {"recipes": set(), "composed": set(), "composed_ratio": {}, "purchases": {}, "stock_entry": {}, "stock_in": {}}
|
||||
@@ -0,0 +1,52 @@
|
||||
from gti_fiscal_agent.ipi_auto_apply import ipi_write_decision
|
||||
|
||||
|
||||
def _decision(ipi_status="IPI_RESOLVED", status="EXACT_NCM", ipi=None):
|
||||
return {"status": status, "ipi_status": ipi_status, "classification": {"ipi": ipi}}
|
||||
|
||||
|
||||
def test_numeric_null_current_is_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "NUMERIC", "rate_numeric": 3.25}), None)
|
||||
assert d["eligible"] is True
|
||||
assert d["proposed_value"] == 3.25
|
||||
assert d["reason_code"] == "CHANGE_REQUIRED"
|
||||
|
||||
|
||||
def test_numeric_same_value_is_not_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "NUMERIC", "rate_numeric": 3.25}), 3.25)
|
||||
assert d["eligible"] is False
|
||||
assert d["reason_code"] == "SAME_VALUE"
|
||||
|
||||
|
||||
def test_numeric_different_value_is_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "NUMERIC", "rate_numeric": 3.25}), 0)
|
||||
assert d["eligible"] is True
|
||||
assert d["proposed_value"] == 3.25
|
||||
|
||||
|
||||
def test_non_numeric_never_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "NON_NUMERIC", "rate_numeric": None}), None)
|
||||
assert d["eligible"] is False
|
||||
assert d["reason_code"] == "RATE_KIND_NON_NUMERIC"
|
||||
|
||||
|
||||
def test_source_blank_never_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "BLANK", "rate_numeric": None}), None)
|
||||
assert d["eligible"] is False
|
||||
assert d["reason_code"] == "RATE_KIND_BLANK"
|
||||
|
||||
|
||||
def test_ambiguous_ipi_status_never_eligible():
|
||||
d = ipi_write_decision(_decision(ipi_status="IPI_CANDIDATES", ipi=None), None)
|
||||
assert d["eligible"] is False
|
||||
|
||||
|
||||
def test_no_match_never_eligible():
|
||||
d = ipi_write_decision(_decision(ipi_status="NO_TIPI_MAPPING_FOUND", ipi=None), None)
|
||||
assert d["eligible"] is False
|
||||
|
||||
|
||||
def test_legacy_ncm_never_eligible():
|
||||
d = ipi_write_decision(_decision(status="LEGACY_NCM", ipi_status="LEGACY_NCM"), None)
|
||||
assert d["eligible"] is False
|
||||
assert d["reason_code"] == "LEGACY_NCM"
|
||||
@@ -0,0 +1,132 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gti_fiscal_agent.company_context import resolve
|
||||
from gti_fiscal_agent.preflight import major_minor, run_preflight
|
||||
|
||||
ROOT = Path(__file__).parents[2]
|
||||
spec = importlib.util.spec_from_file_location("run_agent_service", ROOT / "agent" / "scripts" / "run_agent_service.py")
|
||||
service = importlib.util.module_from_spec(spec); sys.modules[spec.name] = service; spec.loader.exec_module(service)
|
||||
|
||||
CN = "11111111000111"
|
||||
|
||||
|
||||
def env(tmp_path, **over):
|
||||
base = {"GTI_HUB_URL": "http://hub.test", "AGENT_TOKEN": "SECRETTOKENVALUE9", "FIREBIRD_DATABASE_PATH": "x.fdb", "FB_USER": "u", "FB_PASSWORD": "SECRETPASSVALUE9", "FISCAL_MODE": "DRY_RUN", "FIREBIRD_WRITES": "0",
|
||||
"AGENT_STATE_DB": str(tmp_path / "state.sqlite"), "AGENT_LOG_PATH": str(tmp_path / "logs" / "a.log"), "AGENT_VERSION": "1.0.0-rc1"}
|
||||
base.update(over)
|
||||
return base
|
||||
|
||||
|
||||
def http_ok(url, token=None, timeout=10):
|
||||
if url.endswith("/health"):
|
||||
return 200, {"status": "ok"}
|
||||
if url.endswith("/version"):
|
||||
return 200, {"version": "1.0.0-rc1"}
|
||||
return (200 if token == "SECRETTOKENVALUE9" else 401), {}
|
||||
|
||||
|
||||
def fake_probe(company_over=None, caps_missing=()):
|
||||
def probe(env):
|
||||
cols = {"CLASSIFICACAO_FISCAL": (37, 20), "CFOP_ENTRADA": (14, 4), "CST_ENTRADA": (14, 4), "CFOP": (37, 4), "CST": (37, 10), "ALIQUOTA": (16, 8), "CEST": (37, 7), "SITUACAO_TRIBUTARIA": (37, 2),
|
||||
"COD_ENTRADA_COFINS": (37, 3), "COD_SAIDA_COFINS": (37, 3), "ALIQ_COFINS": (37, 8), "COD_ENTRADA_PIS": (37, 3), "COD_SAIDA_PIS": (37, 3), "ALIQ_PIS": (37, 8), "CSOSN": (37, 60), "IPI": (27, 8),
|
||||
"CST_IPI": (14, 3), "IAT": (37, 1), "INCIDENCIA_PISCOFINS": (37, 30), "PISCOFINS": (37, 1), "IPPT": (37, 1), "EX_TIPI": (37, 3), "COD_ORIGEM_MERCADORIA": (8, 4)}
|
||||
for c in caps_missing:
|
||||
cols.pop(c)
|
||||
ev = {"c000004": [{"code": "000001", "cnpj": CN, "uf": "BA", "crt": "1", "optante": "S"}], "emissor": [{"cnpj": CN, "uf": "BA", "crt": "1"}], "nfce_keys": [{"cuf": "29", "cnpj": CN, "n": 5}], "nf_by_filial": {}}
|
||||
ev.update(company_over or {})
|
||||
return {"server_version": "WI-V6.3.9 Firebird 2.5", "ods": "11.2", "tables": {"C000004": {}, "C000025": cols}, "products": 3712, "company": resolve(ev, {})}
|
||||
return probe
|
||||
|
||||
|
||||
def test_preflight_passes_for_a_clean_dry_run_installation(tmp_path):
|
||||
r = run_preflight(env(tmp_path), http_get=http_ok, probe=fake_probe())
|
||||
assert r["PREFLIGHT"] == "PASS" and r["blockers"] == [] and r["effective_firebird_writes"] is False
|
||||
names = {c["name"] for c in r["checks"]}
|
||||
assert {"HUB_REACHABLE", "AUTH", "HUB_VERSION_COMPATIBLE", "FIREBIRD_REACHABLE", "C000004", "C000025", "CAPABILITIES", "COMPANY_CONTEXT", "UF", "CRT", "SCHEMA_TYPE_GUARDS", "SQLITE", "LOG_DIRECTORY", "FISCAL_MODE", "FIREBIRD_WRITES"} <= names
|
||||
assert "SECRETTOKENVALUE9" not in str(r) and "SECRETPASSVALUE9" not in str(r)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("over,blocker", [
|
||||
({"FISCAL_MODE": "DRY_RUN", "FIREBIRD_WRITES": "1"}, "FIREBIRD_WRITES"),
|
||||
({"FISCAL_MODE": ""}, "FISCAL_MODE"),
|
||||
({"AGENT_TOKEN": ""}, "CONFIG"),
|
||||
])
|
||||
def test_preflight_blocks_bad_config(tmp_path, over, blocker):
|
||||
r = run_preflight(env(tmp_path, **over), http_get=http_ok, probe=fake_probe())
|
||||
assert r["PREFLIGHT"] == "FAIL" and any(b.startswith(blocker) for b in r["blockers"])
|
||||
|
||||
|
||||
def test_preflight_fails_on_hub_auth_version_and_missing_capability(tmp_path):
|
||||
def bad_auth(url, token=None, timeout=10):
|
||||
return http_ok(url, "wrong" if token else None)
|
||||
assert any(b.startswith("AUTH") for b in run_preflight(env(tmp_path), http_get=bad_auth, probe=fake_probe())["blockers"])
|
||||
|
||||
def old_hub(url, token=None, timeout=10):
|
||||
return (200, {"version": "0.9.0"}) if url.endswith("/version") else http_ok(url, token)
|
||||
assert any(b.startswith("HUB_VERSION_COMPATIBLE") for b in run_preflight(env(tmp_path), http_get=old_hub, probe=fake_probe())["blockers"])
|
||||
assert any(b.startswith("CAPABILITIES") for b in run_preflight(env(tmp_path), http_get=http_ok, probe=fake_probe(caps_missing=("CEST",)))["blockers"])
|
||||
|
||||
|
||||
def test_preflight_fails_when_company_context_is_ambiguous_or_crt_conflicts(tmp_path):
|
||||
two = {"c000004": [{"code": "1", "cnpj": CN, "uf": "BA", "crt": "1"}, {"code": "2", "cnpj": "22222222000122", "uf": "SP", "crt": "1"}], "emissor": []}
|
||||
r = run_preflight(env(tmp_path), http_get=http_ok, probe=fake_probe(two))
|
||||
assert r["PREFLIGHT"] == "FAIL" and any(b.startswith("COMPANY_CONTEXT") for b in r["blockers"])
|
||||
conflict = {"emissor": [{"cnpj": CN, "uf": "BA", "crt": "3"}]}
|
||||
assert any("CRT_SOURCE_CONFLICT" in b for b in run_preflight(env(tmp_path), http_get=http_ok, probe=fake_probe(conflict))["blockers"])
|
||||
|
||||
|
||||
def test_hub_unreachable_is_a_clear_blocker(tmp_path):
|
||||
def down(url, token=None, timeout=10):
|
||||
raise ConnectionError("nope")
|
||||
r = run_preflight(env(tmp_path), http_get=down, probe=fake_probe())
|
||||
assert r["PREFLIGHT"] == "FAIL" and any(b.startswith("HUB_REACHABLE") for b in r["blockers"])
|
||||
|
||||
|
||||
def test_major_minor():
|
||||
assert major_minor("1.0.0-rc1") == "1.0" and major_minor("1.0.3") == "1.0" and major_minor("0.9.0") != major_minor("1.0.0-rc1")
|
||||
|
||||
|
||||
# ---------------- zero-write guarantee ----------------
|
||||
def test_dry_run_makes_writes_technically_impossible_whatever_firebird_writes_says():
|
||||
for extra in ({"FIREBIRD_WRITES": "1", "FISCAL_MODE": "DRY_RUN"}, {"FIREBIRD_WRITES": "0", "FISCAL_MODE": "AUTO_APPLY"}, {"FIREBIRD_WRITES": "1", "FISCAL_MODE": "dry_run"}):
|
||||
cfg = service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb", **extra})
|
||||
assert cfg.firebird_writes is False
|
||||
assert service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb", "FIREBIRD_WRITES": "1", "FISCAL_MODE": "AUTO_APPLY"}).firebird_writes is True
|
||||
assert service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb", "FIREBIRD_WRITES": "1"}).firebird_writes is True # legacy: unset FISCAL_MODE follows the flag
|
||||
assert service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb"}).fiscal_mode == "DRY_RUN"
|
||||
|
||||
|
||||
def test_write_connection_cannot_be_opened_when_writes_are_disabled(monkeypatch):
|
||||
opened = []
|
||||
monkeypatch.setitem(sys.modules, "fdb", type("F", (), {"connect": staticmethod(lambda **kw: opened.append(kw))}))
|
||||
with pytest.raises(RuntimeError):
|
||||
service.open_write_connection({"FB_USER": "u", "FB_PASSWORD": "SECRETPASSVALUE9"}, "x.fdb", firebird_writes=False)
|
||||
assert opened == []
|
||||
|
||||
|
||||
def test_dry_run_job_never_opens_a_write_connection_nor_calls_the_executor(monkeypatch):
|
||||
class Boom:
|
||||
def __getattr__(self, name):
|
||||
if name in ("execute_guarded_update", "execute_guarded_restore"):
|
||||
raise AssertionError("executor write called in DRY_RUN")
|
||||
return getattr(__import__("scripts.firebird_write_executor", fromlist=["x"]) if False else object(), name, None)
|
||||
cfg = service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb", "FIREBIRD_WRITES": "1", "FISCAL_MODE": "DRY_RUN"})
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: [{"CODIGO": "000001", "PRODUTO": "X", "CLASSIFICACAO_FISCAL": "22030000", "IPI": None}])
|
||||
monkeypatch.setattr(service, "HubClient", lambda url, token: type("H", (), {"classify_batch": lambda self, p: [{"status": "EXACT_NCM", "ipi_status": "IPI_RESOLVED", "classification": {"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.9}},
|
||||
"dataset_versions": {}}]})())
|
||||
monkeypatch.setattr(service, "build_job_coverage", lambda *a, **k: {"TOTAL_PRODUCTS": 1})
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda *a, **k: (_ for _ in ()).throw(AssertionError("write connection opened in DRY_RUN")))
|
||||
|
||||
class State:
|
||||
def log_ipi_write(self, *a): pass
|
||||
|
||||
class Ex:
|
||||
WriteRefused = Exception
|
||||
def execute_guarded_update(self, *a, **k): raise AssertionError("executor write called in DRY_RUN")
|
||||
|
||||
result, error = service.execute_job({"kind": "REPROCESS_ALL", "agent_id": 1, "payload": {}, "job_id": 1}, cfg, Ex(), State(), {})
|
||||
assert error is None and result["writes_attempted"] == 0 and result["firebird_writes_enabled"] is False
|
||||
@@ -0,0 +1,444 @@
|
||||
import re
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
|
||||
from central.app import rtc_reference_data as ref
|
||||
from central.app.cclasstrib_resolver import local_catalog_hash
|
||||
from gti_fiscal_agent import rtc_auto_apply as apply_mod
|
||||
from gti_fiscal_agent import unified_fiscal as uf
|
||||
|
||||
pytestmark = pytest.mark.real_unified_fiscal
|
||||
|
||||
TODAY = date(2026, 9, 24)
|
||||
NORMAL = {"crt": "3"}
|
||||
CLASS_COLS = ("CCLASSTRIB", "CSTIBSCBS", "INICIO_VIGENCIA", "FIM_VIGENCIA", "TIPO_ALIQUOTA", "PRED_IBS", "PRED_CBS", "IND_NFE", "IND_NFCE")
|
||||
SCHEMA = ([("C000025", c) for c in ("CODIGO", "CCLASSTRIB", "CLASSIFICACAO_FISCAL", "PISCOFINS", "INCIDENCIA_PISCOFINS")] + [("RTC_CCLASSTRIB", c) for c in CLASS_COLS] + [("RTC_CST", "CSTIBSCBS"), ("PARAMETROS", "USAR_REFORMA_TRIBUTARIA")])
|
||||
|
||||
|
||||
class FakeFirebird:
|
||||
"""A tiny SGV: PARAMETROS, the RTC catalog (from the official table) and C000025. Every statement is recorded; only SELECT and the single allowed UPDATE are understood."""
|
||||
|
||||
def __init__(self, products, param=(1,), catalog_tweak=None, skip=()):
|
||||
self.products = {c: dict(v) for c, v in products.items()} # code -> {"CCLASSTRIB":..., "CLASSIFICACAO_FISCAL":...}
|
||||
self.param = list(param)
|
||||
self.catalog = {r["code"]: {"cst": r["cst"], "start": r["start"], "end": r["end"], "tipo": r["tipo_aliquota"], "pred_ibs": r["p_red_ibs"], "pred_cbs": r["p_red_cbs"], "ind_nfe": r["ind_nfe"], "ind_nfce": r["ind_nfce"]}
|
||||
for r in ref.OFFICIAL_CCLASSTRIB if r["code"] not in skip and not r["end"]}
|
||||
for code, change in (catalog_tweak or {}).items():
|
||||
self.catalog[code] = {**self.catalog[code], **change}
|
||||
self.cst_table = {r["cst"] for r in ref.OFFICIAL_CCLASSTRIB} # RTC_CST is its own table: a class may point to a CST that is not in it
|
||||
self.crt = ("3",) # C000004.CRT rows
|
||||
self.statements, self.commits, self.rollbacks = [], 0, 0
|
||||
self._pending = {}
|
||||
|
||||
def cursor(self):
|
||||
return FakeCursor(self)
|
||||
|
||||
def commit(self):
|
||||
for code, value in self._pending.items():
|
||||
self.products[code]["CCLASSTRIB"] = value
|
||||
self._pending = {}
|
||||
self.commits += 1
|
||||
|
||||
def rollback(self):
|
||||
self._pending = {}
|
||||
self.rollbacks += 1
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
def __init__(self, db):
|
||||
self.db, self._rows, self.rowcount = db, [], -1
|
||||
|
||||
def execute(self, sql, params=()):
|
||||
db = self.db
|
||||
db.statements.append((sql, tuple(params)))
|
||||
text = " ".join(sql.split())
|
||||
up = text.upper()
|
||||
if up.startswith("SELECT TRIM(RF.RDB$RELATION_NAME)"):
|
||||
self._rows = list(SCHEMA)
|
||||
elif up.startswith("SELECT FIRST 20 USAR_REFORMA_TRIBUTARIA"):
|
||||
self._rows = [(v,) for v in db.param]
|
||||
elif "RDB$RELATION_CONSTRAINTS" in up:
|
||||
self._rows = [(1,)]
|
||||
elif up.startswith("SELECT TRIM(CCLASSTRIB)") and "FROM RTC_CCLASSTRIB" in up:
|
||||
cols = [c.strip() for c in text[len("SELECT "):up.index(" FROM ")].split(",")]
|
||||
rows = []
|
||||
for code, r in db.catalog.items():
|
||||
mapping = {"CSTIBSCBS": r["cst"], "INICIO_VIGENCIA": date.fromisoformat(r["start"]) if r["start"] else None, "FIM_VIGENCIA": None, "TIPO_ALIQUOTA": r["tipo"], "PRED_IBS": r["pred_ibs"],
|
||||
"PRED_CBS": r["pred_cbs"], "IND_NFE": "1" if r["ind_nfe"] else "0", "IND_NFCE": "1" if r["ind_nfce"] else "0"}
|
||||
rows.append(tuple(code if c.startswith("TRIM(CCLASSTRIB") else mapping[re.sub(r"TRIM\((\w+)\)", r"\1", c)] for c in cols))
|
||||
self._rows = rows
|
||||
elif up.startswith("SELECT TRIM(CSTIBSCBS) FROM RTC_CST"):
|
||||
self._rows = [(c,) for c in sorted(db.cst_table)]
|
||||
elif up.startswith("SELECT CODIGO, CCLASSTRIB, CLASSIFICACAO_FISCAL"):
|
||||
self._rows = [(c, v["CCLASSTRIB"], v["CLASSIFICACAO_FISCAL"], "N", "NENHUMA") for c, v in sorted(db.products.items())]
|
||||
elif up.startswith("SELECT DISTINCT TRIM(CRT) FROM C000004"):
|
||||
self._rows = [(c,) for c in db.crt]
|
||||
elif up.startswith("SELECT CCLASSTRIB, CLASSIFICACAO_FISCAL, PRODUTO FROM C000025"):
|
||||
code = params[0]
|
||||
self._rows = [(db.products[code]["CCLASSTRIB"], db.products[code]["CLASSIFICACAO_FISCAL"], db.products[code].get("PRODUTO", ""))] if code in db.products else []
|
||||
if code in db.products and db.products[code].get("DUP"):
|
||||
self._rows = self._rows * 2
|
||||
elif up.startswith("SELECT CCLASSTRIB FROM C000025"):
|
||||
code = params[0]
|
||||
self._rows = [(db.products[code]["CCLASSTRIB"],)]
|
||||
elif up.startswith("UPDATE C000025 SET CCLASSTRIB = NULL WHERE CODIGO = ? AND TRIM(CCLASSTRIB) = ?"):
|
||||
code, written = params
|
||||
product = db.products.get(code)
|
||||
hit = product is not None and (product["CCLASSTRIB"] or "").strip() == written
|
||||
self.rowcount = 1 if hit else 0
|
||||
if hit:
|
||||
db._pending[code] = None
|
||||
elif up.startswith("UPDATE C000025 SET CCLASSTRIB = ? WHERE CODIGO = ?"):
|
||||
value, code = params
|
||||
product = db.products.get(code)
|
||||
hit = product is not None and not (product["CCLASSTRIB"] or "").strip()
|
||||
self.rowcount = (2 if product and product.get("DUP") else 1) if hit else 0
|
||||
if hit and self.rowcount == 1:
|
||||
db._pending[code] = value
|
||||
else:
|
||||
raise AssertionError(f"unexpected statement: {sql}")
|
||||
|
||||
def fetchall(self):
|
||||
return self._rows
|
||||
|
||||
def fetchone(self):
|
||||
return self._rows[0] if self._rows else None
|
||||
|
||||
|
||||
def product(ncm, value=None, **extra):
|
||||
return {"CCLASSTRIB": value, "CLASSIFICACAO_FISCAL": ncm, **extra}
|
||||
|
||||
|
||||
def assess(db, ncm_ok=None, company=NORMAL):
|
||||
return uf.assess_from_connection(db, company, today=TODAY, ncm_valid=ncm_ok if ncm_ok is not None else {c: True for c in db.products})
|
||||
|
||||
|
||||
def candidates_of(db, limit=500):
|
||||
out = assess(db)
|
||||
products = [{"CODIGO": c, "CCLASSTRIB": v["CCLASSTRIB"], "CLASSIFICACAO_FISCAL": v["CLASSIFICACAO_FISCAL"]} for c, v in db.products.items()]
|
||||
cands, total = apply_mod.select_candidates(out["findings"], products, limit=limit)
|
||||
return out, cands, total
|
||||
|
||||
|
||||
def run(db, cands, out, **kw):
|
||||
return apply_mod.apply_candidates(db, cands, company=NORMAL, decision={"local_catalog_hash": out["context"]["local_catalog_hash"]}, now="2026-09-24T20:00:00+00:00", job_id=7, today=TODAY, **kw)
|
||||
|
||||
|
||||
PRODUCTS = {"000001": product("10063021"), "000002": product("04090000"), "000003": product("15171000"), "000004": product("30049099"), "000005": product("39235000", "000001")}
|
||||
|
||||
|
||||
def test_gate_is_driven_by_the_central_mode_only_and_defaults_to_closed():
|
||||
assert apply_mod.write_gate(mode="ENABLED")["allowed"] is True
|
||||
assert apply_mod.write_gate(mode="APPROVAL_REQUIRED", approved=True)["allowed"] is True
|
||||
for kwargs in ({"mode": "DISABLED"}, {"mode": "DRY_RUN"}, {"mode": "APPROVAL_REQUIRED", "approved": False}, {"mode": "ENABLED", "capability": False}, {"mode": "bogus"}, {"mode": ""}):
|
||||
g = apply_mod.write_gate(**kwargs)
|
||||
assert g["allowed"] is False and g["missing"]
|
||||
assert apply_mod.ALLOWED_RTC_WRITE_FIELDS == ("CCLASSTRIB",) and apply_mod.CAPABILITY == "RTC_CCLASSTRIB_WRITE_V1"
|
||||
|
||||
|
||||
def test_only_proven_products_become_candidates():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out, cands, total = candidates_of(db)
|
||||
assert sorted((c["product_code"], c["proposed"], c["rule_id"]) for c in cands) == [("000001", "200003", "RTC-R3"), ("000002", "200034", "RTC-R3"), ("000003", "200003", "RTC-R3")]
|
||||
assert total == 3 and out["context"]["rtc_resolution"]["ready_to_apply"] == 3 and out["context"]["rtc_resolution"]["ambiguous"] == 1 # medicine chapter is ambiguous
|
||||
assert out["context"]["rtc_resolution"]["valid_existing"] == 1 # 000005 already has the right class
|
||||
assert "000004" not in {c["product_code"] for c in cands}
|
||||
|
||||
|
||||
def test_apply_writes_only_cclasstrib_with_audit_and_second_run_writes_nothing():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out, cands, _ = candidates_of(db)
|
||||
result = run(db, cands, out)
|
||||
assert result["write_successes"] == 3 and result["write_attempts"] == 3 and result["aborted"] == []
|
||||
assert {c: db.products[c]["CCLASSTRIB"] for c in ("000001", "000002", "000003")} == {"000001": "200003", "000002": "200034", "000003": "200003"}
|
||||
updates = [s for s, _ in db.statements if s.upper().startswith("UPDATE")]
|
||||
assert len(updates) == 3 and all(re.fullmatch(r"UPDATE C000025 SET CCLASSTRIB = \? WHERE CODIGO = \? AND \(CCLASSTRIB IS NULL OR TRIM\(CCLASSTRIB\) = ''\)", u) for u in updates)
|
||||
assert not [s for s, _ in db.statements if re.match(r"(INSERT|DELETE|ALTER|DROP)", s.strip(), re.I)]
|
||||
audit = result["applied"][0]
|
||||
assert {"product_code", "field", "old_value", "new_value", "rule_id", "evidence", "official_source_version", "local_catalog_hash", "timestamp", "job_id", "outcome"} <= set(audit)
|
||||
assert audit["old_value"] is None and audit["field"] == "CCLASSTRIB" and audit["official_source_version"]["cclasstrib_sha256"] == ref.OFFICIAL_CATALOG["sha256"] and audit["job_id"] == 7
|
||||
out2, cands2, total2 = candidates_of(db) # idempotent: nothing left to write
|
||||
assert cands2 == [] and total2 == 0 and out2["context"]["rtc_resolution"]["valid_existing"] == 4
|
||||
before = len(db.statements)
|
||||
assert run(db, cands2, out2)["write_attempts"] == 0 and len(db.statements) == before
|
||||
|
||||
|
||||
def test_rtc_switched_off_between_decision_and_write_aborts_every_product():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out, cands, _ = candidates_of(db)
|
||||
db.param = [0]
|
||||
result = run(db, cands, out)
|
||||
assert result["write_successes"] == 0 and result["aborted_reasons"] == {"RTC_NO_LONGER_ENABLED": 3} and all(p["CCLASSTRIB"] in (None, "000001") for p in db.products.values())
|
||||
|
||||
|
||||
def test_conflicting_parameter_rows_abort():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out, cands, _ = candidates_of(db)
|
||||
db.param = [1, 0]
|
||||
assert run(db, cands, out)["aborted_reasons"] == {"RTC_NO_LONGER_ENABLED": 3}
|
||||
|
||||
|
||||
def test_catalog_changed_or_invalid_between_decision_and_write_aborts():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out, cands, _ = candidates_of(db)
|
||||
db.catalog["200003"]["pred_ibs"] = "60"
|
||||
assert run(db, cands, out)["aborted_reasons"] == {"LOCAL_CATALOG_CHANGED_SINCE_DECISION": 3}
|
||||
db2 = FakeFirebird(PRODUCTS)
|
||||
out2, cands2, _ = candidates_of(db2)
|
||||
db2.catalog["999999"] = {**db2.catalog["000001"], "cst": "555"} # class pointing to a CST that does not exist -> invalid catalog
|
||||
invalid = apply_mod.apply_candidates(db2, cands2, company=NORMAL, decision={}, now="2026-09-24T20:00:00+00:00", job_id=7, today=TODAY) # no hash pinned: integrity check alone must stop it
|
||||
assert invalid["aborted_reasons"] == {"RTC_CATALOG_INVALID": 3}
|
||||
|
||||
|
||||
def test_product_changed_between_decision_and_write_is_left_alone():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out, cands, _ = candidates_of(db)
|
||||
db.products["000001"]["CLASSIFICACAO_FISCAL"] = "30049099" # NCM edited in the ERP meanwhile
|
||||
db.products["000002"]["CCLASSTRIB"] = "200034" # operator filled it meanwhile (same value)
|
||||
db.products["000003"]["CCLASSTRIB"] = "200034" # ... or with a DIFFERENT value: never overwritten
|
||||
result = run(db, cands, out)
|
||||
assert result["aborted_reasons"] == {"PRODUCT_CHANGED_NCM": 1, "NO_WRITE_REQUIRED": 1, "VALUE_ALREADY_SET_NEVER_OVERWRITTEN": 1} and result["write_successes"] == 0
|
||||
assert db.products["000003"]["CCLASSTRIB"] == "200034" and not [s for s, _ in db.statements if s.upper().startswith("UPDATE")]
|
||||
|
||||
|
||||
def test_duplicate_code_rowcount_protection_and_isolation_of_failures():
|
||||
products = {**PRODUCTS, "000002": product("04090000", DUP=True)}
|
||||
db = FakeFirebird(products)
|
||||
out, cands, _ = candidates_of(db)
|
||||
result = run(db, cands, out)
|
||||
assert result["aborted_reasons"] == {"PRODUCT_ROW_NOT_UNIQUE": 1} and result["write_successes"] == 2 # the ambiguous row is skipped, the rest still applied
|
||||
assert db.products["000002"]["CCLASSTRIB"] is None
|
||||
|
||||
|
||||
def test_existing_different_valid_class_is_a_conflict_blocker_never_a_write():
|
||||
db = FakeFirebird({"000001": product("10063021", "000001")}) # rice already classified as fully taxed: differs from the proven Anexo I class
|
||||
out = assess(db)
|
||||
types = {b["type"] for b in out["findings"]["000001"]["blockers"]}
|
||||
assert types == {"CCLASSTRIB_CONFLICT"} and out["context"]["rtc_resolution"]["conflicts"] == 1
|
||||
_, cands, _ = candidates_of(db)
|
||||
assert cands == []
|
||||
|
||||
|
||||
def test_official_mismatch_of_an_existing_class_is_reported():
|
||||
db = FakeFirebird({"000001": product("39235000", "000001")}, catalog_tweak={"000001": {"ind_nfe": False, "ind_nfce": False}})
|
||||
types = {b["type"] for b in assess(db)["findings"]["000001"]["blockers"]}
|
||||
assert types == {"CCLASSTRIB_OFFICIAL_MISMATCH"}
|
||||
|
||||
|
||||
def test_batch_limit_bounds_the_blast_radius():
|
||||
db = FakeFirebird({f"{i:06d}": product("04090000") for i in range(1, 8)})
|
||||
out, cands, total = candidates_of(db, limit=3)
|
||||
assert len(cands) == 3 and total == 7
|
||||
|
||||
|
||||
def test_simples_never_produces_candidates_even_with_rtc_on():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out = uf.assess_from_connection(db, {"crt": "1"}, today=TODAY, ncm_valid={c: True for c in PRODUCTS})
|
||||
products = [{"CODIGO": c, "CCLASSTRIB": v["CCLASSTRIB"], "CLASSIFICACAO_FISCAL": v["CLASSIFICACAO_FISCAL"]} for c, v in db.products.items()]
|
||||
candidates, total = apply_mod.select_candidates(out["findings"], products)
|
||||
|
||||
assert out["context"]["rtc_resolution"]["ready_to_apply"] == 0
|
||||
assert out["context"]["rtc_resolution"]["not_applicable"] == len(PRODUCTS)
|
||||
assert candidates == [] and total == 0
|
||||
assert not [b for f in out["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
|
||||
assert all(sql.lstrip().upper().startswith("SELECT") for sql, _ in db.statements)
|
||||
|
||||
|
||||
def test_rtc_off_has_no_resolution_and_no_candidates():
|
||||
db = FakeFirebird(PRODUCTS, param=(0,))
|
||||
out, cands, total = candidates_of(db)
|
||||
assert cands == [] and total == 0 and out["context"]["rtc_resolution"] is None
|
||||
|
||||
|
||||
def test_catalog_hash_is_recorded_for_audit():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out = assess(db)
|
||||
schema, catalog, _ = uf.load_rtc_context(db.cursor())
|
||||
assert out["context"]["local_catalog_hash"] == local_catalog_hash(catalog["rows"]) and re.fullmatch(r"[0-9a-f]{64}", out["context"]["local_catalog_hash"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- job integration (gates)
|
||||
import pytest # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _hub_echoes_the_job_authorization(monkeypatch):
|
||||
"""By default the Hub still says what the job said; individual tests change that to prove the pre-write re-check."""
|
||||
from test_run_agent_service import service
|
||||
monkeypatch.setattr(service, "_fetch_authorizations", lambda config, job: dict(job.get("authorizations") or {}))
|
||||
|
||||
|
||||
def _service():
|
||||
from test_run_agent_service import _config, service
|
||||
return service, _config
|
||||
|
||||
|
||||
def _fiscal_for(db):
|
||||
return assess(db)
|
||||
|
||||
|
||||
def _products_of(db):
|
||||
return [{"CODIGO": c, "CCLASSTRIB": v["CCLASSTRIB"], "CLASSIFICACAO_FISCAL": v["CLASSIFICACAO_FISCAL"], "PRODUTO": v.get("PRODUTO", "")} for c, v in db.products.items()]
|
||||
|
||||
|
||||
def _no_write(service, monkeypatch):
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda *a, **k: (_ for _ in ()).throw(AssertionError("no write connection may be opened")))
|
||||
|
||||
|
||||
def test_job_without_a_central_mode_or_with_disabled_or_dry_run_never_writes(monkeypatch):
|
||||
service, _config = _service()
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
fiscal = _fiscal_for(db)
|
||||
_no_write(service, monkeypatch)
|
||||
for auth in ({}, {"rtc_write_mode": "DISABLED"}, {"rtc_write_mode": "DRY_RUN"}, {"rtc_write_mode": "bogus"}, {"rtc_write_mode": "APPROVAL_REQUIRED", "rtc_approved_codes": []}, {"rtc_auto_apply": True}):
|
||||
summary = service._rtc_auto_apply({}, _config(), {"job_id": 1, "authorizations": auth}, _products_of(db), fiscal, NORMAL)
|
||||
assert summary["gate"]["allowed"] is False and summary["ready_total"] == 3 and summary["write_attempts"] == 0 and summary["applied"] == []
|
||||
off = service._rtc_auto_apply({}, _config(), {"job_id": 1, "authorizations": {"rtc_write_mode": "DISABLED"}}, _products_of(db), fiscal, NORMAL)
|
||||
dry = service._rtc_auto_apply({}, _config(), {"job_id": 1, "authorizations": {"rtc_write_mode": "DRY_RUN"}}, _products_of(db), fiscal, NORMAL)
|
||||
assert off["preview"] == [] and off["mode"] == "REPORT_ONLY"
|
||||
assert [c["product_code"] for c in dry["preview"]] == ["000001", "000002", "000003"] and all(c["rule_id"] == "RTC-R3" for c in dry["preview"]) and dry["capability"] == "RTC_CCLASSTRIB_WRITE_V1"
|
||||
|
||||
|
||||
def test_r4_suggestions_are_never_in_the_preview_and_never_written(monkeypatch):
|
||||
service, _config = _service()
|
||||
db = FakeFirebird({"000001": product("39235000"), "000002": product("39235000")})
|
||||
fiscal = _fiscal_for(db)
|
||||
_no_write(service, monkeypatch)
|
||||
summary = service._rtc_auto_apply({}, _config(), {"job_id": 1, "authorizations": {"rtc_write_mode": "ENABLED"}}, _products_of(db), fiscal, NORMAL)
|
||||
assert summary["ready_total"] == 0 and summary["preview"] == [] and summary["write_attempts"] == 0
|
||||
|
||||
|
||||
def test_enabled_mode_applies_without_any_local_switch(monkeypatch):
|
||||
service, _config = _service()
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
fiscal = _fiscal_for(db)
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda env, path: _Closeable(db))
|
||||
summary = service._rtc_auto_apply({}, _config(), {"job_id": 9, "authorizations": {"rtc_write_mode": "ENABLED"}}, _products_of(db), fiscal, NORMAL) # default config: no firebird_writes, no env switch
|
||||
assert summary["mode"] == "ENABLED" and summary["write_successes"] == 3 and summary["allowed_fields"] == ["CCLASSTRIB"] and all(a["job_id"] == 9 and a["authorization_origin"] == "HUB_MODE:ENABLED" for a in summary["applied"])
|
||||
assert [db.products[c]["CCLASSTRIB"] for c in ("000001", "000002", "000003")] == ["200003", "200034", "200003"]
|
||||
a = summary["applied"][0]
|
||||
assert a["rule_version"] and a["company_crt"] == "3" and a["action"] == "APPLY" and a["rtc_status"] == "ENABLED" and a["local_catalog_state"] == "MATCH_SAFE" and a["old_value"] is None
|
||||
|
||||
|
||||
def test_approval_required_writes_only_the_approved_products(monkeypatch):
|
||||
service, _config = _service()
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
fiscal = _fiscal_for(db)
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda env, path: _Closeable(db))
|
||||
summary = service._rtc_auto_apply({}, _config(), {"job_id": 9, "authorizations": {"rtc_write_mode": "APPROVAL_REQUIRED", "rtc_approved_codes": ["000002"]}}, _products_of(db), fiscal, NORMAL)
|
||||
assert summary["write_successes"] == 1 and [db.products[c]["CCLASSTRIB"] for c in ("000001", "000002", "000003")] == [None, "200034", None]
|
||||
|
||||
|
||||
def test_job_never_writes_when_the_erp_says_rtc_is_off_even_in_enabled_mode(monkeypatch):
|
||||
service, _config = _service()
|
||||
db = FakeFirebird(PRODUCTS, param=(0,))
|
||||
fiscal = _fiscal_for(db)
|
||||
_no_write(service, monkeypatch)
|
||||
products = [{"CODIGO": c, "CCLASSTRIB": None, "CLASSIFICACAO_FISCAL": "04090000"} for c in db.products]
|
||||
summary = service._rtc_auto_apply({}, _config(), {"job_id": 1, "authorizations": {"rtc_write_mode": "ENABLED"}}, products, fiscal, NORMAL)
|
||||
assert summary["write_attempts"] == 0 and summary["ready_total"] == 0
|
||||
|
||||
|
||||
def test_company_regime_or_fingerprint_change_aborts_before_the_update():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out, cands, _ = candidates_of(db)
|
||||
db.crt = ("1",) # the company was switched to Simples between decision and write
|
||||
res = run(db, cands, out)
|
||||
assert res["write_successes"] == 0 and res["aborted_reasons"] == {"COMPANY_CONTEXT_CHANGED": 3}
|
||||
db.crt = ("3", "1") # ambiguous company context is also refused
|
||||
assert run(db, cands, out)["aborted_reasons"] == {"COMPANY_CONTEXT_CHANGED": 3}
|
||||
db.crt = ("3",)
|
||||
db.products["000001"]["PRODUTO"] = "RENAMED AFTER THE DECISION"
|
||||
res = run(db, cands, out)
|
||||
assert res["aborted_reasons"] == {"PRODUCT_CHANGED_SINCE_DECISION": 1} and res["write_successes"] == 2 and db.products["000001"]["CCLASSTRIB"] is None
|
||||
assert not any("UPDATE" in q.upper() and "C000004" in q.upper() for q, _ in db.statements)
|
||||
|
||||
|
||||
def test_rollback_restores_only_when_the_value_still_equals_the_written_one_and_reapply_works():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
out, cands, _ = candidates_of(db)
|
||||
first = run(db, cands, out)
|
||||
assert first["write_successes"] == 3
|
||||
items = [{"audit_id": i + 1, "product_code": a["product_code"], "written_value": a["new_value"], "rule_id": a["rule_id"]} for i, a in enumerate(first["applied"])]
|
||||
db.products["000002"]["CCLASSTRIB"] = "200035" # an operator changed it afterwards
|
||||
res = apply_mod.rollback_records(db, items, now="2026-09-24T21:00:00+00:00", job_id=8)
|
||||
by = {r["product_code"]: r for r in res["rollbacks"]}
|
||||
assert res["rolled_back"] == 2 and by["000002"]["outcome"] == "ABORTED" and by["000002"]["reason"] == "ROLLBACK_CONFLICT_VALUE_CHANGED" and db.products["000002"]["CCLASSTRIB"] == "200035"
|
||||
assert by["000001"]["outcome"] == "ROLLED_BACK" and by["000001"]["rollback_of"] == 1 and by["000001"]["old_value"] == "200003" and by["000001"]["new_value"] is None and db.products["000001"]["CCLASSTRIB"] is None
|
||||
again = apply_mod.rollback_records(db, items, now="2026-09-24T21:05:00+00:00", job_id=9)["rollbacks"]
|
||||
assert [r["reason"] for r in again if r["product_code"] == "000001"] == ["ALREADY_EMPTY"]
|
||||
out2, cands2, _ = candidates_of(db) # reapply: the emptied products are candidates again, the changed one is not
|
||||
assert sorted(c["product_code"] for c in cands2) == ["000001", "000003"] and run(db, cands2, out2)["write_successes"] == 2
|
||||
|
||||
|
||||
def test_rollback_uses_one_fixed_statement_and_rejects_malformed_requests():
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
bad = apply_mod.rollback_records(db, [{"audit_id": 1, "product_code": "000001", "written_value": ""}, {"audit_id": 2, "product_code": "000001", "written_value": "200003", "restore_value": "999999"}], now="t")
|
||||
assert [r["reason"] for r in bad["rollbacks"]] == ["ROLLBACK_REQUEST_INVALID"] * 2 and db.statements == []
|
||||
db.products["000001"]["CCLASSTRIB"] = "200003"
|
||||
apply_mod.rollback_records(db, [{"audit_id": 3, "product_code": "000001", "written_value": "200003"}], now="t")
|
||||
writes = [q for q, _ in db.statements if q.strip().upper().startswith(("UPDATE", "INSERT", "DELETE"))]
|
||||
assert writes == ["UPDATE C000025 SET CCLASSTRIB = NULL WHERE CODIGO = ? AND TRIM(CCLASSTRIB) = ?"]
|
||||
|
||||
|
||||
class _Closeable:
|
||||
def __init__(self, db):
|
||||
self._db = db
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self._db, name)
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
def test_firebird_unavailable_for_the_write_connection_fails_the_job_closed_with_zero_writes(monkeypatch):
|
||||
service, _config = _service()
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
fiscal = _fiscal_for(db)
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda *a, **k: (_ for _ in ()).throw(OSError("Firebird unavailable")))
|
||||
import pytest as _pytest
|
||||
with _pytest.raises(OSError):
|
||||
service._rtc_auto_apply({}, _config(), {"job_id": 1, "authorizations": {"rtc_write_mode": "ENABLED"}}, _products_of(db), fiscal, NORMAL)
|
||||
assert all(v["CCLASSTRIB"] is None for c, v in db.products.items() if c != "000005") and not [q for q, _ in db.statements if q.upper().startswith("UPDATE")]
|
||||
|
||||
|
||||
def test_connection_is_closed_even_when_a_product_fails_mid_batch(monkeypatch):
|
||||
service, _config = _service()
|
||||
db = FakeFirebird({**PRODUCTS, "000002": product("04090000", DUP=True)})
|
||||
fiscal = _fiscal_for(db)
|
||||
closed = []
|
||||
conn = _Closeable(db)
|
||||
conn.close = lambda: closed.append(True)
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda env, path: conn)
|
||||
summary = service._rtc_auto_apply({}, _config(), {"job_id": 1, "authorizations": {"rtc_write_mode": "ENABLED"}}, _products_of(db), fiscal, NORMAL)
|
||||
assert closed == [True] and summary["write_successes"] == 2 and summary["aborted_reasons"]
|
||||
|
||||
|
||||
def test_authorization_revoked_after_the_job_was_claimed_means_zero_writes(monkeypatch):
|
||||
service, _config = _service()
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
fiscal = _fiscal_for(db)
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda env, path: _Closeable(db))
|
||||
monkeypatch.setattr(service, "_fetch_authorizations", lambda config, job: {"rtc_write_mode": "DISABLED"}) # operator switched it off while the job was running
|
||||
summary = service._rtc_auto_apply({}, _config(), {"job_id": 5, "authorizations": {"rtc_write_mode": "ENABLED"}}, _products_of(db), fiscal, NORMAL)
|
||||
assert summary["write_successes"] == 0 and summary["aborted_reasons"] == {"AUTHORIZATION_CHANGED": 3}
|
||||
assert [db.products[c]["CCLASSTRIB"] for c in ("000001", "000002", "000003")] == [None, None, None]
|
||||
|
||||
|
||||
def test_unreachable_hub_is_not_authorization_and_approval_removal_is_honoured(monkeypatch):
|
||||
service, _config = _service()
|
||||
db = FakeFirebird(PRODUCTS)
|
||||
fiscal = _fiscal_for(db)
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda env, path: _Closeable(db))
|
||||
|
||||
def down(config, job):
|
||||
raise ConnectionError("hub down")
|
||||
|
||||
monkeypatch.setattr(service, "_fetch_authorizations", down)
|
||||
summary = service._rtc_auto_apply({}, _config(), {"job_id": 6, "authorizations": {"rtc_write_mode": "ENABLED"}}, _products_of(db), fiscal, NORMAL)
|
||||
assert summary["write_successes"] == 0 and summary["aborted_reasons"] == {"AUTHORIZATION_UNVERIFIABLE": 3}
|
||||
monkeypatch.setattr(service, "_fetch_authorizations", lambda config, job: {"rtc_write_mode": "APPROVAL_REQUIRED", "rtc_approved_codes": ["000001"]}) # 000002 was un-approved meanwhile
|
||||
summary = service._rtc_auto_apply({}, _config(), {"job_id": 7, "authorizations": {"rtc_write_mode": "APPROVAL_REQUIRED", "rtc_approved_codes": ["000001", "000002"]}}, _products_of(db), fiscal, NORMAL)
|
||||
assert summary["write_successes"] == 1 and db.products["000001"]["CCLASSTRIB"] == "200003" and db.products["000002"]["CCLASSTRIB"] is None
|
||||
@@ -0,0 +1,515 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPT = Path(__file__).parents[1] / "scripts" / "run_agent_service.py"
|
||||
AUTHORIZED = r"D:\PROJETOS\GTISISTEMAS\Projeto\TESTESOFICIAIS.FDB"
|
||||
|
||||
|
||||
def load_module():
|
||||
spec = importlib.util.spec_from_file_location("run_agent_service", SCRIPT)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
service = load_module()
|
||||
|
||||
|
||||
class _FakeState:
|
||||
def __init__(self):
|
||||
self.logged = []
|
||||
self.identity = None
|
||||
|
||||
def stored_installation_id(self):
|
||||
return self.identity
|
||||
|
||||
def installation_id(self, candidate=None):
|
||||
if self.identity and candidate and self.identity != candidate:
|
||||
raise ValueError("configured installation_id conflicts with persisted identity")
|
||||
if self.identity is None:
|
||||
if not candidate:
|
||||
raise ValueError("installation_id is required")
|
||||
self.identity = candidate
|
||||
return self.identity
|
||||
|
||||
def log_ipi_write(self, *args):
|
||||
self.logged.append(args)
|
||||
|
||||
|
||||
class _FakeExecutor:
|
||||
"""Mirrors the real firebird_write_executor's public surface used by execute_job."""
|
||||
class WriteRefused(Exception):
|
||||
pass
|
||||
|
||||
def __init__(self, path_guard_ok=True, updates=None, fingerprint_guard_ok=True):
|
||||
self.path_guard_ok = path_guard_ok
|
||||
self.updates = updates or {}
|
||||
self.fingerprint_guard_ok = fingerprint_guard_ok
|
||||
|
||||
def assert_authorized_write_path(self, path, authorized_path=None):
|
||||
if not self.path_guard_ok:
|
||||
raise self.WriteRefused(f"path is not the authorized write target: {path}")
|
||||
|
||||
def assert_schema_fingerprint_authorized(self, observed, expected):
|
||||
if not self.fingerprint_guard_ok or observed != expected:
|
||||
raise self.WriteRefused(f"schema fingerprint mismatch: expected {expected!r}, observed {observed!r}")
|
||||
|
||||
def execute_guarded_update(self, connection, *, database_path, proposal, now, authorized_path=None):
|
||||
code = proposal["product_code"]
|
||||
if code in self.updates:
|
||||
raise self.WriteRefused(self.updates[code])
|
||||
record = type("Record", (), {
|
||||
"before_value": proposal["current_value"],
|
||||
"after_value": proposal["proposed_value"],
|
||||
"rowcount": 1,
|
||||
})()
|
||||
return record
|
||||
|
||||
|
||||
def _config(**overrides):
|
||||
base = dict(
|
||||
hub_url="http://localhost:8000", agent_id="a1", agent_token="tok", company_id="c1", company_uf="BA",
|
||||
database_path=AUTHORIZED, firebird_writes=False, poll_interval_seconds=0.01,
|
||||
agent_version="0.1.0", schema_fingerprint="fp", state_db=":memory:", agent_jobs_enabled=True,
|
||||
authorized_write_database_path=None, expected_schema_fingerprint="fp",
|
||||
)
|
||||
base.update(overrides)
|
||||
return type("Config", (), base)()
|
||||
|
||||
|
||||
def test_config_defaults_from_env():
|
||||
env = {"FIREBIRD_DATABASE_PATH": AUTHORIZED}
|
||||
config = service.Config(env)
|
||||
assert config.firebird_writes is False
|
||||
assert config.hub_url == "http://localhost:8000"
|
||||
assert config.poll_interval_seconds == 5.0
|
||||
assert config.agent_jobs_enabled is True
|
||||
|
||||
|
||||
def test_local_agent_jobs_toggle_is_ignored_control_is_central():
|
||||
# Zero-touch: a stale AGENT_JOBS_ENABLED=0 left in an old agent.env must not silence the Agent;
|
||||
# only the Hub (heartbeat processing_enabled) can pause it.
|
||||
config = service.Config({"FIREBIRD_DATABASE_PATH": AUTHORIZED, "AGENT_JOBS_ENABLED": "0"})
|
||||
|
||||
assert config.agent_jobs_enabled is True
|
||||
|
||||
|
||||
def test_config_reads_gti_hub_url_and_writes_flag():
|
||||
env = {"FIREBIRD_DATABASE_PATH": AUTHORIZED, "GTI_HUB_URL": "http://hub.internal", "FIREBIRD_WRITES": "1"}
|
||||
config = service.Config(env)
|
||||
assert config.hub_url == "http://hub.internal"
|
||||
assert config.firebird_writes is True
|
||||
|
||||
|
||||
def test_identity_recovery_persists_existing_hub_installation_id(tmp_path):
|
||||
config = _config(agent_id="2")
|
||||
state = service.AgentState(tmp_path / "agent.db")
|
||||
installation_id = "00000000-0000-4000-8000-000000000222"
|
||||
|
||||
class FakeIdentityClient:
|
||||
calls = 0
|
||||
|
||||
def identity(self):
|
||||
self.calls += 1
|
||||
return {"agent_id": 2, "installation_id": installation_id}
|
||||
|
||||
client = FakeIdentityClient()
|
||||
recovered = service.resolve_installation_id(config, {}, state, client)
|
||||
|
||||
assert recovered == installation_id
|
||||
assert state.installation_id() == installation_id
|
||||
assert client.calls == 1
|
||||
|
||||
restarted = service.resolve_installation_id(config, {}, state, client)
|
||||
assert restarted == installation_id
|
||||
assert client.calls == 1
|
||||
|
||||
|
||||
def test_identity_recovery_fails_closed_for_another_agent_or_missing_id(tmp_path):
|
||||
config = _config(agent_id="2")
|
||||
|
||||
class FakeIdentityClient:
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
def identity(self):
|
||||
return self.payload
|
||||
|
||||
state = service.AgentState(tmp_path / "agent.db")
|
||||
with pytest.raises(RuntimeError, match="identity does not match configured Agent"):
|
||||
service.resolve_installation_id(
|
||||
config,
|
||||
{},
|
||||
state,
|
||||
FakeIdentityClient({"agent_id": 3, "installation_id": "00000000-0000-4000-8000-000000000333"}),
|
||||
)
|
||||
assert state.connection.execute("SELECT COUNT(*) FROM installation_identity").fetchone()[0] == 0
|
||||
|
||||
with pytest.raises(RuntimeError, match="identity response has no installation_id"):
|
||||
service.resolve_installation_id(config, {}, state, FakeIdentityClient({"agent_id": 2}))
|
||||
assert state.connection.execute("SELECT COUNT(*) FROM installation_identity").fetchone()[0] == 0
|
||||
|
||||
|
||||
def test_path_guard_fails_closed_before_any_write(monkeypatch):
|
||||
config = _config(firebird_writes=True, database_path=r"D:\PRODUCAO\ERP.FDB")
|
||||
executor = _FakeExecutor(path_guard_ok=False)
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
called = {"read_products": False}
|
||||
|
||||
def fake_read_products(env, codigos=None):
|
||||
called["read_products"] = True
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(service, "read_products", fake_read_products)
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert result is None
|
||||
assert "not the authorized write target" in error
|
||||
assert called["read_products"] is False # never even reads Firebird once the guard refuses
|
||||
|
||||
|
||||
def test_pilot_write_blocked_when_schema_fingerprint_does_not_match(monkeypatch):
|
||||
# A real client install where the live schema fingerprint drifted from what
|
||||
# this Agent installation was configured/registered to expect.
|
||||
config = _config(firebird_writes=True, schema_fingerprint="observed-live-fp", expected_schema_fingerprint="registered-fp")
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
called = {"read_products": False}
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: called.update(read_products=True) or [])
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert result is None
|
||||
assert "schema fingerprint mismatch" in error
|
||||
assert called["read_products"] is False
|
||||
|
||||
|
||||
def test_pilot_write_allowed_with_explicit_authorized_path_and_matching_fingerprint(monkeypatch):
|
||||
client_path = r"D:\ClienteX\ERP\PRODUCAO.FDB"
|
||||
config = _config(
|
||||
firebird_writes=True, database_path=client_path, authorized_write_database_path=client_path,
|
||||
schema_fingerprint="fp", expected_schema_fingerprint="fp",
|
||||
)
|
||||
products = [{"CODIGO": "000026", "PRODUTO": "x", "CLASSIFICACAO_FISCAL": "03055100", "IPI": None}]
|
||||
decisions = [{
|
||||
"status": "EXACT_NCM", "ipi_status": "IPI_RESOLVED",
|
||||
"classification": {"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.25}},
|
||||
"dataset_versions": {"tipi_ipi": "v1"},
|
||||
}]
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: products)
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, *a, **k): pass
|
||||
def classify_batch(self, items): return decisions
|
||||
monkeypatch.setattr(service, "HubClient", FakeClient)
|
||||
fake_connection = type("Conn", (), {"close": lambda self: None})()
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda env, path: fake_connection)
|
||||
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert error is None
|
||||
assert result["writes_succeeded"] == 1
|
||||
|
||||
|
||||
def test_execute_job_isolates_internal_errors_without_raising(monkeypatch):
|
||||
config = _config()
|
||||
|
||||
def boom(env, codigos=None):
|
||||
raise RuntimeError("firebird unavailable")
|
||||
|
||||
monkeypatch.setattr(service, "read_products", boom)
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_ALL", "agent_id": 1, "payload": {}}
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert result is None
|
||||
assert "firebird unavailable" in error
|
||||
|
||||
|
||||
def test_execute_job_success_summarizes_pipeline(monkeypatch):
|
||||
config = _config()
|
||||
products = [{"CODIGO": "000026", "PRODUTO": "x", "CLASSIFICACAO_FISCAL": "03055100", "IPI": None}]
|
||||
decisions = [{
|
||||
"status": "EXACT_NCM", "ipi_status": "IPI_RESOLVED",
|
||||
"classification": {"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.25}},
|
||||
"dataset_versions": {"tipi_ipi": "v1"},
|
||||
}]
|
||||
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: products)
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, *a, **k): pass
|
||||
def classify_batch(self, items): return decisions
|
||||
monkeypatch.setattr(service, "HubClient", FakeClient)
|
||||
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert error is None
|
||||
assert result["products_processed"] == 1
|
||||
assert result["writes_attempted"] == 0 # firebird_writes disabled by default in _config()
|
||||
|
||||
|
||||
def test_reprocess_products_resolves_real_company_context_not_none(monkeypatch):
|
||||
"""REPROCESS_PRODUCTS used to hardcode company=None for _unified_fiscal (it isn't in SYNC_JOB_KINDS). Company resolution is cheap and
|
||||
deterministic (company_context.resolve, same as every sync job already uses) - it must never silently fall back to None when the company
|
||||
genuinely resolves."""
|
||||
config = _config()
|
||||
products = [{"CODIGO": "000026", "PRODUTO": "x", "CLASSIFICACAO_FISCAL": "03055100", "IPI": None}]
|
||||
decisions = [{"status": "EXACT_NCM", "ipi_status": "IPI_RESOLVED", "classification": {"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.25}}, "dataset_versions": {"tipi_ipi": "v1"}}]
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: products)
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, *a, **k): pass
|
||||
def classify_batch(self, items): return decisions
|
||||
monkeypatch.setattr(service, "HubClient", FakeClient)
|
||||
monkeypatch.setattr(service._firebird_reader, "read_company_context", lambda env: {"uf": "BA", "crt": "1", "company_code": "000001"})
|
||||
|
||||
captured = {}
|
||||
real_unified = service._unified_fiscal
|
||||
|
||||
def spy_unified_fiscal(env, company, ncm_valid=None):
|
||||
captured["company"] = company
|
||||
return real_unified(env, company, ncm_valid)
|
||||
monkeypatch.setattr(service, "_unified_fiscal", spy_unified_fiscal)
|
||||
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert error is None
|
||||
assert captured["company"] == {"crt": "1"} # real CRT, never None, when the company genuinely resolves
|
||||
|
||||
|
||||
def test_reprocess_products_falls_back_to_none_only_when_company_unresolved(monkeypatch):
|
||||
"""No guessing: when the company genuinely cannot be resolved (e.g. Firebird unreachable), company stays None - same as before."""
|
||||
config = _config()
|
||||
products = [{"CODIGO": "000026", "PRODUTO": "x", "CLASSIFICACAO_FISCAL": "03055100", "IPI": None}]
|
||||
decisions = [{"status": "EXACT_NCM", "ipi_status": "IPI_RESOLVED", "classification": {"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.25}}, "dataset_versions": {"tipi_ipi": "v1"}}]
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: products)
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, *a, **k): pass
|
||||
def classify_batch(self, items): return decisions
|
||||
monkeypatch.setattr(service, "HubClient", FakeClient)
|
||||
|
||||
def boom(env):
|
||||
raise RuntimeError("firebird unavailable")
|
||||
monkeypatch.setattr(service._firebird_reader, "read_company_context", boom)
|
||||
|
||||
captured = {}
|
||||
real_unified = service._unified_fiscal
|
||||
|
||||
def spy_unified_fiscal(env, company, ncm_valid=None):
|
||||
captured["company"] = company
|
||||
return real_unified(env, company, ncm_valid)
|
||||
monkeypatch.setattr(service, "_unified_fiscal", spy_unified_fiscal)
|
||||
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert error is None
|
||||
assert captured["company"] is None
|
||||
|
||||
|
||||
class _FakeHubForLoop:
|
||||
"""Simulates: heartbeat flaky twice then OK, one job then nothing, and captures completions."""
|
||||
|
||||
def __init__(self):
|
||||
self.heartbeat_calls = 0
|
||||
self.jobs = [{"job_id": 1, "kind": "REPROCESS_ALL", "agent_id": 1, "payload": {}}]
|
||||
self.completed = []
|
||||
self.failed = []
|
||||
|
||||
def heartbeat(self, *a, **k):
|
||||
self.heartbeat_calls += 1
|
||||
if self.heartbeat_calls == 1:
|
||||
raise ConnectionError("hub down")
|
||||
return {}
|
||||
|
||||
def next_job(self):
|
||||
return self.jobs.pop(0) if self.jobs else None
|
||||
|
||||
def complete_job(self, job_id, result_summary):
|
||||
self.completed.append((job_id, result_summary))
|
||||
|
||||
def fail_job(self, job_id, error):
|
||||
self.failed.append((job_id, error))
|
||||
|
||||
def classify_batch(self, items):
|
||||
return []
|
||||
|
||||
|
||||
class _ExternalStopFlag:
|
||||
"""Mirrors the Windows Service wrapper's stop object: no signal registration,
|
||||
just a plain flag flippable from another thread."""
|
||||
def __init__(self):
|
||||
self.stop = False
|
||||
|
||||
|
||||
def test_run_sends_heartbeat_but_never_polls_when_paused_centrally(monkeypatch, tmp_path):
|
||||
stopper = _ExternalStopFlag()
|
||||
fake_hub = _FakeHubForLoop()
|
||||
installation_id = "00000000-0000-4000-8000-000000000222"
|
||||
calls = {"identity": 0, "next_job": 0, "heartbeat": 0, "heartbeat_installation_id": None}
|
||||
|
||||
def identity():
|
||||
calls["identity"] += 1
|
||||
return {"agent_id": 2, "installation_id": installation_id}
|
||||
|
||||
def heartbeat_metadata(payload):
|
||||
calls["heartbeat"] += 1
|
||||
calls["heartbeat_installation_id"] = payload["installation_id"]
|
||||
stopper.stop = True
|
||||
return {"agent_id": 2, "processing_enabled": False} # paused centrally by the Hub
|
||||
|
||||
def forbidden_next_job():
|
||||
calls["next_job"] += 1
|
||||
raise AssertionError("processing is paused centrally; next_job must not be called")
|
||||
|
||||
fake_hub.identity = identity
|
||||
fake_hub.heartbeat_metadata = heartbeat_metadata
|
||||
fake_hub.next_job = forbidden_next_job
|
||||
monkeypatch.setattr(service, "HubClient", lambda *a, **k: fake_hub)
|
||||
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
||||
monkeypatch.setattr(service.time, "sleep", lambda _seconds: None)
|
||||
|
||||
config = _config(
|
||||
agent_id="2",
|
||||
poll_interval_seconds=0,
|
||||
state_db=str(tmp_path / "agent.db"),
|
||||
)
|
||||
env = {
|
||||
"AGENT_ID": "2",
|
||||
"FIREBIRD_DATABASE_PATH": AUTHORIZED,
|
||||
"FIREBIRD_WRITES": "0",
|
||||
"FISCAL_MODE": "DRY_RUN",
|
||||
}
|
||||
|
||||
service.run(config, env, stopper=stopper)
|
||||
|
||||
assert calls == {
|
||||
"identity": 1,
|
||||
"next_job": 0,
|
||||
"heartbeat": 1,
|
||||
"heartbeat_installation_id": installation_id,
|
||||
}
|
||||
assert service.AgentState(config.state_db).stored_installation_id() == installation_id
|
||||
|
||||
|
||||
def test_company_context_for_heartbeat_resolves_once_and_caches(monkeypatch):
|
||||
"""CRT was never sent at heartbeat: the Agent already proves it (company_context.resolve, via preflight_company_context) for coverage/RTC
|
||||
jobs, but that value never reached the Hub. This locks in the fix: resolved once, cached, never re-queried once solid."""
|
||||
calls = {"n": 0}
|
||||
|
||||
def fake_preflight(env):
|
||||
calls["n"] += 1
|
||||
return {"status": "PASS", "company_uf": "BA", "company_crt": "1", "company_code": "000001"}
|
||||
|
||||
monkeypatch.setattr("gti_fiscal_agent.company_context.preflight_company_context", fake_preflight)
|
||||
result, reason = service._company_context_for_heartbeat({}, None)
|
||||
assert result == {"status": "PASS", "company_uf": "BA", "company_crt": "1", "company_code": "000001"} and reason is None and calls["n"] == 1
|
||||
result2, reason2 = service._company_context_for_heartbeat({}, result) # already cached: never re-queries Firebird
|
||||
assert result2 is result and reason2 is None and calls["n"] == 1
|
||||
|
||||
|
||||
def test_company_context_for_heartbeat_retries_while_unresolved_never_guesses(monkeypatch):
|
||||
def fake_preflight(env):
|
||||
return {"status": "FAIL", "classification": "UNRESOLVED", "reasons": ["UF_SOURCE_UNKNOWN"]}
|
||||
|
||||
monkeypatch.setattr("gti_fiscal_agent.company_context.preflight_company_context", fake_preflight)
|
||||
result, reason = service._company_context_for_heartbeat({}, None)
|
||||
assert result is None and "UNRESOLVED" in reason # never a guessed uf/crt - explicitly unresolved, retried next tick
|
||||
|
||||
|
||||
def test_run_heartbeat_carries_the_auto_discovered_company_crt(monkeypatch, tmp_path):
|
||||
"""End-to-end: once company context resolves, the CRT rides the heartbeat payload the Hub already knows how to store
|
||||
(Installation.company_crt) - closing the transport gap without any Hub change."""
|
||||
stopper = _ExternalStopFlag()
|
||||
fake_hub = _FakeHubForLoop()
|
||||
installation_id = "00000000-0000-4000-8000-000000000333"
|
||||
sent = {}
|
||||
|
||||
def identity():
|
||||
return {"agent_id": 2, "installation_id": installation_id}
|
||||
|
||||
def heartbeat_metadata(payload):
|
||||
sent.update(payload)
|
||||
stopper.stop = True
|
||||
return {"agent_id": 2, "processing_enabled": False}
|
||||
|
||||
fake_hub.identity = identity
|
||||
fake_hub.heartbeat_metadata = heartbeat_metadata
|
||||
fake_hub.next_job = lambda: (_ for _ in ()).throw(AssertionError("paused centrally; next_job must not be called"))
|
||||
monkeypatch.setattr(service, "HubClient", lambda *a, **k: fake_hub)
|
||||
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
||||
monkeypatch.setattr(service.time, "sleep", lambda _seconds: None)
|
||||
monkeypatch.setattr("gti_fiscal_agent.company_context.preflight_company_context",
|
||||
lambda env: {"status": "PASS", "company_uf": "BA", "company_crt": "1", "company_code": "000001", "company_contribuinte_ipi": "N"})
|
||||
|
||||
config = _config(agent_id="2", poll_interval_seconds=0, state_db=str(tmp_path / "agent.db"))
|
||||
env = {"AGENT_ID": "2", "FIREBIRD_DATABASE_PATH": AUTHORIZED, "FIREBIRD_WRITES": "0", "FISCAL_MODE": "DRY_RUN"}
|
||||
|
||||
service.run(config, env, stopper=stopper)
|
||||
|
||||
assert sent["company_uf"] == "BA" and sent["company_crt"] == "1" and sent["company_contribuinte_ipi"] == "N"
|
||||
|
||||
|
||||
def test_run_honors_an_externally_supplied_stopper(monkeypatch):
|
||||
# This is exactly the pattern the Windows Service wrapper uses: it never
|
||||
# constructs GracefulStop() (which would crash off the main thread), it
|
||||
# passes its own stop flag and flips it from the SvcStop handler.
|
||||
config = _config(poll_interval_seconds=0.001)
|
||||
fake_hub = _FakeHubForLoop()
|
||||
fake_hub.jobs = [] # no jobs; loop would run forever without an external stop
|
||||
monkeypatch.setattr(service, "HubClient", lambda *a, **k: fake_hub)
|
||||
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
||||
monkeypatch.setattr(service, "AgentState", lambda path: _FakeState())
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: [])
|
||||
|
||||
stopper = _ExternalStopFlag()
|
||||
calls = {"n": 0}
|
||||
real_sleep = service.time.sleep
|
||||
|
||||
def counting_sleep(seconds):
|
||||
calls["n"] += 1
|
||||
if calls["n"] >= 3:
|
||||
stopper.stop = True
|
||||
|
||||
monkeypatch.setattr(service.time, "sleep", counting_sleep)
|
||||
service.run(config, {"AGENT_ID": "2", "INSTALLATION_ID": "00000000-0000-4000-8000-000000000222"}, stopper=stopper) # no max_iterations — only the external flag can stop it
|
||||
assert stopper.stop is True
|
||||
assert calls["n"] >= 3
|
||||
|
||||
|
||||
def test_run_survives_hub_down_then_processes_one_job_and_stops(monkeypatch):
|
||||
config = _config(poll_interval_seconds=0.001)
|
||||
fake_hub = _FakeHubForLoop()
|
||||
monkeypatch.setattr(service, "HubClient", lambda *a, **k: fake_hub)
|
||||
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
||||
monkeypatch.setattr(service, "AgentState", lambda path: _FakeState())
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: [])
|
||||
monkeypatch.setattr(service, "build_job_coverage", lambda *a, **k: None)
|
||||
monkeypatch.setattr(service.time, "sleep", lambda seconds: None)
|
||||
|
||||
# first heartbeat call raises -> backoff branch; second call onward succeeds.
|
||||
service.run(config, {"AGENT_ID": "2", "INSTALLATION_ID": "00000000-0000-4000-8000-000000000222"}, max_iterations=4)
|
||||
|
||||
assert fake_hub.heartbeat_calls >= 2
|
||||
assert len(fake_hub.completed) == 1
|
||||
assert fake_hub.completed[0][0] == 1
|
||||
assert fake_hub.completed[0][1]["products_processed"] == 0
|
||||
@@ -0,0 +1,7 @@
|
||||
from gti_fiscal_agent.runner import to_hub_product
|
||||
|
||||
|
||||
def test_reference_schema_uses_classificacao_fiscal_as_canonical_ncm():
|
||||
product = {"CODIGO": "000276", "PRODUTO": "ABACAXI M/ JARRA", "NCM": "99999999", "CLASSIFICACAO_FISCAL": "08043000"}
|
||||
|
||||
assert to_hub_product(product)["current_ncm"] == "08043000"
|
||||
@@ -0,0 +1,56 @@
|
||||
import pytest
|
||||
|
||||
from gti_fiscal_agent.state import AgentState
|
||||
|
||||
|
||||
def test_outbox_keeps_failed_work_pending(tmp_path):
|
||||
state = AgentState(tmp_path / "agent.db")
|
||||
assert state.enqueue("000276", "source", "v1", "0.1.0")
|
||||
state.retry("000276")
|
||||
assert state.pending()[0]["status"] == "PENDING_API"
|
||||
state.result_received("000276", "source", "v1", "0.1.0")
|
||||
assert state.enqueue("000276", "source", "v1", "0.1.0") is False
|
||||
|
||||
|
||||
def test_ipi_write_log_records_per_product_audit(tmp_path):
|
||||
state = AgentState(tmp_path / "agent.db")
|
||||
state.log_ipi_write("000276", "fp1", "EXACT_NCM", "IPI_RESOLVED", "SUCCEEDED", None, 3.25, "rowcount=1", "CHANGE_REQUIRED", "2026-09-18T00:00:00Z")
|
||||
log = state.ipi_write_log_for("000276")
|
||||
assert len(log) == 1
|
||||
assert log[0]["write_status"] == "SUCCEEDED"
|
||||
assert log[0]["before_value"] is None
|
||||
assert log[0]["proposed_value"] == "3.25"
|
||||
|
||||
|
||||
def test_installation_identity_requires_an_existing_or_explicit_id(tmp_path):
|
||||
state = AgentState(tmp_path / "agent.db")
|
||||
|
||||
assert state.stored_installation_id() is None
|
||||
with pytest.raises(ValueError, match="installation_id is required"):
|
||||
state.installation_id()
|
||||
|
||||
assert state.connection.execute("SELECT COUNT(*) FROM installation_identity").fetchone()[0] == 0
|
||||
|
||||
|
||||
def test_installation_identity_persists_and_reuses_explicit_id(tmp_path):
|
||||
db_path = tmp_path / "agent.db"
|
||||
state = AgentState(db_path)
|
||||
existing_id = "00000000-0000-4000-8000-000000000222"
|
||||
|
||||
assert state.installation_id(existing_id) == existing_id
|
||||
assert state.installation_id() == existing_id
|
||||
assert state.stored_installation_id() == existing_id
|
||||
|
||||
restarted_state = AgentState(db_path)
|
||||
assert restarted_state.installation_id() == existing_id
|
||||
|
||||
|
||||
def test_installation_identity_rejects_conflicting_or_malformed_ids(tmp_path):
|
||||
state = AgentState(tmp_path / "agent.db")
|
||||
existing_id = "00000000-0000-4000-8000-000000000222"
|
||||
state.installation_id(existing_id)
|
||||
|
||||
with pytest.raises(ValueError, match="conflicts"):
|
||||
state.installation_id("00000000-0000-4000-8000-000000000223")
|
||||
with pytest.raises(ValueError, match="UUID"):
|
||||
state.installation_id("not-a-uuid")
|
||||
@@ -0,0 +1,174 @@
|
||||
import re
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
|
||||
from gti_fiscal_agent import unified_fiscal as uf
|
||||
|
||||
pytestmark = pytest.mark.real_unified_fiscal
|
||||
|
||||
SCHEMA_ROWS = ([("C000025", c) for c in ("CODIGO", "CCLASSTRIB", "COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS", "PISCOFINS", "INCIDENCIA_PISCOFINS")]
|
||||
+ [("RTC_CCLASSTRIB", c) for c in ("CCLASSTRIB", "CSTIBSCBS", "INICIO_VIGENCIA", "FIM_VIGENCIA")] + [("RTC_CST", "CSTIBSCBS")] + [("PARAMETROS", "USAR_REFORMA_TRIBUTARIA")])
|
||||
PRODUCT_COLS = ("CODIGO", "CCLASSTRIB", "COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS", "PISCOFINS", "INCIDENCIA_PISCOFINS")
|
||||
PRODUCTS = [("1", None, None, None, None, None, None, None, "N", "NENHUMA"), ("2", "200003", None, "04", None, None, "04", None, "S", "MONOFÁSICA"), ("3", None, "2", "2", "2", "2", "2", "2", "N", "NENHUMA")]
|
||||
CLASSES = [("000001", "000", date(2025, 5, 5), None), ("200003", "200", date(2025, 5, 5), None)]
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
"""READ ONLY fake: answers only SELECTs and records every statement so tests can prove nothing else ran."""
|
||||
def __init__(self, param=0, params=None, schema=SCHEMA_ROWS, products=PRODUCTS, classes=CLASSES, cst=(("000",), ("200",))):
|
||||
self.param, self.params, self.schema, self.products, self.classes, self.cst, self.statements, self._rows = param, params, schema, products, classes, cst, [], []
|
||||
|
||||
def execute(self, sql, *args):
|
||||
self.statements.append(sql)
|
||||
text = " ".join(sql.split()).upper()
|
||||
assert text.startswith("SELECT"), f"non-SELECT statement: {sql}"
|
||||
if "FROM RDB$RELATION_FIELDS" in text:
|
||||
self._rows = list(self.schema)
|
||||
elif "FROM PARAMETROS" in text:
|
||||
self._rows = [(v,) for v in (self.params if self.params is not None else [self.param])]
|
||||
elif "RDB$RELATION_CONSTRAINTS" in text:
|
||||
self._rows = [(1,)]
|
||||
elif "FROM RTC_CCLASSTRIB" in text:
|
||||
self._rows = list(self.classes)
|
||||
elif "FROM RTC_CST" in text:
|
||||
self._rows = list(self.cst)
|
||||
elif "FROM C000025" in text:
|
||||
self._rows = list(self.products)
|
||||
else:
|
||||
raise AssertionError(sql)
|
||||
|
||||
def fetchall(self):
|
||||
return self._rows
|
||||
|
||||
def fetchone(self):
|
||||
return self._rows[0] if self._rows else None
|
||||
|
||||
|
||||
class FakeConn:
|
||||
def __init__(self, **kw):
|
||||
self.cur = FakeCursor(**kw)
|
||||
|
||||
def cursor(self):
|
||||
return self.cur
|
||||
|
||||
|
||||
def kinds(found):
|
||||
return sorted({b["type"] for b in found["blockers"]})
|
||||
|
||||
|
||||
def test_rtc_off_in_the_erp_is_out_of_scope_and_never_blocks():
|
||||
conn = FakeConn(param=0)
|
||||
out = uf.assess_from_connection(conn, {"crt": "1"}, today=date(2026, 9, 24))
|
||||
assert out["context"]["rtc_enabled"] is False and out["context"]["rtc_supported"] is True and out["context"]["rtc_catalog_valid"] is True
|
||||
assert not [b for f in out["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
|
||||
assert kinds(out["findings"]["1"]) == [] and kinds(out["findings"]["2"]) == [] and kinds(out["findings"]["3"]) == [] # Simples + explicit "no special treatment": not applicable, typed "2" is only a warning
|
||||
assert all(s.strip().upper().startswith("SELECT") for s in conn.cur.statements) # read-only: SELECT statements only
|
||||
|
||||
|
||||
def test_rtc_on_in_the_erp_does_not_make_simples_cclasstrib_applicable_before_effective_date():
|
||||
out = uf.assess_from_connection(FakeConn(param=1), {"crt": "1"}, today=date(2026, 9, 24))
|
||||
assert out["context"]["rtc_enabled"] is True
|
||||
assert out["context"]["cclasstrib_applicability"]["state"] == "NOT_APPLICABLE"
|
||||
assert not [b for f in out["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
|
||||
|
||||
|
||||
def test_rtc_on_reports_simples_cclasstrib_as_not_applicable_and_keeps_other_fields():
|
||||
products = [("1", None, None, None, None, None, None, None, "S", "MONOFÁSICA")]
|
||||
conn = FakeConn(param=1, products=products)
|
||||
|
||||
out = uf.assess_from_connection(conn, {"crt": "1"}, today=date(2026, 9, 25))
|
||||
|
||||
assert out["context"]["cclasstrib_applicability"]["state"] == "NOT_APPLICABLE"
|
||||
assert out["context"]["cclasstrib_applicability"]["source"] == "C000004.CRT + RTC effective-date rules"
|
||||
assert kinds(out["findings"]["1"]) == ["COFINS_CST_MISSING", "PIS_CST_MISSING"]
|
||||
assert all(sql.strip().upper().startswith("SELECT") for sql in conn.cur.statements)
|
||||
|
||||
|
||||
def test_cclasstrib_applicability_change_changes_product_input_fingerprint():
|
||||
simples = uf.assess_from_connection(FakeConn(param=1), {"crt": "1"}, today=date(2026, 9, 25))
|
||||
normal = uf.assess_from_connection(FakeConn(param=1), {"crt": "3"}, today=date(2026, 9, 25))
|
||||
|
||||
assert simples["findings"]["1"]["input_fingerprint"] != normal["findings"]["1"]["input_fingerprint"]
|
||||
assert kinds(simples["findings"]["1"]) == []
|
||||
assert any(blocker["category"] == "RTC" for blocker in normal["findings"]["1"]["blockers"])
|
||||
|
||||
|
||||
def test_simulation_flag_never_touches_the_erp_parameter():
|
||||
conn = FakeConn(param=0)
|
||||
on = uf.assess_from_connection(conn, {"crt": "1"}, rtc_override=True)
|
||||
assert on["context"]["rtc_enabled"] is True and on["context"]["cclasstrib_applicability"]["state"] == "NOT_APPLICABLE"
|
||||
assert not [b for f in on["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
|
||||
assert on["context"]["rtc_param"] == "0" and on["context"]["rtc_source"] == "SIMULATION" and not any("UPDATE" in s.upper() for s in conn.cur.statements)
|
||||
|
||||
|
||||
def test_old_schema_without_rtc_columns_reports_unsupported_and_still_evaluates_pis_cofins():
|
||||
old = [("C000025", c) for c in ("CODIGO", "COD_SAIDA_PIS", "PISCOFINS", "INCIDENCIA_PISCOFINS")]
|
||||
rows = [("1", None, "N", "NENHUMA"), ("2", "049", "N", "NENHUMA")]
|
||||
out = uf.assess_from_connection(FakeConn(schema=old, products=rows, classes=[], cst=()), {"crt": "3"})
|
||||
assert out["context"]["rtc_supported"] is False and out["context"]["rtc_enabled"] is False
|
||||
assert kinds(out["findings"]["1"]) == [] and kinds(out["findings"]["2"]) == ["PIS_CST_INVALID"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value,expected", [(None, False), (0, False), ("0", False), (1, True), ("1", True), ("S", True), ("N", False)])
|
||||
def test_parameter_values_are_read_fail_closed(value, expected):
|
||||
caps = uf.detect_rtc_capabilities({"C000025": {"CCLASSTRIB"}, "RTC_CCLASSTRIB": {"CCLASSTRIB", "CSTIBSCBS"}, "RTC_CST": {"CSTIBSCBS"}}, param_value=value)
|
||||
assert caps["RTC_ENABLED_BY_PARAM"] is expected
|
||||
|
||||
|
||||
def test_catalog_with_orphan_or_duplicate_is_invalid_and_flagged_only_when_rtc_is_on():
|
||||
bad = FakeConn(param=1, classes=CLASSES + [("200003", "200", date(2025, 5, 5), None)]) # duplicate class
|
||||
on = uf.assess_from_connection(bad, {"crt": "3"})
|
||||
assert on["context"]["rtc_catalog_valid"] is False and kinds(on["findings"]["2"]) == ["RTC_CATALOG_INVALID"]
|
||||
off = uf.assess_from_connection(FakeConn(param=0, classes=CLASSES + [("200003", "200", date(2025, 5, 5), None)]), {"crt": "1"})
|
||||
assert not [b for f in off["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
|
||||
|
||||
|
||||
def test_input_fingerprint_changes_with_the_fiscal_values_and_the_rtc_switch():
|
||||
a = uf.assess_from_connection(FakeConn(param=0), {"crt": "1"})["findings"]["1"]["input_fingerprint"]
|
||||
b = uf.assess_from_connection(FakeConn(param=1), {"crt": "1"})["findings"]["1"]["input_fingerprint"]
|
||||
changed = [("1", None, None, "99", None, None, None, None, "N", "NENHUMA")] + PRODUCTS[1:]
|
||||
c = uf.assess_from_connection(FakeConn(param=0, products=changed), {"crt": "1"})["findings"]["1"]["input_fingerprint"]
|
||||
assert len({a, b, c}) == 3 and re.fullmatch(r"[0-9a-f]{64}", a)
|
||||
|
||||
|
||||
def test_rtc_state_is_read_from_this_database_now_with_provenance_and_timestamp():
|
||||
out = uf.assess_from_connection(FakeConn(param=1), {"crt": "1"})
|
||||
ctx = out["context"]
|
||||
assert ctx["rtc_enabled"] is True and ctx["erp_rtc_enabled"] is True and ctx["rtc_source"] == "PARAMETROS.USAR_REFORMA_TRIBUTARIA" and ctx["rtc_param"] == "1" and ctx["rtc_param_rows"] == 1
|
||||
assert ctx["rtc_param_conflict"] is False and ctx["rtc_observed_at"].endswith("+00:00")
|
||||
assert ctx["cclasstrib_applicability"]["state"] == "NOT_APPLICABLE" and ctx["cclasstrib_applicability"]["source"] == "C000004.CRT + RTC effective-date rules"
|
||||
assert ctx["legal_effective_context"]["informational_only"] is True and ctx["legal_effective_context"]["crt"] == "1"
|
||||
|
||||
|
||||
def test_toggle_between_syncs_is_followed_in_both_directions_without_any_cache():
|
||||
conn = FakeConn(param=0)
|
||||
first = uf.assess_from_connection(conn, {"crt": "1"})
|
||||
assert first["context"]["rtc_enabled"] is False and not [b for f in first["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
|
||||
conn.cur.param = 1 # the operator switches the reform on in the ERP between two syncs
|
||||
second = uf.assess_from_connection(conn, {"crt": "1"})
|
||||
assert second["context"]["rtc_enabled"] is True and second["context"]["cclasstrib_applicability"]["state"] == "NOT_APPLICABLE"
|
||||
assert not [b for f in second["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
|
||||
assert second["findings"]["1"]["input_fingerprint"] != first["findings"]["1"]["input_fingerprint"] # a new blocker fingerprint: the old (RTC off) state is not reused
|
||||
conn.cur.param = 0 # and off again
|
||||
third = uf.assess_from_connection(conn, {"crt": "1"})
|
||||
assert third["context"]["rtc_enabled"] is False and not [b for f in third["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
|
||||
assert third["findings"]["1"]["input_fingerprint"] == first["findings"]["1"]["input_fingerprint"]
|
||||
|
||||
|
||||
def test_conflicting_or_repeated_parameter_rows_never_guess_on():
|
||||
same = uf.assess_from_connection(FakeConn(params=[1, 1]), {"crt": "1"})["context"]
|
||||
assert same["rtc_enabled"] is True and same["rtc_param_rows"] == 2 and same["rtc_param_conflict"] is False
|
||||
conflict = uf.assess_from_connection(FakeConn(params=[1, 0]), {"crt": "1"})
|
||||
assert conflict["context"]["rtc_enabled"] is False and conflict["context"]["rtc_param_conflict"] is True
|
||||
assert not [b for f in conflict["findings"].values() for b in f["blockers"] if b["category"] == "RTC"]
|
||||
assert uf.assess_from_connection(FakeConn(params=[]), {"crt": "1"})["context"]["rtc_enabled"] is False
|
||||
|
||||
|
||||
def test_non_canonical_cst_warnings_are_reported_without_blocking():
|
||||
rows = [("1", None, None, "4", None, None, "4", None, "N", "NENHUMA"), ("2", None, None, None, None, None, None, None, "N", "NENHUMA")]
|
||||
out = uf.assess_from_connection(FakeConn(products=rows), {"crt": "3"})
|
||||
assert kinds(out["findings"]["1"]) == [] and kinds(out["findings"]["2"]) == []
|
||||
w = out["context"]["non_canonical_warnings"]
|
||||
assert w["products"] == 1 and w["by_type"] == {"CST_COFINS_NON_CANONICAL_FORMAT": 1, "CST_PIS_NON_CANONICAL_FORMAT": 1} and w["samples"][0]["product_code"] == "1"
|
||||
assert out["context"]["pis_cofins_applicability"] == {"OPTIONAL": 2}
|
||||
@@ -0,0 +1,65 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
|
||||
SERVICE_WRAPPER = Path(__file__).parents[1] / "service" / "gti_agent_windows_service.py"
|
||||
|
||||
|
||||
def load_service_wrapper():
|
||||
spec = importlib.util.spec_from_file_location("gti_agent_windows_service_test", SERVICE_WRAPPER)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_main_enters_scm_dispatcher_when_started_without_command(monkeypatch):
|
||||
service = load_service_wrapper()
|
||||
calls = []
|
||||
|
||||
class FakeServiceManager:
|
||||
def Initialize(self):
|
||||
calls.append("initialize")
|
||||
|
||||
def PrepareToHostSingle(self, service_class):
|
||||
calls.append(("prepare", service_class))
|
||||
|
||||
def StartServiceCtrlDispatcher(self):
|
||||
calls.append("dispatcher")
|
||||
|
||||
def forbidden_handle_command_line(service_class):
|
||||
raise AssertionError("SCM start must not be parsed as a CLI command")
|
||||
|
||||
monkeypatch.setattr(service, "servicemanager", FakeServiceManager())
|
||||
monkeypatch.setattr(service, "win32serviceutil", SimpleNamespace(HandleCommandLine=forbidden_handle_command_line))
|
||||
monkeypatch.setattr(service.sys, "argv", [str(SERVICE_WRAPPER)])
|
||||
|
||||
service.main()
|
||||
|
||||
assert calls == ["initialize", ("prepare", service.GTIFiscalAgentService), "dispatcher"]
|
||||
|
||||
|
||||
def test_main_preserves_pywin32_cli_commands(monkeypatch):
|
||||
service = load_service_wrapper()
|
||||
called = []
|
||||
|
||||
class ForbiddenServiceManager:
|
||||
def __getattr__(self, name):
|
||||
raise AssertionError("CLI commands must not enter the SCM dispatcher")
|
||||
|
||||
monkeypatch.setattr(service, "servicemanager", ForbiddenServiceManager())
|
||||
monkeypatch.setattr(service, "win32serviceutil", SimpleNamespace(HandleCommandLine=lambda cls: called.append(cls)))
|
||||
monkeypatch.setattr(service.sys, "argv", [str(SERVICE_WRAPPER), "start"])
|
||||
|
||||
service.main()
|
||||
|
||||
assert called == [service.GTIFiscalAgentService]
|
||||
|
||||
|
||||
def test_a_fatal_error_in_the_agent_loop_stops_the_service_instead_of_leaving_a_dead_one_running():
|
||||
text = SERVICE_WRAPPER.read_text(encoding="utf-8")
|
||||
body = text.split("agent_service.run(config, os.environ, stopper=self.stop_flag)", 1)[1].split("def main", 1)[0]
|
||||
assert "except BaseException" in body and "logger" not in body.split("except BaseException")[0][-40:] # wrapped in try/except
|
||||
assert "LogErrorMsg" in body and "self.stop_flag.stop = True" in body and "win32event.SetEvent(self.stop_event)" in body
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Agent side of the zero-touch model: delta sync via fingerprint cache, read-only guarantee,
|
||||
blockers/operational summary in the job result, central pause, Firebird credential resolution."""
|
||||
import json
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from test_run_agent_service import AUTHORIZED, _FakeExecutor, _config, service
|
||||
|
||||
|
||||
class FakeHub:
|
||||
def __init__(self):
|
||||
self.classified = []
|
||||
|
||||
def classify_batch(self, products):
|
||||
self.classified.append([p["product_id"] for p in products])
|
||||
return [{"status": "EXACT_NCM" if p["current_ncm"] else "INVALID_NCM", "ipi_status": "IPI_RESOLVED", "classification": {}} for p in products]
|
||||
|
||||
|
||||
def product(code, ncm="03055100", name="X"):
|
||||
return {"CODIGO": code, "PRODUTO": name, "EAN_GTIN": None, "CLASSIFICACAO_FISCAL": ncm, "NCM": None, "CFOP": "5102", "CST": None, "ALIQUOTA": None, "CEST": None, "CSOSN": "102", "IPI": None}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def env_setup(monkeypatch, tmp_path):
|
||||
hub = FakeHub()
|
||||
monkeypatch.setattr(service, "HubClient", lambda *a, **k: hub)
|
||||
monkeypatch.setattr(service, "build_job_coverage", lambda env, client, products, decisions: {"total_products": len(products)})
|
||||
state = service.AgentState(tmp_path / "s.db")
|
||||
return hub, state
|
||||
|
||||
|
||||
def run_job(kind, products, state, monkeypatch, config=None):
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: products)
|
||||
return service.execute_job({"job_id": 1, "kind": kind, "agent_id": 2, "payload": {}}, config or _config(), _FakeExecutor(), state, {})
|
||||
|
||||
|
||||
def test_initial_sync_processes_everything_and_reports_operational_summary(env_setup, monkeypatch):
|
||||
hub, state = env_setup
|
||||
products = [product("1"), product("2"), product("3", ncm="")]
|
||||
result, error = run_job("INITIAL_SYNC", products, state, monkeypatch)
|
||||
assert error is None
|
||||
assert result["products_total"] == 3 and result["products_changed"] == 3 and hub.classified == [["1", "2", "3"]]
|
||||
op = result["operational"]
|
||||
assert op["TOTAL_PRODUCTS"] == 3 and op["COMPLETE"] == 2 and op["SAFE_BLOCKED"] == 1
|
||||
assert op["COMPLETE"] + op["CORRECTED"] + op["SAFE_BLOCKED"] + op["FAILED_TECHNICAL"] == op["TOTAL_PRODUCTS"] # invariant closes
|
||||
assert [b["product_code"] for b in result["blocked_products"]] == ["3"] and result["resolved_products"] == ["1", "2"]
|
||||
|
||||
|
||||
def test_incremental_sync_reclassifies_only_new_and_changed_products(env_setup, monkeypatch):
|
||||
hub, state = env_setup
|
||||
base = [product("1"), product("2"), product("3")]
|
||||
run_job("INITIAL_SYNC", base, state, monkeypatch)
|
||||
hub.classified.clear()
|
||||
|
||||
result, error = run_job("SYNC_INCREMENTAL", base, state, monkeypatch) # nothing changed
|
||||
assert error is None and result["products_changed"] == 0 and hub.classified == [[]] or hub.classified == []
|
||||
assert result["operational"]["TOTAL_PRODUCTS"] == 3 # coverage still complete, rebuilt from the cache
|
||||
|
||||
hub.classified.clear()
|
||||
changed = [product("1"), product("2", ncm="", name="broken now"), product("3"), product("4")] # 2 changed, 4 new
|
||||
result, _ = run_job("SYNC_INCREMENTAL", changed, state, monkeypatch)
|
||||
assert sorted(hub.classified[0]) == ["2", "4"] and result["products_changed"] == 2
|
||||
assert result["operational"]["TOTAL_PRODUCTS"] == 4 and result["operational"]["SAFE_BLOCKED"] == 1
|
||||
|
||||
|
||||
def test_removed_product_leaves_local_cache_but_is_not_reported_as_resolved(env_setup, monkeypatch):
|
||||
hub, state = env_setup
|
||||
run_job("INITIAL_SYNC", [product("1"), product("2")], state, monkeypatch)
|
||||
result, _ = run_job("SYNC_INCREMENTAL", [product("1")], state, monkeypatch)
|
||||
assert state.cached_decision("2") is None and result["operational"]["TOTAL_PRODUCTS"] == 1
|
||||
assert result["resolved_products"] == [] # no history is auto-resolved/deleted for a product that merely disappeared
|
||||
|
||||
|
||||
def test_every_sync_kind_is_read_only(env_setup, monkeypatch):
|
||||
hub, state = env_setup
|
||||
for kind in ("INITIAL_SYNC", "SYNC_INCREMENTAL", "REPROCESS_ALL"):
|
||||
result, error = run_job(kind, [product("1")], state, monkeypatch)
|
||||
assert error is None and result["writes_attempted"] == 0 and result["writes_succeeded"] == 0 and result["firebird_writes_enabled"] is False
|
||||
with pytest.raises(RuntimeError, match="writes are disabled"):
|
||||
service.open_write_connection({}, AUTHORIZED, firebird_writes=False) # no write-capable connection can even be opened
|
||||
|
||||
|
||||
def test_hub_offline_during_job_fails_the_job_cleanly_no_crash(env_setup, monkeypatch):
|
||||
hub, state = env_setup
|
||||
def boom(products): raise ConnectionError("hub down")
|
||||
hub.classify_batch = boom
|
||||
result, error = run_job("SYNC_INCREMENTAL", [product("9")], state, monkeypatch)
|
||||
assert result is None and "ConnectionError" in error # reported as a job failure, the service loop keeps running
|
||||
|
||||
|
||||
def test_firebird_offline_during_job_fails_the_job_cleanly(env_setup, monkeypatch):
|
||||
hub, state = env_setup
|
||||
def down(env, codigos=None): raise OSError("Firebird service not running")
|
||||
monkeypatch.setattr(service, "read_products", down)
|
||||
result, error = service.execute_job({"job_id": 1, "kind": "SYNC_INCREMENTAL", "agent_id": 2, "payload": {}}, _config(), _FakeExecutor(), state, {})
|
||||
assert result is None and "OSError" in error
|
||||
|
||||
|
||||
def test_firebird_credentials_come_from_the_dpapi_file_not_agent_env(monkeypatch, tmp_path):
|
||||
from gti_fiscal_agent import credential_store
|
||||
path = tmp_path / "firebird.bin"
|
||||
monkeypatch.setattr(credential_store, "read_credential", lambda p, env=None: json.dumps({"user": "SYSDBA", "password": "s3cret"}) if str(p) == str(path) else (_ for _ in ()).throw(FileNotFoundError()))
|
||||
env = {"FIREBIRD_DATABASE_PATH": AUTHORIZED, "GTI_FIREBIRD_CREDENTIAL_FILE": str(path)}
|
||||
service.Config(env)
|
||||
assert env["FB_USER"] == "SYSDBA" and env["FB_PASSWORD"] == "s3cret"
|
||||
explicit = {"FIREBIRD_DATABASE_PATH": AUTHORIZED, "FB_PASSWORD": "dev", "FB_USER": "u", "GTI_FIREBIRD_CREDENTIAL_FILE": str(path)}
|
||||
service.Config(explicit)
|
||||
assert explicit["FB_PASSWORD"] == "dev" # an explicit env value still wins (dev override)
|
||||
|
||||
|
||||
def test_central_resume_after_pause_makes_agent_poll_again(monkeypatch, tmp_path):
|
||||
class Stopper: stop = False
|
||||
stopper = Stopper()
|
||||
states = iter([False, True]) # paused, then resumed
|
||||
calls = {"next_job": 0}
|
||||
|
||||
class Hub:
|
||||
def identity(self): return {"agent_id": 2, "installation_id": "00000000-0000-4000-8000-000000000222"}
|
||||
def heartbeat_metadata(self, payload): return {"processing_enabled": next(states)}
|
||||
def next_job(self):
|
||||
calls["next_job"] += 1; stopper.stop = True; return None
|
||||
|
||||
monkeypatch.setattr(service, "HubClient", lambda *a, **k: Hub())
|
||||
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
||||
monkeypatch.setattr(service.time, "sleep", lambda s: None)
|
||||
service.run(_config(agent_id="2", poll_interval_seconds=0, state_db=str(tmp_path / "a.db")), {"AGENT_ID": "2"}, stopper=stopper)
|
||||
assert calls["next_job"] == 1 # first cycle paused (no poll), second cycle resumed (polled)
|
||||
|
||||
|
||||
def test_heartbeats_keep_flowing_while_a_long_job_runs(monkeypatch, tmp_path):
|
||||
beats = []
|
||||
|
||||
class Hub:
|
||||
def identity(self): return {"agent_id": 2, "installation_id": "00000000-0000-4000-8000-000000000222"}
|
||||
def heartbeat_metadata(self, payload): beats.append(payload["status"]); return {"processing_enabled": True}
|
||||
def next_job(self): return {"job_id": 5, "kind": "INITIAL_SYNC", "agent_id": 2, "payload": {}}
|
||||
def complete_job(self, job_id, summary): stopper.stop = True
|
||||
def fail_job(self, job_id, error): stopper.stop = True
|
||||
|
||||
class Stopper: stop = False
|
||||
stopper = Stopper()
|
||||
monkeypatch.setattr(service, "BUSY_HEARTBEAT_SECONDS", 0.05)
|
||||
monkeypatch.setattr(service, "HubClient", lambda *a, **k: Hub())
|
||||
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
||||
monkeypatch.setattr(service, "execute_job", lambda *a, **k: (time.sleep(0.4), ({"ok": 1}, None))[1])
|
||||
service.run(_config(agent_id="2", poll_interval_seconds=0, state_db=str(tmp_path / "a.db")), {"AGENT_ID": "2"}, stopper=stopper)
|
||||
assert beats.count("BUSY") >= 3 # heartbeats during the job, not only before it
|
||||
|
||||
|
||||
def _findings(**by_code):
|
||||
out = {}
|
||||
for code, kinds in by_code.items():
|
||||
out[code] = {"blockers": [{"type": t, "category": c, "reason": "R"} for t, c in kinds], "applicability": {"PIS_COFINS": "REQUIRED", "RTC": "NOT_APPLICABLE"}, "input_fingerprint": "fp" + code, "needs_review": False}
|
||||
return out
|
||||
|
||||
|
||||
def test_unified_fiscal_blockers_change_the_single_coverage_and_the_single_queue(env_setup, monkeypatch):
|
||||
from gti_fiscal_agent import unified_fiscal
|
||||
findings = _findings(**{"2": [("PIS_CST_INVALID", "PIS"), ("COFINS_CST_INVALID", "COFINS")]})
|
||||
monkeypatch.setattr(unified_fiscal, "assess_from_connection", lambda conn, company, **kw: {"context": {"rtc_enabled": False, "rtc_supported": True, "rtc_catalog_valid": True}, "findings": findings})
|
||||
hub, state = env_setup
|
||||
result, error = run_job("INITIAL_SYNC", [product("1"), product("2"), product("3", ncm="")], state, monkeypatch)
|
||||
assert error is None
|
||||
op = result["operational"]
|
||||
assert op["TOTAL_PRODUCTS"] == 3 and op["COMPLETE_PRODUCTS"] == 1 and op["SAFE_BLOCKED"] == 2 and op["BLOCKED_FISCAL_DATA"] == 1 # one number, one definition
|
||||
assert op["COMPLETE_PRODUCTS"] + op["SAFE_BLOCKED"] + op["FAILED_TECHNICAL"] + op["UNCLASSIFIED"] == 3
|
||||
queue = sorted((b["product_code"], b["field"], b["reason_code"]) for b in result["blocked_products"])
|
||||
assert queue == [("2", "COFINS", "COFINS_CST_INVALID"), ("2", "PIS", "PIS_CST_INVALID"), ("3", "NCM", "INVALID_NCM")] # ONE queue: NCM and fiscal blockers together
|
||||
assert result["resolved_products"] == ["1"] # a fiscal blocker keeps the product out of resolved
|
||||
assert result["fiscal_context"]["rtc_enabled"] is False and result["fiscal_context"]["evaluated_all"] is True
|
||||
assert "expanded_scope" not in result
|
||||
|
||||
|
||||
def test_rtc_context_is_reported_but_blockers_only_exist_when_the_evaluator_produced_them(env_setup, monkeypatch):
|
||||
from gti_fiscal_agent import unified_fiscal
|
||||
monkeypatch.setattr(unified_fiscal, "assess_from_connection", lambda conn, company, **kw: {"context": {"rtc_enabled": True, "rtc_supported": True, "rtc_catalog_valid": True}, "findings": _findings(**{"1": [("CCLASSTRIB_MISSING", "RTC")]})})
|
||||
hub, state = env_setup
|
||||
result, _ = run_job("REPROCESS_ALL", [product("1"), product("2")], state, monkeypatch)
|
||||
assert [b["reason_code"] for b in result["blocked_products"]] == ["CCLASSTRIB_MISSING"] and result["blocked_products"][0]["field"] == "RTC"
|
||||
assert result["fiscal_context"]["rtc_enabled"] is True and result["operational"]["BLOCKED_FISCAL_DATA"] == 1
|
||||
|
||||
|
||||
def test_partial_reprocess_does_not_claim_a_full_fiscal_evaluation(env_setup, monkeypatch):
|
||||
hub, state = env_setup
|
||||
result, _ = run_job("REPROCESS_PRODUCTS", [product("1")], state, monkeypatch)
|
||||
assert result["fiscal_context"] is None and result["operational"] is None
|
||||
|
||||
|
||||
def test_rtc_toggle_between_two_syncs_is_followed_by_the_next_job(env_setup, monkeypatch):
|
||||
"""The Agent asks the database EVERY sync: 0 -> 1 makes the next job report RTC blockers, 1 -> 0 makes the following job report none (Hub then closes the old ones)."""
|
||||
from gti_fiscal_agent import unified_fiscal
|
||||
switch = {"on": False}
|
||||
|
||||
def assess(conn, company, **kw):
|
||||
found = _findings(**{"1": [("CCLASSTRIB_MISSING", "RTC")]}) if switch["on"] else {}
|
||||
return {"context": {"rtc_enabled": switch["on"], "rtc_supported": True, "rtc_catalog_valid": True, "rtc_source": "PARAMETROS.USAR_REFORMA_TRIBUTARIA", "rtc_observed_at": "2026-09-24T20:00:00+00:00"}, "findings": found}
|
||||
monkeypatch.setattr(unified_fiscal, "assess_from_connection", assess)
|
||||
hub, state = env_setup
|
||||
products = [product("1"), product("2")]
|
||||
off1, _ = run_job("INITIAL_SYNC", products, state, monkeypatch)
|
||||
switch["on"] = True
|
||||
on, _ = run_job("SYNC_INCREMENTAL", products, state, monkeypatch) # nothing changed in C000025: the delta cache must not hide the RTC change
|
||||
switch["on"] = False
|
||||
off2, _ = run_job("SYNC_INCREMENTAL", products, state, monkeypatch)
|
||||
assert off1["blocked_products"] == [] and off1["fiscal_context"]["rtc_enabled"] is False
|
||||
assert [b["field"] for b in on["blocked_products"]] == ["RTC"] and on["fiscal_context"]["rtc_enabled"] is True and on["operational"]["BLOCKED_FISCAL_DATA"] == 1
|
||||
assert on["fiscal_context"]["rtc_observed_at"] and on["fiscal_context"]["rtc_source"] == "PARAMETROS.USAR_REFORMA_TRIBUTARIA"
|
||||
assert off2["blocked_products"] == [] and off2["fiscal_context"]["rtc_enabled"] is False and off2["fiscal_context"]["evaluated_all"] is True
|
||||
assert off2["operational"]["COMPLETE_PRODUCTS"] == 2
|
||||
Reference in New Issue
Block a user