feat: zero-touch Agent — central scheduler, central pause, delta sync, discovery-driven installer

Hub: installations.processing_enabled / sync_interval_seconds / last_sync_* (migration 0026).
The first heartbeat of an installation without coverage queues exactly one INITIAL_SYNC;
afterwards one SYNC_INCREMENTAL per interval (idempotent, backs off on failure). One fiscal
job at a time per Agent (FIFO, stale RUNNING recovery). Job completion now updates the
coverage snapshot, blockers and JOB_STARTED/COMPLETED/FAILED events. Panel: Sincronizar
agora / Reprocessar tudo / Pausar / Retomar + status; pause is enforced Hub-side.
Agent: local AGENT_JOBS_ENABLED is ignored (central control); SYNC_INCREMENTAL re-classifies
only new/changed products via a per-product fingerprint+decision cache and rebuilds coverage
from it; results carry operational summary and blockers; Firebird credentials come from a
DPAPI firebird.bin. Setup: read-only SGV Firebird discovery (auto-select only when
unambiguous, wizard chooser otherwise), starts the service, waits for the first heartbeat,
upgrade-in-place mode. All jobs stay read-only (FIREBIRD_WRITES=0). Version 1.1.3.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
gti
2026-09-24 08:47:00 -03:00
co-authored by Claude Sonnet 5
parent 9f9e06e9e9
commit bb663a979c
36 changed files with 1409 additions and 118 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
[project]
name = "gti-fiscal-agent"
version = "1.1.2"
version = "1.1.3"
requires-python = ">=3.10"
dependencies = ["httpx>=0.27,<1"]
+58 -12
View File
@@ -31,10 +31,12 @@ sys.path.insert(0, str(PROJECT_ROOT)) # ships central/app engine modules read-o
from gti_fiscal_agent.client import HubClient # noqa: E402
from gti_fiscal_agent.firebird_reader import check_connection, read_products # noqa: E402
from gti_fiscal_agent.pipeline import run_pipeline # noqa: E402
from gti_fiscal_agent.runner import REFERENCE_SCHEMA_V1, to_hub_product # noqa: E402
from gti_fiscal_agent.runner import REFERENCE_SCHEMA_V1, product_fingerprint, to_hub_product # noqa: E402
from central.app.operational_state import classify_product_state, summarize_operational_states # noqa: E402
from gti_fiscal_agent.state import AgentState # noqa: E402
_MAX_BACKOFF_SECONDS = 60
SYNC_JOB_KINDS = ("INITIAL_SYNC", "SYNC_INCREMENTAL", "REPROCESS_ALL") # read-only ERP jobs that refresh coverage
logger = logging.getLogger("gti_fiscal_agent")
@@ -87,6 +89,7 @@ class Config:
self.agent_token = self._resolve_agent_token(env)
self.company_id = env.get("COMPANY_ID", "")
self.company_uf = env.get("COMPANY_UF", "")
_resolve_firebird_credentials(env)
self.database_path = env["FIREBIRD_DATABASE_PATH"]
# FISCAL_MODE gates writes technically: DRY_RUN can never write, whatever FIREBIRD_WRITES says. Unset = legacy
# behaviour (FIREBIRD_WRITES decides). `firebird_writes` below is the EFFECTIVE value used by every write path.
@@ -94,10 +97,12 @@ class Config:
self.fiscal_mode = (env.get("FISCAL_MODE") or ("AUTO_APPLY" if requested else "DRY_RUN")).strip().upper()
self.requested_writes = requested
self.firebird_writes = requested and self.fiscal_mode == "AUTO_APPLY"
jobs_enabled = (env.get("AGENT_JOBS_ENABLED", "1") or "1").strip().upper()
self.agent_jobs_enabled = jobs_enabled not in ("0", "FALSE", "NO")
# Job execution is controlled CENTRALLY (Hub heartbeat: processing_enabled) — there is no
# local toggle, so no installation ever needs its agent.env edited to start/stop work.
# (A legacy AGENT_JOBS_ENABLED line in an old agent.env is deliberately ignored.)
self.agent_jobs_enabled = True
self.poll_interval_seconds = float(env.get("POLL_INTERVAL_SECONDS", "5"))
self.agent_version = env.get("AGENT_VERSION", "1.1.2")
self.agent_version = env.get("AGENT_VERSION", "1.1.3")
self.schema_fingerprint = env.get("SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1)
self.state_db = env.get("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite"))
# Pilot write authorization: both must be explicitly set by the installer for
@@ -125,6 +130,23 @@ class Config:
return ""
def _resolve_firebird_credentials(env):
"""FB_USER/FB_PASSWORD come from the DPAPI-protected firebird.bin written by Setup
(never from a hand-edited agent.env, never sent to the Hub). An explicit FB_PASSWORD in
the environment still wins (dev override)."""
if env.get("FB_PASSWORD"):
return
from gti_fiscal_agent.credential_store import read_credential
path = env.get("GTI_FIREBIRD_CREDENTIAL_FILE") or str(Path(os.environ.get("PROGRAMFILES", "C:/Program Files")) / "GTI" / "FiscalAgent" / "firebird.bin")
try:
data = json.loads(read_credential(path, env=env))
except Exception as exc: # noqa: BLE001 — absent/unreadable: preflight will fail closed with a clear blocker
logger.info("no Firebird credential file usable: %s", type(exc).__name__)
return
env["FB_USER"], env["FB_PASSWORD"] = data.get("user", ""), data.get("password", "")
class GracefulStop:
def __init__(self):
self.stop = False
@@ -183,23 +205,46 @@ def execute_job(job, config, executor, state, env):
logger.info("job %s: read %d product(s) from Firebird", job.get("job_id"), len(products))
client = HubClient(config.hub_url, config.agent_token)
decisions = client.classify_batch([to_hub_product(p) for p in products])
sync_kind = kind in SYNC_JOB_KINDS
cache = sync_kind and hasattr(state, "remember_decision")
# SYNC_INCREMENTAL = delta: only products whose source fingerprint changed (or are new) are
# re-classified by the Hub; INITIAL_SYNC / REPROCESS_ALL / REPROCESS_PRODUCTS process everything they read.
if kind == "SYNC_INCREMENTAL" and cache:
to_process = [p for p in products if state.cached_fingerprint(p["CODIGO"]) != product_fingerprint(p)]
else:
to_process = products
decisions = client.classify_batch([to_hub_product(p) for p in to_process])
write_connection = None
if config.firebird_writes:
write_connection = open_write_connection(env, config.database_path)
try:
summary = run_pipeline(products, decisions, state, config.firebird_writes, write_connection, executor, config.database_path, authorized_path=config.authorized_write_database_path)
summary = run_pipeline(to_process, decisions, state, config.firebird_writes, write_connection, executor, config.database_path, authorized_path=config.authorized_write_database_path)
finally:
if write_connection is not None:
write_connection.close()
coverage = build_job_coverage(env, client, products, decisions) if kind == "REPROCESS_ALL" else None
operational = coverage = None
if sync_kind:
if cache:
for product, decision in zip(to_process, decisions):
state.remember_decision(product["CODIGO"], product_fingerprint(product), decision)
state.forget_missing([p["CODIGO"] for p in products])
all_decisions = [state.cached_decision(p["CODIGO"]) or {} for p in products]
else:
all_decisions = decisions
operational = summarize_operational_states([classify_product_state(p, d) for p, d in zip(products, all_decisions)])
coverage = build_job_coverage(env, client, products, all_decisions)
result_summary = {
"coverage": coverage,
"operational": operational,
"fiscal_mode": getattr(config, "fiscal_mode", None),
"release_version": env.get("AGENT_VERSION") or "1.1.2",
"release_version": env.get("AGENT_VERSION") or "1.1.3",
"products_total": len(products),
"products_changed": len(to_process),
"products_processed": summary["processed"],
"blocked_products": summary["blocked_products"],
"resolved_products": summary["resolved_products"],
"writes_attempted": summary["writes_attempted"],
"writes_succeeded": summary["writes_succeeded"],
"writes_failed": summary["writes_failed"],
@@ -275,9 +320,10 @@ def run(config, env, *, max_iterations=None, stopper=None):
"fiscal_engine_version": env.get("FISCAL_ENGINE_VERSION", config.agent_version), "dataset_versions": {},
}
if hasattr(client, "heartbeat_metadata"):
client.heartbeat_metadata(heartbeat_payload)
hb_response = client.heartbeat_metadata(heartbeat_payload)
else:
client.heartbeat("IDLE", schema_fingerprint=config.schema_fingerprint, agent_version=config.agent_version, last_job_id=last_job_id)
hb_response = client.heartbeat("IDLE", schema_fingerprint=config.schema_fingerprint, agent_version=config.agent_version, last_job_id=last_job_id)
processing_enabled = not (isinstance(hb_response, dict) and hb_response.get("processing_enabled") is False)
if was_down:
logger.info("reconnected to Hub after %ss backoff", backoff)
was_down = False
@@ -289,7 +335,7 @@ def run(config, env, *, max_iterations=None, stopper=None):
backoff = min(backoff * 2, _MAX_BACKOFF_SECONDS)
continue
if not config.agent_jobs_enabled:
if not processing_enabled: # central pause: heartbeat keeps flowing, no job is claimed or executed
time.sleep(config.poll_interval_seconds)
continue
@@ -331,7 +377,7 @@ def run(config, env, *, max_iterations=None, stopper=None):
def enforce_startup_preflight(env):
"""Full read-only preflight at startup (Hub, auth, Firebird, capabilities, company context, guards). Fail closed."""
from gti_fiscal_agent.preflight import run_preflight
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.1.2")
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.1.3")
logger.info("startup preflight: %s", json.dumps({k: report[k] for k in ("PREFLIGHT", "blockers", "fiscal_mode", "effective_firebird_writes")}, ensure_ascii=False))
print(json.dumps({"PREFLIGHT": report["PREFLIGHT"], "blockers": report["blockers"]}, ensure_ascii=False))
if report["PREFLIGHT"] != "PASS":
+1 -1
View File
@@ -9,7 +9,7 @@ import os
import sqlite3
from pathlib import Path
AGENT_VERSION_DEFAULT = "1.1.2"
AGENT_VERSION_DEFAULT = "1.1.3"
REQUIRED_ENV = ("GTI_HUB_URL", "AGENT_TOKEN", "FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")
REQUIRED_CAPABILITIES = ("supports_ncm", "supports_ipi", "supports_cest")
FISCAL_MODES = ("DRY_RUN", "AUTO_APPLY")
+21
View File
@@ -1,4 +1,5 @@
"""Local SQLite outbox; it has no Firebird dependency or write path."""
import json
import sqlite3
from pathlib import Path
import uuid
@@ -8,6 +9,9 @@ class AgentState:
self.connection = sqlite3.connect(Path(path)); self.connection.row_factory = sqlite3.Row
self.connection.executescript("CREATE TABLE IF NOT EXISTS product_state(product_id TEXT PRIMARY KEY, source_fingerprint TEXT NOT NULL, dataset_version TEXT NOT NULL, engine_version TEXT NOT NULL, status TEXT NOT NULL, last_processed_at TEXT, last_result_hash TEXT, reason_code TEXT); CREATE TABLE IF NOT EXISTS outbox(product_id TEXT PRIMARY KEY, source_fingerprint TEXT NOT NULL, status TEXT NOT NULL, retries INTEGER NOT NULL DEFAULT 0); CREATE TABLE IF NOT EXISTS ipi_write_log(product_id TEXT NOT NULL, source_fingerprint TEXT NOT NULL, decision_status TEXT, ipi_status TEXT, write_status TEXT NOT NULL, before_value TEXT, proposed_value TEXT, result TEXT, reason_code TEXT, timestamp TEXT NOT NULL);")
self.connection.execute("CREATE TABLE IF NOT EXISTS installation_identity (id INTEGER PRIMARY KEY CHECK(id=1), installation_id TEXT NOT NULL)")
# Last Hub decision per product + the source fingerprint it was computed for: what makes
# SYNC_INCREMENTAL a real delta (only new/changed products are re-classified).
self.connection.execute("CREATE TABLE IF NOT EXISTS decision_cache (product_id TEXT PRIMARY KEY, source_fingerprint TEXT NOT NULL, decision_json TEXT NOT NULL)")
self.connection.commit()
def stored_installation_id(self):
row = self.connection.execute("SELECT installation_id FROM installation_identity WHERE id=1").fetchone()
@@ -35,6 +39,23 @@ class AgentState:
if row is None or str(uuid.UUID(row["installation_id"])) != value:
raise ValueError("persisted installation_id conflicts with candidate")
return value
def cached_fingerprint(self, product_id):
row = self.connection.execute("SELECT source_fingerprint FROM decision_cache WHERE product_id=?", (product_id,)).fetchone()
return row["source_fingerprint"] if row else None
def cached_decision(self, product_id):
row = self.connection.execute("SELECT decision_json FROM decision_cache WHERE product_id=?", (product_id,)).fetchone()
return json.loads(row["decision_json"]) if row else None
def remember_decision(self, product_id, source_fingerprint, decision):
self.connection.execute("INSERT OR REPLACE INTO decision_cache(product_id,source_fingerprint,decision_json) VALUES(?,?,?)", (product_id, source_fingerprint, json.dumps(decision, default=str)))
self.connection.commit()
def forget_missing(self, present_product_ids):
"""Products no longer in the ERP leave the LOCAL cache only; Hub-side history/blockers are never deleted by this."""
present = set(present_product_ids)
gone = [row["product_id"] for row in self.connection.execute("SELECT product_id FROM decision_cache") if row["product_id"] not in present]
for product_id in gone:
self.connection.execute("DELETE FROM decision_cache WHERE product_id=?", (product_id,))
self.connection.commit()
return len(gone)
def enqueue(self, product_id, source_fingerprint, dataset_version, engine_version):
row = self.connection.execute("SELECT * FROM product_state WHERE product_id=?", (product_id,)).fetchone()
if row and tuple(row[key] for key in ("source_fingerprint", "engine_version", "status")) == (source_fingerprint, engine_version, "RESULT_RECEIVED"): return False
+7 -6
View File
@@ -90,10 +90,12 @@ def test_config_defaults_from_env():
assert config.agent_jobs_enabled is True
def test_config_can_disable_agent_job_polling():
def test_local_agent_jobs_toggle_is_ignored_control_is_central():
# Zero-touch: a stale AGENT_JOBS_ENABLED=0 left in an old agent.env must not silence the Agent;
# only the Hub (heartbeat processing_enabled) can pause it.
config = service.Config({"FIREBIRD_DATABASE_PATH": AUTHORIZED, "AGENT_JOBS_ENABLED": "0"})
assert config.agent_jobs_enabled is False
assert config.agent_jobs_enabled is True
def test_config_reads_gti_hub_url_and_writes_flag():
@@ -294,7 +296,7 @@ class _ExternalStopFlag:
self.stop = False
def test_run_sends_heartbeat_but_never_polls_when_jobs_disabled(monkeypatch, tmp_path):
def test_run_sends_heartbeat_but_never_polls_when_paused_centrally(monkeypatch, tmp_path):
stopper = _ExternalStopFlag()
fake_hub = _FakeHubForLoop()
installation_id = "00000000-0000-4000-8000-000000000222"
@@ -308,11 +310,11 @@ def test_run_sends_heartbeat_but_never_polls_when_jobs_disabled(monkeypatch, tmp
calls["heartbeat"] += 1
calls["heartbeat_installation_id"] = payload["installation_id"]
stopper.stop = True
return {"agent_id": 2}
return {"agent_id": 2, "processing_enabled": False} # paused centrally by the Hub
def forbidden_next_job():
calls["next_job"] += 1
raise AssertionError("jobs are disabled; next_job must not be called")
raise AssertionError("processing is paused centrally; next_job must not be called")
fake_hub.identity = identity
fake_hub.heartbeat_metadata = heartbeat_metadata
@@ -323,7 +325,6 @@ def test_run_sends_heartbeat_but_never_polls_when_jobs_disabled(monkeypatch, tmp
config = _config(
agent_id="2",
agent_jobs_enabled=False,
poll_interval_seconds=0,
state_db=str(tmp_path / "agent.db"),
)
+127
View File
@@ -0,0 +1,127 @@
"""Agent side of the zero-touch model: delta sync via fingerprint cache, read-only guarantee,
blockers/operational summary in the job result, central pause, Firebird credential resolution."""
import json
import pytest
from test_run_agent_service import AUTHORIZED, _FakeExecutor, _config, service
class FakeHub:
def __init__(self):
self.classified = []
def classify_batch(self, products):
self.classified.append([p["product_id"] for p in products])
return [{"status": "EXACT_NCM" if p["current_ncm"] else "INVALID_NCM", "ipi_status": "IPI_RESOLVED", "classification": {}} for p in products]
def product(code, ncm="03055100", name="X"):
return {"CODIGO": code, "PRODUTO": name, "EAN_GTIN": None, "CLASSIFICACAO_FISCAL": ncm, "NCM": None, "CFOP": "5102", "CST": None, "ALIQUOTA": None, "CEST": None, "CSOSN": "102", "IPI": None}
@pytest.fixture
def env_setup(monkeypatch, tmp_path):
hub = FakeHub()
monkeypatch.setattr(service, "HubClient", lambda *a, **k: hub)
monkeypatch.setattr(service, "build_job_coverage", lambda env, client, products, decisions: {"total_products": len(products)})
state = service.AgentState(tmp_path / "s.db")
return hub, state
def run_job(kind, products, state, monkeypatch, config=None):
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: products)
return service.execute_job({"job_id": 1, "kind": kind, "agent_id": 2, "payload": {}}, config or _config(), _FakeExecutor(), state, {})
def test_initial_sync_processes_everything_and_reports_operational_summary(env_setup, monkeypatch):
hub, state = env_setup
products = [product("1"), product("2"), product("3", ncm="")]
result, error = run_job("INITIAL_SYNC", products, state, monkeypatch)
assert error is None
assert result["products_total"] == 3 and result["products_changed"] == 3 and hub.classified == [["1", "2", "3"]]
op = result["operational"]
assert op["TOTAL_PRODUCTS"] == 3 and op["COMPLETE"] == 2 and op["SAFE_BLOCKED"] == 1
assert op["COMPLETE"] + op["CORRECTED"] + op["SAFE_BLOCKED"] + op["FAILED_TECHNICAL"] == op["TOTAL_PRODUCTS"] # invariant closes
assert [b["product_code"] for b in result["blocked_products"]] == ["3"] and result["resolved_products"] == ["1", "2"]
def test_incremental_sync_reclassifies_only_new_and_changed_products(env_setup, monkeypatch):
hub, state = env_setup
base = [product("1"), product("2"), product("3")]
run_job("INITIAL_SYNC", base, state, monkeypatch)
hub.classified.clear()
result, error = run_job("SYNC_INCREMENTAL", base, state, monkeypatch) # nothing changed
assert error is None and result["products_changed"] == 0 and hub.classified == [[]] or hub.classified == []
assert result["operational"]["TOTAL_PRODUCTS"] == 3 # coverage still complete, rebuilt from the cache
hub.classified.clear()
changed = [product("1"), product("2", ncm="", name="broken now"), product("3"), product("4")] # 2 changed, 4 new
result, _ = run_job("SYNC_INCREMENTAL", changed, state, monkeypatch)
assert sorted(hub.classified[0]) == ["2", "4"] and result["products_changed"] == 2
assert result["operational"]["TOTAL_PRODUCTS"] == 4 and result["operational"]["SAFE_BLOCKED"] == 1
def test_removed_product_leaves_local_cache_but_is_not_reported_as_resolved(env_setup, monkeypatch):
hub, state = env_setup
run_job("INITIAL_SYNC", [product("1"), product("2")], state, monkeypatch)
result, _ = run_job("SYNC_INCREMENTAL", [product("1")], state, monkeypatch)
assert state.cached_decision("2") is None and result["operational"]["TOTAL_PRODUCTS"] == 1
assert result["resolved_products"] == [] # no history is auto-resolved/deleted for a product that merely disappeared
def test_every_sync_kind_is_read_only(env_setup, monkeypatch):
hub, state = env_setup
for kind in ("INITIAL_SYNC", "SYNC_INCREMENTAL", "REPROCESS_ALL"):
result, error = run_job(kind, [product("1")], state, monkeypatch)
assert error is None and result["writes_attempted"] == 0 and result["writes_succeeded"] == 0 and result["firebird_writes_enabled"] is False
with pytest.raises(RuntimeError, match="writes are disabled"):
service.open_write_connection({}, AUTHORIZED, firebird_writes=False) # no write-capable connection can even be opened
def test_hub_offline_during_job_fails_the_job_cleanly_no_crash(env_setup, monkeypatch):
hub, state = env_setup
def boom(products): raise ConnectionError("hub down")
hub.classify_batch = boom
result, error = run_job("SYNC_INCREMENTAL", [product("9")], state, monkeypatch)
assert result is None and "ConnectionError" in error # reported as a job failure, the service loop keeps running
def test_firebird_offline_during_job_fails_the_job_cleanly(env_setup, monkeypatch):
hub, state = env_setup
def down(env, codigos=None): raise OSError("Firebird service not running")
monkeypatch.setattr(service, "read_products", down)
result, error = service.execute_job({"job_id": 1, "kind": "SYNC_INCREMENTAL", "agent_id": 2, "payload": {}}, _config(), _FakeExecutor(), state, {})
assert result is None and "OSError" in error
def test_firebird_credentials_come_from_the_dpapi_file_not_agent_env(monkeypatch, tmp_path):
from gti_fiscal_agent import credential_store
path = tmp_path / "firebird.bin"
monkeypatch.setattr(credential_store, "read_credential", lambda p, env=None: json.dumps({"user": "SYSDBA", "password": "s3cret"}) if str(p) == str(path) else (_ for _ in ()).throw(FileNotFoundError()))
env = {"FIREBIRD_DATABASE_PATH": AUTHORIZED, "GTI_FIREBIRD_CREDENTIAL_FILE": str(path)}
service.Config(env)
assert env["FB_USER"] == "SYSDBA" and env["FB_PASSWORD"] == "s3cret"
explicit = {"FIREBIRD_DATABASE_PATH": AUTHORIZED, "FB_PASSWORD": "dev", "FB_USER": "u", "GTI_FIREBIRD_CREDENTIAL_FILE": str(path)}
service.Config(explicit)
assert explicit["FB_PASSWORD"] == "dev" # an explicit env value still wins (dev override)
def test_central_resume_after_pause_makes_agent_poll_again(monkeypatch, tmp_path):
class Stopper: stop = False
stopper = Stopper()
states = iter([False, True]) # paused, then resumed
calls = {"next_job": 0}
class Hub:
def identity(self): return {"agent_id": 2, "installation_id": "00000000-0000-4000-8000-000000000222"}
def heartbeat_metadata(self, payload): return {"processing_enabled": next(states)}
def next_job(self):
calls["next_job"] += 1; stopper.stop = True; return None
monkeypatch.setattr(service, "HubClient", lambda *a, **k: Hub())
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
monkeypatch.setattr(service.time, "sleep", lambda s: None)
service.run(_config(agent_id="2", poll_interval_seconds=0, state_db=str(tmp_path / "a.db")), {"AGENT_ID": "2"}, stopper=stopper)
assert calls["next_job"] == 1 # first cycle paused (no poll), second cycle resumed (polled)