Add 'fiscal/' from commit '102ba84500dbd9655a0471288c3e278652593ca9'
git-subtree-dir: fiscal git-subtree-mainline:d95165fc8fgit-subtree-split:102ba84500
This commit is contained in:
gti
@@ -0,0 +1,582 @@
|
||||
"""Long-running Agent service: heartbeat + poll Hub jobs + execute pipeline.
|
||||
|
||||
startup -> load config -> validate Hub -> validate Firebird -> heartbeat
|
||||
-> poll job -> claim -> execute -> report result -> back to polling.
|
||||
|
||||
No job available: sleep POLL_INTERVAL_SECONDS.
|
||||
Hub unavailable: retry with simple capped backoff; the Agent stays alive.
|
||||
Job fails internally (bad Firebird read, classify error, write refused):
|
||||
the job is reported FAILED with a sanitized error and the Agent keeps polling.
|
||||
SIGINT/SIGTERM: finish the current job (if any) and stop the loop cleanly.
|
||||
|
||||
Written to be runnable as-is today, and easy to later wrap as a Windows
|
||||
Service (a service wrapper would just call main()); no installer here.
|
||||
"""
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_ROOT = Path(__file__).resolve().parents[1]
|
||||
PROJECT_ROOT = AGENT_ROOT.parent
|
||||
sys.path.insert(0, str(AGENT_ROOT / "src"))
|
||||
sys.path.insert(0, str(PROJECT_ROOT)) # ships central/app engine modules read-only (central.app.*)
|
||||
|
||||
from gti_fiscal_agent.client import HubClient # noqa: E402
|
||||
from gti_fiscal_agent import firebird_reader as _firebird_reader, rtc_auto_apply as _rtc_apply, unified_fiscal as _unified # noqa: E402
|
||||
from gti_fiscal_agent.firebird_reader import check_connection, read_products # noqa: E402
|
||||
from gti_fiscal_agent.pipeline import run_pipeline # noqa: E402
|
||||
from gti_fiscal_agent.runner import REFERENCE_SCHEMA_V1, product_fingerprint, to_hub_product # noqa: E402
|
||||
from central.app.operational_state import classify_product_state, summarize_operational_states # noqa: E402
|
||||
from gti_fiscal_agent.state import AgentState # noqa: E402
|
||||
from gti_fiscal_agent import enrichment as _enrichment # noqa: E402
|
||||
|
||||
_MAX_BACKOFF_SECONDS = 60
|
||||
SYNC_JOB_KINDS = ("INITIAL_SYNC", "SYNC_INCREMENTAL", "REPROCESS_ALL") # read-only ERP jobs that refresh coverage
|
||||
|
||||
logger = logging.getLogger("gti_fiscal_agent")
|
||||
|
||||
|
||||
def setup_logging(log_path):
|
||||
"""Rotating local log file. Never logs FB_PASSWORD or AGENT_TOKEN — every
|
||||
log call in this module only references non-secret identifiers/counts."""
|
||||
logger.setLevel(logging.INFO)
|
||||
if logger.handlers:
|
||||
return logger
|
||||
Path(log_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
handler = logging.handlers.RotatingFileHandler(log_path, maxBytes=5_000_000, backupCount=5, encoding="utf-8")
|
||||
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
|
||||
logger.addHandler(handler)
|
||||
logger.addHandler(logging.StreamHandler())
|
||||
return logger
|
||||
|
||||
|
||||
def load_dotenv(path: Path) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
os.environ.setdefault(key.strip(), value.strip())
|
||||
|
||||
|
||||
def load_executor():
|
||||
# The frozen service host has no repo tree; the installer writes
|
||||
# GTI_WRITE_EXECUTOR_PATH pointing at the extracted executor next to
|
||||
# agent.env. Falls back to the dev/repo path for source runs.
|
||||
executor_path = os.environ.get("GTI_WRITE_EXECUTOR_PATH") or (PROJECT_ROOT / "scripts" / "firebird_write_executor.py")
|
||||
spec = importlib.util.spec_from_file_location("firebird_write_executor", executor_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class Config:
|
||||
def __init__(self, env):
|
||||
self.hub_url = env.get("GTI_HUB_URL") or env.get("HUB_URL") or "http://localhost:8000"
|
||||
self.agent_id = env.get("AGENT_ID", "")
|
||||
self.credential_file = env.get("GTI_CREDENTIAL_FILE") or str(Path(os.environ.get("PROGRAMFILES", r"C:\Program Files")) / "GTI" / "FiscalAgent" / "credentials.bin")
|
||||
# The permanent token NEVER lives in agent.env: it is decrypted from
|
||||
# credentials.bin (DPAPI LocalMachine + SYSTEM/Administrators ACL). An
|
||||
# explicit AGENT_TOKEN/AGENT_DEV_TOKEN in env wins (dev override only).
|
||||
self.agent_token = self._resolve_agent_token(env)
|
||||
self.company_id = env.get("COMPANY_ID", "")
|
||||
self.company_uf = env.get("COMPANY_UF", "")
|
||||
_resolve_firebird_credentials(env)
|
||||
self.database_path = env["FIREBIRD_DATABASE_PATH"]
|
||||
# FISCAL_MODE gates writes technically: DRY_RUN can never write, whatever FIREBIRD_WRITES says. Unset = legacy
|
||||
# behaviour (FIREBIRD_WRITES decides). `firebird_writes` below is the EFFECTIVE value used by every write path.
|
||||
requested = env.get("FIREBIRD_WRITES", "0") == "1"
|
||||
self.fiscal_mode = (env.get("FISCAL_MODE") or ("AUTO_APPLY" if requested else "DRY_RUN")).strip().upper()
|
||||
self.requested_writes = requested
|
||||
self.firebird_writes = requested and self.fiscal_mode == "AUTO_APPLY"
|
||||
# Job execution is controlled CENTRALLY (Hub heartbeat: processing_enabled) — there is no
|
||||
# local toggle, so no installation ever needs its agent.env edited to start/stop work.
|
||||
# (A legacy AGENT_JOBS_ENABLED line in an old agent.env is deliberately ignored.)
|
||||
self.agent_jobs_enabled = True
|
||||
self.poll_interval_seconds = float(env.get("POLL_INTERVAL_SECONDS", "5"))
|
||||
# RTC auto-apply is a SEPARATE switch (default OFF): FIREBIRD_WRITES alone never lets the Agent fill cClassTrib.
|
||||
self.rtc_auto_apply_max_per_job = int(env.get("RTC_AUTO_APPLY_MAX_PER_JOB", str(_rtc_apply.DEFAULT_MAX_PER_JOB)))
|
||||
self.agent_version = env.get("AGENT_VERSION", "1.3.9")
|
||||
self.schema_fingerprint = env.get("SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1)
|
||||
self.state_db = env.get("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite"))
|
||||
# Pilot write authorization: both must be explicitly set by the installer for
|
||||
# a real client database. Left unset, both default to the dev-only values,
|
||||
# so a real client path/schema is never silently authorized (see
|
||||
# scripts/firebird_write_executor.py's assert_authorized_write_path).
|
||||
self.authorized_write_database_path = env.get("AUTHORIZED_WRITE_DATABASE_PATH")
|
||||
self.expected_schema_fingerprint = env.get("EXPECTED_SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1)
|
||||
self.log_path = env.get("AGENT_LOG_PATH", str(AGENT_ROOT / "logs" / "agent.log"))
|
||||
|
||||
@staticmethod
|
||||
def _resolve_agent_token(env):
|
||||
"""DPAPI credential from credentials.bin; explicit AGENT_TOKEN env wins
|
||||
(dev override). Fail closed: never log or fabricate a token."""
|
||||
from gti_fiscal_agent.credential_store import install_env_from_credentials
|
||||
|
||||
explicit = env.get("AGENT_TOKEN") or env.get("AGENT_DEV_TOKEN") or ""
|
||||
if explicit:
|
||||
return explicit
|
||||
try:
|
||||
return install_env_from_credentials(env.get("GTI_CREDENTIAL_FILE") or str(
|
||||
Path(os.environ.get("PROGRAMFILES", r"C:\Program Files")) / "GTI" / "FiscalAgent" / "credentials.bin"), env)
|
||||
except Exception as exc: # noqa: BLE001 — fail closed: no token, no heartbeat
|
||||
logger.warning("no usable agent credential: %s", exc)
|
||||
return ""
|
||||
|
||||
|
||||
def _resolve_firebird_credentials(env):
|
||||
"""FB_USER/FB_PASSWORD come from the DPAPI-protected firebird.bin written by Setup
|
||||
(never from a hand-edited agent.env, never sent to the Hub). An explicit FB_PASSWORD in
|
||||
the environment still wins (dev override)."""
|
||||
if env.get("FB_PASSWORD"):
|
||||
return
|
||||
from gti_fiscal_agent.credential_store import read_credential
|
||||
|
||||
path = env.get("GTI_FIREBIRD_CREDENTIAL_FILE") or str(Path(os.environ.get("PROGRAMFILES", "C:/Program Files")) / "GTI" / "FiscalAgent" / "firebird.bin")
|
||||
try:
|
||||
data = json.loads(read_credential(path, env=env))
|
||||
except Exception as exc: # noqa: BLE001 — absent/unreadable: preflight will fail closed with a clear blocker
|
||||
logger.info("no Firebird credential file usable: %s", type(exc).__name__)
|
||||
return
|
||||
env["FB_USER"], env["FB_PASSWORD"] = data.get("user", ""), data.get("password", "")
|
||||
|
||||
|
||||
class GracefulStop:
|
||||
def __init__(self):
|
||||
self.stop = False
|
||||
signal.signal(signal.SIGINT, self._handle)
|
||||
signal.signal(signal.SIGTERM, self._handle)
|
||||
|
||||
def _handle(self, signum, frame):
|
||||
self.stop = True
|
||||
|
||||
|
||||
def build_job_coverage(env, client, products, decisions):
|
||||
"""Company-context-aware coverage over the job's products (read-only). Fails closed when the company context is unresolved."""
|
||||
from gti_fiscal_agent.company_context import mask_cnpj, preflight_company_context
|
||||
from gti_fiscal_agent.coverage import build_refs, compute_coverage
|
||||
from gti_fiscal_agent.firebird_reader import read_evidence
|
||||
ctx = preflight_company_context(env)
|
||||
if ctx["status"] != "PASS":
|
||||
raise RuntimeError("COMPANY_CONTEXT=FAIL: " + "; ".join(ctx.get("reasons") or ["unresolved"]))
|
||||
evidence = read_evidence(env, ctx["company_code"])
|
||||
refs = build_refs(client.get_json, products, ctx["company_uf"])
|
||||
report = compute_coverage(products, decisions, {"uf": ctx["company_uf"], "crt": ctx["company_crt"]}, evidence.get("company_default_csosn"), refs, evidence)
|
||||
report["company_context"] = {"COMPANY_CONTEXT_STATUS": "PASS", "classification": ctx["classification"], "company_code": ctx["company_code"], "company_cnpj": ctx["company_cnpj"], "company_uf": ctx["company_uf"],
|
||||
"company_crt": ctx["company_crt"], "source_of_truth": ctx["source_of_truth"], "confidence": ctx["confidence"], "warnings": ctx["warnings"]}
|
||||
return report
|
||||
|
||||
|
||||
def open_write_connection(env, database_path, firebird_writes=True):
|
||||
if not firebird_writes: # technical guarantee: no write-capable connection exists in DRY_RUN / FIREBIRD_WRITES=0
|
||||
raise RuntimeError("write connection refused: writes are disabled (FISCAL_MODE=DRY_RUN or FIREBIRD_WRITES=0)")
|
||||
import fdb
|
||||
return fdb.connect(
|
||||
database=database_path,
|
||||
user=env["FB_USER"],
|
||||
password=env["FB_PASSWORD"],
|
||||
fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
)
|
||||
|
||||
|
||||
def _unified_fiscal(env, company, ncm_valid=None):
|
||||
"""PIS/COFINS applicability + RTC (only when the ERP has the reform on), read-only (SELECT only). Fail-closed: if it cannot be assessed the job fails
|
||||
instead of reporting a coverage that ignores these fields."""
|
||||
connection = _firebird_reader._connect(env)
|
||||
try:
|
||||
return _unified.assess_from_connection(connection, company, ncm_valid=ncm_valid)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def _fiscal_blocked_products(products, findings):
|
||||
"""One queue entry per (product, blocker). Same shape as every other blocker; `field` is the dashboard category (PIS/COFINS/RTC)."""
|
||||
out = []
|
||||
for product in products:
|
||||
code = str(product.get("CODIGO") or "")
|
||||
found = findings.get(code) or {}
|
||||
for blocker in found.get("blockers") or []:
|
||||
out.append({"company": "", "product_code": code, "gtin": product.get("GTIN") or product.get("EAN_GTIN"), "description": product.get("PRODUTO"), "blocker_type": "BLOCKED_FISCAL_DATA",
|
||||
"field": blocker["category"], "reason_code": blocker["type"], "evidence_summary": {"reason": blocker["reason"], "applicability": found.get("applicability"), **({"human": blocker["human"]} if blocker.get("human") else {})},
|
||||
"input_fingerprint": found.get("input_fingerprint", "")})
|
||||
return out
|
||||
|
||||
|
||||
def _fetch_authorizations(config, job):
|
||||
"""Current central authorization for this installation, straight from the Hub (authenticated). Tests replace this; production never trusts the job's copy for the write itself."""
|
||||
return HubClient(config.hub_url, config.agent_token).authorizations()
|
||||
|
||||
|
||||
def _make_authorizer(config, job, mode, approved_at_claim):
|
||||
"""Re-checks the Hub right before writing (cached for a few seconds): if the mode changed, the product lost its approval, or the Hub cannot be reached, nothing is written (fail closed)."""
|
||||
state = {"at": None, "auth": None}
|
||||
|
||||
def authorize(code):
|
||||
now = time.monotonic()
|
||||
if state["auth"] is None or now - state["at"] > 3.0:
|
||||
try:
|
||||
state["auth"], state["at"] = _fetch_authorizations(config, job) or {}, now
|
||||
except Exception: # noqa: BLE001 - unverifiable authorization is not authorization
|
||||
return "AUTHORIZATION_UNVERIFIABLE"
|
||||
fresh = state["auth"]
|
||||
if fresh.get("rtc_write_mode") != mode:
|
||||
return "AUTHORIZATION_CHANGED"
|
||||
if mode == "APPROVAL_REQUIRED" and str(code) not in {str(c) for c in fresh.get("rtc_approved_codes") or []}:
|
||||
return "AUTHORIZATION_CHANGED"
|
||||
return None
|
||||
return authorize
|
||||
|
||||
|
||||
def _rtc_auto_apply(env, config, job, products, fiscal, company):
|
||||
"""Preview (always) and, only when the Hub-delivered mode permits it, fill CCLASSTRIB for products whose class is PROVEN by an AUTO_APPLY_SAFE rule; also executes
|
||||
Hub-requested rollbacks. Authorization arrives ONLY inside the authenticated job (`authorizations`); there is no local switch. Never raises for a single product."""
|
||||
from datetime import datetime, timezone
|
||||
auth = job.get("authorizations") or {}
|
||||
mode = auth.get("rtc_write_mode") if auth.get("rtc_write_mode") in _rtc_apply.MODES else "DISABLED"
|
||||
approved = {str(c) for c in (auth.get("rtc_approved_codes") or [])}
|
||||
limit = int(getattr(config, "rtc_auto_apply_max_per_job", _rtc_apply.DEFAULT_MAX_PER_JOB))
|
||||
rtc_on = bool(fiscal["context"].get("rtc_enabled"))
|
||||
all_candidates, ready_total = _rtc_apply.select_candidates(fiscal["findings"], products, limit=_rtc_apply.MAX_PREVIEW_CANDIDATES)
|
||||
gate = _rtc_apply.write_gate(mode=mode, capability=True, approved=bool(approved))
|
||||
summary = {"gate": gate, "mode": mode if mode != "DISABLED" else "REPORT_ONLY", "hub_mode": mode, "capability": _rtc_apply.CAPABILITY, "ready_total": ready_total, "max_per_job": limit, "write_attempts": 0, "write_successes": 0,
|
||||
"applied": [], "aborted": [], "aborted_reasons": {}, "rollbacks": [], "allowed_fields": list(_rtc_apply.ALLOWED_RTC_WRITE_FIELDS),
|
||||
"preview": [{k: c[k] for k in ("product_code", "description", "ncm", "current", "proposed", "cst", "rule_id", "basis")} for c in all_candidates] if mode != "DISABLED" and rtc_on else [],
|
||||
"preview_truncated": ready_total > len(all_candidates)}
|
||||
if not rtc_on:
|
||||
return summary
|
||||
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
rollbacks = [r for r in (auth.get("rtc_rollbacks") or []) if isinstance(r, dict)]
|
||||
if mode == "ENABLED":
|
||||
wanted = all_candidates[:limit]
|
||||
elif mode == "APPROVAL_REQUIRED":
|
||||
wanted = [c for c in all_candidates if c["product_code"] in approved][:limit]
|
||||
else:
|
||||
wanted = []
|
||||
if not (rollbacks or (gate["allowed"] and wanted)):
|
||||
return summary
|
||||
connection = open_write_connection(env, config.database_path)
|
||||
try:
|
||||
if rollbacks:
|
||||
summary["rollbacks"] = _rtc_apply.rollback_records(connection, rollbacks, now=now, job_id=job.get("job_id"))["rollbacks"]
|
||||
if gate["allowed"] and wanted:
|
||||
outcome = _rtc_apply.apply_candidates(connection, wanted, company=company or {}, decision={"local_catalog_hash": fiscal["context"].get("local_catalog_hash"), "origin": f"HUB_MODE:{mode}"}, now=now, job_id=job.get("job_id"), authorize=_make_authorizer(config, job, mode, approved))
|
||||
summary.update({k: outcome[k] for k in ("applied", "aborted", "aborted_reasons", "write_attempts", "write_successes")})
|
||||
finally:
|
||||
connection.close()
|
||||
return summary
|
||||
|
||||
|
||||
def execute_job(job, config, executor, state, env):
|
||||
"""Run the pipeline for one claimed job. Never raises for job-domain
|
||||
failures (bad Firebird read, classify error) — returns (result_summary, error)."""
|
||||
kind = job["kind"]
|
||||
payload = job.get("payload") or {}
|
||||
try:
|
||||
if config.firebird_writes:
|
||||
# Layered, per-installation authorization: the path must match this
|
||||
# installation's own explicitly configured authorized_write_database_path
|
||||
# (defaults to the dev-only constant, so a real client path is refused
|
||||
# unless the installer deliberately set it), AND the live schema
|
||||
# fingerprint must match what this installation was configured to expect.
|
||||
executor.assert_authorized_write_path(config.database_path, authorized_path=config.authorized_write_database_path)
|
||||
executor.assert_schema_fingerprint_authorized(config.schema_fingerprint, config.expected_schema_fingerprint)
|
||||
|
||||
codigos = payload.get("codigos") if kind == "REPROCESS_PRODUCTS" else None
|
||||
products = read_products(env, codigos=codigos)
|
||||
logger.info("job %s: read %d product(s) from Firebird", job.get("job_id"), len(products))
|
||||
|
||||
client = HubClient(config.hub_url, config.agent_token)
|
||||
sync_kind = kind in SYNC_JOB_KINDS
|
||||
cache = sync_kind and hasattr(state, "remember_decision")
|
||||
# SYNC_INCREMENTAL = delta: only products whose source fingerprint changed (or are new) are
|
||||
# re-classified by the Hub; INITIAL_SYNC / REPROCESS_ALL / REPROCESS_PRODUCTS process everything they read.
|
||||
if kind == "SYNC_INCREMENTAL" and cache:
|
||||
to_process = [p for p in products if state.cached_fingerprint(p["CODIGO"]) != product_fingerprint(p)]
|
||||
else:
|
||||
to_process = products
|
||||
decisions = client.classify_batch([to_hub_product(p) for p in to_process])
|
||||
|
||||
write_connection = None
|
||||
if config.firebird_writes:
|
||||
write_connection = open_write_connection(env, config.database_path)
|
||||
try:
|
||||
summary = run_pipeline(to_process, decisions, state, config.firebird_writes, write_connection, executor, config.database_path, authorized_path=config.authorized_write_database_path)
|
||||
finally:
|
||||
if write_connection is not None:
|
||||
write_connection.close()
|
||||
|
||||
operational = coverage = fiscal_context = rtc_apply = None
|
||||
fiscal = None
|
||||
if sync_kind:
|
||||
if cache:
|
||||
for product, decision in zip(to_process, decisions):
|
||||
state.remember_decision(product["CODIGO"], product_fingerprint(product), decision)
|
||||
state.forget_missing([p["CODIGO"] for p in products])
|
||||
all_decisions = [state.cached_decision(p["CODIGO"]) or {} for p in products]
|
||||
else:
|
||||
all_decisions = decisions
|
||||
coverage = build_job_coverage(env, client, products, all_decisions)
|
||||
company = {"crt": ((coverage or {}).get("company_context") or {}).get("company_crt")}
|
||||
ncm_ok = {str(p["CODIGO"]): (d or {}).get("status") == "EXACT_NCM" for p, d in zip(products, all_decisions)}
|
||||
fiscal = _unified_fiscal(env, company, ncm_ok)
|
||||
rtc_apply = _rtc_auto_apply(env, config, job, products, fiscal, company)
|
||||
if rtc_apply["write_successes"] or any(r.get("outcome") == "ROLLED_BACK" for r in rtc_apply["rollbacks"]):
|
||||
fiscal = _unified_fiscal(env, company, ncm_ok) # coverage must reflect the fields that were just filled
|
||||
findings = fiscal["findings"]
|
||||
product_states = [classify_product_state(p, d, fiscal=findings.get(str(p["CODIGO"]))) for p, d in zip(products, all_decisions)]
|
||||
operational = summarize_operational_states(product_states)
|
||||
fiscal_context = {**fiscal["context"], "evaluated_all": kind != "REPROCESS_PRODUCTS"}
|
||||
else:
|
||||
# REPROCESS_PRODUCTS (and any other non-sync kind) used to hardcode company=None here - not a missing feature, a real gap: the SAME
|
||||
# deterministic, fail-closed company_context.resolve() every sync job already uses is cheap (no full-table scan) and applies just as
|
||||
# well to a one-off reprocess job. `company` stays None only when the company genuinely cannot be resolved yet - never a silent guess.
|
||||
try:
|
||||
company = {"crt": _firebird_reader.read_company_context(env).get("crt")}
|
||||
except Exception:
|
||||
company = None
|
||||
fiscal = _unified_fiscal(env, company, {str(p["CODIGO"]): (d or {}).get("status") == "EXACT_NCM" for p, d in zip(to_process, decisions)})
|
||||
fiscal_blocked = _fiscal_blocked_products(products, fiscal["findings"])
|
||||
fiscal_codes = {b["product_code"] for b in fiscal_blocked}
|
||||
summary["blocked_products"] = summary["blocked_products"] + fiscal_blocked
|
||||
summary["resolved_products"] = [c for c in summary["resolved_products"] if str(c) not in fiscal_codes]
|
||||
result_summary = {
|
||||
"coverage": coverage,
|
||||
"operational": operational,
|
||||
"fiscal_context": fiscal_context,
|
||||
"rtc_auto_apply": rtc_apply,
|
||||
"fiscal_mode": getattr(config, "fiscal_mode", None),
|
||||
"release_version": env.get("AGENT_VERSION") or "1.3.9",
|
||||
"products_total": len(products),
|
||||
"products_changed": len(to_process),
|
||||
"products_processed": summary["processed"],
|
||||
"blocked_products": summary["blocked_products"],
|
||||
"resolved_products": summary["resolved_products"],
|
||||
"writes_attempted": summary["writes_attempted"],
|
||||
"writes_succeeded": summary["writes_succeeded"],
|
||||
"writes_failed": summary["writes_failed"],
|
||||
"no_ops": summary["no_ops"],
|
||||
"firebird_writes_enabled": config.firebird_writes,
|
||||
"agent_id": job["agent_id"],
|
||||
"schema_fingerprint": config.schema_fingerprint,
|
||||
}
|
||||
logger.info(
|
||||
"job %s finished: processed=%d writes_attempted=%d writes_succeeded=%d writes_failed=%d no_ops=%d",
|
||||
job.get("job_id"), summary["processed"], summary["writes_attempted"], summary["writes_succeeded"], summary["writes_failed"], summary["no_ops"],
|
||||
)
|
||||
if summary["writes_failed"] > 0:
|
||||
return result_summary, f"{summary['writes_failed']} write(s) failed guard checks"
|
||||
return result_summary, None
|
||||
except executor.WriteRefused as error:
|
||||
logger.warning("job %s refused: %s", job.get("job_id"), error)
|
||||
return None, f"write refused: {error}"
|
||||
except Exception as error: # noqa: BLE001 — job-domain isolation boundary
|
||||
logger.warning("job %s failed internally: %s: %s", job.get("job_id"), type(error).__name__, error)
|
||||
return None, f"{type(error).__name__}: {error}"
|
||||
|
||||
|
||||
BUSY_HEARTBEAT_SECONDS = 20
|
||||
|
||||
|
||||
def _busy_heartbeats(client, installation_id, config, job_id, done):
|
||||
while not done.wait(BUSY_HEARTBEAT_SECONDS):
|
||||
try:
|
||||
client.heartbeat_metadata({"installation_id": installation_id, "status": "BUSY", "schema_fingerprint": config.schema_fingerprint, "agent_version": config.agent_version, "last_job_id": job_id})
|
||||
except Exception: # noqa: BLE001 — Hub hiccup mid-job must not affect the job
|
||||
pass
|
||||
|
||||
|
||||
def resolve_installation_id(config, env, state, client):
|
||||
"""Recover and persist the Hub-bound identity; never invent one for an Agent."""
|
||||
persisted = state.stored_installation_id()
|
||||
configured = (env.get("INSTALLATION_ID") or "").strip() or None
|
||||
if persisted is not None:
|
||||
return state.installation_id(configured)
|
||||
if configured is not None:
|
||||
return state.installation_id(configured)
|
||||
|
||||
configured_agent_id = str(env.get("AGENT_ID") or config.agent_id or "").strip()
|
||||
if not configured_agent_id:
|
||||
raise RuntimeError("AGENT_ID is required to recover installation identity")
|
||||
identity = client.identity()
|
||||
if str(identity.get("agent_id") or "") != configured_agent_id:
|
||||
raise RuntimeError("authenticated identity does not match configured Agent")
|
||||
installation_id = identity.get("installation_id")
|
||||
if not installation_id:
|
||||
raise RuntimeError("authenticated identity response has no installation_id")
|
||||
return state.installation_id(str(installation_id))
|
||||
|
||||
|
||||
def _company_context_for_heartbeat(env, cached):
|
||||
"""Resolves company UF/CRT via the SAME deterministic, fail-closed resolver already used for coverage/RTC (`company_context.resolve`,
|
||||
called through `preflight_company_context`) - never a second, separate guess. Previously the heartbeat only ever sent `COMPANY_UF` from an
|
||||
operator-set env var and never sent CRT at all, so `Installation.company_crt` stayed permanently empty in the Hub even though the Agent
|
||||
already proves CRT from C000004/EMISSOR for every coverage job. Resolves once and caches: never re-queries Firebird once solid, retries only
|
||||
while still unresolved (e.g. the database was not yet reachable when the service started)."""
|
||||
if cached is not None:
|
||||
return cached, None
|
||||
try:
|
||||
from gti_fiscal_agent.company_context import preflight_company_context
|
||||
result = preflight_company_context(env)
|
||||
except Exception as error:
|
||||
return None, f"company context not resolved yet: {error}"
|
||||
if result.get("status") != "PASS":
|
||||
return None, f"company context not resolved yet: {result.get('classification')} ({'; '.join(result.get('reasons', []))})"
|
||||
return result, None
|
||||
|
||||
|
||||
def _start_enrichment(config, env, client, stopper):
|
||||
"""New-product enrichment worker (own thread, own SQLite file). It gates ITSELF on the Hub's per-installation features: DISABLED / not configured = zero Firebird activity."""
|
||||
if not hasattr(client, "resolve_product"):
|
||||
return None
|
||||
store = _enrichment.EnrichmentStore(Path(config.state_db).with_name("enrichment_state.db"))
|
||||
worker = _enrichment.EnrichmentWorker(hub=client, store=store, read_conn=lambda: _firebird_reader._connect(env, True), write_conn=lambda: open_write_connection(env, config.database_path),
|
||||
create_reprocess=lambda codes: client.create_job("REPROCESS_PRODUCTS", {"codigos": list(codes), "reason": "ENRICHMENT"}))
|
||||
threading.Thread(target=worker.run_forever, args=(stopper,), name="product-enrichment", daemon=True).start()
|
||||
return worker
|
||||
|
||||
|
||||
def run(config, env, *, max_iterations=None, stopper=None):
|
||||
"""`stopper` defaults to a signal-based GracefulStop (SIGINT/SIGTERM), which
|
||||
only works from the main thread. A host that manages its own lifecycle
|
||||
(e.g. the Windows Service wrapper, which stops the loop via the Service
|
||||
Control Manager instead of OS signals) should pass its own stop object —
|
||||
anything with a `.stop` boolean attribute — and flip it from any thread."""
|
||||
executor = load_executor()
|
||||
state = AgentState(config.state_db)
|
||||
client = HubClient(config.hub_url, config.agent_token)
|
||||
installation_id = resolve_installation_id(config, env, state, client)
|
||||
stopper = stopper if stopper is not None else GracefulStop()
|
||||
if max_iterations is None:
|
||||
_start_enrichment(config, env, client, stopper)
|
||||
|
||||
backoff = 1
|
||||
was_down = False
|
||||
iterations = 0
|
||||
last_job_id = None
|
||||
company_ctx = None
|
||||
while not stopper.stop:
|
||||
if max_iterations is not None and iterations >= max_iterations:
|
||||
break
|
||||
iterations += 1
|
||||
|
||||
try:
|
||||
company_ctx, reason = _company_context_for_heartbeat(env, company_ctx)
|
||||
if reason:
|
||||
logger.debug(reason)
|
||||
heartbeat_payload = {
|
||||
"installation_id": installation_id, "status": "IDLE", "schema_fingerprint": config.schema_fingerprint,
|
||||
"agent_version": config.agent_version, "last_job_id": last_job_id, "display_name": env.get("AGENT_DISPLAY_NAME", ""),
|
||||
"customer_code": env.get("CUSTOMER_CODE"), "hostname": env.get("AGENT_HOSTNAME") or os.environ.get("COMPUTERNAME"),
|
||||
"windows_version": env.get("WINDOWS_VERSION"), "company_code": config.company_id, "company_uf": (company_ctx or {}).get("company_uf") or config.company_uf,
|
||||
"company_crt": (company_ctx or {}).get("company_crt"), "company_contribuinte_ipi": (company_ctx or {}).get("company_contribuinte_ipi"),
|
||||
"database_path": config.database_path, "capabilities": {"heartbeat": True, "jobs": True, "local_logs": True, "RTC_CCLASSTRIB_WRITE_V1": True, "PRODUCT_ENRICHMENT_WRITE_V1": True},
|
||||
"fiscal_engine_version": env.get("FISCAL_ENGINE_VERSION", config.agent_version), "dataset_versions": {},
|
||||
}
|
||||
if hasattr(client, "heartbeat_metadata"):
|
||||
hb_response = client.heartbeat_metadata(heartbeat_payload)
|
||||
else:
|
||||
hb_response = client.heartbeat("IDLE", schema_fingerprint=config.schema_fingerprint, agent_version=config.agent_version, last_job_id=last_job_id)
|
||||
processing_enabled = not (isinstance(hb_response, dict) and hb_response.get("processing_enabled") is False)
|
||||
if was_down:
|
||||
logger.info("reconnected to Hub after %ss backoff", backoff)
|
||||
was_down = False
|
||||
backoff = 1
|
||||
except Exception as error:
|
||||
logger.warning("heartbeat failed, Hub unreachable: %s (retrying in %ss)", error, backoff)
|
||||
was_down = True
|
||||
time.sleep(backoff)
|
||||
backoff = min(backoff * 2, _MAX_BACKOFF_SECONDS)
|
||||
continue
|
||||
|
||||
if not processing_enabled: # central pause: heartbeat keeps flowing, no job is claimed or executed
|
||||
time.sleep(config.poll_interval_seconds)
|
||||
continue
|
||||
|
||||
try:
|
||||
job = client.next_job()
|
||||
except Exception as error:
|
||||
logger.warning("poll failed, Hub unreachable: %s (retrying in %ss)", error, backoff)
|
||||
was_down = True
|
||||
time.sleep(backoff)
|
||||
backoff = min(backoff * 2, _MAX_BACKOFF_SECONDS)
|
||||
continue
|
||||
|
||||
if job is None:
|
||||
time.sleep(config.poll_interval_seconds)
|
||||
continue
|
||||
|
||||
last_job_id = job.get("job_id")
|
||||
logger.info("job %s received: kind=%s", job.get("job_id"), job["kind"])
|
||||
try:
|
||||
if hasattr(client, "heartbeat_metadata"):
|
||||
client.heartbeat_metadata({"installation_id": installation_id, "status": "BUSY", "schema_fingerprint": config.schema_fingerprint, "agent_version": config.agent_version, "last_job_id": last_job_id})
|
||||
else:
|
||||
client.heartbeat("BUSY", schema_fingerprint=config.schema_fingerprint, agent_version=config.agent_version, last_job_id=last_job_id)
|
||||
except Exception:
|
||||
pass # best-effort; the job itself still proceeds
|
||||
|
||||
# A full sync can take minutes: keep the heartbeat alive (BUSY) from a side thread so the
|
||||
# panel never shows a working Agent as OFFLINE. The side thread only talks to the Hub
|
||||
# (no SQLite/Firebird), so it cannot interfere with the job.
|
||||
done = threading.Event()
|
||||
beat = threading.Thread(target=_busy_heartbeats, args=(client, installation_id, config, last_job_id, done), daemon=True)
|
||||
beat.start()
|
||||
try:
|
||||
result_summary, error = execute_job(job, config, executor, state, env)
|
||||
finally:
|
||||
done.set()
|
||||
try:
|
||||
if error is None:
|
||||
client.complete_job(job.get("job_id"), result_summary)
|
||||
logger.info("job %s reported COMPLETED", job.get("job_id"))
|
||||
else:
|
||||
client.fail_job(job.get("job_id"), error)
|
||||
logger.warning("job %s reported FAILED: %s", job.get("job_id"), error)
|
||||
except Exception as report_error:
|
||||
logger.warning("could not report job %s result to Hub: %s", job.get("job_id"), report_error)
|
||||
|
||||
|
||||
def enforce_startup_preflight(env):
|
||||
"""Full read-only preflight at startup (Hub, auth, Firebird, capabilities, company context, guards). Fail closed."""
|
||||
from gti_fiscal_agent.preflight import run_preflight
|
||||
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.3.9")
|
||||
logger.info("startup preflight: %s", json.dumps({k: report[k] for k in ("PREFLIGHT", "blockers", "fiscal_mode", "effective_firebird_writes")}, ensure_ascii=False))
|
||||
print(json.dumps({"PREFLIGHT": report["PREFLIGHT"], "blockers": report["blockers"]}, ensure_ascii=False))
|
||||
if report["PREFLIGHT"] != "PASS":
|
||||
logger.error("startup aborted: preflight failed (fail closed): %s", report["blockers"])
|
||||
raise SystemExit(2)
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--max-iterations", type=int, default=None, help="stop after N poll cycles (used by tests); omit to run forever")
|
||||
args = parser.parse_args()
|
||||
|
||||
load_dotenv(Path(os.environ.get("GTI_AGENT_ENV_FILE") or PROJECT_ROOT / ".env"))
|
||||
config = Config(os.environ)
|
||||
setup_logging(config.log_path)
|
||||
|
||||
logger.info("startup: agent_id=%s company_id=%s agent_version=%s firebird_writes=%s", config.agent_id, config.company_id, config.agent_version, config.firebird_writes)
|
||||
hub_ok = HubClient(config.hub_url, config.agent_token).health()
|
||||
firebird_ok = check_connection(os.environ)
|
||||
logger.info("startup checks: hub_ok=%s firebird_ok=%s", hub_ok, firebird_ok)
|
||||
enforce_startup_preflight(os.environ)
|
||||
|
||||
run(config, os.environ, max_iterations=args.max_iterations)
|
||||
logger.info("shutdown: stopped cleanly")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user