gti
7e83d17533
field_contract.resolve_product was only exercised by its own unit tests. /v1/products/resolve now calls it for every product whose NCM resolves, merging CFOP/CST/CSOSN/PIS-COFINS/CST_IPI/SITUACAO_TRIBUTARIA/IAT/IPPT decisions alongside CLASSIFICACAO_FISCAL/CEST/IPI - reusing the same engine, the same FiscalContextEntry table the blocker-resolution workflow already writes to, no duplicated rules. Closes three real transport/discovery gaps found while wiring it in: - Agent heartbeat never sent company_crt (env-var UF only) or company_contribuinte_ipi, so Installation.company_crt stayed permanently empty even though company_context.resolve() already proves both from C000004/EMISSOR for every coverage job. Resolved once, cached, retried only while unresolved. Migration 0032 adds Installation.company_contribuinte_ipi (NULL=unknown, never coerced to "N"). - The Agent's enrichment payload only ever reported 3 old fields (CLASSIFICACAO_FISCAL/CEST/IPI) as "current", so the Hub could never see the product's own CFOP/CST/CSOSN/... nor the classification columns (TIPO/PROD_COZINHA/PRODUTO_SERVICO/CODGRUPO) infer_role needs - separated into a read-only CURRENT_REPORT_FIELDS list, the write allowlist (PRODUCT_ENRICHMENT_WRITE_V1) untouched. - Added a targeted, product_code-scoped evidence query (firebird_reader.read_product_evidence: recipes/composed/purchases/stock, indexed WHERE, never a full-table scan) so st_withheld/product_role can be proven for a single ad-hoc resolve, not only during a full sync job. Verified read-only against the real database (D:\PROJETOS\GTISISTEMAS\Banco\GTI.FDB): CRT=1, CONTRIBUINTE_IPI=N, company_context.resolve() reaches HIGH confidence; infer_role correctly resolved RESALE/OWN_PRODUCTION for 5/8 sample real products with zero operator questions. 1121 tests passed (tests + central/tests + agent/tests), 0 failed. Version 1.3.7 (single GTI_FISCAL_VERSION shared by Hub and Agent per central/app/version.py's own design - never independently versioned).
451 lines
18 KiB
Python
451 lines
18 KiB
Python
import importlib.util
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
SCRIPT = Path(__file__).parents[1] / "scripts" / "run_agent_service.py"
|
|
AUTHORIZED = r"D:\PROJETOS\GTISISTEMAS\Projeto\TESTESOFICIAIS.FDB"
|
|
|
|
|
|
def load_module():
|
|
spec = importlib.util.spec_from_file_location("run_agent_service", SCRIPT)
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
service = load_module()
|
|
|
|
|
|
class _FakeState:
|
|
def __init__(self):
|
|
self.logged = []
|
|
self.identity = None
|
|
|
|
def stored_installation_id(self):
|
|
return self.identity
|
|
|
|
def installation_id(self, candidate=None):
|
|
if self.identity and candidate and self.identity != candidate:
|
|
raise ValueError("configured installation_id conflicts with persisted identity")
|
|
if self.identity is None:
|
|
if not candidate:
|
|
raise ValueError("installation_id is required")
|
|
self.identity = candidate
|
|
return self.identity
|
|
|
|
def log_ipi_write(self, *args):
|
|
self.logged.append(args)
|
|
|
|
|
|
class _FakeExecutor:
|
|
"""Mirrors the real firebird_write_executor's public surface used by execute_job."""
|
|
class WriteRefused(Exception):
|
|
pass
|
|
|
|
def __init__(self, path_guard_ok=True, updates=None, fingerprint_guard_ok=True):
|
|
self.path_guard_ok = path_guard_ok
|
|
self.updates = updates or {}
|
|
self.fingerprint_guard_ok = fingerprint_guard_ok
|
|
|
|
def assert_authorized_write_path(self, path, authorized_path=None):
|
|
if not self.path_guard_ok:
|
|
raise self.WriteRefused(f"path is not the authorized write target: {path}")
|
|
|
|
def assert_schema_fingerprint_authorized(self, observed, expected):
|
|
if not self.fingerprint_guard_ok or observed != expected:
|
|
raise self.WriteRefused(f"schema fingerprint mismatch: expected {expected!r}, observed {observed!r}")
|
|
|
|
def execute_guarded_update(self, connection, *, database_path, proposal, now, authorized_path=None):
|
|
code = proposal["product_code"]
|
|
if code in self.updates:
|
|
raise self.WriteRefused(self.updates[code])
|
|
record = type("Record", (), {
|
|
"before_value": proposal["current_value"],
|
|
"after_value": proposal["proposed_value"],
|
|
"rowcount": 1,
|
|
})()
|
|
return record
|
|
|
|
|
|
def _config(**overrides):
|
|
base = dict(
|
|
hub_url="http://localhost:8000", agent_id="a1", agent_token="tok", company_id="c1", company_uf="BA",
|
|
database_path=AUTHORIZED, firebird_writes=False, poll_interval_seconds=0.01,
|
|
agent_version="0.1.0", schema_fingerprint="fp", state_db=":memory:", agent_jobs_enabled=True,
|
|
authorized_write_database_path=None, expected_schema_fingerprint="fp",
|
|
)
|
|
base.update(overrides)
|
|
return type("Config", (), base)()
|
|
|
|
|
|
def test_config_defaults_from_env():
|
|
env = {"FIREBIRD_DATABASE_PATH": AUTHORIZED}
|
|
config = service.Config(env)
|
|
assert config.firebird_writes is False
|
|
assert config.hub_url == "http://localhost:8000"
|
|
assert config.poll_interval_seconds == 5.0
|
|
assert config.agent_jobs_enabled is True
|
|
|
|
|
|
def test_local_agent_jobs_toggle_is_ignored_control_is_central():
|
|
# Zero-touch: a stale AGENT_JOBS_ENABLED=0 left in an old agent.env must not silence the Agent;
|
|
# only the Hub (heartbeat processing_enabled) can pause it.
|
|
config = service.Config({"FIREBIRD_DATABASE_PATH": AUTHORIZED, "AGENT_JOBS_ENABLED": "0"})
|
|
|
|
assert config.agent_jobs_enabled is True
|
|
|
|
|
|
def test_config_reads_gti_hub_url_and_writes_flag():
|
|
env = {"FIREBIRD_DATABASE_PATH": AUTHORIZED, "GTI_HUB_URL": "http://hub.internal", "FIREBIRD_WRITES": "1"}
|
|
config = service.Config(env)
|
|
assert config.hub_url == "http://hub.internal"
|
|
assert config.firebird_writes is True
|
|
|
|
|
|
def test_identity_recovery_persists_existing_hub_installation_id(tmp_path):
|
|
config = _config(agent_id="2")
|
|
state = service.AgentState(tmp_path / "agent.db")
|
|
installation_id = "00000000-0000-4000-8000-000000000222"
|
|
|
|
class FakeIdentityClient:
|
|
calls = 0
|
|
|
|
def identity(self):
|
|
self.calls += 1
|
|
return {"agent_id": 2, "installation_id": installation_id}
|
|
|
|
client = FakeIdentityClient()
|
|
recovered = service.resolve_installation_id(config, {}, state, client)
|
|
|
|
assert recovered == installation_id
|
|
assert state.installation_id() == installation_id
|
|
assert client.calls == 1
|
|
|
|
restarted = service.resolve_installation_id(config, {}, state, client)
|
|
assert restarted == installation_id
|
|
assert client.calls == 1
|
|
|
|
|
|
def test_identity_recovery_fails_closed_for_another_agent_or_missing_id(tmp_path):
|
|
config = _config(agent_id="2")
|
|
|
|
class FakeIdentityClient:
|
|
def __init__(self, payload):
|
|
self.payload = payload
|
|
def identity(self):
|
|
return self.payload
|
|
|
|
state = service.AgentState(tmp_path / "agent.db")
|
|
with pytest.raises(RuntimeError, match="identity does not match configured Agent"):
|
|
service.resolve_installation_id(
|
|
config,
|
|
{},
|
|
state,
|
|
FakeIdentityClient({"agent_id": 3, "installation_id": "00000000-0000-4000-8000-000000000333"}),
|
|
)
|
|
assert state.connection.execute("SELECT COUNT(*) FROM installation_identity").fetchone()[0] == 0
|
|
|
|
with pytest.raises(RuntimeError, match="identity response has no installation_id"):
|
|
service.resolve_installation_id(config, {}, state, FakeIdentityClient({"agent_id": 2}))
|
|
assert state.connection.execute("SELECT COUNT(*) FROM installation_identity").fetchone()[0] == 0
|
|
|
|
|
|
def test_path_guard_fails_closed_before_any_write(monkeypatch):
|
|
config = _config(firebird_writes=True, database_path=r"D:\PRODUCAO\ERP.FDB")
|
|
executor = _FakeExecutor(path_guard_ok=False)
|
|
state = _FakeState()
|
|
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
|
|
|
called = {"read_products": False}
|
|
|
|
def fake_read_products(env, codigos=None):
|
|
called["read_products"] = True
|
|
return []
|
|
|
|
monkeypatch.setattr(service, "read_products", fake_read_products)
|
|
result, error = service.execute_job(job, config, executor, state, {})
|
|
assert result is None
|
|
assert "not the authorized write target" in error
|
|
assert called["read_products"] is False # never even reads Firebird once the guard refuses
|
|
|
|
|
|
def test_pilot_write_blocked_when_schema_fingerprint_does_not_match(monkeypatch):
|
|
# A real client install where the live schema fingerprint drifted from what
|
|
# this Agent installation was configured/registered to expect.
|
|
config = _config(firebird_writes=True, schema_fingerprint="observed-live-fp", expected_schema_fingerprint="registered-fp")
|
|
executor = _FakeExecutor()
|
|
state = _FakeState()
|
|
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
|
|
|
called = {"read_products": False}
|
|
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: called.update(read_products=True) or [])
|
|
|
|
result, error = service.execute_job(job, config, executor, state, {})
|
|
assert result is None
|
|
assert "schema fingerprint mismatch" in error
|
|
assert called["read_products"] is False
|
|
|
|
|
|
def test_pilot_write_allowed_with_explicit_authorized_path_and_matching_fingerprint(monkeypatch):
|
|
client_path = r"D:\ClienteX\ERP\PRODUCAO.FDB"
|
|
config = _config(
|
|
firebird_writes=True, database_path=client_path, authorized_write_database_path=client_path,
|
|
schema_fingerprint="fp", expected_schema_fingerprint="fp",
|
|
)
|
|
products = [{"CODIGO": "000026", "PRODUTO": "x", "CLASSIFICACAO_FISCAL": "03055100", "IPI": None}]
|
|
decisions = [{
|
|
"status": "EXACT_NCM", "ipi_status": "IPI_RESOLVED",
|
|
"classification": {"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.25}},
|
|
"dataset_versions": {"tipi_ipi": "v1"},
|
|
}]
|
|
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: products)
|
|
|
|
class FakeClient:
|
|
def __init__(self, *a, **k): pass
|
|
def classify_batch(self, items): return decisions
|
|
monkeypatch.setattr(service, "HubClient", FakeClient)
|
|
fake_connection = type("Conn", (), {"close": lambda self: None})()
|
|
monkeypatch.setattr(service, "open_write_connection", lambda env, path: fake_connection)
|
|
|
|
executor = _FakeExecutor()
|
|
state = _FakeState()
|
|
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
|
|
|
result, error = service.execute_job(job, config, executor, state, {})
|
|
assert error is None
|
|
assert result["writes_succeeded"] == 1
|
|
|
|
|
|
def test_execute_job_isolates_internal_errors_without_raising(monkeypatch):
|
|
config = _config()
|
|
|
|
def boom(env, codigos=None):
|
|
raise RuntimeError("firebird unavailable")
|
|
|
|
monkeypatch.setattr(service, "read_products", boom)
|
|
executor = _FakeExecutor()
|
|
state = _FakeState()
|
|
job = {"kind": "REPROCESS_ALL", "agent_id": 1, "payload": {}}
|
|
|
|
result, error = service.execute_job(job, config, executor, state, {})
|
|
assert result is None
|
|
assert "firebird unavailable" in error
|
|
|
|
|
|
def test_execute_job_success_summarizes_pipeline(monkeypatch):
|
|
config = _config()
|
|
products = [{"CODIGO": "000026", "PRODUTO": "x", "CLASSIFICACAO_FISCAL": "03055100", "IPI": None}]
|
|
decisions = [{
|
|
"status": "EXACT_NCM", "ipi_status": "IPI_RESOLVED",
|
|
"classification": {"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.25}},
|
|
"dataset_versions": {"tipi_ipi": "v1"},
|
|
}]
|
|
|
|
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: products)
|
|
|
|
class FakeClient:
|
|
def __init__(self, *a, **k): pass
|
|
def classify_batch(self, items): return decisions
|
|
monkeypatch.setattr(service, "HubClient", FakeClient)
|
|
|
|
executor = _FakeExecutor()
|
|
state = _FakeState()
|
|
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
|
|
|
result, error = service.execute_job(job, config, executor, state, {})
|
|
assert error is None
|
|
assert result["products_processed"] == 1
|
|
assert result["writes_attempted"] == 0 # firebird_writes disabled by default in _config()
|
|
|
|
|
|
class _FakeHubForLoop:
|
|
"""Simulates: heartbeat flaky twice then OK, one job then nothing, and captures completions."""
|
|
|
|
def __init__(self):
|
|
self.heartbeat_calls = 0
|
|
self.jobs = [{"job_id": 1, "kind": "REPROCESS_ALL", "agent_id": 1, "payload": {}}]
|
|
self.completed = []
|
|
self.failed = []
|
|
|
|
def heartbeat(self, *a, **k):
|
|
self.heartbeat_calls += 1
|
|
if self.heartbeat_calls == 1:
|
|
raise ConnectionError("hub down")
|
|
return {}
|
|
|
|
def next_job(self):
|
|
return self.jobs.pop(0) if self.jobs else None
|
|
|
|
def complete_job(self, job_id, result_summary):
|
|
self.completed.append((job_id, result_summary))
|
|
|
|
def fail_job(self, job_id, error):
|
|
self.failed.append((job_id, error))
|
|
|
|
def classify_batch(self, items):
|
|
return []
|
|
|
|
|
|
class _ExternalStopFlag:
|
|
"""Mirrors the Windows Service wrapper's stop object: no signal registration,
|
|
just a plain flag flippable from another thread."""
|
|
def __init__(self):
|
|
self.stop = False
|
|
|
|
|
|
def test_run_sends_heartbeat_but_never_polls_when_paused_centrally(monkeypatch, tmp_path):
|
|
stopper = _ExternalStopFlag()
|
|
fake_hub = _FakeHubForLoop()
|
|
installation_id = "00000000-0000-4000-8000-000000000222"
|
|
calls = {"identity": 0, "next_job": 0, "heartbeat": 0, "heartbeat_installation_id": None}
|
|
|
|
def identity():
|
|
calls["identity"] += 1
|
|
return {"agent_id": 2, "installation_id": installation_id}
|
|
|
|
def heartbeat_metadata(payload):
|
|
calls["heartbeat"] += 1
|
|
calls["heartbeat_installation_id"] = payload["installation_id"]
|
|
stopper.stop = True
|
|
return {"agent_id": 2, "processing_enabled": False} # paused centrally by the Hub
|
|
|
|
def forbidden_next_job():
|
|
calls["next_job"] += 1
|
|
raise AssertionError("processing is paused centrally; next_job must not be called")
|
|
|
|
fake_hub.identity = identity
|
|
fake_hub.heartbeat_metadata = heartbeat_metadata
|
|
fake_hub.next_job = forbidden_next_job
|
|
monkeypatch.setattr(service, "HubClient", lambda *a, **k: fake_hub)
|
|
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
|
monkeypatch.setattr(service.time, "sleep", lambda _seconds: None)
|
|
|
|
config = _config(
|
|
agent_id="2",
|
|
poll_interval_seconds=0,
|
|
state_db=str(tmp_path / "agent.db"),
|
|
)
|
|
env = {
|
|
"AGENT_ID": "2",
|
|
"FIREBIRD_DATABASE_PATH": AUTHORIZED,
|
|
"FIREBIRD_WRITES": "0",
|
|
"FISCAL_MODE": "DRY_RUN",
|
|
}
|
|
|
|
service.run(config, env, stopper=stopper)
|
|
|
|
assert calls == {
|
|
"identity": 1,
|
|
"next_job": 0,
|
|
"heartbeat": 1,
|
|
"heartbeat_installation_id": installation_id,
|
|
}
|
|
assert service.AgentState(config.state_db).stored_installation_id() == installation_id
|
|
|
|
|
|
def test_company_context_for_heartbeat_resolves_once_and_caches(monkeypatch):
|
|
"""CRT was never sent at heartbeat: the Agent already proves it (company_context.resolve, via preflight_company_context) for coverage/RTC
|
|
jobs, but that value never reached the Hub. This locks in the fix: resolved once, cached, never re-queried once solid."""
|
|
calls = {"n": 0}
|
|
|
|
def fake_preflight(env):
|
|
calls["n"] += 1
|
|
return {"status": "PASS", "company_uf": "BA", "company_crt": "1", "company_code": "000001"}
|
|
|
|
monkeypatch.setattr("gti_fiscal_agent.company_context.preflight_company_context", fake_preflight)
|
|
result, reason = service._company_context_for_heartbeat({}, None)
|
|
assert result == {"status": "PASS", "company_uf": "BA", "company_crt": "1", "company_code": "000001"} and reason is None and calls["n"] == 1
|
|
result2, reason2 = service._company_context_for_heartbeat({}, result) # already cached: never re-queries Firebird
|
|
assert result2 is result and reason2 is None and calls["n"] == 1
|
|
|
|
|
|
def test_company_context_for_heartbeat_retries_while_unresolved_never_guesses(monkeypatch):
|
|
def fake_preflight(env):
|
|
return {"status": "FAIL", "classification": "UNRESOLVED", "reasons": ["UF_SOURCE_UNKNOWN"]}
|
|
|
|
monkeypatch.setattr("gti_fiscal_agent.company_context.preflight_company_context", fake_preflight)
|
|
result, reason = service._company_context_for_heartbeat({}, None)
|
|
assert result is None and "UNRESOLVED" in reason # never a guessed uf/crt - explicitly unresolved, retried next tick
|
|
|
|
|
|
def test_run_heartbeat_carries_the_auto_discovered_company_crt(monkeypatch, tmp_path):
|
|
"""End-to-end: once company context resolves, the CRT rides the heartbeat payload the Hub already knows how to store
|
|
(Installation.company_crt) - closing the transport gap without any Hub change."""
|
|
stopper = _ExternalStopFlag()
|
|
fake_hub = _FakeHubForLoop()
|
|
installation_id = "00000000-0000-4000-8000-000000000333"
|
|
sent = {}
|
|
|
|
def identity():
|
|
return {"agent_id": 2, "installation_id": installation_id}
|
|
|
|
def heartbeat_metadata(payload):
|
|
sent.update(payload)
|
|
stopper.stop = True
|
|
return {"agent_id": 2, "processing_enabled": False}
|
|
|
|
fake_hub.identity = identity
|
|
fake_hub.heartbeat_metadata = heartbeat_metadata
|
|
fake_hub.next_job = lambda: (_ for _ in ()).throw(AssertionError("paused centrally; next_job must not be called"))
|
|
monkeypatch.setattr(service, "HubClient", lambda *a, **k: fake_hub)
|
|
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
|
monkeypatch.setattr(service.time, "sleep", lambda _seconds: None)
|
|
monkeypatch.setattr("gti_fiscal_agent.company_context.preflight_company_context",
|
|
lambda env: {"status": "PASS", "company_uf": "BA", "company_crt": "1", "company_code": "000001", "company_contribuinte_ipi": "N"})
|
|
|
|
config = _config(agent_id="2", poll_interval_seconds=0, state_db=str(tmp_path / "agent.db"))
|
|
env = {"AGENT_ID": "2", "FIREBIRD_DATABASE_PATH": AUTHORIZED, "FIREBIRD_WRITES": "0", "FISCAL_MODE": "DRY_RUN"}
|
|
|
|
service.run(config, env, stopper=stopper)
|
|
|
|
assert sent["company_uf"] == "BA" and sent["company_crt"] == "1" and sent["company_contribuinte_ipi"] == "N"
|
|
|
|
|
|
def test_run_honors_an_externally_supplied_stopper(monkeypatch):
|
|
# This is exactly the pattern the Windows Service wrapper uses: it never
|
|
# constructs GracefulStop() (which would crash off the main thread), it
|
|
# passes its own stop flag and flips it from the SvcStop handler.
|
|
config = _config(poll_interval_seconds=0.001)
|
|
fake_hub = _FakeHubForLoop()
|
|
fake_hub.jobs = [] # no jobs; loop would run forever without an external stop
|
|
monkeypatch.setattr(service, "HubClient", lambda *a, **k: fake_hub)
|
|
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
|
monkeypatch.setattr(service, "AgentState", lambda path: _FakeState())
|
|
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: [])
|
|
|
|
stopper = _ExternalStopFlag()
|
|
calls = {"n": 0}
|
|
real_sleep = service.time.sleep
|
|
|
|
def counting_sleep(seconds):
|
|
calls["n"] += 1
|
|
if calls["n"] >= 3:
|
|
stopper.stop = True
|
|
|
|
monkeypatch.setattr(service.time, "sleep", counting_sleep)
|
|
service.run(config, {"AGENT_ID": "2", "INSTALLATION_ID": "00000000-0000-4000-8000-000000000222"}, stopper=stopper) # no max_iterations — only the external flag can stop it
|
|
assert stopper.stop is True
|
|
assert calls["n"] >= 3
|
|
|
|
|
|
def test_run_survives_hub_down_then_processes_one_job_and_stops(monkeypatch):
|
|
config = _config(poll_interval_seconds=0.001)
|
|
fake_hub = _FakeHubForLoop()
|
|
monkeypatch.setattr(service, "HubClient", lambda *a, **k: fake_hub)
|
|
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
|
monkeypatch.setattr(service, "AgentState", lambda path: _FakeState())
|
|
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: [])
|
|
monkeypatch.setattr(service, "build_job_coverage", lambda *a, **k: None)
|
|
monkeypatch.setattr(service.time, "sleep", lambda seconds: None)
|
|
|
|
# first heartbeat call raises -> backoff branch; second call onward succeeds.
|
|
service.run(config, {"AGENT_ID": "2", "INSTALLATION_ID": "00000000-0000-4000-8000-000000000222"}, max_iterations=4)
|
|
|
|
assert fake_hub.heartbeat_calls >= 2
|
|
assert len(fake_hub.completed) == 1
|
|
assert fake_hub.completed[0][0] == 1
|
|
assert fake_hub.completed[0][1]["products_processed"] == 0
|