chore: establish project baseline
This commit is contained in:
OpenCode
@@ -0,0 +1,41 @@
|
||||
# One of these per Agent installation (one per client machine). Copy to
|
||||
# agent/.env on that machine; never commit it. Get AGENT_ID/AGENT_TOKEN/
|
||||
# COMPANY_ID from central/scripts/register_agent.py, run once per installation.
|
||||
|
||||
GTI_HUB_URL=https://hub.example.com
|
||||
AGENT_ID=
|
||||
AGENT_TOKEN=
|
||||
COMPANY_ID=
|
||||
COMPANY_UF=
|
||||
|
||||
# The client's Firebird ERP database.
|
||||
FIREBIRD_DATABASE_PATH=
|
||||
FB_USER=
|
||||
FB_PASSWORD=
|
||||
FIREBIRD_CLIENT_LIBRARY=
|
||||
|
||||
# Default MUST be 0. Writes only ever happen against exactly the database
|
||||
# named by AUTHORIZED_WRITE_DATABASE_PATH below (never inferred from
|
||||
# FIREBIRD_DATABASE_PATH alone, never from the ".FDB" extension).
|
||||
FIREBIRD_WRITES=0
|
||||
|
||||
# Required to enable writes for THIS installation. Must be set to the exact
|
||||
# same absolute path as FIREBIRD_DATABASE_PATH above — this is the operator's
|
||||
# explicit, per-installation authorization. Leaving it unset means
|
||||
# FIREBIRD_WRITES=1 has no effect against a real client database (it falls
|
||||
# back to the developer-only disposable test database path and therefore
|
||||
# fails closed for anything else).
|
||||
AUTHORIZED_WRITE_DATABASE_PATH=
|
||||
|
||||
# Required to enable writes for THIS installation. Must equal the schema
|
||||
# fingerprint this installation was verified/registered against; if the live
|
||||
# schema fingerprint (reported in heartbeats) ever differs, writes are
|
||||
# refused until an operator re-verifies and updates this value.
|
||||
EXPECTED_SCHEMA_FINGERPRINT=
|
||||
|
||||
POLL_INTERVAL_SECONDS=10
|
||||
AGENT_VERSION=0.1.0
|
||||
|
||||
# Optional overrides:
|
||||
# AGENT_STATE_DB=C:\GTI\Agent\agent_state.sqlite
|
||||
# AGENT_LOG_PATH=C:\GTI\Agent\logs\agent.log
|
||||
@@ -0,0 +1,8 @@
|
||||
[project]
|
||||
name = "gti-fiscal-agent"
|
||||
version = "1.0.0rc1"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = ["httpx>=0.27,<1"]
|
||||
|
||||
[project.optional-dependencies]
|
||||
windows-service = ["pywin32>=306; sys_platform == 'win32'"]
|
||||
@@ -0,0 +1,46 @@
|
||||
"""GTI Fiscal Agent installation preflight (READ-ONLY): python agent/scripts/preflight.py [--env-file FILE] [--json]
|
||||
|
||||
Prints PREFLIGHT=PASS|FAIL and the blockers; exit code 0 on PASS, 2 on FAIL. Secrets are never printed.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(AGENT_ROOT / "src"))
|
||||
from gti_fiscal_agent.preflight import run_preflight # noqa: E402
|
||||
|
||||
|
||||
def load_env(path):
|
||||
if path and Path(path).exists():
|
||||
for line in Path(path).read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
k, _, v = line.partition("=")
|
||||
os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--env-file")
|
||||
ap.add_argument("--json", action="store_true")
|
||||
a = ap.parse_args(argv)
|
||||
load_env(a.env_file or os.environ.get("GTI_AGENT_ENV_FILE") or AGENT_ROOT.parent / ".env")
|
||||
os.environ.setdefault("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite"))
|
||||
os.environ.setdefault("AGENT_LOG_PATH", str(AGENT_ROOT / "logs" / "agent.log"))
|
||||
res = run_preflight(os.environ)
|
||||
if a.json:
|
||||
print(json.dumps(res, ensure_ascii=False, indent=1, default=str))
|
||||
else:
|
||||
for c in res["checks"]:
|
||||
print(f" [{c['status']:<4}] {c['name']:<24} {c['detail']}")
|
||||
print(f"PREFLIGHT={res['PREFLIGHT']}")
|
||||
for b in res["blockers"]:
|
||||
print(f"BLOCKER: {b}")
|
||||
raise SystemExit(0 if res["PREFLIGHT"] == "PASS" else 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Agent preflight: COMPANY_CONTEXT=PASS/FAIL for one Firebird database (READ-ONLY).
|
||||
|
||||
python agent/scripts/preflight_company_context.py [--database <FDB>]
|
||||
|
||||
Configuration (environment or the local ignored .env): COMPANY_CODE / COMPANY_CNPJ (company selection when C000004 is ambiguous),
|
||||
COMPANY_CRT_SOURCE = EMISSOR | C000004 (explicit choice when the two CRT sources disagree). The full CNPJ is never printed.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
from gti_fiscal_agent.company_context import preflight_company_context, preflight_report # noqa: E402
|
||||
|
||||
|
||||
def load_dotenv(path):
|
||||
if path.exists():
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if "=" in line and not line.startswith("#"):
|
||||
k, _, v = line.partition("=")
|
||||
os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--database")
|
||||
a = ap.parse_args(argv)
|
||||
load_dotenv(ROOT / ".env")
|
||||
if a.database:
|
||||
os.environ["FIREBIRD_DATABASE_PATH"] = a.database
|
||||
report = preflight_report(preflight_company_context(os.environ))
|
||||
print(json.dumps(report, ensure_ascii=False, indent=1))
|
||||
raise SystemExit(0 if report["COMPANY_CONTEXT"] == "PASS" else 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Run the agent against a sanitized sample only after an explicit fingerprint gate."""
|
||||
import argparse, json, os, sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).parents[1] / "src"))
|
||||
from gti_fiscal_agent.client import HubClient
|
||||
from gti_fiscal_agent.runner import run_sample
|
||||
from gti_fiscal_agent.state import AgentState
|
||||
|
||||
FIELDS = "CODIGO,PRODUTO,NCM,CLASSIFICACAO_FISCAL,CFOP,CFOP_ENTRADA,CST,CST_ENTRADA,ALIQUOTA,CEST,CSOSN,COD_ENTRADA_COFINS,COD_SAIDA_COFINS,ALIQ_COFINS,COD_ENTRADA_PIS,COD_SAIDA_PIS,ALIQ_PIS,ALIQ_IPI,CST_IPI,ORIGEM,IAT,INCIDENCIA_PISCOFINS,IPPT,ULTIMA_ALTERACAO"
|
||||
|
||||
def read_firebird_products(codes):
|
||||
import fdb
|
||||
connection = fdb.connect(database=os.environ["FIREBIRD_DATABASE_PATH"], user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"], fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"), isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
|
||||
try:
|
||||
cursor = connection.cursor(); cursor.execute(f"SELECT {FIELDS} FROM C000025 WHERE CODIGO IN ({','.join('?' for _ in codes)}) ORDER BY CODIGO", tuple(codes))
|
||||
names = [column[0].strip() for column in cursor.description]
|
||||
return [dict(zip(names, row)) for row in cursor.fetchall()]
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(); parser.add_argument("--sample", required=True); parser.add_argument("--output", required=True); parser.add_argument("--schema-fingerprint", default="")
|
||||
args = parser.parse_args(); sample = json.loads(Path(args.sample).read_text(encoding="utf-8"))
|
||||
codes = [item["CODIGO"] for item in sample.get("sample_without_ncm", [])[:20]]
|
||||
products = read_firebird_products(codes)
|
||||
result = run_sample(products, HubClient(os.getenv("HUB_URL", "http://localhost:8000"), os.getenv("AGENT_DEV_TOKEN", "")), AgentState(Path(args.output).with_suffix(".sqlite")), args.schema_fingerprint)
|
||||
result["products_requested"] = len(products); result["firebird_products_read"] = len(products); result["firebird_access"] = "READ_ONLY_SELECT"
|
||||
output = Path(args.output)
|
||||
if result["results"]:
|
||||
output.write_text(json.dumps(result, indent=2, default=str), encoding="utf-8")
|
||||
elif not output.exists():
|
||||
output.write_text(json.dumps(result, indent=2, default=str), encoding="utf-8")
|
||||
print(json.dumps(result, indent=2))
|
||||
if __name__ == "__main__": main()
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Integrated Agent flow: Firebird read -> Hub /v1/classify/batch -> IPI dry-run
|
||||
-> guarded auto-apply write.
|
||||
|
||||
FIREBIRD_WRITES=0 (default): classifies every product and logs the decision
|
||||
(and what write *would* happen) to the local SQLite audit log; never writes.
|
||||
|
||||
FIREBIRD_WRITES=1: also auto-applies every write_eligible C000025.IPI proposal
|
||||
where rate_kind=NUMERIC, reusing the same schema/path/allowlist/compare-before-write
|
||||
guards as the manual executor. Fails closed before touching the connection unless
|
||||
the configured database is exactly the authorized disposable test database.
|
||||
|
||||
Auto-apply is scoped to table=C000025, field=IPI, rate_kind=NUMERIC only. NT/
|
||||
non-numeric markers, blank rates, ambiguous EX candidates, legacy/invalid NCM,
|
||||
CEST and EX_TIPI are never auto-applied here.
|
||||
"""
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_ROOT = Path(__file__).resolve().parents[1]
|
||||
PROJECT_ROOT = AGENT_ROOT.parent
|
||||
sys.path.insert(0, str(AGENT_ROOT / "src"))
|
||||
|
||||
from gti_fiscal_agent.client import HubClient # noqa: E402
|
||||
from gti_fiscal_agent.firebird_reader import read_products # noqa: E402
|
||||
from gti_fiscal_agent.pipeline import run_pipeline # noqa: E402
|
||||
from gti_fiscal_agent.runner import REFERENCE_SCHEMA_V1, to_hub_product # noqa: E402
|
||||
from gti_fiscal_agent.state import AgentState # noqa: E402
|
||||
|
||||
|
||||
def load_dotenv(path: Path) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
os.environ.setdefault(key.strip(), value.strip())
|
||||
|
||||
|
||||
def load_executor():
|
||||
spec = importlib.util.spec_from_file_location("firebird_write_executor", PROJECT_ROOT / "scripts" / "firebird_write_executor.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--hub-url", default=os.getenv("HUB_URL", "http://localhost:8000"))
|
||||
parser.add_argument("--token", default=os.getenv("AGENT_DEV_TOKEN", ""))
|
||||
parser.add_argument("--state-db", default=str(AGENT_ROOT / "agent_state.sqlite"))
|
||||
parser.add_argument("--schema-fingerprint", default=REFERENCE_SCHEMA_V1)
|
||||
args = parser.parse_args()
|
||||
|
||||
load_dotenv(PROJECT_ROOT / ".env")
|
||||
if not args.token:
|
||||
args.token = os.environ.get("AGENT_DEV_TOKEN", "")
|
||||
|
||||
firebird_writes = os.environ.get("FIREBIRD_WRITES", "0") == "1"
|
||||
executor = load_executor()
|
||||
database_path = os.environ["FIREBIRD_DATABASE_PATH"]
|
||||
|
||||
if firebird_writes:
|
||||
# Fail closed before any product is read or any connection is opened for writing.
|
||||
executor.assert_authorized_write_path(database_path)
|
||||
|
||||
if args.schema_fingerprint != REFERENCE_SCHEMA_V1:
|
||||
print(json.dumps({"status": "SCHEMA_UNKNOWN", "processed": 0}))
|
||||
return
|
||||
|
||||
products = read_products()
|
||||
client = HubClient(args.hub_url, args.token)
|
||||
decisions = client.classify_batch([to_hub_product(p) for p in products])
|
||||
|
||||
state = AgentState(args.state_db)
|
||||
|
||||
write_connection = None
|
||||
if firebird_writes:
|
||||
import fdb
|
||||
write_connection = fdb.connect(
|
||||
database=database_path,
|
||||
user=os.environ["FB_USER"],
|
||||
password=os.environ["FB_PASSWORD"],
|
||||
fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
)
|
||||
|
||||
try:
|
||||
summary = run_pipeline(products, decisions, state, firebird_writes, write_connection, executor, database_path)
|
||||
finally:
|
||||
if write_connection is not None:
|
||||
write_connection.close()
|
||||
|
||||
summary = {"status": "OK", "firebird_writes_enabled": firebird_writes, **summary}
|
||||
print(json.dumps(summary, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,297 @@
|
||||
"""Long-running Agent service: heartbeat + poll Hub jobs + execute pipeline.
|
||||
|
||||
startup -> load config -> validate Hub -> validate Firebird -> heartbeat
|
||||
-> poll job -> claim -> execute -> report result -> back to polling.
|
||||
|
||||
No job available: sleep POLL_INTERVAL_SECONDS.
|
||||
Hub unavailable: retry with simple capped backoff; the Agent stays alive.
|
||||
Job fails internally (bad Firebird read, classify error, write refused):
|
||||
the job is reported FAILED with a sanitized error and the Agent keeps polling.
|
||||
SIGINT/SIGTERM: finish the current job (if any) and stop the loop cleanly.
|
||||
|
||||
Written to be runnable as-is today, and easy to later wrap as a Windows
|
||||
Service (a service wrapper would just call main()); no installer here.
|
||||
"""
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_ROOT = Path(__file__).resolve().parents[1]
|
||||
PROJECT_ROOT = AGENT_ROOT.parent
|
||||
sys.path.insert(0, str(AGENT_ROOT / "src"))
|
||||
sys.path.insert(0, str(PROJECT_ROOT)) # ships central/app engine modules read-only (central.app.*)
|
||||
|
||||
from gti_fiscal_agent.client import HubClient # noqa: E402
|
||||
from gti_fiscal_agent.firebird_reader import check_connection, read_products # noqa: E402
|
||||
from gti_fiscal_agent.pipeline import run_pipeline # noqa: E402
|
||||
from gti_fiscal_agent.runner import REFERENCE_SCHEMA_V1, to_hub_product # noqa: E402
|
||||
from gti_fiscal_agent.state import AgentState # noqa: E402
|
||||
|
||||
_MAX_BACKOFF_SECONDS = 60
|
||||
|
||||
logger = logging.getLogger("gti_fiscal_agent")
|
||||
|
||||
|
||||
def setup_logging(log_path):
|
||||
"""Rotating local log file. Never logs FB_PASSWORD or AGENT_TOKEN — every
|
||||
log call in this module only references non-secret identifiers/counts."""
|
||||
logger.setLevel(logging.INFO)
|
||||
if logger.handlers:
|
||||
return logger
|
||||
Path(log_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
handler = logging.handlers.RotatingFileHandler(log_path, maxBytes=5_000_000, backupCount=5, encoding="utf-8")
|
||||
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
|
||||
logger.addHandler(handler)
|
||||
logger.addHandler(logging.StreamHandler())
|
||||
return logger
|
||||
|
||||
|
||||
def load_dotenv(path: Path) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
os.environ.setdefault(key.strip(), value.strip())
|
||||
|
||||
|
||||
def load_executor():
|
||||
spec = importlib.util.spec_from_file_location("firebird_write_executor", PROJECT_ROOT / "scripts" / "firebird_write_executor.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class Config:
|
||||
def __init__(self, env):
|
||||
self.hub_url = env.get("GTI_HUB_URL") or env.get("HUB_URL") or "http://localhost:8000"
|
||||
self.agent_id = env.get("AGENT_ID", "")
|
||||
self.agent_token = env.get("AGENT_TOKEN") or env.get("AGENT_DEV_TOKEN", "")
|
||||
self.company_id = env.get("COMPANY_ID", "")
|
||||
self.company_uf = env.get("COMPANY_UF", "")
|
||||
self.database_path = env["FIREBIRD_DATABASE_PATH"]
|
||||
# FISCAL_MODE gates writes technically: DRY_RUN can never write, whatever FIREBIRD_WRITES says. Unset = legacy
|
||||
# behaviour (FIREBIRD_WRITES decides). `firebird_writes` below is the EFFECTIVE value used by every write path.
|
||||
requested = env.get("FIREBIRD_WRITES", "0") == "1"
|
||||
self.fiscal_mode = (env.get("FISCAL_MODE") or ("AUTO_APPLY" if requested else "DRY_RUN")).strip().upper()
|
||||
self.requested_writes = requested
|
||||
self.firebird_writes = requested and self.fiscal_mode == "AUTO_APPLY"
|
||||
self.poll_interval_seconds = float(env.get("POLL_INTERVAL_SECONDS", "5"))
|
||||
self.agent_version = env.get("AGENT_VERSION", "1.0.0-rc1")
|
||||
self.schema_fingerprint = env.get("SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1)
|
||||
self.state_db = env.get("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite"))
|
||||
# Pilot write authorization: both must be explicitly set by the installer for
|
||||
# a real client database. Left unset, both default to the dev-only values,
|
||||
# so a real client path/schema is never silently authorized (see
|
||||
# scripts/firebird_write_executor.py's assert_authorized_write_path).
|
||||
self.authorized_write_database_path = env.get("AUTHORIZED_WRITE_DATABASE_PATH")
|
||||
self.expected_schema_fingerprint = env.get("EXPECTED_SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1)
|
||||
self.log_path = env.get("AGENT_LOG_PATH", str(AGENT_ROOT / "logs" / "agent.log"))
|
||||
|
||||
|
||||
class GracefulStop:
|
||||
def __init__(self):
|
||||
self.stop = False
|
||||
signal.signal(signal.SIGINT, self._handle)
|
||||
signal.signal(signal.SIGTERM, self._handle)
|
||||
|
||||
def _handle(self, signum, frame):
|
||||
self.stop = True
|
||||
|
||||
|
||||
def build_job_coverage(env, client, products, decisions):
|
||||
"""Company-context-aware coverage over the job's products (read-only). Fails closed when the company context is unresolved."""
|
||||
from gti_fiscal_agent.company_context import mask_cnpj, preflight_company_context
|
||||
from gti_fiscal_agent.coverage import build_refs, compute_coverage
|
||||
from gti_fiscal_agent.firebird_reader import read_evidence
|
||||
ctx = preflight_company_context(env)
|
||||
if ctx["status"] != "PASS":
|
||||
raise RuntimeError("COMPANY_CONTEXT=FAIL: " + "; ".join(ctx.get("reasons") or ["unresolved"]))
|
||||
evidence = read_evidence(env, ctx["company_code"])
|
||||
refs = build_refs(client.get_json, products, ctx["company_uf"])
|
||||
report = compute_coverage(products, decisions, {"uf": ctx["company_uf"], "crt": ctx["company_crt"]}, evidence.get("company_default_csosn"), refs, evidence)
|
||||
report["company_context"] = {"COMPANY_CONTEXT_STATUS": "PASS", "classification": ctx["classification"], "company_code": ctx["company_code"], "company_cnpj": ctx["company_cnpj"], "company_uf": ctx["company_uf"],
|
||||
"company_crt": ctx["company_crt"], "source_of_truth": ctx["source_of_truth"], "confidence": ctx["confidence"], "warnings": ctx["warnings"]}
|
||||
return report
|
||||
|
||||
|
||||
def open_write_connection(env, database_path, firebird_writes=True):
|
||||
if not firebird_writes: # technical guarantee: no write-capable connection exists in DRY_RUN / FIREBIRD_WRITES=0
|
||||
raise RuntimeError("write connection refused: writes are disabled (FISCAL_MODE=DRY_RUN or FIREBIRD_WRITES=0)")
|
||||
import fdb
|
||||
return fdb.connect(
|
||||
database=database_path,
|
||||
user=env["FB_USER"],
|
||||
password=env["FB_PASSWORD"],
|
||||
fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
)
|
||||
|
||||
|
||||
def execute_job(job, config, executor, state, env):
|
||||
"""Run the pipeline for one claimed job. Never raises for job-domain
|
||||
failures (bad Firebird read, classify error) — returns (result_summary, error)."""
|
||||
kind = job["kind"]
|
||||
payload = job.get("payload") or {}
|
||||
try:
|
||||
if config.firebird_writes:
|
||||
# Layered, per-installation authorization: the path must match this
|
||||
# installation's own explicitly configured authorized_write_database_path
|
||||
# (defaults to the dev-only constant, so a real client path is refused
|
||||
# unless the installer deliberately set it), AND the live schema
|
||||
# fingerprint must match what this installation was configured to expect.
|
||||
executor.assert_authorized_write_path(config.database_path, authorized_path=config.authorized_write_database_path)
|
||||
executor.assert_schema_fingerprint_authorized(config.schema_fingerprint, config.expected_schema_fingerprint)
|
||||
|
||||
codigos = payload.get("codigos") if kind == "REPROCESS_PRODUCTS" else None
|
||||
products = read_products(env, codigos=codigos)
|
||||
logger.info("job %s: read %d product(s) from Firebird", job.get("job_id"), len(products))
|
||||
|
||||
client = HubClient(config.hub_url, config.agent_token)
|
||||
decisions = client.classify_batch([to_hub_product(p) for p in products])
|
||||
|
||||
write_connection = None
|
||||
if config.firebird_writes:
|
||||
write_connection = open_write_connection(env, config.database_path)
|
||||
try:
|
||||
summary = run_pipeline(products, decisions, state, config.firebird_writes, write_connection, executor, config.database_path, authorized_path=config.authorized_write_database_path)
|
||||
finally:
|
||||
if write_connection is not None:
|
||||
write_connection.close()
|
||||
|
||||
coverage = build_job_coverage(env, client, products, decisions) if kind == "REPROCESS_ALL" else None
|
||||
result_summary = {
|
||||
"coverage": coverage,
|
||||
"fiscal_mode": getattr(config, "fiscal_mode", None),
|
||||
"release_version": env.get("AGENT_VERSION") or "1.0.0-rc1",
|
||||
"products_processed": summary["processed"],
|
||||
"writes_attempted": summary["writes_attempted"],
|
||||
"writes_succeeded": summary["writes_succeeded"],
|
||||
"writes_failed": summary["writes_failed"],
|
||||
"no_ops": summary["no_ops"],
|
||||
"firebird_writes_enabled": config.firebird_writes,
|
||||
"agent_id": job["agent_id"],
|
||||
"schema_fingerprint": config.schema_fingerprint,
|
||||
}
|
||||
logger.info(
|
||||
"job %s finished: processed=%d writes_attempted=%d writes_succeeded=%d writes_failed=%d no_ops=%d",
|
||||
job.get("job_id"), summary["processed"], summary["writes_attempted"], summary["writes_succeeded"], summary["writes_failed"], summary["no_ops"],
|
||||
)
|
||||
if summary["writes_failed"] > 0:
|
||||
return result_summary, f"{summary['writes_failed']} write(s) failed guard checks"
|
||||
return result_summary, None
|
||||
except executor.WriteRefused as error:
|
||||
logger.warning("job %s refused: %s", job.get("job_id"), error)
|
||||
return None, f"write refused: {error}"
|
||||
except Exception as error: # noqa: BLE001 — job-domain isolation boundary
|
||||
logger.warning("job %s failed internally: %s: %s", job.get("job_id"), type(error).__name__, error)
|
||||
return None, f"{type(error).__name__}: {error}"
|
||||
|
||||
|
||||
def run(config, env, *, max_iterations=None, stopper=None):
|
||||
"""`stopper` defaults to a signal-based GracefulStop (SIGINT/SIGTERM), which
|
||||
only works from the main thread. A host that manages its own lifecycle
|
||||
(e.g. the Windows Service wrapper, which stops the loop via the Service
|
||||
Control Manager instead of OS signals) should pass its own stop object —
|
||||
anything with a `.stop` boolean attribute — and flip it from any thread."""
|
||||
executor = load_executor()
|
||||
state = AgentState(config.state_db)
|
||||
client = HubClient(config.hub_url, config.agent_token)
|
||||
stopper = stopper if stopper is not None else GracefulStop()
|
||||
|
||||
backoff = 1
|
||||
was_down = False
|
||||
iterations = 0
|
||||
last_job_id = None
|
||||
while not stopper.stop:
|
||||
if max_iterations is not None and iterations >= max_iterations:
|
||||
break
|
||||
iterations += 1
|
||||
|
||||
try:
|
||||
client.heartbeat("IDLE", schema_fingerprint=config.schema_fingerprint, agent_version=config.agent_version, last_job_id=last_job_id)
|
||||
if was_down:
|
||||
logger.info("reconnected to Hub after %ss backoff", backoff)
|
||||
was_down = False
|
||||
backoff = 1
|
||||
except Exception as error:
|
||||
logger.warning("heartbeat failed, Hub unreachable: %s (retrying in %ss)", error, backoff)
|
||||
was_down = True
|
||||
time.sleep(backoff)
|
||||
backoff = min(backoff * 2, _MAX_BACKOFF_SECONDS)
|
||||
continue
|
||||
|
||||
try:
|
||||
job = client.next_job()
|
||||
except Exception as error:
|
||||
logger.warning("poll failed, Hub unreachable: %s (retrying in %ss)", error, backoff)
|
||||
was_down = True
|
||||
time.sleep(backoff)
|
||||
backoff = min(backoff * 2, _MAX_BACKOFF_SECONDS)
|
||||
continue
|
||||
|
||||
if job is None:
|
||||
time.sleep(config.poll_interval_seconds)
|
||||
continue
|
||||
|
||||
last_job_id = job.get("job_id")
|
||||
logger.info("job %s received: kind=%s", job.get("job_id"), job["kind"])
|
||||
try:
|
||||
client.heartbeat("BUSY", schema_fingerprint=config.schema_fingerprint, agent_version=config.agent_version, last_job_id=last_job_id)
|
||||
except Exception:
|
||||
pass # best-effort; the job itself still proceeds
|
||||
|
||||
result_summary, error = execute_job(job, config, executor, state, env)
|
||||
try:
|
||||
if error is None:
|
||||
client.complete_job(job.get("job_id"), result_summary)
|
||||
logger.info("job %s reported COMPLETED", job.get("job_id"))
|
||||
else:
|
||||
client.fail_job(job.get("job_id"), error)
|
||||
logger.warning("job %s reported FAILED: %s", job.get("job_id"), error)
|
||||
except Exception as report_error:
|
||||
logger.warning("could not report job %s result to Hub: %s", job.get("job_id"), report_error)
|
||||
|
||||
|
||||
def enforce_startup_preflight(env):
|
||||
"""Full read-only preflight at startup (Hub, auth, Firebird, capabilities, company context, guards). Fail closed."""
|
||||
from gti_fiscal_agent.preflight import run_preflight
|
||||
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.0.0-rc1")
|
||||
logger.info("startup preflight: %s", json.dumps({k: report[k] for k in ("PREFLIGHT", "blockers", "fiscal_mode", "effective_firebird_writes")}, ensure_ascii=False))
|
||||
print(json.dumps({"PREFLIGHT": report["PREFLIGHT"], "blockers": report["blockers"]}, ensure_ascii=False))
|
||||
if report["PREFLIGHT"] != "PASS":
|
||||
logger.error("startup aborted: preflight failed (fail closed): %s", report["blockers"])
|
||||
raise SystemExit(2)
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--max-iterations", type=int, default=None, help="stop after N poll cycles (used by tests); omit to run forever")
|
||||
args = parser.parse_args()
|
||||
|
||||
load_dotenv(Path(os.environ.get("GTI_AGENT_ENV_FILE") or PROJECT_ROOT / ".env"))
|
||||
config = Config(os.environ)
|
||||
setup_logging(config.log_path)
|
||||
|
||||
logger.info("startup: agent_id=%s company_id=%s agent_version=%s firebird_writes=%s", config.agent_id, config.company_id, config.agent_version, config.firebird_writes)
|
||||
hub_ok = HubClient(config.hub_url, config.agent_token).health()
|
||||
firebird_ok = check_connection(os.environ)
|
||||
logger.info("startup checks: hub_ok=%s firebird_ok=%s", hub_ok, firebird_ok)
|
||||
enforce_startup_preflight(os.environ)
|
||||
|
||||
run(config, os.environ, max_iterations=args.max_iterations)
|
||||
logger.info("shutdown: stopped cleanly")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Windows Service wrapper around run_agent_service.run() — reuses it as-is,
|
||||
does not reimplement any polling/pipeline logic.
|
||||
|
||||
Requires pywin32 (`pip install pywin32`), Windows-only.
|
||||
|
||||
Commands (run from an elevated/administrator terminal):
|
||||
python gti_agent_windows_service.py install # register the service, Startup=Automatic
|
||||
python gti_agent_windows_service.py start
|
||||
python gti_agent_windows_service.py stop
|
||||
python gti_agent_windows_service.py restart
|
||||
python gti_agent_windows_service.py remove # uninstall
|
||||
python gti_agent_windows_service.py --startup=auto --username . --password . install
|
||||
|
||||
Recovery (restart on failure) is not set by pywin32's `install` alone; run once
|
||||
after installing:
|
||||
sc.exe failure "GTIFiscalAgent" reset= 86400 actions= restart/60000/restart/60000/restart/60000
|
||||
|
||||
This module only wraps agent/scripts/run_agent_service.py's `run()` — the
|
||||
actual polling/heartbeat/pipeline logic lives there, unchanged.
|
||||
"""
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(AGENT_ROOT / "scripts"))
|
||||
sys.path.insert(0, str(AGENT_ROOT / "src"))
|
||||
|
||||
import run_agent_service as agent_service # noqa: E402
|
||||
|
||||
try:
|
||||
import servicemanager
|
||||
import win32event
|
||||
import win32service
|
||||
import win32serviceutil
|
||||
except ImportError: # pragma: no cover — only available on Windows with pywin32 installed
|
||||
servicemanager = win32event = win32service = win32serviceutil = None
|
||||
|
||||
|
||||
class _ServiceStopFlag:
|
||||
"""Plain stop flag (no signal.signal registration) — safe to flip from any
|
||||
thread. The Windows Service Control Manager stops the service, not a
|
||||
console SIGINT/SIGTERM, so this replaces GracefulStop for this host."""
|
||||
def __init__(self):
|
||||
self.stop = False
|
||||
|
||||
|
||||
if win32serviceutil is not None:
|
||||
|
||||
class GTIFiscalAgentService(win32serviceutil.ServiceFramework):
|
||||
_svc_name_ = "GTIFiscalAgent"
|
||||
_svc_display_name_ = "GTI Fiscal Agent"
|
||||
_svc_description_ = "Reads the local Firebird ERP, polls the GTI Fiscal Hub for jobs, and applies guarded fiscal updates."
|
||||
|
||||
def __init__(self, args):
|
||||
win32serviceutil.ServiceFramework.__init__(self, args)
|
||||
self.stop_event = win32event.CreateEvent(None, 0, 0, None)
|
||||
self.stop_flag = _ServiceStopFlag()
|
||||
self._thread = None
|
||||
|
||||
def SvcStop(self):
|
||||
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
|
||||
self.stop_flag.stop = True
|
||||
win32event.SetEvent(self.stop_event)
|
||||
|
||||
def SvcDoRun(self):
|
||||
servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE, servicemanager.PYS_SERVICE_STARTED, (self._svc_name_, ""))
|
||||
self._thread = threading.Thread(target=self._run_agent, daemon=True)
|
||||
self._thread.start()
|
||||
win32event.WaitForSingleObject(self.stop_event, win32event.INFINITE)
|
||||
|
||||
def _run_agent(self):
|
||||
import os
|
||||
agent_service.load_dotenv(Path(os.environ.get("GTI_AGENT_ENV_FILE") or agent_service.PROJECT_ROOT / ".env"))
|
||||
config = agent_service.Config(os.environ)
|
||||
agent_service.setup_logging(config.log_path)
|
||||
try:
|
||||
agent_service.enforce_startup_preflight(os.environ) # fail closed: a failing preflight stops the service
|
||||
except SystemExit:
|
||||
self.stop_flag.stop = True
|
||||
servicemanager.LogErrorMsg("GTIFiscalAgent: startup preflight failed; see the agent log")
|
||||
win32event.SetEvent(self.stop_event)
|
||||
return
|
||||
agent_service.run(config, os.environ, stopper=self.stop_flag)
|
||||
|
||||
|
||||
def main():
|
||||
if win32serviceutil is None:
|
||||
raise SystemExit("pywin32 is required: pip install pywin32")
|
||||
win32serviceutil.HandleCommandLine(GTIFiscalAgentService)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
"""Read-only ERP to Fiscal Hub agent."""
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Pure capability detection / schema classification for SGV (C000025/C000004) databases. No I/O, no writes."""
|
||||
|
||||
WRITE_TARGETS = ["CLASSIFICACAO_FISCAL", "CFOP_ENTRADA", "CST_ENTRADA", "CFOP", "CST", "ALIQUOTA", "CEST", "SITUACAO_TRIBUTARIA", "COD_ENTRADA_COFINS",
|
||||
"COD_SAIDA_COFINS", "ALIQ_COFINS", "COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "CSOSN", "IPI", "CST_IPI", "IAT", "INCIDENCIA_PISCOFINS",
|
||||
"PISCOFINS", "IPPT"]
|
||||
ORIGIN_FIELD = "COD_ORIGEM_MERCADORIA"
|
||||
|
||||
# Firebird RDB$FIELD_TYPE -> family
|
||||
FAMILY = {7: "int", 8: "int", 16: "int", 10: "float", 11: "float", 27: "float", 12: "date", 13: "time", 35: "timestamp", 14: "text", 37: "text", 261: "blob"}
|
||||
# accepted families per field (UI edit controls take text or numbers; numeric columns may be int/float)
|
||||
ACCEPT = {"ALIQUOTA": {"float", "int"}, "IPI": {"float", "int"}, "COD_ORIGEM_MERCADORIA": {"int"}}
|
||||
DEFAULT_ACCEPT = {"text", "int"} # text columns; some ERPs keep codes such as CST_ENTRADA in SMALLINT
|
||||
|
||||
CAPABILITIES = {
|
||||
"supports_ncm": ["CLASSIFICACAO_FISCAL"],
|
||||
"supports_cfop_entry": ["CFOP_ENTRADA"],
|
||||
"supports_cfop_exit": ["CFOP"],
|
||||
"supports_cst_entry": ["CST_ENTRADA"],
|
||||
"supports_cst_exit": ["CST"],
|
||||
"supports_icms_rate": ["ALIQUOTA"],
|
||||
"supports_cest": ["CEST"],
|
||||
"supports_csosn": ["CSOSN"],
|
||||
"supports_ipi": ["IPI", "CST_IPI"],
|
||||
"supports_pis": ["COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS"],
|
||||
"supports_cofins": ["COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS"],
|
||||
"supports_origin": [ORIGIN_FIELD],
|
||||
"supports_iat": ["IAT"],
|
||||
"supports_incidence": ["INCIDENCIA_PISCOFINS"],
|
||||
"supports_ippt": ["IPPT"],
|
||||
}
|
||||
|
||||
|
||||
def type_ok(field, type_code):
|
||||
return FAMILY.get(type_code, "other") in ACCEPT.get(field, DEFAULT_ACCEPT)
|
||||
|
||||
|
||||
def detect_capabilities(columns):
|
||||
"""columns: {FIELD: rdb_type_code} of C000025. capability=true only if every required column exists with a usable type."""
|
||||
out = {}
|
||||
for cap, fields in CAPABILITIES.items():
|
||||
out[cap] = all(f in columns and type_ok(f, columns[f]) for f in fields)
|
||||
return out
|
||||
|
||||
|
||||
def field_compatibility(columns, reference_types=None):
|
||||
"""Per write target (+ origin): PRESENT_OK / PRESENT_TYPE_VARIATION / MISSING."""
|
||||
res = {}
|
||||
for f in WRITE_TARGETS + [ORIGIN_FIELD]:
|
||||
if f not in columns:
|
||||
res[f] = "MISSING"
|
||||
elif not type_ok(f, columns[f]):
|
||||
res[f] = "PRESENT_TYPE_INCOMPATIBLE"
|
||||
elif reference_types and reference_types.get(f) is not None and reference_types[f] != columns[f]:
|
||||
res[f] = "PRESENT_TYPE_VARIATION"
|
||||
else:
|
||||
res[f] = "PRESENT_OK"
|
||||
return res
|
||||
|
||||
|
||||
def classify_schema(has_c000004, has_c000025, fingerprint, reference_fingerprint, compat):
|
||||
"""REFERENCE_SCHEMA_MATCH | COMPATIBLE_WITH_VARIATIONS | INCOMPATIBLE. A different hash is not a failure
|
||||
when every required capability exists (origin is optional: it is only PROBABLE in the map)."""
|
||||
if not (has_c000004 and has_c000025):
|
||||
return "INCOMPATIBLE"
|
||||
required = {f: v for f, v in compat.items() if f in WRITE_TARGETS}
|
||||
if any(v in ("MISSING", "PRESENT_TYPE_INCOMPATIBLE") for v in required.values()):
|
||||
return "INCOMPATIBLE"
|
||||
if fingerprint == reference_fingerprint:
|
||||
return "REFERENCE_SCHEMA_MATCH"
|
||||
return "COMPATIBLE_WITH_VARIATIONS"
|
||||
|
||||
|
||||
NAME_KEYWORDS = {"RESTAURANT": ("RESTAURANTE", "PIZZARIA", "LANCHONETE", "CHURRASC"), "RETAIL_MARKET": ("MERCADO", "SUPERMERC", "MERCEARIA", "ATACAD"),
|
||||
"APPAREL": ("MODAS", "CONFEC", "BOUTIQUE", "ROUPAS"), "CONSTRUCTION": ("CONSTRU", "MATERIAIS"), "AUTO_REPAIR": ("OFICINA", "MECANICA", "AUTO PECAS", "AUTOPECAS")}
|
||||
|
||||
|
||||
def infer_segment(signals, company_name=""):
|
||||
"""Evidence-based segment guess from populated tables/flags (never from the file name).
|
||||
signals: {"restaurant_tables_rows": int, "kitchen_products": int, "recipes": int, "service_orders": int, "vehicle_products": int, "grade_items": int, "nfce_items": int}"""
|
||||
ev = []
|
||||
score = {"RESTAURANT": 0, "AUTO_REPAIR": 0, "APPAREL": 0, "RETAIL_MARKET": 0, "CONSTRUCTION": 0}
|
||||
import unicodedata
|
||||
up = unicodedata.normalize("NFKD", company_name or "").encode("ascii", "ignore").decode().upper()
|
||||
for seg_name, kws in NAME_KEYWORDS.items():
|
||||
hit = [k for k in kws if k in up]
|
||||
if hit:
|
||||
score[seg_name] += 1; ev.append(f"company name in C000004 contains {hit[0]} (weak, name is data, not the file name)")
|
||||
if signals.get("restaurant_tables_rows", 0) > 50: score["RESTAURANT"] += 2; ev.append(f"mesas/comandas rows={signals['restaurant_tables_rows']}")
|
||||
if signals.get("kitchen_products", 0) > 0: score["RESTAURANT"] += 1; ev.append(f"PROD_COZINHA products={signals['kitchen_products']}")
|
||||
if signals.get("recipes", 0) > 0: score["RESTAURANT"] += 1; ev.append(f"recipes={signals['recipes']}")
|
||||
if signals.get("service_orders", 0) > 20: score["AUTO_REPAIR"] += 2; ev.append(f"ordens de serviço={signals['service_orders']}")
|
||||
if signals.get("vehicle_products", 0) > 0: score["AUTO_REPAIR"] += 1; ev.append(f"veículos={signals['vehicle_products']}")
|
||||
if signals.get("grade_items", 0) > 50: score["APPAREL"] += 2; ev.append(f"grade (tamanho/cor) rows={signals['grade_items']}")
|
||||
if signals.get("nfce_items", 0) > 1000 and max(score.values()) == 0: score["RETAIL_MARKET"] += 1; ev.append("high-volume NFC-e retail without restaurant/OS/grade signals")
|
||||
best = max(score, key=score.get)
|
||||
return (best if score[best] > 0 else "UNKNOWN"), score, ev
|
||||
@@ -0,0 +1,37 @@
|
||||
import httpx
|
||||
|
||||
class HubClient:
|
||||
def __init__(self, url, token, chunk_size=100): self.url, self.token, self.chunk_size = url.rstrip("/"), token, chunk_size
|
||||
def _headers(self): return {"Authorization": f"Bearer {self.token}"}
|
||||
def classify_batch(self, products):
|
||||
results = []
|
||||
for start in range(0, len(products), self.chunk_size):
|
||||
chunk = products[start:start + self.chunk_size]
|
||||
response = httpx.post(f"{self.url}/v1/classify/batch", json={"products": chunk}, headers=self._headers(), timeout=60)
|
||||
response.raise_for_status(); results.extend(response.json()["results"])
|
||||
return results
|
||||
def get_json(self, path):
|
||||
response = httpx.get(f"{self.url}{path}", headers=self._headers(), timeout=30)
|
||||
response.raise_for_status(); return response.json()
|
||||
def health(self):
|
||||
response = httpx.get(f"{self.url}/health", timeout=10)
|
||||
return response.status_code == 200
|
||||
def heartbeat(self, status, schema_fingerprint=None, agent_version=None, last_job_id=None):
|
||||
payload = {"status": status, "schema_fingerprint": schema_fingerprint, "agent_version": agent_version, "last_job_id": last_job_id}
|
||||
response = httpx.post(f"{self.url}/v1/agents/heartbeat", json=payload, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def next_job(self):
|
||||
response = httpx.get(f"{self.url}/v1/jobs/next", headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def create_job(self, kind, payload=None):
|
||||
response = httpx.post(f"{self.url}/v1/jobs", json={"kind": kind, "payload": payload or {}}, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def get_job(self, job_id):
|
||||
response = httpx.get(f"{self.url}/v1/jobs/{job_id}", headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def complete_job(self, job_id, result_summary):
|
||||
response = httpx.post(f"{self.url}/v1/jobs/{job_id}/complete", json={"result_summary": result_summary}, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def fail_job(self, job_id, error):
|
||||
response = httpx.post(f"{self.url}/v1/jobs/{job_id}/fail", json={"error": str(error)}, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
@@ -0,0 +1,218 @@
|
||||
"""Deterministic, auditable company-context resolution for an Agent installation (READ-ONLY).
|
||||
|
||||
Evidence (proved on 4 SGV databases, see docs/COMPANY_CONTEXT_RESOLUTION_V1.json):
|
||||
* EMISSOR.EMITENTE_CNPJ/UF/CRT is the emitter block read by the emission programs (GtiVendas_NFCe.exe and Envio NF-e.exe both bind
|
||||
QrEmissor.EMITENTE_*; neither reads C000004.CRT/OPTANTE_SIMPLES). 100% of emitted NFC-e access keys carry the CNPJ of the C000004 row
|
||||
whose CNPJ equals EMISSOR.EMITENTE_CNPJ, and cUF=29.
|
||||
* C000004.CRT / OPTANTE_SIMPLES belong to the ERP filial registry (SPED, filial screen) and can disagree with EMISSOR.
|
||||
Never guesses: first row / lowest code / latest row are NOT used. Ambiguity -> FAIL with the reason, unless the operator configures
|
||||
COMPANY_CODE or COMPANY_CNPJ (company) and COMPANY_CRT_SOURCE = EMISSOR | C000004 (explicit choice when the two CRT sources conflict).
|
||||
"""
|
||||
import hashlib
|
||||
import os
|
||||
|
||||
IBGE_UF = {"11": "RO", "12": "AC", "13": "AM", "14": "RR", "15": "PA", "16": "AP", "17": "TO", "21": "MA", "22": "PI", "23": "CE", "24": "RN", "25": "PB", "26": "PE",
|
||||
"27": "AL", "28": "SE", "29": "BA", "31": "MG", "32": "ES", "33": "RJ", "35": "SP", "41": "PR", "42": "SC", "43": "RS", "50": "MS", "51": "MT", "52": "GO",
|
||||
"53": "DF"}
|
||||
VALID_CRT = {"1", "2", "3"}
|
||||
|
||||
|
||||
class CompanyContextError(Exception):
|
||||
def __init__(self, result):
|
||||
super().__init__(f"COMPANY_CONTEXT=FAIL ({result.get('classification')}): {'; '.join(result.get('reasons', []))}")
|
||||
self.result = result
|
||||
|
||||
|
||||
def digits(value):
|
||||
return "".join(c for c in str(value or "") if c.isdigit())
|
||||
|
||||
|
||||
def mask_cnpj(value):
|
||||
d = digits(value)
|
||||
return f"{d[:2]}**********{d[-2:]}" if len(d) >= 4 else None
|
||||
|
||||
|
||||
def cnpj_fingerprint(value):
|
||||
d = digits(value)
|
||||
return hashlib.sha256(d.encode()).hexdigest()[:8] if d else None
|
||||
|
||||
|
||||
def _s(v):
|
||||
return None if v is None else str(v).strip()
|
||||
|
||||
|
||||
def config_from_env(env):
|
||||
crt_source = (env.get("COMPANY_CRT_SOURCE") or "").strip().upper() or None
|
||||
if crt_source == "AUTO": # AUTO = resolve automatically, only when there is no conflict
|
||||
crt_source = None
|
||||
invalid = False
|
||||
else:
|
||||
invalid = bool(crt_source and crt_source not in ("EMISSOR", "C000004"))
|
||||
uf = (env.get("COMPANY_UF") or "").strip().upper() or None
|
||||
return {"company_code": (env.get("COMPANY_CODE") or "").strip() or None, "company_cnpj": digits(env.get("COMPANY_CNPJ")) or None,
|
||||
"crt_source": crt_source if crt_source in ("EMISSOR", "C000004") else None, "crt_source_invalid": invalid,
|
||||
"uf_override": uf}
|
||||
|
||||
|
||||
def resolve(evidence, config=None):
|
||||
"""evidence: {"c000004": [{code, cnpj, ie, uf, crt, optante, name}], "emissor": [{cnpj, uf, crt}],
|
||||
"nfce_keys": [{"cuf", "cnpj", "n"}], "nf_by_filial": {code: n}}. Pure function."""
|
||||
cfg = config or {}
|
||||
out = {"status": "FAIL", "classification": "UNRESOLVED", "company_code": None, "company_cnpj": None, "company_cnpj_fingerprint": None, "company_uf": None, "company_crt": None,
|
||||
"source_of_truth": {}, "confidence": "NONE", "evidence": [], "warnings": [], "reasons": [], "configured": False}
|
||||
|
||||
def fail(cls, reason):
|
||||
out["classification"] = cls; out["reasons"].append(reason); return out
|
||||
|
||||
if cfg.get("crt_source_invalid"):
|
||||
return fail("CONFIG_REQUIRED", "COMPANY_CRT_SOURCE must be EMISSOR or C000004")
|
||||
companies = [{**c, "cnpj": digits(c.get("cnpj"))} for c in evidence.get("c000004", [])]
|
||||
emissors = [{**e, "cnpj": digits(e.get("cnpj"))} for e in evidence.get("emissor", [])]
|
||||
docs = evidence.get("nfce_keys", [])
|
||||
if not companies:
|
||||
return fail("UNRESOLVED", "C000004 has no company rows")
|
||||
|
||||
# ---- company selection -------------------------------------------------------------------
|
||||
selected = None; basis = None
|
||||
by_code, by_cnpj = cfg.get("company_code"), cfg.get("company_cnpj")
|
||||
if by_code or by_cnpj:
|
||||
out["configured"] = True
|
||||
cand = [c for c in companies if (not by_code or c["code"] == by_code) and (not by_cnpj or c["cnpj"] == by_cnpj)]
|
||||
if len(cand) != 1:
|
||||
return fail("CONFIG_REQUIRED", "configured company (COMPANY_CODE/COMPANY_CNPJ) does not identify exactly one C000004 row")
|
||||
selected, basis = cand[0], "CONFIGURED"
|
||||
elif len(companies) == 1:
|
||||
selected, basis = companies[0], "SINGLE_C000004_ROW"
|
||||
elif len(emissors) == 1 and emissors[0]["cnpj"]:
|
||||
match = [c for c in companies if c["cnpj"] == emissors[0]["cnpj"]]
|
||||
if len(match) != 1:
|
||||
return fail("CONFIG_REQUIRED", f"{len(companies)} C000004 rows and the EMISSOR CNPJ matches {len(match)} of them: configure COMPANY_CODE or COMPANY_CNPJ")
|
||||
selected, basis = match[0], "EMISSOR_CNPJ_MATCH"
|
||||
else:
|
||||
return fail("CONFIG_REQUIRED", f"{len(companies)} C000004 rows and no unique EMISSOR row to disambiguate: configure COMPANY_CODE or COMPANY_CNPJ")
|
||||
out["evidence"].append(f"company selected by {basis}")
|
||||
out.update(company_code=selected["code"], company_cnpj=mask_cnpj(selected["cnpj"]), company_cnpj_fingerprint=cnpj_fingerprint(selected["cnpj"]))
|
||||
out["source_of_truth"]["company"] = basis
|
||||
|
||||
# ---- emitter (EMISSOR) must describe the same company --------------------------------------
|
||||
emissor = None
|
||||
if emissors:
|
||||
same = [e for e in emissors if e["cnpj"] and e["cnpj"] == selected["cnpj"]]
|
||||
if same:
|
||||
emissor = same[0]; out["evidence"].append("EMISSOR.EMITENTE_CNPJ equals the selected company CNPJ")
|
||||
else:
|
||||
return fail("CONFLICT", "EMISSOR emitter CNPJ differs from the selected company (unexplained): refusing to guess")
|
||||
else:
|
||||
out["warnings"].append("NO_EMISSOR_ROW")
|
||||
|
||||
# ---- emitted documents must belong to the same CNPJ ------------------------------------------
|
||||
doc_ufs = set()
|
||||
if docs:
|
||||
total = sum(d["n"] for d in docs); own = sum(d["n"] for d in docs if digits(d["cnpj"]) == selected["cnpj"])
|
||||
out["evidence"].append(f"emitted NFC-e keys: {own}/{total} carry the selected CNPJ")
|
||||
if own == 0:
|
||||
return fail("CONFLICT", f"none of the {total} emitted NFC-e keys carries the selected company CNPJ")
|
||||
if own != total: # e.g. a database copy/migration that kept another emitter's history: evidence, not a reason to guess
|
||||
out["warnings"].append(f"HISTORICAL_NFCE_KEYS_FROM_OTHER_CNPJ {total - own}/{total}")
|
||||
doc_ufs = {IBGE_UF.get(d["cuf"]) for d in docs if digits(d["cnpj"]) == selected["cnpj"]} - {None}
|
||||
|
||||
# ---- UF ------------------------------------------------------------------------------------------
|
||||
uf_sources = {}
|
||||
if emissor and _s(emissor.get("uf")): uf_sources["EMISSOR"] = _s(emissor["uf"]).upper()
|
||||
if _s(selected.get("uf")): uf_sources["C000004"] = _s(selected["uf"]).upper()
|
||||
if len(doc_ufs) == 1: uf_sources["DOCUMENT_KEYS"] = next(iter(doc_ufs))
|
||||
if len(doc_ufs) > 1: return fail("CONFLICT", "emitted NFC-e keys carry several UFs")
|
||||
values = set(uf_sources.values())
|
||||
override = cfg.get("uf_override")
|
||||
if len(values) > 1:
|
||||
return fail("CONFLICT", f"UF_SOURCE_CONFLICT {uf_sources}")
|
||||
if override and values and override not in values:
|
||||
return fail("CONFLICT", f"COMPANY_UF={override} differs from the UF proved by the database {sorted(values)}: an override never replaces the resolver silently")
|
||||
if not values and override: # explicit override only when the database proves nothing
|
||||
uf_sources["CONFIG_OVERRIDE"] = override; values = {override}; out["warnings"].append("UF_FROM_EXPLICIT_OVERRIDE")
|
||||
if not values:
|
||||
return fail("UNRESOLVED", "UF_SOURCE_UNKNOWN")
|
||||
out["company_uf"] = next(iter(values)); out["source_of_truth"]["uf"] = "+".join(uf_sources)
|
||||
out["uf_status"] = "UF_SOURCE_CONFIRMED"
|
||||
|
||||
# ---- CRT ------------------------------------------------------------------------------------------
|
||||
e_crt = _s(emissor.get("crt")) if emissor else None
|
||||
c_crt = _s(selected.get("crt"))
|
||||
e_ok, c_ok = e_crt in VALID_CRT, c_crt in VALID_CRT
|
||||
if e_ok and c_ok and e_crt == c_crt:
|
||||
crt, src, status = e_crt, "EMISSOR+C000004", "CRT_SOURCE_CONFIRMED"
|
||||
elif e_ok and c_ok:
|
||||
choice = cfg.get("crt_source")
|
||||
if not choice:
|
||||
out["crt_candidates"] = {"EMISSOR": e_crt, "C000004": c_crt}
|
||||
return fail("CONFLICT", f"CRT_SOURCE_CONFLICT EMISSOR={e_crt} C000004={c_crt}: configure COMPANY_CRT_SOURCE (emission programs read EMISSOR)")
|
||||
crt, src, status = (e_crt if choice == "EMISSOR" else c_crt), f"{choice} (configured; conflict acknowledged)", "CRT_SOURCE_CONFLICT_RESOLVED_BY_CONFIG"
|
||||
out["warnings"].append(f"CRT_CONFLICT EMISSOR={e_crt} C000004={c_crt}")
|
||||
elif e_ok or c_ok:
|
||||
crt, src, status = (e_crt, "EMISSOR", "CRT_SOURCE_SINGLE") if e_ok else (c_crt, "C000004", "CRT_SOURCE_SINGLE")
|
||||
out["warnings"].append("CRT_HAS_A_SINGLE_SOURCE")
|
||||
else:
|
||||
return fail("UNRESOLVED", "CRT_SOURCE_UNKNOWN (no valid CRT in EMISSOR or C000004)")
|
||||
out.update(company_crt=crt, crt_status=status); out["source_of_truth"]["crt"] = src
|
||||
opt = _s(selected.get("optante"))
|
||||
if opt and ((crt in ("1", "2") and opt.upper() != "S") or (crt == "3" and opt.upper() == "S")):
|
||||
out["warnings"].append(f"OPTANTE_SIMPLES={opt} inconsistent with CRT={crt}")
|
||||
|
||||
agree = len(uf_sources) + (2 if status == "CRT_SOURCE_CONFIRMED" else 1)
|
||||
out["confidence"] = "HIGH" if (status == "CRT_SOURCE_CONFIRMED" and len(uf_sources) >= 2) else "MEDIUM"
|
||||
out["status"] = "PASS"
|
||||
out["classification"] = "CONFIGURED" if out["configured"] or status == "CRT_SOURCE_CONFLICT_RESOLVED_BY_CONFIG" else "AUTO_RESOLVED"
|
||||
out["reasons"] = []
|
||||
out["evidence_source_count"] = agree
|
||||
return out
|
||||
|
||||
|
||||
def collect_evidence(cursor):
|
||||
"""Read-only SELECTs. Missing optional tables/columns simply yield less evidence."""
|
||||
cursor.execute("SELECT TRIM(RDB$RELATION_NAME), TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME IN ('C000004','EMISSOR','NFCE','C000061')")
|
||||
cols = {}
|
||||
for t, f in cursor.fetchall():
|
||||
cols.setdefault(t, set()).add(f)
|
||||
ev = {"c000004": [], "emissor": [], "nfce_keys": [], "nf_by_filial": {}}
|
||||
c4 = cols.get("C000004", set())
|
||||
want = [("CODIGO", "code"), ("CNPJ", "cnpj"), ("IE", "ie"), ("UF", "uf"), ("CRT", "crt"), ("OPTANTE_SIMPLES", "optante"), ("FILIAL", "name")]
|
||||
use = [(c, k) for c, k in want if c in c4]
|
||||
if use:
|
||||
cursor.execute(f"SELECT {','.join(c for c, _ in use)} FROM C000004")
|
||||
for row in cursor.fetchall():
|
||||
ev["c000004"].append({k: _s(v) for (_, k), v in zip(use, row)})
|
||||
em = cols.get("EMISSOR", set())
|
||||
want = [("EMITENTE_CNPJ", "cnpj"), ("EMITENTE_UF", "uf"), ("EMITENTE_CRT", "crt")]
|
||||
use = [(c, k) for c, k in want if c in em]
|
||||
if use:
|
||||
cursor.execute(f"SELECT {','.join(c for c, _ in use)} FROM EMISSOR")
|
||||
for row in cursor.fetchall():
|
||||
ev["emissor"].append({k: _s(v) for (_, k), v in zip(use, row)})
|
||||
if "CHAVE" in cols.get("NFCE", set()):
|
||||
cursor.execute("SELECT SUBSTRING(CHAVE FROM 1 FOR 2), SUBSTRING(CHAVE FROM 7 FOR 14), COUNT(*) FROM NFCE WHERE CHAVE IS NOT NULL AND CHAR_LENGTH(TRIM(CHAVE)) = 44 "
|
||||
"GROUP BY SUBSTRING(CHAVE FROM 1 FOR 2), SUBSTRING(CHAVE FROM 7 FOR 14)")
|
||||
ev["nfce_keys"] = [{"cuf": _s(a), "cnpj": _s(b), "n": n} for a, b, n in cursor.fetchall()]
|
||||
if "CODFILIAL" in cols.get("C000061", set()):
|
||||
cursor.execute("SELECT CODFILIAL, COUNT(*) FROM C000061 GROUP BY CODFILIAL")
|
||||
ev["nf_by_filial"] = {_s(a): n for a, n in cursor.fetchall()}
|
||||
return ev
|
||||
|
||||
|
||||
def resolve_from_cursor(cursor, env=None):
|
||||
return resolve(collect_evidence(cursor), config_from_env(env or os.environ))
|
||||
|
||||
|
||||
def preflight_company_context(env=None):
|
||||
"""Opens the configured Firebird database read-only and resolves the company. Never prints the full CNPJ."""
|
||||
from .firebird_reader import _connect
|
||||
env = env or os.environ
|
||||
conn = _connect(env)
|
||||
try:
|
||||
return resolve_from_cursor(conn.cursor(), env)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def preflight_report(result):
|
||||
keys = ("company_code", "company_cnpj", "company_uf", "company_crt", "source_of_truth", "confidence", "classification", "warnings", "reasons")
|
||||
return {"COMPANY_CONTEXT": "PASS" if result["status"] == "PASS" else "FAIL", **{k: result.get(k) for k in keys}}
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Read-only fiscal coverage report for a set of products (used by the Agent's REPROCESS_ALL result and by the multi-database tooling).
|
||||
|
||||
Pure computation over Hub decisions + product rows + local evidence. It reuses the shipped pure engines in central.app
|
||||
(fiscal_decision, product_defaults, icms_context) — no fiscal rule lives here, only guards for the company context:
|
||||
the BA Anexo 1 ST/MVA dataset is applied to UF=BA only and CSOSN logic only under CRT=1.
|
||||
"""
|
||||
from collections import Counter
|
||||
|
||||
PRODUCT_COLUMNS = ("CODIGO", "PRODUTO", "CLASSIFICACAO_FISCAL", "CFOP", "CSOSN", "CST", "CEST", "IPI", "EX_TIPI", "TIPO", "PROD_COZINHA", "PRODUTO_SERVICO", "SITUACAO_TRIBUTARIA")
|
||||
|
||||
|
||||
def _s(v):
|
||||
return None if v is None else str(v).strip()
|
||||
|
||||
|
||||
def compute_coverage(products, decisions, company, company_default, refs, evidence):
|
||||
"""products/decisions aligned lists. company: {"uf","crt"} (resolved). evidence: {"recipes","composed","purchases","stock_entry"}."""
|
||||
from central.app.fiscal_decision import decide_product
|
||||
from central.app.icms_context import csosn_v2, product_role, st_status, withheld_status
|
||||
from central.app.product_defaults import cfop_default
|
||||
uf_ba = company.get("uf") == "BA"
|
||||
crt1 = company.get("crt") == "1"
|
||||
recipes, composed = evidence.get("recipes", set()), evidence.get("composed", set())
|
||||
purchases, stock_entry = evidence.get("purchases", {}), evidence.get("stock_entry", {})
|
||||
m, cfop_values, situ = Counter(), Counter(), Counter()
|
||||
for p, d in zip(products, decisions):
|
||||
p = {c: p.get(c) for c in PRODUCT_COLUMNS}
|
||||
code = _s(p["CODIGO"]); p["CODIGO"] = code
|
||||
cls = d.get("classification") or {}
|
||||
rec = decide_product(p, d, {"uf": company.get("uf") or "", "crt": company.get("crt") or ""}, refs)
|
||||
ncm_status = d.get("status"); raw_ncm = _s(p["CLASSIFICACAO_FISCAL"])
|
||||
m["NCM_MISSING" if not raw_ncm else ("NCM_VALID" if ncm_status == "EXACT_NCM" else ("NCM_LEGACY" if ncm_status == "LEGACY_NCM" else "NCM_INVALID"))] += 1
|
||||
ipi = rec["IPI"]; rk = (cls.get("ipi") or {}).get("rate_kind"); ist = d.get("ipi_status")
|
||||
if ist == "IPI_RESOLVED":
|
||||
m[{"NUMERIC": "IPI_NUMERIC", "NON_NUMERIC": "IPI_NON_NUMERIC", "BLANK": "IPI_SOURCE_BLANK"}.get(rk, "IPI_NO_MATCH")] += 1
|
||||
elif ist == "IPI_CANDIDATES":
|
||||
m["IPI_AMBIGUOUS"] += 1
|
||||
elif ist != "LEGACY_NCM" and raw_ncm:
|
||||
m["IPI_NO_MATCH"] += 1
|
||||
if ipi["action_class"] == "ALREADY_CORRECT":
|
||||
m["IPI_ALREADY_CORRECT"] += 1
|
||||
if ipi["write_eligible"]:
|
||||
m["IPI_CHANGE_REQUIRED"] += 1; m["IPI_WRITE_ELIGIBLE"] += 1
|
||||
cs_ = d.get("cest_status")
|
||||
if cs_ == "CEST_CONFIRMED":
|
||||
m["CEST_CONFIRMED"] += 1
|
||||
elif cs_ == "CEST_CANDIDATES":
|
||||
m["CEST_CANDIDATES"] += 1
|
||||
cf = cfop_default(p, ncm_status, d.get("ba_st_mva_status") if uf_ba else None, refs["cfop_valid"])
|
||||
cfop_values[_s(p["CFOP"])] += 1
|
||||
m["CFOP_RESOLVED" if cf["status"] == "CFOP_DEFAULT_RESOLVED" else "CFOP_CONTEXT_REQUIRED"] += 1
|
||||
ba = cls.get("ba_st_mva") or {}
|
||||
if uf_ba:
|
||||
st, _ = st_status(ncm_status, d.get("ba_st_mva_status"), cs_, p["CEST"], ba.get("cest"), None)
|
||||
else:
|
||||
st = "ST_UNKNOWN"
|
||||
m[st] += 1
|
||||
wh, _ = withheld_status(purchases.get(code, []))
|
||||
m[{"ST_ALREADY_WITHHELD_CONFIRMED": "ST_WITHHOLDING_CONFIRMED", "ST_ALREADY_WITHHELD_NOT_CONFIRMED": "ST_WITHHOLDING_NOT_CONFIRMED"}.get(wh, "ST_WITHHOLDING_UNKNOWN")] += 1
|
||||
role, _ = product_role(p["TIPO"], p["PROD_COZINHA"] == 1, code in recipes, code in composed, bool(_s(p["PRODUTO_SERVICO"])), bool(purchases.get(code)) or stock_entry.get(code, False))
|
||||
m["PRODUCT_ROLE_KNOWN" if role != "UNKNOWN" else "PRODUCT_ROLE_UNKNOWN"] += 1
|
||||
m["ROLE_" + role] += 1
|
||||
state = csosn_v2(p["CSOSN"], company_default, ncm_status, st if uf_ba else "ST_UNKNOWN", wh, role, refs["csosn"])["status"] if crt1 else "CSOSN_CONTEXT_REQUIRED"
|
||||
m["CSOSN_RESOLVED" if state == "CSOSN_RESOLVED" else ("CSOSN_CANDIDATES" if state == "CSOSN_CANDIDATE" else "CSOSN_CONTEXT_REQUIRED")] += 1
|
||||
situ[_s(p["SITUACAO_TRIBUTARIA"])] += 1
|
||||
n = len(products)
|
||||
fill = lambda f: sum(1 for p in products if _s(p.get(f)) not in (None, "")) # noqa: E731
|
||||
return {"TOTAL_PRODUCTS": n, "metrics": dict(m),
|
||||
"IPI_RESOLVED": m["IPI_NUMERIC"] + m["IPI_NON_NUMERIC"] + m["IPI_SOURCE_BLANK"],
|
||||
"quality": {"ncm_valid_share": round(m["NCM_VALID"] / n, 4) if n else None, "cest_filled": fill("CEST"), "csosn_filled": fill("CSOSN"), "cfop_filled": fill("CFOP"), "ipi_filled": fill("IPI"),
|
||||
"cfop_distinct": len(cfop_values), "cfop_top": cfop_values.most_common(6), "situacao_tributaria": dict(situ.most_common(6)),
|
||||
"purchase_items_C000088": sum(len(v) for v in purchases.values()), "products_with_purchase_evidence": len(purchases), "recipes": len(recipes), "composed_products": len(composed)},
|
||||
"guards_applied": {"st_dataset_ba_only": not uf_ba, "csosn_requires_crt1": not crt1}}
|
||||
|
||||
|
||||
def build_refs(get, products, company_uf):
|
||||
"""Reference-data snapshot from the Hub (`get(path) -> json`). Used by the engines for domain validity only."""
|
||||
refs = {}
|
||||
for key, domain in (("csosn", "CSOSN"), ("cst_icms", "CST_ICMS"), ("cst_pis", "CST_PIS"), ("cst_cofins", "CST_COFINS"), ("origem", "ORIGEM_MERCADORIA")):
|
||||
refs[key] = set(get(f"/v1/reference/fiscal_codes/{domain}")["codes"])
|
||||
valid = set()
|
||||
for code in {_s(p.get(f)) for p in products for f in ("CFOP", "CFOP_ENTRADA") if _s(p.get(f))}:
|
||||
if get(f"/v1/reference/cfop/{code}")["found"]:
|
||||
valid.add(code)
|
||||
refs["cfop_valid"] = valid
|
||||
refs["fcp_uf"] = {}
|
||||
fcp = get(f"/v1/reference/fcp/{company_uf}")
|
||||
if fcp.get("found"):
|
||||
refs["fcp_uf"][company_uf] = fcp
|
||||
refs["cbs_rate_2026"] = get("/v1/reference/rtc_cbs_rate/2026").get("rate")
|
||||
refs["cclasstrib_count"] = sum(d["record_count"] for d in get("/v1/datasets/status") if d["kind"] == "RTC_CCLASSTRIB" and d["status"] == "ACTIVE")
|
||||
return refs
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Shared read-only Firebird access for the Agent: full table or a CODIGO subset."""
|
||||
import os
|
||||
|
||||
FIELDS = "CODIGO,PRODUTO,GTIN AS EAN_GTIN,NCM,CLASSIFICACAO_FISCAL,CFOP,CST,ALIQUOTA,CEST,CSOSN,IPI,EX_TIPI"
|
||||
# Present in every audited SGV database but selected only when the column exists (capability-based, never assumed).
|
||||
OPTIONAL_FIELDS = ("TIPO", "PROD_COZINHA", "PRODUTO_SERVICO", "SITUACAO_TRIBUTARIA")
|
||||
|
||||
|
||||
def _select_fields(cursor):
|
||||
cursor.execute("SELECT TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME = 'C000025'")
|
||||
have = {r[0] for r in cursor.fetchall()}
|
||||
return FIELDS + "".join(f",{c}" for c in OPTIONAL_FIELDS if c in have)
|
||||
|
||||
|
||||
def _connect(env, read_only=True):
|
||||
import fdb
|
||||
kwargs = dict(
|
||||
database=env["FIREBIRD_DATABASE_PATH"],
|
||||
user=env["FB_USER"],
|
||||
password=env["FB_PASSWORD"],
|
||||
fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
)
|
||||
if read_only:
|
||||
kwargs["isolation_level"] = fdb.ISOLATION_LEVEL_READ_COMMITED_RO
|
||||
return fdb.connect(**kwargs)
|
||||
|
||||
|
||||
def check_connection(env=None):
|
||||
"""Best-effort read-only connectivity probe. Returns True/False, never raises."""
|
||||
env = env or os.environ
|
||||
try:
|
||||
conn = _connect(env)
|
||||
try:
|
||||
conn.cursor().execute("SELECT 1 FROM RDB$DATABASE")
|
||||
finally:
|
||||
conn.close()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def read_products(env=None, codigos=None):
|
||||
env = env or os.environ
|
||||
conn = _connect(env)
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
fields = _select_fields(cursor)
|
||||
if codigos:
|
||||
placeholders = ",".join("?" for _ in codigos)
|
||||
cursor.execute(f"SELECT {fields} FROM C000025 WHERE CODIGO IN ({placeholders}) ORDER BY CODIGO", tuple(codigos))
|
||||
else:
|
||||
cursor.execute(f"SELECT {fields} FROM C000025 ORDER BY CODIGO")
|
||||
names = [column[0].strip() for column in cursor.description]
|
||||
return [dict(zip(names, row)) for row in cursor.fetchall()]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def parse_company_context(rows):
|
||||
"""Pure: rows of (UF, CRT) from C000004 -> {"uf","crt"}. Fails closed (ValueError) if absent,
|
||||
blank or ambiguous — the engines never fall back to a hardcoded UF/CRT."""
|
||||
pairs = {(str(uf or "").strip().upper(), str(crt or "").strip()) for uf, crt in rows}
|
||||
if len(pairs) != 1:
|
||||
raise ValueError(f"company context unavailable or ambiguous ({len(pairs)} distinct UF/CRT rows in C000004)")
|
||||
uf, crt = next(iter(pairs))
|
||||
if len(uf) != 2 or crt not in ("1", "2", "3"):
|
||||
raise ValueError(f"company context invalid: UF={uf!r} CRT={crt!r}")
|
||||
return {"uf": uf, "crt": crt}
|
||||
|
||||
|
||||
def read_company_context(env=None):
|
||||
"""Resolve the ACTIVE company (UF/CRT) with the deterministic resolver (see company_context.py); fails closed with
|
||||
CompanyContextError when the context is ambiguous, conflicting or unresolved. No hardcoded company values, no first-row fallback."""
|
||||
from .company_context import CompanyContextError, preflight_company_context
|
||||
result = preflight_company_context(env or os.environ)
|
||||
if result["status"] != "PASS":
|
||||
raise CompanyContextError(result)
|
||||
return {"uf": result["company_uf"], "crt": result["company_crt"], "company_code": result["company_code"], "source_of_truth": result["source_of_truth"], "confidence": result["confidence"]}
|
||||
|
||||
|
||||
def read_evidence(env=None, selected_company_code=None):
|
||||
"""Read-only evidence tables used by the coverage report (purchases, stock entries, recipes, compositions) and the company's
|
||||
default CSOSN. Every table/column is optional: a missing one just means less evidence (capability-based)."""
|
||||
from collections import defaultdict
|
||||
env = env or os.environ
|
||||
conn = _connect(env)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT TRIM(RDB$RELATION_NAME), TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME IN ('C000004','C000032','C000088','C000100','RECEITA_INGREDIENTES')")
|
||||
cols = defaultdict(set)
|
||||
for t, f in cur.fetchall():
|
||||
cols[t].add(f)
|
||||
has = lambda t, *c: t in cols and all(x in cols[t] for x in c) # noqa: E731
|
||||
s = lambda v: None if v is None else str(v).strip() # noqa: E731
|
||||
ev = {"recipes": set(), "composed": set(), "purchases": defaultdict(list), "stock_entry": {}, "stock_in": {}, "company_default_csosn": None}
|
||||
if has("RECEITA_INGREDIENTES", "CODRECEITA"):
|
||||
cur.execute("SELECT CODRECEITA FROM RECEITA_INGREDIENTES WHERE CODRECEITA IS NOT NULL")
|
||||
ev["recipes"] = {str(r[0]).zfill(6) for r in cur.fetchall()}
|
||||
if has("C000032", "CODPRODUTO", "TIPO"):
|
||||
cur.execute("SELECT DISTINCT CODPRODUTO FROM C000032 WHERE TIPO = 'COMPOSICAO'")
|
||||
ev["composed"] = {s(r[0]) for r in cur.fetchall()}
|
||||
if has("C000088", "CODPRODUTO", "ICMS_RETIDO", "ICMS_VALORRETIDO", "SUB_VALOR"):
|
||||
cur.execute("SELECT CODPRODUTO, ICMS_RETIDO, ICMS_VALORRETIDO, SUB_VALOR FROM C000088")
|
||||
for cod, ret, vr, sv in cur.fetchall():
|
||||
ev["purchases"][s(cod)].append({"icms_retido": s(ret), "valor_retido": vr, "sub_valor": sv})
|
||||
if has("C000100", "CODPRODUTO", "ULTIMA_ENTRADA", "NOTA_FISCAL"):
|
||||
cur.execute("SELECT CODPRODUTO, ULTIMA_ENTRADA, NOTA_FISCAL FROM C000100")
|
||||
ev["stock_entry"] = {s(r[0]): (r[1] is not None or s(r[2]) not in (None, "", "000000")) for r in cur.fetchall()}
|
||||
if has("C000100", "CODPRODUTO", "ENTRADAS"):
|
||||
cur.execute("SELECT CODPRODUTO, ENTRADAS FROM C000100")
|
||||
ev["stock_in"] = {s(r[0]): bool(r[1] and float(r[1]) > 0) for r in cur.fetchall()}
|
||||
if has("C000004", "ICMS_CSOSN", "CODIGO"):
|
||||
cur.execute("SELECT CODIGO, ICMS_CSOSN FROM C000004")
|
||||
rows = {s(a): s(b) for a, b in cur.fetchall()}
|
||||
ev["company_default_csosn"] = rows.get(selected_company_code) if selected_company_code else (next(iter(rows.values())) if len(rows) == 1 else None)
|
||||
ev["purchases"] = dict(ev["purchases"])
|
||||
return ev
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Decide whether a classified product's IPI rate may be auto-applied.
|
||||
|
||||
Scope is deliberately narrow: C000025.IPI only, and only when the Hub's TIPI
|
||||
match is a single unambiguous numeric rate. Never auto-applies NT/non-numeric
|
||||
markers, blank source rates, ambiguous EX candidates, or legacy/invalid NCM.
|
||||
"""
|
||||
|
||||
|
||||
def ipi_write_decision(decision, current_ipi):
|
||||
"""current_ipi must be a value freshly read from Firebird immediately before
|
||||
this call — None here means the ERP field is genuinely NULL right now, not
|
||||
that it wasn't read."""
|
||||
if decision.get("status") == "LEGACY_NCM":
|
||||
return {"eligible": False, "reason_code": "LEGACY_NCM", "proposed_value": None}
|
||||
if decision.get("ipi_status") != "IPI_RESOLVED":
|
||||
return {"eligible": False, "reason_code": f"IPI_STATUS_{decision.get('ipi_status')}", "proposed_value": None}
|
||||
|
||||
ipi = (decision.get("classification") or {}).get("ipi")
|
||||
if not ipi:
|
||||
return {"eligible": False, "reason_code": "NO_IPI_DECISION", "proposed_value": None}
|
||||
|
||||
rate_kind = ipi.get("rate_kind")
|
||||
if rate_kind != "NUMERIC":
|
||||
return {"eligible": False, "reason_code": f"RATE_KIND_{rate_kind}", "proposed_value": None}
|
||||
|
||||
proposed_value = ipi.get("rate_numeric")
|
||||
if proposed_value is None:
|
||||
return {"eligible": False, "reason_code": "NO_NUMERIC_VALUE", "proposed_value": None}
|
||||
|
||||
if current_ipi is not None and str(current_ipi).strip() == str(proposed_value).strip():
|
||||
return {"eligible": False, "reason_code": "SAME_VALUE", "proposed_value": proposed_value}
|
||||
|
||||
return {"eligible": True, "reason_code": "CHANGE_REQUIRED", "proposed_value": proposed_value}
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Shared per-product IPI auto-apply pipeline: Hub decision -> dry-run ->
|
||||
guarded write -> local SQLite audit. Used by both the manual CLI
|
||||
(run_agent_auto_apply.py) and the long-running job-polling service
|
||||
(run_agent_service.py) so the processing logic exists in exactly one place.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from .ipi_auto_apply import ipi_write_decision
|
||||
from .runner import product_fingerprint
|
||||
|
||||
|
||||
def now():
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def run_pipeline(products, decisions, state, firebird_writes, write_connection, executor, database_path, authorized_path=None):
|
||||
"""Process every (product, decision) pair. Returns a summary dict.
|
||||
|
||||
products come from a fresh Firebird read (current IPI is always "known").
|
||||
decisions come from the Hub's /v1/classify/batch, one per product, same order.
|
||||
"""
|
||||
summary = {
|
||||
"processed": 0,
|
||||
"auto_apply_eligible": 0,
|
||||
"writes_attempted": 0,
|
||||
"writes_succeeded": 0,
|
||||
"writes_failed": 0,
|
||||
"skipped_not_eligible": 0,
|
||||
"no_ops": 0,
|
||||
}
|
||||
for product, decision in zip(products, decisions):
|
||||
code = product["CODIGO"]
|
||||
fingerprint = product_fingerprint(product)
|
||||
current_ipi = product.get("IPI")
|
||||
summary["processed"] += 1
|
||||
|
||||
wd = ipi_write_decision(decision, current_ipi)
|
||||
decision_status = decision.get("status")
|
||||
ipi_status = decision.get("ipi_status")
|
||||
|
||||
if not firebird_writes:
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "DRY_RUN_ONLY", current_ipi, wd["proposed_value"], None, wd["reason_code"], now())
|
||||
if wd["eligible"]:
|
||||
summary["auto_apply_eligible"] += 1
|
||||
elif wd["reason_code"] == "SAME_VALUE":
|
||||
summary["no_ops"] += 1
|
||||
continue
|
||||
|
||||
if not wd["eligible"]:
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "SKIPPED_NOT_ELIGIBLE", current_ipi, wd["proposed_value"], None, wd["reason_code"], now())
|
||||
summary["skipped_not_eligible"] += 1
|
||||
if wd["reason_code"] == "SAME_VALUE":
|
||||
summary["no_ops"] += 1
|
||||
continue
|
||||
|
||||
summary["auto_apply_eligible"] += 1
|
||||
summary["writes_attempted"] += 1
|
||||
proposal = {
|
||||
"product_code": code,
|
||||
"field": "IPI",
|
||||
"current_value": current_ipi,
|
||||
"proposed_value": wd["proposed_value"],
|
||||
"status": "CHANGE_REQUIRED",
|
||||
"write_eligible": True,
|
||||
"source_dataset": "TIPI_IPI",
|
||||
"source_version": decision.get("dataset_versions", {}).get("tipi_ipi"),
|
||||
}
|
||||
try:
|
||||
record = executor.execute_guarded_update(write_connection, database_path=database_path, proposal=proposal, now=now(), authorized_path=authorized_path)
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "SUCCEEDED", record.before_value, record.after_value, f"rowcount={record.rowcount}", wd["reason_code"], now())
|
||||
summary["writes_succeeded"] += 1
|
||||
except executor.WriteRefused as error:
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "FAILED", current_ipi, wd["proposed_value"], str(error), wd["reason_code"], now())
|
||||
summary["writes_failed"] += 1
|
||||
|
||||
return summary
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Single installation preflight for the GTI Fiscal Agent. READ-ONLY: never writes to Firebird, never prints secrets.
|
||||
|
||||
PREFLIGHT=PASS | FAIL (+ blockers, one line each)
|
||||
|
||||
Checks: config, FISCAL_MODE/FIREBIRD_WRITES, Hub reachable/auth/version, Firebird reachable + version + ODS, C000004/C000025,
|
||||
capabilities (minimum: NCM, IPI, CEST), company context (UF/CRT, deterministic resolver), schema/type guards, SQLite state, log dir.
|
||||
"""
|
||||
import os
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_VERSION_DEFAULT = "1.0.0-rc1"
|
||||
REQUIRED_ENV = ("GTI_HUB_URL", "AGENT_TOKEN", "FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")
|
||||
REQUIRED_CAPABILITIES = ("supports_ncm", "supports_ipi", "supports_cest")
|
||||
FISCAL_MODES = ("DRY_RUN", "AUTO_APPLY")
|
||||
|
||||
|
||||
def _check(name, ok, detail="", warn=False):
|
||||
return {"name": name, "status": "PASS" if ok else ("WARN" if warn else "FAIL"), "detail": detail}
|
||||
|
||||
|
||||
def major_minor(version):
|
||||
parts = str(version or "").replace("-", ".").replace("rc", ".").split(".")
|
||||
return ".".join(parts[:2])
|
||||
|
||||
|
||||
def default_http_get(url, token=None, timeout=10):
|
||||
import httpx
|
||||
headers = {"Authorization": f"Bearer {token}"} if token else {}
|
||||
r = httpx.get(url, headers=headers, timeout=timeout)
|
||||
return r.status_code, (r.json() if r.headers.get("content-type", "").startswith("application/json") else {})
|
||||
|
||||
|
||||
def firebird_probe(env):
|
||||
"""Read-only introspection of the configured database."""
|
||||
import fdb
|
||||
from .company_context import collect_evidence, config_from_env, resolve
|
||||
conn = fdb.connect(database=env["FIREBIRD_DATABASE_PATH"], user=env["FB_USER"], password=env["FB_PASSWORD"], fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute("""SELECT TRIM(rf.RDB$RELATION_NAME), TRIM(rf.RDB$FIELD_NAME), f.RDB$FIELD_TYPE, COALESCE(f.RDB$CHARACTER_LENGTH, f.RDB$FIELD_LENGTH)
|
||||
FROM RDB$RELATION_FIELDS rf JOIN RDB$FIELDS f ON f.RDB$FIELD_NAME = rf.RDB$FIELD_SOURCE WHERE rf.RDB$RELATION_NAME IN ('C000004', 'C000025')""")
|
||||
tables = {}
|
||||
for t, c, ty, ln in cur.fetchall():
|
||||
tables.setdefault(t, {})[c] = (ty, ln)
|
||||
products = None
|
||||
if "C000025" in tables:
|
||||
cur.execute("SELECT COUNT(*) FROM C000025")
|
||||
products = cur.fetchone()[0]
|
||||
company = resolve(collect_evidence(cur), config_from_env(env))
|
||||
return {"server_version": getattr(conn, "server_version", None), "ods": f"{getattr(conn, 'ods_version', '?')}.{getattr(conn, 'ods_minor_version', '?')}",
|
||||
"tables": tables, "products": products, "company": company}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def run_preflight(env=None, *, http_get=default_http_get, probe=firebird_probe, agent_version=None):
|
||||
from .capabilities import detect_capabilities, type_ok
|
||||
from .company_context import preflight_report
|
||||
env = env if env is not None else os.environ
|
||||
checks, info = [], {}
|
||||
agent_version = agent_version or env.get("AGENT_VERSION") or AGENT_VERSION_DEFAULT
|
||||
|
||||
missing = [k for k in REQUIRED_ENV if not (env.get(k) or "").strip()]
|
||||
checks.append(_check("CONFIG", not missing, f"missing: {', '.join(missing)}" if missing else "required variables present"))
|
||||
|
||||
mode = (env.get("FISCAL_MODE") or "").strip().upper()
|
||||
writes = (env.get("FIREBIRD_WRITES") or "0").strip() == "1"
|
||||
checks.append(_check("FISCAL_MODE", mode in FISCAL_MODES, f"FISCAL_MODE={mode or 'unset'} (must be one of {FISCAL_MODES})"))
|
||||
dry_conflict = mode == "DRY_RUN" and writes
|
||||
checks.append(_check("FIREBIRD_WRITES", not dry_conflict, "FIREBIRD_WRITES=1 conflicts with FISCAL_MODE=DRY_RUN (writes stay disabled; fix the config)" if dry_conflict else f"requested={int(writes)}"))
|
||||
effective = writes and mode == "AUTO_APPLY"
|
||||
info["effective_firebird_writes"] = effective
|
||||
if effective:
|
||||
ok = bool((env.get("AUTHORIZED_WRITE_DATABASE_PATH") or "").strip() and (env.get("EXPECTED_SCHEMA_FINGERPRINT") or "").strip())
|
||||
checks.append(_check("WRITE_AUTHORIZATION", ok, "AUTO_APPLY needs AUTHORIZED_WRITE_DATABASE_PATH and EXPECTED_SCHEMA_FINGERPRINT (two independent layers)"))
|
||||
|
||||
hub = (env.get("GTI_HUB_URL") or "").rstrip("/")
|
||||
hub_ok = False
|
||||
if hub:
|
||||
try:
|
||||
code, _ = http_get(f"{hub}/health")
|
||||
hub_ok = code == 200
|
||||
checks.append(_check("HUB_REACHABLE", hub_ok, f"GET /health -> {code}"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("HUB_REACHABLE", False, type(e).__name__))
|
||||
if hub_ok:
|
||||
try:
|
||||
code, body = http_get(f"{hub}/version")
|
||||
hub_v = body.get("version")
|
||||
info["hub_version"] = hub_v
|
||||
checks.append(_check("HUB_VERSION_COMPATIBLE", code == 200 and major_minor(hub_v) == major_minor(agent_version), f"hub={hub_v} agent={agent_version} (major.minor must match)"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("HUB_VERSION_COMPATIBLE", False, type(e).__name__))
|
||||
try:
|
||||
code, _ = http_get(f"{hub}/v1/datasets/status", token=env.get("AGENT_TOKEN"))
|
||||
checks.append(_check("AUTH", code == 200, f"agent token -> HTTP {code}"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("AUTH", False, type(e).__name__))
|
||||
|
||||
fb = None
|
||||
if all((env.get(k) or "").strip() for k in ("FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")):
|
||||
try:
|
||||
fb = probe(env)
|
||||
checks.append(_check("FIREBIRD_REACHABLE", True, "read-only connection opened"))
|
||||
info.update(firebird_version=fb.get("server_version"), ods=fb.get("ods"), products=fb.get("products"))
|
||||
checks.append(_check("FIREBIRD_VERSION_ODS", bool(fb.get("server_version")), f"{fb.get('server_version')} / ODS {fb.get('ods')}"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("FIREBIRD_REACHABLE", False, f"{type(e).__name__}: {str(e)[:80]}"))
|
||||
if fb:
|
||||
t = fb["tables"]
|
||||
checks.append(_check("C000004", "C000004" in t, "company table present" if "C000004" in t else "missing"))
|
||||
checks.append(_check("C000025", "C000025" in t, "product table present" if "C000025" in t else "missing"))
|
||||
cols = {c: ty for c, (ty, _) in t.get("C000025", {}).items()}
|
||||
caps = detect_capabilities(cols)
|
||||
info["capabilities"] = caps
|
||||
lacking = [c for c in REQUIRED_CAPABILITIES if not caps.get(c)]
|
||||
checks.append(_check("CAPABILITIES", not lacking, f"missing required: {', '.join(lacking)}" if lacking else f"required present; optional true: {sum(1 for c, v in caps.items() if v and c not in REQUIRED_CAPABILITIES)}"))
|
||||
rep = preflight_report(fb["company"])
|
||||
info["company_context"] = rep
|
||||
checks.append(_check("COMPANY_CONTEXT", rep["COMPANY_CONTEXT"] == "PASS", "; ".join(rep.get("reasons") or []) or f"{rep['classification']} source={rep['source_of_truth']}"))
|
||||
checks.append(_check("UF", bool(rep.get("company_uf")), str(rep.get("company_uf"))))
|
||||
checks.append(_check("CRT", bool(rep.get("company_crt")), str(rep.get("company_crt"))))
|
||||
writable = [f for f in ("IPI", "CEST", "EX_TIPI", "CLASSIFICACAO_FISCAL") if f in t.get("C000025", {})]
|
||||
bad = [f for f in writable if not type_ok(f, t["C000025"][f][0])]
|
||||
lens = {f: t["C000025"][f][1] for f in ("CEST", "EX_TIPI") if f in t.get("C000025", {})}
|
||||
checks.append(_check("SCHEMA_TYPE_GUARDS", not bad and len(writable) >= 3, f"incompatible: {bad}" if bad else f"types ok for {writable}; text lengths {lens} (the executor re-checks length/type before any write)"))
|
||||
|
||||
state_db = env.get("AGENT_STATE_DB") or str(Path.cwd() / "agent_state.sqlite")
|
||||
try:
|
||||
Path(state_db).parent.mkdir(parents=True, exist_ok=True)
|
||||
c = sqlite3.connect(state_db)
|
||||
c.execute("SELECT 1")
|
||||
c.close()
|
||||
checks.append(_check("SQLITE", True, "state database opens"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("SQLITE", False, type(e).__name__))
|
||||
log_path = Path(env.get("AGENT_LOG_PATH") or Path.cwd() / "logs" / "agent.log")
|
||||
try:
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
checks.append(_check("LOG_DIRECTORY", os.access(log_path.parent, os.W_OK), "log directory writable"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("LOG_DIRECTORY", False, type(e).__name__))
|
||||
|
||||
blockers = [f"{c['name']}: {c['detail']}" for c in checks if c["status"] == "FAIL"]
|
||||
return {"PREFLIGHT": "FAIL" if blockers else "PASS", "blockers": blockers, "checks": checks, "agent_version": agent_version, "fiscal_mode": mode or None, **info}
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Gate Firebird reads on the registered schema fingerprint."""
|
||||
import hashlib, json
|
||||
REFERENCE_SCHEMA_V1 = "1198f89897532b643439f06b729e93b189018f56e7d478e95095f8f56977005b"
|
||||
|
||||
def product_fingerprint(product):
|
||||
fields = ("PRODUTO", "EAN_GTIN", "NCM", "CLASSIFICACAO_FISCAL", "CFOP", "CST", "ALIQUOTA", "CEST", "CSOSN")
|
||||
return hashlib.sha256(json.dumps({key: product.get(key) for key in fields}, sort_keys=True, default=str).encode()).hexdigest()
|
||||
def to_hub_product(product):
|
||||
# C000025 has NCM empty and stores the confirmed eight-digit NCM in CLASSIFICACAO_FISCAL.
|
||||
return {"product_id": product["CODIGO"], "description": product.get("PRODUTO", ""), "gtin": product.get("EAN_GTIN"), "current_ncm": product.get("CLASSIFICACAO_FISCAL")}
|
||||
def run_sample(products, client, state, schema_fingerprint):
|
||||
if schema_fingerprint != REFERENCE_SCHEMA_V1: return {"status": "SCHEMA_UNKNOWN", "results": [], "firebird_writes": 0}
|
||||
outbound = []
|
||||
for product in products:
|
||||
fingerprint = product_fingerprint(product)
|
||||
if state.enqueue(product["CODIGO"], fingerprint, "pending", "0.1.0"): outbound.append((product, fingerprint))
|
||||
if not outbound: return {"status": "OK", "results": [], "firebird_writes": 0, "api_requests": 0}
|
||||
try: results = client.classify_batch([to_hub_product(product) for product, _ in outbound])
|
||||
except Exception:
|
||||
for product, _ in outbound: state.retry(product["CODIGO"])
|
||||
return {"status": "API_UNAVAILABLE", "results": [], "firebird_writes": 0}
|
||||
for (product, fingerprint), result in zip(outbound, results): state.result_received(product["CODIGO"], fingerprint, result["dataset_versions"].get("ncm") or "none", result["engine_version"])
|
||||
return {"status": "OK", "results": results, "firebird_writes": 0, "api_requests": 1}
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Local SQLite outbox; it has no Firebird dependency or write path."""
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
class AgentState:
|
||||
def __init__(self, path):
|
||||
self.connection = sqlite3.connect(Path(path)); self.connection.row_factory = sqlite3.Row
|
||||
self.connection.executescript("CREATE TABLE IF NOT EXISTS product_state(product_id TEXT PRIMARY KEY, source_fingerprint TEXT NOT NULL, dataset_version TEXT NOT NULL, engine_version TEXT NOT NULL, status TEXT NOT NULL, last_processed_at TEXT, last_result_hash TEXT, reason_code TEXT); CREATE TABLE IF NOT EXISTS outbox(product_id TEXT PRIMARY KEY, source_fingerprint TEXT NOT NULL, status TEXT NOT NULL, retries INTEGER NOT NULL DEFAULT 0); CREATE TABLE IF NOT EXISTS ipi_write_log(product_id TEXT NOT NULL, source_fingerprint TEXT NOT NULL, decision_status TEXT, ipi_status TEXT, write_status TEXT NOT NULL, before_value TEXT, proposed_value TEXT, result TEXT, reason_code TEXT, timestamp TEXT NOT NULL);")
|
||||
def enqueue(self, product_id, source_fingerprint, dataset_version, engine_version):
|
||||
row = self.connection.execute("SELECT * FROM product_state WHERE product_id=?", (product_id,)).fetchone()
|
||||
if row and tuple(row[key] for key in ("source_fingerprint", "engine_version", "status")) == (source_fingerprint, engine_version, "RESULT_RECEIVED"): return False
|
||||
self.connection.execute("INSERT OR REPLACE INTO product_state(product_id,source_fingerprint,dataset_version,engine_version,status) VALUES(?,?,?,?,?)", (product_id, source_fingerprint, dataset_version, engine_version, "PENDING_API")); self.connection.execute("INSERT OR REPLACE INTO outbox(product_id,source_fingerprint,status) VALUES(?,?,?)", (product_id, source_fingerprint, "PENDING_API")); self.connection.commit(); return True
|
||||
def pending(self): return [dict(row) for row in self.connection.execute("SELECT * FROM outbox WHERE status='PENDING_API'")]
|
||||
def retry(self, product_id): self.connection.execute("UPDATE outbox SET retries=retries+1,status='PENDING_API' WHERE product_id=?", (product_id,)); self.connection.commit()
|
||||
def result_received(self, product_id, source_fingerprint, dataset_version, engine_version):
|
||||
self.connection.execute("UPDATE product_state SET source_fingerprint=?,dataset_version=?,engine_version=?,status='RESULT_RECEIVED' WHERE product_id=?", (source_fingerprint, dataset_version, engine_version, product_id)); self.connection.execute("DELETE FROM outbox WHERE product_id=?", (product_id,)); self.connection.commit()
|
||||
def log_ipi_write(self, product_id, source_fingerprint, decision_status, ipi_status, write_status, before_value, proposed_value, result, reason_code, timestamp):
|
||||
self.connection.execute(
|
||||
"INSERT INTO ipi_write_log(product_id,source_fingerprint,decision_status,ipi_status,write_status,before_value,proposed_value,result,reason_code,timestamp) VALUES(?,?,?,?,?,?,?,?,?,?)",
|
||||
(product_id, source_fingerprint, decision_status, ipi_status, write_status, str(before_value) if before_value is not None else None, str(proposed_value) if proposed_value is not None else None, result, reason_code, timestamp),
|
||||
)
|
||||
self.connection.commit()
|
||||
def ipi_write_log_for(self, product_id):
|
||||
return [dict(row) for row in self.connection.execute("SELECT * FROM ipi_write_log WHERE product_id=? ORDER BY timestamp", (product_id,))]
|
||||
@@ -0,0 +1,3 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).parents[1] / "src"))
|
||||
@@ -0,0 +1,117 @@
|
||||
import pytest
|
||||
|
||||
from gti_fiscal_agent.company_context import CompanyContextError, config_from_env, mask_cnpj, preflight_report, resolve
|
||||
from gti_fiscal_agent.firebird_reader import parse_company_context
|
||||
|
||||
CN_A, CN_B, CN_C = "11111111000111", "22222222000122", "33333333000133"
|
||||
|
||||
|
||||
def co(code, cnpj, uf="BA", crt="1", optante="S"):
|
||||
return {"code": code, "cnpj": cnpj, "uf": uf, "crt": crt, "optante": optante, "name": "X"}
|
||||
|
||||
|
||||
def ev(companies, emissor=None, docs=None):
|
||||
return {"c000004": companies, "emissor": emissor if emissor is not None else [], "nfce_keys": docs or [], "nf_by_filial": {}}
|
||||
|
||||
|
||||
def emis(cnpj=CN_A, uf="BA", crt="1"):
|
||||
return {"cnpj": cnpj, "uf": uf, "crt": crt}
|
||||
|
||||
|
||||
DOC_A = [{"cuf": "29", "cnpj": CN_A, "n": 100}]
|
||||
|
||||
|
||||
def test_single_company_all_sources_agree_is_auto_resolved():
|
||||
r = resolve(ev([co("000001", CN_A)], [emis()], DOC_A))
|
||||
assert r["status"] == "PASS" and r["classification"] == "AUTO_RESOLVED" and r["company_uf"] == "BA" and r["company_crt"] == "1" and r["confidence"] == "HIGH"
|
||||
assert "DOCUMENT_KEYS" in r["source_of_truth"]["uf"]
|
||||
|
||||
|
||||
def test_multiple_rows_resolved_only_through_emitter_identity_never_first_row():
|
||||
rows = [co("000001", CN_B, "SP", None, "N"), co("000005", CN_A), co("000007", CN_C, "BA", None, "N")]
|
||||
r = resolve(ev(rows, [emis(CN_A)], DOC_A))
|
||||
assert r["status"] == "PASS" and r["company_code"] == "000005" and r["source_of_truth"]["company"] == "EMISSOR_CNPJ_MATCH"
|
||||
|
||||
|
||||
def test_multiple_rows_without_disambiguation_requires_config():
|
||||
rows = [co("000001", CN_A), co("000002", CN_B)]
|
||||
r = resolve(ev(rows, []))
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFIG_REQUIRED" and r["company_code"] is None
|
||||
assert resolve(ev(rows, [emis(CN_C)]))["classification"] == "CONFIG_REQUIRED" # emitter matches none
|
||||
ok = resolve(ev(rows, [], []), {"company_code": "000002"})
|
||||
assert ok["status"] == "PASS" and ok["company_code"] == "000002" and ok["classification"] == "CONFIGURED"
|
||||
|
||||
|
||||
def test_configured_company_must_exist_and_agree_with_emitter():
|
||||
rows = [co("000001", CN_A), co("000002", CN_B)]
|
||||
assert resolve(ev(rows, [emis(CN_A)]), {"company_cnpj": "99999999999999"})["classification"] == "CONFIG_REQUIRED"
|
||||
r = resolve(ev(rows, [emis(CN_A)], DOC_A), {"company_code": "000002"})
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFLICT" # selected company differs from the emitter: unexplained
|
||||
|
||||
|
||||
def test_documents_only_from_another_cnpj_conflict_but_mixed_history_is_a_warning():
|
||||
r = resolve(ev([co("000001", CN_A)], [emis()], [{"cuf": "29", "cnpj": CN_B, "n": 10}]))
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFLICT"
|
||||
r = resolve(ev([co("000001", CN_A)], [emis()], [{"cuf": "29", "cnpj": CN_A, "n": 10}, {"cuf": "29", "cnpj": CN_B, "n": 1}]))
|
||||
assert r["status"] == "PASS" and any("HISTORICAL_NFCE_KEYS_FROM_OTHER_CNPJ" in w for w in r["warnings"])
|
||||
|
||||
|
||||
def test_uf_conflict_and_unknown():
|
||||
assert resolve(ev([co("1", CN_A, "BA")], [emis(uf="SP")]))["classification"] == "CONFLICT"
|
||||
assert resolve(ev([co("1", CN_A, "BA")], [emis()], [{"cuf": "35", "cnpj": CN_A, "n": 5}]))["classification"] == "CONFLICT" # key says SP
|
||||
assert resolve(ev([co("1", CN_A, "")], [emis(uf="")]))["classification"] == "UNRESOLVED"
|
||||
|
||||
|
||||
def test_crt_conflict_needs_explicit_source_and_records_it():
|
||||
base = ev([co("1", CN_A, crt="3")], [emis(crt="1")], DOC_A)
|
||||
r = resolve(base)
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFLICT" and r["crt_candidates"] == {"EMISSOR": "1", "C000004": "3"}
|
||||
ok = resolve(base, {"crt_source": "EMISSOR"})
|
||||
assert ok["status"] == "PASS" and ok["company_crt"] == "1" and ok["classification"] == "CONFIGURED" and ok["confidence"] == "MEDIUM"
|
||||
assert resolve(base, {"crt_source": "C000004"})["company_crt"] == "3"
|
||||
assert resolve(base, {"crt_source_invalid": True})["classification"] == "CONFIG_REQUIRED"
|
||||
|
||||
|
||||
def test_crt_unknown_fails_and_single_source_is_flagged():
|
||||
assert resolve(ev([co("1", CN_A, crt=None)], [emis(crt="")]))["classification"] == "UNRESOLVED"
|
||||
r = resolve(ev([co("1", CN_A, crt=None)], [emis(crt="1")]))
|
||||
assert r["status"] == "PASS" and "CRT_HAS_A_SINGLE_SOURCE" in r["warnings"]
|
||||
|
||||
|
||||
def test_optante_inconsistency_is_a_warning_not_a_guess():
|
||||
r = resolve(ev([co("1", CN_A, crt="3", optante="S")], [emis(crt="3")]))
|
||||
assert r["status"] == "PASS" and any("OPTANTE_SIMPLES" in w for w in r["warnings"])
|
||||
|
||||
|
||||
def test_full_cnpj_is_never_exposed():
|
||||
r = resolve(ev([co("1", CN_A)], [emis()], DOC_A))
|
||||
dump = str(preflight_report(r)) + str(r)
|
||||
assert CN_A not in dump and mask_cnpj(CN_A) == "11**********11"
|
||||
|
||||
|
||||
def test_read_company_context_fails_closed(monkeypatch):
|
||||
from gti_fiscal_agent import company_context as cc, firebird_reader as fr
|
||||
monkeypatch.setattr(cc, "preflight_company_context", lambda env: resolve(ev([co("1", CN_A), co("2", CN_B)], [])))
|
||||
with pytest.raises(CompanyContextError):
|
||||
fr.read_company_context({})
|
||||
monkeypatch.setattr(cc, "preflight_company_context", lambda env: resolve(ev([co("1", CN_A)], [emis()], DOC_A)))
|
||||
assert fr.read_company_context({})["uf"] == "BA"
|
||||
|
||||
|
||||
def test_config_from_env_and_legacy_parser_unchanged():
|
||||
assert config_from_env({"COMPANY_CODE": " 000005 ", "COMPANY_CNPJ": "11.111.111/0001-11", "COMPANY_CRT_SOURCE": "emissor"}) == {"company_code": "000005", "company_cnpj": "11111111000111", "crt_source": "EMISSOR", "crt_source_invalid": False, "uf_override": None}
|
||||
assert config_from_env({"COMPANY_CRT_SOURCE": "x"})["crt_source_invalid"] is True
|
||||
assert parse_company_context([("ba", "1")]) == {"uf": "BA", "crt": "1"}
|
||||
|
||||
|
||||
def test_auto_crt_source_and_uf_override_semantics():
|
||||
assert config_from_env({"COMPANY_CRT_SOURCE": "AUTO"}) ["crt_source"] is None and config_from_env({"COMPANY_CRT_SOURCE": "AUTO"})["crt_source_invalid"] is False
|
||||
conflict = ev([co("1", CN_A, crt="3")], [emis(crt="1")], DOC_A)
|
||||
assert resolve(conflict, config_from_env({"COMPANY_CRT_SOURCE": "AUTO"}))["status"] == "FAIL" # AUTO never hides a CRT conflict
|
||||
good = ev([co("1", CN_A)], [emis()], DOC_A)
|
||||
assert resolve(good, {"uf_override": "BA"})["status"] == "PASS"
|
||||
r = resolve(good, {"uf_override": "SP"})
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFLICT" # override never replaces the resolver silently
|
||||
blank = ev([co("1", CN_A, uf="")], [emis(uf="")])
|
||||
r = resolve(blank, {"uf_override": "BA"})
|
||||
assert r["status"] == "PASS" and "UF_FROM_EXPLICIT_OVERRIDE" in r["warnings"]
|
||||
@@ -0,0 +1,52 @@
|
||||
from gti_fiscal_agent.ipi_auto_apply import ipi_write_decision
|
||||
|
||||
|
||||
def _decision(ipi_status="IPI_RESOLVED", status="EXACT_NCM", ipi=None):
|
||||
return {"status": status, "ipi_status": ipi_status, "classification": {"ipi": ipi}}
|
||||
|
||||
|
||||
def test_numeric_null_current_is_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "NUMERIC", "rate_numeric": 3.25}), None)
|
||||
assert d["eligible"] is True
|
||||
assert d["proposed_value"] == 3.25
|
||||
assert d["reason_code"] == "CHANGE_REQUIRED"
|
||||
|
||||
|
||||
def test_numeric_same_value_is_not_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "NUMERIC", "rate_numeric": 3.25}), 3.25)
|
||||
assert d["eligible"] is False
|
||||
assert d["reason_code"] == "SAME_VALUE"
|
||||
|
||||
|
||||
def test_numeric_different_value_is_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "NUMERIC", "rate_numeric": 3.25}), 0)
|
||||
assert d["eligible"] is True
|
||||
assert d["proposed_value"] == 3.25
|
||||
|
||||
|
||||
def test_non_numeric_never_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "NON_NUMERIC", "rate_numeric": None}), None)
|
||||
assert d["eligible"] is False
|
||||
assert d["reason_code"] == "RATE_KIND_NON_NUMERIC"
|
||||
|
||||
|
||||
def test_source_blank_never_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "BLANK", "rate_numeric": None}), None)
|
||||
assert d["eligible"] is False
|
||||
assert d["reason_code"] == "RATE_KIND_BLANK"
|
||||
|
||||
|
||||
def test_ambiguous_ipi_status_never_eligible():
|
||||
d = ipi_write_decision(_decision(ipi_status="IPI_CANDIDATES", ipi=None), None)
|
||||
assert d["eligible"] is False
|
||||
|
||||
|
||||
def test_no_match_never_eligible():
|
||||
d = ipi_write_decision(_decision(ipi_status="NO_TIPI_MAPPING_FOUND", ipi=None), None)
|
||||
assert d["eligible"] is False
|
||||
|
||||
|
||||
def test_legacy_ncm_never_eligible():
|
||||
d = ipi_write_decision(_decision(status="LEGACY_NCM", ipi_status="LEGACY_NCM"), None)
|
||||
assert d["eligible"] is False
|
||||
assert d["reason_code"] == "LEGACY_NCM"
|
||||
@@ -0,0 +1,132 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gti_fiscal_agent.company_context import resolve
|
||||
from gti_fiscal_agent.preflight import major_minor, run_preflight
|
||||
|
||||
ROOT = Path(__file__).parents[2]
|
||||
spec = importlib.util.spec_from_file_location("run_agent_service", ROOT / "agent" / "scripts" / "run_agent_service.py")
|
||||
service = importlib.util.module_from_spec(spec); sys.modules[spec.name] = service; spec.loader.exec_module(service)
|
||||
|
||||
CN = "11111111000111"
|
||||
|
||||
|
||||
def env(tmp_path, **over):
|
||||
base = {"GTI_HUB_URL": "http://hub.test", "AGENT_TOKEN": "SECRETTOKENVALUE9", "FIREBIRD_DATABASE_PATH": "x.fdb", "FB_USER": "u", "FB_PASSWORD": "SECRETPASSVALUE9", "FISCAL_MODE": "DRY_RUN", "FIREBIRD_WRITES": "0",
|
||||
"AGENT_STATE_DB": str(tmp_path / "state.sqlite"), "AGENT_LOG_PATH": str(tmp_path / "logs" / "a.log"), "AGENT_VERSION": "1.0.0-rc1"}
|
||||
base.update(over)
|
||||
return base
|
||||
|
||||
|
||||
def http_ok(url, token=None, timeout=10):
|
||||
if url.endswith("/health"):
|
||||
return 200, {"status": "ok"}
|
||||
if url.endswith("/version"):
|
||||
return 200, {"version": "1.0.0-rc1"}
|
||||
return (200 if token == "SECRETTOKENVALUE9" else 401), {}
|
||||
|
||||
|
||||
def fake_probe(company_over=None, caps_missing=()):
|
||||
def probe(env):
|
||||
cols = {"CLASSIFICACAO_FISCAL": (37, 20), "CFOP_ENTRADA": (14, 4), "CST_ENTRADA": (14, 4), "CFOP": (37, 4), "CST": (37, 10), "ALIQUOTA": (16, 8), "CEST": (37, 7), "SITUACAO_TRIBUTARIA": (37, 2),
|
||||
"COD_ENTRADA_COFINS": (37, 3), "COD_SAIDA_COFINS": (37, 3), "ALIQ_COFINS": (37, 8), "COD_ENTRADA_PIS": (37, 3), "COD_SAIDA_PIS": (37, 3), "ALIQ_PIS": (37, 8), "CSOSN": (37, 60), "IPI": (27, 8),
|
||||
"CST_IPI": (14, 3), "IAT": (37, 1), "INCIDENCIA_PISCOFINS": (37, 30), "PISCOFINS": (37, 1), "IPPT": (37, 1), "EX_TIPI": (37, 3), "COD_ORIGEM_MERCADORIA": (8, 4)}
|
||||
for c in caps_missing:
|
||||
cols.pop(c)
|
||||
ev = {"c000004": [{"code": "000001", "cnpj": CN, "uf": "BA", "crt": "1", "optante": "S"}], "emissor": [{"cnpj": CN, "uf": "BA", "crt": "1"}], "nfce_keys": [{"cuf": "29", "cnpj": CN, "n": 5}], "nf_by_filial": {}}
|
||||
ev.update(company_over or {})
|
||||
return {"server_version": "WI-V6.3.9 Firebird 2.5", "ods": "11.2", "tables": {"C000004": {}, "C000025": cols}, "products": 3712, "company": resolve(ev, {})}
|
||||
return probe
|
||||
|
||||
|
||||
def test_preflight_passes_for_a_clean_dry_run_installation(tmp_path):
|
||||
r = run_preflight(env(tmp_path), http_get=http_ok, probe=fake_probe())
|
||||
assert r["PREFLIGHT"] == "PASS" and r["blockers"] == [] and r["effective_firebird_writes"] is False
|
||||
names = {c["name"] for c in r["checks"]}
|
||||
assert {"HUB_REACHABLE", "AUTH", "HUB_VERSION_COMPATIBLE", "FIREBIRD_REACHABLE", "C000004", "C000025", "CAPABILITIES", "COMPANY_CONTEXT", "UF", "CRT", "SCHEMA_TYPE_GUARDS", "SQLITE", "LOG_DIRECTORY", "FISCAL_MODE", "FIREBIRD_WRITES"} <= names
|
||||
assert "SECRETTOKENVALUE9" not in str(r) and "SECRETPASSVALUE9" not in str(r)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("over,blocker", [
|
||||
({"FISCAL_MODE": "DRY_RUN", "FIREBIRD_WRITES": "1"}, "FIREBIRD_WRITES"),
|
||||
({"FISCAL_MODE": ""}, "FISCAL_MODE"),
|
||||
({"AGENT_TOKEN": ""}, "CONFIG"),
|
||||
])
|
||||
def test_preflight_blocks_bad_config(tmp_path, over, blocker):
|
||||
r = run_preflight(env(tmp_path, **over), http_get=http_ok, probe=fake_probe())
|
||||
assert r["PREFLIGHT"] == "FAIL" and any(b.startswith(blocker) for b in r["blockers"])
|
||||
|
||||
|
||||
def test_preflight_fails_on_hub_auth_version_and_missing_capability(tmp_path):
|
||||
def bad_auth(url, token=None, timeout=10):
|
||||
return http_ok(url, "wrong" if token else None)
|
||||
assert any(b.startswith("AUTH") for b in run_preflight(env(tmp_path), http_get=bad_auth, probe=fake_probe())["blockers"])
|
||||
|
||||
def old_hub(url, token=None, timeout=10):
|
||||
return (200, {"version": "0.9.0"}) if url.endswith("/version") else http_ok(url, token)
|
||||
assert any(b.startswith("HUB_VERSION_COMPATIBLE") for b in run_preflight(env(tmp_path), http_get=old_hub, probe=fake_probe())["blockers"])
|
||||
assert any(b.startswith("CAPABILITIES") for b in run_preflight(env(tmp_path), http_get=http_ok, probe=fake_probe(caps_missing=("CEST",)))["blockers"])
|
||||
|
||||
|
||||
def test_preflight_fails_when_company_context_is_ambiguous_or_crt_conflicts(tmp_path):
|
||||
two = {"c000004": [{"code": "1", "cnpj": CN, "uf": "BA", "crt": "1"}, {"code": "2", "cnpj": "22222222000122", "uf": "SP", "crt": "1"}], "emissor": []}
|
||||
r = run_preflight(env(tmp_path), http_get=http_ok, probe=fake_probe(two))
|
||||
assert r["PREFLIGHT"] == "FAIL" and any(b.startswith("COMPANY_CONTEXT") for b in r["blockers"])
|
||||
conflict = {"emissor": [{"cnpj": CN, "uf": "BA", "crt": "3"}]}
|
||||
assert any("CRT_SOURCE_CONFLICT" in b for b in run_preflight(env(tmp_path), http_get=http_ok, probe=fake_probe(conflict))["blockers"])
|
||||
|
||||
|
||||
def test_hub_unreachable_is_a_clear_blocker(tmp_path):
|
||||
def down(url, token=None, timeout=10):
|
||||
raise ConnectionError("nope")
|
||||
r = run_preflight(env(tmp_path), http_get=down, probe=fake_probe())
|
||||
assert r["PREFLIGHT"] == "FAIL" and any(b.startswith("HUB_REACHABLE") for b in r["blockers"])
|
||||
|
||||
|
||||
def test_major_minor():
|
||||
assert major_minor("1.0.0-rc1") == "1.0" and major_minor("1.0.3") == "1.0" and major_minor("0.9.0") != major_minor("1.0.0-rc1")
|
||||
|
||||
|
||||
# ---------------- zero-write guarantee ----------------
|
||||
def test_dry_run_makes_writes_technically_impossible_whatever_firebird_writes_says():
|
||||
for extra in ({"FIREBIRD_WRITES": "1", "FISCAL_MODE": "DRY_RUN"}, {"FIREBIRD_WRITES": "0", "FISCAL_MODE": "AUTO_APPLY"}, {"FIREBIRD_WRITES": "1", "FISCAL_MODE": "dry_run"}):
|
||||
cfg = service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb", **extra})
|
||||
assert cfg.firebird_writes is False
|
||||
assert service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb", "FIREBIRD_WRITES": "1", "FISCAL_MODE": "AUTO_APPLY"}).firebird_writes is True
|
||||
assert service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb", "FIREBIRD_WRITES": "1"}).firebird_writes is True # legacy: unset FISCAL_MODE follows the flag
|
||||
assert service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb"}).fiscal_mode == "DRY_RUN"
|
||||
|
||||
|
||||
def test_write_connection_cannot_be_opened_when_writes_are_disabled(monkeypatch):
|
||||
opened = []
|
||||
monkeypatch.setitem(sys.modules, "fdb", type("F", (), {"connect": staticmethod(lambda **kw: opened.append(kw))}))
|
||||
with pytest.raises(RuntimeError):
|
||||
service.open_write_connection({"FB_USER": "u", "FB_PASSWORD": "SECRETPASSVALUE9"}, "x.fdb", firebird_writes=False)
|
||||
assert opened == []
|
||||
|
||||
|
||||
def test_dry_run_job_never_opens_a_write_connection_nor_calls_the_executor(monkeypatch):
|
||||
class Boom:
|
||||
def __getattr__(self, name):
|
||||
if name in ("execute_guarded_update", "execute_guarded_restore"):
|
||||
raise AssertionError("executor write called in DRY_RUN")
|
||||
return getattr(__import__("scripts.firebird_write_executor", fromlist=["x"]) if False else object(), name, None)
|
||||
cfg = service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb", "FIREBIRD_WRITES": "1", "FISCAL_MODE": "DRY_RUN"})
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: [{"CODIGO": "000001", "PRODUTO": "X", "CLASSIFICACAO_FISCAL": "22030000", "IPI": None}])
|
||||
monkeypatch.setattr(service, "HubClient", lambda url, token: type("H", (), {"classify_batch": lambda self, p: [{"status": "EXACT_NCM", "ipi_status": "IPI_RESOLVED", "classification": {"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.9}},
|
||||
"dataset_versions": {}}]})())
|
||||
monkeypatch.setattr(service, "build_job_coverage", lambda *a, **k: {"TOTAL_PRODUCTS": 1})
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda *a, **k: (_ for _ in ()).throw(AssertionError("write connection opened in DRY_RUN")))
|
||||
|
||||
class State:
|
||||
def log_ipi_write(self, *a): pass
|
||||
|
||||
class Ex:
|
||||
WriteRefused = Exception
|
||||
def execute_guarded_update(self, *a, **k): raise AssertionError("executor write called in DRY_RUN")
|
||||
|
||||
result, error = service.execute_job({"kind": "REPROCESS_ALL", "agent_id": 1, "payload": {}, "job_id": 1}, cfg, Ex(), State(), {})
|
||||
assert error is None and result["writes_attempted"] == 0 and result["firebird_writes_enabled"] is False
|
||||
@@ -0,0 +1,272 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPT = Path(__file__).parents[1] / "scripts" / "run_agent_service.py"
|
||||
AUTHORIZED = r"D:\PROJETOS\GTISISTEMAS\Projeto\TESTESOFICIAIS.FDB"
|
||||
|
||||
|
||||
def load_module():
|
||||
spec = importlib.util.spec_from_file_location("run_agent_service", SCRIPT)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
service = load_module()
|
||||
|
||||
|
||||
class _FakeState:
|
||||
def __init__(self):
|
||||
self.logged = []
|
||||
|
||||
def log_ipi_write(self, *args):
|
||||
self.logged.append(args)
|
||||
|
||||
|
||||
class _FakeExecutor:
|
||||
"""Mirrors the real firebird_write_executor's public surface used by execute_job."""
|
||||
class WriteRefused(Exception):
|
||||
pass
|
||||
|
||||
def __init__(self, path_guard_ok=True, updates=None, fingerprint_guard_ok=True):
|
||||
self.path_guard_ok = path_guard_ok
|
||||
self.updates = updates or {}
|
||||
self.fingerprint_guard_ok = fingerprint_guard_ok
|
||||
|
||||
def assert_authorized_write_path(self, path, authorized_path=None):
|
||||
if not self.path_guard_ok:
|
||||
raise self.WriteRefused(f"path is not the authorized write target: {path}")
|
||||
|
||||
def assert_schema_fingerprint_authorized(self, observed, expected):
|
||||
if not self.fingerprint_guard_ok or observed != expected:
|
||||
raise self.WriteRefused(f"schema fingerprint mismatch: expected {expected!r}, observed {observed!r}")
|
||||
|
||||
def execute_guarded_update(self, connection, *, database_path, proposal, now, authorized_path=None):
|
||||
code = proposal["product_code"]
|
||||
if code in self.updates:
|
||||
raise self.WriteRefused(self.updates[code])
|
||||
record = type("Record", (), {
|
||||
"before_value": proposal["current_value"],
|
||||
"after_value": proposal["proposed_value"],
|
||||
"rowcount": 1,
|
||||
})()
|
||||
return record
|
||||
|
||||
|
||||
def _config(**overrides):
|
||||
base = dict(
|
||||
hub_url="http://localhost:8000", agent_id="a1", agent_token="tok", company_id="c1", company_uf="BA",
|
||||
database_path=AUTHORIZED, firebird_writes=False, poll_interval_seconds=0.01,
|
||||
agent_version="0.1.0", schema_fingerprint="fp", state_db=":memory:",
|
||||
authorized_write_database_path=None, expected_schema_fingerprint="fp",
|
||||
)
|
||||
base.update(overrides)
|
||||
return type("Config", (), base)()
|
||||
|
||||
|
||||
def test_config_defaults_from_env():
|
||||
env = {"FIREBIRD_DATABASE_PATH": AUTHORIZED}
|
||||
config = service.Config(env)
|
||||
assert config.firebird_writes is False
|
||||
assert config.hub_url == "http://localhost:8000"
|
||||
assert config.poll_interval_seconds == 5.0
|
||||
|
||||
|
||||
def test_config_reads_gti_hub_url_and_writes_flag():
|
||||
env = {"FIREBIRD_DATABASE_PATH": AUTHORIZED, "GTI_HUB_URL": "http://hub.internal", "FIREBIRD_WRITES": "1"}
|
||||
config = service.Config(env)
|
||||
assert config.hub_url == "http://hub.internal"
|
||||
assert config.firebird_writes is True
|
||||
|
||||
|
||||
def test_path_guard_fails_closed_before_any_write(monkeypatch):
|
||||
config = _config(firebird_writes=True, database_path=r"D:\PRODUCAO\ERP.FDB")
|
||||
executor = _FakeExecutor(path_guard_ok=False)
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
called = {"read_products": False}
|
||||
|
||||
def fake_read_products(env, codigos=None):
|
||||
called["read_products"] = True
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(service, "read_products", fake_read_products)
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert result is None
|
||||
assert "not the authorized write target" in error
|
||||
assert called["read_products"] is False # never even reads Firebird once the guard refuses
|
||||
|
||||
|
||||
def test_pilot_write_blocked_when_schema_fingerprint_does_not_match(monkeypatch):
|
||||
# A real client install where the live schema fingerprint drifted from what
|
||||
# this Agent installation was configured/registered to expect.
|
||||
config = _config(firebird_writes=True, schema_fingerprint="observed-live-fp", expected_schema_fingerprint="registered-fp")
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
called = {"read_products": False}
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: called.update(read_products=True) or [])
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert result is None
|
||||
assert "schema fingerprint mismatch" in error
|
||||
assert called["read_products"] is False
|
||||
|
||||
|
||||
def test_pilot_write_allowed_with_explicit_authorized_path_and_matching_fingerprint(monkeypatch):
|
||||
client_path = r"D:\ClienteX\ERP\PRODUCAO.FDB"
|
||||
config = _config(
|
||||
firebird_writes=True, database_path=client_path, authorized_write_database_path=client_path,
|
||||
schema_fingerprint="fp", expected_schema_fingerprint="fp",
|
||||
)
|
||||
products = [{"CODIGO": "000026", "PRODUTO": "x", "CLASSIFICACAO_FISCAL": "03055100", "IPI": None}]
|
||||
decisions = [{
|
||||
"status": "EXACT_NCM", "ipi_status": "IPI_RESOLVED",
|
||||
"classification": {"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.25}},
|
||||
"dataset_versions": {"tipi_ipi": "v1"},
|
||||
}]
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: products)
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, *a, **k): pass
|
||||
def classify_batch(self, items): return decisions
|
||||
monkeypatch.setattr(service, "HubClient", FakeClient)
|
||||
fake_connection = type("Conn", (), {"close": lambda self: None})()
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda env, path: fake_connection)
|
||||
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert error is None
|
||||
assert result["writes_succeeded"] == 1
|
||||
|
||||
|
||||
def test_execute_job_isolates_internal_errors_without_raising(monkeypatch):
|
||||
config = _config()
|
||||
|
||||
def boom(env, codigos=None):
|
||||
raise RuntimeError("firebird unavailable")
|
||||
|
||||
monkeypatch.setattr(service, "read_products", boom)
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_ALL", "agent_id": 1, "payload": {}}
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert result is None
|
||||
assert "firebird unavailable" in error
|
||||
|
||||
|
||||
def test_execute_job_success_summarizes_pipeline(monkeypatch):
|
||||
config = _config()
|
||||
products = [{"CODIGO": "000026", "PRODUTO": "x", "CLASSIFICACAO_FISCAL": "03055100", "IPI": None}]
|
||||
decisions = [{
|
||||
"status": "EXACT_NCM", "ipi_status": "IPI_RESOLVED",
|
||||
"classification": {"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.25}},
|
||||
"dataset_versions": {"tipi_ipi": "v1"},
|
||||
}]
|
||||
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: products)
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, *a, **k): pass
|
||||
def classify_batch(self, items): return decisions
|
||||
monkeypatch.setattr(service, "HubClient", FakeClient)
|
||||
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert error is None
|
||||
assert result["products_processed"] == 1
|
||||
assert result["writes_attempted"] == 0 # firebird_writes disabled by default in _config()
|
||||
|
||||
|
||||
class _FakeHubForLoop:
|
||||
"""Simulates: heartbeat flaky twice then OK, one job then nothing, and captures completions."""
|
||||
|
||||
def __init__(self):
|
||||
self.heartbeat_calls = 0
|
||||
self.jobs = [{"job_id": 1, "kind": "REPROCESS_ALL", "agent_id": 1, "payload": {}}]
|
||||
self.completed = []
|
||||
self.failed = []
|
||||
|
||||
def heartbeat(self, *a, **k):
|
||||
self.heartbeat_calls += 1
|
||||
if self.heartbeat_calls == 1:
|
||||
raise ConnectionError("hub down")
|
||||
return {}
|
||||
|
||||
def next_job(self):
|
||||
return self.jobs.pop(0) if self.jobs else None
|
||||
|
||||
def complete_job(self, job_id, result_summary):
|
||||
self.completed.append((job_id, result_summary))
|
||||
|
||||
def fail_job(self, job_id, error):
|
||||
self.failed.append((job_id, error))
|
||||
|
||||
def classify_batch(self, items):
|
||||
return []
|
||||
|
||||
|
||||
class _ExternalStopFlag:
|
||||
"""Mirrors the Windows Service wrapper's stop object: no signal registration,
|
||||
just a plain flag flippable from another thread."""
|
||||
def __init__(self):
|
||||
self.stop = False
|
||||
|
||||
|
||||
def test_run_honors_an_externally_supplied_stopper(monkeypatch):
|
||||
# This is exactly the pattern the Windows Service wrapper uses: it never
|
||||
# constructs GracefulStop() (which would crash off the main thread), it
|
||||
# passes its own stop flag and flips it from the SvcStop handler.
|
||||
config = _config(poll_interval_seconds=0.001)
|
||||
fake_hub = _FakeHubForLoop()
|
||||
fake_hub.jobs = [] # no jobs; loop would run forever without an external stop
|
||||
monkeypatch.setattr(service, "HubClient", lambda *a, **k: fake_hub)
|
||||
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
||||
monkeypatch.setattr(service, "AgentState", lambda path: _FakeState())
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: [])
|
||||
|
||||
stopper = _ExternalStopFlag()
|
||||
calls = {"n": 0}
|
||||
real_sleep = service.time.sleep
|
||||
|
||||
def counting_sleep(seconds):
|
||||
calls["n"] += 1
|
||||
if calls["n"] >= 3:
|
||||
stopper.stop = True
|
||||
|
||||
monkeypatch.setattr(service.time, "sleep", counting_sleep)
|
||||
service.run(config, {}, stopper=stopper) # no max_iterations — only the external flag can stop it
|
||||
assert stopper.stop is True
|
||||
assert calls["n"] >= 3
|
||||
|
||||
|
||||
def test_run_survives_hub_down_then_processes_one_job_and_stops(monkeypatch):
|
||||
config = _config(poll_interval_seconds=0.001)
|
||||
fake_hub = _FakeHubForLoop()
|
||||
monkeypatch.setattr(service, "HubClient", lambda *a, **k: fake_hub)
|
||||
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
||||
monkeypatch.setattr(service, "AgentState", lambda path: _FakeState())
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: [])
|
||||
monkeypatch.setattr(service, "build_job_coverage", lambda *a, **k: None)
|
||||
monkeypatch.setattr(service.time, "sleep", lambda seconds: None)
|
||||
|
||||
# first heartbeat call raises -> backoff branch; second call onward succeeds.
|
||||
service.run(config, {}, max_iterations=4)
|
||||
|
||||
assert fake_hub.heartbeat_calls >= 2
|
||||
assert len(fake_hub.completed) == 1
|
||||
assert fake_hub.completed[0][0] == 1
|
||||
assert fake_hub.completed[0][1]["products_processed"] == 0
|
||||
@@ -0,0 +1,7 @@
|
||||
from gti_fiscal_agent.runner import to_hub_product
|
||||
|
||||
|
||||
def test_reference_schema_uses_classificacao_fiscal_as_canonical_ncm():
|
||||
product = {"CODIGO": "000276", "PRODUTO": "ABACAXI M/ JARRA", "NCM": "99999999", "CLASSIFICACAO_FISCAL": "08043000"}
|
||||
|
||||
assert to_hub_product(product)["current_ncm"] == "08043000"
|
||||
@@ -0,0 +1,20 @@
|
||||
from gti_fiscal_agent.state import AgentState
|
||||
|
||||
|
||||
def test_outbox_keeps_failed_work_pending(tmp_path):
|
||||
state = AgentState(tmp_path / "agent.db")
|
||||
assert state.enqueue("000276", "source", "v1", "0.1.0")
|
||||
state.retry("000276")
|
||||
assert state.pending()[0]["status"] == "PENDING_API"
|
||||
state.result_received("000276", "source", "v1", "0.1.0")
|
||||
assert state.enqueue("000276", "source", "v1", "0.1.0") is False
|
||||
|
||||
|
||||
def test_ipi_write_log_records_per_product_audit(tmp_path):
|
||||
state = AgentState(tmp_path / "agent.db")
|
||||
state.log_ipi_write("000276", "fp1", "EXACT_NCM", "IPI_RESOLVED", "SUCCEEDED", None, 3.25, "rowcount=1", "CHANGE_REQUIRED", "2026-09-18T00:00:00Z")
|
||||
log = state.ipi_write_log_for("000276")
|
||||
assert len(log) == 1
|
||||
assert log[0]["write_status"] == "SUCCEEDED"
|
||||
assert log[0]["before_value"] is None
|
||||
assert log[0]["proposed_value"] == "3.25"
|
||||
Reference in New Issue
Block a user