feat: enrichment Hub API, feature flags UI/gating, NCM refresher, Agent enrichment worker with tests

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

central/app/datasets/updater.py

@@ -0,0 +1,85 @@
+"""Automatic refresh of the official NCM dataset (Siscomex JSON): staging -> validate -> atomic promotion, keeping the last valid dataset on ANY failure.
+
+Every attempt leaves a SourceRefreshRun (url, time, sha256, version label, record count, status, error) for the panel. Nothing here touches Agents or providers.
+"""
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from datetime import datetime, timedelta, timezone
+
+import httpx
+from sqlalchemy import select
+
+from ..config import settings
+from ..models import Dataset, SourceRefreshRun
+from .ncm import parse_ncm_payload
+from .service import DatasetService
+
+logger = logging.getLogger("gti_fiscal_hub")
+MIN_RATIO = 0.9          # a payload with < 90% of the active record count is treated as a broken download, never promoted
+INTERVAL = timedelta(hours=24)
+RETRY_AFTER_FAILURE = timedelta(hours=6)
+
+
+def _fetch(url: str) -> bytes:
+    response = httpx.get(url, timeout=max(settings.http_timeout_seconds, 60), follow_redirects=True, headers={"User-Agent": "GTIFiscalHub/1.0 (official dataset refresh)"})
+    response.raise_for_status()
+    return response.content
+
+
+def refresh_ncm(session, fetch=_fetch, url: str | None = None) -> SourceRefreshRun:
+    url = url or settings.ncm_dataset_url
+    run = SourceRefreshRun(source="NCM", source_url=url, status="RUNNING")
+    session.add(run)
+    session.commit()
+    try:
+        content = fetch(url)                                                       # staging: bytes in memory, nothing written to the active dataset yet
+        parsed = parse_ncm_payload(content)                                        # schema validation
+        run.sha256, run.version_label, run.record_count = parsed.sha256, (parsed.version or "")[:64], len(parsed.entries)
+        active = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
+        if active is not None and active.sha256 == parsed.sha256:
+            run.status = "UNCHANGED"
+        else:
+            if active is not None and len(parsed.entries) < MIN_RATIO * (active.record_count or 0):
+                raise ValueError(f"implausible record count {len(parsed.entries)} vs active {active.record_count}")
+            DatasetService(session).import_payload(parsed, url, datetime.now(timezone.utc))      # atomic: one transaction, old ACTIVE superseded only at the end
+            run.status = "UPDATED"
+    except Exception as error:                                                     # keep the last valid dataset
+        session.rollback()
+        run = session.get(SourceRefreshRun, run.id)
+        run.status, run.error = "FAILED", f"{type(error).__name__}: {error}"[:300]
+        logger.warning("NCM refresh failed: %s", run.error)
+    run.finished_at = datetime.now(timezone.utc)
+    session.commit()
+    return run
+
+
+def last_runs(session, source: str = "NCM", limit: int = 5):
+    return session.scalars(select(SourceRefreshRun).where(SourceRefreshRun.source == source).order_by(SourceRefreshRun.id.desc()).limit(limit)).all()
+
+
+def _due(session) -> bool:
+    last = session.scalar(select(SourceRefreshRun).where(SourceRefreshRun.source == "NCM", SourceRefreshRun.status != "RUNNING").order_by(SourceRefreshRun.id.desc()).limit(1))
+    if last is None or last.finished_at is None:
+        return True
+    finished = last.finished_at if last.finished_at.tzinfo else last.finished_at.replace(tzinfo=timezone.utc)
+    return datetime.now(timezone.utc) - finished >= (RETRY_AFTER_FAILURE if last.status == "FAILED" else INTERVAL)
+
+
+def start_scheduler(session_factory, first_delay: float = 90.0):
+    """Daemon thread: checks hourly whether a refresh is due (survives restarts because due-ness is derived from SourceRefreshRun)."""
+    def loop():
+        time.sleep(first_delay)
+        while True:
+            try:
+                with session_factory() as session:
+                    if _due(session):
+                        refresh_ncm(session)
+            except Exception:                                                      # never let the scheduler die
+                logger.exception("official refresh loop error")
+            time.sleep(3600)
+    thread = threading.Thread(target=loop, name="official-refresh", daemon=True)
+    thread.start()
+    return thread