92 lines
4.8 KiB
Python
92 lines
4.8 KiB
Python
"""Automatic refresh of the official NCM dataset (Siscomex JSON): staging -> validate -> atomic promotion, keeping the last valid dataset on ANY failure.
|
|
|
|
Every attempt leaves a SourceRefreshRun (url, time, sha256, version label, record count, status, error) for the panel. Nothing here touches Agents or providers.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import threading
|
|
import time
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import httpx
|
|
from sqlalchemy import select
|
|
|
|
from ..config import settings
|
|
from ..models import Dataset, SourceRefreshRun
|
|
from .ncm import parse_ncm_payload
|
|
from .service import DatasetService
|
|
|
|
logger = logging.getLogger("gti_fiscal_hub")
|
|
MIN_RATIO = 0.9 # a payload with < 90% of the active record count is treated as a broken download, never promoted
|
|
INTERVAL = timedelta(hours=24)
|
|
RETRY_AFTER_FAILURE = timedelta(hours=6)
|
|
|
|
|
|
def _fetch(url: str) -> bytes:
|
|
response = httpx.get(url, timeout=max(settings.http_timeout_seconds, 60), follow_redirects=True, headers={"User-Agent": "GTIFiscalHub/1.0 (official dataset refresh)"})
|
|
response.raise_for_status()
|
|
return response.content
|
|
|
|
|
|
def refresh_ncm(session, fetch=_fetch, url: str | None = None) -> SourceRefreshRun:
|
|
url = url or settings.ncm_dataset_url
|
|
run = SourceRefreshRun(source="NCM", source_url=url, status="RUNNING")
|
|
session.add(run)
|
|
session.commit()
|
|
try:
|
|
content = fetch(url) # staging: bytes in memory, nothing written to the active dataset yet
|
|
parsed = parse_ncm_payload(content) # schema validation
|
|
run.sha256, run.version_label, run.record_count = parsed.sha256, (parsed.version or "")[:64], len(parsed.entries)
|
|
active = session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
|
|
if active is not None and active.sha256 == parsed.sha256:
|
|
run.status = "UNCHANGED"
|
|
else:
|
|
if active is not None and len(parsed.entries) < MIN_RATIO * (active.record_count or 0):
|
|
raise ValueError(f"implausible record count {len(parsed.entries)} vs active {active.record_count}")
|
|
DatasetService(session).import_payload(parsed, url, datetime.now(timezone.utc)) # atomic: one transaction, old ACTIVE superseded only at the end
|
|
run.status = "UPDATED"
|
|
try: # the KB is never deleted: only flagged STALE_REVIEW_REQUIRED when its NCM left the official dataset
|
|
from ..enrichment import history, kb
|
|
current = history.official_sets(session)
|
|
run.error = f"kb_revalidated: {kb.revalidate(session, lambda v: v in current)}"[:300]
|
|
except Exception as error: # noqa: BLE001 - revalidation trouble must not undo a valid dataset promotion
|
|
run.error = f"kb_revalidate_failed: {type(error).__name__}"[:300]
|
|
except Exception as error: # keep the last valid dataset
|
|
session.rollback()
|
|
run = session.get(SourceRefreshRun, run.id)
|
|
run.status, run.error = "FAILED", f"{type(error).__name__}: {error}"[:300]
|
|
logger.warning("NCM refresh failed: %s", run.error)
|
|
run.finished_at = datetime.now(timezone.utc)
|
|
session.commit()
|
|
return run
|
|
|
|
|
|
def last_runs(session, source: str = "NCM", limit: int = 5):
|
|
return session.scalars(select(SourceRefreshRun).where(SourceRefreshRun.source == source).order_by(SourceRefreshRun.id.desc()).limit(limit)).all()
|
|
|
|
|
|
def _due(session) -> bool:
|
|
last = session.scalar(select(SourceRefreshRun).where(SourceRefreshRun.source == "NCM", SourceRefreshRun.status != "RUNNING").order_by(SourceRefreshRun.id.desc()).limit(1))
|
|
if last is None or last.finished_at is None:
|
|
return True
|
|
finished = last.finished_at if last.finished_at.tzinfo else last.finished_at.replace(tzinfo=timezone.utc)
|
|
return datetime.now(timezone.utc) - finished >= (RETRY_AFTER_FAILURE if last.status == "FAILED" else INTERVAL)
|
|
|
|
|
|
def start_scheduler(session_factory, first_delay: float = 90.0):
|
|
"""Daemon thread: checks hourly whether a refresh is due (survives restarts because due-ness is derived from SourceRefreshRun)."""
|
|
def loop():
|
|
time.sleep(first_delay)
|
|
while True:
|
|
try:
|
|
with session_factory() as session:
|
|
if _due(session):
|
|
refresh_ncm(session)
|
|
except Exception: # never let the scheduler die
|
|
logger.exception("official refresh loop error")
|
|
time.sleep(3600)
|
|
thread = threading.Thread(target=loop, name="official-refresh", daemon=True)
|
|
thread.start()
|
|
return thread
|