28 lines
1.0 KiB
Python
28 lines
1.0 KiB
Python
"""Download and promote the official CONFAZ CV142_18 HTML CEST source."""
|
|
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
|
|
from central.app.config import settings
|
|
from central.app.datasets.cest import parse_cest_html
|
|
from central.app.datasets.service import DatasetService
|
|
from central.app.db import SessionLocal
|
|
|
|
|
|
def main():
|
|
response = httpx.get(settings.cest_dataset_url, timeout=settings.http_timeout_seconds, follow_redirects=True)
|
|
response.raise_for_status()
|
|
parsed = parse_cest_html(response.content)
|
|
path = Path(settings.dataset_storage_path)
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
(path / f"{parsed.sha256}.html").write_bytes(response.content)
|
|
with SessionLocal() as session:
|
|
dataset = DatasetService(session).import_cest_payload(parsed, settings.cest_dataset_url, datetime.now(timezone.utc))
|
|
print({"id": dataset.id, "status": dataset.status, "sha256": dataset.sha256, "record_count": dataset.record_count, "version": parsed.version})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|