GTI_GERENCIADOR_FISCAL/fiscal/central/app/datasets/service.py

"""Versioned NCM dataset lifecycle with atomic activation."""
from datetime import datetime
from sqlalchemy import select, update
from sqlalchemy.orm import Session
from ..models import (
    AnpProductCode, BaStMvaEntry, BaStMvaNcmSh, CestEntry, CestNcmEntry, CfopEntry, Dataset,
    FcpRate, FiscalCodeEntry, LegacyNcmEntry, NcmEntry, NcmUtribEntry, RtcBiofuelIndexEntry,
    RtcCredPresEntry, RtcCClassTribEntry, RtcCstIbsCbsEntry, RtcCbsRateEntry, TipiEntry,
)


def _digits(value):
    return "".join(char for char in str(value or "") if char.isdigit())
from .cest import CestPayload
from .ncm import NcmPayload


class DatasetService:
    def __init__(self, session: Session): self.session = session

    def import_payload(self, payload: NcmPayload, source_url: str, downloaded_at: datetime, *, minimum_entries: int = 1000) -> Dataset:
        existing = self.session.scalar(select(Dataset).where(Dataset.provider == "Siscomex", Dataset.sha256 == payload.sha256))
        if existing: return existing
        if len(payload.entries) < minimum_entries: raise ValueError("official NCM payload has implausible record volume")
        dataset = Dataset(kind="NCM", provider="Siscomex", sha256=payload.sha256, version_label=payload.version, act=payload.act, source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
        self.session.add(dataset); self.session.flush()
        seen = set()
        for entry in payload.entries:
            key = (entry.code, entry.valid_from, entry.valid_to)
            if key in seen: raise ValueError("duplicate NCM entry")
            seen.add(key); self.session.add(NcmEntry(dataset_id=dataset.id, code=entry.code, description=entry.description, valid_from=entry.valid_from, valid_to=entry.valid_to))
        self.session.flush()
        if len(seen) != len(payload.entries): raise ValueError("NCM insert count mismatch")
        dataset.record_count = len(seen); dataset.status = "READY"
        self.session.execute(update(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
        dataset.status = "ACTIVE"
        self.session.commit()
        return dataset

    def active_entries(self):
        return self.session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE")).all()

    def import_tipi_payload(self, payload, source_url, downloaded_at):
        existing = self.session.scalar(select(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.sha256 == payload.sha256, Dataset.version_label == payload.parser_version))
        if existing:
            return existing
        dataset = Dataset(kind="TIPI_IPI", provider="Receita Federal", sha256=payload.sha256, raw_sha256=payload.sha256, version_label=payload.parser_version, act="TIPI", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
        self.session.add(dataset); self.session.flush()
        try:
            for source_row, entry in enumerate(payload.entries, start=1):
                text = entry.rate or None
                try: rate = float(str(entry.rate).replace(",", ".")) if text else None
                except ValueError: rate = None
                self.session.add(TipiEntry(dataset_id=dataset.id, ncm=entry.ncm, ex=entry.ex, description=entry.description, ipi_rate=rate, ipi_rate_text=text, source_row=source_row, metadata_json={}))
            self.session.flush()
            if self.session.query(TipiEntry).filter_by(dataset_id=dataset.id).count() != len(payload.entries): raise ValueError("TIPI insert count mismatch")
            dataset.record_count = len(payload.entries); dataset.status = "READY"
            self.session.execute(update(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
            dataset.status = "ACTIVE"; self.session.commit(); return dataset
        except Exception:
            self.session.rollback(); raise

    def import_ba_st_mva(self, records, raw_sha256, source_url, downloaded_at):
        existing = self.session.scalar(select(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.sha256 == raw_sha256, Dataset.version_label == "ba-st-mva-pdf-v1"))
        if existing: return existing
        dataset = Dataset(kind="BA_ST_MVA", provider="SEFAZ-BA", sha256=raw_sha256, raw_sha256=raw_sha256, version_label="ba-st-mva-pdf-v1", act="RICMS/BA Anexo 1", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
        self.session.add(dataset); self.session.flush()
        try:
            for record in records:
                ncm_codes = record.pop("ncm_sh") or []
                entry = BaStMvaEntry(dataset_id=dataset.id, **record, metadata_json={})
                self.session.add(entry); self.session.flush()
                for code in dict.fromkeys(ncm_codes): self.session.add(BaStMvaNcmSh(entry_id=entry.id, ncm_sh=code, metadata_json={}))
            self.session.flush()
            if self.session.query(BaStMvaEntry).filter_by(dataset_id=dataset.id).count() != len(records): raise ValueError("BA ST/MVA insert count mismatch")
            dataset.record_count = len(records); dataset.status = "READY"
            self.session.execute(update(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
            dataset.status = "ACTIVE"; self.session.commit(); return dataset
        except Exception:
            self.session.rollback(); raise

    def import_cest_payload(self, payload: CestPayload, source_url: str, downloaded_at: datetime) -> Dataset:
        existing = self.session.scalar(select(Dataset).where(Dataset.provider == "CONFAZ", Dataset.sha256 == payload.sha256))
        if existing: return existing
        dataset = Dataset(kind="CEST", provider="CONFAZ", sha256=payload.sha256, raw_sha256=payload.sha256, version_label=payload.version, act="Convenio ICMS 142/18", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
        self.session.add(dataset); self.session.flush()
        canonical = {}
        for entry in payload.entries:
            existing_entry = canonical.get(entry.code)
            if existing_entry is None:
                canonical[entry.code] = [entry, set(entry.ncm_codes)]
            else:
                existing_entry[1].update(entry.ncm_codes)
        for entry, ncm_codes in canonical.values():
            row = CestEntry(dataset_id=dataset.id, code=entry.code, description=entry.description, annex=entry.annex)
            self.session.add(row); self.session.flush()
            for ncm_code in sorted(ncm_codes):
                self.session.add(CestNcmEntry(cest_entry_id=row.id, ncm_code=ncm_code))
        dataset.record_count = len(canonical); dataset.status = "READY"
        self.session.execute(update(Dataset).where(Dataset.kind == "CEST", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
        dataset.status = "ACTIVE"; self.session.commit()
        return dataset

    def cest_for_ncm(self, code):
        rows = self.session.execute(select(CestEntry, CestNcmEntry.ncm_code).join(CestNcmEntry).join(Dataset).where(Dataset.kind == "CEST", Dataset.status == "ACTIVE")).all()
        return list(dict.fromkeys(entry for entry, ncm_code in rows if code.startswith(ncm_code)))

    def legacy_ncm(self, code):
        entry = self.session.get(LegacyNcmEntry, code)
        if not entry or entry.successor_code is not None:
            return None
        return {"code": entry.code, "source": entry.source, "source_date": entry.source_date, "reason": entry.reason}

    def tipi_for_ncm(self, code):
        code = _digits(code)
        return self.session.scalars(select(TipiEntry).join(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.status == "ACTIVE", TipiEntry.ncm == code)).all()

    def ba_st_mva_for_ncm(self, code):
        code = _digits(code)
        rows = self.session.execute(select(BaStMvaEntry, BaStMvaNcmSh.ncm_sh).join(BaStMvaNcmSh, BaStMvaNcmSh.entry_id == BaStMvaEntry.id).join(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE")).all()
        return list(dict.fromkeys(entry for entry, ncm_sh in rows if _digits(ncm_sh) and code.startswith(_digits(ncm_sh))))

    def import_moc_tables(self, payload, source_url, downloaded_at):
        existing = self.session.scalar(select(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.sha256 == payload.sha256, Dataset.version_label == payload.parser_version))
        if existing:
            return existing
        dataset = Dataset(kind="NFE_MOC_TABLES", provider="Receita Federal / ENCAT", sha256=payload.sha256, raw_sha256=payload.sha256, version_label=payload.parser_version, act="MOC 7.0 Anexo I", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
        self.session.add(dataset); self.session.flush()
        try:
            for entry in payload.entries:
                self.session.add(FiscalCodeEntry(dataset_id=dataset.id, domain=entry.domain, code=entry.code, description=entry.description, source_section=entry.source_section, metadata_json=entry.metadata))
            self.session.flush()
            if self.session.query(FiscalCodeEntry).filter_by(dataset_id=dataset.id).count() != len(payload.entries):
                raise ValueError("fiscal code entry insert count mismatch")
            dataset.record_count = len(payload.entries); dataset.status = "READY"
            self.session.execute(update(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
            dataset.status = "ACTIVE"; self.session.commit(); return dataset
        except Exception:
            self.session.rollback(); raise

    def fiscal_codes(self, domain, code=None):
        query = select(FiscalCodeEntry).join(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.status == "ACTIVE", FiscalCodeEntry.domain == domain)
        if code is not None:
            query = query.where(FiscalCodeEntry.code == code)
        return self.session.scalars(query).all()

    def _import_generic(self, *, kind, provider, act, sha256, source_url, downloaded_at, entries, model_cls, row_kwargs, extra=()):
        existing = self.session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.sha256 == sha256))
        if existing:
            return existing
        dataset = Dataset(kind=kind, provider=provider, sha256=sha256, raw_sha256=sha256, version_label=sha256[:16], act=act, source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
        self.session.add(dataset); self.session.flush()
        try:
            for entry in entries:
                self.session.add(model_cls(dataset_id=dataset.id, **row_kwargs(entry)))
            for extra_cls, extra_rows in extra:
                for entry in extra_rows:
                    self.session.add(extra_cls(dataset_id=dataset.id, **entry))
            self.session.flush()
            if self.session.query(model_cls).filter_by(dataset_id=dataset.id).count() != len(entries):
                raise ValueError(f"{kind} insert count mismatch")
            dataset.record_count = len(entries); dataset.status = "READY"
            self.session.execute(update(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
            dataset.status = "ACTIVE"; self.session.commit(); return dataset
        except Exception:
            self.session.rollback(); raise

    def import_cfop(self, payload, source_url, downloaded_at):
        return self._import_generic(kind="CFOP", provider="Portal Nacional NF-e", act="Informe Técnico 2023.002", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=CfopEntry, row_kwargs=lambda e: e)

    def import_fcp(self, payload, source_url, downloaded_at):
        return self._import_generic(kind="FCP", provider="Portal Nacional NF-e", act="Tabela de Alíquotas FCP por UF", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=FcpRate, row_kwargs=lambda e: e)

    def import_anp(self, payload, source_url, downloaded_at):
        return self._import_generic(kind="ANP", provider="Portal Nacional NF-e", act="Tabela de Código de Produto ANP", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=AnpProductCode, row_kwargs=lambda e: e)

    def import_ncm_utrib(self, payload, source_url, downloaded_at):
        return self._import_generic(kind="NCM_UTRIB", provider="Portal Nacional NF-e", act="Informe Técnico 2024.001 - NCM x uTrib", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=NcmUtribEntry, row_kwargs=lambda e: e)

    def import_rtc_biofuel_index(self, payload, source_url, downloaded_at):
        return self._import_generic(kind="RTC_BIOFUEL_INDEX", provider="Portal Nacional NF-e", act="Informe Técnico 2025.004 - pBioObrigatorio", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcBiofuelIndexEntry, row_kwargs=lambda e: e)

    def import_rtc_cred_pres(self, payload, source_url, downloaded_at):
        return self._import_generic(kind="RTC_CCREDPRES", provider="Portal Nacional NF-e", act="cCredPres — LC 214/2025", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcCredPresEntry, row_kwargs=lambda e: e)

    def import_rtc_cclasstrib(self, payload, source_url, downloaded_at):
        return self._import_generic(kind="RTC_CCLASSTRIB", provider="Portal Nacional NF-e", act="cClassTrib — IBS/CBS (LC 214/2025)", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcCClassTribEntry, row_kwargs=lambda e: e, extra=((RtcCstIbsCbsEntry, payload.extra_entries),))

    def import_rtc_cbs_rates(self, payload, source_url, downloaded_at):
        return self._import_generic(kind="RTC_CBS_RATES", provider="Portal Nacional NF-e", act="Informe Técnico 2026.002 - Alíquotas da CBS", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcCbsRateEntry, row_kwargs=lambda e: e)

    def rtc_cclasstrib(self, code=None):
        query = select(RtcCClassTribEntry).join(Dataset).where(Dataset.kind == "RTC_CCLASSTRIB", Dataset.status == "ACTIVE")
        if code is not None:
            query = query.where(RtcCClassTribEntry.cclasstrib == str(code))
        return self.session.scalars(query).all()

    def rtc_cbs_rate(self, year):
        return self.session.scalar(select(RtcCbsRateEntry).join(Dataset).where(Dataset.kind == "RTC_CBS_RATES", Dataset.status == "ACTIVE", RtcCbsRateEntry.year == int(year)))

    def cfop_lookup(self, code=None):
        query = select(CfopEntry).join(Dataset).where(Dataset.kind == "CFOP", Dataset.status == "ACTIVE")
        if code is not None:
            query = query.where(CfopEntry.code == str(code))
        return self.session.scalars(query).all()

    def fcp_for_uf(self, uf):
        return self.session.scalar(select(FcpRate).join(Dataset).where(Dataset.kind == "FCP", Dataset.status == "ACTIVE", FcpRate.uf == uf.upper()))

    def anp_code_exists(self, cprod_anp):
        return self.session.scalar(select(AnpProductCode).join(Dataset).where(Dataset.kind == "ANP", Dataset.status == "ACTIVE", AnpProductCode.cprod_anp == str(cprod_anp))) is not None

    def ncm_utrib_for_ncm(self, ncm):
        ncm = _digits(ncm)
        return self.session.scalar(select(NcmUtribEntry).join(Dataset).where(Dataset.kind == "NCM_UTRIB", Dataset.status == "ACTIVE", NcmUtribEntry.ncm == ncm))

    def rtc_biofuel_for_anp(self, cprod_anp):
        return self.session.scalars(select(RtcBiofuelIndexEntry).join(Dataset).where(Dataset.kind == "RTC_BIOFUEL_INDEX", Dataset.status == "ACTIVE", RtcBiofuelIndexEntry.cprod_anp == str(cprod_anp), RtcBiofuelIndexEntry.is_historical.is_(False))).all()

    def rtc_cred_pres(self, code=None):
        query = select(RtcCredPresEntry).join(Dataset).where(Dataset.kind == "RTC_CCREDPRES", Dataset.status == "ACTIVE")
        if code is not None:
            query = query.where(RtcCredPresEntry.code == str(code))
        return self.session.scalars(query).all()

    def dataset_versions(self):
        active = self.session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
        cest = self.session.scalar(select(Dataset).where(Dataset.kind == "CEST", Dataset.status == "ACTIVE"))
        tipi = self.session.scalar(select(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.status == "ACTIVE"))
        mva = self.session.scalar(select(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE"))
        moc = self.session.scalar(select(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.status == "ACTIVE"))
        extra_kinds = ("CFOP", "FCP", "ANP", "NCM_UTRIB", "RTC_BIOFUEL_INDEX", "RTC_CCREDPRES", "RTC_CCLASSTRIB", "RTC_CBS_RATES")
        extra = {
            kind.lower(): self.session.scalar(select(Dataset.sha256).where(Dataset.kind == kind, Dataset.status == "ACTIVE"))
            for kind in extra_kinds
        }
        return {
            "ncm": active.sha256 if active else None,
            "cest": cest.sha256 if cest else None,
            "tipi_ipi": tipi.sha256 if tipi else None,
            "ba_st_mva": mva.sha256 if mva else None,
            "nfe_moc_tables": moc.sha256 if moc else None,
            **extra,
        }