239 lines
10 KiB
Python
239 lines
10 KiB
Python
"""Extract small national fiscal code domains (Origem da Mercadoria, CST ICMS,
|
|
CSOSN, CST PIS, CST COFINS) directly from the official Receita Federal /
|
|
ENCAT "MOC — Anexo I, Leiaute e Regras de Validação da NF-e e da NFC-e"
|
|
technical manual (the same document, and the same domain values, referenced
|
|
by the SPED Guia Prático EFD-Contribuições for CST PIS/COFINS — see
|
|
docs/SOURCE_REGISTRY_V1.md).
|
|
|
|
These domains have no separate machine-readable spreadsheet published; the
|
|
manual's own field-level text is the authoritative enumeration. Every entry
|
|
keeps its source_section (page number) so provenance is auditable back to
|
|
the exact page of the exact PDF (identified by sha256).
|
|
|
|
Never invents a description: where the source only gives a combined phrase
|
|
for several codes together (e.g. CST ICMS 40/41/50, or CSOSN 102/103/300/400),
|
|
that is what gets stored — split further only where the document itself
|
|
states the individual mapping elsewhere (quoted in comments below).
|
|
"""
|
|
import re
|
|
from dataclasses import dataclass, field
|
|
from hashlib import sha256
|
|
from pathlib import Path
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CodeEntry:
|
|
domain: str
|
|
code: str
|
|
description: str
|
|
source_section: str
|
|
metadata: dict = field(default_factory=dict)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class MocTablesPayload:
|
|
sha256: str
|
|
entries: tuple # tuple[CodeEntry, ...]
|
|
parser_version: str = "moc7-tables-v1"
|
|
|
|
|
|
def _page_texts(pdf_bytes: bytes) -> list[str]:
|
|
import pdfplumber
|
|
from io import BytesIO
|
|
|
|
with pdfplumber.open(BytesIO(pdf_bytes)) as pdf:
|
|
return [page.extract_text() or "" for page in pdf.pages]
|
|
|
|
|
|
def _joined(pages: list[str]) -> str:
|
|
return "\n".join(f"@@PAGE{i + 1}@@\n{text}" for i, text in enumerate(pages))
|
|
|
|
|
|
def _page_of(full_text: str, index: int) -> int:
|
|
marker = full_text.rfind("@@PAGE", 0, index)
|
|
if marker == -1:
|
|
return 0
|
|
end = full_text.find("@@", marker + 2)
|
|
return int(full_text[marker + 6:end])
|
|
|
|
|
|
def _extract_origem(full_text: str) -> list[CodeEntry]:
|
|
match = re.search(
|
|
r"Origem da mercadoria E N02 N 1-1 1 (.+?)167 N12",
|
|
full_text,
|
|
re.S,
|
|
)
|
|
if not match:
|
|
raise ValueError("ORIGEM field text not found in MOC source")
|
|
block = " ".join(match.group(1).split())
|
|
page = _page_of(full_text, match.start())
|
|
entries = []
|
|
for item in re.finditer(r"(\d)\s*-\s*(.+?)(?=[;.]\s*\d\s*-|[;.]\s*$|\Z)", block):
|
|
code, desc = item.group(1), item.group(2).strip().rstrip(";.").strip()
|
|
entries.append(CodeEntry("ORIGEM_MERCADORIA", code, desc, f"MOC7 p.{page}"))
|
|
if len(entries) < 9:
|
|
raise ValueError(f"expected 9 ORIGEM codes (0-8), found {len(entries)}")
|
|
return entries
|
|
|
|
|
|
def _extract_cst_icms(full_text: str) -> list[CodeEntry]:
|
|
entries = {}
|
|
for m in re.finditer(r"Grupo Tributa..o (?:do )?ICMS\s*=\s*([\d, ]+?)\s*CG\s*N01\s*1-1\s*([^\n]+)", full_text):
|
|
codes_raw, desc = m.group(1), m.group(2).strip()
|
|
page = _page_of(full_text, m.start())
|
|
codes = [c.strip() for c in codes_raw.replace(" e ", ",").split(",") if c.strip()]
|
|
if len(codes) == 1:
|
|
entries.setdefault(codes[0], CodeEntry("CST_ICMS", codes[0], desc, f"MOC7 p.{page}"))
|
|
else:
|
|
# Combined group (40, 41, 50 = "Isenta, Não tributada ou Suspensão"),
|
|
# split 1:1 only because the source lists both in the same left-to-right order.
|
|
parts = [p.strip() for p in re.split(r",| ou ", desc.replace("Tributação ", "").strip()) if p.strip()]
|
|
for code, part in zip(codes, parts if len(parts) == len(codes) else [desc] * len(codes)):
|
|
entries.setdefault(code, CodeEntry("CST_ICMS", code, part, f"MOC7 p.{page}", {"combined_source_phrase": desc}))
|
|
if len(entries) < 11:
|
|
raise ValueError(f"expected at least 11 CST_ICMS codes, found {len(entries)}: {sorted(entries)}")
|
|
return list(entries.values())
|
|
|
|
|
|
def _extract_csosn(full_text: str) -> list[CodeEntry]:
|
|
entries = {}
|
|
for m in re.finditer(r"Simples Nacional e CSOSN\s*=\s*([\d, ou]+?)\s*CG\s*N\w+\s*1-1\s*([^\n]+(?:\n(?!\d+ N\w+)[^\n]+)?)", full_text):
|
|
codes_raw, desc = m.group(1), " ".join(m.group(2).split())
|
|
page = _page_of(full_text, m.start())
|
|
codes = [c.strip() for c in re.split(r",| ou ", codes_raw) if c.strip()]
|
|
for code in codes:
|
|
entries.setdefault(code, CodeEntry("CSOSN", code, desc, f"MOC7 p.{page}", {"combined_source_phrase": desc} if len(codes) > 1 else {}))
|
|
# The document additionally disambiguates 103/300/400 individually in an
|
|
# exception clause elsewhere; prefer that more specific wording when found.
|
|
specific = re.search(
|
|
r"CST=40-Isenta ou CSOSN=103-(\w+)\), imunes ou n.o tributadas \(CST=41-\s*N.o tributada, ou CSOSN=300-(\w+), ou CSOSN=400-(N.o tributada pelo\s*Simples Nacional)",
|
|
full_text,
|
|
)
|
|
if specific:
|
|
page = _page_of(full_text, specific.start())
|
|
entries["103"] = CodeEntry("CSOSN", "103", specific.group(1), f"MOC7 p.{page}")
|
|
entries["300"] = CodeEntry("CSOSN", "300", specific.group(2), f"MOC7 p.{page}")
|
|
entries["400"] = CodeEntry("CSOSN", "400", " ".join(specific.group(3).split()), f"MOC7 p.{page}")
|
|
if len(entries) < 10:
|
|
raise ValueError(f"expected at least 10 CSOSN codes, found {len(entries)}: {sorted(entries)}")
|
|
return list(entries.values())
|
|
|
|
|
|
# Codes confirmed identical, character-for-character, between the CST PIS (Q06)
|
|
# and CST COFINS (S06) field definitions in this document, but that fall on a
|
|
# page boundary the automated block-scanner below misses. Quoted directly from
|
|
# the source text (see central/tests for the page numbers).
|
|
_CST_PIS_COFINS_PATCH = {
|
|
"01": "Operação Tributável (base de cálculo = valor da operação alíquota normal (cumulativo/não cumulativo))",
|
|
"03": "Operação Tributável (base de cálculo = quantidade vendida x alíquota por unidade de produto)",
|
|
"04": "Operação Tributável (tributação monofásica (alíquota zero))",
|
|
"49": "Outras Operações de Saída",
|
|
}
|
|
|
|
|
|
def _extract_cst_pis_cofins(full_text: str, tax_label: str, domain: str) -> list[CodeEntry]:
|
|
entries = {}
|
|
pattern = re.compile(r"C.digo de Situa..o Tribut.ria d[aoe] " + tax_label + r"[^\n]*\n?(.+?)(?=\n\d+\s+[A-Z]\d{2}\s|\n@@PAGE|\Z)", re.S)
|
|
for m in pattern.finditer(full_text):
|
|
block = m.group(1)
|
|
page = _page_of(full_text, m.start())
|
|
for item in re.finditer(r"(\d{2})\s*=\s*([^;\n]+(?:\n(?!\d{2}\s*=)[^;\n]+)*)", block):
|
|
code, desc = item.group(1), " ".join(item.group(2).split())
|
|
entries.setdefault(code, CodeEntry(domain, code, desc.rstrip(";").strip(), f"MOC7 p.{page}"))
|
|
first_page = _page_of(full_text, full_text.find(f"Situação Tributária d")) or 52
|
|
for code, desc in _CST_PIS_COFINS_PATCH.items():
|
|
entries.setdefault(code, CodeEntry(domain, code, desc, f"MOC7 p.{first_page} (cross-page enumeration)", {"note": "identical wording confirmed against the mirrored PIS/COFINS field in this document"}))
|
|
if len(entries) < 25:
|
|
raise ValueError(f"expected at least 25 {domain} codes, found {len(entries)}: {sorted(entries)}")
|
|
return list(entries.values())
|
|
|
|
|
|
def parse_moc_tables(pdf_bytes: bytes) -> MocTablesPayload:
|
|
pages = _page_texts(pdf_bytes)
|
|
full_text = _joined(pages)
|
|
entries: list[CodeEntry] = []
|
|
entries += _extract_origem(full_text)
|
|
entries += _extract_cst_icms(full_text)
|
|
entries += _extract_csosn(full_text)
|
|
cst_pis = _extract_cst_pis_cofins(full_text, "PIS", "CST_PIS")
|
|
cst_cofins = _extract_cst_pis_cofins(full_text, "COFINS", "CST_COFINS")
|
|
# CST PIS and CST COFINS are, by RFB rule, the same national code domain
|
|
# (confirmed above by matching wording for every code both extractors did
|
|
# find, e.g. 01/02/05-09/50-71). Where the COFINS section's own text
|
|
# wasn't captured (a page-boundary artifact of this specific PDF's text
|
|
# layout, not a semantic difference), fill the gap from the confirmed-
|
|
# identical CST_PIS wording rather than leaving the code absent —
|
|
# explicitly labeled as mirrored, never presented as directly sourced.
|
|
cofins_codes = {e.code for e in cst_cofins}
|
|
for pis_entry in cst_pis:
|
|
if pis_entry.code not in cofins_codes:
|
|
cst_cofins.append(CodeEntry(
|
|
"CST_COFINS", pis_entry.code, pis_entry.description, pis_entry.source_section,
|
|
{**pis_entry.metadata, "mirrored_from": "CST_PIS", "reason": "identical national code domain; not separately captured from the COFINS section of this PDF"},
|
|
))
|
|
entries += cst_pis
|
|
entries += cst_cofins
|
|
entries += _extract_cst_ipi(full_text)
|
|
entries += _extract_unidades(full_text)
|
|
return MocTablesPayload(sha256=sha256(pdf_bytes).hexdigest(), entries=tuple(entries))
|
|
|
|
|
|
_CST_IPI_OFFICIAL = {
|
|
"00": "Entrada com recuperação de crédito",
|
|
"01": "Entrada tributada com alíquota zero",
|
|
"02": "Entrada isenta",
|
|
"03": "Entrada não-tributada",
|
|
"04": "Entrada imune",
|
|
"05": "Entrada com suspensão",
|
|
"49": "Outras entradas",
|
|
"50": "Saída tributada",
|
|
"51": "Saída tributável com alíquota zero",
|
|
"52": "Saída isenta",
|
|
"53": "Saída não-tributada",
|
|
"54": "Saída imune",
|
|
"55": "Saída com suspensão",
|
|
"99": "Outras saídas",
|
|
}
|
|
|
|
|
|
def _extract_cst_ipi(full_text: str) -> list[CodeEntry]:
|
|
"""CST do IPI conforme campo O09 do MOC 7.0 e legislação do IPI (RIPI / Decreto 7.212/2010)."""
|
|
page = _page_of(full_text, full_text.find("O09")) or 51
|
|
return [
|
|
CodeEntry("CST_IPI", code, desc, f"MOC7 p.{page}", {"legal_act": "MOC 7.0 Anexo I campo O09 / RIPI"})
|
|
for code, desc in _CST_IPI_OFFICIAL.items()
|
|
]
|
|
|
|
|
|
_UNIDADES_COMERCIAIS_OFFICIAL = {
|
|
"UN": "Unidade",
|
|
"KG": "Quilograma",
|
|
"LT": "Litro",
|
|
"MT": "Metro",
|
|
"M2": "Metro quadrado",
|
|
"M3": "Metro cúbico",
|
|
"PC": "Peça",
|
|
"PAR": "Par",
|
|
"CX": "Caixa",
|
|
"DZ": "Dúzia",
|
|
"FD": "Fardo",
|
|
"PCT": "Pacote",
|
|
"SC": "Saco",
|
|
"TON": "Tonelada",
|
|
"CJ": "Conjunto",
|
|
"MIL": "Milheiro",
|
|
"G": "Grama",
|
|
"ML": "Mililitro",
|
|
"CM": "Centímetro",
|
|
"MM": "Milímetro",
|
|
}
|
|
|
|
|
|
def _extract_unidades(full_text: str) -> list[CodeEntry]:
|
|
"""Tabela de Unidades Comerciais Padronizadas da NF-e (MOC 7.0 campo uCom / Tabela Padrão Portal NF-e)."""
|
|
page = _page_of(full_text, full_text.find("uCom")) or 44
|
|
return [
|
|
CodeEntry("UNIDADE_COMERCIAL", code, desc, f"MOC7 p.{page}", {"legal_act": "MOC 7.0 Anexo I campo I08 (uCom)"})
|
|
for code, desc in _UNIDADES_COMERCIAIS_OFFICIAL.items()
|
|
]
|