chore: establish project baseline
This commit is contained in:
@@ -0,0 +1,39 @@
|
||||
# Hub + local dev-only Firebird settings. Copy to .env locally; never commit
|
||||
# credentials or database URLs (this file is a template, .env is gitignored).
|
||||
#
|
||||
# For a real client Agent installation, use agent/.env.example instead — this
|
||||
# file's FIREBIRD_* values are for the disposable dev/test database only.
|
||||
|
||||
FIREBIRD_DATABASE_PATH=
|
||||
# Default MUST be 0. Only ever set to 1 against the authorized disposable
|
||||
# test database (see CLAUDE.md "Firebird write policy"); the agent fails
|
||||
# closed if the configured path is not exactly that file.
|
||||
FIREBIRD_WRITES=0
|
||||
|
||||
# --- Hub / PostgreSQL ---
|
||||
# DATABASE_URL takes precedence when set. If you override POSTGRES_DB/POSTGRES_USER
|
||||
# below, also set DATABASE_URL explicitly to match — the fallback URL built from
|
||||
# POSTGRES_PASSWORD alone assumes the gti/gti_fiscal defaults.
|
||||
DATABASE_URL=postgresql+psycopg://gti:development-only@localhost:5432/gti_fiscal
|
||||
POSTGRES_DB=gti_fiscal
|
||||
POSTGRES_USER=gti
|
||||
POSTGRES_PASSWORD=
|
||||
HUB_PORT=8000
|
||||
BASE_URL=http://localhost:8000
|
||||
APP_ENV=development
|
||||
LOG_LEVEL=INFO
|
||||
AGENT_ONLINE_THRESHOLD_SECONDS=120
|
||||
|
||||
# Local-dev-only agent bootstrap. Leave unset in any shared/pilot environment —
|
||||
# real Agents are created with central/scripts/register_agent.py.
|
||||
AGENT_DEV_TOKEN=
|
||||
|
||||
NCM_DATASET_URL=https://portalunico.siscomex.gov.br/classif/api/publico/nomenclatura/download/json
|
||||
CEST_DATASET_URL=https://www.confaz.fazenda.gov.br/legislacao/convenios/2018/CV142_18
|
||||
DATASET_STORAGE_PATH=./data/datasets
|
||||
HTTP_TIMEOUT_SECONDS=30
|
||||
|
||||
# Company context (Agent). Leave empty when C000004 has one row and CRT sources agree.
|
||||
# COMPANY_CODE= # C000004.CODIGO of the active company (when C000004 has several rows and EMISSOR cannot disambiguate)
|
||||
# COMPANY_CNPJ= # alternative selector (digits only); a stable, verifiable identifier
|
||||
# COMPANY_CRT_SOURCE= # EMISSOR | C000004 — required when the two CRT values disagree (emission programs read EMISSOR)
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
*.FDB
|
||||
*.FDB-journal
|
||||
*.FDB.lck
|
||||
.env
|
||||
__pycache__/
|
||||
.pytest_cache/
|
||||
.superpowers/
|
||||
agent/*.sqlite
|
||||
agent/logs/
|
||||
agent/*.log
|
||||
|
||||
dist/
|
||||
|
||||
# --- release closure: secrets, runtime state, local configs, temporary artifacts ---
|
||||
.env.*
|
||||
!.env.example
|
||||
*.env
|
||||
!*.env.example
|
||||
*.secret
|
||||
secrets/
|
||||
credentials*
|
||||
*.pem
|
||||
*.key
|
||||
*.pfx
|
||||
*.log
|
||||
logs/
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
*.db
|
||||
*.dump
|
||||
*.zip
|
||||
Banco/
|
||||
bancos_teste/
|
||||
.playwright-mcp/
|
||||
.superpowers/
|
||||
deploy/hub/backups/
|
||||
deploy/hub/.env
|
||||
docs/runs/multi_db/_*
|
||||
scripts/_*
|
||||
.worktrees/
|
||||
*.tmp
|
||||
*.bak
|
||||
*.gz
|
||||
*.pkl
|
||||
@@ -0,0 +1,139 @@
|
||||
# Task 1 Report
|
||||
|
||||
## Status
|
||||
|
||||
Implemented the read-only discovery foundation. The FDB database was not opened,
|
||||
inspected, or modified.
|
||||
|
||||
## Files Changed
|
||||
|
||||
- `src/gti_discovery/__init__.py`: exports the read-only public interface.
|
||||
- `src/gti_discovery/read_only.py`: validates one `SELECT`/`WITH` statement and
|
||||
exposes `ReadOnlyConnection.execute` without connection write methods.
|
||||
- `tests/test_read_only.py`: covers accepted queries, normalization, prohibited
|
||||
statements, comments, multiple statements, quoted literals, and delegation.
|
||||
- `pyproject.toml`: defines the package and `src`-layout pytest configuration.
|
||||
- `.env.example`: documents an empty database-path placeholder without secrets.
|
||||
- `CLAUDE.md`: records the strict read-only project constraints and test command.
|
||||
|
||||
## Tests
|
||||
|
||||
Command:
|
||||
|
||||
```text
|
||||
python -m pytest tests/test_read_only.py -q
|
||||
```
|
||||
|
||||
Result: `18 passed in 0.04s`.
|
||||
|
||||
`git diff --check` also completed without whitespace errors.
|
||||
|
||||
## Concerns
|
||||
|
||||
- Commit was not created because Git has no configured author identity (`Author
|
||||
identity unknown`). Git configuration was not changed.
|
||||
- Pytest emits an existing `pytest-asyncio` deprecation warning because its event
|
||||
loop fixture scope is not configured; the focused tests still pass.
|
||||
- The validator intentionally rejects all SQL comments and any prohibited keyword
|
||||
outside quoted values. It does not attempt to parse the full Firebird grammar.
|
||||
- The wrapper expects the injected driver connection to provide `execute(sql,
|
||||
params)`; no Firebird driver is installed or required for this offline task.
|
||||
|
||||
## Reviewer Fixes
|
||||
|
||||
- Replaced the retained raw driver connection with a narrow execute-only
|
||||
delegate and added coverage proving `_connection`, `commit`, and `rollback`
|
||||
are unavailable on `ReadOnlyConnection`.
|
||||
- Added rejection for selectable procedure calls, `GEN_ID`, generator value
|
||||
expressions, `WITH LOCK`, and `FOR UPDATE`. The wrapper cannot enforce the
|
||||
driver's transaction/read-only mode, so deployment must use a read-only
|
||||
database account or driver-level setting.
|
||||
- Replaced global `split`/`join` normalization with whitespace collapsing only
|
||||
outside quoted literals; quoted contents are preserved byte-for-byte.
|
||||
- Added regression tests for side-effecting SELECT constructs and literal
|
||||
preservation.
|
||||
|
||||
## Fix Verification
|
||||
|
||||
Exact command:
|
||||
|
||||
```text
|
||||
python -m pytest tests -q
|
||||
```
|
||||
|
||||
Exact output:
|
||||
|
||||
```text
|
||||
C:\Users\Gustavo\AppData\Local\Programs\Python\Python312\Lib\site-packages\pytest_asyncio\plugin.py:299: PytestDeprecationWarning: The configuration option "asyncio_default_fixture_loop_scope" is unset.
|
||||
The event loop scope for asynchronous fixtures will default to the "fixture" caching scope. Future versions of pytest-asyncio will default the fixture scope to "function". Set the default fixture loop scope explicitly to avoid unexpected behavior.
|
||||
|
||||
warnings.warn(PytestDeprecationWarning(_DEFAULT_FIXTURE_LOOP_SCOPE_UNSET))
|
||||
....................... [100%]
|
||||
23 passed in 0.06s
|
||||
```
|
||||
|
||||
## Re-review Fixes
|
||||
|
||||
- Replaced the instance-bound driver delegate with an opaque-token lookup in a
|
||||
module-private registry. `ReadOnlyConnection` stores only the token; its
|
||||
bound `execute` method has the wrapper as `__self__` and no closure.
|
||||
- Added a regression test that checks `__self__`, `__closure__`, wrapper slots,
|
||||
and reachable slot values cannot expose the raw driver or `commit`.
|
||||
- Added conservative rejection of unknown and qualified function calls, while
|
||||
allowing a small deterministic-function allowlist. This covers selectable
|
||||
procedures, `GEN_ID`, UDF/external-function style calls, generator values,
|
||||
locking, and update forms. True read-only enforcement still requires a
|
||||
database read-only account or read-only transaction/driver configuration.
|
||||
|
||||
## Re-review Verification
|
||||
|
||||
Exact command:
|
||||
|
||||
```text
|
||||
python -m pytest tests -q
|
||||
```
|
||||
|
||||
Exact output:
|
||||
|
||||
```text
|
||||
C:\Users\Gustavo\AppData\Local\Programs\Python\Python312\Lib\site-packages\pytest_asyncio\plugin.py:299: PytestDeprecationWarning: The configuration option "asyncio_default_fixture_loop_scope" is unset.
|
||||
The event loop scope for asynchronous fixtures will default to the "fixture" caching scope. Future versions of pytest-asyncio will default the fixture scope to "function". Set the default fixture loop scope explicitly to avoid unexpected behavior.
|
||||
|
||||
warnings.warn(PytestDeprecationWarning(_DEFAULT_FIXTURE_LOOP_SCOPE_UNSET))
|
||||
.......................... [100%]
|
||||
26 passed in 0.07s
|
||||
```
|
||||
|
||||
## Third Fix Round
|
||||
|
||||
- Removed the module-level bound-method registry. `ReadOnlyConnection` now
|
||||
accepts only a narrow adapter-supplied execute callable and exposes no driver
|
||||
or connection attributes. Python process-level introspection is explicitly
|
||||
outside this guarantee.
|
||||
- Rejected all function-like SQL calls except a small deterministic-function
|
||||
allowlist, including qualified names with arbitrary whitespace around dots.
|
||||
Selectable procedures in `FROM`, generator calls, `GEN_ID`, `EXECUTE`, UDF,
|
||||
and external-function-style calls are rejected conservatively.
|
||||
- Documented that actual read-only enforcement requires a Firebird read-only
|
||||
account or read-only transaction/driver configuration.
|
||||
- Added regression coverage for spaced qualified calls and the practical public
|
||||
API boundary.
|
||||
|
||||
## Third Fix Verification
|
||||
|
||||
Exact command:
|
||||
|
||||
```text
|
||||
python -m pytest tests -q
|
||||
```
|
||||
|
||||
Exact output:
|
||||
|
||||
```text
|
||||
C:\Users\Gustavo\AppData\Local\Programs\Python\Python312\Lib\site-packages\pytest_asyncio\plugin.py:299: PytestDeprecationWarning: The configuration option "asyncio_default_fixture_loop_scope" is unset.
|
||||
The event loop scope for asynchronous fixtures will default to the "fixture" caching scope. Future versions of pytest-asyncio will default the fixture scope to "function". Set the default fixture loop scope explicitly to avoid unexpected behavior.
|
||||
|
||||
warnings.warn(PytestDeprecationWarning(_DEFAULT_FIXTURE_LOOP_SCOPE_UNSET))
|
||||
................................... [100%]
|
||||
35 passed in 0.09s
|
||||
```
|
||||
@@ -0,0 +1,78 @@
|
||||
# GTI Discovery
|
||||
|
||||
This project is strictly read-only. Discovery code must use a database-level
|
||||
read-only account or transaction/driver mode; lexical SQL validation is only
|
||||
defense-in-depth. Discovery code must not perform DML or DDL,
|
||||
modify database objects or permissions, or store credentials and other secrets.
|
||||
|
||||
Run the offline tests with:
|
||||
|
||||
```text
|
||||
python -m pytest tests -q
|
||||
```
|
||||
|
||||
## Firebird write policy
|
||||
|
||||
Default policy:
|
||||
- All client/production Firebird databases are STRICTLY READ-ONLY.
|
||||
- Never perform INSERT, UPDATE, DELETE, DDL, permission changes, or destructive operations
|
||||
against any client/production ERP database unless explicitly authorized in a future policy change.
|
||||
|
||||
Explicit development exception:
|
||||
|
||||
The following database is a disposable TEST DATABASE and is explicitly authorized for writes:
|
||||
|
||||
```
|
||||
D:\PROJETOS\GTISISTEMAS\Projeto\TESTESOFICIAIS.FDB
|
||||
```
|
||||
|
||||
For this exact file only, the agent MAY perform:
|
||||
|
||||
- SELECT
|
||||
- INSERT
|
||||
- UPDATE
|
||||
- DELETE
|
||||
- DDL
|
||||
- transactional tests
|
||||
- rollback/recovery tests
|
||||
- destructive development experiments when necessary
|
||||
|
||||
This database has external intact backups and can be restored.
|
||||
|
||||
Safety requirement:
|
||||
|
||||
Before ANY Firebird write, the code must verify that the canonical absolute database path is exactly:
|
||||
|
||||
```
|
||||
D:\PROJETOS\GTISISTEMAS\Projeto\TESTESOFICIAIS.FDB
|
||||
```
|
||||
|
||||
If the path differs in any way, writes must fail closed.
|
||||
|
||||
Never infer write authorization from filename alone.
|
||||
Never enable writes to other Firebird databases.
|
||||
|
||||
Secrets:
|
||||
- credentials remain local in ignored environment files;
|
||||
- never log, commit, or expose passwords.
|
||||
|
||||
Development priority:
|
||||
- TESTESOFICIAIS.FDB may be modified freely as needed to validate the MVP.
|
||||
- Do not over-engineer rollback protections for this disposable test database unless they are part
|
||||
of the product behavior being validated.
|
||||
|
||||
## GTI Fiscal write-target rule
|
||||
|
||||
The SGV keeps its fiscal defaults in the canonical product record (`C000025`); every other routine consumes them.
|
||||
GTI Fiscal therefore writes ONLY to the canonical product record, and only fields whose mapping is proven
|
||||
(`docs/SGV_PRODUCT_FISCAL_UI_MAP_V1.json`, `write_target=true`). It never writes to NFCE, NFCE_ITENS, item tables,
|
||||
issued documents or any other transactional/historical table; those are read-only evidence for validation.
|
||||
Company context (UF, CRT) is read from `C000004` at run time — never hardcoded.
|
||||
|
||||
## Local completion mission — exact write allowlist (operator-authorized)
|
||||
|
||||
For the "GTI FISCAL LOCAL COMPLETION V1" mission the operator explicitly authorized writes to the disposable COPIES listed, by exact canonical path, in
|
||||
`docs/LOCAL_WRITE_ALLOWLIST_V1.json` (CONSTRUCAO, MMMODAS, RINCAO, UNIAO under `Projeto\Banco\`, plus TESTESOFICIAIS.FDB). Authorization is by exact
|
||||
path only — never by directory or `.FDB` extension. Any other Firebird under `D:\PROJETOS\GTISISTEMAS` stays strictly READ-ONLY. Writes go through the
|
||||
guarded executor (path allowlist, company-context guard, schema/type/length guard, compare-before-write, rowcount, transaction, audit) and only to
|
||||
`C000025` fields whose contract in `docs/FISCAL_FIELD_CONTRACT_V1.json` has `WRITE_ALLOWED=true`. This authorization does not extend to production or client databases.
|
||||
@@ -0,0 +1,41 @@
|
||||
# One of these per Agent installation (one per client machine). Copy to
|
||||
# agent/.env on that machine; never commit it. Get AGENT_ID/AGENT_TOKEN/
|
||||
# COMPANY_ID from central/scripts/register_agent.py, run once per installation.
|
||||
|
||||
GTI_HUB_URL=https://hub.example.com
|
||||
AGENT_ID=
|
||||
AGENT_TOKEN=
|
||||
COMPANY_ID=
|
||||
COMPANY_UF=
|
||||
|
||||
# The client's Firebird ERP database.
|
||||
FIREBIRD_DATABASE_PATH=
|
||||
FB_USER=
|
||||
FB_PASSWORD=
|
||||
FIREBIRD_CLIENT_LIBRARY=
|
||||
|
||||
# Default MUST be 0. Writes only ever happen against exactly the database
|
||||
# named by AUTHORIZED_WRITE_DATABASE_PATH below (never inferred from
|
||||
# FIREBIRD_DATABASE_PATH alone, never from the ".FDB" extension).
|
||||
FIREBIRD_WRITES=0
|
||||
|
||||
# Required to enable writes for THIS installation. Must be set to the exact
|
||||
# same absolute path as FIREBIRD_DATABASE_PATH above — this is the operator's
|
||||
# explicit, per-installation authorization. Leaving it unset means
|
||||
# FIREBIRD_WRITES=1 has no effect against a real client database (it falls
|
||||
# back to the developer-only disposable test database path and therefore
|
||||
# fails closed for anything else).
|
||||
AUTHORIZED_WRITE_DATABASE_PATH=
|
||||
|
||||
# Required to enable writes for THIS installation. Must equal the schema
|
||||
# fingerprint this installation was verified/registered against; if the live
|
||||
# schema fingerprint (reported in heartbeats) ever differs, writes are
|
||||
# refused until an operator re-verifies and updates this value.
|
||||
EXPECTED_SCHEMA_FINGERPRINT=
|
||||
|
||||
POLL_INTERVAL_SECONDS=10
|
||||
AGENT_VERSION=0.1.0
|
||||
|
||||
# Optional overrides:
|
||||
# AGENT_STATE_DB=C:\GTI\Agent\agent_state.sqlite
|
||||
# AGENT_LOG_PATH=C:\GTI\Agent\logs\agent.log
|
||||
@@ -0,0 +1,8 @@
|
||||
[project]
|
||||
name = "gti-fiscal-agent"
|
||||
version = "1.0.0rc1"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = ["httpx>=0.27,<1"]
|
||||
|
||||
[project.optional-dependencies]
|
||||
windows-service = ["pywin32>=306; sys_platform == 'win32'"]
|
||||
@@ -0,0 +1,46 @@
|
||||
"""GTI Fiscal Agent installation preflight (READ-ONLY): python agent/scripts/preflight.py [--env-file FILE] [--json]
|
||||
|
||||
Prints PREFLIGHT=PASS|FAIL and the blockers; exit code 0 on PASS, 2 on FAIL. Secrets are never printed.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(AGENT_ROOT / "src"))
|
||||
from gti_fiscal_agent.preflight import run_preflight # noqa: E402
|
||||
|
||||
|
||||
def load_env(path):
|
||||
if path and Path(path).exists():
|
||||
for line in Path(path).read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
k, _, v = line.partition("=")
|
||||
os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--env-file")
|
||||
ap.add_argument("--json", action="store_true")
|
||||
a = ap.parse_args(argv)
|
||||
load_env(a.env_file or os.environ.get("GTI_AGENT_ENV_FILE") or AGENT_ROOT.parent / ".env")
|
||||
os.environ.setdefault("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite"))
|
||||
os.environ.setdefault("AGENT_LOG_PATH", str(AGENT_ROOT / "logs" / "agent.log"))
|
||||
res = run_preflight(os.environ)
|
||||
if a.json:
|
||||
print(json.dumps(res, ensure_ascii=False, indent=1, default=str))
|
||||
else:
|
||||
for c in res["checks"]:
|
||||
print(f" [{c['status']:<4}] {c['name']:<24} {c['detail']}")
|
||||
print(f"PREFLIGHT={res['PREFLIGHT']}")
|
||||
for b in res["blockers"]:
|
||||
print(f"BLOCKER: {b}")
|
||||
raise SystemExit(0 if res["PREFLIGHT"] == "PASS" else 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Agent preflight: COMPANY_CONTEXT=PASS/FAIL for one Firebird database (READ-ONLY).
|
||||
|
||||
python agent/scripts/preflight_company_context.py [--database <FDB>]
|
||||
|
||||
Configuration (environment or the local ignored .env): COMPANY_CODE / COMPANY_CNPJ (company selection when C000004 is ambiguous),
|
||||
COMPANY_CRT_SOURCE = EMISSOR | C000004 (explicit choice when the two CRT sources disagree). The full CNPJ is never printed.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "agent" / "src"))
|
||||
from gti_fiscal_agent.company_context import preflight_company_context, preflight_report # noqa: E402
|
||||
|
||||
|
||||
def load_dotenv(path):
|
||||
if path.exists():
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if "=" in line and not line.startswith("#"):
|
||||
k, _, v = line.partition("=")
|
||||
os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--database")
|
||||
a = ap.parse_args(argv)
|
||||
load_dotenv(ROOT / ".env")
|
||||
if a.database:
|
||||
os.environ["FIREBIRD_DATABASE_PATH"] = a.database
|
||||
report = preflight_report(preflight_company_context(os.environ))
|
||||
print(json.dumps(report, ensure_ascii=False, indent=1))
|
||||
raise SystemExit(0 if report["COMPANY_CONTEXT"] == "PASS" else 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Run the agent against a sanitized sample only after an explicit fingerprint gate."""
|
||||
import argparse, json, os, sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).parents[1] / "src"))
|
||||
from gti_fiscal_agent.client import HubClient
|
||||
from gti_fiscal_agent.runner import run_sample
|
||||
from gti_fiscal_agent.state import AgentState
|
||||
|
||||
FIELDS = "CODIGO,PRODUTO,NCM,CLASSIFICACAO_FISCAL,CFOP,CFOP_ENTRADA,CST,CST_ENTRADA,ALIQUOTA,CEST,CSOSN,COD_ENTRADA_COFINS,COD_SAIDA_COFINS,ALIQ_COFINS,COD_ENTRADA_PIS,COD_SAIDA_PIS,ALIQ_PIS,ALIQ_IPI,CST_IPI,ORIGEM,IAT,INCIDENCIA_PISCOFINS,IPPT,ULTIMA_ALTERACAO"
|
||||
|
||||
def read_firebird_products(codes):
|
||||
import fdb
|
||||
connection = fdb.connect(database=os.environ["FIREBIRD_DATABASE_PATH"], user=os.environ["FB_USER"], password=os.environ["FB_PASSWORD"], fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"), isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
|
||||
try:
|
||||
cursor = connection.cursor(); cursor.execute(f"SELECT {FIELDS} FROM C000025 WHERE CODIGO IN ({','.join('?' for _ in codes)}) ORDER BY CODIGO", tuple(codes))
|
||||
names = [column[0].strip() for column in cursor.description]
|
||||
return [dict(zip(names, row)) for row in cursor.fetchall()]
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(); parser.add_argument("--sample", required=True); parser.add_argument("--output", required=True); parser.add_argument("--schema-fingerprint", default="")
|
||||
args = parser.parse_args(); sample = json.loads(Path(args.sample).read_text(encoding="utf-8"))
|
||||
codes = [item["CODIGO"] for item in sample.get("sample_without_ncm", [])[:20]]
|
||||
products = read_firebird_products(codes)
|
||||
result = run_sample(products, HubClient(os.getenv("HUB_URL", "http://localhost:8000"), os.getenv("AGENT_DEV_TOKEN", "")), AgentState(Path(args.output).with_suffix(".sqlite")), args.schema_fingerprint)
|
||||
result["products_requested"] = len(products); result["firebird_products_read"] = len(products); result["firebird_access"] = "READ_ONLY_SELECT"
|
||||
output = Path(args.output)
|
||||
if result["results"]:
|
||||
output.write_text(json.dumps(result, indent=2, default=str), encoding="utf-8")
|
||||
elif not output.exists():
|
||||
output.write_text(json.dumps(result, indent=2, default=str), encoding="utf-8")
|
||||
print(json.dumps(result, indent=2))
|
||||
if __name__ == "__main__": main()
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Integrated Agent flow: Firebird read -> Hub /v1/classify/batch -> IPI dry-run
|
||||
-> guarded auto-apply write.
|
||||
|
||||
FIREBIRD_WRITES=0 (default): classifies every product and logs the decision
|
||||
(and what write *would* happen) to the local SQLite audit log; never writes.
|
||||
|
||||
FIREBIRD_WRITES=1: also auto-applies every write_eligible C000025.IPI proposal
|
||||
where rate_kind=NUMERIC, reusing the same schema/path/allowlist/compare-before-write
|
||||
guards as the manual executor. Fails closed before touching the connection unless
|
||||
the configured database is exactly the authorized disposable test database.
|
||||
|
||||
Auto-apply is scoped to table=C000025, field=IPI, rate_kind=NUMERIC only. NT/
|
||||
non-numeric markers, blank rates, ambiguous EX candidates, legacy/invalid NCM,
|
||||
CEST and EX_TIPI are never auto-applied here.
|
||||
"""
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_ROOT = Path(__file__).resolve().parents[1]
|
||||
PROJECT_ROOT = AGENT_ROOT.parent
|
||||
sys.path.insert(0, str(AGENT_ROOT / "src"))
|
||||
|
||||
from gti_fiscal_agent.client import HubClient # noqa: E402
|
||||
from gti_fiscal_agent.firebird_reader import read_products # noqa: E402
|
||||
from gti_fiscal_agent.pipeline import run_pipeline # noqa: E402
|
||||
from gti_fiscal_agent.runner import REFERENCE_SCHEMA_V1, to_hub_product # noqa: E402
|
||||
from gti_fiscal_agent.state import AgentState # noqa: E402
|
||||
|
||||
|
||||
def load_dotenv(path: Path) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
os.environ.setdefault(key.strip(), value.strip())
|
||||
|
||||
|
||||
def load_executor():
|
||||
spec = importlib.util.spec_from_file_location("firebird_write_executor", PROJECT_ROOT / "scripts" / "firebird_write_executor.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--hub-url", default=os.getenv("HUB_URL", "http://localhost:8000"))
|
||||
parser.add_argument("--token", default=os.getenv("AGENT_DEV_TOKEN", ""))
|
||||
parser.add_argument("--state-db", default=str(AGENT_ROOT / "agent_state.sqlite"))
|
||||
parser.add_argument("--schema-fingerprint", default=REFERENCE_SCHEMA_V1)
|
||||
args = parser.parse_args()
|
||||
|
||||
load_dotenv(PROJECT_ROOT / ".env")
|
||||
if not args.token:
|
||||
args.token = os.environ.get("AGENT_DEV_TOKEN", "")
|
||||
|
||||
firebird_writes = os.environ.get("FIREBIRD_WRITES", "0") == "1"
|
||||
executor = load_executor()
|
||||
database_path = os.environ["FIREBIRD_DATABASE_PATH"]
|
||||
|
||||
if firebird_writes:
|
||||
# Fail closed before any product is read or any connection is opened for writing.
|
||||
executor.assert_authorized_write_path(database_path)
|
||||
|
||||
if args.schema_fingerprint != REFERENCE_SCHEMA_V1:
|
||||
print(json.dumps({"status": "SCHEMA_UNKNOWN", "processed": 0}))
|
||||
return
|
||||
|
||||
products = read_products()
|
||||
client = HubClient(args.hub_url, args.token)
|
||||
decisions = client.classify_batch([to_hub_product(p) for p in products])
|
||||
|
||||
state = AgentState(args.state_db)
|
||||
|
||||
write_connection = None
|
||||
if firebird_writes:
|
||||
import fdb
|
||||
write_connection = fdb.connect(
|
||||
database=database_path,
|
||||
user=os.environ["FB_USER"],
|
||||
password=os.environ["FB_PASSWORD"],
|
||||
fb_library_name=os.environ.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
)
|
||||
|
||||
try:
|
||||
summary = run_pipeline(products, decisions, state, firebird_writes, write_connection, executor, database_path)
|
||||
finally:
|
||||
if write_connection is not None:
|
||||
write_connection.close()
|
||||
|
||||
summary = {"status": "OK", "firebird_writes_enabled": firebird_writes, **summary}
|
||||
print(json.dumps(summary, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,297 @@
|
||||
"""Long-running Agent service: heartbeat + poll Hub jobs + execute pipeline.
|
||||
|
||||
startup -> load config -> validate Hub -> validate Firebird -> heartbeat
|
||||
-> poll job -> claim -> execute -> report result -> back to polling.
|
||||
|
||||
No job available: sleep POLL_INTERVAL_SECONDS.
|
||||
Hub unavailable: retry with simple capped backoff; the Agent stays alive.
|
||||
Job fails internally (bad Firebird read, classify error, write refused):
|
||||
the job is reported FAILED with a sanitized error and the Agent keeps polling.
|
||||
SIGINT/SIGTERM: finish the current job (if any) and stop the loop cleanly.
|
||||
|
||||
Written to be runnable as-is today, and easy to later wrap as a Windows
|
||||
Service (a service wrapper would just call main()); no installer here.
|
||||
"""
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_ROOT = Path(__file__).resolve().parents[1]
|
||||
PROJECT_ROOT = AGENT_ROOT.parent
|
||||
sys.path.insert(0, str(AGENT_ROOT / "src"))
|
||||
sys.path.insert(0, str(PROJECT_ROOT)) # ships central/app engine modules read-only (central.app.*)
|
||||
|
||||
from gti_fiscal_agent.client import HubClient # noqa: E402
|
||||
from gti_fiscal_agent.firebird_reader import check_connection, read_products # noqa: E402
|
||||
from gti_fiscal_agent.pipeline import run_pipeline # noqa: E402
|
||||
from gti_fiscal_agent.runner import REFERENCE_SCHEMA_V1, to_hub_product # noqa: E402
|
||||
from gti_fiscal_agent.state import AgentState # noqa: E402
|
||||
|
||||
_MAX_BACKOFF_SECONDS = 60
|
||||
|
||||
logger = logging.getLogger("gti_fiscal_agent")
|
||||
|
||||
|
||||
def setup_logging(log_path):
|
||||
"""Rotating local log file. Never logs FB_PASSWORD or AGENT_TOKEN — every
|
||||
log call in this module only references non-secret identifiers/counts."""
|
||||
logger.setLevel(logging.INFO)
|
||||
if logger.handlers:
|
||||
return logger
|
||||
Path(log_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
handler = logging.handlers.RotatingFileHandler(log_path, maxBytes=5_000_000, backupCount=5, encoding="utf-8")
|
||||
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
|
||||
logger.addHandler(handler)
|
||||
logger.addHandler(logging.StreamHandler())
|
||||
return logger
|
||||
|
||||
|
||||
def load_dotenv(path: Path) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
os.environ.setdefault(key.strip(), value.strip())
|
||||
|
||||
|
||||
def load_executor():
|
||||
spec = importlib.util.spec_from_file_location("firebird_write_executor", PROJECT_ROOT / "scripts" / "firebird_write_executor.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class Config:
|
||||
def __init__(self, env):
|
||||
self.hub_url = env.get("GTI_HUB_URL") or env.get("HUB_URL") or "http://localhost:8000"
|
||||
self.agent_id = env.get("AGENT_ID", "")
|
||||
self.agent_token = env.get("AGENT_TOKEN") or env.get("AGENT_DEV_TOKEN", "")
|
||||
self.company_id = env.get("COMPANY_ID", "")
|
||||
self.company_uf = env.get("COMPANY_UF", "")
|
||||
self.database_path = env["FIREBIRD_DATABASE_PATH"]
|
||||
# FISCAL_MODE gates writes technically: DRY_RUN can never write, whatever FIREBIRD_WRITES says. Unset = legacy
|
||||
# behaviour (FIREBIRD_WRITES decides). `firebird_writes` below is the EFFECTIVE value used by every write path.
|
||||
requested = env.get("FIREBIRD_WRITES", "0") == "1"
|
||||
self.fiscal_mode = (env.get("FISCAL_MODE") or ("AUTO_APPLY" if requested else "DRY_RUN")).strip().upper()
|
||||
self.requested_writes = requested
|
||||
self.firebird_writes = requested and self.fiscal_mode == "AUTO_APPLY"
|
||||
self.poll_interval_seconds = float(env.get("POLL_INTERVAL_SECONDS", "5"))
|
||||
self.agent_version = env.get("AGENT_VERSION", "1.0.0-rc1")
|
||||
self.schema_fingerprint = env.get("SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1)
|
||||
self.state_db = env.get("AGENT_STATE_DB", str(AGENT_ROOT / "agent_state.sqlite"))
|
||||
# Pilot write authorization: both must be explicitly set by the installer for
|
||||
# a real client database. Left unset, both default to the dev-only values,
|
||||
# so a real client path/schema is never silently authorized (see
|
||||
# scripts/firebird_write_executor.py's assert_authorized_write_path).
|
||||
self.authorized_write_database_path = env.get("AUTHORIZED_WRITE_DATABASE_PATH")
|
||||
self.expected_schema_fingerprint = env.get("EXPECTED_SCHEMA_FINGERPRINT", REFERENCE_SCHEMA_V1)
|
||||
self.log_path = env.get("AGENT_LOG_PATH", str(AGENT_ROOT / "logs" / "agent.log"))
|
||||
|
||||
|
||||
class GracefulStop:
|
||||
def __init__(self):
|
||||
self.stop = False
|
||||
signal.signal(signal.SIGINT, self._handle)
|
||||
signal.signal(signal.SIGTERM, self._handle)
|
||||
|
||||
def _handle(self, signum, frame):
|
||||
self.stop = True
|
||||
|
||||
|
||||
def build_job_coverage(env, client, products, decisions):
|
||||
"""Company-context-aware coverage over the job's products (read-only). Fails closed when the company context is unresolved."""
|
||||
from gti_fiscal_agent.company_context import mask_cnpj, preflight_company_context
|
||||
from gti_fiscal_agent.coverage import build_refs, compute_coverage
|
||||
from gti_fiscal_agent.firebird_reader import read_evidence
|
||||
ctx = preflight_company_context(env)
|
||||
if ctx["status"] != "PASS":
|
||||
raise RuntimeError("COMPANY_CONTEXT=FAIL: " + "; ".join(ctx.get("reasons") or ["unresolved"]))
|
||||
evidence = read_evidence(env, ctx["company_code"])
|
||||
refs = build_refs(client.get_json, products, ctx["company_uf"])
|
||||
report = compute_coverage(products, decisions, {"uf": ctx["company_uf"], "crt": ctx["company_crt"]}, evidence.get("company_default_csosn"), refs, evidence)
|
||||
report["company_context"] = {"COMPANY_CONTEXT_STATUS": "PASS", "classification": ctx["classification"], "company_code": ctx["company_code"], "company_cnpj": ctx["company_cnpj"], "company_uf": ctx["company_uf"],
|
||||
"company_crt": ctx["company_crt"], "source_of_truth": ctx["source_of_truth"], "confidence": ctx["confidence"], "warnings": ctx["warnings"]}
|
||||
return report
|
||||
|
||||
|
||||
def open_write_connection(env, database_path, firebird_writes=True):
|
||||
if not firebird_writes: # technical guarantee: no write-capable connection exists in DRY_RUN / FIREBIRD_WRITES=0
|
||||
raise RuntimeError("write connection refused: writes are disabled (FISCAL_MODE=DRY_RUN or FIREBIRD_WRITES=0)")
|
||||
import fdb
|
||||
return fdb.connect(
|
||||
database=database_path,
|
||||
user=env["FB_USER"],
|
||||
password=env["FB_PASSWORD"],
|
||||
fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
)
|
||||
|
||||
|
||||
def execute_job(job, config, executor, state, env):
|
||||
"""Run the pipeline for one claimed job. Never raises for job-domain
|
||||
failures (bad Firebird read, classify error) — returns (result_summary, error)."""
|
||||
kind = job["kind"]
|
||||
payload = job.get("payload") or {}
|
||||
try:
|
||||
if config.firebird_writes:
|
||||
# Layered, per-installation authorization: the path must match this
|
||||
# installation's own explicitly configured authorized_write_database_path
|
||||
# (defaults to the dev-only constant, so a real client path is refused
|
||||
# unless the installer deliberately set it), AND the live schema
|
||||
# fingerprint must match what this installation was configured to expect.
|
||||
executor.assert_authorized_write_path(config.database_path, authorized_path=config.authorized_write_database_path)
|
||||
executor.assert_schema_fingerprint_authorized(config.schema_fingerprint, config.expected_schema_fingerprint)
|
||||
|
||||
codigos = payload.get("codigos") if kind == "REPROCESS_PRODUCTS" else None
|
||||
products = read_products(env, codigos=codigos)
|
||||
logger.info("job %s: read %d product(s) from Firebird", job.get("job_id"), len(products))
|
||||
|
||||
client = HubClient(config.hub_url, config.agent_token)
|
||||
decisions = client.classify_batch([to_hub_product(p) for p in products])
|
||||
|
||||
write_connection = None
|
||||
if config.firebird_writes:
|
||||
write_connection = open_write_connection(env, config.database_path)
|
||||
try:
|
||||
summary = run_pipeline(products, decisions, state, config.firebird_writes, write_connection, executor, config.database_path, authorized_path=config.authorized_write_database_path)
|
||||
finally:
|
||||
if write_connection is not None:
|
||||
write_connection.close()
|
||||
|
||||
coverage = build_job_coverage(env, client, products, decisions) if kind == "REPROCESS_ALL" else None
|
||||
result_summary = {
|
||||
"coverage": coverage,
|
||||
"fiscal_mode": getattr(config, "fiscal_mode", None),
|
||||
"release_version": env.get("AGENT_VERSION") or "1.0.0-rc1",
|
||||
"products_processed": summary["processed"],
|
||||
"writes_attempted": summary["writes_attempted"],
|
||||
"writes_succeeded": summary["writes_succeeded"],
|
||||
"writes_failed": summary["writes_failed"],
|
||||
"no_ops": summary["no_ops"],
|
||||
"firebird_writes_enabled": config.firebird_writes,
|
||||
"agent_id": job["agent_id"],
|
||||
"schema_fingerprint": config.schema_fingerprint,
|
||||
}
|
||||
logger.info(
|
||||
"job %s finished: processed=%d writes_attempted=%d writes_succeeded=%d writes_failed=%d no_ops=%d",
|
||||
job.get("job_id"), summary["processed"], summary["writes_attempted"], summary["writes_succeeded"], summary["writes_failed"], summary["no_ops"],
|
||||
)
|
||||
if summary["writes_failed"] > 0:
|
||||
return result_summary, f"{summary['writes_failed']} write(s) failed guard checks"
|
||||
return result_summary, None
|
||||
except executor.WriteRefused as error:
|
||||
logger.warning("job %s refused: %s", job.get("job_id"), error)
|
||||
return None, f"write refused: {error}"
|
||||
except Exception as error: # noqa: BLE001 — job-domain isolation boundary
|
||||
logger.warning("job %s failed internally: %s: %s", job.get("job_id"), type(error).__name__, error)
|
||||
return None, f"{type(error).__name__}: {error}"
|
||||
|
||||
|
||||
def run(config, env, *, max_iterations=None, stopper=None):
|
||||
"""`stopper` defaults to a signal-based GracefulStop (SIGINT/SIGTERM), which
|
||||
only works from the main thread. A host that manages its own lifecycle
|
||||
(e.g. the Windows Service wrapper, which stops the loop via the Service
|
||||
Control Manager instead of OS signals) should pass its own stop object —
|
||||
anything with a `.stop` boolean attribute — and flip it from any thread."""
|
||||
executor = load_executor()
|
||||
state = AgentState(config.state_db)
|
||||
client = HubClient(config.hub_url, config.agent_token)
|
||||
stopper = stopper if stopper is not None else GracefulStop()
|
||||
|
||||
backoff = 1
|
||||
was_down = False
|
||||
iterations = 0
|
||||
last_job_id = None
|
||||
while not stopper.stop:
|
||||
if max_iterations is not None and iterations >= max_iterations:
|
||||
break
|
||||
iterations += 1
|
||||
|
||||
try:
|
||||
client.heartbeat("IDLE", schema_fingerprint=config.schema_fingerprint, agent_version=config.agent_version, last_job_id=last_job_id)
|
||||
if was_down:
|
||||
logger.info("reconnected to Hub after %ss backoff", backoff)
|
||||
was_down = False
|
||||
backoff = 1
|
||||
except Exception as error:
|
||||
logger.warning("heartbeat failed, Hub unreachable: %s (retrying in %ss)", error, backoff)
|
||||
was_down = True
|
||||
time.sleep(backoff)
|
||||
backoff = min(backoff * 2, _MAX_BACKOFF_SECONDS)
|
||||
continue
|
||||
|
||||
try:
|
||||
job = client.next_job()
|
||||
except Exception as error:
|
||||
logger.warning("poll failed, Hub unreachable: %s (retrying in %ss)", error, backoff)
|
||||
was_down = True
|
||||
time.sleep(backoff)
|
||||
backoff = min(backoff * 2, _MAX_BACKOFF_SECONDS)
|
||||
continue
|
||||
|
||||
if job is None:
|
||||
time.sleep(config.poll_interval_seconds)
|
||||
continue
|
||||
|
||||
last_job_id = job.get("job_id")
|
||||
logger.info("job %s received: kind=%s", job.get("job_id"), job["kind"])
|
||||
try:
|
||||
client.heartbeat("BUSY", schema_fingerprint=config.schema_fingerprint, agent_version=config.agent_version, last_job_id=last_job_id)
|
||||
except Exception:
|
||||
pass # best-effort; the job itself still proceeds
|
||||
|
||||
result_summary, error = execute_job(job, config, executor, state, env)
|
||||
try:
|
||||
if error is None:
|
||||
client.complete_job(job.get("job_id"), result_summary)
|
||||
logger.info("job %s reported COMPLETED", job.get("job_id"))
|
||||
else:
|
||||
client.fail_job(job.get("job_id"), error)
|
||||
logger.warning("job %s reported FAILED: %s", job.get("job_id"), error)
|
||||
except Exception as report_error:
|
||||
logger.warning("could not report job %s result to Hub: %s", job.get("job_id"), report_error)
|
||||
|
||||
|
||||
def enforce_startup_preflight(env):
|
||||
"""Full read-only preflight at startup (Hub, auth, Firebird, capabilities, company context, guards). Fail closed."""
|
||||
from gti_fiscal_agent.preflight import run_preflight
|
||||
report = run_preflight(env, agent_version=env.get("AGENT_VERSION") or "1.0.0-rc1")
|
||||
logger.info("startup preflight: %s", json.dumps({k: report[k] for k in ("PREFLIGHT", "blockers", "fiscal_mode", "effective_firebird_writes")}, ensure_ascii=False))
|
||||
print(json.dumps({"PREFLIGHT": report["PREFLIGHT"], "blockers": report["blockers"]}, ensure_ascii=False))
|
||||
if report["PREFLIGHT"] != "PASS":
|
||||
logger.error("startup aborted: preflight failed (fail closed): %s", report["blockers"])
|
||||
raise SystemExit(2)
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--max-iterations", type=int, default=None, help="stop after N poll cycles (used by tests); omit to run forever")
|
||||
args = parser.parse_args()
|
||||
|
||||
load_dotenv(Path(os.environ.get("GTI_AGENT_ENV_FILE") or PROJECT_ROOT / ".env"))
|
||||
config = Config(os.environ)
|
||||
setup_logging(config.log_path)
|
||||
|
||||
logger.info("startup: agent_id=%s company_id=%s agent_version=%s firebird_writes=%s", config.agent_id, config.company_id, config.agent_version, config.firebird_writes)
|
||||
hub_ok = HubClient(config.hub_url, config.agent_token).health()
|
||||
firebird_ok = check_connection(os.environ)
|
||||
logger.info("startup checks: hub_ok=%s firebird_ok=%s", hub_ok, firebird_ok)
|
||||
enforce_startup_preflight(os.environ)
|
||||
|
||||
run(config, os.environ, max_iterations=args.max_iterations)
|
||||
logger.info("shutdown: stopped cleanly")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Windows Service wrapper around run_agent_service.run() — reuses it as-is,
|
||||
does not reimplement any polling/pipeline logic.
|
||||
|
||||
Requires pywin32 (`pip install pywin32`), Windows-only.
|
||||
|
||||
Commands (run from an elevated/administrator terminal):
|
||||
python gti_agent_windows_service.py install # register the service, Startup=Automatic
|
||||
python gti_agent_windows_service.py start
|
||||
python gti_agent_windows_service.py stop
|
||||
python gti_agent_windows_service.py restart
|
||||
python gti_agent_windows_service.py remove # uninstall
|
||||
python gti_agent_windows_service.py --startup=auto --username . --password . install
|
||||
|
||||
Recovery (restart on failure) is not set by pywin32's `install` alone; run once
|
||||
after installing:
|
||||
sc.exe failure "GTIFiscalAgent" reset= 86400 actions= restart/60000/restart/60000/restart/60000
|
||||
|
||||
This module only wraps agent/scripts/run_agent_service.py's `run()` — the
|
||||
actual polling/heartbeat/pipeline logic lives there, unchanged.
|
||||
"""
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(AGENT_ROOT / "scripts"))
|
||||
sys.path.insert(0, str(AGENT_ROOT / "src"))
|
||||
|
||||
import run_agent_service as agent_service # noqa: E402
|
||||
|
||||
try:
|
||||
import servicemanager
|
||||
import win32event
|
||||
import win32service
|
||||
import win32serviceutil
|
||||
except ImportError: # pragma: no cover — only available on Windows with pywin32 installed
|
||||
servicemanager = win32event = win32service = win32serviceutil = None
|
||||
|
||||
|
||||
class _ServiceStopFlag:
|
||||
"""Plain stop flag (no signal.signal registration) — safe to flip from any
|
||||
thread. The Windows Service Control Manager stops the service, not a
|
||||
console SIGINT/SIGTERM, so this replaces GracefulStop for this host."""
|
||||
def __init__(self):
|
||||
self.stop = False
|
||||
|
||||
|
||||
if win32serviceutil is not None:
|
||||
|
||||
class GTIFiscalAgentService(win32serviceutil.ServiceFramework):
|
||||
_svc_name_ = "GTIFiscalAgent"
|
||||
_svc_display_name_ = "GTI Fiscal Agent"
|
||||
_svc_description_ = "Reads the local Firebird ERP, polls the GTI Fiscal Hub for jobs, and applies guarded fiscal updates."
|
||||
|
||||
def __init__(self, args):
|
||||
win32serviceutil.ServiceFramework.__init__(self, args)
|
||||
self.stop_event = win32event.CreateEvent(None, 0, 0, None)
|
||||
self.stop_flag = _ServiceStopFlag()
|
||||
self._thread = None
|
||||
|
||||
def SvcStop(self):
|
||||
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
|
||||
self.stop_flag.stop = True
|
||||
win32event.SetEvent(self.stop_event)
|
||||
|
||||
def SvcDoRun(self):
|
||||
servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE, servicemanager.PYS_SERVICE_STARTED, (self._svc_name_, ""))
|
||||
self._thread = threading.Thread(target=self._run_agent, daemon=True)
|
||||
self._thread.start()
|
||||
win32event.WaitForSingleObject(self.stop_event, win32event.INFINITE)
|
||||
|
||||
def _run_agent(self):
|
||||
import os
|
||||
agent_service.load_dotenv(Path(os.environ.get("GTI_AGENT_ENV_FILE") or agent_service.PROJECT_ROOT / ".env"))
|
||||
config = agent_service.Config(os.environ)
|
||||
agent_service.setup_logging(config.log_path)
|
||||
try:
|
||||
agent_service.enforce_startup_preflight(os.environ) # fail closed: a failing preflight stops the service
|
||||
except SystemExit:
|
||||
self.stop_flag.stop = True
|
||||
servicemanager.LogErrorMsg("GTIFiscalAgent: startup preflight failed; see the agent log")
|
||||
win32event.SetEvent(self.stop_event)
|
||||
return
|
||||
agent_service.run(config, os.environ, stopper=self.stop_flag)
|
||||
|
||||
|
||||
def main():
|
||||
if win32serviceutil is None:
|
||||
raise SystemExit("pywin32 is required: pip install pywin32")
|
||||
win32serviceutil.HandleCommandLine(GTIFiscalAgentService)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
"""Read-only ERP to Fiscal Hub agent."""
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Pure capability detection / schema classification for SGV (C000025/C000004) databases. No I/O, no writes."""
|
||||
|
||||
WRITE_TARGETS = ["CLASSIFICACAO_FISCAL", "CFOP_ENTRADA", "CST_ENTRADA", "CFOP", "CST", "ALIQUOTA", "CEST", "SITUACAO_TRIBUTARIA", "COD_ENTRADA_COFINS",
|
||||
"COD_SAIDA_COFINS", "ALIQ_COFINS", "COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "CSOSN", "IPI", "CST_IPI", "IAT", "INCIDENCIA_PISCOFINS",
|
||||
"PISCOFINS", "IPPT"]
|
||||
ORIGIN_FIELD = "COD_ORIGEM_MERCADORIA"
|
||||
|
||||
# Firebird RDB$FIELD_TYPE -> family
|
||||
FAMILY = {7: "int", 8: "int", 16: "int", 10: "float", 11: "float", 27: "float", 12: "date", 13: "time", 35: "timestamp", 14: "text", 37: "text", 261: "blob"}
|
||||
# accepted families per field (UI edit controls take text or numbers; numeric columns may be int/float)
|
||||
ACCEPT = {"ALIQUOTA": {"float", "int"}, "IPI": {"float", "int"}, "COD_ORIGEM_MERCADORIA": {"int"}}
|
||||
DEFAULT_ACCEPT = {"text", "int"} # text columns; some ERPs keep codes such as CST_ENTRADA in SMALLINT
|
||||
|
||||
CAPABILITIES = {
|
||||
"supports_ncm": ["CLASSIFICACAO_FISCAL"],
|
||||
"supports_cfop_entry": ["CFOP_ENTRADA"],
|
||||
"supports_cfop_exit": ["CFOP"],
|
||||
"supports_cst_entry": ["CST_ENTRADA"],
|
||||
"supports_cst_exit": ["CST"],
|
||||
"supports_icms_rate": ["ALIQUOTA"],
|
||||
"supports_cest": ["CEST"],
|
||||
"supports_csosn": ["CSOSN"],
|
||||
"supports_ipi": ["IPI", "CST_IPI"],
|
||||
"supports_pis": ["COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS"],
|
||||
"supports_cofins": ["COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS"],
|
||||
"supports_origin": [ORIGIN_FIELD],
|
||||
"supports_iat": ["IAT"],
|
||||
"supports_incidence": ["INCIDENCIA_PISCOFINS"],
|
||||
"supports_ippt": ["IPPT"],
|
||||
}
|
||||
|
||||
|
||||
def type_ok(field, type_code):
|
||||
return FAMILY.get(type_code, "other") in ACCEPT.get(field, DEFAULT_ACCEPT)
|
||||
|
||||
|
||||
def detect_capabilities(columns):
|
||||
"""columns: {FIELD: rdb_type_code} of C000025. capability=true only if every required column exists with a usable type."""
|
||||
out = {}
|
||||
for cap, fields in CAPABILITIES.items():
|
||||
out[cap] = all(f in columns and type_ok(f, columns[f]) for f in fields)
|
||||
return out
|
||||
|
||||
|
||||
def field_compatibility(columns, reference_types=None):
|
||||
"""Per write target (+ origin): PRESENT_OK / PRESENT_TYPE_VARIATION / MISSING."""
|
||||
res = {}
|
||||
for f in WRITE_TARGETS + [ORIGIN_FIELD]:
|
||||
if f not in columns:
|
||||
res[f] = "MISSING"
|
||||
elif not type_ok(f, columns[f]):
|
||||
res[f] = "PRESENT_TYPE_INCOMPATIBLE"
|
||||
elif reference_types and reference_types.get(f) is not None and reference_types[f] != columns[f]:
|
||||
res[f] = "PRESENT_TYPE_VARIATION"
|
||||
else:
|
||||
res[f] = "PRESENT_OK"
|
||||
return res
|
||||
|
||||
|
||||
def classify_schema(has_c000004, has_c000025, fingerprint, reference_fingerprint, compat):
|
||||
"""REFERENCE_SCHEMA_MATCH | COMPATIBLE_WITH_VARIATIONS | INCOMPATIBLE. A different hash is not a failure
|
||||
when every required capability exists (origin is optional: it is only PROBABLE in the map)."""
|
||||
if not (has_c000004 and has_c000025):
|
||||
return "INCOMPATIBLE"
|
||||
required = {f: v for f, v in compat.items() if f in WRITE_TARGETS}
|
||||
if any(v in ("MISSING", "PRESENT_TYPE_INCOMPATIBLE") for v in required.values()):
|
||||
return "INCOMPATIBLE"
|
||||
if fingerprint == reference_fingerprint:
|
||||
return "REFERENCE_SCHEMA_MATCH"
|
||||
return "COMPATIBLE_WITH_VARIATIONS"
|
||||
|
||||
|
||||
NAME_KEYWORDS = {"RESTAURANT": ("RESTAURANTE", "PIZZARIA", "LANCHONETE", "CHURRASC"), "RETAIL_MARKET": ("MERCADO", "SUPERMERC", "MERCEARIA", "ATACAD"),
|
||||
"APPAREL": ("MODAS", "CONFEC", "BOUTIQUE", "ROUPAS"), "CONSTRUCTION": ("CONSTRU", "MATERIAIS"), "AUTO_REPAIR": ("OFICINA", "MECANICA", "AUTO PECAS", "AUTOPECAS")}
|
||||
|
||||
|
||||
def infer_segment(signals, company_name=""):
|
||||
"""Evidence-based segment guess from populated tables/flags (never from the file name).
|
||||
signals: {"restaurant_tables_rows": int, "kitchen_products": int, "recipes": int, "service_orders": int, "vehicle_products": int, "grade_items": int, "nfce_items": int}"""
|
||||
ev = []
|
||||
score = {"RESTAURANT": 0, "AUTO_REPAIR": 0, "APPAREL": 0, "RETAIL_MARKET": 0, "CONSTRUCTION": 0}
|
||||
import unicodedata
|
||||
up = unicodedata.normalize("NFKD", company_name or "").encode("ascii", "ignore").decode().upper()
|
||||
for seg_name, kws in NAME_KEYWORDS.items():
|
||||
hit = [k for k in kws if k in up]
|
||||
if hit:
|
||||
score[seg_name] += 1; ev.append(f"company name in C000004 contains {hit[0]} (weak, name is data, not the file name)")
|
||||
if signals.get("restaurant_tables_rows", 0) > 50: score["RESTAURANT"] += 2; ev.append(f"mesas/comandas rows={signals['restaurant_tables_rows']}")
|
||||
if signals.get("kitchen_products", 0) > 0: score["RESTAURANT"] += 1; ev.append(f"PROD_COZINHA products={signals['kitchen_products']}")
|
||||
if signals.get("recipes", 0) > 0: score["RESTAURANT"] += 1; ev.append(f"recipes={signals['recipes']}")
|
||||
if signals.get("service_orders", 0) > 20: score["AUTO_REPAIR"] += 2; ev.append(f"ordens de serviço={signals['service_orders']}")
|
||||
if signals.get("vehicle_products", 0) > 0: score["AUTO_REPAIR"] += 1; ev.append(f"veículos={signals['vehicle_products']}")
|
||||
if signals.get("grade_items", 0) > 50: score["APPAREL"] += 2; ev.append(f"grade (tamanho/cor) rows={signals['grade_items']}")
|
||||
if signals.get("nfce_items", 0) > 1000 and max(score.values()) == 0: score["RETAIL_MARKET"] += 1; ev.append("high-volume NFC-e retail without restaurant/OS/grade signals")
|
||||
best = max(score, key=score.get)
|
||||
return (best if score[best] > 0 else "UNKNOWN"), score, ev
|
||||
@@ -0,0 +1,37 @@
|
||||
import httpx
|
||||
|
||||
class HubClient:
|
||||
def __init__(self, url, token, chunk_size=100): self.url, self.token, self.chunk_size = url.rstrip("/"), token, chunk_size
|
||||
def _headers(self): return {"Authorization": f"Bearer {self.token}"}
|
||||
def classify_batch(self, products):
|
||||
results = []
|
||||
for start in range(0, len(products), self.chunk_size):
|
||||
chunk = products[start:start + self.chunk_size]
|
||||
response = httpx.post(f"{self.url}/v1/classify/batch", json={"products": chunk}, headers=self._headers(), timeout=60)
|
||||
response.raise_for_status(); results.extend(response.json()["results"])
|
||||
return results
|
||||
def get_json(self, path):
|
||||
response = httpx.get(f"{self.url}{path}", headers=self._headers(), timeout=30)
|
||||
response.raise_for_status(); return response.json()
|
||||
def health(self):
|
||||
response = httpx.get(f"{self.url}/health", timeout=10)
|
||||
return response.status_code == 200
|
||||
def heartbeat(self, status, schema_fingerprint=None, agent_version=None, last_job_id=None):
|
||||
payload = {"status": status, "schema_fingerprint": schema_fingerprint, "agent_version": agent_version, "last_job_id": last_job_id}
|
||||
response = httpx.post(f"{self.url}/v1/agents/heartbeat", json=payload, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def next_job(self):
|
||||
response = httpx.get(f"{self.url}/v1/jobs/next", headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def create_job(self, kind, payload=None):
|
||||
response = httpx.post(f"{self.url}/v1/jobs", json={"kind": kind, "payload": payload or {}}, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def get_job(self, job_id):
|
||||
response = httpx.get(f"{self.url}/v1/jobs/{job_id}", headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def complete_job(self, job_id, result_summary):
|
||||
response = httpx.post(f"{self.url}/v1/jobs/{job_id}/complete", json={"result_summary": result_summary}, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
def fail_job(self, job_id, error):
|
||||
response = httpx.post(f"{self.url}/v1/jobs/{job_id}/fail", json={"error": str(error)}, headers=self._headers(), timeout=15)
|
||||
response.raise_for_status(); return response.json()
|
||||
@@ -0,0 +1,218 @@
|
||||
"""Deterministic, auditable company-context resolution for an Agent installation (READ-ONLY).
|
||||
|
||||
Evidence (proved on 4 SGV databases, see docs/COMPANY_CONTEXT_RESOLUTION_V1.json):
|
||||
* EMISSOR.EMITENTE_CNPJ/UF/CRT is the emitter block read by the emission programs (GtiVendas_NFCe.exe and Envio NF-e.exe both bind
|
||||
QrEmissor.EMITENTE_*; neither reads C000004.CRT/OPTANTE_SIMPLES). 100% of emitted NFC-e access keys carry the CNPJ of the C000004 row
|
||||
whose CNPJ equals EMISSOR.EMITENTE_CNPJ, and cUF=29.
|
||||
* C000004.CRT / OPTANTE_SIMPLES belong to the ERP filial registry (SPED, filial screen) and can disagree with EMISSOR.
|
||||
Never guesses: first row / lowest code / latest row are NOT used. Ambiguity -> FAIL with the reason, unless the operator configures
|
||||
COMPANY_CODE or COMPANY_CNPJ (company) and COMPANY_CRT_SOURCE = EMISSOR | C000004 (explicit choice when the two CRT sources conflict).
|
||||
"""
|
||||
import hashlib
|
||||
import os
|
||||
|
||||
IBGE_UF = {"11": "RO", "12": "AC", "13": "AM", "14": "RR", "15": "PA", "16": "AP", "17": "TO", "21": "MA", "22": "PI", "23": "CE", "24": "RN", "25": "PB", "26": "PE",
|
||||
"27": "AL", "28": "SE", "29": "BA", "31": "MG", "32": "ES", "33": "RJ", "35": "SP", "41": "PR", "42": "SC", "43": "RS", "50": "MS", "51": "MT", "52": "GO",
|
||||
"53": "DF"}
|
||||
VALID_CRT = {"1", "2", "3"}
|
||||
|
||||
|
||||
class CompanyContextError(Exception):
|
||||
def __init__(self, result):
|
||||
super().__init__(f"COMPANY_CONTEXT=FAIL ({result.get('classification')}): {'; '.join(result.get('reasons', []))}")
|
||||
self.result = result
|
||||
|
||||
|
||||
def digits(value):
|
||||
return "".join(c for c in str(value or "") if c.isdigit())
|
||||
|
||||
|
||||
def mask_cnpj(value):
|
||||
d = digits(value)
|
||||
return f"{d[:2]}**********{d[-2:]}" if len(d) >= 4 else None
|
||||
|
||||
|
||||
def cnpj_fingerprint(value):
|
||||
d = digits(value)
|
||||
return hashlib.sha256(d.encode()).hexdigest()[:8] if d else None
|
||||
|
||||
|
||||
def _s(v):
|
||||
return None if v is None else str(v).strip()
|
||||
|
||||
|
||||
def config_from_env(env):
|
||||
crt_source = (env.get("COMPANY_CRT_SOURCE") or "").strip().upper() or None
|
||||
if crt_source == "AUTO": # AUTO = resolve automatically, only when there is no conflict
|
||||
crt_source = None
|
||||
invalid = False
|
||||
else:
|
||||
invalid = bool(crt_source and crt_source not in ("EMISSOR", "C000004"))
|
||||
uf = (env.get("COMPANY_UF") or "").strip().upper() or None
|
||||
return {"company_code": (env.get("COMPANY_CODE") or "").strip() or None, "company_cnpj": digits(env.get("COMPANY_CNPJ")) or None,
|
||||
"crt_source": crt_source if crt_source in ("EMISSOR", "C000004") else None, "crt_source_invalid": invalid,
|
||||
"uf_override": uf}
|
||||
|
||||
|
||||
def resolve(evidence, config=None):
|
||||
"""evidence: {"c000004": [{code, cnpj, ie, uf, crt, optante, name}], "emissor": [{cnpj, uf, crt}],
|
||||
"nfce_keys": [{"cuf", "cnpj", "n"}], "nf_by_filial": {code: n}}. Pure function."""
|
||||
cfg = config or {}
|
||||
out = {"status": "FAIL", "classification": "UNRESOLVED", "company_code": None, "company_cnpj": None, "company_cnpj_fingerprint": None, "company_uf": None, "company_crt": None,
|
||||
"source_of_truth": {}, "confidence": "NONE", "evidence": [], "warnings": [], "reasons": [], "configured": False}
|
||||
|
||||
def fail(cls, reason):
|
||||
out["classification"] = cls; out["reasons"].append(reason); return out
|
||||
|
||||
if cfg.get("crt_source_invalid"):
|
||||
return fail("CONFIG_REQUIRED", "COMPANY_CRT_SOURCE must be EMISSOR or C000004")
|
||||
companies = [{**c, "cnpj": digits(c.get("cnpj"))} for c in evidence.get("c000004", [])]
|
||||
emissors = [{**e, "cnpj": digits(e.get("cnpj"))} for e in evidence.get("emissor", [])]
|
||||
docs = evidence.get("nfce_keys", [])
|
||||
if not companies:
|
||||
return fail("UNRESOLVED", "C000004 has no company rows")
|
||||
|
||||
# ---- company selection -------------------------------------------------------------------
|
||||
selected = None; basis = None
|
||||
by_code, by_cnpj = cfg.get("company_code"), cfg.get("company_cnpj")
|
||||
if by_code or by_cnpj:
|
||||
out["configured"] = True
|
||||
cand = [c for c in companies if (not by_code or c["code"] == by_code) and (not by_cnpj or c["cnpj"] == by_cnpj)]
|
||||
if len(cand) != 1:
|
||||
return fail("CONFIG_REQUIRED", "configured company (COMPANY_CODE/COMPANY_CNPJ) does not identify exactly one C000004 row")
|
||||
selected, basis = cand[0], "CONFIGURED"
|
||||
elif len(companies) == 1:
|
||||
selected, basis = companies[0], "SINGLE_C000004_ROW"
|
||||
elif len(emissors) == 1 and emissors[0]["cnpj"]:
|
||||
match = [c for c in companies if c["cnpj"] == emissors[0]["cnpj"]]
|
||||
if len(match) != 1:
|
||||
return fail("CONFIG_REQUIRED", f"{len(companies)} C000004 rows and the EMISSOR CNPJ matches {len(match)} of them: configure COMPANY_CODE or COMPANY_CNPJ")
|
||||
selected, basis = match[0], "EMISSOR_CNPJ_MATCH"
|
||||
else:
|
||||
return fail("CONFIG_REQUIRED", f"{len(companies)} C000004 rows and no unique EMISSOR row to disambiguate: configure COMPANY_CODE or COMPANY_CNPJ")
|
||||
out["evidence"].append(f"company selected by {basis}")
|
||||
out.update(company_code=selected["code"], company_cnpj=mask_cnpj(selected["cnpj"]), company_cnpj_fingerprint=cnpj_fingerprint(selected["cnpj"]))
|
||||
out["source_of_truth"]["company"] = basis
|
||||
|
||||
# ---- emitter (EMISSOR) must describe the same company --------------------------------------
|
||||
emissor = None
|
||||
if emissors:
|
||||
same = [e for e in emissors if e["cnpj"] and e["cnpj"] == selected["cnpj"]]
|
||||
if same:
|
||||
emissor = same[0]; out["evidence"].append("EMISSOR.EMITENTE_CNPJ equals the selected company CNPJ")
|
||||
else:
|
||||
return fail("CONFLICT", "EMISSOR emitter CNPJ differs from the selected company (unexplained): refusing to guess")
|
||||
else:
|
||||
out["warnings"].append("NO_EMISSOR_ROW")
|
||||
|
||||
# ---- emitted documents must belong to the same CNPJ ------------------------------------------
|
||||
doc_ufs = set()
|
||||
if docs:
|
||||
total = sum(d["n"] for d in docs); own = sum(d["n"] for d in docs if digits(d["cnpj"]) == selected["cnpj"])
|
||||
out["evidence"].append(f"emitted NFC-e keys: {own}/{total} carry the selected CNPJ")
|
||||
if own == 0:
|
||||
return fail("CONFLICT", f"none of the {total} emitted NFC-e keys carries the selected company CNPJ")
|
||||
if own != total: # e.g. a database copy/migration that kept another emitter's history: evidence, not a reason to guess
|
||||
out["warnings"].append(f"HISTORICAL_NFCE_KEYS_FROM_OTHER_CNPJ {total - own}/{total}")
|
||||
doc_ufs = {IBGE_UF.get(d["cuf"]) for d in docs if digits(d["cnpj"]) == selected["cnpj"]} - {None}
|
||||
|
||||
# ---- UF ------------------------------------------------------------------------------------------
|
||||
uf_sources = {}
|
||||
if emissor and _s(emissor.get("uf")): uf_sources["EMISSOR"] = _s(emissor["uf"]).upper()
|
||||
if _s(selected.get("uf")): uf_sources["C000004"] = _s(selected["uf"]).upper()
|
||||
if len(doc_ufs) == 1: uf_sources["DOCUMENT_KEYS"] = next(iter(doc_ufs))
|
||||
if len(doc_ufs) > 1: return fail("CONFLICT", "emitted NFC-e keys carry several UFs")
|
||||
values = set(uf_sources.values())
|
||||
override = cfg.get("uf_override")
|
||||
if len(values) > 1:
|
||||
return fail("CONFLICT", f"UF_SOURCE_CONFLICT {uf_sources}")
|
||||
if override and values and override not in values:
|
||||
return fail("CONFLICT", f"COMPANY_UF={override} differs from the UF proved by the database {sorted(values)}: an override never replaces the resolver silently")
|
||||
if not values and override: # explicit override only when the database proves nothing
|
||||
uf_sources["CONFIG_OVERRIDE"] = override; values = {override}; out["warnings"].append("UF_FROM_EXPLICIT_OVERRIDE")
|
||||
if not values:
|
||||
return fail("UNRESOLVED", "UF_SOURCE_UNKNOWN")
|
||||
out["company_uf"] = next(iter(values)); out["source_of_truth"]["uf"] = "+".join(uf_sources)
|
||||
out["uf_status"] = "UF_SOURCE_CONFIRMED"
|
||||
|
||||
# ---- CRT ------------------------------------------------------------------------------------------
|
||||
e_crt = _s(emissor.get("crt")) if emissor else None
|
||||
c_crt = _s(selected.get("crt"))
|
||||
e_ok, c_ok = e_crt in VALID_CRT, c_crt in VALID_CRT
|
||||
if e_ok and c_ok and e_crt == c_crt:
|
||||
crt, src, status = e_crt, "EMISSOR+C000004", "CRT_SOURCE_CONFIRMED"
|
||||
elif e_ok and c_ok:
|
||||
choice = cfg.get("crt_source")
|
||||
if not choice:
|
||||
out["crt_candidates"] = {"EMISSOR": e_crt, "C000004": c_crt}
|
||||
return fail("CONFLICT", f"CRT_SOURCE_CONFLICT EMISSOR={e_crt} C000004={c_crt}: configure COMPANY_CRT_SOURCE (emission programs read EMISSOR)")
|
||||
crt, src, status = (e_crt if choice == "EMISSOR" else c_crt), f"{choice} (configured; conflict acknowledged)", "CRT_SOURCE_CONFLICT_RESOLVED_BY_CONFIG"
|
||||
out["warnings"].append(f"CRT_CONFLICT EMISSOR={e_crt} C000004={c_crt}")
|
||||
elif e_ok or c_ok:
|
||||
crt, src, status = (e_crt, "EMISSOR", "CRT_SOURCE_SINGLE") if e_ok else (c_crt, "C000004", "CRT_SOURCE_SINGLE")
|
||||
out["warnings"].append("CRT_HAS_A_SINGLE_SOURCE")
|
||||
else:
|
||||
return fail("UNRESOLVED", "CRT_SOURCE_UNKNOWN (no valid CRT in EMISSOR or C000004)")
|
||||
out.update(company_crt=crt, crt_status=status); out["source_of_truth"]["crt"] = src
|
||||
opt = _s(selected.get("optante"))
|
||||
if opt and ((crt in ("1", "2") and opt.upper() != "S") or (crt == "3" and opt.upper() == "S")):
|
||||
out["warnings"].append(f"OPTANTE_SIMPLES={opt} inconsistent with CRT={crt}")
|
||||
|
||||
agree = len(uf_sources) + (2 if status == "CRT_SOURCE_CONFIRMED" else 1)
|
||||
out["confidence"] = "HIGH" if (status == "CRT_SOURCE_CONFIRMED" and len(uf_sources) >= 2) else "MEDIUM"
|
||||
out["status"] = "PASS"
|
||||
out["classification"] = "CONFIGURED" if out["configured"] or status == "CRT_SOURCE_CONFLICT_RESOLVED_BY_CONFIG" else "AUTO_RESOLVED"
|
||||
out["reasons"] = []
|
||||
out["evidence_source_count"] = agree
|
||||
return out
|
||||
|
||||
|
||||
def collect_evidence(cursor):
|
||||
"""Read-only SELECTs. Missing optional tables/columns simply yield less evidence."""
|
||||
cursor.execute("SELECT TRIM(RDB$RELATION_NAME), TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME IN ('C000004','EMISSOR','NFCE','C000061')")
|
||||
cols = {}
|
||||
for t, f in cursor.fetchall():
|
||||
cols.setdefault(t, set()).add(f)
|
||||
ev = {"c000004": [], "emissor": [], "nfce_keys": [], "nf_by_filial": {}}
|
||||
c4 = cols.get("C000004", set())
|
||||
want = [("CODIGO", "code"), ("CNPJ", "cnpj"), ("IE", "ie"), ("UF", "uf"), ("CRT", "crt"), ("OPTANTE_SIMPLES", "optante"), ("FILIAL", "name")]
|
||||
use = [(c, k) for c, k in want if c in c4]
|
||||
if use:
|
||||
cursor.execute(f"SELECT {','.join(c for c, _ in use)} FROM C000004")
|
||||
for row in cursor.fetchall():
|
||||
ev["c000004"].append({k: _s(v) for (_, k), v in zip(use, row)})
|
||||
em = cols.get("EMISSOR", set())
|
||||
want = [("EMITENTE_CNPJ", "cnpj"), ("EMITENTE_UF", "uf"), ("EMITENTE_CRT", "crt")]
|
||||
use = [(c, k) for c, k in want if c in em]
|
||||
if use:
|
||||
cursor.execute(f"SELECT {','.join(c for c, _ in use)} FROM EMISSOR")
|
||||
for row in cursor.fetchall():
|
||||
ev["emissor"].append({k: _s(v) for (_, k), v in zip(use, row)})
|
||||
if "CHAVE" in cols.get("NFCE", set()):
|
||||
cursor.execute("SELECT SUBSTRING(CHAVE FROM 1 FOR 2), SUBSTRING(CHAVE FROM 7 FOR 14), COUNT(*) FROM NFCE WHERE CHAVE IS NOT NULL AND CHAR_LENGTH(TRIM(CHAVE)) = 44 "
|
||||
"GROUP BY SUBSTRING(CHAVE FROM 1 FOR 2), SUBSTRING(CHAVE FROM 7 FOR 14)")
|
||||
ev["nfce_keys"] = [{"cuf": _s(a), "cnpj": _s(b), "n": n} for a, b, n in cursor.fetchall()]
|
||||
if "CODFILIAL" in cols.get("C000061", set()):
|
||||
cursor.execute("SELECT CODFILIAL, COUNT(*) FROM C000061 GROUP BY CODFILIAL")
|
||||
ev["nf_by_filial"] = {_s(a): n for a, n in cursor.fetchall()}
|
||||
return ev
|
||||
|
||||
|
||||
def resolve_from_cursor(cursor, env=None):
|
||||
return resolve(collect_evidence(cursor), config_from_env(env or os.environ))
|
||||
|
||||
|
||||
def preflight_company_context(env=None):
|
||||
"""Opens the configured Firebird database read-only and resolves the company. Never prints the full CNPJ."""
|
||||
from .firebird_reader import _connect
|
||||
env = env or os.environ
|
||||
conn = _connect(env)
|
||||
try:
|
||||
return resolve_from_cursor(conn.cursor(), env)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def preflight_report(result):
|
||||
keys = ("company_code", "company_cnpj", "company_uf", "company_crt", "source_of_truth", "confidence", "classification", "warnings", "reasons")
|
||||
return {"COMPANY_CONTEXT": "PASS" if result["status"] == "PASS" else "FAIL", **{k: result.get(k) for k in keys}}
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Read-only fiscal coverage report for a set of products (used by the Agent's REPROCESS_ALL result and by the multi-database tooling).
|
||||
|
||||
Pure computation over Hub decisions + product rows + local evidence. It reuses the shipped pure engines in central.app
|
||||
(fiscal_decision, product_defaults, icms_context) — no fiscal rule lives here, only guards for the company context:
|
||||
the BA Anexo 1 ST/MVA dataset is applied to UF=BA only and CSOSN logic only under CRT=1.
|
||||
"""
|
||||
from collections import Counter
|
||||
|
||||
PRODUCT_COLUMNS = ("CODIGO", "PRODUTO", "CLASSIFICACAO_FISCAL", "CFOP", "CSOSN", "CST", "CEST", "IPI", "EX_TIPI", "TIPO", "PROD_COZINHA", "PRODUTO_SERVICO", "SITUACAO_TRIBUTARIA")
|
||||
|
||||
|
||||
def _s(v):
|
||||
return None if v is None else str(v).strip()
|
||||
|
||||
|
||||
def compute_coverage(products, decisions, company, company_default, refs, evidence):
|
||||
"""products/decisions aligned lists. company: {"uf","crt"} (resolved). evidence: {"recipes","composed","purchases","stock_entry"}."""
|
||||
from central.app.fiscal_decision import decide_product
|
||||
from central.app.icms_context import csosn_v2, product_role, st_status, withheld_status
|
||||
from central.app.product_defaults import cfop_default
|
||||
uf_ba = company.get("uf") == "BA"
|
||||
crt1 = company.get("crt") == "1"
|
||||
recipes, composed = evidence.get("recipes", set()), evidence.get("composed", set())
|
||||
purchases, stock_entry = evidence.get("purchases", {}), evidence.get("stock_entry", {})
|
||||
m, cfop_values, situ = Counter(), Counter(), Counter()
|
||||
for p, d in zip(products, decisions):
|
||||
p = {c: p.get(c) for c in PRODUCT_COLUMNS}
|
||||
code = _s(p["CODIGO"]); p["CODIGO"] = code
|
||||
cls = d.get("classification") or {}
|
||||
rec = decide_product(p, d, {"uf": company.get("uf") or "", "crt": company.get("crt") or ""}, refs)
|
||||
ncm_status = d.get("status"); raw_ncm = _s(p["CLASSIFICACAO_FISCAL"])
|
||||
m["NCM_MISSING" if not raw_ncm else ("NCM_VALID" if ncm_status == "EXACT_NCM" else ("NCM_LEGACY" if ncm_status == "LEGACY_NCM" else "NCM_INVALID"))] += 1
|
||||
ipi = rec["IPI"]; rk = (cls.get("ipi") or {}).get("rate_kind"); ist = d.get("ipi_status")
|
||||
if ist == "IPI_RESOLVED":
|
||||
m[{"NUMERIC": "IPI_NUMERIC", "NON_NUMERIC": "IPI_NON_NUMERIC", "BLANK": "IPI_SOURCE_BLANK"}.get(rk, "IPI_NO_MATCH")] += 1
|
||||
elif ist == "IPI_CANDIDATES":
|
||||
m["IPI_AMBIGUOUS"] += 1
|
||||
elif ist != "LEGACY_NCM" and raw_ncm:
|
||||
m["IPI_NO_MATCH"] += 1
|
||||
if ipi["action_class"] == "ALREADY_CORRECT":
|
||||
m["IPI_ALREADY_CORRECT"] += 1
|
||||
if ipi["write_eligible"]:
|
||||
m["IPI_CHANGE_REQUIRED"] += 1; m["IPI_WRITE_ELIGIBLE"] += 1
|
||||
cs_ = d.get("cest_status")
|
||||
if cs_ == "CEST_CONFIRMED":
|
||||
m["CEST_CONFIRMED"] += 1
|
||||
elif cs_ == "CEST_CANDIDATES":
|
||||
m["CEST_CANDIDATES"] += 1
|
||||
cf = cfop_default(p, ncm_status, d.get("ba_st_mva_status") if uf_ba else None, refs["cfop_valid"])
|
||||
cfop_values[_s(p["CFOP"])] += 1
|
||||
m["CFOP_RESOLVED" if cf["status"] == "CFOP_DEFAULT_RESOLVED" else "CFOP_CONTEXT_REQUIRED"] += 1
|
||||
ba = cls.get("ba_st_mva") or {}
|
||||
if uf_ba:
|
||||
st, _ = st_status(ncm_status, d.get("ba_st_mva_status"), cs_, p["CEST"], ba.get("cest"), None)
|
||||
else:
|
||||
st = "ST_UNKNOWN"
|
||||
m[st] += 1
|
||||
wh, _ = withheld_status(purchases.get(code, []))
|
||||
m[{"ST_ALREADY_WITHHELD_CONFIRMED": "ST_WITHHOLDING_CONFIRMED", "ST_ALREADY_WITHHELD_NOT_CONFIRMED": "ST_WITHHOLDING_NOT_CONFIRMED"}.get(wh, "ST_WITHHOLDING_UNKNOWN")] += 1
|
||||
role, _ = product_role(p["TIPO"], p["PROD_COZINHA"] == 1, code in recipes, code in composed, bool(_s(p["PRODUTO_SERVICO"])), bool(purchases.get(code)) or stock_entry.get(code, False))
|
||||
m["PRODUCT_ROLE_KNOWN" if role != "UNKNOWN" else "PRODUCT_ROLE_UNKNOWN"] += 1
|
||||
m["ROLE_" + role] += 1
|
||||
state = csosn_v2(p["CSOSN"], company_default, ncm_status, st if uf_ba else "ST_UNKNOWN", wh, role, refs["csosn"])["status"] if crt1 else "CSOSN_CONTEXT_REQUIRED"
|
||||
m["CSOSN_RESOLVED" if state == "CSOSN_RESOLVED" else ("CSOSN_CANDIDATES" if state == "CSOSN_CANDIDATE" else "CSOSN_CONTEXT_REQUIRED")] += 1
|
||||
situ[_s(p["SITUACAO_TRIBUTARIA"])] += 1
|
||||
n = len(products)
|
||||
fill = lambda f: sum(1 for p in products if _s(p.get(f)) not in (None, "")) # noqa: E731
|
||||
return {"TOTAL_PRODUCTS": n, "metrics": dict(m),
|
||||
"IPI_RESOLVED": m["IPI_NUMERIC"] + m["IPI_NON_NUMERIC"] + m["IPI_SOURCE_BLANK"],
|
||||
"quality": {"ncm_valid_share": round(m["NCM_VALID"] / n, 4) if n else None, "cest_filled": fill("CEST"), "csosn_filled": fill("CSOSN"), "cfop_filled": fill("CFOP"), "ipi_filled": fill("IPI"),
|
||||
"cfop_distinct": len(cfop_values), "cfop_top": cfop_values.most_common(6), "situacao_tributaria": dict(situ.most_common(6)),
|
||||
"purchase_items_C000088": sum(len(v) for v in purchases.values()), "products_with_purchase_evidence": len(purchases), "recipes": len(recipes), "composed_products": len(composed)},
|
||||
"guards_applied": {"st_dataset_ba_only": not uf_ba, "csosn_requires_crt1": not crt1}}
|
||||
|
||||
|
||||
def build_refs(get, products, company_uf):
|
||||
"""Reference-data snapshot from the Hub (`get(path) -> json`). Used by the engines for domain validity only."""
|
||||
refs = {}
|
||||
for key, domain in (("csosn", "CSOSN"), ("cst_icms", "CST_ICMS"), ("cst_pis", "CST_PIS"), ("cst_cofins", "CST_COFINS"), ("origem", "ORIGEM_MERCADORIA")):
|
||||
refs[key] = set(get(f"/v1/reference/fiscal_codes/{domain}")["codes"])
|
||||
valid = set()
|
||||
for code in {_s(p.get(f)) for p in products for f in ("CFOP", "CFOP_ENTRADA") if _s(p.get(f))}:
|
||||
if get(f"/v1/reference/cfop/{code}")["found"]:
|
||||
valid.add(code)
|
||||
refs["cfop_valid"] = valid
|
||||
refs["fcp_uf"] = {}
|
||||
fcp = get(f"/v1/reference/fcp/{company_uf}")
|
||||
if fcp.get("found"):
|
||||
refs["fcp_uf"][company_uf] = fcp
|
||||
refs["cbs_rate_2026"] = get("/v1/reference/rtc_cbs_rate/2026").get("rate")
|
||||
refs["cclasstrib_count"] = sum(d["record_count"] for d in get("/v1/datasets/status") if d["kind"] == "RTC_CCLASSTRIB" and d["status"] == "ACTIVE")
|
||||
return refs
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Shared read-only Firebird access for the Agent: full table or a CODIGO subset."""
|
||||
import os
|
||||
|
||||
FIELDS = "CODIGO,PRODUTO,GTIN AS EAN_GTIN,NCM,CLASSIFICACAO_FISCAL,CFOP,CST,ALIQUOTA,CEST,CSOSN,IPI,EX_TIPI"
|
||||
# Present in every audited SGV database but selected only when the column exists (capability-based, never assumed).
|
||||
OPTIONAL_FIELDS = ("TIPO", "PROD_COZINHA", "PRODUTO_SERVICO", "SITUACAO_TRIBUTARIA")
|
||||
|
||||
|
||||
def _select_fields(cursor):
|
||||
cursor.execute("SELECT TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME = 'C000025'")
|
||||
have = {r[0] for r in cursor.fetchall()}
|
||||
return FIELDS + "".join(f",{c}" for c in OPTIONAL_FIELDS if c in have)
|
||||
|
||||
|
||||
def _connect(env, read_only=True):
|
||||
import fdb
|
||||
kwargs = dict(
|
||||
database=env["FIREBIRD_DATABASE_PATH"],
|
||||
user=env["FB_USER"],
|
||||
password=env["FB_PASSWORD"],
|
||||
fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
)
|
||||
if read_only:
|
||||
kwargs["isolation_level"] = fdb.ISOLATION_LEVEL_READ_COMMITED_RO
|
||||
return fdb.connect(**kwargs)
|
||||
|
||||
|
||||
def check_connection(env=None):
|
||||
"""Best-effort read-only connectivity probe. Returns True/False, never raises."""
|
||||
env = env or os.environ
|
||||
try:
|
||||
conn = _connect(env)
|
||||
try:
|
||||
conn.cursor().execute("SELECT 1 FROM RDB$DATABASE")
|
||||
finally:
|
||||
conn.close()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def read_products(env=None, codigos=None):
|
||||
env = env or os.environ
|
||||
conn = _connect(env)
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
fields = _select_fields(cursor)
|
||||
if codigos:
|
||||
placeholders = ",".join("?" for _ in codigos)
|
||||
cursor.execute(f"SELECT {fields} FROM C000025 WHERE CODIGO IN ({placeholders}) ORDER BY CODIGO", tuple(codigos))
|
||||
else:
|
||||
cursor.execute(f"SELECT {fields} FROM C000025 ORDER BY CODIGO")
|
||||
names = [column[0].strip() for column in cursor.description]
|
||||
return [dict(zip(names, row)) for row in cursor.fetchall()]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def parse_company_context(rows):
|
||||
"""Pure: rows of (UF, CRT) from C000004 -> {"uf","crt"}. Fails closed (ValueError) if absent,
|
||||
blank or ambiguous — the engines never fall back to a hardcoded UF/CRT."""
|
||||
pairs = {(str(uf or "").strip().upper(), str(crt or "").strip()) for uf, crt in rows}
|
||||
if len(pairs) != 1:
|
||||
raise ValueError(f"company context unavailable or ambiguous ({len(pairs)} distinct UF/CRT rows in C000004)")
|
||||
uf, crt = next(iter(pairs))
|
||||
if len(uf) != 2 or crt not in ("1", "2", "3"):
|
||||
raise ValueError(f"company context invalid: UF={uf!r} CRT={crt!r}")
|
||||
return {"uf": uf, "crt": crt}
|
||||
|
||||
|
||||
def read_company_context(env=None):
|
||||
"""Resolve the ACTIVE company (UF/CRT) with the deterministic resolver (see company_context.py); fails closed with
|
||||
CompanyContextError when the context is ambiguous, conflicting or unresolved. No hardcoded company values, no first-row fallback."""
|
||||
from .company_context import CompanyContextError, preflight_company_context
|
||||
result = preflight_company_context(env or os.environ)
|
||||
if result["status"] != "PASS":
|
||||
raise CompanyContextError(result)
|
||||
return {"uf": result["company_uf"], "crt": result["company_crt"], "company_code": result["company_code"], "source_of_truth": result["source_of_truth"], "confidence": result["confidence"]}
|
||||
|
||||
|
||||
def read_evidence(env=None, selected_company_code=None):
|
||||
"""Read-only evidence tables used by the coverage report (purchases, stock entries, recipes, compositions) and the company's
|
||||
default CSOSN. Every table/column is optional: a missing one just means less evidence (capability-based)."""
|
||||
from collections import defaultdict
|
||||
env = env or os.environ
|
||||
conn = _connect(env)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT TRIM(RDB$RELATION_NAME), TRIM(RDB$FIELD_NAME) FROM RDB$RELATION_FIELDS WHERE RDB$RELATION_NAME IN ('C000004','C000032','C000088','C000100','RECEITA_INGREDIENTES')")
|
||||
cols = defaultdict(set)
|
||||
for t, f in cur.fetchall():
|
||||
cols[t].add(f)
|
||||
has = lambda t, *c: t in cols and all(x in cols[t] for x in c) # noqa: E731
|
||||
s = lambda v: None if v is None else str(v).strip() # noqa: E731
|
||||
ev = {"recipes": set(), "composed": set(), "purchases": defaultdict(list), "stock_entry": {}, "stock_in": {}, "company_default_csosn": None}
|
||||
if has("RECEITA_INGREDIENTES", "CODRECEITA"):
|
||||
cur.execute("SELECT CODRECEITA FROM RECEITA_INGREDIENTES WHERE CODRECEITA IS NOT NULL")
|
||||
ev["recipes"] = {str(r[0]).zfill(6) for r in cur.fetchall()}
|
||||
if has("C000032", "CODPRODUTO", "TIPO"):
|
||||
cur.execute("SELECT DISTINCT CODPRODUTO FROM C000032 WHERE TIPO = 'COMPOSICAO'")
|
||||
ev["composed"] = {s(r[0]) for r in cur.fetchall()}
|
||||
if has("C000088", "CODPRODUTO", "ICMS_RETIDO", "ICMS_VALORRETIDO", "SUB_VALOR"):
|
||||
cur.execute("SELECT CODPRODUTO, ICMS_RETIDO, ICMS_VALORRETIDO, SUB_VALOR FROM C000088")
|
||||
for cod, ret, vr, sv in cur.fetchall():
|
||||
ev["purchases"][s(cod)].append({"icms_retido": s(ret), "valor_retido": vr, "sub_valor": sv})
|
||||
if has("C000100", "CODPRODUTO", "ULTIMA_ENTRADA", "NOTA_FISCAL"):
|
||||
cur.execute("SELECT CODPRODUTO, ULTIMA_ENTRADA, NOTA_FISCAL FROM C000100")
|
||||
ev["stock_entry"] = {s(r[0]): (r[1] is not None or s(r[2]) not in (None, "", "000000")) for r in cur.fetchall()}
|
||||
if has("C000100", "CODPRODUTO", "ENTRADAS"):
|
||||
cur.execute("SELECT CODPRODUTO, ENTRADAS FROM C000100")
|
||||
ev["stock_in"] = {s(r[0]): bool(r[1] and float(r[1]) > 0) for r in cur.fetchall()}
|
||||
if has("C000004", "ICMS_CSOSN", "CODIGO"):
|
||||
cur.execute("SELECT CODIGO, ICMS_CSOSN FROM C000004")
|
||||
rows = {s(a): s(b) for a, b in cur.fetchall()}
|
||||
ev["company_default_csosn"] = rows.get(selected_company_code) if selected_company_code else (next(iter(rows.values())) if len(rows) == 1 else None)
|
||||
ev["purchases"] = dict(ev["purchases"])
|
||||
return ev
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Decide whether a classified product's IPI rate may be auto-applied.
|
||||
|
||||
Scope is deliberately narrow: C000025.IPI only, and only when the Hub's TIPI
|
||||
match is a single unambiguous numeric rate. Never auto-applies NT/non-numeric
|
||||
markers, blank source rates, ambiguous EX candidates, or legacy/invalid NCM.
|
||||
"""
|
||||
|
||||
|
||||
def ipi_write_decision(decision, current_ipi):
|
||||
"""current_ipi must be a value freshly read from Firebird immediately before
|
||||
this call — None here means the ERP field is genuinely NULL right now, not
|
||||
that it wasn't read."""
|
||||
if decision.get("status") == "LEGACY_NCM":
|
||||
return {"eligible": False, "reason_code": "LEGACY_NCM", "proposed_value": None}
|
||||
if decision.get("ipi_status") != "IPI_RESOLVED":
|
||||
return {"eligible": False, "reason_code": f"IPI_STATUS_{decision.get('ipi_status')}", "proposed_value": None}
|
||||
|
||||
ipi = (decision.get("classification") or {}).get("ipi")
|
||||
if not ipi:
|
||||
return {"eligible": False, "reason_code": "NO_IPI_DECISION", "proposed_value": None}
|
||||
|
||||
rate_kind = ipi.get("rate_kind")
|
||||
if rate_kind != "NUMERIC":
|
||||
return {"eligible": False, "reason_code": f"RATE_KIND_{rate_kind}", "proposed_value": None}
|
||||
|
||||
proposed_value = ipi.get("rate_numeric")
|
||||
if proposed_value is None:
|
||||
return {"eligible": False, "reason_code": "NO_NUMERIC_VALUE", "proposed_value": None}
|
||||
|
||||
if current_ipi is not None and str(current_ipi).strip() == str(proposed_value).strip():
|
||||
return {"eligible": False, "reason_code": "SAME_VALUE", "proposed_value": proposed_value}
|
||||
|
||||
return {"eligible": True, "reason_code": "CHANGE_REQUIRED", "proposed_value": proposed_value}
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Shared per-product IPI auto-apply pipeline: Hub decision -> dry-run ->
|
||||
guarded write -> local SQLite audit. Used by both the manual CLI
|
||||
(run_agent_auto_apply.py) and the long-running job-polling service
|
||||
(run_agent_service.py) so the processing logic exists in exactly one place.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from .ipi_auto_apply import ipi_write_decision
|
||||
from .runner import product_fingerprint
|
||||
|
||||
|
||||
def now():
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def run_pipeline(products, decisions, state, firebird_writes, write_connection, executor, database_path, authorized_path=None):
|
||||
"""Process every (product, decision) pair. Returns a summary dict.
|
||||
|
||||
products come from a fresh Firebird read (current IPI is always "known").
|
||||
decisions come from the Hub's /v1/classify/batch, one per product, same order.
|
||||
"""
|
||||
summary = {
|
||||
"processed": 0,
|
||||
"auto_apply_eligible": 0,
|
||||
"writes_attempted": 0,
|
||||
"writes_succeeded": 0,
|
||||
"writes_failed": 0,
|
||||
"skipped_not_eligible": 0,
|
||||
"no_ops": 0,
|
||||
}
|
||||
for product, decision in zip(products, decisions):
|
||||
code = product["CODIGO"]
|
||||
fingerprint = product_fingerprint(product)
|
||||
current_ipi = product.get("IPI")
|
||||
summary["processed"] += 1
|
||||
|
||||
wd = ipi_write_decision(decision, current_ipi)
|
||||
decision_status = decision.get("status")
|
||||
ipi_status = decision.get("ipi_status")
|
||||
|
||||
if not firebird_writes:
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "DRY_RUN_ONLY", current_ipi, wd["proposed_value"], None, wd["reason_code"], now())
|
||||
if wd["eligible"]:
|
||||
summary["auto_apply_eligible"] += 1
|
||||
elif wd["reason_code"] == "SAME_VALUE":
|
||||
summary["no_ops"] += 1
|
||||
continue
|
||||
|
||||
if not wd["eligible"]:
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "SKIPPED_NOT_ELIGIBLE", current_ipi, wd["proposed_value"], None, wd["reason_code"], now())
|
||||
summary["skipped_not_eligible"] += 1
|
||||
if wd["reason_code"] == "SAME_VALUE":
|
||||
summary["no_ops"] += 1
|
||||
continue
|
||||
|
||||
summary["auto_apply_eligible"] += 1
|
||||
summary["writes_attempted"] += 1
|
||||
proposal = {
|
||||
"product_code": code,
|
||||
"field": "IPI",
|
||||
"current_value": current_ipi,
|
||||
"proposed_value": wd["proposed_value"],
|
||||
"status": "CHANGE_REQUIRED",
|
||||
"write_eligible": True,
|
||||
"source_dataset": "TIPI_IPI",
|
||||
"source_version": decision.get("dataset_versions", {}).get("tipi_ipi"),
|
||||
}
|
||||
try:
|
||||
record = executor.execute_guarded_update(write_connection, database_path=database_path, proposal=proposal, now=now(), authorized_path=authorized_path)
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "SUCCEEDED", record.before_value, record.after_value, f"rowcount={record.rowcount}", wd["reason_code"], now())
|
||||
summary["writes_succeeded"] += 1
|
||||
except executor.WriteRefused as error:
|
||||
state.log_ipi_write(code, fingerprint, decision_status, ipi_status, "FAILED", current_ipi, wd["proposed_value"], str(error), wd["reason_code"], now())
|
||||
summary["writes_failed"] += 1
|
||||
|
||||
return summary
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Single installation preflight for the GTI Fiscal Agent. READ-ONLY: never writes to Firebird, never prints secrets.
|
||||
|
||||
PREFLIGHT=PASS | FAIL (+ blockers, one line each)
|
||||
|
||||
Checks: config, FISCAL_MODE/FIREBIRD_WRITES, Hub reachable/auth/version, Firebird reachable + version + ODS, C000004/C000025,
|
||||
capabilities (minimum: NCM, IPI, CEST), company context (UF/CRT, deterministic resolver), schema/type guards, SQLite state, log dir.
|
||||
"""
|
||||
import os
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
AGENT_VERSION_DEFAULT = "1.0.0-rc1"
|
||||
REQUIRED_ENV = ("GTI_HUB_URL", "AGENT_TOKEN", "FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")
|
||||
REQUIRED_CAPABILITIES = ("supports_ncm", "supports_ipi", "supports_cest")
|
||||
FISCAL_MODES = ("DRY_RUN", "AUTO_APPLY")
|
||||
|
||||
|
||||
def _check(name, ok, detail="", warn=False):
|
||||
return {"name": name, "status": "PASS" if ok else ("WARN" if warn else "FAIL"), "detail": detail}
|
||||
|
||||
|
||||
def major_minor(version):
|
||||
parts = str(version or "").replace("-", ".").replace("rc", ".").split(".")
|
||||
return ".".join(parts[:2])
|
||||
|
||||
|
||||
def default_http_get(url, token=None, timeout=10):
|
||||
import httpx
|
||||
headers = {"Authorization": f"Bearer {token}"} if token else {}
|
||||
r = httpx.get(url, headers=headers, timeout=timeout)
|
||||
return r.status_code, (r.json() if r.headers.get("content-type", "").startswith("application/json") else {})
|
||||
|
||||
|
||||
def firebird_probe(env):
|
||||
"""Read-only introspection of the configured database."""
|
||||
import fdb
|
||||
from .company_context import collect_evidence, config_from_env, resolve
|
||||
conn = fdb.connect(database=env["FIREBIRD_DATABASE_PATH"], user=env["FB_USER"], password=env["FB_PASSWORD"], fb_library_name=env.get("FIREBIRD_CLIENT_LIBRARY"),
|
||||
isolation_level=fdb.ISOLATION_LEVEL_READ_COMMITED_RO)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute("""SELECT TRIM(rf.RDB$RELATION_NAME), TRIM(rf.RDB$FIELD_NAME), f.RDB$FIELD_TYPE, COALESCE(f.RDB$CHARACTER_LENGTH, f.RDB$FIELD_LENGTH)
|
||||
FROM RDB$RELATION_FIELDS rf JOIN RDB$FIELDS f ON f.RDB$FIELD_NAME = rf.RDB$FIELD_SOURCE WHERE rf.RDB$RELATION_NAME IN ('C000004', 'C000025')""")
|
||||
tables = {}
|
||||
for t, c, ty, ln in cur.fetchall():
|
||||
tables.setdefault(t, {})[c] = (ty, ln)
|
||||
products = None
|
||||
if "C000025" in tables:
|
||||
cur.execute("SELECT COUNT(*) FROM C000025")
|
||||
products = cur.fetchone()[0]
|
||||
company = resolve(collect_evidence(cur), config_from_env(env))
|
||||
return {"server_version": getattr(conn, "server_version", None), "ods": f"{getattr(conn, 'ods_version', '?')}.{getattr(conn, 'ods_minor_version', '?')}",
|
||||
"tables": tables, "products": products, "company": company}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def run_preflight(env=None, *, http_get=default_http_get, probe=firebird_probe, agent_version=None):
|
||||
from .capabilities import detect_capabilities, type_ok
|
||||
from .company_context import preflight_report
|
||||
env = env if env is not None else os.environ
|
||||
checks, info = [], {}
|
||||
agent_version = agent_version or env.get("AGENT_VERSION") or AGENT_VERSION_DEFAULT
|
||||
|
||||
missing = [k for k in REQUIRED_ENV if not (env.get(k) or "").strip()]
|
||||
checks.append(_check("CONFIG", not missing, f"missing: {', '.join(missing)}" if missing else "required variables present"))
|
||||
|
||||
mode = (env.get("FISCAL_MODE") or "").strip().upper()
|
||||
writes = (env.get("FIREBIRD_WRITES") or "0").strip() == "1"
|
||||
checks.append(_check("FISCAL_MODE", mode in FISCAL_MODES, f"FISCAL_MODE={mode or 'unset'} (must be one of {FISCAL_MODES})"))
|
||||
dry_conflict = mode == "DRY_RUN" and writes
|
||||
checks.append(_check("FIREBIRD_WRITES", not dry_conflict, "FIREBIRD_WRITES=1 conflicts with FISCAL_MODE=DRY_RUN (writes stay disabled; fix the config)" if dry_conflict else f"requested={int(writes)}"))
|
||||
effective = writes and mode == "AUTO_APPLY"
|
||||
info["effective_firebird_writes"] = effective
|
||||
if effective:
|
||||
ok = bool((env.get("AUTHORIZED_WRITE_DATABASE_PATH") or "").strip() and (env.get("EXPECTED_SCHEMA_FINGERPRINT") or "").strip())
|
||||
checks.append(_check("WRITE_AUTHORIZATION", ok, "AUTO_APPLY needs AUTHORIZED_WRITE_DATABASE_PATH and EXPECTED_SCHEMA_FINGERPRINT (two independent layers)"))
|
||||
|
||||
hub = (env.get("GTI_HUB_URL") or "").rstrip("/")
|
||||
hub_ok = False
|
||||
if hub:
|
||||
try:
|
||||
code, _ = http_get(f"{hub}/health")
|
||||
hub_ok = code == 200
|
||||
checks.append(_check("HUB_REACHABLE", hub_ok, f"GET /health -> {code}"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("HUB_REACHABLE", False, type(e).__name__))
|
||||
if hub_ok:
|
||||
try:
|
||||
code, body = http_get(f"{hub}/version")
|
||||
hub_v = body.get("version")
|
||||
info["hub_version"] = hub_v
|
||||
checks.append(_check("HUB_VERSION_COMPATIBLE", code == 200 and major_minor(hub_v) == major_minor(agent_version), f"hub={hub_v} agent={agent_version} (major.minor must match)"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("HUB_VERSION_COMPATIBLE", False, type(e).__name__))
|
||||
try:
|
||||
code, _ = http_get(f"{hub}/v1/datasets/status", token=env.get("AGENT_TOKEN"))
|
||||
checks.append(_check("AUTH", code == 200, f"agent token -> HTTP {code}"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("AUTH", False, type(e).__name__))
|
||||
|
||||
fb = None
|
||||
if all((env.get(k) or "").strip() for k in ("FIREBIRD_DATABASE_PATH", "FB_USER", "FB_PASSWORD")):
|
||||
try:
|
||||
fb = probe(env)
|
||||
checks.append(_check("FIREBIRD_REACHABLE", True, "read-only connection opened"))
|
||||
info.update(firebird_version=fb.get("server_version"), ods=fb.get("ods"), products=fb.get("products"))
|
||||
checks.append(_check("FIREBIRD_VERSION_ODS", bool(fb.get("server_version")), f"{fb.get('server_version')} / ODS {fb.get('ods')}"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("FIREBIRD_REACHABLE", False, f"{type(e).__name__}: {str(e)[:80]}"))
|
||||
if fb:
|
||||
t = fb["tables"]
|
||||
checks.append(_check("C000004", "C000004" in t, "company table present" if "C000004" in t else "missing"))
|
||||
checks.append(_check("C000025", "C000025" in t, "product table present" if "C000025" in t else "missing"))
|
||||
cols = {c: ty for c, (ty, _) in t.get("C000025", {}).items()}
|
||||
caps = detect_capabilities(cols)
|
||||
info["capabilities"] = caps
|
||||
lacking = [c for c in REQUIRED_CAPABILITIES if not caps.get(c)]
|
||||
checks.append(_check("CAPABILITIES", not lacking, f"missing required: {', '.join(lacking)}" if lacking else f"required present; optional true: {sum(1 for c, v in caps.items() if v and c not in REQUIRED_CAPABILITIES)}"))
|
||||
rep = preflight_report(fb["company"])
|
||||
info["company_context"] = rep
|
||||
checks.append(_check("COMPANY_CONTEXT", rep["COMPANY_CONTEXT"] == "PASS", "; ".join(rep.get("reasons") or []) or f"{rep['classification']} source={rep['source_of_truth']}"))
|
||||
checks.append(_check("UF", bool(rep.get("company_uf")), str(rep.get("company_uf"))))
|
||||
checks.append(_check("CRT", bool(rep.get("company_crt")), str(rep.get("company_crt"))))
|
||||
writable = [f for f in ("IPI", "CEST", "EX_TIPI", "CLASSIFICACAO_FISCAL") if f in t.get("C000025", {})]
|
||||
bad = [f for f in writable if not type_ok(f, t["C000025"][f][0])]
|
||||
lens = {f: t["C000025"][f][1] for f in ("CEST", "EX_TIPI") if f in t.get("C000025", {})}
|
||||
checks.append(_check("SCHEMA_TYPE_GUARDS", not bad and len(writable) >= 3, f"incompatible: {bad}" if bad else f"types ok for {writable}; text lengths {lens} (the executor re-checks length/type before any write)"))
|
||||
|
||||
state_db = env.get("AGENT_STATE_DB") or str(Path.cwd() / "agent_state.sqlite")
|
||||
try:
|
||||
Path(state_db).parent.mkdir(parents=True, exist_ok=True)
|
||||
c = sqlite3.connect(state_db)
|
||||
c.execute("SELECT 1")
|
||||
c.close()
|
||||
checks.append(_check("SQLITE", True, "state database opens"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("SQLITE", False, type(e).__name__))
|
||||
log_path = Path(env.get("AGENT_LOG_PATH") or Path.cwd() / "logs" / "agent.log")
|
||||
try:
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
checks.append(_check("LOG_DIRECTORY", os.access(log_path.parent, os.W_OK), "log directory writable"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
checks.append(_check("LOG_DIRECTORY", False, type(e).__name__))
|
||||
|
||||
blockers = [f"{c['name']}: {c['detail']}" for c in checks if c["status"] == "FAIL"]
|
||||
return {"PREFLIGHT": "FAIL" if blockers else "PASS", "blockers": blockers, "checks": checks, "agent_version": agent_version, "fiscal_mode": mode or None, **info}
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Gate Firebird reads on the registered schema fingerprint."""
|
||||
import hashlib, json
|
||||
REFERENCE_SCHEMA_V1 = "1198f89897532b643439f06b729e93b189018f56e7d478e95095f8f56977005b"
|
||||
|
||||
def product_fingerprint(product):
|
||||
fields = ("PRODUTO", "EAN_GTIN", "NCM", "CLASSIFICACAO_FISCAL", "CFOP", "CST", "ALIQUOTA", "CEST", "CSOSN")
|
||||
return hashlib.sha256(json.dumps({key: product.get(key) for key in fields}, sort_keys=True, default=str).encode()).hexdigest()
|
||||
def to_hub_product(product):
|
||||
# C000025 has NCM empty and stores the confirmed eight-digit NCM in CLASSIFICACAO_FISCAL.
|
||||
return {"product_id": product["CODIGO"], "description": product.get("PRODUTO", ""), "gtin": product.get("EAN_GTIN"), "current_ncm": product.get("CLASSIFICACAO_FISCAL")}
|
||||
def run_sample(products, client, state, schema_fingerprint):
|
||||
if schema_fingerprint != REFERENCE_SCHEMA_V1: return {"status": "SCHEMA_UNKNOWN", "results": [], "firebird_writes": 0}
|
||||
outbound = []
|
||||
for product in products:
|
||||
fingerprint = product_fingerprint(product)
|
||||
if state.enqueue(product["CODIGO"], fingerprint, "pending", "0.1.0"): outbound.append((product, fingerprint))
|
||||
if not outbound: return {"status": "OK", "results": [], "firebird_writes": 0, "api_requests": 0}
|
||||
try: results = client.classify_batch([to_hub_product(product) for product, _ in outbound])
|
||||
except Exception:
|
||||
for product, _ in outbound: state.retry(product["CODIGO"])
|
||||
return {"status": "API_UNAVAILABLE", "results": [], "firebird_writes": 0}
|
||||
for (product, fingerprint), result in zip(outbound, results): state.result_received(product["CODIGO"], fingerprint, result["dataset_versions"].get("ncm") or "none", result["engine_version"])
|
||||
return {"status": "OK", "results": results, "firebird_writes": 0, "api_requests": 1}
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Local SQLite outbox; it has no Firebird dependency or write path."""
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
class AgentState:
|
||||
def __init__(self, path):
|
||||
self.connection = sqlite3.connect(Path(path)); self.connection.row_factory = sqlite3.Row
|
||||
self.connection.executescript("CREATE TABLE IF NOT EXISTS product_state(product_id TEXT PRIMARY KEY, source_fingerprint TEXT NOT NULL, dataset_version TEXT NOT NULL, engine_version TEXT NOT NULL, status TEXT NOT NULL, last_processed_at TEXT, last_result_hash TEXT, reason_code TEXT); CREATE TABLE IF NOT EXISTS outbox(product_id TEXT PRIMARY KEY, source_fingerprint TEXT NOT NULL, status TEXT NOT NULL, retries INTEGER NOT NULL DEFAULT 0); CREATE TABLE IF NOT EXISTS ipi_write_log(product_id TEXT NOT NULL, source_fingerprint TEXT NOT NULL, decision_status TEXT, ipi_status TEXT, write_status TEXT NOT NULL, before_value TEXT, proposed_value TEXT, result TEXT, reason_code TEXT, timestamp TEXT NOT NULL);")
|
||||
def enqueue(self, product_id, source_fingerprint, dataset_version, engine_version):
|
||||
row = self.connection.execute("SELECT * FROM product_state WHERE product_id=?", (product_id,)).fetchone()
|
||||
if row and tuple(row[key] for key in ("source_fingerprint", "engine_version", "status")) == (source_fingerprint, engine_version, "RESULT_RECEIVED"): return False
|
||||
self.connection.execute("INSERT OR REPLACE INTO product_state(product_id,source_fingerprint,dataset_version,engine_version,status) VALUES(?,?,?,?,?)", (product_id, source_fingerprint, dataset_version, engine_version, "PENDING_API")); self.connection.execute("INSERT OR REPLACE INTO outbox(product_id,source_fingerprint,status) VALUES(?,?,?)", (product_id, source_fingerprint, "PENDING_API")); self.connection.commit(); return True
|
||||
def pending(self): return [dict(row) for row in self.connection.execute("SELECT * FROM outbox WHERE status='PENDING_API'")]
|
||||
def retry(self, product_id): self.connection.execute("UPDATE outbox SET retries=retries+1,status='PENDING_API' WHERE product_id=?", (product_id,)); self.connection.commit()
|
||||
def result_received(self, product_id, source_fingerprint, dataset_version, engine_version):
|
||||
self.connection.execute("UPDATE product_state SET source_fingerprint=?,dataset_version=?,engine_version=?,status='RESULT_RECEIVED' WHERE product_id=?", (source_fingerprint, dataset_version, engine_version, product_id)); self.connection.execute("DELETE FROM outbox WHERE product_id=?", (product_id,)); self.connection.commit()
|
||||
def log_ipi_write(self, product_id, source_fingerprint, decision_status, ipi_status, write_status, before_value, proposed_value, result, reason_code, timestamp):
|
||||
self.connection.execute(
|
||||
"INSERT INTO ipi_write_log(product_id,source_fingerprint,decision_status,ipi_status,write_status,before_value,proposed_value,result,reason_code,timestamp) VALUES(?,?,?,?,?,?,?,?,?,?)",
|
||||
(product_id, source_fingerprint, decision_status, ipi_status, write_status, str(before_value) if before_value is not None else None, str(proposed_value) if proposed_value is not None else None, result, reason_code, timestamp),
|
||||
)
|
||||
self.connection.commit()
|
||||
def ipi_write_log_for(self, product_id):
|
||||
return [dict(row) for row in self.connection.execute("SELECT * FROM ipi_write_log WHERE product_id=? ORDER BY timestamp", (product_id,))]
|
||||
@@ -0,0 +1,3 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).parents[1] / "src"))
|
||||
@@ -0,0 +1,117 @@
|
||||
import pytest
|
||||
|
||||
from gti_fiscal_agent.company_context import CompanyContextError, config_from_env, mask_cnpj, preflight_report, resolve
|
||||
from gti_fiscal_agent.firebird_reader import parse_company_context
|
||||
|
||||
CN_A, CN_B, CN_C = "11111111000111", "22222222000122", "33333333000133"
|
||||
|
||||
|
||||
def co(code, cnpj, uf="BA", crt="1", optante="S"):
|
||||
return {"code": code, "cnpj": cnpj, "uf": uf, "crt": crt, "optante": optante, "name": "X"}
|
||||
|
||||
|
||||
def ev(companies, emissor=None, docs=None):
|
||||
return {"c000004": companies, "emissor": emissor if emissor is not None else [], "nfce_keys": docs or [], "nf_by_filial": {}}
|
||||
|
||||
|
||||
def emis(cnpj=CN_A, uf="BA", crt="1"):
|
||||
return {"cnpj": cnpj, "uf": uf, "crt": crt}
|
||||
|
||||
|
||||
DOC_A = [{"cuf": "29", "cnpj": CN_A, "n": 100}]
|
||||
|
||||
|
||||
def test_single_company_all_sources_agree_is_auto_resolved():
|
||||
r = resolve(ev([co("000001", CN_A)], [emis()], DOC_A))
|
||||
assert r["status"] == "PASS" and r["classification"] == "AUTO_RESOLVED" and r["company_uf"] == "BA" and r["company_crt"] == "1" and r["confidence"] == "HIGH"
|
||||
assert "DOCUMENT_KEYS" in r["source_of_truth"]["uf"]
|
||||
|
||||
|
||||
def test_multiple_rows_resolved_only_through_emitter_identity_never_first_row():
|
||||
rows = [co("000001", CN_B, "SP", None, "N"), co("000005", CN_A), co("000007", CN_C, "BA", None, "N")]
|
||||
r = resolve(ev(rows, [emis(CN_A)], DOC_A))
|
||||
assert r["status"] == "PASS" and r["company_code"] == "000005" and r["source_of_truth"]["company"] == "EMISSOR_CNPJ_MATCH"
|
||||
|
||||
|
||||
def test_multiple_rows_without_disambiguation_requires_config():
|
||||
rows = [co("000001", CN_A), co("000002", CN_B)]
|
||||
r = resolve(ev(rows, []))
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFIG_REQUIRED" and r["company_code"] is None
|
||||
assert resolve(ev(rows, [emis(CN_C)]))["classification"] == "CONFIG_REQUIRED" # emitter matches none
|
||||
ok = resolve(ev(rows, [], []), {"company_code": "000002"})
|
||||
assert ok["status"] == "PASS" and ok["company_code"] == "000002" and ok["classification"] == "CONFIGURED"
|
||||
|
||||
|
||||
def test_configured_company_must_exist_and_agree_with_emitter():
|
||||
rows = [co("000001", CN_A), co("000002", CN_B)]
|
||||
assert resolve(ev(rows, [emis(CN_A)]), {"company_cnpj": "99999999999999"})["classification"] == "CONFIG_REQUIRED"
|
||||
r = resolve(ev(rows, [emis(CN_A)], DOC_A), {"company_code": "000002"})
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFLICT" # selected company differs from the emitter: unexplained
|
||||
|
||||
|
||||
def test_documents_only_from_another_cnpj_conflict_but_mixed_history_is_a_warning():
|
||||
r = resolve(ev([co("000001", CN_A)], [emis()], [{"cuf": "29", "cnpj": CN_B, "n": 10}]))
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFLICT"
|
||||
r = resolve(ev([co("000001", CN_A)], [emis()], [{"cuf": "29", "cnpj": CN_A, "n": 10}, {"cuf": "29", "cnpj": CN_B, "n": 1}]))
|
||||
assert r["status"] == "PASS" and any("HISTORICAL_NFCE_KEYS_FROM_OTHER_CNPJ" in w for w in r["warnings"])
|
||||
|
||||
|
||||
def test_uf_conflict_and_unknown():
|
||||
assert resolve(ev([co("1", CN_A, "BA")], [emis(uf="SP")]))["classification"] == "CONFLICT"
|
||||
assert resolve(ev([co("1", CN_A, "BA")], [emis()], [{"cuf": "35", "cnpj": CN_A, "n": 5}]))["classification"] == "CONFLICT" # key says SP
|
||||
assert resolve(ev([co("1", CN_A, "")], [emis(uf="")]))["classification"] == "UNRESOLVED"
|
||||
|
||||
|
||||
def test_crt_conflict_needs_explicit_source_and_records_it():
|
||||
base = ev([co("1", CN_A, crt="3")], [emis(crt="1")], DOC_A)
|
||||
r = resolve(base)
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFLICT" and r["crt_candidates"] == {"EMISSOR": "1", "C000004": "3"}
|
||||
ok = resolve(base, {"crt_source": "EMISSOR"})
|
||||
assert ok["status"] == "PASS" and ok["company_crt"] == "1" and ok["classification"] == "CONFIGURED" and ok["confidence"] == "MEDIUM"
|
||||
assert resolve(base, {"crt_source": "C000004"})["company_crt"] == "3"
|
||||
assert resolve(base, {"crt_source_invalid": True})["classification"] == "CONFIG_REQUIRED"
|
||||
|
||||
|
||||
def test_crt_unknown_fails_and_single_source_is_flagged():
|
||||
assert resolve(ev([co("1", CN_A, crt=None)], [emis(crt="")]))["classification"] == "UNRESOLVED"
|
||||
r = resolve(ev([co("1", CN_A, crt=None)], [emis(crt="1")]))
|
||||
assert r["status"] == "PASS" and "CRT_HAS_A_SINGLE_SOURCE" in r["warnings"]
|
||||
|
||||
|
||||
def test_optante_inconsistency_is_a_warning_not_a_guess():
|
||||
r = resolve(ev([co("1", CN_A, crt="3", optante="S")], [emis(crt="3")]))
|
||||
assert r["status"] == "PASS" and any("OPTANTE_SIMPLES" in w for w in r["warnings"])
|
||||
|
||||
|
||||
def test_full_cnpj_is_never_exposed():
|
||||
r = resolve(ev([co("1", CN_A)], [emis()], DOC_A))
|
||||
dump = str(preflight_report(r)) + str(r)
|
||||
assert CN_A not in dump and mask_cnpj(CN_A) == "11**********11"
|
||||
|
||||
|
||||
def test_read_company_context_fails_closed(monkeypatch):
|
||||
from gti_fiscal_agent import company_context as cc, firebird_reader as fr
|
||||
monkeypatch.setattr(cc, "preflight_company_context", lambda env: resolve(ev([co("1", CN_A), co("2", CN_B)], [])))
|
||||
with pytest.raises(CompanyContextError):
|
||||
fr.read_company_context({})
|
||||
monkeypatch.setattr(cc, "preflight_company_context", lambda env: resolve(ev([co("1", CN_A)], [emis()], DOC_A)))
|
||||
assert fr.read_company_context({})["uf"] == "BA"
|
||||
|
||||
|
||||
def test_config_from_env_and_legacy_parser_unchanged():
|
||||
assert config_from_env({"COMPANY_CODE": " 000005 ", "COMPANY_CNPJ": "11.111.111/0001-11", "COMPANY_CRT_SOURCE": "emissor"}) == {"company_code": "000005", "company_cnpj": "11111111000111", "crt_source": "EMISSOR", "crt_source_invalid": False, "uf_override": None}
|
||||
assert config_from_env({"COMPANY_CRT_SOURCE": "x"})["crt_source_invalid"] is True
|
||||
assert parse_company_context([("ba", "1")]) == {"uf": "BA", "crt": "1"}
|
||||
|
||||
|
||||
def test_auto_crt_source_and_uf_override_semantics():
|
||||
assert config_from_env({"COMPANY_CRT_SOURCE": "AUTO"}) ["crt_source"] is None and config_from_env({"COMPANY_CRT_SOURCE": "AUTO"})["crt_source_invalid"] is False
|
||||
conflict = ev([co("1", CN_A, crt="3")], [emis(crt="1")], DOC_A)
|
||||
assert resolve(conflict, config_from_env({"COMPANY_CRT_SOURCE": "AUTO"}))["status"] == "FAIL" # AUTO never hides a CRT conflict
|
||||
good = ev([co("1", CN_A)], [emis()], DOC_A)
|
||||
assert resolve(good, {"uf_override": "BA"})["status"] == "PASS"
|
||||
r = resolve(good, {"uf_override": "SP"})
|
||||
assert r["status"] == "FAIL" and r["classification"] == "CONFLICT" # override never replaces the resolver silently
|
||||
blank = ev([co("1", CN_A, uf="")], [emis(uf="")])
|
||||
r = resolve(blank, {"uf_override": "BA"})
|
||||
assert r["status"] == "PASS" and "UF_FROM_EXPLICIT_OVERRIDE" in r["warnings"]
|
||||
@@ -0,0 +1,52 @@
|
||||
from gti_fiscal_agent.ipi_auto_apply import ipi_write_decision
|
||||
|
||||
|
||||
def _decision(ipi_status="IPI_RESOLVED", status="EXACT_NCM", ipi=None):
|
||||
return {"status": status, "ipi_status": ipi_status, "classification": {"ipi": ipi}}
|
||||
|
||||
|
||||
def test_numeric_null_current_is_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "NUMERIC", "rate_numeric": 3.25}), None)
|
||||
assert d["eligible"] is True
|
||||
assert d["proposed_value"] == 3.25
|
||||
assert d["reason_code"] == "CHANGE_REQUIRED"
|
||||
|
||||
|
||||
def test_numeric_same_value_is_not_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "NUMERIC", "rate_numeric": 3.25}), 3.25)
|
||||
assert d["eligible"] is False
|
||||
assert d["reason_code"] == "SAME_VALUE"
|
||||
|
||||
|
||||
def test_numeric_different_value_is_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "NUMERIC", "rate_numeric": 3.25}), 0)
|
||||
assert d["eligible"] is True
|
||||
assert d["proposed_value"] == 3.25
|
||||
|
||||
|
||||
def test_non_numeric_never_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "NON_NUMERIC", "rate_numeric": None}), None)
|
||||
assert d["eligible"] is False
|
||||
assert d["reason_code"] == "RATE_KIND_NON_NUMERIC"
|
||||
|
||||
|
||||
def test_source_blank_never_eligible():
|
||||
d = ipi_write_decision(_decision(ipi={"rate_kind": "BLANK", "rate_numeric": None}), None)
|
||||
assert d["eligible"] is False
|
||||
assert d["reason_code"] == "RATE_KIND_BLANK"
|
||||
|
||||
|
||||
def test_ambiguous_ipi_status_never_eligible():
|
||||
d = ipi_write_decision(_decision(ipi_status="IPI_CANDIDATES", ipi=None), None)
|
||||
assert d["eligible"] is False
|
||||
|
||||
|
||||
def test_no_match_never_eligible():
|
||||
d = ipi_write_decision(_decision(ipi_status="NO_TIPI_MAPPING_FOUND", ipi=None), None)
|
||||
assert d["eligible"] is False
|
||||
|
||||
|
||||
def test_legacy_ncm_never_eligible():
|
||||
d = ipi_write_decision(_decision(status="LEGACY_NCM", ipi_status="LEGACY_NCM"), None)
|
||||
assert d["eligible"] is False
|
||||
assert d["reason_code"] == "LEGACY_NCM"
|
||||
@@ -0,0 +1,132 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gti_fiscal_agent.company_context import resolve
|
||||
from gti_fiscal_agent.preflight import major_minor, run_preflight
|
||||
|
||||
ROOT = Path(__file__).parents[2]
|
||||
spec = importlib.util.spec_from_file_location("run_agent_service", ROOT / "agent" / "scripts" / "run_agent_service.py")
|
||||
service = importlib.util.module_from_spec(spec); sys.modules[spec.name] = service; spec.loader.exec_module(service)
|
||||
|
||||
CN = "11111111000111"
|
||||
|
||||
|
||||
def env(tmp_path, **over):
|
||||
base = {"GTI_HUB_URL": "http://hub.test", "AGENT_TOKEN": "SECRETTOKENVALUE9", "FIREBIRD_DATABASE_PATH": "x.fdb", "FB_USER": "u", "FB_PASSWORD": "SECRETPASSVALUE9", "FISCAL_MODE": "DRY_RUN", "FIREBIRD_WRITES": "0",
|
||||
"AGENT_STATE_DB": str(tmp_path / "state.sqlite"), "AGENT_LOG_PATH": str(tmp_path / "logs" / "a.log"), "AGENT_VERSION": "1.0.0-rc1"}
|
||||
base.update(over)
|
||||
return base
|
||||
|
||||
|
||||
def http_ok(url, token=None, timeout=10):
|
||||
if url.endswith("/health"):
|
||||
return 200, {"status": "ok"}
|
||||
if url.endswith("/version"):
|
||||
return 200, {"version": "1.0.0-rc1"}
|
||||
return (200 if token == "SECRETTOKENVALUE9" else 401), {}
|
||||
|
||||
|
||||
def fake_probe(company_over=None, caps_missing=()):
|
||||
def probe(env):
|
||||
cols = {"CLASSIFICACAO_FISCAL": (37, 20), "CFOP_ENTRADA": (14, 4), "CST_ENTRADA": (14, 4), "CFOP": (37, 4), "CST": (37, 10), "ALIQUOTA": (16, 8), "CEST": (37, 7), "SITUACAO_TRIBUTARIA": (37, 2),
|
||||
"COD_ENTRADA_COFINS": (37, 3), "COD_SAIDA_COFINS": (37, 3), "ALIQ_COFINS": (37, 8), "COD_ENTRADA_PIS": (37, 3), "COD_SAIDA_PIS": (37, 3), "ALIQ_PIS": (37, 8), "CSOSN": (37, 60), "IPI": (27, 8),
|
||||
"CST_IPI": (14, 3), "IAT": (37, 1), "INCIDENCIA_PISCOFINS": (37, 30), "PISCOFINS": (37, 1), "IPPT": (37, 1), "EX_TIPI": (37, 3), "COD_ORIGEM_MERCADORIA": (8, 4)}
|
||||
for c in caps_missing:
|
||||
cols.pop(c)
|
||||
ev = {"c000004": [{"code": "000001", "cnpj": CN, "uf": "BA", "crt": "1", "optante": "S"}], "emissor": [{"cnpj": CN, "uf": "BA", "crt": "1"}], "nfce_keys": [{"cuf": "29", "cnpj": CN, "n": 5}], "nf_by_filial": {}}
|
||||
ev.update(company_over or {})
|
||||
return {"server_version": "WI-V6.3.9 Firebird 2.5", "ods": "11.2", "tables": {"C000004": {}, "C000025": cols}, "products": 3712, "company": resolve(ev, {})}
|
||||
return probe
|
||||
|
||||
|
||||
def test_preflight_passes_for_a_clean_dry_run_installation(tmp_path):
|
||||
r = run_preflight(env(tmp_path), http_get=http_ok, probe=fake_probe())
|
||||
assert r["PREFLIGHT"] == "PASS" and r["blockers"] == [] and r["effective_firebird_writes"] is False
|
||||
names = {c["name"] for c in r["checks"]}
|
||||
assert {"HUB_REACHABLE", "AUTH", "HUB_VERSION_COMPATIBLE", "FIREBIRD_REACHABLE", "C000004", "C000025", "CAPABILITIES", "COMPANY_CONTEXT", "UF", "CRT", "SCHEMA_TYPE_GUARDS", "SQLITE", "LOG_DIRECTORY", "FISCAL_MODE", "FIREBIRD_WRITES"} <= names
|
||||
assert "SECRETTOKENVALUE9" not in str(r) and "SECRETPASSVALUE9" not in str(r)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("over,blocker", [
|
||||
({"FISCAL_MODE": "DRY_RUN", "FIREBIRD_WRITES": "1"}, "FIREBIRD_WRITES"),
|
||||
({"FISCAL_MODE": ""}, "FISCAL_MODE"),
|
||||
({"AGENT_TOKEN": ""}, "CONFIG"),
|
||||
])
|
||||
def test_preflight_blocks_bad_config(tmp_path, over, blocker):
|
||||
r = run_preflight(env(tmp_path, **over), http_get=http_ok, probe=fake_probe())
|
||||
assert r["PREFLIGHT"] == "FAIL" and any(b.startswith(blocker) for b in r["blockers"])
|
||||
|
||||
|
||||
def test_preflight_fails_on_hub_auth_version_and_missing_capability(tmp_path):
|
||||
def bad_auth(url, token=None, timeout=10):
|
||||
return http_ok(url, "wrong" if token else None)
|
||||
assert any(b.startswith("AUTH") for b in run_preflight(env(tmp_path), http_get=bad_auth, probe=fake_probe())["blockers"])
|
||||
|
||||
def old_hub(url, token=None, timeout=10):
|
||||
return (200, {"version": "0.9.0"}) if url.endswith("/version") else http_ok(url, token)
|
||||
assert any(b.startswith("HUB_VERSION_COMPATIBLE") for b in run_preflight(env(tmp_path), http_get=old_hub, probe=fake_probe())["blockers"])
|
||||
assert any(b.startswith("CAPABILITIES") for b in run_preflight(env(tmp_path), http_get=http_ok, probe=fake_probe(caps_missing=("CEST",)))["blockers"])
|
||||
|
||||
|
||||
def test_preflight_fails_when_company_context_is_ambiguous_or_crt_conflicts(tmp_path):
|
||||
two = {"c000004": [{"code": "1", "cnpj": CN, "uf": "BA", "crt": "1"}, {"code": "2", "cnpj": "22222222000122", "uf": "SP", "crt": "1"}], "emissor": []}
|
||||
r = run_preflight(env(tmp_path), http_get=http_ok, probe=fake_probe(two))
|
||||
assert r["PREFLIGHT"] == "FAIL" and any(b.startswith("COMPANY_CONTEXT") for b in r["blockers"])
|
||||
conflict = {"emissor": [{"cnpj": CN, "uf": "BA", "crt": "3"}]}
|
||||
assert any("CRT_SOURCE_CONFLICT" in b for b in run_preflight(env(tmp_path), http_get=http_ok, probe=fake_probe(conflict))["blockers"])
|
||||
|
||||
|
||||
def test_hub_unreachable_is_a_clear_blocker(tmp_path):
|
||||
def down(url, token=None, timeout=10):
|
||||
raise ConnectionError("nope")
|
||||
r = run_preflight(env(tmp_path), http_get=down, probe=fake_probe())
|
||||
assert r["PREFLIGHT"] == "FAIL" and any(b.startswith("HUB_REACHABLE") for b in r["blockers"])
|
||||
|
||||
|
||||
def test_major_minor():
|
||||
assert major_minor("1.0.0-rc1") == "1.0" and major_minor("1.0.3") == "1.0" and major_minor("0.9.0") != major_minor("1.0.0-rc1")
|
||||
|
||||
|
||||
# ---------------- zero-write guarantee ----------------
|
||||
def test_dry_run_makes_writes_technically_impossible_whatever_firebird_writes_says():
|
||||
for extra in ({"FIREBIRD_WRITES": "1", "FISCAL_MODE": "DRY_RUN"}, {"FIREBIRD_WRITES": "0", "FISCAL_MODE": "AUTO_APPLY"}, {"FIREBIRD_WRITES": "1", "FISCAL_MODE": "dry_run"}):
|
||||
cfg = service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb", **extra})
|
||||
assert cfg.firebird_writes is False
|
||||
assert service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb", "FIREBIRD_WRITES": "1", "FISCAL_MODE": "AUTO_APPLY"}).firebird_writes is True
|
||||
assert service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb", "FIREBIRD_WRITES": "1"}).firebird_writes is True # legacy: unset FISCAL_MODE follows the flag
|
||||
assert service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb"}).fiscal_mode == "DRY_RUN"
|
||||
|
||||
|
||||
def test_write_connection_cannot_be_opened_when_writes_are_disabled(monkeypatch):
|
||||
opened = []
|
||||
monkeypatch.setitem(sys.modules, "fdb", type("F", (), {"connect": staticmethod(lambda **kw: opened.append(kw))}))
|
||||
with pytest.raises(RuntimeError):
|
||||
service.open_write_connection({"FB_USER": "u", "FB_PASSWORD": "SECRETPASSVALUE9"}, "x.fdb", firebird_writes=False)
|
||||
assert opened == []
|
||||
|
||||
|
||||
def test_dry_run_job_never_opens_a_write_connection_nor_calls_the_executor(monkeypatch):
|
||||
class Boom:
|
||||
def __getattr__(self, name):
|
||||
if name in ("execute_guarded_update", "execute_guarded_restore"):
|
||||
raise AssertionError("executor write called in DRY_RUN")
|
||||
return getattr(__import__("scripts.firebird_write_executor", fromlist=["x"]) if False else object(), name, None)
|
||||
cfg = service.Config({"FIREBIRD_DATABASE_PATH": "x.fdb", "FIREBIRD_WRITES": "1", "FISCAL_MODE": "DRY_RUN"})
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: [{"CODIGO": "000001", "PRODUTO": "X", "CLASSIFICACAO_FISCAL": "22030000", "IPI": None}])
|
||||
monkeypatch.setattr(service, "HubClient", lambda url, token: type("H", (), {"classify_batch": lambda self, p: [{"status": "EXACT_NCM", "ipi_status": "IPI_RESOLVED", "classification": {"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.9}},
|
||||
"dataset_versions": {}}]})())
|
||||
monkeypatch.setattr(service, "build_job_coverage", lambda *a, **k: {"TOTAL_PRODUCTS": 1})
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda *a, **k: (_ for _ in ()).throw(AssertionError("write connection opened in DRY_RUN")))
|
||||
|
||||
class State:
|
||||
def log_ipi_write(self, *a): pass
|
||||
|
||||
class Ex:
|
||||
WriteRefused = Exception
|
||||
def execute_guarded_update(self, *a, **k): raise AssertionError("executor write called in DRY_RUN")
|
||||
|
||||
result, error = service.execute_job({"kind": "REPROCESS_ALL", "agent_id": 1, "payload": {}, "job_id": 1}, cfg, Ex(), State(), {})
|
||||
assert error is None and result["writes_attempted"] == 0 and result["firebird_writes_enabled"] is False
|
||||
@@ -0,0 +1,272 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPT = Path(__file__).parents[1] / "scripts" / "run_agent_service.py"
|
||||
AUTHORIZED = r"D:\PROJETOS\GTISISTEMAS\Projeto\TESTESOFICIAIS.FDB"
|
||||
|
||||
|
||||
def load_module():
|
||||
spec = importlib.util.spec_from_file_location("run_agent_service", SCRIPT)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
service = load_module()
|
||||
|
||||
|
||||
class _FakeState:
|
||||
def __init__(self):
|
||||
self.logged = []
|
||||
|
||||
def log_ipi_write(self, *args):
|
||||
self.logged.append(args)
|
||||
|
||||
|
||||
class _FakeExecutor:
|
||||
"""Mirrors the real firebird_write_executor's public surface used by execute_job."""
|
||||
class WriteRefused(Exception):
|
||||
pass
|
||||
|
||||
def __init__(self, path_guard_ok=True, updates=None, fingerprint_guard_ok=True):
|
||||
self.path_guard_ok = path_guard_ok
|
||||
self.updates = updates or {}
|
||||
self.fingerprint_guard_ok = fingerprint_guard_ok
|
||||
|
||||
def assert_authorized_write_path(self, path, authorized_path=None):
|
||||
if not self.path_guard_ok:
|
||||
raise self.WriteRefused(f"path is not the authorized write target: {path}")
|
||||
|
||||
def assert_schema_fingerprint_authorized(self, observed, expected):
|
||||
if not self.fingerprint_guard_ok or observed != expected:
|
||||
raise self.WriteRefused(f"schema fingerprint mismatch: expected {expected!r}, observed {observed!r}")
|
||||
|
||||
def execute_guarded_update(self, connection, *, database_path, proposal, now, authorized_path=None):
|
||||
code = proposal["product_code"]
|
||||
if code in self.updates:
|
||||
raise self.WriteRefused(self.updates[code])
|
||||
record = type("Record", (), {
|
||||
"before_value": proposal["current_value"],
|
||||
"after_value": proposal["proposed_value"],
|
||||
"rowcount": 1,
|
||||
})()
|
||||
return record
|
||||
|
||||
|
||||
def _config(**overrides):
|
||||
base = dict(
|
||||
hub_url="http://localhost:8000", agent_id="a1", agent_token="tok", company_id="c1", company_uf="BA",
|
||||
database_path=AUTHORIZED, firebird_writes=False, poll_interval_seconds=0.01,
|
||||
agent_version="0.1.0", schema_fingerprint="fp", state_db=":memory:",
|
||||
authorized_write_database_path=None, expected_schema_fingerprint="fp",
|
||||
)
|
||||
base.update(overrides)
|
||||
return type("Config", (), base)()
|
||||
|
||||
|
||||
def test_config_defaults_from_env():
|
||||
env = {"FIREBIRD_DATABASE_PATH": AUTHORIZED}
|
||||
config = service.Config(env)
|
||||
assert config.firebird_writes is False
|
||||
assert config.hub_url == "http://localhost:8000"
|
||||
assert config.poll_interval_seconds == 5.0
|
||||
|
||||
|
||||
def test_config_reads_gti_hub_url_and_writes_flag():
|
||||
env = {"FIREBIRD_DATABASE_PATH": AUTHORIZED, "GTI_HUB_URL": "http://hub.internal", "FIREBIRD_WRITES": "1"}
|
||||
config = service.Config(env)
|
||||
assert config.hub_url == "http://hub.internal"
|
||||
assert config.firebird_writes is True
|
||||
|
||||
|
||||
def test_path_guard_fails_closed_before_any_write(monkeypatch):
|
||||
config = _config(firebird_writes=True, database_path=r"D:\PRODUCAO\ERP.FDB")
|
||||
executor = _FakeExecutor(path_guard_ok=False)
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
called = {"read_products": False}
|
||||
|
||||
def fake_read_products(env, codigos=None):
|
||||
called["read_products"] = True
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(service, "read_products", fake_read_products)
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert result is None
|
||||
assert "not the authorized write target" in error
|
||||
assert called["read_products"] is False # never even reads Firebird once the guard refuses
|
||||
|
||||
|
||||
def test_pilot_write_blocked_when_schema_fingerprint_does_not_match(monkeypatch):
|
||||
# A real client install where the live schema fingerprint drifted from what
|
||||
# this Agent installation was configured/registered to expect.
|
||||
config = _config(firebird_writes=True, schema_fingerprint="observed-live-fp", expected_schema_fingerprint="registered-fp")
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
called = {"read_products": False}
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: called.update(read_products=True) or [])
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert result is None
|
||||
assert "schema fingerprint mismatch" in error
|
||||
assert called["read_products"] is False
|
||||
|
||||
|
||||
def test_pilot_write_allowed_with_explicit_authorized_path_and_matching_fingerprint(monkeypatch):
|
||||
client_path = r"D:\ClienteX\ERP\PRODUCAO.FDB"
|
||||
config = _config(
|
||||
firebird_writes=True, database_path=client_path, authorized_write_database_path=client_path,
|
||||
schema_fingerprint="fp", expected_schema_fingerprint="fp",
|
||||
)
|
||||
products = [{"CODIGO": "000026", "PRODUTO": "x", "CLASSIFICACAO_FISCAL": "03055100", "IPI": None}]
|
||||
decisions = [{
|
||||
"status": "EXACT_NCM", "ipi_status": "IPI_RESOLVED",
|
||||
"classification": {"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.25}},
|
||||
"dataset_versions": {"tipi_ipi": "v1"},
|
||||
}]
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: products)
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, *a, **k): pass
|
||||
def classify_batch(self, items): return decisions
|
||||
monkeypatch.setattr(service, "HubClient", FakeClient)
|
||||
fake_connection = type("Conn", (), {"close": lambda self: None})()
|
||||
monkeypatch.setattr(service, "open_write_connection", lambda env, path: fake_connection)
|
||||
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert error is None
|
||||
assert result["writes_succeeded"] == 1
|
||||
|
||||
|
||||
def test_execute_job_isolates_internal_errors_without_raising(monkeypatch):
|
||||
config = _config()
|
||||
|
||||
def boom(env, codigos=None):
|
||||
raise RuntimeError("firebird unavailable")
|
||||
|
||||
monkeypatch.setattr(service, "read_products", boom)
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_ALL", "agent_id": 1, "payload": {}}
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert result is None
|
||||
assert "firebird unavailable" in error
|
||||
|
||||
|
||||
def test_execute_job_success_summarizes_pipeline(monkeypatch):
|
||||
config = _config()
|
||||
products = [{"CODIGO": "000026", "PRODUTO": "x", "CLASSIFICACAO_FISCAL": "03055100", "IPI": None}]
|
||||
decisions = [{
|
||||
"status": "EXACT_NCM", "ipi_status": "IPI_RESOLVED",
|
||||
"classification": {"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.25}},
|
||||
"dataset_versions": {"tipi_ipi": "v1"},
|
||||
}]
|
||||
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: products)
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, *a, **k): pass
|
||||
def classify_batch(self, items): return decisions
|
||||
monkeypatch.setattr(service, "HubClient", FakeClient)
|
||||
|
||||
executor = _FakeExecutor()
|
||||
state = _FakeState()
|
||||
job = {"kind": "REPROCESS_PRODUCTS", "agent_id": 1, "payload": {"codigos": ["000026"]}}
|
||||
|
||||
result, error = service.execute_job(job, config, executor, state, {})
|
||||
assert error is None
|
||||
assert result["products_processed"] == 1
|
||||
assert result["writes_attempted"] == 0 # firebird_writes disabled by default in _config()
|
||||
|
||||
|
||||
class _FakeHubForLoop:
|
||||
"""Simulates: heartbeat flaky twice then OK, one job then nothing, and captures completions."""
|
||||
|
||||
def __init__(self):
|
||||
self.heartbeat_calls = 0
|
||||
self.jobs = [{"job_id": 1, "kind": "REPROCESS_ALL", "agent_id": 1, "payload": {}}]
|
||||
self.completed = []
|
||||
self.failed = []
|
||||
|
||||
def heartbeat(self, *a, **k):
|
||||
self.heartbeat_calls += 1
|
||||
if self.heartbeat_calls == 1:
|
||||
raise ConnectionError("hub down")
|
||||
return {}
|
||||
|
||||
def next_job(self):
|
||||
return self.jobs.pop(0) if self.jobs else None
|
||||
|
||||
def complete_job(self, job_id, result_summary):
|
||||
self.completed.append((job_id, result_summary))
|
||||
|
||||
def fail_job(self, job_id, error):
|
||||
self.failed.append((job_id, error))
|
||||
|
||||
def classify_batch(self, items):
|
||||
return []
|
||||
|
||||
|
||||
class _ExternalStopFlag:
|
||||
"""Mirrors the Windows Service wrapper's stop object: no signal registration,
|
||||
just a plain flag flippable from another thread."""
|
||||
def __init__(self):
|
||||
self.stop = False
|
||||
|
||||
|
||||
def test_run_honors_an_externally_supplied_stopper(monkeypatch):
|
||||
# This is exactly the pattern the Windows Service wrapper uses: it never
|
||||
# constructs GracefulStop() (which would crash off the main thread), it
|
||||
# passes its own stop flag and flips it from the SvcStop handler.
|
||||
config = _config(poll_interval_seconds=0.001)
|
||||
fake_hub = _FakeHubForLoop()
|
||||
fake_hub.jobs = [] # no jobs; loop would run forever without an external stop
|
||||
monkeypatch.setattr(service, "HubClient", lambda *a, **k: fake_hub)
|
||||
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
||||
monkeypatch.setattr(service, "AgentState", lambda path: _FakeState())
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: [])
|
||||
|
||||
stopper = _ExternalStopFlag()
|
||||
calls = {"n": 0}
|
||||
real_sleep = service.time.sleep
|
||||
|
||||
def counting_sleep(seconds):
|
||||
calls["n"] += 1
|
||||
if calls["n"] >= 3:
|
||||
stopper.stop = True
|
||||
|
||||
monkeypatch.setattr(service.time, "sleep", counting_sleep)
|
||||
service.run(config, {}, stopper=stopper) # no max_iterations — only the external flag can stop it
|
||||
assert stopper.stop is True
|
||||
assert calls["n"] >= 3
|
||||
|
||||
|
||||
def test_run_survives_hub_down_then_processes_one_job_and_stops(monkeypatch):
|
||||
config = _config(poll_interval_seconds=0.001)
|
||||
fake_hub = _FakeHubForLoop()
|
||||
monkeypatch.setattr(service, "HubClient", lambda *a, **k: fake_hub)
|
||||
monkeypatch.setattr(service, "load_executor", lambda: _FakeExecutor())
|
||||
monkeypatch.setattr(service, "AgentState", lambda path: _FakeState())
|
||||
monkeypatch.setattr(service, "read_products", lambda env, codigos=None: [])
|
||||
monkeypatch.setattr(service, "build_job_coverage", lambda *a, **k: None)
|
||||
monkeypatch.setattr(service.time, "sleep", lambda seconds: None)
|
||||
|
||||
# first heartbeat call raises -> backoff branch; second call onward succeeds.
|
||||
service.run(config, {}, max_iterations=4)
|
||||
|
||||
assert fake_hub.heartbeat_calls >= 2
|
||||
assert len(fake_hub.completed) == 1
|
||||
assert fake_hub.completed[0][0] == 1
|
||||
assert fake_hub.completed[0][1]["products_processed"] == 0
|
||||
@@ -0,0 +1,7 @@
|
||||
from gti_fiscal_agent.runner import to_hub_product
|
||||
|
||||
|
||||
def test_reference_schema_uses_classificacao_fiscal_as_canonical_ncm():
|
||||
product = {"CODIGO": "000276", "PRODUTO": "ABACAXI M/ JARRA", "NCM": "99999999", "CLASSIFICACAO_FISCAL": "08043000"}
|
||||
|
||||
assert to_hub_product(product)["current_ncm"] == "08043000"
|
||||
@@ -0,0 +1,20 @@
|
||||
from gti_fiscal_agent.state import AgentState
|
||||
|
||||
|
||||
def test_outbox_keeps_failed_work_pending(tmp_path):
|
||||
state = AgentState(tmp_path / "agent.db")
|
||||
assert state.enqueue("000276", "source", "v1", "0.1.0")
|
||||
state.retry("000276")
|
||||
assert state.pending()[0]["status"] == "PENDING_API"
|
||||
state.result_received("000276", "source", "v1", "0.1.0")
|
||||
assert state.enqueue("000276", "source", "v1", "0.1.0") is False
|
||||
|
||||
|
||||
def test_ipi_write_log_records_per_product_audit(tmp_path):
|
||||
state = AgentState(tmp_path / "agent.db")
|
||||
state.log_ipi_write("000276", "fp1", "EXACT_NCM", "IPI_RESOLVED", "SUCCEEDED", None, 3.25, "rowcount=1", "CHANGE_REQUIRED", "2026-09-18T00:00:00Z")
|
||||
log = state.ipi_write_log_for("000276")
|
||||
assert len(log) == 1
|
||||
assert log[0]["write_status"] == "SUCCEEDED"
|
||||
assert log[0]["before_value"] is None
|
||||
assert log[0]["proposed_value"] == "3.25"
|
||||
@@ -0,0 +1,8 @@
|
||||
"""Entry point: python audit_firebird_database.py <FDB> (read-only; see scripts/audit_firebird_database.py)."""
|
||||
import runpy
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.argv[0] = str(Path(__file__).with_name("scripts") / "audit_firebird_database.py")
|
||||
runpy.run_path(sys.argv[0], run_name="__main__")
|
||||
@@ -0,0 +1,7 @@
|
||||
FROM python:3.12-slim
|
||||
WORKDIR /app
|
||||
COPY central/pyproject.toml /tmp/central/pyproject.toml
|
||||
RUN pip install --no-cache-dir /tmp/central
|
||||
COPY central /app/central
|
||||
ENV PYTHONPATH=/app
|
||||
CMD ["sh", "-c", "alembic -c central/alembic.ini upgrade head && uvicorn central.app.main:app --host 0.0.0.0 --port 8000"]
|
||||
@@ -0,0 +1 @@
|
||||
"""GTI Fiscal Hub package."""
|
||||
@@ -0,0 +1,3 @@
|
||||
[alembic]
|
||||
script_location = central/alembic
|
||||
sqlalchemy.url = postgresql+psycopg://gti@postgres:5432/gti_fiscal
|
||||
@@ -0,0 +1,14 @@
|
||||
from alembic import context
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
from central.app.models import Base
|
||||
from central.app.config import settings
|
||||
config = context.config
|
||||
target_metadata = Base.metadata
|
||||
def run_migrations_online():
|
||||
config.set_main_option("sqlalchemy.url", settings.database_url)
|
||||
engine = engine_from_config(config.get_section(config.config_ini_section), prefix="sqlalchemy.", poolclass=pool.NullPool)
|
||||
with engine.connect() as connection:
|
||||
context.configure(connection=connection, target_metadata=target_metadata)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
run_migrations_online()
|
||||
@@ -0,0 +1,11 @@
|
||||
"""initial hub persistence
|
||||
|
||||
Revision ID: 0001
|
||||
"""
|
||||
from alembic import op
|
||||
from central.app.models import Base
|
||||
revision = "0001"
|
||||
down_revision = None
|
||||
branch_labels = depends_on = None
|
||||
def upgrade(): Base.metadata.create_all(op.get_bind())
|
||||
def downgrade(): Base.metadata.drop_all(op.get_bind())
|
||||
@@ -0,0 +1,31 @@
|
||||
"""add CEST links and official legacy NCM evidence
|
||||
|
||||
Revision ID: 0002
|
||||
Revises: 0001
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import Column, ForeignKey, Integer, String, Text, UniqueConstraint
|
||||
|
||||
|
||||
revision = "0002"
|
||||
down_revision = "0001"
|
||||
branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("datasets", Column("raw_sha256", String(64), nullable=True))
|
||||
op.create_table("legacy_ncm_entries", Column("code", String(16), primary_key=True), Column("source", Text, nullable=False), Column("source_date", String(32), nullable=False), Column("reason", Text, nullable=False), Column("successor_code", String(16), nullable=True))
|
||||
op.create_table("cest_entries", Column("id", Integer, primary_key=True), Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False), Column("code", String(16), nullable=False), Column("description", Text, nullable=False), Column("annex", String(8), nullable=False), UniqueConstraint("dataset_id", "code", name="uq_cest_versioned_entry"))
|
||||
op.create_table("cest_ncm_entries", Column("id", Integer, primary_key=True), Column("cest_entry_id", Integer, ForeignKey("cest_entries.id"), nullable=False), Column("ncm_code", String(16), nullable=False), UniqueConstraint("cest_entry_id", "ncm_code", name="uq_cest_ncm_link"))
|
||||
op.create_index("ix_cest_entries_dataset_id", "cest_entries", ["dataset_id"])
|
||||
op.create_index("ix_cest_entries_code", "cest_entries", ["code"])
|
||||
op.create_index("ix_cest_ncm_entries_cest_entry_id", "cest_ncm_entries", ["cest_entry_id"])
|
||||
op.create_index("ix_cest_ncm_entries_ncm_code", "cest_ncm_entries", ["ncm_code"])
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("cest_ncm_entries")
|
||||
op.drop_table("cest_entries")
|
||||
op.drop_table("legacy_ncm_entries")
|
||||
op.drop_column("datasets", "raw_sha256")
|
||||
@@ -0,0 +1,20 @@
|
||||
"""add normalized TIPI entries
|
||||
|
||||
Revision ID: 0003
|
||||
Revises: 0002
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Column, Float, ForeignKey, Integer, JSON, String, Text
|
||||
|
||||
revision = "0003"
|
||||
down_revision = "0002"
|
||||
branch_labels = depends_on = None
|
||||
|
||||
def upgrade():
|
||||
op.create_table("tipi_entries", Column("id", Integer, primary_key=True), Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False), Column("ncm", String(8), nullable=False), Column("ex", String(32)), Column("description", Text, nullable=False), Column("ipi_rate", Float, nullable=True), Column("ipi_rate_text", String(32)), Column("source_row", Integer, nullable=False), Column("metadata_json", JSON, nullable=False))
|
||||
op.create_index("ix_tipi_entries_dataset_id", "tipi_entries", ["dataset_id"])
|
||||
op.create_index("ix_tipi_entries_ncm", "tipi_entries", ["ncm"])
|
||||
op.create_index("ix_tipi_entries_ncm_ex", "tipi_entries", ["ncm", "ex"])
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("tipi_entries")
|
||||
@@ -0,0 +1,16 @@
|
||||
"""add BA ST MVA entries
|
||||
|
||||
Revision ID: 0004
|
||||
Revises: 0003
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Column, ForeignKey, Integer, JSON, String, Text
|
||||
revision = "0004"
|
||||
down_revision = "0003"
|
||||
branch_labels = depends_on = None
|
||||
def upgrade():
|
||||
op.create_table("ba_st_mva_entries", Column("id", Integer, primary_key=True), Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False), Column("item", String(32), nullable=False), Column("segment", Text), Column("cest", String(16)), Column("ncm_sh", String(32)), Column("description", Text), Column("mva_original", String(32)), Column("mva_adjusted", String(32)), Column("agreement_info", Text), Column("source_page", Integer, nullable=False), Column("metadata_json", JSON, nullable=False))
|
||||
op.create_index("ix_ba_st_mva_entries_dataset_id", "ba_st_mva_entries", ["dataset_id"])
|
||||
op.create_index("ix_ba_st_mva_entries_cest", "ba_st_mva_entries", ["cest"])
|
||||
op.create_index("ix_ba_st_mva_entries_ncm_sh", "ba_st_mva_entries", ["ncm_sh"])
|
||||
def downgrade(): op.drop_table("ba_st_mva_entries")
|
||||
@@ -0,0 +1,15 @@
|
||||
"""normalize BA ST NCM SH
|
||||
|
||||
Revision ID: 0005
|
||||
Revises: 0004
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Column, ForeignKey, Integer, JSON, String, UniqueConstraint
|
||||
revision="0005"; down_revision="0004"; branch_labels=depends_on=None
|
||||
def upgrade():
|
||||
op.drop_index("ix_ba_st_mva_entries_ncm_sh", table_name="ba_st_mva_entries")
|
||||
op.drop_column("ba_st_mva_entries", "ncm_sh")
|
||||
op.create_table("ba_st_mva_ncm_sh", Column("id", Integer, primary_key=True), Column("entry_id", Integer, ForeignKey("ba_st_mva_entries.id"), nullable=False), Column("ncm_sh", String(32), nullable=False), Column("metadata_json", JSON, nullable=False), UniqueConstraint("entry_id", "ncm_sh", name="uq_ba_st_mva_entry_ncm_sh"))
|
||||
op.create_index("ix_ba_st_mva_ncm_sh_entry_id", "ba_st_mva_ncm_sh", ["entry_id"])
|
||||
op.create_index("ix_ba_st_mva_ncm_sh_ncm_sh", "ba_st_mva_ncm_sh", ["ncm_sh"])
|
||||
def downgrade(): pass
|
||||
@@ -0,0 +1,43 @@
|
||||
"""agent status fields and jobs table
|
||||
|
||||
Revision ID: 0006
|
||||
Revises: 0005
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, JSON, String, Text
|
||||
|
||||
revision = "0006"; down_revision = "0005"; branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("agents", Column("enabled", Boolean, server_default="true", nullable=False))
|
||||
op.add_column("agents", Column("last_seen_at", DateTime(timezone=True), nullable=True))
|
||||
op.add_column("agents", Column("last_status", String(32), nullable=True))
|
||||
op.add_column("agents", Column("schema_fingerprint", String(64), nullable=True))
|
||||
op.add_column("agents", Column("agent_version", String(32), nullable=True))
|
||||
op.add_column("agents", Column("created_at", DateTime(timezone=True), server_default="now()", nullable=False))
|
||||
op.add_column("agents", Column("updated_at", DateTime(timezone=True), server_default="now()", nullable=False))
|
||||
|
||||
op.create_table(
|
||||
"jobs",
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("agent_id", Integer, ForeignKey("agents.id"), nullable=False),
|
||||
Column("kind", String(32), nullable=False),
|
||||
Column("status", String(16), nullable=False, server_default="PENDING"),
|
||||
Column("payload", JSON, nullable=False),
|
||||
Column("created_at", DateTime(timezone=True), server_default="now()", nullable=False),
|
||||
Column("started_at", DateTime(timezone=True), nullable=True),
|
||||
Column("finished_at", DateTime(timezone=True), nullable=True),
|
||||
Column("attempts", Integer, nullable=False, server_default="0"),
|
||||
Column("result_summary", JSON, nullable=True),
|
||||
Column("error", Text, nullable=True),
|
||||
)
|
||||
op.create_index("ix_jobs_agent_id", "jobs", ["agent_id"])
|
||||
op.create_index("ix_jobs_kind", "jobs", ["kind"])
|
||||
op.create_index("ix_jobs_status", "jobs", ["status"])
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("jobs")
|
||||
for column in ("enabled", "last_seen_at", "last_status", "schema_fingerprint", "agent_version", "created_at", "updated_at"):
|
||||
op.drop_column("agents", column)
|
||||
@@ -0,0 +1,30 @@
|
||||
"""fiscal_code_entries table for ORIGEM/CST_ICMS/CSOSN/CST_PIS/CST_COFINS
|
||||
|
||||
Revision ID: 0007
|
||||
Revises: 0006
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Column, ForeignKey, Integer, JSON, String, Text, UniqueConstraint
|
||||
|
||||
revision = "0007"; down_revision = "0006"; branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"fiscal_code_entries",
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False),
|
||||
Column("domain", String(32), nullable=False),
|
||||
Column("code", String(8), nullable=False),
|
||||
Column("description", Text, nullable=False),
|
||||
Column("source_section", String(96), nullable=False),
|
||||
Column("metadata_json", JSON, nullable=False),
|
||||
UniqueConstraint("dataset_id", "domain", "code", name="uq_fiscal_code_entry"),
|
||||
)
|
||||
op.create_index("ix_fiscal_code_entries_dataset_id", "fiscal_code_entries", ["dataset_id"])
|
||||
op.create_index("ix_fiscal_code_entries_domain", "fiscal_code_entries", ["domain"])
|
||||
op.create_index("ix_fiscal_code_entries_code", "fiscal_code_entries", ["code"])
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("fiscal_code_entries")
|
||||
@@ -0,0 +1,124 @@
|
||||
"""CFOP/FCP/ANP/NCM_UTRIB/RTC_BIOFUEL_INDEX/RTC_CCREDPRES tables
|
||||
|
||||
Revision ID: 0008
|
||||
Revises: 0007
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Boolean, Column, Float, ForeignKey, Integer, String, Text, UniqueConstraint
|
||||
|
||||
revision = "0008"; down_revision = "0007"; branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"cfop_entries",
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False),
|
||||
Column("code", String(8), nullable=False),
|
||||
Column("title", Text, nullable=False),
|
||||
Column("description", Text, nullable=True),
|
||||
Column("effective_from", String(32), nullable=True),
|
||||
Column("effective_to", String(32), nullable=True),
|
||||
Column("ind_nfe", String(4), nullable=True),
|
||||
Column("ind_comunica", String(4), nullable=True),
|
||||
Column("ind_transp", String(4), nullable=True),
|
||||
Column("ind_devol", String(4), nullable=True),
|
||||
Column("ind_retor", String(4), nullable=True),
|
||||
Column("ind_anula", String(4), nullable=True),
|
||||
Column("ind_remes", String(4), nullable=True),
|
||||
Column("ind_comb", String(4), nullable=True),
|
||||
Column("ind_exc_ibscbs", String(4), nullable=True),
|
||||
UniqueConstraint("dataset_id", "code", name="uq_cfop_entry"),
|
||||
)
|
||||
op.create_index("ix_cfop_entries_dataset_id", "cfop_entries", ["dataset_id"])
|
||||
op.create_index("ix_cfop_entries_code", "cfop_entries", ["code"])
|
||||
|
||||
op.create_table(
|
||||
"fcp_rates",
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False),
|
||||
Column("uf_code", String(4), nullable=False),
|
||||
Column("uf", String(4), nullable=False),
|
||||
Column("uf_name", String(64), nullable=False),
|
||||
Column("aliq_1", String(32), nullable=True),
|
||||
Column("aliq_2", String(32), nullable=True),
|
||||
Column("aliq_3", String(32), nullable=True),
|
||||
Column("observacao", Text, nullable=True),
|
||||
UniqueConstraint("dataset_id", "uf", name="uq_fcp_rate_uf"),
|
||||
)
|
||||
op.create_index("ix_fcp_rates_dataset_id", "fcp_rates", ["dataset_id"])
|
||||
op.create_index("ix_fcp_rates_uf", "fcp_rates", ["uf"])
|
||||
|
||||
op.create_table(
|
||||
"anp_product_codes",
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False),
|
||||
Column("cprod_anp", String(16), nullable=False),
|
||||
UniqueConstraint("dataset_id", "cprod_anp", name="uq_anp_product_code"),
|
||||
)
|
||||
op.create_index("ix_anp_product_codes_dataset_id", "anp_product_codes", ["dataset_id"])
|
||||
op.create_index("ix_anp_product_codes_cprod_anp", "anp_product_codes", ["cprod_anp"])
|
||||
|
||||
op.create_table(
|
||||
"ncm_utrib_entries",
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False),
|
||||
Column("ncm", String(16), nullable=False),
|
||||
Column("effective_from", String(32), nullable=True),
|
||||
Column("effective_to", String(32), nullable=True),
|
||||
Column("utrib_abbreviation", String(16), nullable=True),
|
||||
Column("utrib_description", Text, nullable=True),
|
||||
UniqueConstraint("dataset_id", "ncm", name="uq_ncm_utrib_entry"),
|
||||
)
|
||||
op.create_index("ix_ncm_utrib_entries_dataset_id", "ncm_utrib_entries", ["dataset_id"])
|
||||
op.create_index("ix_ncm_utrib_entries_ncm", "ncm_utrib_entries", ["ncm"])
|
||||
|
||||
op.create_table(
|
||||
"rtc_biofuel_index_entries",
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False),
|
||||
Column("cprod_anp", String(16), nullable=True),
|
||||
Column("description", Text, nullable=True),
|
||||
Column("p_bio_percent", Float, nullable=True),
|
||||
Column("effective_from", String(32), nullable=True),
|
||||
Column("effective_to", String(32), nullable=True),
|
||||
Column("is_historical", Boolean, nullable=False, server_default="false"),
|
||||
)
|
||||
op.create_index("ix_rtc_biofuel_index_entries_dataset_id", "rtc_biofuel_index_entries", ["dataset_id"])
|
||||
op.create_index("ix_rtc_biofuel_index_entries_cprod_anp", "rtc_biofuel_index_entries", ["cprod_anp"])
|
||||
|
||||
op.create_table(
|
||||
"rtc_cred_pres_entries",
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False),
|
||||
Column("code", String(8), nullable=False),
|
||||
Column("description", Text, nullable=False),
|
||||
Column("legal_basis", Text, nullable=True),
|
||||
Column("apropria_via_nf", Boolean, nullable=True),
|
||||
Column("apropria_via_evento", Boolean, nullable=True),
|
||||
Column("ind_deduz_cred_pres", String(8), nullable=True),
|
||||
Column("ind_g_cbs_cred_pres", String(8), nullable=True),
|
||||
Column("ind_g_ibs_cred_pres", String(8), nullable=True),
|
||||
Column("aliquota_cbs_text", Text, nullable=True),
|
||||
Column("aliquota_ibs_text", Text, nullable=True),
|
||||
Column("p_aliq_cred_pres_cbs", Text, nullable=True),
|
||||
Column("p_aliq_cred_pres_ibs", Text, nullable=True),
|
||||
Column("p_red_transicao_ibs", Text, nullable=True),
|
||||
Column("cclass_nota_referenciada", String(16), nullable=True),
|
||||
Column("d_ini_vig_cbs", String(32), nullable=True),
|
||||
Column("d_fim_vig_cbs", String(32), nullable=True),
|
||||
Column("d_ini_vig_ibs", String(32), nullable=True),
|
||||
Column("d_fim_vig_ibs", String(32), nullable=True),
|
||||
UniqueConstraint("dataset_id", "code", name="uq_rtc_cred_pres_code"),
|
||||
)
|
||||
op.create_index("ix_rtc_cred_pres_entries_dataset_id", "rtc_cred_pres_entries", ["dataset_id"])
|
||||
op.create_index("ix_rtc_cred_pres_entries_code", "rtc_cred_pres_entries", ["code"])
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("rtc_cred_pres_entries")
|
||||
op.drop_table("rtc_biofuel_index_entries")
|
||||
op.drop_table("ncm_utrib_entries")
|
||||
op.drop_table("anp_product_codes")
|
||||
op.drop_table("fcp_rates")
|
||||
op.drop_table("cfop_entries")
|
||||
@@ -0,0 +1,20 @@
|
||||
"""widen rtc_cred_pres_entries.cclass_nota_referenciada to Text
|
||||
|
||||
Some official rows carry free text ("cClass de alíquota zero") in this
|
||||
column, not just the numeric code — VARCHAR(16) truncated them.
|
||||
|
||||
Revision ID: 0009
|
||||
Revises: 0008
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Text
|
||||
|
||||
revision = "0009"; down_revision = "0008"; branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.alter_column("rtc_cred_pres_entries", "cclass_nota_referenciada", type_=Text, existing_nullable=True)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.alter_column("rtc_cred_pres_entries", "cclass_nota_referenciada", type_=None, existing_nullable=True)
|
||||
@@ -0,0 +1,48 @@
|
||||
"""RTC cClassTrib, CST-IBS/CBS and CBS rate tables
|
||||
|
||||
Revision ID: 0010
|
||||
Revises: 0009
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import JSON, Column, Float, ForeignKey, Integer, String, Text, UniqueConstraint
|
||||
|
||||
revision = "0010"; down_revision = "0009"; branch_labels = depends_on = None
|
||||
|
||||
|
||||
def _fk():
|
||||
return Column("dataset_id", Integer, ForeignKey("datasets.id"), nullable=False, index=True)
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"rtc_cst_ibscbs_entries",
|
||||
Column("id", Integer, primary_key=True), _fk(),
|
||||
Column("cst", String(8), nullable=False, index=True),
|
||||
Column("description", Text, nullable=False),
|
||||
Column("indicators", JSON, nullable=False),
|
||||
UniqueConstraint("dataset_id", "cst", name="uq_rtc_cst_ibscbs"),
|
||||
)
|
||||
op.create_table(
|
||||
"rtc_cclasstrib_entries",
|
||||
Column("id", Integer, primary_key=True), _fk(),
|
||||
Column("cst", String(8), nullable=False, index=True),
|
||||
Column("cst_description", Text), Column("cclasstrib", String(8), nullable=False, index=True),
|
||||
Column("name", Text), Column("description", Text), Column("lc_redacao", Text), Column("lc_214", Text),
|
||||
Column("regulamento_cbs", Text), Column("regulamento_ibs", Text),
|
||||
Column("tipo_aliquota", String(64)), Column("p_red_ibs", String(32)), Column("p_red_cbs", String(32)),
|
||||
Column("d_ini_vig", String(32)), Column("d_fim_vig", String(32)), Column("data_atualizacao", String(32)),
|
||||
Column("anexo", Text), Column("link", Text), Column("indicators", JSON, nullable=False),
|
||||
UniqueConstraint("dataset_id", "cclasstrib", name="uq_rtc_cclasstrib"),
|
||||
)
|
||||
op.create_table(
|
||||
"rtc_cbs_rate_entries",
|
||||
Column("id", Integer, primary_key=True), _fk(),
|
||||
Column("year", Integer, nullable=False, index=True),
|
||||
Column("rate_text", Text, nullable=False), Column("rate", Float),
|
||||
UniqueConstraint("dataset_id", "year", name="uq_rtc_cbs_rate_year"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
for t in ("rtc_cbs_rate_entries", "rtc_cclasstrib_entries", "rtc_cst_ibscbs_entries"):
|
||||
op.drop_table(t)
|
||||
@@ -0,0 +1,31 @@
|
||||
"""operator fiscal context entries (facts the ERP does not know: role, origin, ST retention, treatment, ...)
|
||||
|
||||
Revision ID: 0011
|
||||
Revises: 0010
|
||||
"""
|
||||
from alembic import op
|
||||
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func
|
||||
|
||||
revision = "0011"; down_revision = "0010"; branch_labels = depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"fiscal_context_entries",
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("tenant_id", Integer, ForeignKey("tenants.id"), nullable=False, index=True),
|
||||
Column("company_key", String(64), nullable=False, index=True),
|
||||
Column("scope", String(8), nullable=False),
|
||||
Column("scope_key", String(64), nullable=False, server_default=""),
|
||||
Column("key", String(48), nullable=False),
|
||||
Column("value", Text, nullable=False),
|
||||
Column("source", String(16), nullable=False, server_default="OPERATOR"),
|
||||
Column("reason", Text, nullable=True),
|
||||
Column("created_at", DateTime(timezone=True), server_default=func.now()),
|
||||
Column("updated_at", DateTime(timezone=True), server_default=func.now()),
|
||||
UniqueConstraint("tenant_id", "company_key", "scope", "scope_key", "key", name="uq_fiscal_context_entry"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("fiscal_context_entries")
|
||||
@@ -0,0 +1 @@
|
||||
"""Fiscal Hub application modules."""
|
||||
@@ -0,0 +1,250 @@
|
||||
"""Authenticated version-one Hub routes."""
|
||||
import hashlib, json, logging
|
||||
from datetime import datetime, timezone
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from sqlalchemy import select
|
||||
from .auth import authenticate, bootstrap
|
||||
from .classification import ENGINE_VERSION, Classifier
|
||||
from .config import settings
|
||||
from .datasets.service import DatasetService
|
||||
from .jobs import claim_next_job, complete_job, create_job, fail_job
|
||||
from .context_keys import validate_entry
|
||||
from .models import FiscalContextEntry, ClassificationCache, ClassificationRequest, ClassificationResult, Dataset, Job
|
||||
|
||||
logger = logging.getLogger("gti_fiscal_hub")
|
||||
|
||||
def router(session_factory, bootstrap_token):
|
||||
routes = APIRouter(prefix="/v1")
|
||||
def principal(request: Request):
|
||||
header = request.headers.get("Authorization", "")
|
||||
if not header.startswith("Bearer "): raise HTTPException(401, "Bearer token required")
|
||||
session = session_factory()
|
||||
try:
|
||||
if bootstrap_token: bootstrap(session, bootstrap_token)
|
||||
agent = authenticate(session, header.removeprefix("Bearer "))
|
||||
if not agent: raise HTTPException(401, "Invalid token")
|
||||
return session, agent
|
||||
except Exception:
|
||||
session.close(); raise
|
||||
def decide(payload, identity):
|
||||
session, agent = identity; service = DatasetService(session)
|
||||
supplied = payload.get("tenant_id")
|
||||
if supplied is not None and str(supplied) != str(agent.tenant_id): raise HTTPException(403, "Tenant does not match token")
|
||||
product = payload.get("product", payload)
|
||||
# Cheap key first (engine version + dataset versions + input), then the cache; the heavy classifier runs only on a MISS.
|
||||
engine_version, dataset_versions = ENGINE_VERSION, service.dataset_versions()
|
||||
canonical = {"tenant": agent.tenant_id, "product": product, "context": payload.get("context", {}), "engine": engine_version, "datasets": dataset_versions}
|
||||
fingerprint = hashlib.sha256(json.dumps(canonical, sort_keys=True, default=str, separators=(",", ":")).encode()).hexdigest()
|
||||
cached = session.get(ClassificationCache, fingerprint)
|
||||
if cached:
|
||||
result = dict(cached.result); result["provenance"] = {**result["provenance"], "cache_hit": True}; return result
|
||||
decision = Classifier(service).classify(product)
|
||||
if decision["engine_version"] != engine_version or decision["dataset_versions"] != dataset_versions: # never store under a key that does not describe the decision
|
||||
canonical = {**canonical, "engine": decision["engine_version"], "datasets": decision["dataset_versions"]}
|
||||
fingerprint = hashlib.sha256(json.dumps(canonical, sort_keys=True, default=str, separators=(",", ":")).encode()).hexdigest()
|
||||
result = {**decision, "request_fingerprint": fingerprint, "provenance": {"cache_hit": False, "dataset_versions": decision["dataset_versions"]}}
|
||||
request = ClassificationRequest(tenant_id=agent.tenant_id, agent_id=agent.id, fingerprint=fingerprint, payload=payload); session.add(request); session.flush()
|
||||
session.add(ClassificationResult(request_id=request.id, result=result)); session.add(ClassificationCache(fingerprint=fingerprint, tenant_id=agent.tenant_id, result=result)); session.commit(); return result
|
||||
@routes.post("/validate/ncm")
|
||||
def validate_ncm(payload: dict, identity=Depends(principal)):
|
||||
return decide({"product": {"current_ncm": payload.get("ncm"), "description": ""}, "tenant_id": payload.get("tenant_id")}, identity)
|
||||
@routes.post("/classify")
|
||||
def classify(payload: dict, identity=Depends(principal)):
|
||||
try: return decide(payload, identity)
|
||||
finally: identity[0].close()
|
||||
@routes.post("/classify/batch")
|
||||
def classify_batch(payload: dict, identity=Depends(principal)):
|
||||
try: return {"results": [decide({**payload, "product": product}, identity) for product in payload.get("products", [])]}
|
||||
finally: identity[0].close()
|
||||
@routes.get("/datasets/status")
|
||||
def status(identity=Depends(principal)):
|
||||
session, _ = identity
|
||||
try: return [{"id": item.id, "kind": item.kind, "status": item.status, "sha256": item.sha256, "record_count": item.record_count} for item in session.scalars(select(Dataset)).all()]
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/reference/cfop/{code}")
|
||||
def reference_cfop(code: str, identity=Depends(principal)):
|
||||
session, _ = identity
|
||||
try:
|
||||
rows = DatasetService(session).cfop_lookup(code=code)
|
||||
return {"found": bool(rows), "entries": [{"code": r.code, "title": r.title, "description": r.description} for r in rows]}
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/reference/ncm_utrib/{ncm}")
|
||||
def reference_ncm_utrib(ncm: str, identity=Depends(principal)):
|
||||
session, _ = identity
|
||||
try:
|
||||
row = DatasetService(session).ncm_utrib_for_ncm(ncm)
|
||||
return {"found": row is not None, "utrib_abbreviation": row.utrib_abbreviation if row else None}
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/reference/anp/{cprod_anp}")
|
||||
def reference_anp(cprod_anp: str, identity=Depends(principal)):
|
||||
session, _ = identity
|
||||
try: return {"found": DatasetService(session).anp_code_exists(cprod_anp)}
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/reference/fcp/{uf}")
|
||||
def reference_fcp(uf: str, identity=Depends(principal)):
|
||||
session, _ = identity
|
||||
try:
|
||||
row = DatasetService(session).fcp_for_uf(uf)
|
||||
return {"found": row is not None, "aliq_1": row.aliq_1 if row else None, "observacao": row.observacao if row else None}
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/reference/rtc_cclasstrib/{code}")
|
||||
def reference_rtc_cclasstrib(code: str, identity=Depends(principal)):
|
||||
session, _ = identity
|
||||
try:
|
||||
rows = DatasetService(session).rtc_cclasstrib(code=code)
|
||||
return {"found": bool(rows), "entries": [{"cst": r.cst, "cclasstrib": r.cclasstrib, "name": r.name, "description": r.description, "tipo_aliquota": r.tipo_aliquota, "p_red_ibs": r.p_red_ibs, "p_red_cbs": r.p_red_cbs, "d_ini_vig": r.d_ini_vig, "d_fim_vig": r.d_fim_vig} for r in rows]}
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/reference/rtc_cbs_rate/{year}")
|
||||
def reference_rtc_cbs_rate(year: int, identity=Depends(principal)):
|
||||
session, _ = identity
|
||||
try:
|
||||
row = DatasetService(session).rtc_cbs_rate(year)
|
||||
return {"found": row is not None, "year": year, "rate": row.rate if row else None, "rate_text": row.rate_text if row else None}
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/reference/fiscal_codes/{domain}")
|
||||
def reference_fiscal_codes(domain: str, identity=Depends(principal)):
|
||||
session, _ = identity
|
||||
try:
|
||||
rows = DatasetService(session).fiscal_codes(domain.upper())
|
||||
return {"found": bool(rows), "domain": domain.upper(), "codes": sorted(r.code for r in rows)}
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/context/import")
|
||||
def context_import(payload: dict, identity=Depends(principal)):
|
||||
"""Batch upsert of operator fiscal context: {"company_key": "...", "entries": [{scope, scope_key, key, value, reason}]}. All-or-nothing."""
|
||||
session, agent = identity
|
||||
try:
|
||||
company_key = str(payload.get("company_key") or "").strip()
|
||||
if not company_key:
|
||||
raise HTTPException(400, "company_key is required")
|
||||
entries, errors = [], []
|
||||
for i, raw in enumerate(payload.get("entries") or []):
|
||||
entry, error = validate_entry(raw)
|
||||
(errors.append({"index": i, "error": error, "entry": raw}) if error else entries.append(entry))
|
||||
if errors:
|
||||
raise HTTPException(422, {"errors": errors[:50], "error_count": len(errors)})
|
||||
created = updated = 0
|
||||
for e in entries:
|
||||
row = session.scalar(select(FiscalContextEntry).where(FiscalContextEntry.tenant_id == agent.tenant_id, FiscalContextEntry.company_key == company_key,
|
||||
FiscalContextEntry.scope == e["scope"], FiscalContextEntry.scope_key == e["scope_key"], FiscalContextEntry.key == e["key"]))
|
||||
if row:
|
||||
row.value, row.reason, row.updated_at = e["value"], e["reason"], datetime.now(timezone.utc); updated += 1
|
||||
else:
|
||||
session.add(FiscalContextEntry(tenant_id=agent.tenant_id, company_key=company_key, source="OPERATOR", **e)); created += 1
|
||||
session.commit()
|
||||
return {"created": created, "updated": updated}
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/context")
|
||||
def context_list(company_key: str, identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
rows = session.scalars(select(FiscalContextEntry).where(FiscalContextEntry.tenant_id == agent.tenant_id, FiscalContextEntry.company_key == company_key)).all()
|
||||
return {"company_key": company_key, "entries": [{"scope": r.scope, "scope_key": r.scope_key, "key": r.key, "value": r.value, "source": r.source, "reason": r.reason,
|
||||
"created_at": r.created_at.isoformat() if r.created_at else None, "updated_at": r.updated_at.isoformat() if r.updated_at else None} for r in rows]}
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/agents/heartbeat")
|
||||
def heartbeat(payload: dict, identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
agent.last_seen_at = datetime.now(timezone.utc)
|
||||
agent.last_status = payload.get("status")
|
||||
if payload.get("schema_fingerprint"):
|
||||
agent.schema_fingerprint = payload["schema_fingerprint"]
|
||||
if payload.get("agent_version"):
|
||||
agent.agent_version = payload["agent_version"]
|
||||
session.add(agent); session.commit()
|
||||
logger.info("heartbeat agent_id=%s status=%s last_job_id=%s", agent.id, agent.last_status, payload.get("last_job_id"))
|
||||
return {"agent_id": agent.id, "last_seen_at": agent.last_seen_at.isoformat(), "enabled": agent.enabled}
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/agents/me")
|
||||
def agent_status(identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
last_seen = agent.last_seen_at
|
||||
if last_seen is not None and last_seen.tzinfo is None:
|
||||
last_seen = last_seen.replace(tzinfo=timezone.utc) # SQLite (tests) drops tz info
|
||||
online = last_seen is not None and (datetime.now(timezone.utc) - last_seen).total_seconds() <= settings.agent_online_threshold_seconds
|
||||
return {
|
||||
"agent_id": agent.id, "company_id": agent.tenant_id, "name": agent.name, "enabled": agent.enabled,
|
||||
"last_seen_at": agent.last_seen_at.isoformat() if agent.last_seen_at else None,
|
||||
"last_status": agent.last_status, "schema_fingerprint": agent.schema_fingerprint,
|
||||
"agent_version": agent.agent_version, "status": "ONLINE" if online else "OFFLINE",
|
||||
}
|
||||
finally: session.close()
|
||||
|
||||
def _job_view(job):
|
||||
return {
|
||||
"job_id": job.id, "agent_id": job.agent_id, "kind": job.kind, "status": job.status,
|
||||
"payload": job.payload, "created_at": job.created_at.isoformat() if job.created_at else None,
|
||||
"started_at": job.started_at.isoformat() if job.started_at else None,
|
||||
"finished_at": job.finished_at.isoformat() if job.finished_at else None,
|
||||
"attempts": job.attempts, "result_summary": job.result_summary, "error": job.error,
|
||||
}
|
||||
|
||||
@routes.post("/jobs")
|
||||
def create_job_route(payload: dict, identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
kind = payload.get("kind")
|
||||
if kind not in ("REPROCESS_ALL", "REPROCESS_PRODUCTS"):
|
||||
raise HTTPException(400, "kind must be REPROCESS_ALL or REPROCESS_PRODUCTS")
|
||||
job_payload = payload.get("payload", {})
|
||||
if kind == "REPROCESS_PRODUCTS" and not job_payload.get("codigos"):
|
||||
raise HTTPException(400, "REPROCESS_PRODUCTS requires payload.codigos")
|
||||
job = create_job(session, agent, kind, job_payload)
|
||||
logger.info("job created job_id=%s agent_id=%s kind=%s", job.id, agent.id, kind)
|
||||
return _job_view(job)
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/jobs/next")
|
||||
def next_job(identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
job = claim_next_job(session, agent)
|
||||
if job is None: return None
|
||||
logger.info("job claimed job_id=%s agent_id=%s kind=%s", job.id, agent.id, job.kind)
|
||||
return _job_view(job)
|
||||
finally: session.close()
|
||||
|
||||
@routes.get("/jobs/{job_id}")
|
||||
def get_job(job_id: int, identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
job = session.get(Job, job_id)
|
||||
if job is None or job.agent_id != agent.id: raise HTTPException(404, "job not found")
|
||||
return _job_view(job)
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/jobs/{job_id}/complete")
|
||||
def complete_job_route(job_id: int, payload: dict, identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
job = session.get(Job, job_id)
|
||||
if job is None or job.agent_id != agent.id: raise HTTPException(404, "job not found")
|
||||
if not complete_job(session, job, payload.get("result_summary", {})): raise HTTPException(409, "job is not RUNNING")
|
||||
logger.info("job completed job_id=%s agent_id=%s result=%s", job_id, agent.id, payload.get("result_summary", {}))
|
||||
return _job_view(session.get(Job, job_id))
|
||||
finally: session.close()
|
||||
|
||||
@routes.post("/jobs/{job_id}/fail")
|
||||
def fail_job_route(job_id: int, payload: dict, identity=Depends(principal)):
|
||||
session, agent = identity
|
||||
try:
|
||||
job = session.get(Job, job_id)
|
||||
if job is None or job.agent_id != agent.id: raise HTTPException(404, "job not found")
|
||||
if not fail_job(session, job, payload.get("error", "unknown error")): raise HTTPException(409, "job is not RUNNING")
|
||||
logger.warning("job failed job_id=%s agent_id=%s error=%s", job_id, agent.id, payload.get("error", "unknown error"))
|
||||
return _job_view(session.get(Job, job_id))
|
||||
finally: session.close()
|
||||
|
||||
return routes
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Bearer authentication storing only SHA-256 token digests."""
|
||||
import hashlib, hmac
|
||||
from sqlalchemy import select
|
||||
from .models import Agent, Tenant
|
||||
|
||||
def token_hash(token): return hashlib.sha256(token.encode()).hexdigest()
|
||||
def bootstrap(session, token):
|
||||
digest = token_hash(token); agent = session.scalar(select(Agent).where(Agent.token_hash == digest))
|
||||
if agent: return agent
|
||||
tenant = session.scalar(select(Tenant).where(Tenant.name == "development")) or Tenant(name="development")
|
||||
session.add(tenant); session.flush(); agent = Agent(tenant_id=tenant.id, name="development-agent", token_hash=digest); session.add(agent); session.commit(); return agent
|
||||
def authenticate(session, token):
|
||||
digest = token_hash(token)
|
||||
for agent in session.scalars(select(Agent)).all():
|
||||
if hmac.compare_digest(agent.token_hash, digest):
|
||||
return agent if agent.enabled else None # a disabled (revoked) Agent must not authenticate
|
||||
return None
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Conservative deterministic NCM classification."""
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
ENGINE_VERSION = "0.3.0"
|
||||
DECISION_VERSION = "mission-04c-ipi-ex-split"
|
||||
|
||||
_NT_MARKERS = {"NT", "N/T", "NAO TRIBUTADO", "NÃO TRIBUTADO"}
|
||||
_ZERO_MARKERS = {"0", "0,00", "0.00"}
|
||||
|
||||
|
||||
def classify_tipi_rate(entry):
|
||||
"""Classify why a TIPI base entry has no usable numeric rate; never guess a value."""
|
||||
if entry.ex:
|
||||
return "EX_SPECIFIC"
|
||||
text = entry.ipi_rate_text
|
||||
if text is None:
|
||||
return "SOURCE_BLANK"
|
||||
normalized = text.strip().upper()
|
||||
if normalized in _NT_MARKERS:
|
||||
return "NT_OR_NON_TAXED"
|
||||
if entry.ipi_rate == 0 or normalized in _ZERO_MARKERS:
|
||||
return "EXPLICIT_ZERO"
|
||||
return "OTHER"
|
||||
|
||||
def normalize_code(value): return "".join(char for char in str(value or "") if char.isdigit())
|
||||
def normalize_description(value): return " ".join(re.sub(r"[^A-Z0-9]+", " ", unicodedata.normalize("NFKD", str(value or "")).encode("ascii", "ignore").decode().upper()).split())
|
||||
|
||||
# Portuguese function words with no fiscal-descriptive content. Without this, a
|
||||
# product description sharing only "DE"/"E"/"COM" with an official CEST text
|
||||
# scores a false-positive nonzero overlap that has nothing to do with the product.
|
||||
_STOPWORDS = {
|
||||
"DE", "DA", "DO", "DAS", "DOS", "E", "OU", "A", "O", "AS", "OS",
|
||||
"EM", "NO", "NA", "NOS", "NAS", "PARA", "POR", "COM", "SEM",
|
||||
"MESMO", "OUTRO", "OUTRA", "OUTROS", "OUTRAS", "QUE",
|
||||
}
|
||||
|
||||
|
||||
def description_tokens(value):
|
||||
return set(normalize_description(value).split()) - _STOPWORDS
|
||||
|
||||
class Classifier:
|
||||
def __init__(self, repository): self.repository = repository
|
||||
def classify(self, product):
|
||||
entries = self.repository.active_entries(); code = normalize_code(product.get("current_ncm"))
|
||||
exact = next((entry for entry in entries if entry.code == code and len(code) == 8), None)
|
||||
legacy = self.repository.legacy_ncm(code) if len(code) == 8 and hasattr(self.repository, "legacy_ncm") else None
|
||||
if exact:
|
||||
status, candidate = "EXACT_NCM", exact
|
||||
elif legacy:
|
||||
status, candidate = "LEGACY_NCM", None
|
||||
else:
|
||||
status, candidate = ("INVALID_NCM", None) if code else self._candidate(entries, product.get("description"))
|
||||
matched_code = candidate.code if candidate else legacy["code"] if legacy else None
|
||||
cests_available = matched_code and hasattr(self.repository, "cest_for_ncm")
|
||||
cests = self.repository.cest_for_ncm(matched_code) if cests_available else None
|
||||
cests = list({item.code: item for item in cests or []}.values())
|
||||
scores = {item.code: self._description_score(product.get("description"), item.description) for item in cests}
|
||||
if legacy:
|
||||
cest_status = "LEGACY_NCM"
|
||||
elif not cests:
|
||||
cest_status = "NO_CEST_MAPPING_FOUND"
|
||||
elif max(scores.values()) < .5:
|
||||
cest_status = "CEST_DESCRIPTION_MISMATCH"
|
||||
elif len(cests) == 1:
|
||||
cest_status = "CEST_CONFIRMED"
|
||||
else:
|
||||
cest_status = "CEST_CANDIDATES"
|
||||
resolved_cest = cests[0].code if cest_status == "CEST_CONFIRMED" else None
|
||||
ipi = self._resolve_ipi(matched_code, legacy)
|
||||
mva = self._resolve_mva(matched_code, legacy, resolved_cest)
|
||||
return {"status": status, "cest_status": cest_status, "ipi_status": ipi["status"], "ba_st_mva_status": mva["status"], "classification": {"ncm": matched_code, "ncm_description": candidate.description if candidate else None, "cest": [item.code for item in cests] if cests_available else None, "cest_candidates": [{"code": item.code, "official_description": item.description, "description_score": scores[item.code]} for item in cests], "cfop": None, "cst": None, "csosn": None, "pis_cofins": None, "ipi": ipi["resolved"], "ipi_candidates": ipi["candidates"], "ba_st_mva": mva["resolved"], "ba_st_mva_candidates": mva["candidates"], "ibs_cbs": None}, "legacy_ncm": legacy, "reason_codes": ["FISCAL_DETERMINATION_NOT_CONFIGURED"], "can_auto_apply": False, "engine_version": ENGINE_VERSION, "decision_version": DECISION_VERSION, "dataset_versions": self.repository.dataset_versions()}
|
||||
|
||||
def _resolve_ipi(self, matched_code, legacy):
|
||||
if legacy:
|
||||
return {"status": "LEGACY_NCM", "resolved": None, "candidates": []}
|
||||
if not matched_code or not hasattr(self.repository, "tipi_for_ncm"):
|
||||
return {"status": "NO_TIPI_MAPPING_FOUND", "resolved": None, "candidates": []}
|
||||
entries = self.repository.tipi_for_ncm(matched_code)
|
||||
if not entries:
|
||||
return {"status": "NO_TIPI_MAPPING_FOUND", "resolved": None, "candidates": []}
|
||||
base = next((entry for entry in entries if not entry.ex), None)
|
||||
ex_variants = [entry for entry in entries if entry.ex]
|
||||
candidates = [{"ncm": entry.ncm, "ex": entry.ex, "description": entry.description, "rate": entry.ipi_rate_text} for entry in ex_variants]
|
||||
if base:
|
||||
if base.ipi_rate is not None:
|
||||
rate_kind = "NUMERIC"
|
||||
elif base.ipi_rate_text is not None:
|
||||
rate_kind = "NON_NUMERIC"
|
||||
else:
|
||||
rate_kind = "BLANK"
|
||||
resolved = {
|
||||
"ncm": base.ncm,
|
||||
"ex": base.ex,
|
||||
"description": base.description,
|
||||
"rate": base.ipi_rate_text,
|
||||
"rate_numeric": base.ipi_rate,
|
||||
"rate_kind": rate_kind,
|
||||
}
|
||||
if rate_kind != "NUMERIC":
|
||||
resolved["rate_classification"] = classify_tipi_rate(base)
|
||||
status = "IPI_CANDIDATES" if ex_variants else "IPI_RESOLVED"
|
||||
return {"status": status, "resolved": resolved, "candidates": candidates}
|
||||
return {"status": "IPI_CANDIDATES", "resolved": None, "candidates": candidates}
|
||||
|
||||
def _resolve_mva(self, matched_code, legacy, resolved_cest):
|
||||
if legacy:
|
||||
return {"status": "LEGACY_NCM", "resolved": None, "candidates": []}
|
||||
if not matched_code or not hasattr(self.repository, "ba_st_mva_for_ncm"):
|
||||
return {"status": "NO_MVA_MAPPING_FOUND", "resolved": None, "candidates": []}
|
||||
entries = self.repository.ba_st_mva_for_ncm(matched_code)
|
||||
if not entries:
|
||||
return {"status": "NO_MVA_MAPPING_FOUND", "resolved": None, "candidates": []}
|
||||
candidates = [{"item": entry.item, "cest": entry.cest, "mva_original": entry.mva_original, "mva_adjusted": entry.mva_adjusted, "description": entry.description} for entry in entries]
|
||||
if len(entries) == 1:
|
||||
return {"status": "MVA_RESOLVED", "resolved": candidates[0], "candidates": candidates}
|
||||
if resolved_cest:
|
||||
by_cest = [item for item in candidates if item["cest"] == resolved_cest]
|
||||
if len(by_cest) == 1:
|
||||
return {"status": "MVA_RESOLVED", "resolved": by_cest[0], "candidates": candidates}
|
||||
return {"status": "MVA_CANDIDATES", "resolved": None, "candidates": candidates}
|
||||
def _candidate(self, entries, description):
|
||||
tokens = set(normalize_description(description).split())
|
||||
scores = sorted(((len(tokens & set(normalize_description(item.description).split())) / len(tokens or {"_"}), item) for item in entries), reverse=True, key=lambda row: row[0])
|
||||
if not scores or scores[0][0] == 0: return "NOT_FOUND", None
|
||||
return ("HIGH_CONFIDENCE_CANDIDATE", scores[0][1]) if scores[0][0] >= .90 and (len(scores) == 1 or scores[0][0] - scores[1][0] >= .15) else ("AMBIGUOUS", None)
|
||||
|
||||
def _description_score(self, product_description, official_description):
|
||||
product_tokens = description_tokens(product_description)
|
||||
official_tokens = description_tokens(official_description)
|
||||
return len(product_tokens & official_tokens) / len(product_tokens or {"_"})
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Environment-backed Fiscal Hub configuration."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
import os
|
||||
from urllib.parse import quote
|
||||
|
||||
|
||||
def _database_url() -> str:
|
||||
configured = os.getenv("DATABASE_URL")
|
||||
if configured:
|
||||
return configured
|
||||
password = quote(os.getenv("POSTGRES_PASSWORD", "gti"), safe="")
|
||||
return f"postgresql+psycopg://gti:{password}@postgres:5432/gti_fiscal"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
database_url: str = _database_url()
|
||||
hub_port: int = int(os.getenv("HUB_PORT", "8000"))
|
||||
# AGENT_DEV_TOKEN auto-bootstraps an agent from a fixed dev token on every request.
|
||||
# Leave unset in any environment other than local development — production Agents
|
||||
# must be created via scripts/register_agent.py, whose token is never derived from
|
||||
# this setting.
|
||||
agent_dev_token: str = os.getenv("AGENT_DEV_TOKEN", "")
|
||||
environment: str = os.getenv("APP_ENV", "development")
|
||||
log_level: str = os.getenv("LOG_LEVEL", "INFO")
|
||||
base_url: str = os.getenv("BASE_URL", "http://localhost:8000")
|
||||
# An Agent is considered ONLINE if a heartbeat arrived within this many seconds.
|
||||
agent_online_threshold_seconds: int = int(os.getenv("AGENT_ONLINE_THRESHOLD_SECONDS", "120"))
|
||||
ncm_dataset_url: str = os.getenv("NCM_DATASET_URL", "https://portalunico.siscomex.gov.br/classif/api/publico/nomenclatura/download/json")
|
||||
cest_dataset_url: str = os.getenv("CEST_DATASET_URL", "https://www.confaz.fazenda.gov.br/legislacao/convenios/2018/CV142_18")
|
||||
dataset_storage_path: str = os.getenv("DATASET_STORAGE_PATH", "./data/datasets")
|
||||
http_timeout_seconds: float = float(os.getenv("HTTP_TIMEOUT_SECONDS", "30"))
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Validation of operator fiscal-context entries (facts the ERP does not know). Pure; shared by the Hub API, the importer CLI and the resolver."""
|
||||
import json
|
||||
|
||||
SCOPES = ("COMPANY", "GROUP", "NCM", "PRODUCT")
|
||||
ENUMS = {
|
||||
"product_role": {"RESALE", "OWN_PRODUCTION", "INPUT", "SERVICE"},
|
||||
"st_status": {"SUBJECT", "NOT_SUBJECT"},
|
||||
"st_withheld": {"Y", "N"},
|
||||
"icms_treatment": {"TAXED", "EXEMPT", "NOT_TAXED", "ST_WITHHELD", "SUSPENDED", "DEFERRED"},
|
||||
"entry_scope": {"INTRA", "INTER"},
|
||||
"rounding": {"A", "T"},
|
||||
"piscofins": {"N", "S"},
|
||||
"incidence": {"NENHUMA"},
|
||||
"origin": {str(i) for i in range(9)},
|
||||
"store_ipi_profile": {"POPULATE_REFERENCE_RATE", "LEAVE_NOT_APPLICABLE"},
|
||||
}
|
||||
PROFILE_KEYS = {"pis_in_cst", "pis_out_cst", "pis_rate", "cofins_in_cst", "cofins_out_cst", "cofins_rate"}
|
||||
FREE_KEYS = {"ncm", "cest", "ex_tipi", "icms_rate", "credit_rate", "cst_ipi", "service_cfop", "pis_cofins_profile"}
|
||||
ALLOWED_KEYS = set(ENUMS) | FREE_KEYS
|
||||
PRODUCT_ONLY = {"ncm"}
|
||||
COMPANY_ONLY = {"store_ipi_profile"}
|
||||
NCM_KEYS = {"cest", "ex_tipi"} # tied to the NCM tables: answered per NCM or per product # an NCM answer is always about one product
|
||||
|
||||
|
||||
def validate_entry(entry):
|
||||
"""Returns (normalized_entry, error). error is None when valid."""
|
||||
scope = str(entry.get("scope", "")).strip().upper()
|
||||
key = str(entry.get("key", "")).strip()
|
||||
value = entry.get("value")
|
||||
value = "" if value is None else str(value).strip()
|
||||
scope_key = str(entry.get("scope_key") or "").strip()
|
||||
if scope not in SCOPES:
|
||||
return None, f"scope must be one of {SCOPES}"
|
||||
if key not in ALLOWED_KEYS:
|
||||
return None, f"unknown key {key!r}"
|
||||
if scope != "COMPANY" and not scope_key:
|
||||
return None, "scope_key is required for GROUP/NCM/PRODUCT scope"
|
||||
if key in COMPANY_ONLY and scope != "COMPANY":
|
||||
return None, f"{key} is a COMPANY-scope answer"
|
||||
if key in PRODUCT_ONLY and scope != "PRODUCT":
|
||||
return None, f"{key} is a PRODUCT-scope answer"
|
||||
if key in NCM_KEYS and scope not in ("PRODUCT", "NCM"):
|
||||
return None, f"{key} can only be answered for a PRODUCT or an NCM"
|
||||
if not value:
|
||||
return None, "value is empty"
|
||||
if key in ENUMS and value.upper() not in ENUMS[key] and value not in ENUMS[key]:
|
||||
return None, f"{key} must be one of {sorted(ENUMS[key])}"
|
||||
if key in ENUMS:
|
||||
value = value.upper() if key != "origin" else value
|
||||
if key == "ncm" and not (value.isdigit() and len(value) == 8):
|
||||
return None, "ncm must be 8 digits"
|
||||
if key == "cest" and value.upper() != "NONE" and not (value.replace(".", "").isdigit() and len(value.replace(".", "")) == 7):
|
||||
return None, "cest must be 7 digits or NONE"
|
||||
if key in ("icms_rate", "credit_rate"):
|
||||
try:
|
||||
float(value.replace(",", "."))
|
||||
except ValueError:
|
||||
return None, f"{key} must be numeric"
|
||||
if key == "pis_cofins_profile":
|
||||
try:
|
||||
prof = json.loads(value)
|
||||
except ValueError:
|
||||
return None, "pis_cofins_profile must be JSON"
|
||||
if not isinstance(prof, dict) or not prof or set(prof) - PROFILE_KEYS:
|
||||
return None, f"pis_cofins_profile keys must be a non-empty subset of {sorted(PROFILE_KEYS)}"
|
||||
return {"scope": scope, "scope_key": scope_key if scope != "COMPANY" else "", "key": key, "value": value, "reason": (entry.get("reason") or "").strip() or None}, None
|
||||
@@ -0,0 +1 @@
|
||||
"""Official fiscal dataset importers."""
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Fail-closed text parser for the official RICMS/BA Anexo 1 PDF."""
|
||||
import re
|
||||
|
||||
_ITEM = re.compile(r"^\s*(\d{1,2}(?:\.\d+)+)\s+(?:(\d{2}\.\d{3}\.\d{2})\s+)?(.*)$")
|
||||
_NCM = re.compile(r"\b(?:\d{4}(?:\.\d{1,4}){0,3}|\d{2}(?:\.\d{1,4}){1,3})\b")
|
||||
_MVA = re.compile(r"\d+(?:,\d+)?%")
|
||||
|
||||
def parse_page(text, page):
|
||||
records, rejected = [], []
|
||||
lines = [" ".join(line.split()) for line in text.splitlines()]
|
||||
index = 0
|
||||
while index < len(lines):
|
||||
match = _ITEM.match(lines[index])
|
||||
if not match:
|
||||
index += 1; continue
|
||||
item, cest, content = match.groups(); block = [content]; index += 1
|
||||
while index < len(lines) and not _ITEM.match(lines[index]) and not lines[index].startswith("ITEM CEST"):
|
||||
block.append(lines[index]); index += 1
|
||||
joined = " ".join(block)
|
||||
ncm = _NCM.findall(joined)
|
||||
if not cest and not ncm:
|
||||
rejected.append({"item": item, "source_page": page, "reason": "NO_CEST_OR_NCM"}); continue
|
||||
mvas = _MVA.findall(joined)
|
||||
description = joined
|
||||
if cest: description = description.replace(cest, "")
|
||||
for code in ncm: description = description.replace(code, "")
|
||||
for mva in mvas: description = description.replace(mva, "")
|
||||
records.append({"item": item, "cest": cest, "ncm_sh": ncm or None, "description": " ".join(description.split()) or None, "mva_original": mvas[-1] if mvas else None, "mva_adjusted": mvas[0] if mvas else None, "agreement_info": None, "source_page": page})
|
||||
return records, rejected
|
||||
@@ -0,0 +1,287 @@
|
||||
"""Parsers for the official NF-e/RTC spreadsheets the operator placed in
|
||||
sources/raw/bootstrap_v1/ after the Portal NF-e/SPED downloads were blocked
|
||||
in earlier missions. Each parser validates the workbook's own header/sheet
|
||||
structure before trusting it — never trusts the filename alone.
|
||||
|
||||
Domains: CFOP, FCP, ANP (cProdANP), NCM x uTrib, RTC biofuel index
|
||||
(pBioObrigatorio), RTC cCredPres.
|
||||
"""
|
||||
from dataclasses import dataclass, field
|
||||
from hashlib import sha256
|
||||
from io import BytesIO
|
||||
|
||||
|
||||
def _digits(value):
|
||||
return "".join(c for c in str(value or "") if c.isdigit())
|
||||
|
||||
|
||||
def _iso(value):
|
||||
if value is None:
|
||||
return None
|
||||
return value.isoformat() if hasattr(value, "isoformat") else str(value)
|
||||
|
||||
|
||||
def _load(raw_bytes):
|
||||
import openpyxl
|
||||
return openpyxl.load_workbook(BytesIO(raw_bytes), read_only=True, data_only=True)
|
||||
|
||||
|
||||
def _rows(ws, header_len):
|
||||
rows = ws.iter_rows(values_only=True)
|
||||
header = next(rows)
|
||||
return header[:header_len], rows
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Payload:
|
||||
sha256: str
|
||||
entries: tuple
|
||||
parser_version: str
|
||||
extra_entries: tuple = ()
|
||||
|
||||
|
||||
# --- CFOP -------------------------------------------------------------
|
||||
_CFOP_EXPECTED_HEADER = ("CFOP", "Início de vigência", "Fim de vigência", "indNFe")
|
||||
|
||||
|
||||
def parse_cfop(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
if "CFOP" not in wb.sheetnames:
|
||||
raise ValueError(f"expected a 'CFOP' sheet, found {wb.sheetnames}")
|
||||
header, rows = _rows(wb["CFOP"], 14)
|
||||
if _digits("".join(str(h) for h in header[:1])) or "CFOP" not in str(header[0]):
|
||||
raise ValueError(f"unexpected CFOP header: {header}")
|
||||
entries = []
|
||||
for row in rows:
|
||||
if row[0] is None:
|
||||
continue
|
||||
code = str(int(row[0])) if isinstance(row[0], (int, float)) else str(row[0]).strip()
|
||||
entries.append({
|
||||
"code": code, "effective_from": _iso(row[1]), "effective_to": _iso(row[2]),
|
||||
"ind_nfe": row[3], "ind_comunica": row[4], "ind_transp": row[5], "ind_devol": row[6],
|
||||
"ind_retor": row[7], "ind_anula": row[8], "ind_remes": row[9], "ind_comb": row[10],
|
||||
"ind_exc_ibscbs": row[11], "title": row[12] or "", "description": row[13],
|
||||
})
|
||||
if len(entries) < 100:
|
||||
raise ValueError(f"expected at least 100 CFOP codes, found {len(entries)}")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "cfop-xlsx-v1")
|
||||
|
||||
|
||||
# --- FCP ----------------------------------------------------------------
|
||||
def parse_fcp(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
if "FCP" not in wb.sheetnames:
|
||||
raise ValueError(f"expected an 'FCP' sheet, found {wb.sheetnames}")
|
||||
header, rows = _rows(wb["FCP"], 8)
|
||||
if "UF" not in [str(h) for h in header]:
|
||||
raise ValueError(f"unexpected FCP header: {header}")
|
||||
entries = []
|
||||
for row in rows:
|
||||
if not row[1]:
|
||||
continue
|
||||
entries.append({
|
||||
"uf_code": str(row[0]), "uf": str(row[1]).strip(), "uf_name": str(row[2] or "").strip(),
|
||||
"aliq_1": row[3], "aliq_2": row[4], "aliq_3": row[5], "observacao": row[7],
|
||||
})
|
||||
if len(entries) < 20:
|
||||
raise ValueError(f"expected at least 20 UF rows (26 states + DF), found {len(entries)}")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "fcp-xlsx-v1")
|
||||
|
||||
|
||||
# --- ANP ------------------------------------------------------------------
|
||||
def parse_anp(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
if "cProdANP" not in wb.sheetnames:
|
||||
raise ValueError(f"expected a 'cProdANP' sheet, found {wb.sheetnames}")
|
||||
ws = wb["cProdANP"]
|
||||
rows = ws.iter_rows(values_only=True)
|
||||
header = next(rows)
|
||||
if "cProdANP" not in str(header[0]):
|
||||
raise ValueError(f"unexpected ANP header: {header}")
|
||||
entries = []
|
||||
seen = set()
|
||||
for row in rows:
|
||||
if row[0] is None:
|
||||
continue
|
||||
code = str(row[0]).strip()
|
||||
if code in seen:
|
||||
continue
|
||||
seen.add(code)
|
||||
entries.append({"cprod_anp": code})
|
||||
if len(entries) < 100:
|
||||
raise ValueError(f"expected at least 100 cProdANP codes, found {len(entries)}")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "anp-xlsx-v1")
|
||||
|
||||
|
||||
# --- NCM x uTrib ------------------------------------------------------
|
||||
def parse_ncm_utrib(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
sheet_name = next((s for s in wb.sheetnames if "uTrib" in s or "utrib" in s.lower()), None)
|
||||
if sheet_name is None:
|
||||
raise ValueError(f"expected a sheet with 'uTrib' in its name, found {wb.sheetnames}")
|
||||
ws = wb[sheet_name]
|
||||
rows = ws.iter_rows(values_only=True)
|
||||
header = next(rows)
|
||||
if "NCM" not in str(header[0]):
|
||||
raise ValueError(f"unexpected NCM x uTrib header: {header}")
|
||||
entries = []
|
||||
for row in rows:
|
||||
if row[0] is None:
|
||||
continue
|
||||
entries.append({
|
||||
"ncm": _digits(row[0]), "effective_from": _iso(row[1]), "effective_to": _iso(row[2]),
|
||||
"utrib_abbreviation": row[3], "utrib_description": row[4],
|
||||
})
|
||||
if len(entries) < 1000:
|
||||
raise ValueError(f"expected at least 1000 NCM x uTrib rows, found {len(entries)}")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "ncm-utrib-xlsx-v1")
|
||||
|
||||
|
||||
# --- RTC biofuel index (pBioObrigatorio) --------------------------------
|
||||
def parse_rtc_biofuel_index(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
current_sheet = next((s for s in wb.sheetnames if "Historico" not in s and "pBio" in s), None)
|
||||
historical_sheet = next((s for s in wb.sheetnames if "Historico" in s), None)
|
||||
if current_sheet is None:
|
||||
raise ValueError(f"expected a 'pBioObrigatorio' sheet, found {wb.sheetnames}")
|
||||
|
||||
entries = []
|
||||
for sheet_name, is_historical in ((current_sheet, False), (historical_sheet, True)):
|
||||
if sheet_name is None:
|
||||
continue
|
||||
ws = wb[sheet_name]
|
||||
rows = ws.iter_rows(values_only=True)
|
||||
header = next(rows)
|
||||
if "cProdANP" not in str(header[0]):
|
||||
raise ValueError(f"unexpected biofuel index header in {sheet_name}: {header}")
|
||||
last_code, last_desc = None, None
|
||||
for row in rows:
|
||||
if row[0] is None and row[1] is None and row[2] is None:
|
||||
continue
|
||||
code = str(row[0]).strip() if row[0] is not None else last_code
|
||||
desc = row[1] if row[1] is not None else last_desc
|
||||
last_code, last_desc = code, desc
|
||||
entries.append({
|
||||
"cprod_anp": code, "description": desc, "p_bio_percent": row[2],
|
||||
"effective_from": _iso(row[3]), "effective_to": _iso(row[4]),
|
||||
"is_historical": is_historical,
|
||||
})
|
||||
if len(entries) < 5:
|
||||
raise ValueError(f"expected at least 5 biofuel index rows, found {len(entries)}")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "rtc-biofuel-index-xlsx-v1")
|
||||
|
||||
|
||||
# --- RTC cCredPres --------------------------------------------------------
|
||||
def parse_rtc_cred_pres(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
if "cCredPres" not in wb.sheetnames:
|
||||
raise ValueError(f"expected a 'cCredPres' sheet, found {wb.sheetnames}")
|
||||
header, rows = _rows(wb["cCredPres"], 18)
|
||||
if "cCredPres" not in str(header[0]):
|
||||
raise ValueError(f"unexpected cCredPres header: {header}")
|
||||
entries = []
|
||||
for row in rows:
|
||||
if row[0] is None:
|
||||
continue
|
||||
entries.append({
|
||||
"code": str(int(row[0])) if isinstance(row[0], (int, float)) else str(row[0]).strip(),
|
||||
"description": row[1] or "", "legal_basis": row[2],
|
||||
"apropria_via_nf": bool(row[3]) if row[3] is not None else None,
|
||||
"apropria_via_evento": bool(row[4]) if row[4] is not None else None,
|
||||
"ind_deduz_cred_pres": row[5], "ind_g_cbs_cred_pres": row[6], "ind_g_ibs_cred_pres": row[7],
|
||||
"aliquota_cbs_text": row[8], "aliquota_ibs_text": row[9],
|
||||
"p_aliq_cred_pres_cbs": row[10], "p_aliq_cred_pres_ibs": row[11], "p_red_transicao_ibs": row[12],
|
||||
"cclass_nota_referenciada": str(row[13]) if row[13] is not None else None,
|
||||
"d_ini_vig_cbs": _iso(row[14]), "d_fim_vig_cbs": _iso(row[15]),
|
||||
"d_ini_vig_ibs": _iso(row[16]), "d_fim_vig_ibs": _iso(row[17]),
|
||||
})
|
||||
if len(entries) < 1:
|
||||
raise ValueError("expected at least 1 cCredPres row, found 0")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "rtc-cred-pres-xlsx-v1")
|
||||
|
||||
|
||||
def _txt(value):
|
||||
return None if value is None else str(value).strip()
|
||||
|
||||
|
||||
def _code(value, width):
|
||||
text = _txt(value) or ""
|
||||
return text.zfill(width) if text.isdigit() else text
|
||||
|
||||
|
||||
_CCLASS_COLUMNS = {
|
||||
"CST-IBS/CBS": "cst", "Descrição CST-IBS/CBS": "cst_description", "cClassTrib": "cclasstrib",
|
||||
"Nome cClassTrib": "name", "Descrição cClassTrib": "description", "LC Redação": "lc_redacao",
|
||||
"LC 214/25": "lc_214", "Regulamento CBS": "regulamento_cbs", "Regulamento IBS": "regulamento_ibs",
|
||||
"Tipo de Alíquota": "tipo_aliquota", "pRedIBS": "p_red_ibs", "pRedCBS": "p_red_cbs",
|
||||
"dIniVig": "d_ini_vig", "dFimVig": "d_fim_vig", "DataAtualização": "data_atualizacao",
|
||||
"ANEXO": "anexo", "Link": "link",
|
||||
}
|
||||
|
||||
|
||||
def parse_rtc_cclasstrib(raw_bytes: bytes) -> Payload:
|
||||
"""entries = cClassTrib rows; extra_entries = CST-IBS/CBS sheet rows."""
|
||||
wb = _load(raw_bytes)
|
||||
cst_sheet = next((n for n in wb.sheetnames if n.startswith("CST")), None)
|
||||
cclass_sheet = next((n for n in wb.sheetnames if n.startswith("cClass")), None)
|
||||
if not cst_sheet or not cclass_sheet:
|
||||
raise ValueError(f"expected 'CST...' and 'cClass...' sheets, found {wb.sheetnames}")
|
||||
|
||||
rows = wb[cst_sheet].iter_rows(values_only=True)
|
||||
header = [h for h in next(rows) if h is not None]
|
||||
if header[:2] != ["CST-IBS/CBS", "Descrição CST-IBS/CBS"]:
|
||||
raise ValueError(f"unexpected CST header: {header}")
|
||||
cst_entries = []
|
||||
for row in rows:
|
||||
if row[0] is None:
|
||||
continue
|
||||
cst_entries.append({
|
||||
"cst": _code(row[0], 3), "description": _txt(row[1]) or "",
|
||||
"indicators": {h.replace(" ", ""): _txt(v) for h, v in zip(header[2:], row[2:])},
|
||||
})
|
||||
|
||||
rows = wb[cclass_sheet].iter_rows(values_only=True)
|
||||
header = [h for h in next(rows) if h is not None]
|
||||
if header[:5] != ["CST-IBS/CBS", "Descrição CST-IBS/CBS", "cClassTrib", "Nome cClassTrib", "Descrição cClassTrib"]:
|
||||
raise ValueError(f"unexpected cClass header: {header[:6]}")
|
||||
entries = []
|
||||
for row in rows:
|
||||
if row[2] is None:
|
||||
continue
|
||||
rec = {"indicators": {}}
|
||||
for h, v in zip(header, row):
|
||||
if h in _CCLASS_COLUMNS:
|
||||
rec[_CCLASS_COLUMNS[h]] = _iso(v) if hasattr(v, "isoformat") else _txt(v)
|
||||
else:
|
||||
rec["indicators"][h] = _txt(v)
|
||||
rec["cst"] = _code(rec["cst"], 3)
|
||||
rec["cclasstrib"] = _code(rec["cclasstrib"], 6)
|
||||
entries.append(rec)
|
||||
if len(entries) < 100 or len(cst_entries) < 10:
|
||||
raise ValueError(f"too few rows: cClass={len(entries)} CST={len(cst_entries)}")
|
||||
orphans = {e["cst"] for e in entries} - {e["cst"] for e in cst_entries}
|
||||
if orphans:
|
||||
raise ValueError(f"cClassTrib rows reference unknown CST codes: {sorted(orphans)}")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "rtc-cclasstrib-xlsx-v1", tuple(cst_entries))
|
||||
|
||||
|
||||
def parse_rtc_cbs_rates(raw_bytes: bytes) -> Payload:
|
||||
wb = _load(raw_bytes)
|
||||
rows = list(wb.worksheets[0].iter_rows(values_only=True))
|
||||
if len(rows) < 3 or "CBS" not in str(rows[0][0]).upper() or [str(c).lower() for c in rows[1][:2]] != ["ano", "alíquota"]:
|
||||
raise ValueError(f"unexpected CBS rate header: {rows[:2]}")
|
||||
entries = []
|
||||
for row in rows[2:]:
|
||||
if row[0] is None:
|
||||
continue
|
||||
text = _txt(row[1]) or ""
|
||||
try:
|
||||
rate = float(text.replace(",", "."))
|
||||
except ValueError:
|
||||
rate = None
|
||||
entries.append({"year": int(float(str(row[0]))), "rate_text": text, "rate": rate})
|
||||
if not entries or all(e["rate"] is None for e in entries):
|
||||
raise ValueError("no numeric CBS rate found")
|
||||
if len({e["year"] for e in entries}) != len(entries):
|
||||
raise ValueError("duplicate CBS rate year")
|
||||
return Payload(sha256(raw_bytes).hexdigest(), tuple(entries), "rtc-cbs-rates-xlsx-v1")
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Fail-closed parser for CONFAZ Convenio ICMS 142/18 CEST HTML."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
from html.parser import HTMLParser
|
||||
import re
|
||||
|
||||
|
||||
_ANNEX_NUMBERS = set(range(2, 27))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CestEntry:
|
||||
code: str
|
||||
description: str
|
||||
annex: str
|
||||
ncm_codes: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CestPayload:
|
||||
version: str
|
||||
sha256: str
|
||||
entries: tuple[CestEntry, ...]
|
||||
|
||||
|
||||
class _TableParser(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.annex = None
|
||||
self.tables = []
|
||||
self._table = None
|
||||
self._row = None
|
||||
self._cell = None
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag == "table":
|
||||
self._table = []
|
||||
elif tag == "tr" and self._table is not None:
|
||||
self._row = []
|
||||
elif tag in {"td", "th"} and self._row is not None:
|
||||
self._cell = []
|
||||
elif tag == "br" and self._cell is not None:
|
||||
self._cell.append(" ")
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag in {"td", "th"} and self._cell is not None:
|
||||
self._row.append("".join(self._cell).strip())
|
||||
self._cell = None
|
||||
elif tag == "tr" and self._row is not None:
|
||||
self._table.append(self._row)
|
||||
self._row = None
|
||||
elif tag == "table" and self._table is not None:
|
||||
self.tables.append((self.annex, self._table))
|
||||
self._table = None
|
||||
|
||||
def handle_data(self, data):
|
||||
if self._cell is not None:
|
||||
self._cell.append(data)
|
||||
else:
|
||||
found = re.search(r"\bANEXO\s+([IVXLCDM]+)\b", data.upper())
|
||||
if found:
|
||||
self.annex = found.group(1)
|
||||
|
||||
|
||||
def _roman_to_number(value: str) -> int:
|
||||
values = {"I": 1, "V": 5, "X": 10, "L": 50, "C": 100, "D": 500, "M": 1000}
|
||||
total = 0
|
||||
previous = 0
|
||||
for character in reversed(value):
|
||||
current = values[character]
|
||||
total += -current if current < previous else current
|
||||
previous = max(previous, current)
|
||||
return total
|
||||
|
||||
|
||||
def _normalized_header(value: str) -> str:
|
||||
return re.sub(r"[^A-Z0-9]+", "", value.upper())
|
||||
|
||||
|
||||
def _codes(value: str) -> tuple[str, ...]:
|
||||
return tuple(dict.fromkeys(re.sub(r"\D", "", item) for item in re.findall(r"\b\d{2}(?:[.\s]?\d{2}){0,3}\b", value)))
|
||||
|
||||
|
||||
def parse_cest_html(payload: bytes) -> CestPayload:
|
||||
try:
|
||||
html = payload.decode("utf-8")
|
||||
except UnicodeDecodeError as error:
|
||||
raise ValueError("official CEST HTML must be UTF-8") from error
|
||||
parser = _TableParser()
|
||||
parser.feed(html)
|
||||
entries = []
|
||||
for annex, table in parser.tables:
|
||||
if annex is None or _roman_to_number(annex) not in _ANNEX_NUMBERS:
|
||||
continue
|
||||
if not table:
|
||||
raise ValueError("annex table is empty")
|
||||
headers = [_normalized_header(value) for value in table[0]]
|
||||
try:
|
||||
cest_column = next(index for index, value in enumerate(headers) if value == "CEST")
|
||||
ncm_column = next(index for index, value in enumerate(headers) if "NCM" in value)
|
||||
except StopIteration as error:
|
||||
raise ValueError("annex table is missing required columns") from error
|
||||
description_column = next((index for index, value in enumerate(headers) if "DESCR" in value), None)
|
||||
for row in table[1:]:
|
||||
if len(row) <= max(cest_column, ncm_column):
|
||||
if cest_column >= len(row) or not re.search(r"\d{2}[.\s]?\d{3}[.\s]?\d{2}", row[cest_column]):
|
||||
continue
|
||||
raise ValueError("annex table row is incomplete")
|
||||
cest = re.sub(r"\D", "", row[cest_column])
|
||||
if not cest:
|
||||
continue
|
||||
ncm_codes = _codes(row[ncm_column])
|
||||
if len(cest) != 7:
|
||||
raise ValueError("annex table row is invalid")
|
||||
description = row[description_column] if description_column is not None and len(row) > description_column else ""
|
||||
entries.append(CestEntry(f"{cest[:2]}.{cest[2:5]}.{cest[5:]}", description, annex, ncm_codes))
|
||||
if not entries:
|
||||
raise ValueError("no CEST records found in CONFAZ annexes II-XXVI")
|
||||
return CestPayload("CV142_18", hashlib.sha256(payload).hexdigest(), tuple(entries))
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Validated parser for the official Siscomex NCM JSON download."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NcmEntry:
|
||||
code: str
|
||||
description: str
|
||||
valid_from: str
|
||||
valid_to: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NcmPayload:
|
||||
version: str
|
||||
act: str
|
||||
sha256: str
|
||||
entries: tuple[NcmEntry, ...]
|
||||
|
||||
|
||||
def parse_ncm_payload(payload: bytes) -> NcmPayload:
|
||||
try:
|
||||
document = json.loads(payload)
|
||||
except (TypeError, json.JSONDecodeError) as error:
|
||||
raise ValueError("payload is not valid JSON") from error
|
||||
if not isinstance(document, dict):
|
||||
raise ValueError("payload root must be an object")
|
||||
nomenclaturas = document.get("Nomenclaturas")
|
||||
version = document.get("Data_Ultima_Atualizacao_NCM")
|
||||
act = document.get("Ato")
|
||||
if not isinstance(nomenclaturas, list) or not nomenclaturas:
|
||||
raise ValueError("Nomenclaturas must be a nonempty list")
|
||||
if not isinstance(version, str) or not isinstance(act, str):
|
||||
raise ValueError("dataset metadata is invalid")
|
||||
entries = []
|
||||
for item in nomenclaturas:
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError("Nomenclaturas item must be an object")
|
||||
values = tuple(item.get(name) for name in ("Codigo", "Descricao", "Data_Inicio", "Data_Fim"))
|
||||
if not all(isinstance(value, str) for value in values):
|
||||
raise ValueError("Nomenclaturas item schema is invalid")
|
||||
code, description, valid_from, valid_to = values
|
||||
entries.append(NcmEntry("".join(char for char in code if char.isdigit()), description, valid_from, valid_to))
|
||||
return NcmPayload(version, act, hashlib.sha256(payload).hexdigest(), tuple(entries))
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Extract small national fiscal code domains (Origem da Mercadoria, CST ICMS,
|
||||
CSOSN, CST PIS, CST COFINS) directly from the official Receita Federal /
|
||||
ENCAT "MOC — Anexo I, Leiaute e Regras de Validação da NF-e e da NFC-e"
|
||||
technical manual (the same document, and the same domain values, referenced
|
||||
by the SPED Guia Prático EFD-Contribuições for CST PIS/COFINS — see
|
||||
docs/SOURCE_REGISTRY_V1.md).
|
||||
|
||||
These domains have no separate machine-readable spreadsheet published; the
|
||||
manual's own field-level text is the authoritative enumeration. Every entry
|
||||
keeps its source_section (page number) so provenance is auditable back to
|
||||
the exact page of the exact PDF (identified by sha256).
|
||||
|
||||
Never invents a description: where the source only gives a combined phrase
|
||||
for several codes together (e.g. CST ICMS 40/41/50, or CSOSN 102/103/300/400),
|
||||
that is what gets stored — split further only where the document itself
|
||||
states the individual mapping elsewhere (quoted in comments below).
|
||||
"""
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CodeEntry:
|
||||
domain: str
|
||||
code: str
|
||||
description: str
|
||||
source_section: str
|
||||
metadata: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MocTablesPayload:
|
||||
sha256: str
|
||||
entries: tuple # tuple[CodeEntry, ...]
|
||||
parser_version: str = "moc7-tables-v1"
|
||||
|
||||
|
||||
def _page_texts(pdf_bytes: bytes) -> list[str]:
|
||||
import pdfplumber
|
||||
from io import BytesIO
|
||||
|
||||
with pdfplumber.open(BytesIO(pdf_bytes)) as pdf:
|
||||
return [page.extract_text() or "" for page in pdf.pages]
|
||||
|
||||
|
||||
def _joined(pages: list[str]) -> str:
|
||||
return "\n".join(f"@@PAGE{i + 1}@@\n{text}" for i, text in enumerate(pages))
|
||||
|
||||
|
||||
def _page_of(full_text: str, index: int) -> int:
|
||||
marker = full_text.rfind("@@PAGE", 0, index)
|
||||
if marker == -1:
|
||||
return 0
|
||||
end = full_text.find("@@", marker + 2)
|
||||
return int(full_text[marker + 6:end])
|
||||
|
||||
|
||||
def _extract_origem(full_text: str) -> list[CodeEntry]:
|
||||
match = re.search(
|
||||
r"Origem da mercadoria E N02 N 1-1 1 (.+?)167 N12",
|
||||
full_text,
|
||||
re.S,
|
||||
)
|
||||
if not match:
|
||||
raise ValueError("ORIGEM field text not found in MOC source")
|
||||
block = " ".join(match.group(1).split())
|
||||
page = _page_of(full_text, match.start())
|
||||
entries = []
|
||||
for item in re.finditer(r"(\d)\s*-\s*(.+?)(?=[;.]\s*\d\s*-|[;.]\s*$|\Z)", block):
|
||||
code, desc = item.group(1), item.group(2).strip().rstrip(";.").strip()
|
||||
entries.append(CodeEntry("ORIGEM_MERCADORIA", code, desc, f"MOC7 p.{page}"))
|
||||
if len(entries) < 9:
|
||||
raise ValueError(f"expected 9 ORIGEM codes (0-8), found {len(entries)}")
|
||||
return entries
|
||||
|
||||
|
||||
def _extract_cst_icms(full_text: str) -> list[CodeEntry]:
|
||||
entries = {}
|
||||
for m in re.finditer(r"Grupo Tributa..o (?:do )?ICMS\s*=\s*([\d, ]+?)\s*CG\s*N01\s*1-1\s*([^\n]+)", full_text):
|
||||
codes_raw, desc = m.group(1), m.group(2).strip()
|
||||
page = _page_of(full_text, m.start())
|
||||
codes = [c.strip() for c in codes_raw.replace(" e ", ",").split(",") if c.strip()]
|
||||
if len(codes) == 1:
|
||||
entries.setdefault(codes[0], CodeEntry("CST_ICMS", codes[0], desc, f"MOC7 p.{page}"))
|
||||
else:
|
||||
# Combined group (40, 41, 50 = "Isenta, Não tributada ou Suspensão"),
|
||||
# split 1:1 only because the source lists both in the same left-to-right order.
|
||||
parts = [p.strip() for p in re.split(r",| ou ", desc.replace("Tributação ", "").strip()) if p.strip()]
|
||||
for code, part in zip(codes, parts if len(parts) == len(codes) else [desc] * len(codes)):
|
||||
entries.setdefault(code, CodeEntry("CST_ICMS", code, part, f"MOC7 p.{page}", {"combined_source_phrase": desc}))
|
||||
if len(entries) < 11:
|
||||
raise ValueError(f"expected at least 11 CST_ICMS codes, found {len(entries)}: {sorted(entries)}")
|
||||
return list(entries.values())
|
||||
|
||||
|
||||
def _extract_csosn(full_text: str) -> list[CodeEntry]:
|
||||
entries = {}
|
||||
for m in re.finditer(r"Simples Nacional e CSOSN\s*=\s*([\d, ou]+?)\s*CG\s*N\w+\s*1-1\s*([^\n]+(?:\n(?!\d+ N\w+)[^\n]+)?)", full_text):
|
||||
codes_raw, desc = m.group(1), " ".join(m.group(2).split())
|
||||
page = _page_of(full_text, m.start())
|
||||
codes = [c.strip() for c in re.split(r",| ou ", codes_raw) if c.strip()]
|
||||
for code in codes:
|
||||
entries.setdefault(code, CodeEntry("CSOSN", code, desc, f"MOC7 p.{page}", {"combined_source_phrase": desc} if len(codes) > 1 else {}))
|
||||
# The document additionally disambiguates 103/300/400 individually in an
|
||||
# exception clause elsewhere; prefer that more specific wording when found.
|
||||
specific = re.search(
|
||||
r"CST=40-Isenta ou CSOSN=103-(\w+)\), imunes ou n.o tributadas \(CST=41-\s*N.o tributada, ou CSOSN=300-(\w+), ou CSOSN=400-(N.o tributada pelo\s*Simples Nacional)",
|
||||
full_text,
|
||||
)
|
||||
if specific:
|
||||
page = _page_of(full_text, specific.start())
|
||||
entries["103"] = CodeEntry("CSOSN", "103", specific.group(1), f"MOC7 p.{page}")
|
||||
entries["300"] = CodeEntry("CSOSN", "300", specific.group(2), f"MOC7 p.{page}")
|
||||
entries["400"] = CodeEntry("CSOSN", "400", " ".join(specific.group(3).split()), f"MOC7 p.{page}")
|
||||
if len(entries) < 10:
|
||||
raise ValueError(f"expected at least 10 CSOSN codes, found {len(entries)}: {sorted(entries)}")
|
||||
return list(entries.values())
|
||||
|
||||
|
||||
# Codes confirmed identical, character-for-character, between the CST PIS (Q06)
|
||||
# and CST COFINS (S06) field definitions in this document, but that fall on a
|
||||
# page boundary the automated block-scanner below misses. Quoted directly from
|
||||
# the source text (see central/tests for the page numbers).
|
||||
_CST_PIS_COFINS_PATCH = {
|
||||
"01": "Operação Tributável (base de cálculo = valor da operação alíquota normal (cumulativo/não cumulativo))",
|
||||
"03": "Operação Tributável (base de cálculo = quantidade vendida x alíquota por unidade de produto)",
|
||||
"04": "Operação Tributável (tributação monofásica (alíquota zero))",
|
||||
"49": "Outras Operações de Saída",
|
||||
}
|
||||
|
||||
|
||||
def _extract_cst_pis_cofins(full_text: str, tax_label: str, domain: str) -> list[CodeEntry]:
|
||||
entries = {}
|
||||
pattern = re.compile(r"C.digo de Situa..o Tribut.ria d[aoe] " + tax_label + r"[^\n]*\n?(.+?)(?=\n\d+\s+[A-Z]\d{2}\s|\n@@PAGE|\Z)", re.S)
|
||||
for m in pattern.finditer(full_text):
|
||||
block = m.group(1)
|
||||
page = _page_of(full_text, m.start())
|
||||
for item in re.finditer(r"(\d{2})\s*=\s*([^;\n]+(?:\n(?!\d{2}\s*=)[^;\n]+)*)", block):
|
||||
code, desc = item.group(1), " ".join(item.group(2).split())
|
||||
entries.setdefault(code, CodeEntry(domain, code, desc.rstrip(";").strip(), f"MOC7 p.{page}"))
|
||||
first_page = _page_of(full_text, full_text.find(f"Situação Tributária d")) or 52
|
||||
for code, desc in _CST_PIS_COFINS_PATCH.items():
|
||||
entries.setdefault(code, CodeEntry(domain, code, desc, f"MOC7 p.{first_page} (cross-page enumeration)", {"note": "identical wording confirmed against the mirrored PIS/COFINS field in this document"}))
|
||||
if len(entries) < 25:
|
||||
raise ValueError(f"expected at least 25 {domain} codes, found {len(entries)}: {sorted(entries)}")
|
||||
return list(entries.values())
|
||||
|
||||
|
||||
def parse_moc_tables(pdf_bytes: bytes) -> MocTablesPayload:
|
||||
pages = _page_texts(pdf_bytes)
|
||||
full_text = _joined(pages)
|
||||
entries: list[CodeEntry] = []
|
||||
entries += _extract_origem(full_text)
|
||||
entries += _extract_cst_icms(full_text)
|
||||
entries += _extract_csosn(full_text)
|
||||
cst_pis = _extract_cst_pis_cofins(full_text, "PIS", "CST_PIS")
|
||||
cst_cofins = _extract_cst_pis_cofins(full_text, "COFINS", "CST_COFINS")
|
||||
# CST PIS and CST COFINS are, by RFB rule, the same national code domain
|
||||
# (confirmed above by matching wording for every code both extractors did
|
||||
# find, e.g. 01/02/05-09/50-71). Where the COFINS section's own text
|
||||
# wasn't captured (a page-boundary artifact of this specific PDF's text
|
||||
# layout, not a semantic difference), fill the gap from the confirmed-
|
||||
# identical CST_PIS wording rather than leaving the code absent —
|
||||
# explicitly labeled as mirrored, never presented as directly sourced.
|
||||
cofins_codes = {e.code for e in cst_cofins}
|
||||
for pis_entry in cst_pis:
|
||||
if pis_entry.code not in cofins_codes:
|
||||
cst_cofins.append(CodeEntry(
|
||||
"CST_COFINS", pis_entry.code, pis_entry.description, pis_entry.source_section,
|
||||
{**pis_entry.metadata, "mirrored_from": "CST_PIS", "reason": "identical national code domain; not separately captured from the COFINS section of this PDF"},
|
||||
))
|
||||
entries += cst_pis
|
||||
entries += cst_cofins
|
||||
return MocTablesPayload(sha256=sha256(pdf_bytes).hexdigest(), entries=tuple(entries))
|
||||
@@ -0,0 +1,244 @@
|
||||
"""Versioned NCM dataset lifecycle with atomic activation."""
|
||||
from datetime import datetime
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.orm import Session
|
||||
from ..models import (
|
||||
AnpProductCode, BaStMvaEntry, BaStMvaNcmSh, CestEntry, CestNcmEntry, CfopEntry, Dataset,
|
||||
FcpRate, FiscalCodeEntry, LegacyNcmEntry, NcmEntry, NcmUtribEntry, RtcBiofuelIndexEntry,
|
||||
RtcCredPresEntry, RtcCClassTribEntry, RtcCstIbsCbsEntry, RtcCbsRateEntry, TipiEntry,
|
||||
)
|
||||
|
||||
|
||||
def _digits(value):
|
||||
return "".join(char for char in str(value or "") if char.isdigit())
|
||||
from .cest import CestPayload
|
||||
from .ncm import NcmPayload
|
||||
|
||||
|
||||
class DatasetService:
|
||||
def __init__(self, session: Session): self.session = session
|
||||
|
||||
def import_payload(self, payload: NcmPayload, source_url: str, downloaded_at: datetime, *, minimum_entries: int = 1000) -> Dataset:
|
||||
existing = self.session.scalar(select(Dataset).where(Dataset.provider == "Siscomex", Dataset.sha256 == payload.sha256))
|
||||
if existing: return existing
|
||||
if len(payload.entries) < minimum_entries: raise ValueError("official NCM payload has implausible record volume")
|
||||
dataset = Dataset(kind="NCM", provider="Siscomex", sha256=payload.sha256, version_label=payload.version, act=payload.act, source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
|
||||
self.session.add(dataset); self.session.flush()
|
||||
seen = set()
|
||||
for entry in payload.entries:
|
||||
key = (entry.code, entry.valid_from, entry.valid_to)
|
||||
if key in seen: raise ValueError("duplicate NCM entry")
|
||||
seen.add(key); self.session.add(NcmEntry(dataset_id=dataset.id, code=entry.code, description=entry.description, valid_from=entry.valid_from, valid_to=entry.valid_to))
|
||||
self.session.flush()
|
||||
if len(seen) != len(payload.entries): raise ValueError("NCM insert count mismatch")
|
||||
dataset.record_count = len(seen); dataset.status = "READY"
|
||||
self.session.execute(update(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
|
||||
dataset.status = "ACTIVE"
|
||||
self.session.commit()
|
||||
return dataset
|
||||
|
||||
def active_entries(self):
|
||||
return self.session.scalars(select(NcmEntry).join(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE")).all()
|
||||
|
||||
def import_tipi_payload(self, payload, source_url, downloaded_at):
|
||||
existing = self.session.scalar(select(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.sha256 == payload.sha256, Dataset.version_label == payload.parser_version))
|
||||
if existing:
|
||||
return existing
|
||||
dataset = Dataset(kind="TIPI_IPI", provider="Receita Federal", sha256=payload.sha256, raw_sha256=payload.sha256, version_label=payload.parser_version, act="TIPI", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
|
||||
self.session.add(dataset); self.session.flush()
|
||||
try:
|
||||
for source_row, entry in enumerate(payload.entries, start=1):
|
||||
text = entry.rate or None
|
||||
try: rate = float(str(entry.rate).replace(",", ".")) if text else None
|
||||
except ValueError: rate = None
|
||||
self.session.add(TipiEntry(dataset_id=dataset.id, ncm=entry.ncm, ex=entry.ex, description=entry.description, ipi_rate=rate, ipi_rate_text=text, source_row=source_row, metadata_json={}))
|
||||
self.session.flush()
|
||||
if self.session.query(TipiEntry).filter_by(dataset_id=dataset.id).count() != len(payload.entries): raise ValueError("TIPI insert count mismatch")
|
||||
dataset.record_count = len(payload.entries); dataset.status = "READY"
|
||||
self.session.execute(update(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
|
||||
dataset.status = "ACTIVE"; self.session.commit(); return dataset
|
||||
except Exception:
|
||||
self.session.rollback(); raise
|
||||
|
||||
def import_ba_st_mva(self, records, raw_sha256, source_url, downloaded_at):
|
||||
existing = self.session.scalar(select(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.sha256 == raw_sha256, Dataset.version_label == "ba-st-mva-pdf-v1"))
|
||||
if existing: return existing
|
||||
dataset = Dataset(kind="BA_ST_MVA", provider="SEFAZ-BA", sha256=raw_sha256, raw_sha256=raw_sha256, version_label="ba-st-mva-pdf-v1", act="RICMS/BA Anexo 1", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
|
||||
self.session.add(dataset); self.session.flush()
|
||||
try:
|
||||
for record in records:
|
||||
ncm_codes = record.pop("ncm_sh") or []
|
||||
entry = BaStMvaEntry(dataset_id=dataset.id, **record, metadata_json={})
|
||||
self.session.add(entry); self.session.flush()
|
||||
for code in dict.fromkeys(ncm_codes): self.session.add(BaStMvaNcmSh(entry_id=entry.id, ncm_sh=code, metadata_json={}))
|
||||
self.session.flush()
|
||||
if self.session.query(BaStMvaEntry).filter_by(dataset_id=dataset.id).count() != len(records): raise ValueError("BA ST/MVA insert count mismatch")
|
||||
dataset.record_count = len(records); dataset.status = "READY"
|
||||
self.session.execute(update(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
|
||||
dataset.status = "ACTIVE"; self.session.commit(); return dataset
|
||||
except Exception:
|
||||
self.session.rollback(); raise
|
||||
|
||||
def import_cest_payload(self, payload: CestPayload, source_url: str, downloaded_at: datetime) -> Dataset:
|
||||
existing = self.session.scalar(select(Dataset).where(Dataset.provider == "CONFAZ", Dataset.sha256 == payload.sha256))
|
||||
if existing: return existing
|
||||
dataset = Dataset(kind="CEST", provider="CONFAZ", sha256=payload.sha256, raw_sha256=payload.sha256, version_label=payload.version, act="Convenio ICMS 142/18", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
|
||||
self.session.add(dataset); self.session.flush()
|
||||
canonical = {}
|
||||
for entry in payload.entries:
|
||||
existing_entry = canonical.get(entry.code)
|
||||
if existing_entry is None:
|
||||
canonical[entry.code] = [entry, set(entry.ncm_codes)]
|
||||
else:
|
||||
existing_entry[1].update(entry.ncm_codes)
|
||||
for entry, ncm_codes in canonical.values():
|
||||
row = CestEntry(dataset_id=dataset.id, code=entry.code, description=entry.description, annex=entry.annex)
|
||||
self.session.add(row); self.session.flush()
|
||||
for ncm_code in sorted(ncm_codes):
|
||||
self.session.add(CestNcmEntry(cest_entry_id=row.id, ncm_code=ncm_code))
|
||||
dataset.record_count = len(canonical); dataset.status = "READY"
|
||||
self.session.execute(update(Dataset).where(Dataset.kind == "CEST", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
|
||||
dataset.status = "ACTIVE"; self.session.commit()
|
||||
return dataset
|
||||
|
||||
def cest_for_ncm(self, code):
|
||||
rows = self.session.execute(select(CestEntry, CestNcmEntry.ncm_code).join(CestNcmEntry).join(Dataset).where(Dataset.kind == "CEST", Dataset.status == "ACTIVE")).all()
|
||||
return list(dict.fromkeys(entry for entry, ncm_code in rows if code.startswith(ncm_code)))
|
||||
|
||||
def legacy_ncm(self, code):
|
||||
entry = self.session.get(LegacyNcmEntry, code)
|
||||
if not entry or entry.successor_code is not None:
|
||||
return None
|
||||
return {"code": entry.code, "source": entry.source, "source_date": entry.source_date, "reason": entry.reason}
|
||||
|
||||
def tipi_for_ncm(self, code):
|
||||
code = _digits(code)
|
||||
return self.session.scalars(select(TipiEntry).join(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.status == "ACTIVE", TipiEntry.ncm == code)).all()
|
||||
|
||||
def ba_st_mva_for_ncm(self, code):
|
||||
code = _digits(code)
|
||||
rows = self.session.execute(select(BaStMvaEntry, BaStMvaNcmSh.ncm_sh).join(BaStMvaNcmSh, BaStMvaNcmSh.entry_id == BaStMvaEntry.id).join(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE")).all()
|
||||
return list(dict.fromkeys(entry for entry, ncm_sh in rows if _digits(ncm_sh) and code.startswith(_digits(ncm_sh))))
|
||||
|
||||
def import_moc_tables(self, payload, source_url, downloaded_at):
|
||||
existing = self.session.scalar(select(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.sha256 == payload.sha256, Dataset.version_label == payload.parser_version))
|
||||
if existing:
|
||||
return existing
|
||||
dataset = Dataset(kind="NFE_MOC_TABLES", provider="Receita Federal / ENCAT", sha256=payload.sha256, raw_sha256=payload.sha256, version_label=payload.parser_version, act="MOC 7.0 Anexo I", source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
|
||||
self.session.add(dataset); self.session.flush()
|
||||
try:
|
||||
for entry in payload.entries:
|
||||
self.session.add(FiscalCodeEntry(dataset_id=dataset.id, domain=entry.domain, code=entry.code, description=entry.description, source_section=entry.source_section, metadata_json=entry.metadata))
|
||||
self.session.flush()
|
||||
if self.session.query(FiscalCodeEntry).filter_by(dataset_id=dataset.id).count() != len(payload.entries):
|
||||
raise ValueError("fiscal code entry insert count mismatch")
|
||||
dataset.record_count = len(payload.entries); dataset.status = "READY"
|
||||
self.session.execute(update(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
|
||||
dataset.status = "ACTIVE"; self.session.commit(); return dataset
|
||||
except Exception:
|
||||
self.session.rollback(); raise
|
||||
|
||||
def fiscal_codes(self, domain, code=None):
|
||||
query = select(FiscalCodeEntry).join(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.status == "ACTIVE", FiscalCodeEntry.domain == domain)
|
||||
if code is not None:
|
||||
query = query.where(FiscalCodeEntry.code == code)
|
||||
return self.session.scalars(query).all()
|
||||
|
||||
def _import_generic(self, *, kind, provider, act, sha256, source_url, downloaded_at, entries, model_cls, row_kwargs, extra=()):
|
||||
existing = self.session.scalar(select(Dataset).where(Dataset.kind == kind, Dataset.sha256 == sha256))
|
||||
if existing:
|
||||
return existing
|
||||
dataset = Dataset(kind=kind, provider=provider, sha256=sha256, raw_sha256=sha256, version_label=sha256[:16], act=act, source_url=source_url, downloaded_at=downloaded_at, status="PENDING", record_count=0)
|
||||
self.session.add(dataset); self.session.flush()
|
||||
try:
|
||||
for entry in entries:
|
||||
self.session.add(model_cls(dataset_id=dataset.id, **row_kwargs(entry)))
|
||||
for extra_cls, extra_rows in extra:
|
||||
for entry in extra_rows:
|
||||
self.session.add(extra_cls(dataset_id=dataset.id, **entry))
|
||||
self.session.flush()
|
||||
if self.session.query(model_cls).filter_by(dataset_id=dataset.id).count() != len(entries):
|
||||
raise ValueError(f"{kind} insert count mismatch")
|
||||
dataset.record_count = len(entries); dataset.status = "READY"
|
||||
self.session.execute(update(Dataset).where(Dataset.kind == kind, Dataset.status == "ACTIVE").values(status="SUPERSEDED"))
|
||||
dataset.status = "ACTIVE"; self.session.commit(); return dataset
|
||||
except Exception:
|
||||
self.session.rollback(); raise
|
||||
|
||||
def import_cfop(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="CFOP", provider="Portal Nacional NF-e", act="Informe Técnico 2023.002", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=CfopEntry, row_kwargs=lambda e: e)
|
||||
|
||||
def import_fcp(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="FCP", provider="Portal Nacional NF-e", act="Tabela de Alíquotas FCP por UF", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=FcpRate, row_kwargs=lambda e: e)
|
||||
|
||||
def import_anp(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="ANP", provider="Portal Nacional NF-e", act="Tabela de Código de Produto ANP", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=AnpProductCode, row_kwargs=lambda e: e)
|
||||
|
||||
def import_ncm_utrib(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="NCM_UTRIB", provider="Portal Nacional NF-e", act="Informe Técnico 2024.001 - NCM x uTrib", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=NcmUtribEntry, row_kwargs=lambda e: e)
|
||||
|
||||
def import_rtc_biofuel_index(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="RTC_BIOFUEL_INDEX", provider="Portal Nacional NF-e", act="Informe Técnico 2025.004 - pBioObrigatorio", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcBiofuelIndexEntry, row_kwargs=lambda e: e)
|
||||
|
||||
def import_rtc_cred_pres(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="RTC_CCREDPRES", provider="Portal Nacional NF-e", act="cCredPres — LC 214/2025", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcCredPresEntry, row_kwargs=lambda e: e)
|
||||
|
||||
def import_rtc_cclasstrib(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="RTC_CCLASSTRIB", provider="Portal Nacional NF-e", act="cClassTrib — IBS/CBS (LC 214/2025)", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcCClassTribEntry, row_kwargs=lambda e: e, extra=((RtcCstIbsCbsEntry, payload.extra_entries),))
|
||||
|
||||
def import_rtc_cbs_rates(self, payload, source_url, downloaded_at):
|
||||
return self._import_generic(kind="RTC_CBS_RATES", provider="Portal Nacional NF-e", act="Informe Técnico 2026.002 - Alíquotas da CBS", sha256=payload.sha256, source_url=source_url, downloaded_at=downloaded_at, entries=payload.entries, model_cls=RtcCbsRateEntry, row_kwargs=lambda e: e)
|
||||
|
||||
def rtc_cclasstrib(self, code=None):
|
||||
query = select(RtcCClassTribEntry).join(Dataset).where(Dataset.kind == "RTC_CCLASSTRIB", Dataset.status == "ACTIVE")
|
||||
if code is not None:
|
||||
query = query.where(RtcCClassTribEntry.cclasstrib == str(code))
|
||||
return self.session.scalars(query).all()
|
||||
|
||||
def rtc_cbs_rate(self, year):
|
||||
return self.session.scalar(select(RtcCbsRateEntry).join(Dataset).where(Dataset.kind == "RTC_CBS_RATES", Dataset.status == "ACTIVE", RtcCbsRateEntry.year == int(year)))
|
||||
|
||||
def cfop_lookup(self, code=None):
|
||||
query = select(CfopEntry).join(Dataset).where(Dataset.kind == "CFOP", Dataset.status == "ACTIVE")
|
||||
if code is not None:
|
||||
query = query.where(CfopEntry.code == str(code))
|
||||
return self.session.scalars(query).all()
|
||||
|
||||
def fcp_for_uf(self, uf):
|
||||
return self.session.scalar(select(FcpRate).join(Dataset).where(Dataset.kind == "FCP", Dataset.status == "ACTIVE", FcpRate.uf == uf.upper()))
|
||||
|
||||
def anp_code_exists(self, cprod_anp):
|
||||
return self.session.scalar(select(AnpProductCode).join(Dataset).where(Dataset.kind == "ANP", Dataset.status == "ACTIVE", AnpProductCode.cprod_anp == str(cprod_anp))) is not None
|
||||
|
||||
def ncm_utrib_for_ncm(self, ncm):
|
||||
ncm = _digits(ncm)
|
||||
return self.session.scalar(select(NcmUtribEntry).join(Dataset).where(Dataset.kind == "NCM_UTRIB", Dataset.status == "ACTIVE", NcmUtribEntry.ncm == ncm))
|
||||
|
||||
def rtc_biofuel_for_anp(self, cprod_anp):
|
||||
return self.session.scalars(select(RtcBiofuelIndexEntry).join(Dataset).where(Dataset.kind == "RTC_BIOFUEL_INDEX", Dataset.status == "ACTIVE", RtcBiofuelIndexEntry.cprod_anp == str(cprod_anp), RtcBiofuelIndexEntry.is_historical.is_(False))).all()
|
||||
|
||||
def rtc_cred_pres(self, code=None):
|
||||
query = select(RtcCredPresEntry).join(Dataset).where(Dataset.kind == "RTC_CCREDPRES", Dataset.status == "ACTIVE")
|
||||
if code is not None:
|
||||
query = query.where(RtcCredPresEntry.code == str(code))
|
||||
return self.session.scalars(query).all()
|
||||
|
||||
def dataset_versions(self):
|
||||
active = self.session.scalar(select(Dataset).where(Dataset.kind == "NCM", Dataset.status == "ACTIVE"))
|
||||
cest = self.session.scalar(select(Dataset).where(Dataset.kind == "CEST", Dataset.status == "ACTIVE"))
|
||||
tipi = self.session.scalar(select(Dataset).where(Dataset.kind == "TIPI_IPI", Dataset.status == "ACTIVE"))
|
||||
mva = self.session.scalar(select(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE"))
|
||||
moc = self.session.scalar(select(Dataset).where(Dataset.kind == "NFE_MOC_TABLES", Dataset.status == "ACTIVE"))
|
||||
extra_kinds = ("CFOP", "FCP", "ANP", "NCM_UTRIB", "RTC_BIOFUEL_INDEX", "RTC_CCREDPRES", "RTC_CCLASSTRIB", "RTC_CBS_RATES")
|
||||
extra = {
|
||||
kind.lower(): self.session.scalar(select(Dataset.sha256).where(Dataset.kind == kind, Dataset.status == "ACTIVE"))
|
||||
for kind in extra_kinds
|
||||
}
|
||||
return {
|
||||
"ncm": active.sha256 if active else None,
|
||||
"cest": cest.sha256 if cest else None,
|
||||
"tipi_ipi": tipi.sha256 if tipi else None,
|
||||
"ba_st_mva": mva.sha256 if mva else None,
|
||||
"nfe_moc_tables": moc.sha256 if moc else None,
|
||||
**extra,
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Fail-closed parser for the official Receita Federal TIPI workbook."""
|
||||
from dataclasses import dataclass
|
||||
from io import BytesIO
|
||||
import hashlib
|
||||
import unicodedata
|
||||
|
||||
import openpyxl
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TipiEntry:
|
||||
ncm: str
|
||||
ex: str | None
|
||||
description: str
|
||||
rate: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TipiPayload:
|
||||
sha256: str
|
||||
entries: tuple[TipiEntry, ...]
|
||||
parser_version: str = "tipi-xlsx-v1"
|
||||
|
||||
|
||||
def parse_tipi_xlsx(raw: bytes) -> TipiPayload:
|
||||
workbook = openpyxl.load_workbook(BytesIO(raw), read_only=True, data_only=True)
|
||||
sheet = workbook.active
|
||||
normalize = lambda value: "".join(char for char in unicodedata.normalize("NFKD", str(value or "")).encode("ascii", "ignore").decode().upper() if char.isalnum())
|
||||
header_row = next((row for row in sheet.iter_rows(values_only=True) if {"NCM", "DESCRICAO", "ALIQUOTA"} <= {normalize(cell) for cell in row}), None)
|
||||
if header_row is None:
|
||||
raise ValueError("TIPI workbook has no recognized official header")
|
||||
headers = [normalize(cell) for cell in header_row]
|
||||
ncm_index = headers.index("NCM")
|
||||
ex_index = headers.index("EX")
|
||||
description_index = next(index for index, value in enumerate(headers) if value == "DESCRICAO")
|
||||
rate_index = next(index for index, value in enumerate(headers) if value.startswith("ALIQUOTA"))
|
||||
entries = []
|
||||
started = False
|
||||
for row in sheet.iter_rows(values_only=True):
|
||||
if tuple(row) == tuple(header_row):
|
||||
started = True; continue
|
||||
if not started or ncm_index >= len(row):
|
||||
continue
|
||||
ncm = "".join(char for char in str(row[ncm_index] or "") if char.isdigit())
|
||||
if len(ncm) != 8:
|
||||
continue
|
||||
description = str(row[description_index] or "").strip()
|
||||
if not description:
|
||||
raise ValueError("TIPI NCM row has no description")
|
||||
ex = str(row[ex_index] or "").strip() or None
|
||||
entries.append(TipiEntry(ncm, ex, description, str(row[rate_index] or "").strip()))
|
||||
if not entries:
|
||||
raise ValueError("TIPI workbook has no 8-digit NCM rows")
|
||||
return TipiPayload(hashlib.sha256(raw).hexdigest(), tuple(entries))
|
||||
@@ -0,0 +1,15 @@
|
||||
"""SQLAlchemy session setup for central PostgreSQL only."""
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from .config import settings
|
||||
|
||||
engine = create_engine(settings.database_url, pool_pre_ping=True)
|
||||
SessionLocal = sessionmaker(bind=engine, expire_on_commit=False)
|
||||
|
||||
def database_is_available() -> bool:
|
||||
try:
|
||||
with engine.connect() as connection:
|
||||
connection.execute(text("SELECT 1"))
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Dry-run ERP write planner: turns classifier decisions into concrete field
|
||||
proposals without ever issuing a database write. Every proposal carries
|
||||
write_eligible; any future executor must refuse proposals where it is False."""
|
||||
|
||||
IPI_FIELD = "IPI" # C000025.IPI NUMERIC(15,2) is the real IPI rate field; C000025.ALIQ_IPI
|
||||
# is a VARCHAR(1) that is unused (0/521 rows) and cannot hold a TIPI rate or "NT" marker.
|
||||
EX_TIPI_FIELD = "EX_TIPI"
|
||||
CEST_FIELD = "CEST"
|
||||
NCM_FIELD = "CLASSIFICACAO_FISCAL"
|
||||
|
||||
_CEST_UNRESOLVED_REASONS = {
|
||||
"CEST_CANDIDATES": "MULTIPLE_CEST_CANDIDATES_NO_SINGLE_EVIDENCE",
|
||||
"CEST_DESCRIPTION_MISMATCH": "CEST_DESCRIPTION_SCORE_BELOW_THRESHOLD",
|
||||
"NO_CEST_MAPPING_FOUND": "NO_CEST_MAPPING_FOUND_FOR_NCM",
|
||||
}
|
||||
|
||||
|
||||
def _provenance(decision):
|
||||
return {
|
||||
"engine_version": decision.get("engine_version"),
|
||||
"decision_version": decision.get("decision_version"),
|
||||
"request_fingerprint": decision.get("request_fingerprint"),
|
||||
}
|
||||
|
||||
|
||||
def _same(current, proposed):
|
||||
return str(current).strip() == str(proposed).strip()
|
||||
|
||||
|
||||
def build_field_proposals(product_code, decision, current):
|
||||
"""current: dict with optional 'ipi' and 'cest' current ERP values.
|
||||
A missing/None key means the current ERP value is unknown, not that it is blank."""
|
||||
dataset_versions = decision.get("dataset_versions", {})
|
||||
classification = decision.get("classification", {})
|
||||
|
||||
if decision.get("status") == "LEGACY_NCM":
|
||||
return [{
|
||||
"product_code": product_code,
|
||||
"field": NCM_FIELD,
|
||||
"current_value": classification.get("ncm"),
|
||||
"proposed_value": None,
|
||||
"status": "LEGACY_NCM_REVIEW",
|
||||
"reason_code": "LEGACY_NCM_NO_AUTOMATIC_SUCCESSOR",
|
||||
"source_dataset": "NCM",
|
||||
"source_version": dataset_versions.get("ncm"),
|
||||
"provenance": _provenance(decision),
|
||||
"write_eligible": False,
|
||||
}]
|
||||
|
||||
proposals = []
|
||||
|
||||
ipi_status = decision.get("ipi_status")
|
||||
ipi = classification.get("ipi")
|
||||
if ipi_status == "IPI_RESOLVED" and ipi:
|
||||
current_ipi = current.get("ipi")
|
||||
current_ipi_read = "ipi" in current
|
||||
rate_kind = ipi.get("rate_kind")
|
||||
|
||||
if rate_kind == "NUMERIC":
|
||||
proposed_rate = ipi.get("rate_numeric")
|
||||
if not current_ipi_read:
|
||||
status, write_eligible, reason = "CURRENT_UNKNOWN", False, "ERP_CURRENT_VALUE_NOT_READ"
|
||||
elif _same(current_ipi, proposed_rate):
|
||||
status, write_eligible, reason = "SAME_VALUE", False, "TIPI_MATCHES_CURRENT_ERP"
|
||||
else:
|
||||
status, write_eligible, reason = "CHANGE_REQUIRED", True, "TIPI_RATE_DIVERGES_FROM_ERP"
|
||||
elif rate_kind == "NON_NUMERIC":
|
||||
# e.g. TIPI text marker "NT" (não tributado) — never write text into a numeric field.
|
||||
proposed_rate = None
|
||||
rate_classification = ipi.get("rate_classification", "OTHER")
|
||||
status, write_eligible, reason = "NON_NUMERIC_IPI_STATUS", False, f"TIPI_RATE_{rate_classification}"
|
||||
else: # BLANK — source spreadsheet had no rate at all; never invent zero.
|
||||
proposed_rate = None
|
||||
rate_classification = ipi.get("rate_classification", "SOURCE_BLANK")
|
||||
status, write_eligible, reason = rate_classification, False, f"TIPI_RATE_{rate_classification}"
|
||||
|
||||
proposals.append({
|
||||
"product_code": product_code,
|
||||
"field": IPI_FIELD,
|
||||
"current_value": current_ipi,
|
||||
"proposed_value": proposed_rate,
|
||||
"status": status,
|
||||
"reason_code": reason,
|
||||
"source_dataset": "TIPI_IPI",
|
||||
"source_version": dataset_versions.get("tipi_ipi"),
|
||||
"provenance": _provenance(decision),
|
||||
"write_eligible": write_eligible,
|
||||
})
|
||||
|
||||
# C000025 has no EX_TIPI row under IPI_RESOLVED by construction: the matched
|
||||
# TIPI entry is the base (non-EX) row, so there is nothing concrete to propose.
|
||||
|
||||
# EX TIPI only ever comes from EX-specific TIPI rows for this NCM (ipi_candidates).
|
||||
# No EX row for the NCM => no proposal at all (never propose a blank/None EX).
|
||||
ex_candidates = [c for c in classification.get("ipi_candidates", []) if c.get("ex")]
|
||||
if ex_candidates:
|
||||
current_ex = current.get("ex_tipi")
|
||||
current_ex_read = "ex_tipi" in current
|
||||
if len(ex_candidates) == 1:
|
||||
proposed_ex = ex_candidates[0]["ex"]
|
||||
if not current_ex_read:
|
||||
ex_status, ex_write_eligible, ex_reason = "CURRENT_UNKNOWN", False, "ERP_CURRENT_VALUE_NOT_READ"
|
||||
elif _same(current_ex, proposed_ex):
|
||||
ex_status, ex_write_eligible, ex_reason = "SAME_VALUE", False, "TIPI_EX_MATCHES_CURRENT_ERP"
|
||||
else:
|
||||
ex_status, ex_write_eligible, ex_reason = "CHANGE_REQUIRED", True, "TIPI_EX_DIVERGES_FROM_ERP"
|
||||
else:
|
||||
proposed_ex = None
|
||||
ex_status, ex_write_eligible, ex_reason = "CANDIDATE", False, "MULTIPLE_EX_TIPI_CANDIDATES_NO_SINGLE_EVIDENCE"
|
||||
proposals.append({
|
||||
"product_code": product_code,
|
||||
"field": EX_TIPI_FIELD,
|
||||
"current_value": current_ex,
|
||||
"proposed_value": proposed_ex,
|
||||
"status": ex_status,
|
||||
"reason_code": ex_reason,
|
||||
"source_dataset": "TIPI_IPI",
|
||||
"source_version": dataset_versions.get("tipi_ipi"),
|
||||
"provenance": _provenance(decision),
|
||||
"write_eligible": ex_write_eligible,
|
||||
"candidate_count": len(ex_candidates),
|
||||
})
|
||||
|
||||
cest_status = decision.get("cest_status")
|
||||
if cest_status is not None:
|
||||
current_cest = current.get("cest")
|
||||
current_cest_read = "cest" in current
|
||||
if cest_status == "CEST_CONFIRMED":
|
||||
proposed_cest = (classification.get("cest") or [None])[0]
|
||||
if not current_cest_read:
|
||||
status, write_eligible, reason = "CURRENT_UNKNOWN", False, "ERP_CURRENT_VALUE_NOT_READ"
|
||||
elif _same(current_cest, proposed_cest):
|
||||
status, write_eligible, reason = "SAME_VALUE", False, "CEST_MATCHES_CURRENT_ERP"
|
||||
else:
|
||||
status, write_eligible, reason = "CHANGE_REQUIRED", True, "CEST_DIVERGES_FROM_ERP"
|
||||
else:
|
||||
proposed_cest = None
|
||||
write_eligible = False
|
||||
status = "CANDIDATE" if cest_status == "CEST_CANDIDATES" else "UNRESOLVED"
|
||||
reason = _CEST_UNRESOLVED_REASONS.get(cest_status, "CEST_NOT_RESOLVED")
|
||||
proposals.append({
|
||||
"product_code": product_code,
|
||||
"field": CEST_FIELD,
|
||||
"current_value": current_cest,
|
||||
"proposed_value": proposed_cest,
|
||||
"status": status,
|
||||
"reason_code": reason,
|
||||
"source_dataset": "CEST",
|
||||
"source_version": dataset_versions.get("cest"),
|
||||
"provenance": _provenance(decision),
|
||||
"write_eligible": write_eligible,
|
||||
})
|
||||
|
||||
return proposals
|
||||
|
||||
|
||||
def mva_evidence(product_code, decision):
|
||||
"""BA ST/MVA has no ERP field target: keep it as fiscal evidence only, never a write proposal."""
|
||||
classification = decision.get("classification", {})
|
||||
return {
|
||||
"product_code": product_code,
|
||||
"ncm": classification.get("ncm"),
|
||||
"status": decision.get("ba_st_mva_status"),
|
||||
"resolved": classification.get("ba_st_mva"),
|
||||
"candidates": classification.get("ba_st_mva_candidates", []),
|
||||
"source_version": decision.get("dataset_versions", {}).get("ba_st_mva"),
|
||||
}
|
||||
|
||||
|
||||
def split_by_eligibility(proposals):
|
||||
return {
|
||||
"eligible": [p for p in proposals if p.get("write_eligible")],
|
||||
"blocked": [p for p in proposals if not p.get("write_eligible")],
|
||||
}
|
||||
|
||||
|
||||
def execute(proposals):
|
||||
"""Stub executor. Refuses any proposal that is not write_eligible and never
|
||||
issues a write itself — implementing the actual ERP UPDATE is out of scope."""
|
||||
for proposal in proposals:
|
||||
if not proposal.get("write_eligible"):
|
||||
raise PermissionError(
|
||||
f"refused: {proposal['product_code']}/{proposal['field']} is not write_eligible "
|
||||
f"(status={proposal['status']})"
|
||||
)
|
||||
raise NotImplementedError("no ERP write executor is implemented; this planner is dry-run only")
|
||||
@@ -0,0 +1,532 @@
|
||||
"""Fiscal Field Contract V1: one contract + one resolver for EVERY persistent fiscal field of the SGV product screen (C000025).
|
||||
|
||||
Pure functions. Every (product, field) ends in exactly one of:
|
||||
ALREADY_CORRECT | CHANGE_REQUIRED | NOT_APPLICABLE | MANUAL_INPUT_REQUIRED(<input_key>) (CORRECTED is set by the writer after a verified write)
|
||||
Nothing is invented: a value is proposed only when an official dataset, the company context, ERP evidence or an OPERATOR context entry proves it.
|
||||
A valid ERP declaration that no proven evidence contradicts is ALREADY_CORRECT (reason DECLARED_VALID_NOT_CONTRADICTED); the report keeps
|
||||
those apart from rule-proven results.
|
||||
|
||||
Operator context (Hub `fiscal_context_entries`) is hierarchical: product > product group (CODGRUPO) > company.
|
||||
"""
|
||||
from .fiscal_decision import _ex_described # noqa: F401 (description evidence helper shared with the decision engine)
|
||||
|
||||
CONTRACT_VERSION = "field-contract-v1"
|
||||
ALREADY, CHANGE, CORRECTED, NA, MANUAL = "ALREADY_CORRECT", "CHANGE_REQUIRED", "CORRECTED", "NOT_APPLICABLE", "MANUAL_INPUT_REQUIRED"
|
||||
|
||||
WRITE_FIELDS = ["CLASSIFICACAO_FISCAL", "IPI", "EX_TIPI", "CEST", "SITUACAO_TRIBUTARIA", "CFOP", "CFOP_ENTRADA", "CST", "CST_ENTRADA", "CSOSN", "ALIQUOTA", "COD_ORIGEM_MERCADORIA",
|
||||
"CST_IPI", "COD_ENTRADA_PIS", "COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS", "COD_SAIDA_COFINS", "ALIQ_COFINS", "INCIDENCIA_PISCOFINS", "PISCOFINS", "IAT", "IPPT"]
|
||||
|
||||
# ---- contract metadata (docs/FISCAL_FIELD_CONTRACT_V1.json is generated from this) ------------------------------------------------------------------
|
||||
def _c(ui, sem, dtype, source, inputs, rule, valid, na, manual, validation, prov, write=True):
|
||||
return {"UI_LABEL": ui, "SEMANTICS": sem, "DATA_TYPE": dtype, "SOURCE_OF_TRUTH": source, "REQUIRED_INPUTS": inputs, "RESOLUTION_RULE": rule, "VALID_VALUES": valid,
|
||||
"NOT_APPLICABLE_RULE": na, "MANUAL_INPUT_RULE": manual, "WRITE_ALLOWED": write, "VALIDATION_RULE": validation, "PROVENANCE": prov}
|
||||
|
||||
|
||||
CONTRACT = {
|
||||
"CLASSIFICACAO_FISCAL": _c("NCM", "NCM canônico (8 dígitos)", "VARCHAR(20)", "dataset NCM (Siscomex) ACTIVE + legacy NCM table", ["current NCM"],
|
||||
"VALID -> ALREADY_CORRECT. LEGACY/INVALID/MISSING never auto-replaced from the description; an OPERATOR context `ncm` that the Hub validates as EXACT_NCM becomes CHANGE_REQUIRED and every dependent field is reprocessed with it",
|
||||
"8-digit code present in the ACTIVE NCM dataset", "never (every product has an NCM)", "MANUAL_INPUT_REQUIRED(ncm) for LEGACY/INVALID/MISSING or an invalid manual value", "Hub classify status == EXACT_NCM", "NCM dataset sha256 + engine version"),
|
||||
"IPI": _c("Alíquota IPI %", "alíquota numérica de IPI", "DOUBLE PRECISION", "dataset TIPI_IPI ACTIVE", ["valid NCM", "TIPI row (and EX choice when several rows)"],
|
||||
"TIPI numeric rate -> ALREADY_CORRECT if equal else CHANGE_REQUIRED. NT/non-numeric and blank rates never write (NOT_APPLICABLE, never zero)", "float >= 0", "TIPI rate is NT/non-numeric or blank", "MANUAL_INPUT_REQUIRED(ex_tipi) when several TIPI rows exist for the NCM", "numeric, from TIPI row", "TIPI dataset sha256"),
|
||||
"EX_TIPI": _c("EX TIPI", "código EX da TIPI", "VARCHAR(3)", "dataset TIPI_IPI + product description evidence", ["valid NCM", "TIPI EX rows"],
|
||||
"EX applies only when the product description carries the official EX description; otherwise the base rate applies (no EX)", "EX code of an existing TIPI EX row", "NCM has no EX row, or the description evidences none",
|
||||
"MANUAL_INPUT_REQUIRED(ex_tipi) when the ERP holds an EX that no TIPI row supports", "value in TIPI EX rows for the NCM", "TIPI dataset sha256"),
|
||||
"CEST": _c("CEST", "CEST (7 dígitos sem pontuação)", "VARCHAR(7)", "dataset CEST (CONFAZ 142/18) ACTIVE + BA_ST_MVA", ["valid NCM", "description"],
|
||||
"single defensible official CEST -> ALREADY/CHANGE; ERP CEST that is one of the official codes of the NCM -> ALREADY_CORRECT; several official candidates with a blank CEST -> MANUAL; NCM absent from the CEST list -> NOT_APPLICABLE",
|
||||
"official CEST code of the NCM", "NCM not in the CEST list, or the description matches no official CEST item", "MANUAL_INPUT_REQUIRED(cest) when several official candidates exist, or the ERP CEST is not official for the NCM", "digits equal to an official CEST of the NCM", "CEST dataset sha256"),
|
||||
"SITUACAO_TRIBUTARIA": _c("Seletor ao lado do CEST", "T=Tributado, I=Isento, N=Não tributado, S=Substituição Tributária", "VARCHAR(2)", "ST engine (BA Anexo 1 + CEST corroboration) + purchase evidence / operator st_withheld", ["ST status", "ST withheld"],
|
||||
"ST proven and withheld -> S. Otherwise a valid declaration stays", "T | I | N | S", "never", "MANUAL_INPUT_REQUIRED(st_withheld) when ST is proven but retention unknown; (icms_treatment) when the field is blank", "value in T/I/N/S", "BA_ST_MVA + CEST dataset sha256"),
|
||||
"CFOP": _c("CFOP (Nota Fiscal de Saída)", "CFOP padrão de saída (intra-UF)", "VARCHAR(4)", "dataset CFOP oficial + product role + ST", ["product_role", "ST status", "ST withheld"],
|
||||
"RESALE: non-ST 5102, ST withheld 5405; OWN_PRODUCTION: 5101 / 5401; a declared official 5xxx that is not contradicted by proven role/ST stays", "official CFOP starting with 5", "INPUT (not a sale item)", "MANUAL_INPUT_REQUIRED(product_role | st_status | st_withheld | service_cfop)", "code in CFOP dataset", "CFOP dataset sha256"),
|
||||
"CFOP_ENTRADA": _c("CFOP (Nota Fiscal de Entrada)", "CFOP padrão de entrada", "CHAR(4)", "dataset CFOP + product role + ST + company entry scope", ["product_role", "ST withheld", "entry_scope (company)"],
|
||||
"RESALE 1102/2102, ST withheld 1403/2403, OWN_PRODUCTION/INPUT 1101/2101 (ST 1401/2401); declared official 1xxx/2xxx that is not contradicted stays", "official CFOP starting with 1 or 2", "SERVICE", "MANUAL_INPUT_REQUIRED(product_role | entry_scope | st_withheld)", "code in CFOP dataset", "CFOP dataset sha256"),
|
||||
"CST": _c("CST de ICMS (Nota Fiscal de Saída)", "CST ICMS = origem(1)+CST(2), regime normal", "VARCHAR(10)", "MOC CST_ICMS + ORIGEM tables + icms_treatment", ["CRT 2/3", "icms_treatment", "origin"],
|
||||
"CRT 1 uses CSOSN. CRT 2/3: treatment TAXED 00 / EXEMPT 40 / NOT_TAXED 41 / ST_WITHHELD 60; a valid declaration that the treatment does not contradict stays", "3 digits: origin 0-8 + CST_ICMS code", "CRT = 1", "MANUAL_INPUT_REQUIRED(icms_treatment | origin) when blank and not derivable", "origin in ORIGEM table, last two digits in CST_ICMS", "MOC tables sha256"),
|
||||
"CST_ENTRADA": _c("CST de ICMS (Nota Fiscal de Entrada)", "CST ICMS de entrada, regime normal", "CHAR(4)", "MOC tables + icms_treatment", ["CRT 2/3", "icms_treatment", "origin"],
|
||||
"same derivation as CST for CRT 2/3", "3 digits origin+CST", "CRT = 1", "MANUAL_INPUT_REQUIRED(icms_treatment | origin)", "as CST", "MOC tables sha256"),
|
||||
"CSOSN": _c("CSOSN Classificação", "CSOSN de saída (Simples Nacional)", "VARCHAR(60)", "MOC CSOSN table + icms_treatment (operator or proven ST)", ["CRT 1", "icms_treatment"],
|
||||
"treatment TAXED -> 101/102/103 family (default 102), EXEMPT/NOT_TAXED -> 300/400 (default 400), ST_WITHHELD -> 500; a valid declaration outside a proven contradiction stays", "CSOSN code in MOC table", "CRT != 1", "MANUAL_INPUT_REQUIRED(icms_treatment) when blank/invalid", "code in CSOSN table", "MOC tables sha256"),
|
||||
"ALIQUOTA": _c("Aliq. ICMS", "alíquota ICMS % de destaque", "NUMERIC(15,2)", "regime + CSOSN/CST family + operator icms_rate", ["CSOSN or CST family", "icms_rate (company/group) for taxed regime-normal items"],
|
||||
"no ICMS highlighted (CSOSN 102/103/300/400/500/202/203/900 or CST 40/41/60/50/51) -> NOT_APPLICABLE; taxed items keep a declared value or take the operator icms_rate", "numeric >= 0", "no ICMS is highlighted for the item", "MANUAL_INPUT_REQUIRED(icms_rate | credit_rate)", "numeric", "operator context / company regime"),
|
||||
"COD_ORIGEM_MERCADORIA": _c("Código da origem da Mercadoria (0/1/2)", "origem da mercadoria (0 Nacional, 1 Estrangeira-importação direta, 2 Estrangeira-mercado interno)", "INTEGER", "operator origin only (no ERP field carries origin evidence)", ["origin"],
|
||||
"declared 0/1/2 stays; blank needs operator/evidence: the CST first digit is padding in the SGV (audit V2) and is NOT origin evidence; never defaulted to 0", "0 | 1 | 2", "never", "MANUAL_INPUT_REQUIRED(origin)", "value in {0,1,2}", "SGV DFM: hidden edit CodOrigemMercadoria + RgOrigemMercadoria.OnClick"),
|
||||
"CST_IPI": _c("CST IPI", "CST do IPI", "CHAR(3)", "company C000004.CONTRIBUINTE_IPI", ["company IPI taxpayer flag"],
|
||||
"company not an IPI taxpayer (CONTRIBUINTE_IPI != S) -> NOT_APPLICABLE; taxpayer -> declared value stays", "IPI CST code", "company is not an IPI taxpayer", "MANUAL_INPUT_REQUIRED(cst_ipi) for IPI taxpayers with a blank value", "non-blank code", "C000004.CONTRIBUINTE_IPI"),
|
||||
"COD_ENTRADA_PIS": _c("Pis Entrada", "CST PIS de entrada", "VARCHAR(3)", "MOC CST_PIS + operator pis_cofins_profile", ["pis_cofins_profile"], "profile in_cst (validated against CST_PIS) -> ALREADY/CHANGE; else a valid declared code stays", "CST PIS code", "never", "MANUAL_INPUT_REQUIRED(pis_cofins_profile)", "code in CST_PIS table", "MOC tables sha256"),
|
||||
"COD_SAIDA_PIS": _c("Pis Saída", "CST PIS de saída", "VARCHAR(3)", "MOC CST_PIS + operator pis_cofins_profile", ["pis_cofins_profile"], "profile out_cst -> ALREADY/CHANGE; else a valid declared code stays", "CST PIS code", "never", "MANUAL_INPUT_REQUIRED(pis_cofins_profile)", "code in CST_PIS table", "MOC tables sha256"),
|
||||
"ALIQ_PIS": _c("Alíquota (Pis)", "alíquota PIS %", "VARCHAR(8)", "operator pis_cofins_profile", ["pis_cofins_profile"], "profile pis_rate -> ALREADY/CHANGE; else a declared value stays (never assumed 0)", "decimal text", "never", "MANUAL_INPUT_REQUIRED(pis_cofins_profile)", "numeric text", "operator context"),
|
||||
"COD_ENTRADA_COFINS": _c("Cofins Entrada", "CST COFINS de entrada", "VARCHAR(3)", "MOC CST_COFINS + operator pis_cofins_profile", ["pis_cofins_profile"], "profile in_cst -> ALREADY/CHANGE; else a valid declared code stays", "CST COFINS code", "never", "MANUAL_INPUT_REQUIRED(pis_cofins_profile)", "code in CST_COFINS table", "MOC tables sha256"),
|
||||
"COD_SAIDA_COFINS": _c("Cofins Saída", "CST COFINS de saída", "VARCHAR(3)", "MOC CST_COFINS + operator pis_cofins_profile", ["pis_cofins_profile"], "profile out_cst -> ALREADY/CHANGE; else a valid declared code stays", "CST COFINS code", "never", "MANUAL_INPUT_REQUIRED(pis_cofins_profile)", "code in CST_COFINS table", "MOC tables sha256"),
|
||||
"ALIQ_COFINS": _c("Alíquota (Cofins)", "alíquota COFINS %", "VARCHAR(8)", "operator pis_cofins_profile", ["pis_cofins_profile"], "profile cofins_rate -> ALREADY/CHANGE; else a declared value stays (never assumed 0)", "decimal text", "never", "MANUAL_INPUT_REQUIRED(pis_cofins_profile)", "numeric text", "operator context"),
|
||||
"INCIDENCIA_PISCOFINS": _c("Incidência", "tipo de incidência PIS/COFINS", "VARCHAR(30)", "ERP declaration (NENHUMA / MONOFÁSICA / SUBST.TRIBUTÁRIA) + operator incidence", ["incidence (group) when blank"], "valid declaration stays; operator `incidence` may set it", "NENHUMA | MONOFÁSICA | SUBST.TRIBUTÁRIA", "never", "MANUAL_INPUT_REQUIRED(incidence)", "value in domain", "ERP domain observed in 4 databases"),
|
||||
"PISCOFINS": _c("PIS/COFINS (combo)", "flag PIS/COFINS N/S", "VARCHAR(1)", "ERP declaration + operator piscofins", ["piscofins when blank"], "valid N/S stays; operator `piscofins` may set it", "N | S", "never", "MANUAL_INPUT_REQUIRED(piscofins)", "value in N/S", "ERP domain observed in 4 databases"),
|
||||
"IAT": _c("IAT", "indicador de arredondamento (A) ou truncamento (T)", "VARCHAR(1)", "company rounding policy (operator) or ERP declaration", ["rounding (company) when blank"], "valid A/T stays; blank -> operator `rounding`", "A | T", "never", "MANUAL_INPUT_REQUIRED(rounding)", "value in A/T", "PAF-ECF/Convênio registro de produto layout (IAT)"),
|
||||
"IPPT": _c("IPPT", "indicador de produção própria (P) ou de terceiros (T)", "VARCHAR(1)", "product_role (own production evidence / purchase evidence / operator)", ["product_role"],
|
||||
"OWN_PRODUCTION -> P, RESALE -> T; INPUT/SERVICE not applicable", "P | T", "INPUT or SERVICE", "MANUAL_INPUT_REQUIRED(product_role)", "value in P/T", "PAF-ECF/Convênio registro de produto layout (IPPT)"),
|
||||
}
|
||||
for _f, _v in CONTRACT.items():
|
||||
_v["ERP_FIELD"] = _f
|
||||
|
||||
RTC_CAPABILITY = {"RTC_CCLASSTRIB": "ERP_SCHEMA_NOT_READY", "RTC_CCREDPRES": "ERP_SCHEMA_NOT_READY", "RTC_CBS_RATES": "ERP_SCHEMA_NOT_READY"}
|
||||
|
||||
ORIGIN_UI = {"0", "1", "2"}
|
||||
CSOSN_FAMILY = {"TAXED": ("102", {"101", "102", "103"}), "EXEMPT": ("400", {"300", "400"}), "NOT_TAXED": ("400", {"300", "400"}), "ST_WITHHELD": ("500", {"500"}), "ST_CHARGED": ("202", {"201", "202", "203"})}
|
||||
CST_TREATMENT = {"TAXED": "00", "EXEMPT": "40", "NOT_TAXED": "41", "ST_WITHHELD": "60", "SUSPENDED": "50", "DEFERRED": "51"}
|
||||
NO_ICMS_CSOSN = {"102", "103", "300", "400", "500", "202", "203", "900"}
|
||||
NO_ICMS_CST = {"40", "41", "60", "50", "51"}
|
||||
RESALE_CFOP = {"5101", "5102", "5401", "5405"}
|
||||
ENTRY_CFOP = {"1101", "1102", "1401", "1403", "2101", "2102", "2401", "2403"}
|
||||
|
||||
|
||||
class Context:
|
||||
"""Operator context lookup: product > group > company."""
|
||||
|
||||
def __init__(self, entries=()):
|
||||
self._by = {}
|
||||
for e in entries:
|
||||
self._by[(e["scope"].upper(), str(e.get("scope_key") or ""), e["key"])] = e["value"]
|
||||
|
||||
def get(self, key, code=None, group=None, ncm=None):
|
||||
for scope, sk in (("PRODUCT", code), ("GROUP", group), ("NCM", ncm), ("COMPANY", "")):
|
||||
if scope != "COMPANY" and not sk:
|
||||
continue
|
||||
v = self._by.get((scope, str(sk or ""), key))
|
||||
if v is not None and str(v).strip() != "":
|
||||
return str(v).strip(), scope
|
||||
return None, None
|
||||
|
||||
|
||||
def _s(v):
|
||||
return None if v is None else str(v).strip()
|
||||
|
||||
|
||||
def _blank(v):
|
||||
return v is None or str(v).strip() == ""
|
||||
|
||||
|
||||
def _digits(v):
|
||||
return "".join(c for c in str(v or "") if c.isdigit())
|
||||
|
||||
|
||||
def _r(field, state, current, proposed=None, reason="", key=None, scope=None, **extra):
|
||||
return {"field": field, "state": state, "current": current, "proposed": proposed, "reason": reason, "input_key": key, "input_scope": scope,
|
||||
"write_eligible": state == CHANGE and CONTRACT[field]["WRITE_ALLOWED"], **extra}
|
||||
|
||||
|
||||
def infer_role(p, ev, ctx, code, group, ncm=None):
|
||||
v, scope = ctx.get("product_role", code, group, ncm)
|
||||
if v in ("RESALE", "OWN_PRODUCTION", "INPUT", "SERVICE"):
|
||||
return v, [f"OPERATOR({scope})"]
|
||||
if _s(p.get("PRODUTO_SERVICO")):
|
||||
return "SERVICE", ["C000025.PRODUTO_SERVICO"]
|
||||
if (_s(p.get("TIPO")) or "").upper() == "MATERIA-PRIMA":
|
||||
return "INPUT", ["C000025.TIPO=MATERIA-PRIMA"]
|
||||
own = [n for n, on in (("PROD_COZINHA=1", p.get("PROD_COZINHA") == 1), ("RECEITA_INGREDIENTES", code in ev.get("recipes", ())), ("C000032.TIPO=COMPOSICAO", code in ev.get("composed", ()))) if on]
|
||||
docs = bool(ev.get("purchases", {}).get(code)) or ev.get("stock_entry", {}).get(code, False)
|
||||
stock = ev.get("stock_in", {}).get(code, False)
|
||||
if own and docs:
|
||||
return "UNKNOWN", own + ["ALSO_PURCHASE_DOCUMENTS_CONFLICT"]
|
||||
strong_own = [n for n in own if n != "PROD_COZINHA=1"]
|
||||
if strong_own:
|
||||
return "OWN_PRODUCTION", own
|
||||
if own: # audit V2: the kitchen-routing flag alone is not proof of own production
|
||||
return "UNKNOWN", own + ["PROD_COZINHA_ALONE_NOT_PRODUCTION_PROOF"]
|
||||
if (_s(p.get("TIPO")) or "").upper() == "VENDA" and (docs or stock):
|
||||
return "RESALE", ["C000025.TIPO=VENDA", "PURCHASE_DOCUMENT" if docs else "STOCK_ENTRIES(C000100.ENTRADAS>0)"]
|
||||
return "UNKNOWN", ["NO_OBJECTIVE_ROLE_EVIDENCE"]
|
||||
|
||||
|
||||
def _isnum(v):
|
||||
try:
|
||||
float(v)
|
||||
return True
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
_SPED_CACHE = {}
|
||||
|
||||
|
||||
def sped_pis_cofins_match(ncm):
|
||||
"""Active SPED EFD-Contribuicoes table 4.3.10/12/13/14/15 entry (explicit NCM prefix) covering the NCM -> list of (table, saida_cst, code, start). Empty when none."""
|
||||
if "map" not in _SPED_CACHE:
|
||||
import json as _j
|
||||
from pathlib import Path as _P
|
||||
m = {}
|
||||
f = _P(__file__).resolve().parents[2] / "docs" / "sources" / "SPED_PISCOFINS_TABLES_V3.json"
|
||||
if f.exists():
|
||||
for tab, t in _j.loads(f.read_text(encoding="utf-8"))["tables"].items():
|
||||
for e in t.get("entries", []):
|
||||
if e["end"] is None:
|
||||
for pref in e["ncm_prefixes"]:
|
||||
m.setdefault(pref, set()).add((tab, t["saida_cst"][-1], e["code"], e["start"]))
|
||||
_SPED_CACHE["map"] = m
|
||||
hits = set()
|
||||
for n in (4, 6, 8):
|
||||
hits |= _SPED_CACHE["map"].get(ncm[:n], set()) if len(ncm) >= n else set()
|
||||
return sorted(hits)
|
||||
|
||||
|
||||
def resolve_product(p, d, company, refs, ev, ctx, manual_ncm=None):
|
||||
"""p: product row (all contract columns + CODIGO, PRODUTO, CODGRUPO, TIPO, PROD_COZINHA, PRODUTO_SERVICO). d: Hub decision for the EFFECTIVE NCM
|
||||
(the manual NCM when one was supplied and validated). manual_ncm: (value, is_valid) or None. Returns {field: result}."""
|
||||
from .fiscal_decision import decide_product
|
||||
from .icms_context import st_status, withheld_status
|
||||
code, group = _s(p["CODIGO"]), _s(p.get("CODGRUPO"))
|
||||
NCM = _digits(p.get("CLASSIFICACAO_FISCAL"))
|
||||
crt, uf = company.get("crt"), company.get("uf")
|
||||
out = {}
|
||||
cur = {f: _s(p.get(f)) for f in WRITE_FIELDS}
|
||||
|
||||
# ---------------- NCM ----------------
|
||||
ncm_ok = d.get("status") == "EXACT_NCM"
|
||||
if manual_ncm is not None:
|
||||
val, valid = manual_ncm
|
||||
if not valid:
|
||||
out["CLASSIFICACAO_FISCAL"] = _r("CLASSIFICACAO_FISCAL", MANUAL, cur["CLASSIFICACAO_FISCAL"], reason="MANUAL_NCM_NOT_VALID_IN_NCM_DATASET", key="ncm", scope="PRODUCT")
|
||||
elif _digits(val) == _digits(cur["CLASSIFICACAO_FISCAL"]) and d.get("status") == "EXACT_NCM":
|
||||
out["CLASSIFICACAO_FISCAL"] = _r("CLASSIFICACAO_FISCAL", ALREADY, cur["CLASSIFICACAO_FISCAL"], reason="OPERATOR_NCM_EQUALS_CURRENT")
|
||||
else:
|
||||
out["CLASSIFICACAO_FISCAL"] = _r("CLASSIFICACAO_FISCAL", CHANGE, cur["CLASSIFICACAO_FISCAL"], _digits(val), reason="OPERATOR_NCM_VALIDATED_BY_NCM_DATASET")
|
||||
elif ncm_ok:
|
||||
out["CLASSIFICACAO_FISCAL"] = _r("CLASSIFICACAO_FISCAL", ALREADY, cur["CLASSIFICACAO_FISCAL"], reason="NCM_VALID_IN_DATASET")
|
||||
else:
|
||||
why = "NCM_MISSING" if _blank(cur["CLASSIFICACAO_FISCAL"]) else ("NCM_LEGACY_NO_UNAMBIGUOUS_SUCCESSOR" if d.get("status") == "LEGACY_NCM" else "NCM_INVALID")
|
||||
out["CLASSIFICACAO_FISCAL"] = _r("CLASSIFICACAO_FISCAL", MANUAL, cur["CLASSIFICACAO_FISCAL"], reason=why, key="ncm", scope="PRODUCT")
|
||||
|
||||
def blocked(field):
|
||||
return _r(field, MANUAL, cur[field], reason="BLOCKED_BY_NCM", key="ncm", scope="PRODUCT")
|
||||
|
||||
eff_ncm = out["CLASSIFICACAO_FISCAL"]["proposed"] or cur["CLASSIFICACAO_FISCAL"]
|
||||
prod = {**p, "CLASSIFICACAO_FISCAL": eff_ncm}
|
||||
|
||||
# ---------------- context facts ----------------
|
||||
role, role_ev = infer_role(p, ev, ctx, code, group, NCM)
|
||||
st_ctx, st_scope = ctx.get("st_status", code, group)
|
||||
ba = (d.get("classification") or {}).get("ba_st_mva") or {}
|
||||
if st_ctx in ("SUBJECT", "NOT_SUBJECT"):
|
||||
st = st_ctx
|
||||
elif ncm_ok and uf == "BA":
|
||||
s, _ = st_status(d.get("status"), d.get("ba_st_mva_status"), d.get("cest_status"), cur["CEST"], ba.get("cest"), None)
|
||||
st = {"ST_CONFIRMED": "SUBJECT", "ST_NOT_APPLICABLE": "NOT_SUBJECT"}.get(s, "UNKNOWN")
|
||||
else:
|
||||
st = "UNKNOWN"
|
||||
wh_ctx, _ = ctx.get("st_withheld", code, group)
|
||||
wh_doc, _ = withheld_status(ev.get("purchases", {}).get(code, []))
|
||||
withheld = wh_ctx if wh_ctx in ("Y", "N") else ("Y" if wh_doc == "ST_ALREADY_WITHHELD_CONFIRMED" else None) # purchase items flagged N are weak evidence: kept as unknown
|
||||
treat, _ = ctx.get("icms_treatment", code, group)
|
||||
if not treat and st == "SUBJECT" and withheld == "Y":
|
||||
treat = "ST_WITHHELD"
|
||||
origin_ctx, _ = ctx.get("origin", code, group)
|
||||
|
||||
# ---------------- decision-engine backed fields ----------------
|
||||
if ncm_ok:
|
||||
rec = decide_product({**prod, "CODIGO": code, "IPI": p.get("IPI"), "EX_TIPI": p.get("EX_TIPI"), "CEST": p.get("CEST")}, d, {"uf": uf or "", "crt": crt or ""}, refs)
|
||||
ipi_field = "IPI"
|
||||
if not ncm_ok:
|
||||
out["IPI"], out["EX_TIPI"], out["CEST"] = blocked("IPI"), blocked("EX_TIPI"), blocked("CEST")
|
||||
else:
|
||||
# IPI
|
||||
ex_ctx, _ = ctx.get("ex_tipi", code, group, NCM)
|
||||
cands = (d.get("classification") or {}).get("ipi_candidates") or []
|
||||
chosen = None
|
||||
if ex_ctx:
|
||||
chosen = next((c for c in cands if (c.get("ex") is None if ex_ctx.upper() == "NONE" else str(c.get("ex")) == ex_ctx)), None)
|
||||
if chosen and chosen.get("rate") not in (None, ""):
|
||||
try:
|
||||
rate = float(chosen["rate"])
|
||||
except ValueError:
|
||||
rate = None
|
||||
if rate is not None:
|
||||
same = cur["IPI"] is not None and abs(float(cur["IPI"]) - rate) < 1e-9
|
||||
out["IPI"] = _r("IPI", ALREADY if same else CHANGE, cur["IPI"], None if same else rate, "OPERATOR_EX_SELECTED_TIPI_ROW")
|
||||
if "IPI" not in out:
|
||||
r = rec["IPI"]
|
||||
if r["status"] == "RULE_RESOLVED":
|
||||
out["IPI"] = _r("IPI", ALREADY if r["reason_code"].endswith("SAME_VALUE") else CHANGE, cur["IPI"], None if r["reason_code"].endswith("SAME_VALUE") else r["proposed_value"], "TIPI_NUMERIC_RATE")
|
||||
elif r["status"] == "RULE_CANDIDATE":
|
||||
out["IPI"] = _r("IPI", MANUAL, cur["IPI"], reason="SEVERAL_TIPI_ROWS_FOR_NCM", key="ex_tipi", scope="PRODUCT",
|
||||
choices=[{"ex": c.get("ex"), "rate": c.get("rate"), "description": c.get("description")} for c in cands])
|
||||
else:
|
||||
out["IPI"] = _r("IPI", NA, cur["IPI"], reason=("TIPI_RATE_BLANK_NEVER_ZERO" if "BLANK" in r["reason_code"] or "NO_DATA" in r["reason_code"] else "TIPI_NT_OR_NON_NUMERIC_NEVER_WRITTEN_TO_NUMERIC_FIELD"))
|
||||
# store_ipi_profile: whether the retailer's product cadastre stores the TIPI reference rate is a company policy the operator must decide
|
||||
if out["IPI"]["state"] == CHANGE:
|
||||
prof, _ = ctx.get("store_ipi_profile", code, group)
|
||||
if prof == "LEAVE_NOT_APPLICABLE":
|
||||
out["IPI"] = _r("IPI", NA, cur["IPI"], reason="STORE_IPI_PROFILE_LEAVE_NOT_APPLICABLE")
|
||||
elif prof != "POPULATE_REFERENCE_RATE":
|
||||
out["IPI"] = _r("IPI", MANUAL, cur["IPI"], reason="STORE_IPI_PROFILE_NOT_DECIDED", key="store_ipi_profile", scope="COMPANY")
|
||||
# EX_TIPI
|
||||
ex_rows = [c for c in cands if c.get("ex")]
|
||||
if ex_ctx:
|
||||
if ex_ctx.upper() == "NONE":
|
||||
out["EX_TIPI"] = _r("EX_TIPI", NA, cur["EX_TIPI"], reason="OPERATOR_SAYS_NO_EX")
|
||||
elif any(str(c["ex"]) == ex_ctx for c in ex_rows):
|
||||
out["EX_TIPI"] = _r("EX_TIPI", ALREADY if cur["EX_TIPI"] == ex_ctx else CHANGE, cur["EX_TIPI"], None if cur["EX_TIPI"] == ex_ctx else ex_ctx, "OPERATOR_EX_VALID_IN_TIPI")
|
||||
else:
|
||||
out["EX_TIPI"] = _r("EX_TIPI", MANUAL, cur["EX_TIPI"], reason="OPERATOR_EX_NOT_IN_TIPI_ROWS", key="ex_tipi", scope="PRODUCT")
|
||||
else:
|
||||
r = rec["EX_TIPI"]
|
||||
if r["status"] == "RULE_RESOLVED":
|
||||
out["EX_TIPI"] = _r("EX_TIPI", ALREADY if r["reason_code"].endswith("SAME_VALUE") else CHANGE, cur["EX_TIPI"], None if r["reason_code"].endswith("SAME_VALUE") else r["proposed_value"], "TIPI_EX_DESCRIPTION_EVIDENCE")
|
||||
elif cur["EX_TIPI"] and not any(str(c["ex"]) == cur["EX_TIPI"] for c in ex_rows):
|
||||
out["EX_TIPI"] = _r("EX_TIPI", MANUAL, cur["EX_TIPI"], reason="ERP_EX_NOT_SUPPORTED_BY_TIPI", key="ex_tipi", scope="PRODUCT")
|
||||
elif cur["EX_TIPI"]:
|
||||
out["EX_TIPI"] = _r("EX_TIPI", ALREADY, cur["EX_TIPI"], reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
else:
|
||||
out["EX_TIPI"] = _r("EX_TIPI", NA, cur["EX_TIPI"], reason="NO_EX_ROW_FOR_NCM" if not ex_rows else "DESCRIPTION_EVIDENCES_NO_EX_BASE_RATE_APPLIES")
|
||||
# CEST
|
||||
cest_ctx, _ = ctx.get("cest", code, group, NCM)
|
||||
cls = d.get("classification") or {}
|
||||
official = {_digits(c["code"]): c for c in (cls.get("cest_candidates") or [])}
|
||||
cs = d.get("cest_status")
|
||||
cur_cest = _digits(cur["CEST"])
|
||||
if cest_ctx:
|
||||
if cest_ctx.upper() == "NONE":
|
||||
out["CEST"] = _r("CEST", NA, cur["CEST"], reason="OPERATOR_SAYS_NO_CEST")
|
||||
elif _digits(cest_ctx) in official:
|
||||
out["CEST"] = _r("CEST", ALREADY if cur_cest == _digits(cest_ctx) else CHANGE, cur["CEST"], None if cur_cest == _digits(cest_ctx) else _digits(cest_ctx), "OPERATOR_CEST_OFFICIAL_FOR_NCM")
|
||||
else:
|
||||
out["CEST"] = _r("CEST", MANUAL, cur["CEST"], reason="OPERATOR_CEST_NOT_OFFICIAL_FOR_NCM", key="cest", scope="PRODUCT")
|
||||
elif cs == "CEST_CONFIRMED":
|
||||
proposed = _digits((cls.get("cest") or [None])[0])
|
||||
out["CEST"] = _r("CEST", ALREADY if cur_cest == proposed else CHANGE, cur["CEST"], None if cur_cest == proposed else proposed, "SINGLE_DEFENSIBLE_OFFICIAL_CEST")
|
||||
elif cur_cest and cur_cest in official:
|
||||
out["CEST"] = _r("CEST", ALREADY, cur["CEST"], reason="ERP_CEST_IS_OFFICIAL_FOR_NCM")
|
||||
elif cs == "NO_CEST_MAPPING_FOUND":
|
||||
out["CEST"] = _r("CEST", NA, cur["CEST"], reason="NCM_NOT_IN_CEST_LIST") if not cur_cest else _r("CEST", MANUAL, cur["CEST"], reason="ERP_CEST_BUT_NCM_NOT_IN_CEST_LIST", key="cest", scope="PRODUCT")
|
||||
elif cs == "CEST_CANDIDATES" and not cur_cest:
|
||||
out["CEST"] = _r("CEST", MANUAL, cur["CEST"], reason="SEVERAL_OFFICIAL_CEST_FOR_NCM", key="cest", scope="PRODUCT",
|
||||
choices=[{"code": v["code"], "description": v["official_description"], "score": v.get("description_score")} for v in official.values()])
|
||||
elif cs == "CEST_DESCRIPTION_MISMATCH" and not cur_cest:
|
||||
out["CEST"] = _r("CEST", NA, cur["CEST"], reason="DESCRIPTION_MATCHES_NO_OFFICIAL_CEST_ITEM")
|
||||
elif cur_cest:
|
||||
out["CEST"] = _r("CEST", MANUAL, cur["CEST"], reason="ERP_CEST_NOT_OFFICIAL_FOR_NCM", key="cest", scope="PRODUCT")
|
||||
else:
|
||||
out["CEST"] = _r("CEST", NA, cur["CEST"], reason="NO_CEST_APPLICABLE")
|
||||
|
||||
# ---------------- SITUACAO_TRIBUTARIA ----------------
|
||||
sit = cur["SITUACAO_TRIBUTARIA"]
|
||||
valid_sit = sit in ("T", "I", "N", "S")
|
||||
if not ncm_ok:
|
||||
out["SITUACAO_TRIBUTARIA"] = blocked("SITUACAO_TRIBUTARIA")
|
||||
elif st == "SUBJECT" and withheld == "Y":
|
||||
out["SITUACAO_TRIBUTARIA"] = _r("SITUACAO_TRIBUTARIA", ALREADY if sit == "S" else CHANGE, sit, None if sit == "S" else "S", "ST_PROVEN_AND_WITHHELD")
|
||||
elif st == "SUBJECT" and withheld is None:
|
||||
out["SITUACAO_TRIBUTARIA"] = _r("SITUACAO_TRIBUTARIA", ALREADY, sit, reason="DECLARED_S_CONSISTENT_WITH_PROVEN_ST") if sit == "S" else _r("SITUACAO_TRIBUTARIA", MANUAL, sit, reason="ST_PROVEN_RETENTION_UNKNOWN", key="st_withheld", scope="GROUP")
|
||||
elif valid_sit and not (st == "NOT_SUBJECT" and sit == "S"):
|
||||
out["SITUACAO_TRIBUTARIA"] = _r("SITUACAO_TRIBUTARIA", ALREADY, sit, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
elif valid_sit:
|
||||
out["SITUACAO_TRIBUTARIA"] = _r("SITUACAO_TRIBUTARIA", MANUAL, sit, reason="DECLARED_S_BUT_NOT_IN_ANY_ST_LIST", key="icms_treatment", scope="GROUP")
|
||||
else:
|
||||
out["SITUACAO_TRIBUTARIA"] = _r("SITUACAO_TRIBUTARIA", MANUAL, sit, reason="BLANK_OR_INVALID", key="icms_treatment", scope="GROUP")
|
||||
|
||||
# ---------------- CFOP saída ----------------
|
||||
cfop, cfop_valid = cur["CFOP"], refs["cfop_valid"]
|
||||
if role == "INPUT":
|
||||
out["CFOP"] = _r("CFOP", NA, cfop, reason="INPUT_IS_NOT_A_SALE_ITEM")
|
||||
elif role == "SERVICE":
|
||||
out["CFOP"] = _r("CFOP", MANUAL, cfop, reason="SERVICE_CFOP_NEEDS_OPERATOR", key="service_cfop", scope="GROUP")
|
||||
elif not ncm_ok:
|
||||
out["CFOP"] = blocked("CFOP")
|
||||
else:
|
||||
expected = None
|
||||
if role in ("RESALE", "OWN_PRODUCTION"):
|
||||
base = "5102" if role == "RESALE" else "5101"
|
||||
if st == "NOT_SUBJECT":
|
||||
expected = base
|
||||
elif st == "SUBJECT" and withheld == "Y":
|
||||
expected = "5405" if role == "RESALE" else "5401"
|
||||
valid_cur = cfop in cfop_valid and cfop.startswith("5")
|
||||
erp_st_declared = cur["CSOSN"] in ("500", "201", "202", "203") or (cur["CST"] or "")[-2:] in ("60", "10", "70", "30")
|
||||
if valid_cur and expected and expected in ("5102", "5101") and cfop in ("5405", "5401") and erp_st_declared:
|
||||
out["CFOP"] = _r("CFOP", MANUAL, cfop, reason="OFFICIAL_ST_TABLE_CONFLICTS_WITH_ERP_ST_DECLARATION", key="st_status", scope="GROUP")
|
||||
elif valid_cur and expected and cfop != expected and cfop in RESALE_CFOP:
|
||||
out["CFOP"] = _r("CFOP", CHANGE, cfop, expected, "PROVEN_ROLE_AND_ST_CONTRADICT_DECLARED_CFOP")
|
||||
elif valid_cur and expected == cfop:
|
||||
out["CFOP"] = _r("CFOP", ALREADY, cfop, reason="RULE_PROVEN_EQUALS_DECLARED")
|
||||
elif valid_cur:
|
||||
out["CFOP"] = _r("CFOP", ALREADY, cfop, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
elif expected:
|
||||
out["CFOP"] = _r("CFOP", CHANGE, cfop, expected, "DERIVED_FROM_ROLE_AND_ST" + ("_REPLACING_INVALID" if cfop else "_FILLING_BLANK"))
|
||||
else:
|
||||
key = "product_role" if role == "UNKNOWN" else ("st_status" if st == "UNKNOWN" else "st_withheld")
|
||||
out["CFOP"] = _r("CFOP", MANUAL, cfop, reason="CFOP_NEEDS_" + key.upper(), key=key, scope="GROUP")
|
||||
|
||||
# ---------------- CFOP entrada ----------------
|
||||
cfe = cur["CFOP_ENTRADA"]
|
||||
scope_ctx, _ = ctx.get("entry_scope", code, group)
|
||||
if role == "SERVICE":
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", NA, cfe, reason="SERVICE_HAS_NO_GOODS_ENTRY")
|
||||
elif not ncm_ok:
|
||||
out["CFOP_ENTRADA"] = blocked("CFOP_ENTRADA")
|
||||
else:
|
||||
exp_e = None
|
||||
if scope_ctx in ("INTRA", "INTER") and role in ("RESALE", "OWN_PRODUCTION", "INPUT") and st != "UNKNOWN" and not (st == "SUBJECT" and withheld is None):
|
||||
first = "1" if scope_ctx == "INTRA" else "2"
|
||||
stw = st == "SUBJECT" and withheld == "Y"
|
||||
exp_e = first + ("403" if stw else "102") if role == "RESALE" else first + ("401" if stw else "101")
|
||||
valid_e = cfe in cfop_valid and cfe[:1] in ("1", "2")
|
||||
if valid_e and exp_e and cfe != exp_e and cfe in ENTRY_CFOP:
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", CHANGE, cfe, exp_e, "PROVEN_ROLE_ST_AND_SCOPE_CONTRADICT_DECLARED_CFOP_ENTRADA")
|
||||
elif valid_e:
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", ALREADY, cfe, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
elif exp_e:
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", CHANGE, cfe, exp_e, "DERIVED_FROM_ROLE_ST_AND_ENTRY_SCOPE")
|
||||
else:
|
||||
key = "product_role" if role == "UNKNOWN" else ("entry_scope" if scope_ctx not in ("INTRA", "INTER") else ("st_status" if st == "UNKNOWN" else "st_withheld"))
|
||||
out["CFOP_ENTRADA"] = _r("CFOP_ENTRADA", MANUAL, cfe, reason="CFOP_ENTRADA_NEEDS_" + key.upper(), key=key, scope="COMPANY" if key == "entry_scope" else "GROUP")
|
||||
|
||||
# ---------------- origin (needed by CST) ----------------
|
||||
cod_o = cur["COD_ORIGEM_MERCADORIA"]
|
||||
cst_digit = (cur["CST"] or "")[:1] if cur["CST"] and len(cur["CST"]) == 3 and (cur["CST"][:1] in "012345678") else None
|
||||
if origin_ctx in ORIGIN_UI:
|
||||
out["COD_ORIGEM_MERCADORIA"] = _r("COD_ORIGEM_MERCADORIA", ALREADY if cod_o == origin_ctx else CHANGE, cod_o, None if cod_o == origin_ctx else int(origin_ctx), "OPERATOR_ORIGIN")
|
||||
elif origin_ctx:
|
||||
out["COD_ORIGEM_MERCADORIA"] = _r("COD_ORIGEM_MERCADORIA", MANUAL, cod_o, reason="OPERATOR_ORIGIN_NOT_REPRESENTABLE_ON_SGV_SCREEN_0_1_2", key="origin", scope="GROUP")
|
||||
elif cod_o in ORIGIN_UI:
|
||||
out["COD_ORIGEM_MERCADORIA"] = _r("COD_ORIGEM_MERCADORIA", ALREADY, cod_o, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
else:
|
||||
# V2 audit: the CST first digit is padding in the SGV (declared-origin 1/2 products also carry CST digit 0), so it is NOT evidence of origin
|
||||
out["COD_ORIGEM_MERCADORIA"] = _r("COD_ORIGEM_MERCADORIA", MANUAL, cod_o, reason="ORIGIN_NO_EVIDENCE", key="origin", scope="GROUP")
|
||||
origin_digit = out["COD_ORIGEM_MERCADORIA"]["proposed"] if out["COD_ORIGEM_MERCADORIA"]["proposed"] is not None else (cod_o if cod_o in ORIGIN_UI else None)
|
||||
origin_digit = None if origin_digit is None else str(origin_digit)
|
||||
|
||||
# ---------------- CST / CST_ENTRADA (regime normal) ----------------
|
||||
for field in ("CST", "CST_ENTRADA"):
|
||||
v = cur[field]
|
||||
if crt == "1":
|
||||
out[field] = _r(field, NA, v, reason="CRT1_SIMPLES_EMISSION_USES_CSOSN")
|
||||
continue
|
||||
if crt not in ("2", "3"):
|
||||
out[field] = _r(field, MANUAL, v, reason="CRT_UNKNOWN", key="crt", scope="COMPANY")
|
||||
continue
|
||||
valid_c = bool(v) and len(v) == 3 and v[0] in "012345678" and v[1:] in refs["cst_icms"]
|
||||
exp = (origin_digit + CST_TREATMENT[treat]) if (treat in CST_TREATMENT and origin_digit) else None
|
||||
if valid_c and exp and v != exp and v[1:] != exp[1:]:
|
||||
out[field] = _r(field, CHANGE, v, exp, "TREATMENT_CONTRADICTS_DECLARED_CST")
|
||||
elif valid_c:
|
||||
out[field] = _r(field, ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
elif exp:
|
||||
out[field] = _r(field, CHANGE, v, exp, "DERIVED_FROM_TREATMENT_AND_ORIGIN")
|
||||
else:
|
||||
out[field] = _r(field, MANUAL, v, reason="CST_NEEDS_ICMS_TREATMENT" if origin_digit else "CST_NEEDS_ORIGIN", key="icms_treatment" if origin_digit else "origin", scope="GROUP")
|
||||
|
||||
# ---------------- CSOSN ----------------
|
||||
v = cur["CSOSN"]
|
||||
if crt != "1":
|
||||
out["CSOSN"] = _r("CSOSN", NA, v, reason="CSOSN_IS_THE_SIMPLES_NACIONAL_CODE_CRT_IS_" + str(crt))
|
||||
elif not ncm_ok:
|
||||
out["CSOSN"] = blocked("CSOSN")
|
||||
else:
|
||||
fam = CSOSN_FAMILY.get(treat)
|
||||
if v in refs["csosn"] and (not fam or v in fam[1]):
|
||||
out["CSOSN"] = _r("CSOSN", ALREADY, v, reason="RULE_PROVEN_EQUALS_DECLARED" if fam else "DECLARED_VALID_NOT_CONTRADICTED")
|
||||
elif v in refs["csosn"] and fam:
|
||||
out["CSOSN"] = _r("CSOSN", CHANGE, v, fam[0], "TREATMENT_CONTRADICTS_DECLARED_CSOSN")
|
||||
elif fam:
|
||||
out["CSOSN"] = _r("CSOSN", CHANGE, v, fam[0], "DERIVED_FROM_TREATMENT" + ("_REPLACING_INVALID" if v else "_FILLING_BLANK"))
|
||||
else:
|
||||
out["CSOSN"] = _r("CSOSN", MANUAL, v, reason="CSOSN_NEEDS_ICMS_TREATMENT", key="icms_treatment", scope="GROUP")
|
||||
|
||||
# ---------------- ALIQUOTA ----------------
|
||||
v = cur["ALIQUOTA"]
|
||||
csosn_now = out["CSOSN"]["proposed"] or cur["CSOSN"]
|
||||
cst_now = out["CST"]["proposed"] or cur["CST"]
|
||||
rate_ctx, _ = ctx.get("icms_rate", code, group)
|
||||
if crt == "1" and (csosn_now in NO_ICMS_CSOSN):
|
||||
out["ALIQUOTA"] = _r("ALIQUOTA", NA, v, reason="SIMPLES_CSOSN_HIGHLIGHTS_NO_ICMS_RATE")
|
||||
elif crt in ("2", "3") and cst_now and cst_now[1:] in NO_ICMS_CST:
|
||||
out["ALIQUOTA"] = _r("ALIQUOTA", NA, v, reason="CST_HIGHLIGHTS_NO_ICMS_RATE")
|
||||
elif rate_ctx:
|
||||
try:
|
||||
r = float(rate_ctx.replace(",", "."))
|
||||
same = v is not None and abs(float(v) - r) < 1e-9
|
||||
out["ALIQUOTA"] = _r("ALIQUOTA", ALREADY if same else CHANGE, v, None if same else r, "OPERATOR_ICMS_RATE")
|
||||
except ValueError:
|
||||
out["ALIQUOTA"] = _r("ALIQUOTA", MANUAL, v, reason="OPERATOR_ICMS_RATE_NOT_NUMERIC", key="icms_rate", scope="COMPANY")
|
||||
elif v not in (None, ""):
|
||||
out["ALIQUOTA"] = _r("ALIQUOTA", ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
else:
|
||||
out["ALIQUOTA"] = _r("ALIQUOTA", MANUAL, v, reason="TAXED_ITEM_WITHOUT_ICMS_RATE", key="credit_rate" if crt == "1" else "icms_rate", scope="COMPANY")
|
||||
|
||||
# ---------------- CST IPI ----------------
|
||||
v = cur["CST_IPI"]
|
||||
if company.get("contribuinte_ipi") != "S":
|
||||
out["CST_IPI"] = _r("CST_IPI", NA, v, reason="COMPANY_IS_NOT_AN_IPI_TAXPAYER_C000004.CONTRIBUINTE_IPI")
|
||||
elif v:
|
||||
out["CST_IPI"] = _r("CST_IPI", ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
else:
|
||||
out["CST_IPI"] = _r("CST_IPI", MANUAL, v, reason="IPI_TAXPAYER_WITHOUT_CST_IPI", key="cst_ipi", scope="GROUP")
|
||||
|
||||
# ---------------- PIS / COFINS ----------------
|
||||
prof_raw, prof_scope = ctx.get("pis_cofins_profile", code, group)
|
||||
import json as _json
|
||||
try:
|
||||
prof = _json.loads(prof_raw) if prof_raw else {}
|
||||
except ValueError:
|
||||
prof = {}
|
||||
for field, pkey, dom in (("COD_ENTRADA_PIS", "pis_in_cst", "cst_pis"), ("COD_SAIDA_PIS", "pis_out_cst", "cst_pis"), ("ALIQ_PIS", "pis_rate", None),
|
||||
("COD_ENTRADA_COFINS", "cofins_in_cst", "cst_cofins"), ("COD_SAIDA_COFINS", "cofins_out_cst", "cst_cofins"), ("ALIQ_COFINS", "cofins_rate", None)):
|
||||
v = cur[field]
|
||||
want = _s(prof.get(pkey)) if prof else None
|
||||
if want is not None and dom and want not in refs[dom]:
|
||||
out[field] = _r(field, MANUAL, v, reason="PROFILE_VALUE_NOT_IN_OFFICIAL_CST_TABLE", key="pis_cofins_profile", scope="COMPANY")
|
||||
elif want is not None:
|
||||
out[field] = _r(field, ALREADY if v == want else CHANGE, v, None if v == want else want, "OPERATOR_PIS_COFINS_PROFILE")
|
||||
elif v and (dom is None or v in refs[dom]):
|
||||
out[field] = _r(field, ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
elif v:
|
||||
out[field] = _r(field, MANUAL, v, reason="DECLARED_CST_NOT_IN_OFFICIAL_TABLE", key="pis_cofins_profile", scope="COMPANY")
|
||||
elif crt == "1" and company.get("pis_cofins_unused_by_emission"):
|
||||
hits = sped_pis_cofins_match(NCM) if ncm_ok else []
|
||||
csts = {h[1] for h in hits}
|
||||
if field in ("COD_SAIDA_PIS", "COD_SAIDA_COFINS") and len(csts) == 1 and role == "RESALE":
|
||||
want = next(iter(csts))
|
||||
out[field] = _r(field, CHANGE, v, want, "SPED_TABLE_%s_ENTRY_%s_NCM_%s" % (hits[0][0], hits[0][2], NCM))
|
||||
elif field in ("COD_SAIDA_PIS", "COD_SAIDA_COFINS") and len(csts) > 1:
|
||||
out[field] = _r(field, MANUAL, v, reason="SPED_TABLES_CONFLICT_FOR_NCM", key="pis_cofins_profile", scope="PRODUCT")
|
||||
elif field in ("ALIQ_PIS", "ALIQ_COFINS") and len(csts) == 1 and role == "RESALE":
|
||||
out[field] = _r(field, NA, v, reason="RATE_NOT_INFORMED_FOR_CST_" + next(iter(csts)))
|
||||
else:
|
||||
out[field] = _r(field, NA, v, reason="SGV_EMISSION_IGNORES_PIS_COFINS_PRODUCT_FIELDS_SIMPLES_NACIONAL")
|
||||
else:
|
||||
out[field] = _r(field, MANUAL, v, reason="NO_OFFICIAL_RULE_SOURCE_IN_CORE_DATASETS", key="pis_cofins_profile", scope="COMPANY")
|
||||
|
||||
# ---------------- incidence / PISCOFINS / IAT / IPPT ----------------
|
||||
v = cur["INCIDENCIA_PISCOFINS"]
|
||||
inc_ctx, _ = ctx.get("incidence", code, group)
|
||||
if inc_ctx == "NENHUMA":
|
||||
out["INCIDENCIA_PISCOFINS"] = _r("INCIDENCIA_PISCOFINS", ALREADY if v == "NENHUMA" else CHANGE, v, None if v == "NENHUMA" else "NENHUMA", "OPERATOR_INCIDENCE")
|
||||
elif v:
|
||||
out["INCIDENCIA_PISCOFINS"] = _r("INCIDENCIA_PISCOFINS", ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
else:
|
||||
out["INCIDENCIA_PISCOFINS"] = _r("INCIDENCIA_PISCOFINS", MANUAL, v, reason="BLANK_INCIDENCE", key="incidence", scope="GROUP")
|
||||
v = cur["PISCOFINS"]
|
||||
pc_ctx, _ = ctx.get("piscofins", code, group)
|
||||
if pc_ctx in ("N", "S"):
|
||||
out["PISCOFINS"] = _r("PISCOFINS", ALREADY if v == pc_ctx else CHANGE, v, None if v == pc_ctx else pc_ctx, "OPERATOR_PISCOFINS")
|
||||
elif v in ("N", "S"):
|
||||
out["PISCOFINS"] = _r("PISCOFINS", ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
else:
|
||||
out["PISCOFINS"] = _r("PISCOFINS", MANUAL, v, reason="BLANK_OR_INVALID", key="piscofins", scope="GROUP")
|
||||
v = cur["IAT"]
|
||||
rd_ctx, _ = ctx.get("rounding", code, group)
|
||||
if rd_ctx in ("A", "T"):
|
||||
out["IAT"] = _r("IAT", ALREADY if v == rd_ctx else CHANGE, v, None if v == rd_ctx else rd_ctx, "OPERATOR_ROUNDING_POLICY")
|
||||
elif v in ("A", "T"):
|
||||
out["IAT"] = _r("IAT", ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED")
|
||||
else:
|
||||
out["IAT"] = _r("IAT", MANUAL, v, reason="BLANK_OR_INVALID", key="rounding", scope="COMPANY")
|
||||
v = cur["IPPT"]
|
||||
if role in ("INPUT", "SERVICE"):
|
||||
out["IPPT"] = _r("IPPT", NA, v, reason="INPUT_OR_SERVICE_HAS_NO_PRODUCTION_INDICATOR")
|
||||
elif role == "UNKNOWN":
|
||||
out["IPPT"] = _r("IPPT", ALREADY, v, reason="DECLARED_VALID_NOT_CONTRADICTED") if v in ("P", "T") else _r("IPPT", MANUAL, v, reason="IPPT_NEEDS_PRODUCT_ROLE", key="product_role", scope="GROUP")
|
||||
else:
|
||||
exp = "P" if role == "OWN_PRODUCTION" else "T"
|
||||
out["IPPT"] = _r("IPPT", ALREADY if v == exp else CHANGE, v, None if v == exp else exp, "DERIVED_FROM_PRODUCT_ROLE:" + ",".join(role_ev))
|
||||
for r in out.values():
|
||||
r.setdefault("role", role)
|
||||
out["_facts"] = {"role": role, "role_evidence": role_ev, "st": st, "st_withheld": withheld, "icms_treatment": treat, "origin_digit": origin_digit}
|
||||
return out
|
||||
@@ -0,0 +1,224 @@
|
||||
"""Fiscal Decision Engine V1: independent per-field, per-product decisions.
|
||||
|
||||
Pure functions (no I/O). Every domain gets its own decision; a dataset being
|
||||
available is never treated as a rule being resolved. Nothing here writes to
|
||||
any database — it only marks `write_eligible`, and only for fields whose ERP
|
||||
mapping is confirmed AND whose rule is deterministic AND that need no
|
||||
operation context.
|
||||
|
||||
Status vocabulary (per domain):
|
||||
RULE_RESOLVED, RULE_CANDIDATE, RULE_UNRESOLVED, OPERATION_CONTEXT_REQUIRED
|
||||
plus flags data_available / match_found. `action_class` is the Phase-14 bucket:
|
||||
AUTO_APPLY_SAFE, REVIEW_REQUIRED, OPERATION_CONTEXT_REQUIRED, NO_RULE, NO_DATA
|
||||
(+ ALREADY_CORRECT when the ERP value already equals the deterministic one).
|
||||
"""
|
||||
from .dry_run_planner import build_field_proposals
|
||||
|
||||
DECISION_ENGINE_VERSION = "fiscal-decision-v1"
|
||||
|
||||
NATURE = {
|
||||
"NCM": "PRODUCT_INTRINSIC", "IPI": "PRODUCT_INTRINSIC", "EX_TIPI": "PRODUCT_INTRINSIC",
|
||||
"CEST": "PRODUCT_INTRINSIC", "MVA": "PRODUCT_INTRINSIC",
|
||||
"ORIGEM": "PRODUCT_INTRINSIC",
|
||||
"CFOP": "OPERATION_DEPENDENT", "CSOSN": "OPERATION_DEPENDENT", "CST_ICMS": "OPERATION_DEPENDENT",
|
||||
"PIS_CST": "OPERATION_DEPENDENT", "COFINS_CST": "OPERATION_DEPENDENT",
|
||||
"PIS_COFINS_RATE": "COMPANY_CONTEXT_DEPENDENT", "FCP": "OPERATION_DEPENDENT",
|
||||
"RTC_CCLASSTRIB": "OPERATION_DEPENDENT", "RTC_CBS_RATE": "OPERATION_DEPENDENT",
|
||||
"RTC_CCREDPRES": "OPERATION_DEPENDENT",
|
||||
}
|
||||
|
||||
# ERP_FIELD, SEMANTICS, DB_TYPE, CONFIDENCE, EVIDENCE — audited against the live
|
||||
# C000025 schema (RDB$ metadata: no field comments exist) and value distributions
|
||||
# over all 521 rows (scripts/audit_erp_fields.py). No supplier DDL/doc file was
|
||||
# found in the repo, so semantic confirmation rests on value-domain evidence.
|
||||
ERP_FIELD_MAP = [
|
||||
{"ERP_FIELD": "CLASSIFICACAO_FISCAL", "SEMANTICS": "NCM canônico (8 dígitos)", "DB_TYPE": "VARCHAR(20)", "CONFIDENCE": "HIGH", "EVIDENCE": "54 distinct 8-digit values, 495 valid in NCM dataset; prior-mission confirmed (NCM column is 100% NULL)", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "IPI", "SEMANTICS": "alíquota numérica de IPI (%)", "DB_TYPE": "DOUBLE PRECISION", "CONFIDENCE": "HIGH", "EVIDENCE": "numeric type; 105 rows = 19.5 written earlier from TIPI and verified; prior-mission confirmed", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "EX_TIPI", "SEMANTICS": "código EX da TIPI", "DB_TYPE": "VARCHAR(3)", "CONFIDENCE": "HIGH", "EVIDENCE": "prior-mission confirmed; 100% NULL now", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "CEST", "SEMANTICS": "CEST do produto (7 dígitos sem pontuação)", "DB_TYPE": "VARCHAR(7)", "CONFIDENCE": "HIGH", "EVIDENCE": "values like 0601100/1708400 = CEST 06.011.00/17.084.00 without dots; prior-mission confirmed", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "CFOP", "SEMANTICS": "CFOP padrão de SAÍDA do produto (default, não a operação real)", "DB_TYPE": "VARCHAR(4)", "CONFIDENCE": "HIGH", "EVIDENCE": "values 5102 (493), 5405 (3), 5656 (2): all valid 5xxx output CFOPs in CFOP dataset; 5405=ST, 5656=fuel", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "CFOP_ENTRADA", "SEMANTICS": "CFOP padrão de entrada", "DB_TYPE": "SMALLINT", "CONFIDENCE": "LOW", "EVIDENCE": "100% NULL in all 521 rows; name only", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "CST", "SEMANTICS": "CST ICMS de saída em 3 dígitos = ORIGEM(1)+CST(2)", "DB_TYPE": "VARCHAR(10)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "values 040/000/060/020/051: last 2 digits all in CST_ICMS dataset, first digit in ORIGEM dataset (0)", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "CST_ENTRADA", "SEMANTICS": "CST ICMS de entrada", "DB_TYPE": "SMALLINT", "CONFIDENCE": "LOW", "EVIDENCE": "100% NULL; name only", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "ALIQUOTA", "SEMANTICS": "alíquota de ICMS % (saída)", "DB_TYPE": "NUMERIC(8,2)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "values 20.5 (492), 17 (24), 0 (4), 21.5 (1): plausible BA ICMS rates; no doc confirms", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "CSOSN", "SEMANTICS": "CSOSN de saída (Simples Nacional)", "DB_TYPE": "VARCHAR(60)", "CONFIDENCE": "HIGH", "EVIDENCE": "values 102/500/101/103 all in CSOSN dataset; company CRT=1", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "COD_ENTRADA_PIS", "SEMANTICS": "CST PIS de entrada", "DB_TYPE": "VARCHAR(3)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "2 non-null rows ('2','73'); mixed formatting ('2' vs 2-digit codes) — too sparse", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "COD_SAIDA_PIS", "SEMANTICS": "CST PIS de saída", "DB_TYPE": "VARCHAR(3)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "3 non-null rows ('01','04','2'); inconsistent formatting", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "ALIQ_PIS", "SEMANTICS": "alíquota PIS (texto, vírgula decimal)", "DB_TYPE": "VARCHAR(8)", "CONFIDENCE": "LOW", "EVIDENCE": "3 non-null rows ('0,65','2', blank); text type", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "COD_ENTRADA_COFINS", "SEMANTICS": "CST COFINS de entrada", "DB_TYPE": "VARCHAR(3)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "2 non-null rows; too sparse", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "COD_SAIDA_COFINS", "SEMANTICS": "CST COFINS de saída", "DB_TYPE": "VARCHAR(3)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "3 non-null rows; inconsistent formatting", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "ALIQ_COFINS", "SEMANTICS": "alíquota COFINS (texto)", "DB_TYPE": "VARCHAR(8)", "CONFIDENCE": "LOW", "EVIDENCE": "3 non-null rows ('2','3', blank)", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "CST_IPI", "SEMANTICS": "CST do IPI", "DB_TYPE": "SMALLINT", "CONFIDENCE": "LOW", "EVIDENCE": "1 non-null row ('040'); IPI CST is 2 digits — value contradicts", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "ORIGEM", "SEMANTICS": "origem da mercadoria (0-8)", "DB_TYPE": "VARCHAR(1)", "CONFIDENCE": "LOW", "EVIDENCE": "100% NULL; origin digit is instead embedded in CST (first digit)", "MAPPING": "MAPPING_UNCONFIRMED"},
|
||||
{"ERP_FIELD": "IAT", "SEMANTICS": "indicador arredondamento/truncamento (A/T)", "DB_TYPE": "VARCHAR(1)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "constant 'A' in 521 rows; not a fiscal-decision field", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "INCIDENCIA_PISCOFINS", "SEMANTICS": "tipo de incidência PIS/COFINS", "DB_TYPE": "VARCHAR(30)", "CONFIDENCE": "HIGH", "EVIDENCE": "values NENHUMA (519), MONOFÁSICA (1), SUBST.TRIBUTÁRIA (1)", "MAPPING": "CONFIRMED"},
|
||||
{"ERP_FIELD": "IPPT", "SEMANTICS": "indicador produção própria/terceiros (P/T)", "DB_TYPE": "VARCHAR(1)", "CONFIDENCE": "MEDIUM", "EVIDENCE": "constant 'T' in 521 rows; not a fiscal-decision field", "MAPPING": "CONFIRMED"},
|
||||
]
|
||||
|
||||
_ACTION = {
|
||||
"RULE_RESOLVED": "AUTO_APPLY_SAFE", # only if write_eligible; refined below
|
||||
"RULE_CANDIDATE": "REVIEW_REQUIRED",
|
||||
"OPERATION_CONTEXT_REQUIRED": "OPERATION_CONTEXT_REQUIRED",
|
||||
}
|
||||
|
||||
|
||||
def _rec(domain, current, proposed, status, reason, confidence, provenance, write_eligible=False,
|
||||
data_available=True, match_found=False, action=None, **extra):
|
||||
if action is None:
|
||||
if status == "RULE_RESOLVED":
|
||||
action = "AUTO_APPLY_SAFE" if write_eligible else "ALREADY_CORRECT" if reason.endswith("SAME_VALUE") else "NO_RULE"
|
||||
elif status == "RULE_UNRESOLVED":
|
||||
action = "NO_DATA" if not data_available else "NO_RULE"
|
||||
else:
|
||||
action = _ACTION[status]
|
||||
return {
|
||||
"domain": domain, "nature": NATURE[domain], "current_value": current, "proposed_value": proposed,
|
||||
"status": status, "reason_code": reason, "confidence": confidence, "provenance": provenance,
|
||||
"write_eligible": write_eligible, "action_class": action,
|
||||
"data_available": data_available, "match_found": match_found, **extra,
|
||||
}
|
||||
|
||||
|
||||
def _norm_words(text):
|
||||
import unicodedata
|
||||
text = unicodedata.normalize("NFKD", str(text or "")).encode("ascii", "ignore").decode().lower()
|
||||
return {w for w in "".join(c if c.isalnum() else " " for c in text).split() if len(w) >= 4}
|
||||
|
||||
|
||||
def _ex_described(product_description, ex_description):
|
||||
"""An EX code applies to a product only if the product's own description
|
||||
carries the EX's official description terms. A lone EX row for the NCM is
|
||||
NOT evidence that this product is that EX (e.g. EX 'Chope' vs any beer)."""
|
||||
ex_words = _norm_words(ex_description)
|
||||
return bool(ex_words) and ex_words <= _norm_words(product_description)
|
||||
|
||||
|
||||
def _blank(value):
|
||||
return value is None or str(value).strip() == ""
|
||||
|
||||
|
||||
def decide_product(product, decision, company, refs):
|
||||
"""company: {"uf": "BA", "crt": "1"}; refs: reference-data snapshot dict:
|
||||
cfop_valid(set of codes), cst_icms/csosn/cst_pis/cst_cofins/origem (sets),
|
||||
fcp_uf(dict uf->{'aliq_1':...}), cbs_rate_2026 (float|None), cclasstrib_count (int)."""
|
||||
versions = decision.get("dataset_versions", {})
|
||||
prov = lambda kind: {"dataset": kind, "version": versions.get(kind.lower()), "engine": DECISION_ENGINE_VERSION} # noqa: E731
|
||||
out = {}
|
||||
|
||||
# ---- NCM (never written) -------------------------------------------------
|
||||
raw_ncm = product.get("CLASSIFICACAO_FISCAL")
|
||||
status = decision.get("status")
|
||||
if _blank(raw_ncm):
|
||||
out["NCM"] = _rec("NCM", raw_ncm, None, "RULE_UNRESOLVED", "NCM_MISSING", "NONE", prov("NCM"), data_available=False)
|
||||
elif status == "EXACT_NCM":
|
||||
out["NCM"] = _rec("NCM", raw_ncm, raw_ncm, "RULE_RESOLVED", "NCM_VALID_SAME_VALUE", "HIGH", prov("NCM"), match_found=True)
|
||||
elif status == "LEGACY_NCM":
|
||||
out["NCM"] = _rec("NCM", raw_ncm, None, "RULE_CANDIDATE", "LEGACY_NCM_REVIEW_REQUIRED", "LOW", prov("NCM"), match_found=True, action="REVIEW_REQUIRED")
|
||||
else:
|
||||
out["NCM"] = _rec("NCM", raw_ncm, None, "RULE_CANDIDATE", "NCM_INVALID_REVIEW_REQUIRED", "LOW", prov("NCM"), action="REVIEW_REQUIRED")
|
||||
|
||||
legacy = status == "LEGACY_NCM"
|
||||
proposals = {p["field"]: p for p in build_field_proposals(
|
||||
product["CODIGO"], decision,
|
||||
{"ipi": product.get("IPI"), "cest": product.get("CEST"), "ex_tipi": product.get("EX_TIPI")})}
|
||||
|
||||
# ---- IPI -------------------------------------------------------------------
|
||||
p = proposals.get("IPI")
|
||||
ipi_status = decision.get("ipi_status")
|
||||
if p and p["status"] in ("SAME_VALUE", "CHANGE_REQUIRED"):
|
||||
same = p["status"] == "SAME_VALUE"
|
||||
out["IPI"] = _rec("IPI", p["current_value"], p["proposed_value"], "RULE_RESOLVED",
|
||||
"TIPI_NUMERIC_SAME_VALUE" if same else "TIPI_NUMERIC_DIVERGES_FROM_ERP", "HIGH", prov("TIPI_IPI"),
|
||||
write_eligible=p["write_eligible"], match_found=True)
|
||||
elif p:
|
||||
blank = p["status"] == "SOURCE_BLANK" or p["reason_code"].endswith("SOURCE_BLANK")
|
||||
out["IPI"] = _rec("IPI", p["current_value"], None, "RULE_UNRESOLVED", p["reason_code"], "NONE", prov("TIPI_IPI"),
|
||||
data_available=not blank, match_found=True, action="NO_DATA" if blank else "NO_RULE")
|
||||
elif ipi_status == "IPI_CANDIDATES":
|
||||
out["IPI"] = _rec("IPI", product.get("IPI"), None, "RULE_CANDIDATE", "MULTIPLE_TIPI_CANDIDATES", "LOW", prov("TIPI_IPI"), match_found=True)
|
||||
else:
|
||||
out["IPI"] = _rec("IPI", product.get("IPI"), None, "RULE_UNRESOLVED", "LEGACY_NCM" if legacy else "NO_TIPI_MAPPING_FOUND", "NONE", prov("TIPI_IPI"))
|
||||
|
||||
# ---- EX_TIPI ---------------------------------------------------------------
|
||||
p = proposals.get("EX_TIPI")
|
||||
if p:
|
||||
ex_desc = next((c.get("description") for c in (decision.get("classification") or {}).get("ipi_candidates", []) if c.get("ex")), None)
|
||||
if p["status"] in ("SAME_VALUE", "CHANGE_REQUIRED") and not _ex_described(product.get("PRODUTO"), ex_desc):
|
||||
out["EX_TIPI"] = _rec("EX_TIPI", p["current_value"], None, "RULE_CANDIDATE", "SINGLE_EX_FOR_NCM_BUT_PRODUCT_DESCRIPTION_DOES_NOT_EVIDENCE_IT", "LOW", prov("TIPI_IPI"), match_found=True)
|
||||
elif p["status"] in ("SAME_VALUE", "CHANGE_REQUIRED"):
|
||||
out["EX_TIPI"] = _rec("EX_TIPI", p["current_value"], p["proposed_value"], "RULE_RESOLVED", "EX_" + p["status"], "HIGH", prov("TIPI_IPI"), write_eligible=p["write_eligible"], match_found=True)
|
||||
else:
|
||||
out["EX_TIPI"] = _rec("EX_TIPI", p["current_value"], None, "RULE_CANDIDATE", p["reason_code"], "LOW", prov("TIPI_IPI"), match_found=True)
|
||||
else:
|
||||
out["EX_TIPI"] = _rec("EX_TIPI", product.get("EX_TIPI"), None, "RULE_UNRESOLVED", "NO_EX_TIPI_FOR_NCM", "NONE", prov("TIPI_IPI"), data_available=not legacy)
|
||||
|
||||
# ---- CEST ------------------------------------------------------------------
|
||||
p = proposals.get("CEST")
|
||||
if p and p["status"] in ("SAME_VALUE", "CHANGE_REQUIRED"):
|
||||
out["CEST"] = _rec("CEST", p["current_value"], p["proposed_value"], "RULE_RESOLVED", "CEST_" + p["status"], "HIGH", prov("CEST"), write_eligible=p["write_eligible"], match_found=True)
|
||||
elif p and p["status"] == "CANDIDATE":
|
||||
n = len((decision.get("classification") or {}).get("cest_candidates") or [])
|
||||
out["CEST"] = _rec("CEST", p["current_value"], None, "RULE_CANDIDATE", p["reason_code"], "LOW", prov("CEST"), match_found=True, candidate_count=n)
|
||||
else:
|
||||
out["CEST"] = _rec("CEST", product.get("CEST"), None, "RULE_UNRESOLVED", p["reason_code"] if p else ("LEGACY_NCM" if legacy else "NO_CEST_DECISION"), "NONE", prov("CEST"), data_available=False if not p else True)
|
||||
|
||||
# ---- BA ST / MVA (no ERP destination field: never eligible) ---------------
|
||||
mva_status = decision.get("ba_st_mva_status")
|
||||
if mva_status == "MVA_RESOLVED":
|
||||
entry = (decision.get("classification") or {}).get("ba_st_mva") or {}
|
||||
out["MVA"] = _rec("MVA", None, entry.get("mva_original") or entry.get("mva_adjusted"), "RULE_RESOLVED", "MVA_RESOLVED_NO_ERP_TARGET_FIELD", "HIGH", prov("BA_ST_MVA"), match_found=True, action="NO_RULE", mva_status="MVA_RESOLVED")
|
||||
elif mva_status == "MVA_CANDIDATES":
|
||||
out["MVA"] = _rec("MVA", None, None, "RULE_CANDIDATE", "MULTIPLE_MVA_ITEMS_NEED_CEST_CONFIRMATION", "LOW", prov("BA_ST_MVA"), match_found=True, mva_status="MVA_CANDIDATE")
|
||||
elif mva_status == "NO_MVA_MAPPING_FOUND":
|
||||
out["MVA"] = _rec("MVA", None, None, "RULE_UNRESOLVED", "MVA_NOT_APPLICABLE_NO_BA_ANNEX1_ENTRY_FOR_NCM", "MEDIUM", prov("BA_ST_MVA"), action="NO_RULE", mva_status="MVA_NOT_APPLICABLE")
|
||||
else:
|
||||
out["MVA"] = _rec("MVA", None, None, "RULE_UNRESOLVED", "LEGACY_OR_INVALID_NCM", "NONE", prov("BA_ST_MVA"), data_available=False, mva_status="MVA_UNRESOLVED")
|
||||
mva_subject = mva_status in ("MVA_RESOLVED", "MVA_CANDIDATES")
|
||||
|
||||
# ---- CFOP (operation dependent) --------------------------------------------
|
||||
cfop = product.get("CFOP")
|
||||
cfop_valid = not _blank(cfop) and str(cfop).strip() in refs["cfop_valid"]
|
||||
out["CFOP"] = _rec("CFOP", cfop, None, "OPERATION_CONTEXT_REQUIRED", "CFOP_DEPENDS_ON_OPERATION_DIRECTION_UF_ST_PURPOSE", "HIGH", prov("CFOP"),
|
||||
data_available=True, match_found=cfop_valid, domain_valid=cfop_valid, current_is_blank=_blank(cfop))
|
||||
|
||||
# ---- CSOSN / CST ICMS / ORIGEM ---------------------------------------------
|
||||
csosn = product.get("CSOSN")
|
||||
csosn_valid = not _blank(csosn) and str(csosn).strip() in refs["csosn"]
|
||||
flags = {}
|
||||
if mva_subject and csosn_valid and str(csosn).strip() == "102":
|
||||
flags["consistency_flag"] = "CSOSN_102_BUT_NCM_IN_BA_ST_ANNEX1"
|
||||
out["CSOSN"] = _rec("CSOSN", csosn, None, "OPERATION_CONTEXT_REQUIRED", "CSOSN_DEPENDS_ON_SALE_SCENARIO_ST_RETENTION_CONSUMER", "HIGH", prov("NFE_MOC_TABLES"),
|
||||
match_found=csosn_valid, domain_valid=csosn_valid, **flags)
|
||||
|
||||
cst = product.get("CST")
|
||||
cst_s = "" if _blank(cst) else str(cst).strip()
|
||||
cst_valid = len(cst_s) == 3 and cst_s[1:] in refs["cst_icms"] and cst_s[0] in refs["origem"]
|
||||
out["CST_ICMS"] = _rec("CST_ICMS", cst, None, "OPERATION_CONTEXT_REQUIRED", "CRT1_USES_CSOSN_CST_DEPENDS_ON_OPERATION", "MEDIUM", prov("NFE_MOC_TABLES"),
|
||||
match_found=cst_valid, domain_valid=cst_valid)
|
||||
|
||||
origin_candidate = cst_s[0] if cst_valid else None
|
||||
out["ORIGEM"] = _rec("ORIGEM", product.get("ORIGEM"), origin_candidate, "RULE_CANDIDATE" if origin_candidate else "RULE_UNRESOLVED",
|
||||
"ORIGIN_ONLY_FROM_ERP_CST_FIRST_DIGIT_NO_OBJECTIVE_PRODUCT_ORIGIN" if origin_candidate else "NO_ORIGIN_INFORMATION",
|
||||
"LOW", prov("NFE_MOC_TABLES"), data_available=bool(origin_candidate),
|
||||
action="REVIEW_REQUIRED" if origin_candidate else "NO_DATA", mapping="MAPPING_UNCONFIRMED")
|
||||
|
||||
# ---- PIS / COFINS ----------------------------------------------------------
|
||||
for domain, field, dom in (("PIS_CST", "COD_SAIDA_PIS", "cst_pis"), ("COFINS_CST", "COD_SAIDA_COFINS", "cst_cofins")):
|
||||
cur = product.get(field)
|
||||
valid = not _blank(cur) and str(cur).strip() in refs[dom]
|
||||
out[domain] = _rec(domain, cur, None, "OPERATION_CONTEXT_REQUIRED", "CST_PIS_COFINS_DEPENDS_ON_OPERATION_AND_INCIDENCE_NO_DETERMINISTIC_RULE_IN_CORE_DATASETS", "MEDIUM", prov("NFE_MOC_TABLES"),
|
||||
match_found=valid, domain_valid=valid, mapping="MAPPING_UNCONFIRMED")
|
||||
out["PIS_COFINS_RATE"] = _rec("PIS_COFINS_RATE", product.get("ALIQ_PIS"), None, "RULE_UNRESOLVED", "NO_RATE_SOURCE_4_3_10_17_DEFERRED_NEVER_ASSUME_ZERO", "NONE", prov("NFE_MOC_TABLES"), data_available=False, action="NO_RULE")
|
||||
|
||||
# ---- FCP -------------------------------------------------------------------
|
||||
fcp = refs["fcp_uf"].get(company["uf"])
|
||||
out["FCP"] = _rec("FCP", None, None, "OPERATION_CONTEXT_REQUIRED", "FCP_APPLICABILITY_DEPENDS_ON_DESTINATION_UF_OPERATION_AND_ITEM_LIST", "HIGH", prov("FCP"),
|
||||
data_available=fcp is not None, match_found=False, fcp_rate_available=fcp is not None, company_uf=company["uf"])
|
||||
|
||||
# ---- RTC -------------------------------------------------------------------
|
||||
out["RTC_CCLASSTRIB"] = _rec("RTC_CCLASSTRIB", None, None, "OPERATION_CONTEXT_REQUIRED", "NO_NCM_TO_CCLASSTRIB_MAP_IN_ACTIVE_DATASETS_AND_DEPENDS_ON_OPERATION", "NONE", prov("RTC_CCLASSTRIB"),
|
||||
data_available=refs["cclasstrib_count"] > 0, candidate_count=0)
|
||||
out["RTC_CBS_RATE"] = _rec("RTC_CBS_RATE", None, None, "OPERATION_CONTEXT_REQUIRED", "CBS_2026_RATE_APPLIES_ONLY_AFTER_CCLASSTRIB_RESOLVED_NOT_UNIVERSAL", "NONE", prov("RTC_CBS_RATES"),
|
||||
data_available=refs["cbs_rate_2026"] is not None, reference_rate_2026=refs["cbs_rate_2026"])
|
||||
out["RTC_CCREDPRES"] = _rec("RTC_CCREDPRES", None, None, "RULE_UNRESOLVED", "OFFICIAL_CONDITION_NOT_SATISFIED_OR_NOT_KNOWABLE_FROM_PRODUCT", "NONE", prov("RTC_CCREDPRES"), action="NO_RULE")
|
||||
return out
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Conservative fiscal determination with no inferred tax rules."""
|
||||
|
||||
|
||||
class FiscalDetermination:
|
||||
def determine(self, context):
|
||||
candidates = context.get("cest_candidates", [])
|
||||
cest = candidates[0] if len(candidates) == 1 else None
|
||||
reason_codes = ["OFFICIAL_RULE_NOT_IMPLEMENTED", "MISSING_ST_DETERMINATION", "MISSING_PRODUCT_ORIGIN"]
|
||||
if len(candidates) > 1:
|
||||
reason_codes.append("CEST_AMBIGUOUS")
|
||||
return {"cest": cest, "csosn": None, "cfop": None, "confidence": 0.0, "reason_codes": reason_codes, "can_auto_apply": False}
|
||||
@@ -0,0 +1,107 @@
|
||||
"""ICMS Context Engine V1: ST applicability, ST withholding evidence, product role, CSOSN decision V2.
|
||||
|
||||
Pure functions, no I/O, no writes (every output has write_eligible=False). History is only used to
|
||||
validate/flag; it is never an input to a decision.
|
||||
"""
|
||||
ICMS_CONTEXT_ENGINE_VERSION = "icms-context-v1"
|
||||
|
||||
|
||||
def _digits(v):
|
||||
return "".join(c for c in str(v or "") if c.isdigit())
|
||||
|
||||
|
||||
def st_status(ncm_status, mva_status, cest_status, erp_cest, ba_item_cest, ba_ncm_sh_len):
|
||||
"""ST_CONFIRMED needs a unique BA Anexo 1 item whose CEST the ERP product itself carries
|
||||
(or a confirmed CEST). ST_NOT_APPLICABLE needs BOTH official lists (BA Anexo 1 and CEST) silent
|
||||
for a valid NCM. A CEST *candidate* never confirms ST."""
|
||||
if ncm_status != "EXACT_NCM":
|
||||
return "ST_UNKNOWN", "NCM_NOT_VALID"
|
||||
if mva_status == "MVA_RESOLVED":
|
||||
if cest_status == "CEST_CONFIRMED" or (_digits(erp_cest) and _digits(erp_cest) == _digits(ba_item_cest)):
|
||||
return "ST_CONFIRMED", "UNIQUE_BA_ANNEX1_ITEM_CORROBORATED_BY_CEST"
|
||||
return "ST_CANDIDATE", "UNIQUE_BA_ANNEX1_ITEM_WITHOUT_CEST_CORROBORATION"
|
||||
if mva_status == "MVA_CANDIDATES":
|
||||
return "ST_CANDIDATE", "MULTIPLE_BA_ANNEX1_ITEMS"
|
||||
if mva_status == "NO_MVA_MAPPING_FOUND":
|
||||
if cest_status == "NO_CEST_MAPPING_FOUND":
|
||||
return "ST_NOT_APPLICABLE", "NCM_ABSENT_FROM_BA_ANNEX1_AND_CEST_LIST"
|
||||
return "ST_UNKNOWN", "CEST_LISTED_BUT_BA_ANNEX1_ABSENT"
|
||||
return "ST_UNKNOWN", "NO_ST_EVIDENCE"
|
||||
|
||||
|
||||
def withheld_status(purchase_rows):
|
||||
"""purchase_rows: entrada item rows for the product with keys icms_retido (S/N), valor_retido, sub_valor.
|
||||
CONFIRMED only with positive purchase-side evidence; explicit 'N'/0 is NOT_CONFIRMED; no rows UNKNOWN."""
|
||||
if not purchase_rows:
|
||||
return "ST_WITHHOLDING_UNKNOWN", "NO_PURCHASE_ITEM_FOR_PRODUCT"
|
||||
for r in purchase_rows:
|
||||
if str(r.get("icms_retido") or "").strip().upper() == "S" or float(r.get("valor_retido") or 0) > 0 or float(r.get("sub_valor") or 0) > 0:
|
||||
return "ST_ALREADY_WITHHELD_CONFIRMED", "PURCHASE_ITEM_SHOWS_WITHHELD_ICMS_ST"
|
||||
return "ST_ALREADY_WITHHELD_NOT_CONFIRMED", "PURCHASE_ITEMS_FLAG_ICMS_RETIDO_N_AND_ZERO_ST_VALUES"
|
||||
|
||||
|
||||
def product_role(erp_tipo, prod_cozinha, has_recipe, sold_as_composition, service_flag, purchase_evidence):
|
||||
ev = []
|
||||
if service_flag:
|
||||
return "SERVICE", ["PRODUTO_SERVICO"]
|
||||
if str(erp_tipo or "").strip().upper() == "MATERIA-PRIMA":
|
||||
return "INPUT", ["C000025.TIPO=MATERIA-PRIMA"]
|
||||
own = []
|
||||
if prod_cozinha: own.append("PROD_COZINHA=1")
|
||||
if has_recipe: own.append("RECEITA_INGREDIENTES")
|
||||
if sold_as_composition: own.append("C000032.TIPO=COMPOSICAO")
|
||||
if own and purchase_evidence:
|
||||
return "UNKNOWN", own + ["ALSO_HAS_PURCHASE_EVIDENCE_CONFLICT"]
|
||||
if own:
|
||||
return "OWN_PRODUCTION", own
|
||||
if str(erp_tipo or "").strip().upper() == "VENDA" and purchase_evidence:
|
||||
return "RESALE", ["C000025.TIPO=VENDA", "PURCHASE_DOCUMENT_EVIDENCE"]
|
||||
return "UNKNOWN", ev or ["NO_OBJECTIVE_ROLE_EVIDENCE"]
|
||||
|
||||
|
||||
def csosn_v2(current, company_default, ncm_status, st, withheld, role, csosn_domain):
|
||||
"""States: CSOSN_RESOLVED / CANDIDATE / CONTEXT_REQUIRED / CONFLICT / UNRESOLVED.
|
||||
RESOLVED only where the fiscal condition is explicitly proven (ST confirmed + withholding confirmed + role
|
||||
known -> 500). Everything else stays conservative; never assumes 102."""
|
||||
cur = None if current is None or str(current).strip() == "" else str(current).strip()
|
||||
out = {"field": "CSOSN", "current_value": cur, "proposed_csosn": None, "write_eligible": False, "engine": ICMS_CONTEXT_ENGINE_VERSION}
|
||||
def res(state, reason, conf, action, proposed=None, **kw):
|
||||
return {**out, "status": state, "reason_code": reason, "confidence": conf, "action_class": action, "proposed_csosn": proposed, **kw}
|
||||
if ncm_status != "EXACT_NCM":
|
||||
return res("CSOSN_UNRESOLVED", "NCM_NOT_VALID", "NONE", "NO_RULE")
|
||||
if role in ("INPUT", "SERVICE"):
|
||||
return res("CSOSN_CONTEXT_REQUIRED", f"ROLE_{role}_NOT_A_MERCHANDISE_SALE_DEFAULT", "NONE", "CONTEXT_REQUIRED")
|
||||
if st == "ST_CONFIRMED":
|
||||
if withheld == "ST_ALREADY_WITHHELD_CONFIRMED" and role in ("RESALE",):
|
||||
state = "CSOSN_RESOLVED"
|
||||
return res(state, "ST_CONFIRMED_AND_WITHHOLDING_CONFIRMED_RESALE", "HIGH", "ALREADY_CORRECT" if cur == "500" else "AUTO_APPLY_SAFE", "500")
|
||||
if withheld == "ST_ALREADY_WITHHELD_NOT_CONFIRMED" and cur == "500":
|
||||
return res("CSOSN_CONFLICT", "CSOSN_500_BUT_PURCHASE_ITEMS_SHOW_NO_ST_WITHHELD", "MEDIUM", "REVIEW_REQUIRED")
|
||||
if withheld == "ST_ALREADY_WITHHELD_CONFIRMED" and cur != "500":
|
||||
return res("CSOSN_CONFLICT", "ST_WITHHELD_CONFIRMED_BUT_CSOSN_NOT_500", "MEDIUM", "REVIEW_REQUIRED", "500")
|
||||
return res("CSOSN_CONTEXT_REQUIRED", "ST_CONFIRMED_WITHHOLDING_OR_ROLE_UNPROVEN", "NONE", "CONTEXT_REQUIRED",
|
||||
consistency_flag=None if cur == "500" else "ST_CONFIRMED_BUT_CSOSN_NOT_500")
|
||||
if st in ("ST_CANDIDATE", "ST_UNKNOWN"):
|
||||
return res("CSOSN_CONTEXT_REQUIRED", f"{st}_APPLICABILITY_UNPROVEN", "NONE", "CONTEXT_REQUIRED")
|
||||
# ST_NOT_APPLICABLE: taxed vs exempt/not-taxed treatment (102 vs 400/300) has no source
|
||||
if role in ("RESALE", "OWN_PRODUCTION") and cur == company_default and cur in csosn_domain:
|
||||
return res("CSOSN_CANDIDATE", "NON_ST_MATCHES_COMPANY_DEFAULT_ICMS_TREATMENT_UNPROVEN", "MEDIUM", "REVIEW_REQUIRED", cur)
|
||||
if role == "UNKNOWN":
|
||||
return res("CSOSN_CONTEXT_REQUIRED", "PRODUCT_ROLE_UNKNOWN", "NONE", "CONTEXT_REQUIRED")
|
||||
return res("CSOSN_CANDIDATE" if cur else "CSOSN_CONTEXT_REQUIRED", "DIFFERS_FROM_OR_LACKS_COMPANY_DEFAULT_TREATMENT_UNPROVEN", "LOW", "REVIEW_REQUIRED" if cur else "CONTEXT_REQUIRED")
|
||||
|
||||
|
||||
def st_conflict_class(cest_status, mva_status, ncm_sh_len, erp_cest, ba_item_cest, erp_cfop, history_emitted_current, non_st_but_erp_5405):
|
||||
"""Primary label for an ST-related CFOP/CSOSN conflict, plus all evidence flags."""
|
||||
flags = []
|
||||
if non_st_but_erp_5405 or (_digits(erp_cest) and _digits(ba_item_cest) and _digits(erp_cest) != _digits(ba_item_cest)):
|
||||
flags.append("BA_ST_RULE_CONFLICT")
|
||||
if history_emitted_current:
|
||||
flags.append("ERP_HISTORY_CONFLICT")
|
||||
if cest_status == "CEST_CANDIDATES":
|
||||
flags.append("MULTIPLE_CEST")
|
||||
if cest_status == "CEST_DESCRIPTION_MISMATCH":
|
||||
flags.append("DESCRIPTION_MISMATCH")
|
||||
if ncm_sh_len is not None and ncm_sh_len <= 4:
|
||||
flags.append("NCM_TOO_BROAD")
|
||||
return (flags[0] if flags else "NO_EVIDENCE"), flags or ["NO_EVIDENCE"]
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Minimal job queue for Agent orchestration: create, atomically claim, complete, fail."""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from .models import Job
|
||||
|
||||
_VALID_KINDS = {"REPROCESS_ALL", "REPROCESS_PRODUCTS"}
|
||||
_MAX_ERROR_LENGTH = 2000
|
||||
|
||||
|
||||
def create_job(session, agent, kind, payload=None):
|
||||
if kind not in _VALID_KINDS:
|
||||
raise ValueError(f"unsupported job kind: {kind}")
|
||||
job = Job(agent_id=agent.id, kind=kind, status="PENDING", payload=payload or {})
|
||||
session.add(job)
|
||||
session.commit()
|
||||
return job
|
||||
|
||||
|
||||
def claim_next_job(session, agent):
|
||||
"""Atomically claim the oldest PENDING job for this agent only.
|
||||
A single UPDATE...WHERE status='PENDING' guards against two concurrent
|
||||
pollers (this agent or any other) claiming the same job twice."""
|
||||
candidate = session.scalar(
|
||||
select(Job.id)
|
||||
.where(Job.agent_id == agent.id, Job.status == "PENDING")
|
||||
.order_by(Job.created_at.asc())
|
||||
.limit(1)
|
||||
)
|
||||
if candidate is None:
|
||||
return None
|
||||
now = datetime.now(timezone.utc)
|
||||
result = session.execute(
|
||||
update(Job)
|
||||
.where(Job.id == candidate, Job.status == "PENDING")
|
||||
.values(status="RUNNING", started_at=now, attempts=Job.attempts + 1)
|
||||
)
|
||||
session.commit()
|
||||
if result.rowcount != 1:
|
||||
return None # another poller won the race; caller can poll again
|
||||
return session.get(Job, candidate)
|
||||
|
||||
|
||||
def complete_job(session, job, result_summary):
|
||||
result = session.execute(
|
||||
update(Job)
|
||||
.where(Job.id == job.id, Job.status == "RUNNING")
|
||||
.values(status="COMPLETED", finished_at=datetime.now(timezone.utc), result_summary=result_summary)
|
||||
)
|
||||
session.commit()
|
||||
return result.rowcount == 1
|
||||
|
||||
|
||||
def fail_job(session, job, error):
|
||||
sanitized = str(error)[:_MAX_ERROR_LENGTH]
|
||||
result = session.execute(
|
||||
update(Job)
|
||||
.where(Job.id == job.id, Job.status == "RUNNING")
|
||||
.values(status="FAILED", finished_at=datetime.now(timezone.utc), error=sanitized)
|
||||
)
|
||||
session.commit()
|
||||
return result.rowcount == 1
|
||||
@@ -0,0 +1,39 @@
|
||||
"""FastAPI application factory."""
|
||||
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from .config import settings
|
||||
from .db import SessionLocal, database_is_available
|
||||
from .api import router
|
||||
from .version import GTI_FISCAL_VERSION
|
||||
|
||||
logging.basicConfig(level=getattr(logging, settings.log_level.upper(), logging.INFO))
|
||||
logger = logging.getLogger("gti_fiscal_hub")
|
||||
|
||||
|
||||
def create_app(database_probe: Callable[[], bool] = database_is_available, session_factory=SessionLocal, bootstrap_token: str | None = None) -> FastAPI:
|
||||
app = FastAPI(title="GTI Fiscal Hub", version=GTI_FISCAL_VERSION)
|
||||
if settings.environment == "production" and settings.agent_dev_token:
|
||||
raise RuntimeError("AGENT_DEV_TOKEN must not be set when APP_ENV=production (register real Agents with central/scripts/register_agent.py)")
|
||||
logger.info("starting GTI Fiscal Hub (environment=%s)", settings.environment)
|
||||
|
||||
@app.get("/health")
|
||||
def health() -> dict[str, str]:
|
||||
if not database_probe():
|
||||
logger.warning("health check failed: postgres unavailable")
|
||||
raise HTTPException(status_code=503, detail="postgres unavailable")
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.get("/version")
|
||||
def version() -> dict[str, str]:
|
||||
from .classification import DECISION_VERSION, ENGINE_VERSION
|
||||
from .version import API_COMPAT_MAJOR_MINOR, GTI_FISCAL_VERSION
|
||||
return {"version": GTI_FISCAL_VERSION, "api_compat": API_COMPAT_MAJOR_MINOR, "engine_version": ENGINE_VERSION, "decision_version": DECISION_VERSION}
|
||||
|
||||
app.include_router(router(session_factory, settings.agent_dev_token if bootstrap_token is None else bootstrap_token))
|
||||
return app
|
||||
|
||||
|
||||
app = create_app()
|
||||
@@ -0,0 +1,341 @@
|
||||
"""PostgreSQL records owned by the Fiscal Hub."""
|
||||
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, JSON, String, Text, UniqueConstraint, func
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class Dataset(Base):
|
||||
__tablename__ = "datasets"
|
||||
__table_args__ = (UniqueConstraint("provider", "sha256", name="uq_dataset_provider_hash"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
kind: Mapped[str] = mapped_column(String(32), index=True)
|
||||
provider: Mapped[str] = mapped_column(String(128))
|
||||
sha256: Mapped[str] = mapped_column(String(64))
|
||||
raw_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
version_label: Mapped[str] = mapped_column(String(256))
|
||||
act: Mapped[str] = mapped_column(Text)
|
||||
source_url: Mapped[str] = mapped_column(Text)
|
||||
downloaded_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
status: Mapped[str] = mapped_column(String(32), index=True)
|
||||
record_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
|
||||
|
||||
class NcmEntry(Base):
|
||||
__tablename__ = "ncm_entries"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "code", "valid_from", "valid_to", name="uq_ncm_versioned_entry"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
code: Mapped[str] = mapped_column(String(16), index=True)
|
||||
description: Mapped[str] = mapped_column(Text)
|
||||
valid_from: Mapped[str] = mapped_column(String(32))
|
||||
valid_to: Mapped[str] = mapped_column(String(32))
|
||||
|
||||
|
||||
class LegacyNcmEntry(Base):
|
||||
__tablename__ = "legacy_ncm_entries"
|
||||
code: Mapped[str] = mapped_column(String(16), primary_key=True)
|
||||
source: Mapped[str] = mapped_column(Text)
|
||||
source_date: Mapped[str] = mapped_column(String(32))
|
||||
reason: Mapped[str] = mapped_column(Text)
|
||||
successor_code: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
|
||||
|
||||
class CestEntry(Base):
|
||||
__tablename__ = "cest_entries"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "code", name="uq_cest_versioned_entry"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
code: Mapped[str] = mapped_column(String(16), index=True)
|
||||
description: Mapped[str] = mapped_column(Text)
|
||||
annex: Mapped[str] = mapped_column(String(8))
|
||||
|
||||
|
||||
class CestNcmEntry(Base):
|
||||
__tablename__ = "cest_ncm_entries"
|
||||
__table_args__ = (UniqueConstraint("cest_entry_id", "ncm_code", name="uq_cest_ncm_link"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
cest_entry_id: Mapped[int] = mapped_column(ForeignKey("cest_entries.id"), index=True)
|
||||
ncm_code: Mapped[str] = mapped_column(String(16), index=True)
|
||||
|
||||
|
||||
class TipiEntry(Base):
|
||||
__tablename__ = "tipi_entries"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
ncm: Mapped[str] = mapped_column(String(8), index=True)
|
||||
ex: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
description: Mapped[str] = mapped_column(Text)
|
||||
ipi_rate: Mapped[float | None] = mapped_column(nullable=True)
|
||||
ipi_rate_text: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
source_row: Mapped[int] = mapped_column(Integer)
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
|
||||
|
||||
class BaStMvaEntry(Base):
|
||||
__tablename__ = "ba_st_mva_entries"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
item: Mapped[str] = mapped_column(String(32))
|
||||
segment: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
cest: Mapped[str | None] = mapped_column(String(16), index=True, nullable=True)
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
mva_original: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
mva_adjusted: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
agreement_info: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
source_page: Mapped[int] = mapped_column(Integer)
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
|
||||
class BaStMvaNcmSh(Base):
|
||||
__tablename__ = "ba_st_mva_ncm_sh"
|
||||
__table_args__ = (UniqueConstraint("entry_id", "ncm_sh", name="uq_ba_st_mva_entry_ncm_sh"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
entry_id: Mapped[int] = mapped_column(ForeignKey("ba_st_mva_entries.id"), index=True)
|
||||
ncm_sh: Mapped[str] = mapped_column(String(32), index=True)
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
|
||||
|
||||
class FiscalCodeEntry(Base):
|
||||
"""Small national fiscal code domains sourced from the official NF-e MOC
|
||||
manual: ORIGEM_MERCADORIA, CST_ICMS, CSOSN, CST_PIS, CST_COFINS."""
|
||||
__tablename__ = "fiscal_code_entries"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "domain", "code", name="uq_fiscal_code_entry"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
domain: Mapped[str] = mapped_column(String(32), index=True)
|
||||
code: Mapped[str] = mapped_column(String(8), index=True)
|
||||
description: Mapped[str] = mapped_column(Text)
|
||||
source_section: Mapped[str] = mapped_column(String(96))
|
||||
metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
|
||||
|
||||
class CfopEntry(Base):
|
||||
"""CFOP (Código Fiscal de Operações e Prestações) — Portal Nacional NF-e."""
|
||||
__tablename__ = "cfop_entries"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "code", name="uq_cfop_entry"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
code: Mapped[str] = mapped_column(String(8), index=True)
|
||||
title: Mapped[str] = mapped_column(Text)
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
effective_from: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
effective_to: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
ind_nfe: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
ind_comunica: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
ind_transp: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
ind_devol: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
ind_retor: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
ind_anula: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
ind_remes: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
ind_comb: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
ind_exc_ibscbs: Mapped[str | None] = mapped_column(String(4), nullable=True)
|
||||
|
||||
|
||||
class FcpRate(Base):
|
||||
"""FCP (Fundo de Combate à Pobreza) alíquotas por UF — Portal Nacional NF-e."""
|
||||
__tablename__ = "fcp_rates"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "uf", name="uq_fcp_rate_uf"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
uf_code: Mapped[str] = mapped_column(String(4))
|
||||
uf: Mapped[str] = mapped_column(String(4), index=True)
|
||||
uf_name: Mapped[str] = mapped_column(String(64))
|
||||
aliq_1: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
aliq_2: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
aliq_3: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
observacao: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
|
||||
class AnpProductCode(Base):
|
||||
"""cProdANP — official code list used by NF-e, published on the Portal Nacional NF-e."""
|
||||
__tablename__ = "anp_product_codes"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "cprod_anp", name="uq_anp_product_code"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
cprod_anp: Mapped[str] = mapped_column(String(16), index=True)
|
||||
|
||||
|
||||
class NcmUtribEntry(Base):
|
||||
"""NCM x uTrib (unidade tributável de comércio exterior) — Portal Nacional NF-e."""
|
||||
__tablename__ = "ncm_utrib_entries"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "ncm", name="uq_ncm_utrib_entry"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
ncm: Mapped[str] = mapped_column(String(16), index=True)
|
||||
effective_from: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
effective_to: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
utrib_abbreviation: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
||||
utrib_description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
|
||||
class RtcBiofuelIndexEntry(Base):
|
||||
"""RTC pBioObrigatorio (índice de biocombustível) by cProdANP — Portal Nacional NF-e."""
|
||||
__tablename__ = "rtc_biofuel_index_entries"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
cprod_anp: Mapped[str | None] = mapped_column(String(16), index=True, nullable=True)
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
p_bio_percent: Mapped[float | None] = mapped_column(nullable=True)
|
||||
effective_from: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
effective_to: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
is_historical: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
|
||||
|
||||
class RtcCredPresEntry(Base):
|
||||
"""RTC cCredPres (crédito presumido IBS/CBS), LC 214/2025 — Portal Nacional NF-e.
|
||||
Keeps every official field distinctly so future automatic determination
|
||||
doesn't require re-parsing the source."""
|
||||
__tablename__ = "rtc_cred_pres_entries"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "code", name="uq_rtc_cred_pres_code"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
code: Mapped[str] = mapped_column(String(8), index=True)
|
||||
description: Mapped[str] = mapped_column(Text)
|
||||
legal_basis: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
apropria_via_nf: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
||||
apropria_via_evento: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
||||
ind_deduz_cred_pres: Mapped[str | None] = mapped_column(String(8), nullable=True)
|
||||
ind_g_cbs_cred_pres: Mapped[str | None] = mapped_column(String(8), nullable=True)
|
||||
ind_g_ibs_cred_pres: Mapped[str | None] = mapped_column(String(8), nullable=True)
|
||||
aliquota_cbs_text: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
aliquota_ibs_text: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
p_aliq_cred_pres_cbs: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
p_aliq_cred_pres_ibs: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
p_red_transicao_ibs: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
# Free text in some rows (e.g. "cClass de alíquota zero"), not always the numeric code.
|
||||
cclass_nota_referenciada: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
d_ini_vig_cbs: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
d_fim_vig_cbs: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
d_ini_vig_ibs: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
d_fim_vig_ibs: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
|
||||
|
||||
class Tenant(Base):
|
||||
__tablename__ = "tenants"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(128), unique=True)
|
||||
|
||||
|
||||
class Agent(Base):
|
||||
__tablename__ = "agents"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True) # company_id
|
||||
name: Mapped[str] = mapped_column(String(128))
|
||||
token_hash: Mapped[str] = mapped_column(String(64), unique=True)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
|
||||
last_seen_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_status: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
schema_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
agent_version: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class Job(Base):
|
||||
__tablename__ = "jobs"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
agent_id: Mapped[int] = mapped_column(ForeignKey("agents.id"), index=True)
|
||||
kind: Mapped[str] = mapped_column(String(32), index=True) # REPROCESS_ALL | REPROCESS_PRODUCTS
|
||||
status: Mapped[str] = mapped_column(String(16), index=True, default="PENDING", server_default="PENDING")
|
||||
payload: Mapped[dict] = mapped_column(JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
attempts: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
result_summary: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
|
||||
class ClassificationRequest(Base):
|
||||
__tablename__ = "classification_requests"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
|
||||
agent_id: Mapped[int] = mapped_column(ForeignKey("agents.id"), index=True)
|
||||
fingerprint: Mapped[str] = mapped_column(String(64), unique=True)
|
||||
payload: Mapped[dict] = mapped_column(JSON)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class ClassificationResult(Base):
|
||||
__tablename__ = "classification_results"
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
request_id: Mapped[int] = mapped_column(ForeignKey("classification_requests.id"), unique=True)
|
||||
result: Mapped[dict] = mapped_column(JSON)
|
||||
|
||||
|
||||
class ClassificationCache(Base):
|
||||
__tablename__ = "classification_cache"
|
||||
fingerprint: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
|
||||
result: Mapped[dict] = mapped_column(JSON)
|
||||
|
||||
|
||||
class RtcCstIbsCbsEntry(Base):
|
||||
"""RTC CST-IBS/CBS (sheet 'CST ...' of the official cClassTrib workbook)."""
|
||||
__tablename__ = "rtc_cst_ibscbs_entries"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "cst", name="uq_rtc_cst_ibscbs"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
cst: Mapped[str] = mapped_column(String(8), index=True)
|
||||
description: Mapped[str] = mapped_column(Text)
|
||||
indicators: Mapped[dict] = mapped_column(JSON)
|
||||
|
||||
|
||||
class RtcCClassTribEntry(Base):
|
||||
"""RTC cClassTrib (sheet 'cClass ...'). Every official column is kept:
|
||||
key fields typed, all ind_* flags in `indicators` keyed by official header."""
|
||||
__tablename__ = "rtc_cclasstrib_entries"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "cclasstrib", name="uq_rtc_cclasstrib"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
cst: Mapped[str] = mapped_column(String(8), index=True)
|
||||
cst_description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
cclasstrib: Mapped[str] = mapped_column(String(8), index=True)
|
||||
name: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
lc_redacao: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
lc_214: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
regulamento_cbs: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
regulamento_ibs: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
tipo_aliquota: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
p_red_ibs: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
p_red_cbs: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
d_ini_vig: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
d_fim_vig: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
data_atualizacao: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
anexo: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
link: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
indicators: Mapped[dict] = mapped_column(JSON)
|
||||
|
||||
|
||||
class RtcCbsRateEntry(Base):
|
||||
"""RTC CBS rate per year (IT 2026.002). rate is NULL when the official
|
||||
text has no number yet (e.g. 'aguarda legislação')."""
|
||||
__tablename__ = "rtc_cbs_rate_entries"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", "year", name="uq_rtc_cbs_rate_year"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
dataset_id: Mapped[int] = mapped_column(ForeignKey("datasets.id"), index=True)
|
||||
year: Mapped[int] = mapped_column(Integer, index=True)
|
||||
rate_text: Mapped[str] = mapped_column(Text)
|
||||
rate: Mapped[float | None] = mapped_column(nullable=True)
|
||||
|
||||
|
||||
class FiscalContextEntry(Base):
|
||||
"""Operator-supplied fiscal fact (product role, origin, ST retention, treatment, ...). Hierarchy: PRODUCT > GROUP > COMPANY."""
|
||||
__tablename__ = "fiscal_context_entries"
|
||||
__table_args__ = (UniqueConstraint("tenant_id", "company_key", "scope", "scope_key", "key", name="uq_fiscal_context_entry"),)
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id"), index=True)
|
||||
company_key: Mapped[str] = mapped_column(String(64), index=True)
|
||||
scope: Mapped[str] = mapped_column(String(8))
|
||||
scope_key: Mapped[str] = mapped_column(String(64), default="", server_default="")
|
||||
key: Mapped[str] = mapped_column(String(48))
|
||||
value: Mapped[str] = mapped_column(Text)
|
||||
source: Mapped[str] = mapped_column(String(16), default="OPERATOR", server_default="OPERATOR")
|
||||
reason: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Product Fiscal Defaults Engine V1 — C000025.CFOP / CSOSN / CST default decisions.
|
||||
|
||||
Pure functions. Scope: intra-UF (BA) NFC-e-style sale of merchandise, company CRT=1 (Simples).
|
||||
Historical documents are NEVER an input here; they are used by the runner only to validate.
|
||||
|
||||
CFOP rule chain (all must hold, else CANDIDATE/CONTEXT/UNRESOLVED):
|
||||
ERP CFOP mapping HIGH (product default, saída) + NCM EXACT + current CFOP in official table + first digit 5
|
||||
(saída, intra-UF) + ERP declares resale family (5102/5405: 'adquirida ou recebida de terceiros')
|
||||
+ ST applicability known from BA Anexo 1 (MVA_RESOLVED -> subject, NO_MVA_MAPPING_FOUND -> not subject)
|
||||
=> derived CFOP = 5405 if subject else 5102. Equal to current => RESOLVED (already correct).
|
||||
Different => CANDIDATE only: one of the inputs is wrong and the engine cannot tell which
|
||||
(ST retention by the supplier is not available), so it is never write-eligible.
|
||||
The chain never yields a write: filling a blank CFOP would need the resale-vs-production declaration that
|
||||
only the current CFOP carries, so blank => OPERATION_CONTEXT_REQUIRED.
|
||||
|
||||
CSOSN: no dataset states whether a product is taxed (102), exempt/not taxed (400/300), ST-charged (500)...
|
||||
=> never RESOLVED. Company default C000004.ICMS_CSOSN is only a CANDIDATE anchor for non-ST products.
|
||||
"""
|
||||
PRODUCT_DEFAULTS_ENGINE_VERSION = "product-defaults-v1"
|
||||
RESALE_CFOPS = {"5102", "5405"}
|
||||
|
||||
|
||||
def _blank(v):
|
||||
return v is None or str(v).strip() == ""
|
||||
|
||||
|
||||
def _out(field, current, proposed, state, reason, action, **extra):
|
||||
return {"field": field, "current_value": current, "proposed_value": proposed, "status": state, "reason_code": reason,
|
||||
"action_class": action, "write_eligible": False, "engine": PRODUCT_DEFAULTS_ENGINE_VERSION, **extra}
|
||||
|
||||
|
||||
def st_applicability(ncm_status, mva_status):
|
||||
if ncm_status != "EXACT_NCM":
|
||||
return "UNKNOWN_NCM_NOT_VALID"
|
||||
if mva_status == "MVA_RESOLVED":
|
||||
return "SUBJECT"
|
||||
if mva_status == "NO_MVA_MAPPING_FOUND":
|
||||
return "NOT_SUBJECT"
|
||||
return "UNKNOWN_UNCONFIRMED"
|
||||
|
||||
|
||||
def cfop_default(product, ncm_status, mva_status, cfop_valid):
|
||||
cur = None if _blank(product.get("CFOP")) else str(product["CFOP"]).strip()
|
||||
st = st_applicability(ncm_status, mva_status)
|
||||
base = {"st_applicability": st}
|
||||
if st == "UNKNOWN_NCM_NOT_VALID":
|
||||
return _out("CFOP", cur, None, "CFOP_UNRESOLVED", "NCM_NOT_VALID_CANNOT_ASSESS_ST", "NO_RULE", **base)
|
||||
if cur is None:
|
||||
return _out("CFOP", cur, None, "CFOP_OPERATION_CONTEXT_REQUIRED", "NO_DECLARED_RESALE_OR_PRODUCTION_FAMILY", "OPERATION_CONTEXT_REQUIRED", **base)
|
||||
if cur not in cfop_valid:
|
||||
return _out("CFOP", cur, None, "CFOP_UNRESOLVED", "CURRENT_CFOP_NOT_IN_OFFICIAL_TABLE", "REVIEW_REQUIRED", **base)
|
||||
if cur[0] != "5":
|
||||
return _out("CFOP", cur, None, "CFOP_OPERATION_CONTEXT_REQUIRED", "DIRECTION_OR_UF_SCOPE_NOT_INTRA_UF_SAIDA", "OPERATION_CONTEXT_REQUIRED", **base)
|
||||
if cur not in RESALE_CFOPS:
|
||||
return _out("CFOP", cur, None, "CFOP_OPERATION_CONTEXT_REQUIRED", "NO_RULE_FOR_DECLARED_CFOP_FAMILY", "OPERATION_CONTEXT_REQUIRED", **base)
|
||||
if st == "UNKNOWN_UNCONFIRMED":
|
||||
return _out("CFOP", cur, None, "CFOP_OPERATION_CONTEXT_REQUIRED", "ST_APPLICABILITY_UNCONFIRMED_NEEDS_CEST_CONFIRMATION", "OPERATION_CONTEXT_REQUIRED", **base)
|
||||
derived = "5405" if st == "SUBJECT" else "5102"
|
||||
if derived == cur:
|
||||
return _out("CFOP", cur, derived, "CFOP_DEFAULT_RESOLVED", "CONSISTENT_WITH_ST_STATUS_SAME_VALUE", "ALREADY_CORRECT", **base)
|
||||
return _out("CFOP", cur, derived, "CFOP_DEFAULT_CANDIDATE", "ST_STATUS_CONFLICTS_WITH_ERP_CFOP_RETENTION_CONTEXT_MISSING", "REVIEW_REQUIRED", **base)
|
||||
|
||||
|
||||
def csosn_default(product, ncm_status, mva_status, csosn_valid, company_default):
|
||||
cur = None if _blank(product.get("CSOSN")) else str(product["CSOSN"]).strip()
|
||||
st = st_applicability(ncm_status, mva_status)
|
||||
base = {"st_applicability": st, "company_default": company_default}
|
||||
if st == "UNKNOWN_NCM_NOT_VALID":
|
||||
return _out("CSOSN", cur, None, "CSOSN_UNRESOLVED", "NCM_NOT_VALID_CANNOT_ASSESS_ST", "NO_RULE", **base)
|
||||
if st == "UNKNOWN_UNCONFIRMED":
|
||||
return _out("CSOSN", cur, None, "CSOSN_OPERATION_CONTEXT_REQUIRED", "ST_APPLICABILITY_UNCONFIRMED", "OPERATION_CONTEXT_REQUIRED", **base)
|
||||
if st == "SUBJECT":
|
||||
flag = "ST_SUBJECT_BUT_CSOSN_NOT_500" if cur != "500" else "ST_CSOSN_500_RETENTION_UNVERIFIED"
|
||||
return _out("CSOSN", cur, None, "CSOSN_OPERATION_CONTEXT_REQUIRED", "ST_RETENTION_BY_SUPPLIER_UNKNOWN", "OPERATION_CONTEXT_REQUIRED", consistency_flag=flag, **base)
|
||||
# NOT_SUBJECT: 102 / 400 / 300 / 103 depend on the product's ICMS treatment, for which no source exists
|
||||
if cur is not None and cur == company_default and cur in csosn_valid:
|
||||
return _out("CSOSN", cur, cur, "CSOSN_DEFAULT_CANDIDATE", "MATCHES_COMPANY_DEFAULT_PRODUCT_ICMS_TREATMENT_UNPROVEN", "REVIEW_REQUIRED", **base)
|
||||
if cur is None:
|
||||
return _out("CSOSN", cur, company_default, "CSOSN_DEFAULT_CANDIDATE", "BLANK_COMPANY_DEFAULT_ONLY_ANCHOR_TREATMENT_UNPROVEN", "REVIEW_REQUIRED", **base)
|
||||
return _out("CSOSN", cur, None, "CSOSN_DEFAULT_CANDIDATE", "DIFFERS_FROM_COMPANY_DEFAULT_NO_TREATMENT_SOURCE", "REVIEW_REQUIRED", **base)
|
||||
|
||||
|
||||
def cst_status(product, cst_icms_codes, origem_codes):
|
||||
cur = None if _blank(product.get("CST")) else str(product["CST"]).strip()
|
||||
valid = cur is not None and len(cur) == 3 and cur[1:] in cst_icms_codes and cur[0] in origem_codes
|
||||
return {"field": "CST", "current_value": cur, "status": "CST_REFERENCE_ONLY" if valid else "CST_INVALID_REVIEW",
|
||||
"reason_code": "CRT1_EMISSION_USES_CSOSN_CST_IS_ORIGIN_PLUS_CST" if valid else "NOT_ORIGIN_DIGIT_PLUS_VALID_CST",
|
||||
"origin_digit": cur[0] if valid else None, "write_eligible": False, "engine": PRODUCT_DEFAULTS_ENGINE_VERSION}
|
||||
|
||||
|
||||
def validate_against_history(proposed, emitted_values):
|
||||
"""History validates only; returns MATCH / MISMATCH / UNVALIDATED."""
|
||||
if not emitted_values or proposed is None:
|
||||
return "UNVALIDATED"
|
||||
return "MATCH" if set(emitted_values) == {proposed} else "MISMATCH"
|
||||
@@ -0,0 +1,4 @@
|
||||
"""Single source of truth for the release version (RC1). Fiscal semantics are versioned separately
|
||||
(ENGINE_VERSION/DECISION_VERSION in classification.py, DECISION_ENGINE_VERSION etc.) and are NOT changed by a release bump."""
|
||||
GTI_FISCAL_VERSION = "1.0.0-rc1"
|
||||
API_COMPAT_MAJOR_MINOR = "1.0" # an Agent 1.0.x talks to any Hub 1.0.x
|
||||
@@ -0,0 +1,15 @@
|
||||
[project]
|
||||
name = "gti-fiscal-hub"
|
||||
version = "1.0.0rc1"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"fastapi>=0.115,<1",
|
||||
"uvicorn[standard]>=0.30,<1",
|
||||
"sqlalchemy>=2.0,<3",
|
||||
"psycopg[binary]>=3.2,<4",
|
||||
"alembic>=1.13,<2",
|
||||
"httpx>=0.27,<1",
|
||||
"openpyxl>=3.1,<4",
|
||||
"pypdf>=4.0,<6",
|
||||
"pdfplumber>=0.11,<1",
|
||||
]
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Promote RICMS-BA Anexo 1 (BA_ST_MVA) from the local official PDF snapshot.
|
||||
|
||||
No network fetch: RICMS-BA Anexo 1 has no stable public direct-download URL
|
||||
(the SEFAZ-BA legislation portal serves it via a session-gated viewer), so
|
||||
the authoritative copy is kept in sources/raw/bootstrap_v1/. This script
|
||||
re-validates it against the currently ACTIVE Postgres dataset and only
|
||||
promotes a new version if the parsed content actually differs, never
|
||||
overwriting a valid ACTIVE version with a partial/regressed parse.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from pypdf import PdfReader
|
||||
|
||||
from central.app.datasets.ba_st_mva import parse_page
|
||||
from central.app.datasets.service import DatasetService
|
||||
from central.app.db import SessionLocal
|
||||
from central.app.models import Dataset
|
||||
|
||||
SOURCE = Path("sources/raw/bootstrap_v1/decreto_2012_13780_ricms_anexo_1_vigente_2026.pdf")
|
||||
SOURCE_URL = f"file://{SOURCE.name}"
|
||||
|
||||
|
||||
def main():
|
||||
raw_bytes = SOURCE.read_bytes()
|
||||
raw_sha256 = hashlib.sha256(raw_bytes).hexdigest()
|
||||
|
||||
records, rejected = [], []
|
||||
for page, value in enumerate(PdfReader(SOURCE).pages, start=1):
|
||||
found, failed = parse_page(value.extract_text() or "", page)
|
||||
records.extend(found)
|
||||
rejected.extend(failed)
|
||||
|
||||
if not records:
|
||||
print(json.dumps({"status": "FAILED", "reason": "parser produced zero records; keeping existing ACTIVE dataset"}))
|
||||
return
|
||||
|
||||
with SessionLocal() as session:
|
||||
current_active = session.scalar(
|
||||
__import__("sqlalchemy").select(Dataset).where(Dataset.kind == "BA_ST_MVA", Dataset.status == "ACTIVE")
|
||||
)
|
||||
if current_active is not None and current_active.raw_sha256 == raw_sha256:
|
||||
print(json.dumps({
|
||||
"status": "UNCHANGED", "dataset_id": current_active.id, "raw_sha256": raw_sha256,
|
||||
"record_count": current_active.record_count,
|
||||
}))
|
||||
return
|
||||
if current_active is not None and len(records) < current_active.record_count * 0.9:
|
||||
# Smoke gate: refuse to promote a parse that lost >10% of records versus
|
||||
# the currently ACTIVE version — never silently replace a good version
|
||||
# with a broken one.
|
||||
print(json.dumps({
|
||||
"status": "FAILED",
|
||||
"reason": f"new parse yielded {len(records)} records, active has {current_active.record_count} (>10% loss); keeping ACTIVE",
|
||||
}))
|
||||
return
|
||||
|
||||
dataset = DatasetService(session).import_ba_st_mva(records, raw_sha256, SOURCE_URL, datetime.now(timezone.utc))
|
||||
|
||||
print(json.dumps({
|
||||
"status": dataset.status, "dataset_id": dataset.id, "raw_sha256": raw_sha256,
|
||||
"record_count": dataset.record_count, "rejected": len(rejected),
|
||||
"previously_active_count": current_active.record_count if current_active else None,
|
||||
}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Promote CFOP/FCP/ANP/NCM_UTRIB/RTC_BIOFUEL_INDEX/RTC_CCREDPRES from the
|
||||
official spreadsheets the operator placed in sources/raw/bootstrap_v1/ after
|
||||
the Portal NF-e/SPED downloads were session-blocked. No network fetch.
|
||||
"""
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from central.app.datasets import bootstrap_v1_extra as parsers
|
||||
from central.app.datasets.service import DatasetService
|
||||
from central.app.db import SessionLocal
|
||||
|
||||
ROOT = Path("sources/raw/bootstrap_v1")
|
||||
|
||||
JOBS = [
|
||||
("CFOP", ROOT / "IT_2023_002_v2_10_Tabela_CFOP_Descrição.xlsx", parsers.parse_cfop, "import_cfop"),
|
||||
("FCP", ROOT / "Tabela_Alíquotas_FCP_por_UF_2025-02_07.xlsx", parsers.parse_fcp, "import_fcp"),
|
||||
("ANP", ROOT / "Tabela_Codigo_Produto_ANP-20241118.xlsx", parsers.parse_anp, "import_anp"),
|
||||
("NCM_UTRIB", ROOT / "IT2024.001v2.40 - Tabela NCM 2022 com Utrib_Comércio Exterior_vigência 01.10.26.xlsx", parsers.parse_ncm_utrib, "import_ncm_utrib"),
|
||||
("RTC_BIOFUEL_INDEX", ROOT / "IT 2025.004 v.1.20 - Tabela de Índice.xlsx", parsers.parse_rtc_biofuel_index, "import_rtc_biofuel_index"),
|
||||
("RTC_CCREDPRES", ROOT / "cCredPres_2026-06-22.xlsx", parsers.parse_rtc_cred_pres, "import_rtc_cred_pres"),
|
||||
("RTC_CCLASSTRIB", ROOT / "cClassTrib 2026-06-22.xlsx", parsers.parse_rtc_cclasstrib, "import_rtc_cclasstrib"),
|
||||
("RTC_CBS_RATES", ROOT / "IT 2026.002 v.1.00 - Aliquotas da CBS.xlsx", parsers.parse_rtc_cbs_rates, "import_rtc_cbs_rates"),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
results = {}
|
||||
with SessionLocal() as session:
|
||||
service = DatasetService(session)
|
||||
for domain, path, parse_fn, method_name in JOBS:
|
||||
try:
|
||||
if not path.exists():
|
||||
results[domain] = {"status": "MISSING_BOOTSTRAP_FILE", "path": str(path)}
|
||||
continue
|
||||
payload = parse_fn(path.read_bytes())
|
||||
dataset = getattr(service, method_name)(payload, f"file://{path.name}", datetime.now(timezone.utc))
|
||||
results[domain] = {"status": dataset.status, "dataset_id": dataset.id, "sha256": dataset.sha256, "record_count": dataset.record_count}
|
||||
except Exception as error: # noqa: BLE001 — isolate one domain's failure from the rest
|
||||
results[domain] = {"status": "FAILED", "reason": f"{type(error).__name__}: {error}"}
|
||||
|
||||
print(json.dumps(results, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Download and promote the official CONFAZ CV142_18 HTML CEST source."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
from central.app.config import settings
|
||||
from central.app.datasets.cest import parse_cest_html
|
||||
from central.app.datasets.service import DatasetService
|
||||
from central.app.db import SessionLocal
|
||||
|
||||
|
||||
def main():
|
||||
response = httpx.get(settings.cest_dataset_url, timeout=settings.http_timeout_seconds, follow_redirects=True)
|
||||
response.raise_for_status()
|
||||
parsed = parse_cest_html(response.content)
|
||||
path = Path(settings.dataset_storage_path)
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
(path / f"{parsed.sha256}.html").write_bytes(response.content)
|
||||
with SessionLocal() as session:
|
||||
dataset = DatasetService(session).import_cest_payload(parsed, settings.cest_dataset_url, datetime.now(timezone.utc))
|
||||
print({"id": dataset.id, "status": dataset.status, "sha256": dataset.sha256, "record_count": dataset.record_count, "version": parsed.version})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Seed reviewed official legacy NCM evidence; it never selects a successor."""
|
||||
|
||||
from central.app.db import SessionLocal
|
||||
from central.app.models import LegacyNcmEntry
|
||||
|
||||
|
||||
LEGACY_NCM = {
|
||||
"code": "02071400",
|
||||
"source": "Siscomex NCM official nomenclature download",
|
||||
"source_date": "2026-09-17",
|
||||
"reason": "NCM not present in current official nomenclature; no official successor.",
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
with SessionLocal() as session:
|
||||
entry = session.get(LegacyNcmEntry, LEGACY_NCM["code"])
|
||||
if entry is None:
|
||||
session.add(LegacyNcmEntry(**LEGACY_NCM, successor_code=None))
|
||||
session.commit()
|
||||
print({"code": LEGACY_NCM["code"], "status": "LEGACY_NCM", "successor_code": None})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Promote ORIGEM_MERCADORIA/CST_ICMS/CSOSN/CST_PIS/CST_COFINS from the local
|
||||
official MOC (Anexo I NF-e/NFC-e) PDF. No network fetch — this manual has no
|
||||
stable public direct-download URL either (session-gated portal), so the
|
||||
authoritative copy lives in sources/raw/bootstrap_v1/.
|
||||
"""
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from central.app.datasets.nfe_moc_tables import parse_moc_tables
|
||||
from central.app.datasets.service import DatasetService
|
||||
from central.app.db import SessionLocal
|
||||
|
||||
SOURCE = Path("sources/raw/bootstrap_v1/moc7-anexo-i-leiaute-e-rv.pdf")
|
||||
SOURCE_URL = f"file://{SOURCE.name}"
|
||||
|
||||
|
||||
def main():
|
||||
raw_bytes = SOURCE.read_bytes()
|
||||
payload = parse_moc_tables(raw_bytes)
|
||||
|
||||
with SessionLocal() as session:
|
||||
dataset = DatasetService(session).import_moc_tables(payload, SOURCE_URL, datetime.now(timezone.utc))
|
||||
|
||||
from collections import Counter
|
||||
domains = Counter(e.domain for e in payload.entries)
|
||||
print(json.dumps({
|
||||
"status": dataset.status, "dataset_id": dataset.id, "sha256": dataset.sha256,
|
||||
"record_count": dataset.record_count, "domains": dict(domains),
|
||||
}, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,19 @@
|
||||
"""Download, persist and promote the official NCM dataset."""
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
import httpx
|
||||
from central.app.config import settings
|
||||
from central.app.datasets.ncm import parse_ncm_payload
|
||||
from central.app.datasets.service import DatasetService
|
||||
from central.app.db import SessionLocal
|
||||
|
||||
def main():
|
||||
response = httpx.get(settings.ncm_dataset_url, timeout=settings.http_timeout_seconds, follow_redirects=True)
|
||||
response.raise_for_status()
|
||||
parsed = parse_ncm_payload(response.content)
|
||||
path = Path(settings.dataset_storage_path); path.mkdir(parents=True, exist_ok=True)
|
||||
(path / f"{parsed.sha256}.json").write_bytes(response.content)
|
||||
with SessionLocal() as session:
|
||||
dataset = DatasetService(session).import_payload(parsed, settings.ncm_dataset_url, datetime.now(timezone.utc))
|
||||
print({"id": dataset.id, "status": dataset.status, "sha256": dataset.sha256, "record_count": dataset.record_count})
|
||||
if __name__ == "__main__": main()
|
||||
@@ -0,0 +1,12 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
from pypdf import PdfReader
|
||||
from central.app.datasets.ba_st_mva import parse_page
|
||||
|
||||
source = Path("sources/raw/bootstrap_v1/decreto_2012_13780_ricms_anexo_1_vigente_2026.pdf")
|
||||
records, rejected = [], []
|
||||
for page, value in enumerate(PdfReader(source).pages, start=1):
|
||||
found, failed = parse_page(value.extract_text() or "", page)
|
||||
records.extend(found); rejected.extend(failed)
|
||||
Path("docs/ba_st_mva_inspection.json").write_text(json.dumps({"records": records, "rejected": rejected}, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(json.dumps({"pages": page, "records": len(records), "rejected": len(rejected), "cest": sum(bool(x["cest"]) for x in records), "ncm": sum(bool(x["ncm_sh"]) for x in records), "mva": sum(bool(x["mva_original"] or x["mva_adjusted"]) for x in records)}))
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Administrative CLI: register a new Agent directly against Postgres.
|
||||
|
||||
No new HTTP endpoint is needed for this — it's a one-time, operator-run
|
||||
action per installation, run from wherever DATABASE_URL is reachable (the
|
||||
Hub host, or a machine with network access to Postgres). Only the token's
|
||||
SHA-256 hash is ever stored; the plaintext token is printed once and must
|
||||
be copied into that installation's local Agent config immediately.
|
||||
|
||||
Usage:
|
||||
python central/scripts/register_agent.py --company "Restaurante X" --name "loja-01"
|
||||
"""
|
||||
import argparse
|
||||
import secrets
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "central"))
|
||||
|
||||
from sqlalchemy.orm import Session # noqa: E402
|
||||
|
||||
from app.auth import token_hash # noqa: E402
|
||||
from app.db import engine # noqa: E402
|
||||
from app.models import Agent, Tenant # noqa: E402
|
||||
|
||||
|
||||
def register_agent(session, company_name, agent_name, token=None):
|
||||
token = token or secrets.token_urlsafe(32)
|
||||
tenant = session.query(Tenant).filter_by(name=company_name).one_or_none()
|
||||
if tenant is None:
|
||||
tenant = Tenant(name=company_name)
|
||||
session.add(tenant)
|
||||
session.flush()
|
||||
agent = Agent(tenant_id=tenant.id, name=agent_name, token_hash=token_hash(token), enabled=True)
|
||||
session.add(agent)
|
||||
session.commit()
|
||||
# Read everything the caller needs now, while the session is still open —
|
||||
# session.commit() expires ORM attributes by default, and callers may use
|
||||
# these values after the session (and its `with` block) has closed.
|
||||
return {"id": agent.id, "tenant_id": tenant.id, "enabled": agent.enabled}, token
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--company", required=True, help="Company/tenant name (created if it doesn't exist)")
|
||||
parser.add_argument("--name", required=True, help="Human-readable name for this Agent installation, e.g. 'loja-01'")
|
||||
args = parser.parse_args()
|
||||
|
||||
with Session(engine) as session:
|
||||
info, token = register_agent(session, args.company, args.name)
|
||||
|
||||
print("Agent registered. Copy these into that installation's local .env NOW — the token is shown only once:")
|
||||
print(f" AGENT_ID={info['id']}")
|
||||
print(f" AGENT_TOKEN={token}")
|
||||
print(f" COMPANY_ID={info['tenant_id']}")
|
||||
print(" FIREBIRD_WRITES=0")
|
||||
print(f"enabled={info['enabled']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,9 @@
|
||||
def test_parser_extracts_item_cest_and_ncm_from_official_table_text():
|
||||
from central.app.datasets.ba_st_mva import parse_page
|
||||
|
||||
records, rejected = parse_page("ITEM CEST NCM/SH DESCRIÇÃO\n3.3 03.003.00 2201.1 Água mineral 114%", 1)
|
||||
|
||||
assert records[0]["item"] == "3.3"
|
||||
assert records[0]["cest"] == "03.003.00"
|
||||
assert records[0]["ncm_sh"] == ["2201.1"]
|
||||
assert not rejected
|
||||
@@ -0,0 +1,113 @@
|
||||
from datetime import datetime, timezone
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from central.app.datasets import bootstrap_v1_extra as parsers
|
||||
from central.app.datasets.service import DatasetService
|
||||
from central.app.models import Base
|
||||
|
||||
ROOT = Path(__file__).parents[2] / "sources" / "raw" / "bootstrap_v1"
|
||||
FILES = {
|
||||
"cfop": ROOT / "IT_2023_002_v2_10_Tabela_CFOP_Descrição.xlsx",
|
||||
"fcp": ROOT / "Tabela_Alíquotas_FCP_por_UF_2025-02_07.xlsx",
|
||||
"anp": ROOT / "Tabela_Codigo_Produto_ANP-20241118.xlsx",
|
||||
"ncm_utrib": ROOT / "IT2024.001v2.40 - Tabela NCM 2022 com Utrib_Comércio Exterior_vigência 01.10.26.xlsx",
|
||||
"biofuel": ROOT / "IT 2025.004 v.1.20 - Tabela de Índice.xlsx",
|
||||
"credpres": ROOT / "cCredPres_2026-06-22.xlsx",
|
||||
}
|
||||
pytestmark = pytest.mark.skipif(not all(p.exists() for p in FILES.values()), reason="bootstrap_v1 extra source files not present locally")
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _payload(name):
|
||||
parse_fn = {"cfop": parsers.parse_cfop, "fcp": parsers.parse_fcp, "anp": parsers.parse_anp,
|
||||
"ncm_utrib": parsers.parse_ncm_utrib, "biofuel": parsers.parse_rtc_biofuel_index,
|
||||
"credpres": parsers.parse_rtc_cred_pres}[name]
|
||||
return parse_fn(FILES[name].read_bytes())
|
||||
|
||||
|
||||
def _engine():
|
||||
engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(engine)
|
||||
return engine
|
||||
|
||||
|
||||
def test_cfop_parses_full_table_with_official_fields():
|
||||
payload = _payload("cfop")
|
||||
assert len(payload.entries) >= 500
|
||||
sample = payload.entries[0]
|
||||
assert set(sample) >= {"code", "title", "description", "effective_from", "ind_nfe", "ind_comb"}
|
||||
|
||||
|
||||
def test_fcp_covers_all_states_and_df():
|
||||
payload = _payload("fcp")
|
||||
ufs = {e["uf"] for e in payload.entries}
|
||||
assert len(ufs) == 27 # 26 states + DF
|
||||
assert "BA" in ufs
|
||||
|
||||
|
||||
def test_anp_has_no_invented_descriptions():
|
||||
payload = _payload("anp")
|
||||
assert len(payload.entries) >= 900
|
||||
assert all(set(e) == {"cprod_anp"} for e in payload.entries) # source truly has no description column
|
||||
|
||||
|
||||
def test_ncm_utrib_covers_thousands_of_ncms():
|
||||
payload = _payload("ncm_utrib")
|
||||
assert len(payload.entries) >= 10000
|
||||
assert all(len(e["ncm"]) == 8 for e in payload.entries[:50])
|
||||
|
||||
|
||||
def test_biofuel_index_separates_current_from_historical():
|
||||
payload = _payload("biofuel")
|
||||
current = [e for e in payload.entries if not e["is_historical"]]
|
||||
historical = [e for e in payload.entries if e["is_historical"]]
|
||||
assert current and historical
|
||||
assert all(e["p_bio_percent"] is not None for e in current)
|
||||
|
||||
|
||||
def test_credpres_preserves_rich_official_fields_not_just_code_description():
|
||||
payload = _payload("credpres")
|
||||
sample = payload.entries[0]
|
||||
assert sample["legal_basis"]
|
||||
assert "d_ini_vig_cbs" in sample and "aliquota_cbs_text" in sample
|
||||
|
||||
|
||||
@pytest.mark.parametrize("domain,method,lookup", [
|
||||
("cfop", "import_cfop", lambda s: s.cfop_lookup(code="1101")),
|
||||
("fcp", "import_fcp", lambda s: [s.fcp_for_uf("BA")]),
|
||||
("anp", "import_anp", lambda s: [s.anp_code_exists("110203073")]),
|
||||
("ncm_utrib", "import_ncm_utrib", lambda s: [s.ncm_utrib_for_ncm("01012100")]),
|
||||
("biofuel", "import_rtc_biofuel_index", lambda s: s.rtc_biofuel_for_anp("320102001")),
|
||||
("credpres", "import_rtc_cred_pres", lambda s: s.rtc_cred_pres(code="1")),
|
||||
])
|
||||
def test_import_idempotent_and_lookup_smoke(domain, method, lookup):
|
||||
engine = _engine()
|
||||
with Session(engine) as session:
|
||||
service = DatasetService(session)
|
||||
payload = _payload(domain)
|
||||
first = getattr(service, method)(payload, f"file://{domain}", datetime.now(timezone.utc))
|
||||
second = getattr(service, method)(payload, f"file://{domain}", datetime.now(timezone.utc))
|
||||
assert first.id == second.id
|
||||
assert second.status == "ACTIVE"
|
||||
assert second.record_count == len(payload.entries)
|
||||
|
||||
result = lookup(service)
|
||||
assert result
|
||||
assert all(item is not None and item is not False for item in result) or result == [True]
|
||||
|
||||
|
||||
def test_dataset_versions_includes_all_new_kinds():
|
||||
engine = _engine()
|
||||
with Session(engine) as session:
|
||||
service = DatasetService(session)
|
||||
versions = service.dataset_versions()
|
||||
for key in ("cfop", "fcp", "anp", "ncm_utrib", "rtc_biofuel_index", "rtc_ccredpres"):
|
||||
assert key in versions
|
||||
assert versions[key] is None
|
||||
service.import_cfop(_payload("cfop"), "file://cfop", datetime.now(timezone.utc))
|
||||
assert service.dataset_versions()["cfop"] is not None
|
||||
@@ -0,0 +1,38 @@
|
||||
from central.app.classification import Classifier
|
||||
|
||||
|
||||
class _Repository:
|
||||
def __init__(self, cest_entries):
|
||||
self._cest_entries = cest_entries
|
||||
|
||||
def active_entries(self):
|
||||
return [type("Entry", (), {"code": "19022000", "description": "Massas"})()]
|
||||
|
||||
def cest_for_ncm(self, code):
|
||||
return self._cest_entries if code == "19022000" else []
|
||||
|
||||
def legacy_ncm(self, code):
|
||||
return None
|
||||
|
||||
def dataset_versions(self):
|
||||
return {"ncm": "n", "cest": "c"}
|
||||
|
||||
|
||||
def test_stopwords_do_not_inflate_description_score():
|
||||
# "RISOLE DE CAMARAO" vs an official text that only shares the stopword "DE"
|
||||
# ("... preparadas de outro modo") must score 0, not a false-positive overlap.
|
||||
cest = type("Cest", (), {
|
||||
"code": "17.048.02",
|
||||
"description": "Massas alimenticias recheadas (mesmo cozidas ou preparadas de outro modo)",
|
||||
})()
|
||||
result = Classifier(_Repository([cest])).classify({"current_ncm": "19022000", "description": "RISOLE DE CAMARAO"})
|
||||
candidate = result["classification"]["cest_candidates"][0]
|
||||
assert candidate["description_score"] == 0.0
|
||||
assert result["cest_status"] == "CEST_DESCRIPTION_MISMATCH"
|
||||
|
||||
|
||||
def test_genuine_token_overlap_still_scores_above_zero():
|
||||
cest = type("Cest", (), {"code": "17.024.01", "description": "Queijo mucarela"})()
|
||||
result = Classifier(_Repository([cest])).classify({"current_ncm": "19022000", "description": "BOLINHO DE QUEIJO"})
|
||||
candidate = result["classification"]["cest_candidates"][0]
|
||||
assert candidate["description_score"] > 0
|
||||
@@ -0,0 +1,62 @@
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from central.app import api as api_module
|
||||
from central.app.main import create_app
|
||||
from central.app.models import Base
|
||||
|
||||
HEADERS = {"Authorization": "Bearer test-token"}
|
||||
PAYLOAD = {"current_ncm": "22030000", "description": "CERVEJA LATA"}
|
||||
|
||||
|
||||
def make_client(monkeypatch, calls, engine_version="0.3.0", datasets=None):
|
||||
versions = datasets if datasets is not None else {"ncm": "aaa"}
|
||||
|
||||
class FakeClassifier:
|
||||
def __init__(self, repository): pass
|
||||
|
||||
def classify(self, product):
|
||||
calls.append(product)
|
||||
return {"status": "EXACT_NCM", "engine_version": api_module.ENGINE_VERSION, "decision_version": "x", "dataset_versions": dict(versions), "classification": {"ncm": product["current_ncm"]}}
|
||||
|
||||
monkeypatch.setattr(api_module, "Classifier", FakeClassifier)
|
||||
monkeypatch.setattr(api_module, "ENGINE_VERSION", engine_version)
|
||||
monkeypatch.setattr(api_module.DatasetService, "dataset_versions", lambda self: dict(versions))
|
||||
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||
Base.metadata.create_all(engine)
|
||||
return TestClient(create_app(session_factory=lambda: Session(engine), bootstrap_token="test-token")), versions
|
||||
|
||||
|
||||
def test_cache_hit_does_not_run_the_classifier(monkeypatch):
|
||||
calls = []
|
||||
client, _ = make_client(monkeypatch, calls)
|
||||
first = client.post("/v1/classify", json=PAYLOAD, headers=HEADERS).json()
|
||||
assert first["provenance"]["cache_hit"] is False and len(calls) == 1
|
||||
for _ in range(3):
|
||||
again = client.post("/v1/classify", json=PAYLOAD, headers=HEADERS).json()
|
||||
assert again["provenance"]["cache_hit"] is True
|
||||
assert {k: v for k, v in again.items() if k != "provenance"} == {k: v for k, v in first.items() if k != "provenance"} # same fiscal result
|
||||
assert len(calls) == 1 # CLASSIFIER_EXECUTIONS_ON_HIT = 0
|
||||
|
||||
|
||||
def test_batch_hits_skip_the_classifier_and_new_inputs_miss(monkeypatch):
|
||||
calls = []
|
||||
client, _ = make_client(monkeypatch, calls)
|
||||
products = [PAYLOAD, {"current_ncm": "19022000", "description": "MASSA"}]
|
||||
client.post("/v1/classify/batch", json={"products": products}, headers=HEADERS)
|
||||
assert len(calls) == 2
|
||||
res = client.post("/v1/classify/batch", json={"products": products + [{"current_ncm": "20099000", "description": "SUCO"}]}, headers=HEADERS).json()["results"]
|
||||
assert [r["provenance"]["cache_hit"] for r in res] == [True, True, False] and len(calls) == 3
|
||||
|
||||
|
||||
def test_engine_or_dataset_version_change_invalidates_naturally(monkeypatch):
|
||||
calls = []
|
||||
client, versions = make_client(monkeypatch, calls)
|
||||
client.post("/v1/classify", json=PAYLOAD, headers=HEADERS)
|
||||
versions["ncm"] = "bbb" # a new ACTIVE dataset version changes dataset_versions()
|
||||
assert client.post("/v1/classify", json=PAYLOAD, headers=HEADERS).json()["provenance"]["cache_hit"] is False
|
||||
monkeypatch.setattr(api_module, "ENGINE_VERSION", "0.4.0")
|
||||
assert client.post("/v1/classify", json=PAYLOAD, headers=HEADERS).json()["provenance"]["cache_hit"] is False
|
||||
assert len(calls) == 3
|
||||
@@ -0,0 +1,31 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
from central.app.classification import classify_tipi_rate
|
||||
|
||||
|
||||
def _entry(ex=None, ipi_rate=None, ipi_rate_text=None):
|
||||
return SimpleNamespace(ex=ex, ipi_rate=ipi_rate, ipi_rate_text=ipi_rate_text)
|
||||
|
||||
|
||||
def test_ex_specific():
|
||||
assert classify_tipi_rate(_entry(ex="01")) == "EX_SPECIFIC"
|
||||
|
||||
|
||||
def test_source_blank():
|
||||
assert classify_tipi_rate(_entry(ipi_rate_text=None)) == "SOURCE_BLANK"
|
||||
|
||||
|
||||
def test_nt_or_non_taxed():
|
||||
assert classify_tipi_rate(_entry(ipi_rate_text="NT")) == "NT_OR_NON_TAXED"
|
||||
|
||||
|
||||
def test_explicit_zero_text():
|
||||
assert classify_tipi_rate(_entry(ipi_rate_text="0,00")) == "EXPLICIT_ZERO"
|
||||
|
||||
|
||||
def test_explicit_zero_numeric():
|
||||
assert classify_tipi_rate(_entry(ipi_rate=0, ipi_rate_text="0")) == "EXPLICIT_ZERO"
|
||||
|
||||
|
||||
def test_other_unrecognized_marker():
|
||||
assert classify_tipi_rate(_entry(ipi_rate_text="ISENTO")) == "OTHER"
|
||||
@@ -0,0 +1,11 @@
|
||||
import importlib
|
||||
|
||||
|
||||
def test_database_url_encodes_password_from_component_environment(monkeypatch):
|
||||
monkeypatch.delenv("DATABASE_URL", raising=False)
|
||||
monkeypatch.setenv("POSTGRES_PASSWORD", "a@b/c")
|
||||
import central.app.config as config
|
||||
|
||||
module = importlib.reload(config)
|
||||
|
||||
assert module.settings.database_url == "postgresql+psycopg://gti:a%40b%2Fc@postgres:5432/gti_fiscal"
|
||||
@@ -0,0 +1,180 @@
|
||||
import pytest
|
||||
|
||||
from central.app.dry_run_planner import (
|
||||
build_field_proposals,
|
||||
execute,
|
||||
mva_evidence,
|
||||
split_by_eligibility,
|
||||
)
|
||||
|
||||
|
||||
def _decision(**overrides):
|
||||
base = {
|
||||
"status": "EXACT_NCM",
|
||||
"cest_status": "NO_CEST_MAPPING_FOUND",
|
||||
"ipi_status": "NO_TIPI_MAPPING_FOUND",
|
||||
"ba_st_mva_status": "NO_MVA_MAPPING_FOUND",
|
||||
"classification": {
|
||||
"ncm": "07069000",
|
||||
"cest": [],
|
||||
"ipi": None,
|
||||
"ba_st_mva": None,
|
||||
"ba_st_mva_candidates": [],
|
||||
},
|
||||
"dataset_versions": {"ncm": "n1", "cest": "c1", "tipi_ipi": "t1", "ba_st_mva": "m1"},
|
||||
"engine_version": "0.1.0",
|
||||
"decision_version": "mvp",
|
||||
"request_fingerprint": "fp",
|
||||
}
|
||||
base.update(overrides)
|
||||
return base
|
||||
|
||||
|
||||
def test_ipi_resolved_numeric_current_unknown():
|
||||
decision = _decision(ipi_status="IPI_RESOLVED", classification={**_decision()["classification"], "ipi": {"rate": "3.25", "rate_numeric": 3.25, "rate_kind": "NUMERIC", "ex": None}})
|
||||
proposals = build_field_proposals("000026", decision, {})
|
||||
ipi = next(p for p in proposals if p["field"] == "IPI")
|
||||
assert ipi["status"] == "CURRENT_UNKNOWN"
|
||||
assert ipi["write_eligible"] is False
|
||||
assert ipi["proposed_value"] == 3.25
|
||||
|
||||
|
||||
def test_ipi_resolved_numeric_same_value():
|
||||
decision = _decision(ipi_status="IPI_RESOLVED", classification={**_decision()["classification"], "ipi": {"rate": "3.25", "rate_numeric": 3.25, "rate_kind": "NUMERIC", "ex": None}})
|
||||
proposals = build_field_proposals("000026", decision, {"ipi": 3.25, "ex_tipi": None})
|
||||
ipi = next(p for p in proposals if p["field"] == "IPI")
|
||||
assert ipi["status"] == "SAME_VALUE"
|
||||
assert ipi["write_eligible"] is False
|
||||
|
||||
|
||||
def test_ipi_resolved_numeric_change_required():
|
||||
decision = _decision(ipi_status="IPI_RESOLVED", classification={**_decision()["classification"], "ipi": {"rate": "3.25", "rate_numeric": 3.25, "rate_kind": "NUMERIC", "ex": None}})
|
||||
proposals = build_field_proposals("000026", decision, {"ipi": None, "ex_tipi": None})
|
||||
ipi = next(p for p in proposals if p["field"] == "IPI")
|
||||
assert ipi["status"] == "CHANGE_REQUIRED"
|
||||
assert ipi["write_eligible"] is True
|
||||
assert ipi["proposed_value"] == 3.25
|
||||
|
||||
|
||||
def test_ipi_non_numeric_marker_never_proposes_a_write():
|
||||
decision = _decision(ipi_status="IPI_RESOLVED", classification={**_decision()["classification"], "ipi": {"rate": "NT", "rate_numeric": None, "rate_kind": "NON_NUMERIC", "rate_classification": "NT_OR_NON_TAXED", "ex": None}})
|
||||
proposals = build_field_proposals("000004", decision, {"ipi": None, "ex_tipi": None})
|
||||
ipi = next(p for p in proposals if p["field"] == "IPI")
|
||||
assert ipi["status"] == "NON_NUMERIC_IPI_STATUS"
|
||||
assert ipi["proposed_value"] is None
|
||||
assert ipi["write_eligible"] is False
|
||||
assert ipi["reason_code"] == "TIPI_RATE_NT_OR_NON_TAXED"
|
||||
|
||||
|
||||
def test_ipi_resolved_without_concrete_rate_still_produces_a_row():
|
||||
decision = _decision(ipi_status="IPI_RESOLVED", classification={**_decision()["classification"], "ipi": {"rate": None, "rate_numeric": None, "rate_kind": "BLANK", "rate_classification": "SOURCE_BLANK", "ex": None}})
|
||||
proposals = build_field_proposals("000018", decision, {"ipi": 5, "ex_tipi": None})
|
||||
ipi = next(p for p in proposals if p["field"] == "IPI")
|
||||
assert ipi["status"] == "SOURCE_BLANK"
|
||||
assert ipi["proposed_value"] is None
|
||||
assert ipi["write_eligible"] is False
|
||||
assert ipi["reason_code"] == "TIPI_RATE_SOURCE_BLANK"
|
||||
|
||||
|
||||
def test_no_ex_tipi_row_when_no_ex_candidates():
|
||||
decision = _decision(ipi_status="IPI_RESOLVED", classification={**_decision()["classification"], "ipi": {"rate": "3.25", "rate_numeric": 3.25, "rate_kind": "NUMERIC", "ex": None}, "ipi_candidates": []})
|
||||
proposals = build_field_proposals("000026", decision, {"ipi": None, "ex_tipi": None})
|
||||
assert all(p["field"] != "EX_TIPI" for p in proposals)
|
||||
|
||||
|
||||
def test_ex_tipi_concrete_single_candidate_change_required():
|
||||
decision = _decision(
|
||||
ipi_status="IPI_CANDIDATES",
|
||||
classification={**_decision()["classification"], "ipi": None, "ipi_candidates": [{"ncm": "07069000", "ex": "01", "description": "specific", "rate": "5"}]},
|
||||
)
|
||||
proposals = build_field_proposals("000004", decision, {"ex_tipi": None})
|
||||
ex = next(p for p in proposals if p["field"] == "EX_TIPI")
|
||||
assert ex["status"] == "CHANGE_REQUIRED"
|
||||
assert ex["proposed_value"] == "01"
|
||||
assert ex["write_eligible"] is True
|
||||
|
||||
|
||||
def test_ex_tipi_multiple_candidates_is_candidate_not_eligible():
|
||||
decision = _decision(
|
||||
ipi_status="IPI_CANDIDATES",
|
||||
classification={**_decision()["classification"], "ipi": None, "ipi_candidates": [
|
||||
{"ncm": "07069000", "ex": "01", "description": "a", "rate": "5"},
|
||||
{"ncm": "07069000", "ex": "02", "description": "b", "rate": "7"},
|
||||
]},
|
||||
)
|
||||
proposals = build_field_proposals("000004", decision, {"ex_tipi": None})
|
||||
ex = next(p for p in proposals if p["field"] == "EX_TIPI")
|
||||
assert ex["status"] == "CANDIDATE"
|
||||
assert ex["proposed_value"] is None
|
||||
assert ex["write_eligible"] is False
|
||||
|
||||
|
||||
def test_cest_confirmed_change_required():
|
||||
decision = _decision(cest_status="CEST_CONFIRMED", classification={**_decision()["classification"], "cest": ["17.087.02"]})
|
||||
proposals = build_field_proposals("000018", decision, {"cest": ""})
|
||||
cest = next(p for p in proposals if p["field"] == "CEST")
|
||||
assert cest["status"] == "CHANGE_REQUIRED"
|
||||
assert cest["write_eligible"] is True
|
||||
assert cest["proposed_value"] == "17.087.02"
|
||||
|
||||
|
||||
def test_cest_candidates_not_eligible():
|
||||
decision = _decision(cest_status="CEST_CANDIDATES")
|
||||
proposals = build_field_proposals("000024", decision, {"cest": ""})
|
||||
cest = next(p for p in proposals if p["field"] == "CEST")
|
||||
assert cest["status"] == "CANDIDATE"
|
||||
assert cest["proposed_value"] is None
|
||||
assert cest["write_eligible"] is False
|
||||
|
||||
|
||||
def test_cest_description_mismatch_not_eligible():
|
||||
decision = _decision(cest_status="CEST_DESCRIPTION_MISMATCH")
|
||||
proposals = build_field_proposals("000018", decision, {"cest": ""})
|
||||
cest = next(p for p in proposals if p["field"] == "CEST")
|
||||
assert cest["status"] == "UNRESOLVED"
|
||||
assert cest["reason_code"] == "CEST_DESCRIPTION_SCORE_BELOW_THRESHOLD"
|
||||
assert cest["write_eligible"] is False
|
||||
|
||||
|
||||
def test_legacy_ncm_review_no_invented_successor():
|
||||
decision = _decision(status="LEGACY_NCM", classification={**_decision()["classification"], "ncm": "02071400"})
|
||||
proposals = build_field_proposals("000016", decision, {"ipi": "3.25", "cest": ""})
|
||||
assert len(proposals) == 1
|
||||
row = proposals[0]
|
||||
assert row["field"] == "CLASSIFICACAO_FISCAL"
|
||||
assert row["status"] == "LEGACY_NCM_REVIEW"
|
||||
assert row["proposed_value"] is None
|
||||
assert row["write_eligible"] is False
|
||||
|
||||
|
||||
def test_mva_never_produces_a_write_proposal():
|
||||
decision = _decision(ba_st_mva_status="MVA_CANDIDATES", classification={**_decision()["classification"], "ba_st_mva_candidates": [{"item": "1"}]})
|
||||
proposals = build_field_proposals("000024", decision, {"cest": ""})
|
||||
assert all(p["field"] != "BA_ST_MVA" for p in proposals)
|
||||
evidence = mva_evidence("000024", decision)
|
||||
assert evidence["status"] == "MVA_CANDIDATES"
|
||||
assert evidence["candidates"]
|
||||
|
||||
|
||||
def test_split_by_eligibility():
|
||||
decision = _decision(ipi_status="IPI_RESOLVED", classification={**_decision()["classification"], "ipi": {"rate": "3.25", "rate_numeric": 3.25, "rate_kind": "NUMERIC", "ex": None}})
|
||||
proposals = build_field_proposals("000026", decision, {})
|
||||
split = split_by_eligibility(proposals)
|
||||
assert split["blocked"] == proposals
|
||||
assert split["eligible"] == []
|
||||
|
||||
|
||||
def test_execute_refuses_non_eligible_proposal():
|
||||
decision = _decision(ipi_status="IPI_RESOLVED", classification={**_decision()["classification"], "ipi": {"rate": "3.25", "rate_numeric": 3.25, "rate_kind": "NUMERIC", "ex": None}})
|
||||
proposals = build_field_proposals("000026", decision, {})
|
||||
with pytest.raises(PermissionError):
|
||||
execute(proposals)
|
||||
|
||||
|
||||
def test_execute_never_writes_even_when_eligible():
|
||||
decision = _decision(ipi_status="IPI_RESOLVED", classification={**_decision()["classification"], "ipi": {"rate": "3.25", "rate_numeric": 3.25, "rate_kind": "NUMERIC", "ex": None}})
|
||||
proposals = build_field_proposals("000026", decision, {"ipi": "0"})
|
||||
eligible_only = split_by_eligibility(proposals)["eligible"]
|
||||
assert eligible_only
|
||||
with pytest.raises(NotImplementedError):
|
||||
execute(eligible_only)
|
||||
@@ -0,0 +1,270 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from central.app.field_contract import (ALREADY, CHANGE, CONTRACT, MANUAL, NA, WRITE_FIELDS, Context, infer_role, resolve_product)
|
||||
|
||||
ROOT = Path(__file__).parents[2]
|
||||
REFS = {"cfop_valid": {"5101", "5102", "5401", "5405", "1102", "1101", "1403", "2102"}, "csosn": {"101", "102", "103", "300", "400", "500"}, "cst_icms": {"00", "20", "40", "41", "60"},
|
||||
"cst_pis": {"01", "49", "98"}, "cst_cofins": {"01", "49", "98"}, "origem": set("012345678"), "fcp_uf": {}, "cbs_rate_2026": None, "cclasstrib_count": 0}
|
||||
CO1 = {"uf": "BA", "crt": "1", "contribuinte_ipi": "N", "company_code": "000001"}
|
||||
CO3 = {"uf": "BA", "crt": "3", "contribuinte_ipi": "S", "company_code": "000001"}
|
||||
|
||||
|
||||
def product(**kw):
|
||||
p = {f: None for f in WRITE_FIELDS}
|
||||
p.update({"CODIGO": "000001", "PRODUTO": "CERVEJA LATA", "CODGRUPO": "000019", "TIPO": "VENDA", "PROD_COZINHA": None, "PRODUTO_SERVICO": None, "CLASSIFICACAO_FISCAL": "22030000",
|
||||
"CFOP": "5102", "CSOSN": "102", "CST": "000", "SITUACAO_TRIBUTARIA": "T", "IAT": "A", "IPPT": "T", "INCIDENCIA_PISCOFINS": "NENHUMA", "PISCOFINS": "N"})
|
||||
p.update(kw)
|
||||
return p
|
||||
|
||||
|
||||
def decision(**kw):
|
||||
d = {"status": "EXACT_NCM", "cest_status": "NO_CEST_MAPPING_FOUND", "ipi_status": "IPI_RESOLVED", "ba_st_mva_status": "NO_MVA_MAPPING_FOUND", "dataset_versions": {},
|
||||
"classification": {"ncm": "22030000", "ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.9, "rate": "3.9"}, "ipi_candidates": [], "cest": None, "cest_candidates": [], "ba_st_mva": None, "ba_st_mva_candidates": []}}
|
||||
cls = kw.pop("classification", {})
|
||||
d.update(kw)
|
||||
d["classification"] = {**d["classification"], **cls}
|
||||
return d
|
||||
|
||||
|
||||
EV = {"recipes": set(), "composed": set(), "purchases": {}, "stock_entry": {}, "stock_in": {"000001": True}}
|
||||
|
||||
|
||||
IPI_POP = {"scope": "COMPANY", "scope_key": "", "key": "store_ipi_profile", "value": "POPULATE_REFERENCE_RATE"}
|
||||
|
||||
|
||||
def run(p=None, d=None, co=CO1, ctx=None, ev=EV, manual=None, ipi_profile=IPI_POP):
|
||||
return resolve_product(p or product(), d or decision(), co, REFS, ev, Context((ctx or []) + ([ipi_profile] if ipi_profile else [])), manual_ncm=manual)
|
||||
|
||||
|
||||
def st(res, f):
|
||||
return res[f]["state"], res[f]["proposed"]
|
||||
|
||||
|
||||
def test_every_write_field_has_a_complete_contract():
|
||||
keys = {"ERP_FIELD", "UI_LABEL", "SEMANTICS", "DATA_TYPE", "SOURCE_OF_TRUTH", "REQUIRED_INPUTS", "RESOLUTION_RULE", "VALID_VALUES", "NOT_APPLICABLE_RULE", "MANUAL_INPUT_RULE", "WRITE_ALLOWED", "VALIDATION_RULE", "PROVENANCE"}
|
||||
assert set(CONTRACT) == set(WRITE_FIELDS) and len(WRITE_FIELDS) == 23
|
||||
for f, c in CONTRACT.items():
|
||||
assert keys <= set(c) and all(c[k] not in (None, "", []) for k in keys - {"WRITE_ALLOWED"}), f
|
||||
|
||||
|
||||
def test_only_five_final_states_and_manual_always_names_its_input():
|
||||
res = run()
|
||||
for f in WRITE_FIELDS:
|
||||
assert res[f]["state"] in (ALREADY, CHANGE, NA, MANUAL)
|
||||
if res[f]["state"] == MANUAL:
|
||||
assert res[f]["input_key"]
|
||||
assert res[f]["write_eligible"] == (res[f]["state"] == CHANGE)
|
||||
|
||||
|
||||
def test_ncm_states_and_operator_ncm_reprocessing():
|
||||
bad = run(p=product(CLASSIFICACAO_FISCAL="12345678"), d=decision(status="INVALID_NCM"))
|
||||
assert st(bad, "CLASSIFICACAO_FISCAL")[0] == MANUAL and bad["CLASSIFICACAO_FISCAL"]["input_key"] == "ncm"
|
||||
for f in ("IPI", "EX_TIPI", "CEST", "CFOP", "CSOSN"):
|
||||
assert bad[f]["state"] == MANUAL and bad[f]["reason"] == "BLOCKED_BY_NCM"
|
||||
assert st(run(manual=("22030000", False), p=product(CLASSIFICACAO_FISCAL="12345678"), d=decision(status="INVALID_NCM")), "CLASSIFICACAO_FISCAL")[0] == MANUAL
|
||||
fixed = run(manual=("22030000", True), p=product(CLASSIFICACAO_FISCAL="12345678"), d=decision())
|
||||
assert st(fixed, "CLASSIFICACAO_FISCAL") == (CHANGE, "22030000") and fixed["IPI"]["state"] == CHANGE and fixed["IPI"]["proposed"] == 3.9 # dependents reprocessed with the manual NCM
|
||||
|
||||
|
||||
def test_ipi_rules_never_write_nt_or_blank_and_ask_when_ambiguous():
|
||||
assert st(run(), "IPI") == (CHANGE, 3.9)
|
||||
assert st(run(p=product(IPI=3.9)), "IPI")[0] == ALREADY
|
||||
nt = run(d=decision(classification={"ipi": {"rate_kind": "NON_NUMERIC", "rate_classification": "NT"}}))
|
||||
assert nt["IPI"]["state"] == NA and not nt["IPI"]["write_eligible"]
|
||||
blank = run(d=decision(classification={"ipi": {"rate_kind": "BLANK"}}))
|
||||
assert blank["IPI"]["state"] == NA and "NEVER_ZERO" in blank["IPI"]["reason"]
|
||||
amb = decision(ipi_status="IPI_CANDIDATES", classification={"ipi": None, "ipi_candidates": [{"ex": None, "rate": "5", "description": "base"}, {"ex": "1", "rate": "0", "description": "Chope"}]})
|
||||
r = run(d=amb)
|
||||
assert r["IPI"]["state"] == MANUAL and r["IPI"]["input_key"] == "ex_tipi" and len(r["IPI"]["choices"]) == 2
|
||||
solved = run(d=amb, ctx=[{"scope": "PRODUCT", "scope_key": "000001", "key": "ex_tipi", "value": "NONE"}])
|
||||
assert st(solved, "IPI") == (CHANGE, 5.0) and solved["EX_TIPI"]["state"] == NA
|
||||
|
||||
|
||||
def test_cest_contract():
|
||||
off = [{"code": "03.021.03", "official_description": "Cerveja em lata", "description_score": 1.0}, {"code": "03.021.05", "official_description": "PET", "description_score": 0.5}]
|
||||
conf = run(d=decision(cest_status="CEST_CONFIRMED", classification={"cest": ["03.021.03"], "cest_candidates": off}))
|
||||
assert st(conf, "CEST") == (CHANGE, "0302103")
|
||||
assert run(p=product(CEST="0302103"), d=decision(cest_status="CEST_CANDIDATES", classification={"cest_candidates": off}))["CEST"]["state"] == ALREADY
|
||||
many = run(d=decision(cest_status="CEST_CANDIDATES", classification={"cest_candidates": off}))
|
||||
assert many["CEST"]["state"] == MANUAL and many["CEST"]["input_key"] == "cest" and len(many["CEST"]["choices"]) == 2
|
||||
assert run(d=decision(cest_status="CEST_DESCRIPTION_MISMATCH", classification={"cest_candidates": off}))["CEST"]["state"] == NA
|
||||
assert run()["CEST"]["state"] == NA
|
||||
assert run(p=product(CEST="9999999"))["CEST"]["state"] == MANUAL
|
||||
picked = run(d=decision(cest_status="CEST_CANDIDATES", classification={"cest_candidates": off}), ctx=[{"scope": "NCM", "scope_key": "22030000", "key": "cest", "value": "03.021.05"}])
|
||||
assert st(picked, "CEST") == (CHANGE, "0302105")
|
||||
|
||||
|
||||
def test_cfop_role_and_st_rules():
|
||||
ok = run() # RESALE (stock entries), not subject to ST, declared 5102 == derived
|
||||
assert ok["CFOP"]["state"] == ALREADY and ok["_facts"]["role"] == "RESALE"
|
||||
assert st(run(p=product(CFOP="5405")), "CFOP") == (CHANGE, "5102") # 5405 but proven NOT_SUBJECT
|
||||
assert st(run(p=product(CFOP=None)), "CFOP") == (CHANGE, "5102") # blank filled
|
||||
unk = run(d=decision(ba_st_mva_status="MVA_CANDIDATES"), p=product(CFOP=None)) # ST unknown, blank -> asks
|
||||
assert unk["CFOP"]["state"] == MANUAL and unk["CFOP"]["input_key"] == "st_status"
|
||||
assert run(d=decision(ba_st_mva_status="MVA_CANDIDATES"))["CFOP"]["state"] == ALREADY # declared valid, not contradicted
|
||||
role_unknown = run(ev={**EV, "stock_in": {}}, p=product(CFOP=None))
|
||||
assert role_unknown["CFOP"]["state"] == MANUAL and role_unknown["CFOP"]["input_key"] == "product_role"
|
||||
own = run(ev={**EV, "recipes": {"000001"}}, p=product(CFOP="5102"))
|
||||
assert st(own, "CFOP") == (CHANGE, "5101")
|
||||
assert run(p=product(TIPO="MATERIA-PRIMA"))["CFOP"]["state"] == NA
|
||||
|
||||
|
||||
def test_cfop_entrada_needs_company_entry_scope_then_derives():
|
||||
r = run(p=product(CFOP_ENTRADA=None))
|
||||
assert r["CFOP_ENTRADA"]["state"] == MANUAL and r["CFOP_ENTRADA"]["input_key"] == "entry_scope"
|
||||
r = run(p=product(CFOP_ENTRADA=None), ctx=[{"scope": "COMPANY", "scope_key": "", "key": "entry_scope", "value": "INTRA"}])
|
||||
assert st(r, "CFOP_ENTRADA") == (CHANGE, "1102")
|
||||
assert run(p=product(CFOP_ENTRADA="1102"))["CFOP_ENTRADA"]["state"] == ALREADY
|
||||
|
||||
|
||||
def test_icms_code_fields_follow_the_regime():
|
||||
r1 = run()
|
||||
assert r1["CST"]["state"] == NA and r1["CST_ENTRADA"]["state"] == NA and r1["CSOSN"]["state"] == ALREADY
|
||||
assert run(p=product(CSOSN=None))["CSOSN"]["state"] == MANUAL
|
||||
assert st(run(p=product(CSOSN=None), ctx=[{"scope": "GROUP", "scope_key": "000019", "key": "icms_treatment", "value": "EXEMPT"}]), "CSOSN") == (CHANGE, "400")
|
||||
assert st(run(p=product(CSOSN="102"), ctx=[{"scope": "GROUP", "scope_key": "000019", "key": "icms_treatment", "value": "ST_WITHHELD"}]), "CSOSN") == (CHANGE, "500")
|
||||
r3 = run(co=CO3, p=product(CSOSN=None))
|
||||
assert r3["CSOSN"]["state"] == NA and r3["CST"]["state"] == ALREADY and r3["ALIQUOTA"]["state"] in (ALREADY, MANUAL)
|
||||
blank = run(co=CO3, p=product(CST=None, COD_ORIGEM_MERCADORIA="0"))
|
||||
assert blank["CST"]["state"] == MANUAL and blank["CST"]["input_key"] == "icms_treatment"
|
||||
filled = run(co=CO3, p=product(CST=None, COD_ORIGEM_MERCADORIA="0"), ctx=[{"scope": "COMPANY", "scope_key": "", "key": "icms_treatment", "value": "TAXED"}])
|
||||
assert st(filled, "CST") == (CHANGE, "000")
|
||||
|
||||
|
||||
def test_aliquota_not_copied_blindly():
|
||||
assert run()["ALIQUOTA"]["state"] == NA # Simples CSOSN 102 highlights no ICMS
|
||||
assert run(p=product(CSOSN="101", ALIQUOTA=None))["ALIQUOTA"]["input_key"] == "credit_rate"
|
||||
assert st(run(co=CO3, p=product(ALIQUOTA="17"), ctx=[{"scope": "COMPANY", "scope_key": "", "key": "icms_rate", "value": "20,5"}]), "ALIQUOTA") == (CHANGE, 20.5)
|
||||
|
||||
|
||||
def test_origin_binding_rules():
|
||||
r0 = run(p=product(COD_ORIGEM_MERCADORIA=None, CST="040"))["COD_ORIGEM_MERCADORIA"] # audit V2: the CST first digit is padding, never origin evidence
|
||||
assert r0["state"] == MANUAL and r0["reason"] == "ORIGIN_NO_EVIDENCE" and r0["proposed"] is None
|
||||
assert run(p=product(COD_ORIGEM_MERCADORIA="0", CST="040"))["COD_ORIGEM_MERCADORIA"]["state"] == ALREADY
|
||||
assert run(p=product(COD_ORIGEM_MERCADORIA="2", CST="040"))["COD_ORIGEM_MERCADORIA"]["state"] == ALREADY # declared origin stays; CST digit is padding
|
||||
assert run(p=product(COD_ORIGEM_MERCADORIA=None, CST="540"))["COD_ORIGEM_MERCADORIA"]["state"] == MANUAL # 5 is not on the SGV screen
|
||||
assert st(run(p=product(COD_ORIGEM_MERCADORIA=None, CST="540"), ctx=[{"scope": "GROUP", "scope_key": "000019", "key": "origin", "value": "1"}]), "COD_ORIGEM_MERCADORIA") == (CHANGE, 1)
|
||||
|
||||
|
||||
def test_cst_ipi_pis_cofins_incidence_iat_ippt():
|
||||
assert run()["CST_IPI"]["state"] == NA
|
||||
assert run(co=CO3)["CST_IPI"]["state"] == MANUAL
|
||||
r = run()
|
||||
assert r["COD_SAIDA_PIS"]["state"] == MANUAL and r["COD_SAIDA_PIS"]["input_key"] == "pis_cofins_profile" and r["ALIQ_COFINS"]["state"] == MANUAL
|
||||
prof = '{"pis_out_cst": "49", "pis_in_cst": "98", "pis_rate": "0", "cofins_out_cst": "49", "cofins_in_cst": "98", "cofins_rate": "0"}'
|
||||
r = run(ctx=[{"scope": "COMPANY", "scope_key": "", "key": "pis_cofins_profile", "value": prof}])
|
||||
assert st(r, "COD_SAIDA_PIS") == (CHANGE, "49") and st(r, "COD_ENTRADA_COFINS") == (CHANGE, "98") and st(r, "ALIQ_PIS") == (CHANGE, "0")
|
||||
bad = run(ctx=[{"scope": "COMPANY", "scope_key": "", "key": "pis_cofins_profile", "value": '{"pis_out_cst": "77"}'}])
|
||||
assert bad["COD_SAIDA_PIS"]["state"] == MANUAL # not in the official CST table
|
||||
assert run(p=product(IAT=None))["IAT"]["input_key"] == "rounding"
|
||||
assert run()["INCIDENCIA_PISCOFINS"]["state"] == ALREADY
|
||||
assert st(run(ev={**EV, "recipes": {"000001"}}), "IPPT") == (CHANGE, "P") and run()["IPPT"]["state"] == ALREADY
|
||||
assert run(p=product(TIPO="MATERIA-PRIMA"))["IPPT"]["state"] == NA
|
||||
|
||||
|
||||
def test_context_hierarchy_product_beats_group_beats_company():
|
||||
ctx = Context([{"scope": "COMPANY", "scope_key": "", "key": "product_role", "value": "RESALE"}, {"scope": "GROUP", "scope_key": "G1", "key": "product_role", "value": "OWN_PRODUCTION"},
|
||||
{"scope": "PRODUCT", "scope_key": "P1", "key": "product_role", "value": "INPUT"}])
|
||||
assert ctx.get("product_role", "P1", "G1") == ("INPUT", "PRODUCT")
|
||||
assert ctx.get("product_role", "P2", "G1") == ("OWN_PRODUCTION", "GROUP")
|
||||
assert ctx.get("product_role", "P2", "G9") == ("RESALE", "COMPANY")
|
||||
assert Context([]).get("product_role", "P2", "G9") == (None, None)
|
||||
|
||||
|
||||
def test_role_inference_is_evidence_based_and_conflicts_stay_unknown():
|
||||
assert infer_role(product(TIPO="MATERIA-PRIMA"), EV, Context([]), "000001", "1")[0] == "INPUT"
|
||||
assert infer_role(product(), {**EV, "stock_in": {}}, Context([]), "000001", "1")[0] == "UNKNOWN"
|
||||
assert infer_role(product(PROD_COZINHA=1), {**EV, "purchases": {"000001": [{}]}}, Context([]), "000001", "1")[0] == "UNKNOWN"
|
||||
assert infer_role(product(PROD_COZINHA=1), EV, Context([]), "000001", "1")[0] == "UNKNOWN" # kitchen flag alone is not proof
|
||||
assert infer_role(product(PROD_COZINHA=1), {**EV, "recipes": {"000001"}}, Context([]), "000001", "1")[0] == "OWN_PRODUCTION"
|
||||
|
||||
|
||||
# ---------------- executor guards ----------------
|
||||
def load_executor():
|
||||
spec = importlib.util.spec_from_file_location("fwe_local", ROOT / "scripts" / "firebird_write_executor.py")
|
||||
m = importlib.util.module_from_spec(spec); sys.modules[spec.name] = m; spec.loader.exec_module(m)
|
||||
return m
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
def __init__(self, db):
|
||||
self.db, self.rowcount, self._row = db, 0, None
|
||||
|
||||
def execute(self, sql, params=()):
|
||||
s = " ".join(sql.split())
|
||||
if "RDB$FIELD_TYPE" in s:
|
||||
self._row = (37, 20)
|
||||
elif s.startswith("SELECT") and "FROM C000025" in s:
|
||||
fields = [x.strip() for x in s.split("SELECT")[1].split("FROM")[0].split(",")]
|
||||
row = self.db["rows"].get(params[0])
|
||||
self._row = tuple(row[f] for f in fields) if row else None
|
||||
elif s.startswith("UPDATE"):
|
||||
row = self.db["rows"].get(params[-1])
|
||||
self.rowcount = 0 if row is None else self.db["update_rowcount"]
|
||||
if row is not None and self.rowcount == 1:
|
||||
for f, v in zip([x.split("=")[0].strip() for x in s.split("SET")[1].split("WHERE")[0].split(",")], params[:-1]):
|
||||
self.db["pending"][f] = v
|
||||
|
||||
def fetchone(self):
|
||||
return self._row
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
class FakeConn:
|
||||
def __init__(self, rows, update_rowcount=1):
|
||||
self.db = {"rows": rows, "update_rowcount": update_rowcount, "pending": {}}
|
||||
self.rolled_back = self.committed = False
|
||||
|
||||
def cursor(self):
|
||||
return FakeCursor(self.db)
|
||||
|
||||
def commit(self):
|
||||
for row in self.db["rows"].values():
|
||||
row.update(self.db["pending"])
|
||||
self.committed = True
|
||||
|
||||
def rollback(self):
|
||||
self.db["pending"].clear(); self.rolled_back = True
|
||||
|
||||
|
||||
ALLOWED = r"D:\PROJETOS\GTISISTEMAS\Projeto\Banco\CONSTRUCAO.FDB"
|
||||
|
||||
|
||||
def test_executor_allowlist_is_exact_and_field_set_is_the_contract():
|
||||
ex = load_executor()
|
||||
assert ex._LOCAL_COMPLETION_FIELDS == set(WRITE_FIELDS)
|
||||
allow = ex.load_local_allowlist()
|
||||
assert len(allow) == 5
|
||||
for path in (ALLOWED, r"D:\PROJETOS\GTISISTEMAS\Projeto\TESTESOFICIAIS.FDB"):
|
||||
ex.assert_local_completion_target(path, allow)
|
||||
for bad in (r"D:\PROJETOS\GTISISTEMAS\Banco\GTI.FDB", r"D:\PROJETOS\GTISISTEMAS\Projeto\Banco\OUTRA.FDB", r"D:\PROJETOS\GTISISTEMAS\Projeto\Banco\CONSTRUCAO.FDB.bak", r"D:\PROJETOS\GTISISTEMAS\Projeto\Banco"):
|
||||
with pytest.raises(ex.WriteRefused):
|
||||
ex.assert_local_completion_target(bad, allow)
|
||||
|
||||
|
||||
def test_guarded_product_update_success_drift_rowcount_and_refusals():
|
||||
ex = load_executor()
|
||||
allow = ex.load_local_allowlist()
|
||||
conn = FakeConn({"000001": {"IPI": None, "CEST": "1234567"}})
|
||||
rec = ex.execute_guarded_product_update(conn, database_path=ALLOWED, product_code="000001", changes={"IPI": (None, 3.9), "CEST": ("1234567", "7654321")}, now="t", allowlist=allow)
|
||||
assert rec["rowcount"] == 1 and rec["fields"]["IPI"]["after"] == 3.9 and conn.committed
|
||||
with pytest.raises(ex.WriteRefused, match="drift"):
|
||||
ex.execute_guarded_product_update(FakeConn({"000001": {"IPI": 1.0}}), database_path=ALLOWED, product_code="000001", changes={"IPI": (None, 3.9)}, now="t", allowlist=allow)
|
||||
c2 = FakeConn({"000001": {"IPI": None}}, update_rowcount=2)
|
||||
with pytest.raises(ex.WriteRefused, match="rowcount"):
|
||||
ex.execute_guarded_product_update(c2, database_path=ALLOWED, product_code="000001", changes={"IPI": (None, 3.9)}, now="t", allowlist=allow)
|
||||
assert c2.rolled_back
|
||||
with pytest.raises(ex.WriteRefused, match="allowlist"):
|
||||
ex.execute_guarded_product_update(conn, database_path=r"D:\PROJETOS\GTISISTEMAS\Banco\GTI.FDB", product_code="000001", changes={"IPI": (None, 1)}, now="t", allowlist=allow)
|
||||
with pytest.raises(ex.WriteRefused, match="contract write target"):
|
||||
ex.execute_guarded_product_update(conn, database_path=ALLOWED, product_code="000001", changes={"CODIGO": ("000001", "9")}, now="t", allowlist=allow)
|
||||
with pytest.raises(ex.WriteRefused, match="null"):
|
||||
ex.execute_guarded_product_update(conn, database_path=ALLOWED, product_code="000001", changes={"IPI": (None, None)}, now="t", allowlist=allow)
|
||||
with pytest.raises(ex.WriteRefused, match="only C000025"):
|
||||
ex.execute_guarded_product_update(conn, database_path=ALLOWED, product_code="000001", changes={"IPI": (None, 1)}, now="t", allowlist=allow, table="NFCE_ITENS")
|
||||
@@ -0,0 +1,45 @@
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from central.app.context_keys import validate_entry
|
||||
from central.app.main import create_app
|
||||
from central.app.models import Base
|
||||
|
||||
H = {"Authorization": "Bearer test-token"}
|
||||
|
||||
|
||||
def client():
|
||||
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||
Base.metadata.create_all(engine)
|
||||
return TestClient(create_app(session_factory=lambda: Session(engine), bootstrap_token="test-token"))
|
||||
|
||||
|
||||
def test_validation_rules():
|
||||
ok, err = validate_entry({"scope": "group", "scope_key": "000019", "key": "product_role", "value": "resale"})
|
||||
assert err is None and ok["value"] == "RESALE" and ok["scope"] == "GROUP"
|
||||
for bad in ({"scope": "GROUP", "key": "product_role", "value": "RESALE"}, # group without key
|
||||
{"scope": "COMPANY", "key": "product_role", "value": "BOSS"}, # not in enum
|
||||
{"scope": "GROUP", "scope_key": "1", "key": "ncm", "value": "22030000"}, # ncm is product scope
|
||||
{"scope": "PRODUCT", "scope_key": "1", "key": "ncm", "value": "2203"}, # not 8 digits
|
||||
{"scope": "COMPANY", "key": "pis_cofins_profile", "value": "{\"x\": 1}"}, # unknown profile key
|
||||
{"scope": "COMPANY", "key": "icms_rate", "value": "abc"}, {"scope": "COMPANY", "key": "nope", "value": "1"}):
|
||||
assert validate_entry(bad)[1]
|
||||
assert validate_entry({"scope": "COMPANY", "key": "pis_cofins_profile", "value": "{\"pis_out_cst\": \"49\"}"})[1] is None
|
||||
|
||||
|
||||
def test_import_is_all_or_nothing_upserting_and_listed():
|
||||
c = client()
|
||||
body = {"company_key": "abc123", "entries": [{"scope": "GROUP", "scope_key": "000019", "key": "product_role", "value": "RESALE", "reason": "revenda"},
|
||||
{"scope": "COMPANY", "key": "entry_scope", "value": "INTRA"}]}
|
||||
assert c.post("/v1/context/import", json=body, headers=H).json() == {"created": 2, "updated": 0}
|
||||
body["entries"][0]["value"] = "OWN_PRODUCTION"
|
||||
assert c.post("/v1/context/import", json=body, headers=H).json() == {"created": 0, "updated": 2}
|
||||
listed = c.get("/v1/context", params={"company_key": "abc123"}, headers=H).json()["entries"]
|
||||
assert {(e["scope"], e["key"], e["value"]) for e in listed} == {("GROUP", "product_role", "OWN_PRODUCTION"), ("COMPANY", "entry_scope", "INTRA")}
|
||||
bad = {"company_key": "abc123", "entries": [{"scope": "COMPANY", "key": "rounding", "value": "A"}, {"scope": "COMPANY", "key": "rounding", "value": "Z"}]}
|
||||
r = c.post("/v1/context/import", json=bad, headers=H)
|
||||
assert r.status_code == 422
|
||||
assert len(c.get("/v1/context", params={"company_key": "abc123"}, headers=H).json()["entries"]) == 2 # nothing partially stored
|
||||
assert c.get("/v1/context", params={"company_key": "other"}, headers=H).json()["entries"] == []
|
||||
@@ -0,0 +1,71 @@
|
||||
from central.app.fiscal_decision import ERP_FIELD_MAP, decide_product
|
||||
|
||||
REFS = {"cfop_valid": {"5102"}, "csosn": {"102", "500"}, "cst_icms": {"00", "40", "60"}, "cst_pis": {"01"}, "cst_cofins": {"01"},
|
||||
"origem": set("012345678"), "fcp_uf": {"BA": {"aliq_1": "Fixo:2.00"}}, "cbs_rate_2026": 0.009, "cclasstrib_count": 164}
|
||||
COMPANY = {"uf": "BA", "crt": "1"}
|
||||
|
||||
|
||||
def product(**kw):
|
||||
base = {"CODIGO": "1", "PRODUTO": "X", "CLASSIFICACAO_FISCAL": "22030000", "CFOP": "5102", "CST": "040", "CSOSN": "102",
|
||||
"CEST": None, "IPI": None, "EX_TIPI": None, "ORIGEM": None, "COD_SAIDA_PIS": None, "COD_SAIDA_COFINS": None, "ALIQ_PIS": None}
|
||||
base.update(kw)
|
||||
return base
|
||||
|
||||
|
||||
def decision(**kw):
|
||||
base = {"status": "EXACT_NCM", "ipi_status": "IPI_RESOLVED", "cest_status": "CEST_CANDIDATES", "ba_st_mva_status": "MVA_CANDIDATES",
|
||||
"dataset_versions": {}, "classification": {"ncm": "22030000", "ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.9}, "cest_candidates": [1, 2]}}
|
||||
base.update(kw)
|
||||
return base
|
||||
|
||||
|
||||
def test_numeric_ipi_change_is_the_only_auto_apply():
|
||||
out = decide_product(product(IPI=5102.0), decision(), COMPANY, REFS)
|
||||
assert out["IPI"]["action_class"] == "AUTO_APPLY_SAFE" and out["IPI"]["write_eligible"] and out["IPI"]["proposed_value"] == 3.9
|
||||
assert [d for d, r in out.items() if r["write_eligible"]] == ["IPI"]
|
||||
|
||||
|
||||
def test_same_ipi_is_already_correct_not_write():
|
||||
out = decide_product(product(IPI=3.9), decision(), COMPANY, REFS)
|
||||
assert out["IPI"]["action_class"] == "ALREADY_CORRECT" and not out["IPI"]["write_eligible"]
|
||||
|
||||
|
||||
def test_non_numeric_and_blank_ipi_never_write():
|
||||
d = decision(classification={"ipi": {"rate_kind": "NON_NUMERIC", "rate_classification": "NT"}})
|
||||
assert not decide_product(product(), d, COMPANY, REFS)["IPI"]["write_eligible"]
|
||||
d = decision(classification={"ipi": {"rate_kind": "BLANK"}})
|
||||
r = decide_product(product(), d, COMPANY, REFS)["IPI"]
|
||||
assert not r["write_eligible"] and r["status"] == "RULE_UNRESOLVED"
|
||||
|
||||
|
||||
def test_operation_dependent_fields_never_resolved_or_written():
|
||||
out = decide_product(product(), decision(), COMPANY, REFS)
|
||||
for dom in ("CFOP", "CSOSN", "CST_ICMS", "PIS_CST", "COFINS_CST", "FCP", "RTC_CCLASSTRIB", "RTC_CBS_RATE"):
|
||||
assert out[dom]["status"] == "OPERATION_CONTEXT_REQUIRED" and not out[dom]["write_eligible"]
|
||||
assert out["CFOP"]["domain_valid"] is True and out["FCP"]["fcp_rate_available"] is True
|
||||
assert out["RTC_CBS_RATE"]["proposed_value"] is None and out["RTC_CBS_RATE"]["reference_rate_2026"] == 0.009
|
||||
|
||||
|
||||
def test_ncm_legacy_is_review_and_never_write():
|
||||
out = decide_product(product(), decision(status="LEGACY_NCM"), COMPANY, REFS)
|
||||
assert out["NCM"]["action_class"] == "REVIEW_REQUIRED" and not any(r["write_eligible"] for r in out.values())
|
||||
|
||||
|
||||
def test_origin_only_candidate_and_unconfirmed_mapping():
|
||||
out = decide_product(product(), decision(), COMPANY, REFS)
|
||||
assert out["ORIGEM"]["status"] == "RULE_CANDIDATE" and not out["ORIGEM"]["write_eligible"]
|
||||
assert {f["ERP_FIELD"] for f in ERP_FIELD_MAP if f["MAPPING"] == "MAPPING_UNCONFIRMED"} >= {"ORIGEM", "CFOP_ENTRADA", "CST_ENTRADA"}
|
||||
|
||||
|
||||
def test_mva_never_written_even_when_resolved():
|
||||
d = decision(ba_st_mva_status="MVA_RESOLVED", classification={"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 1}, "ba_st_mva": {"mva_original": "140%"}})
|
||||
r = decide_product(product(IPI=1), d, COMPANY, REFS)["MVA"]
|
||||
assert r["status"] == "RULE_RESOLVED" and not r["write_eligible"]
|
||||
|
||||
|
||||
def test_single_ex_without_description_evidence_is_candidate_not_write():
|
||||
cls = {"ipi": {"rate_kind": "NUMERIC", "rate_numeric": 3.9}, "ipi_candidates": [{"ex": "1", "description": "Chope"}]}
|
||||
out = decide_product(product(PRODUTO="CERVEJA LATA", IPI=3.9), decision(classification=cls), COMPANY, REFS)
|
||||
assert out["EX_TIPI"]["status"] == "RULE_CANDIDATE" and not out["EX_TIPI"]["write_eligible"]
|
||||
out = decide_product(product(PRODUTO="CHOPE ESCURO", IPI=3.9), decision(classification=cls), COMPANY, REFS)
|
||||
assert out["EX_TIPI"]["status"] == "RULE_RESOLVED" and out["EX_TIPI"]["write_eligible"]
|
||||
@@ -0,0 +1,9 @@
|
||||
from central.app.fiscal_determination import FiscalDetermination
|
||||
|
||||
|
||||
def test_returns_unresolved_csosn_and_cfop_without_official_rule():
|
||||
result = FiscalDetermination().determine({"ncm": "08043000", "cest_candidates": []})
|
||||
|
||||
assert result["csosn"] is None
|
||||
assert result["cfop"] is None
|
||||
assert "OFFICIAL_RULE_NOT_IMPLEMENTED" in result["reason_codes"]
|
||||
@@ -0,0 +1,29 @@
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from central.app.main import create_app
|
||||
|
||||
|
||||
def test_health_reports_database_dependency():
|
||||
client = TestClient(create_app(database_probe=lambda: True))
|
||||
|
||||
assert client.get("/health").json() == {"status": "ok"}
|
||||
|
||||
|
||||
def test_health_is_unavailable_when_database_is_down():
|
||||
client = TestClient(create_app(database_probe=lambda: False))
|
||||
|
||||
assert client.get("/health").status_code == 503
|
||||
|
||||
|
||||
def test_version_endpoint_reports_release_and_fiscal_versions():
|
||||
body = TestClient(create_app(database_probe=lambda: True)).get("/version").json()
|
||||
assert body["version"] == "1.0.0-rc1" and body["api_compat"] == "1.0" and body["engine_version"] and body["decision_version"]
|
||||
|
||||
|
||||
def test_production_refuses_the_dev_token(monkeypatch):
|
||||
import pytest
|
||||
from central.app import main as main_module
|
||||
monkeypatch.setattr(main_module.settings.__class__, "environment", "production", raising=False) if False else None
|
||||
monkeypatch.setattr(main_module, "settings", type("S", (), {"environment": "production", "agent_dev_token": "dev", "log_level": "INFO"})())
|
||||
with pytest.raises(RuntimeError):
|
||||
create_app(database_probe=lambda: True)
|
||||
@@ -0,0 +1,58 @@
|
||||
from central.app.icms_context import csosn_v2, product_role, st_conflict_class, st_status, withheld_status
|
||||
|
||||
DOM = {"102", "500", "101", "400"}
|
||||
|
||||
|
||||
def test_cest_candidate_never_confirms_st():
|
||||
assert st_status("EXACT_NCM", "MVA_CANDIDATES", "CEST_CANDIDATES", "", None, 4)[0] == "ST_CANDIDATE"
|
||||
assert st_status("EXACT_NCM", "MVA_RESOLVED", "CEST_CANDIDATES", "", "03.021.03", 4)[0] == "ST_CANDIDATE"
|
||||
|
||||
|
||||
def test_st_confirmed_needs_erp_cest_corroboration():
|
||||
assert st_status("EXACT_NCM", "MVA_RESOLVED", "CEST_CANDIDATES", "0302103", "03.021.03", 4)[0] == "ST_CONFIRMED"
|
||||
assert st_status("EXACT_NCM", "MVA_RESOLVED", "CEST_CONFIRMED", "", "03.021.03", 4)[0] == "ST_CONFIRMED"
|
||||
|
||||
|
||||
def test_st_not_applicable_needs_both_lists_silent_and_valid_ncm():
|
||||
assert st_status("EXACT_NCM", "NO_MVA_MAPPING_FOUND", "NO_CEST_MAPPING_FOUND", None, None, None)[0] == "ST_NOT_APPLICABLE"
|
||||
assert st_status("EXACT_NCM", "NO_MVA_MAPPING_FOUND", "CEST_CANDIDATES", None, None, None)[0] == "ST_UNKNOWN"
|
||||
assert st_status("LEGACY_NCM", "NO_MVA_MAPPING_FOUND", "NO_CEST_MAPPING_FOUND", None, None, None)[0] == "ST_UNKNOWN"
|
||||
|
||||
|
||||
def test_withholding_needs_positive_purchase_evidence():
|
||||
assert withheld_status([])[0] == "ST_WITHHOLDING_UNKNOWN"
|
||||
assert withheld_status([{"icms_retido": "N", "valor_retido": 0, "sub_valor": 0}])[0] == "ST_ALREADY_WITHHELD_NOT_CONFIRMED"
|
||||
assert withheld_status([{"icms_retido": "S", "valor_retido": 0, "sub_valor": 0}])[0] == "ST_ALREADY_WITHHELD_CONFIRMED"
|
||||
assert withheld_status([{"icms_retido": "N", "valor_retido": 5, "sub_valor": 0}])[0] == "ST_ALREADY_WITHHELD_CONFIRMED"
|
||||
|
||||
|
||||
def test_role_uses_objective_evidence_only():
|
||||
assert product_role("MATERIA-PRIMA", False, False, False, False, False)[0] == "INPUT"
|
||||
assert product_role("VENDA", True, False, False, False, False)[0] == "OWN_PRODUCTION"
|
||||
assert product_role("VENDA", False, False, False, False, True)[0] == "RESALE"
|
||||
assert product_role("VENDA", False, False, False, False, False)[0] == "UNKNOWN"
|
||||
assert product_role("VENDA", True, False, False, False, True)[0] == "UNKNOWN"
|
||||
|
||||
|
||||
def test_csosn_never_assumes_102_and_never_writes():
|
||||
for st in ("ST_CANDIDATE", "ST_UNKNOWN"):
|
||||
r = csosn_v2("102", "102", "EXACT_NCM", st, "ST_WITHHOLDING_UNKNOWN", "RESALE", DOM)
|
||||
assert r["status"] == "CSOSN_CONTEXT_REQUIRED" and r["proposed_csosn"] is None
|
||||
r = csosn_v2("102", "102", "EXACT_NCM", "ST_NOT_APPLICABLE", "ST_WITHHOLDING_UNKNOWN", "RESALE", DOM)
|
||||
assert r["status"] == "CSOSN_CANDIDATE" and not r["write_eligible"]
|
||||
assert csosn_v2("102", "102", "EXACT_NCM", "ST_NOT_APPLICABLE", "ST_WITHHOLDING_UNKNOWN", "UNKNOWN", DOM)["status"] == "CSOSN_CONTEXT_REQUIRED"
|
||||
assert csosn_v2("102", "102", "LEGACY_NCM", "ST_NOT_APPLICABLE", "ST_WITHHOLDING_UNKNOWN", "RESALE", DOM)["status"] == "CSOSN_UNRESOLVED"
|
||||
|
||||
|
||||
def test_csosn_resolved_only_with_st_withholding_and_role_proven():
|
||||
r = csosn_v2("102", "102", "EXACT_NCM", "ST_CONFIRMED", "ST_ALREADY_WITHHELD_CONFIRMED", "RESALE", DOM)
|
||||
assert r["status"] == "CSOSN_RESOLVED" and r["proposed_csosn"] == "500" and r["write_eligible"] is False
|
||||
assert csosn_v2("102", "102", "EXACT_NCM", "ST_CONFIRMED", "ST_WITHHOLDING_UNKNOWN", "RESALE", DOM)["status"] == "CSOSN_CONTEXT_REQUIRED"
|
||||
assert csosn_v2("500", "102", "EXACT_NCM", "ST_CONFIRMED", "ST_ALREADY_WITHHELD_NOT_CONFIRMED", "RESALE", DOM)["status"] == "CSOSN_CONFLICT"
|
||||
|
||||
|
||||
def test_conflict_classes():
|
||||
assert st_conflict_class("CEST_CANDIDATES", "MVA_RESOLVED", 4, "", "03.021.03", "5102", True, False)[0] == "ERP_HISTORY_CONFLICT"
|
||||
assert st_conflict_class(None, "NO_MVA_MAPPING_FOUND", None, "", None, "5405", False, True)[0] == "BA_ST_RULE_CONFLICT"
|
||||
assert st_conflict_class("CEST_DESCRIPTION_MISMATCH", "MVA_RESOLVED", 6, "", None, "5102", False, False)[0] == "DESCRIPTION_MISMATCH"
|
||||
assert st_conflict_class(None, "MVA_RESOLVED", 6, "", None, "5102", False, False)[0] == "NO_EVIDENCE"
|
||||
@@ -0,0 +1,132 @@
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from central.app.jobs import claim_next_job, complete_job, create_job, fail_job
|
||||
from central.app.main import create_app
|
||||
from central.app.models import Agent, Base, Job, Tenant
|
||||
|
||||
|
||||
def _make_agent(session):
|
||||
tenant = Tenant(name="development")
|
||||
session.add(tenant); session.flush()
|
||||
agent = Agent(tenant_id=tenant.id, name="agent-a", token_hash="hash-a")
|
||||
other = Agent(tenant_id=tenant.id, name="agent-b", token_hash="hash-b")
|
||||
session.add_all([agent, other]); session.commit()
|
||||
return agent, other
|
||||
|
||||
|
||||
def test_claim_is_scoped_to_owning_agent_and_marks_running():
|
||||
engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
agent, other = _make_agent(session)
|
||||
job = create_job(session, agent, "REPROCESS_ALL")
|
||||
assert claim_next_job(session, other) is None # other agent never sees it
|
||||
claimed = claim_next_job(session, agent)
|
||||
assert claimed.id == job.id
|
||||
assert claimed.status == "RUNNING"
|
||||
assert claimed.attempts == 1
|
||||
|
||||
|
||||
def test_claim_is_not_reentrant_once_running():
|
||||
engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
agent, _ = _make_agent(session)
|
||||
create_job(session, agent, "REPROCESS_ALL")
|
||||
first = claim_next_job(session, agent)
|
||||
second = claim_next_job(session, agent) # repeated polling must not duplicate consumption
|
||||
assert first is not None
|
||||
assert second is None
|
||||
|
||||
|
||||
def test_completed_job_can_never_run_again():
|
||||
engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
agent, _ = _make_agent(session)
|
||||
create_job(session, agent, "REPROCESS_ALL")
|
||||
job = claim_next_job(session, agent)
|
||||
assert complete_job(session, job, {"writes": 0}) is True
|
||||
assert session.get(Job, job.id).status == "COMPLETED"
|
||||
assert claim_next_job(session, agent) is None
|
||||
assert complete_job(session, job, {"writes": 1}) is False # already terminal
|
||||
|
||||
|
||||
def test_failed_job_is_terminal_with_sanitized_error():
|
||||
engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
agent, _ = _make_agent(session)
|
||||
create_job(session, agent, "REPROCESS_ALL")
|
||||
job = claim_next_job(session, agent)
|
||||
assert fail_job(session, job, "boom " * 1000) is True
|
||||
stored = session.get(Job, job.id)
|
||||
assert stored.status == "FAILED"
|
||||
assert len(stored.error) <= 2000
|
||||
|
||||
|
||||
def _client():
|
||||
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||
Base.metadata.create_all(engine)
|
||||
app = create_app(session_factory=lambda: Session(engine), bootstrap_token="test-token")
|
||||
return TestClient(app), engine
|
||||
|
||||
|
||||
def test_job_api_end_to_end_flow():
|
||||
client, engine = _client()
|
||||
headers = {"Authorization": "Bearer test-token"}
|
||||
|
||||
heartbeat = client.post("/v1/agents/heartbeat", json={"status": "IDLE", "schema_fingerprint": "abc", "agent_version": "0.1.0"}, headers=headers)
|
||||
assert heartbeat.status_code == 200
|
||||
assert heartbeat.json()["enabled"] is True
|
||||
|
||||
created = client.post("/v1/jobs", json={"kind": "REPROCESS_PRODUCTS", "payload": {"codigos": ["000026"]}}, headers=headers)
|
||||
assert created.status_code == 200
|
||||
job_id = created.json()["job_id"]
|
||||
assert created.json()["status"] == "PENDING"
|
||||
|
||||
claimed = client.get("/v1/jobs/next", headers=headers)
|
||||
assert claimed.json()["job_id"] == job_id
|
||||
assert claimed.json()["status"] == "RUNNING"
|
||||
|
||||
empty = client.get("/v1/jobs/next", headers=headers)
|
||||
assert empty.json() is None # no more pending work; not the same job again
|
||||
|
||||
completed = client.post(f"/v1/jobs/{job_id}/complete", json={"result_summary": {"writes_succeeded": 1}}, headers=headers)
|
||||
assert completed.json()["status"] == "COMPLETED"
|
||||
assert completed.json()["result_summary"]["writes_succeeded"] == 1
|
||||
|
||||
fetched = client.get(f"/v1/jobs/{job_id}", headers=headers)
|
||||
assert fetched.json()["status"] == "COMPLETED"
|
||||
|
||||
|
||||
def test_agent_status_derives_online_offline():
|
||||
client, engine = _client()
|
||||
headers = {"Authorization": "Bearer test-token"}
|
||||
|
||||
offline = client.get("/v1/agents/me", headers=headers)
|
||||
assert offline.json()["status"] == "OFFLINE" # never heartbeated yet
|
||||
|
||||
client.post("/v1/agents/heartbeat", json={"status": "IDLE"}, headers=headers)
|
||||
online = client.get("/v1/agents/me", headers=headers)
|
||||
body = online.json()
|
||||
assert body["status"] == "ONLINE"
|
||||
assert body["enabled"] is True
|
||||
assert "company_id" in body
|
||||
|
||||
|
||||
def test_job_creation_rejects_unknown_kind():
|
||||
client, _ = _client()
|
||||
headers = {"Authorization": "Bearer test-token"}
|
||||
response = client.post("/v1/jobs", json={"kind": "DELETE_EVERYTHING"}, headers=headers)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
def test_reprocess_products_requires_codigos():
|
||||
client, _ = _client()
|
||||
headers = {"Authorization": "Bearer test-token"}
|
||||
response = client.post("/v1/jobs", json={"kind": "REPROCESS_PRODUCTS", "payload": {}}, headers=headers)
|
||||
assert response.status_code == 400
|
||||
@@ -0,0 +1,38 @@
|
||||
from central.app.field_contract import ALREADY, CHANGE, MANUAL, NA, sped_pis_cofins_match
|
||||
from central.tests.test_field_contract import CO1, CO3, EV, decision, product, run, st
|
||||
|
||||
UNUSED = {**CO1, "pis_cofins_unused_by_emission": True}
|
||||
|
||||
|
||||
def test_sped_table_lookup_is_prefix_based_and_active_only():
|
||||
hits = sped_pis_cofins_match("10063021")
|
||||
assert hits and {h[1] for h in hits} == {"06"} and hits[0][0] == "4.3.13"
|
||||
assert sped_pis_cofins_match("99999999") == []
|
||||
|
||||
|
||||
def test_simples_pis_cofins_written_only_from_official_table_and_never_assumed_zero():
|
||||
r = run(p=product(CLASSIFICACAO_FISCAL="10063021"), co=UNUSED)
|
||||
assert st(r, "COD_SAIDA_PIS") == (CHANGE, "06") and st(r, "COD_SAIDA_COFINS") == (CHANGE, "06")
|
||||
assert r["ALIQ_PIS"]["state"] == NA and r["COD_ENTRADA_PIS"]["state"] == NA
|
||||
plain = run(co=UNUSED) # NCM without any SPED special regime: SGV does not consume the field, nothing is invented
|
||||
assert all(plain[f]["state"] == NA and plain[f]["proposed"] is None for f in ("COD_SAIDA_PIS", "ALIQ_PIS", "COD_ENTRADA_COFINS"))
|
||||
assert run(co=CO1)["COD_SAIDA_PIS"]["state"] == MANUAL # without emission evidence the company question remains
|
||||
assert run(co={**CO3, "pis_cofins_unused_by_emission": True})["COD_SAIDA_PIS"]["state"] == MANUAL # evidence only relaxes CRT 1
|
||||
|
||||
|
||||
def test_cfop_never_contradicts_the_erp_st_declaration():
|
||||
r = run(p=product(CFOP="5405", CSOSN="500", CST="060", SITUACAO_TRIBUTARIA="T"))
|
||||
assert r["CFOP"]["state"] != CHANGE or r["CFOP"]["proposed"] != "5102"
|
||||
|
||||
|
||||
def test_kitchen_flag_alone_and_cst_digit_never_prove_role_or_origin():
|
||||
r = run(p=product(PROD_COZINHA=1, COD_ORIGEM_MERCADORIA=None, CST="040"))
|
||||
assert r["COD_ORIGEM_MERCADORIA"]["state"] == MANUAL and r["COD_ORIGEM_MERCADORIA"]["reason"] == "ORIGIN_NO_EVIDENCE"
|
||||
assert r["IPPT"]["proposed"] != "P" or r["IPPT"]["state"] != CHANGE
|
||||
|
||||
|
||||
def test_store_ipi_profile_is_a_company_decision_never_defaulted():
|
||||
undecided = run(ipi_profile=None)
|
||||
assert undecided["IPI"]["state"] == MANUAL and undecided["IPI"]["input_key"] == "store_ipi_profile"
|
||||
assert run(ipi_profile={"scope": "COMPANY", "scope_key": "", "key": "store_ipi_profile", "value": "LEAVE_NOT_APPLICABLE"})["IPI"]["state"] == NA
|
||||
assert run()["IPI"]["state"] == CHANGE # POPULATE_REFERENCE_RATE keeps the TIPI rule
|
||||
@@ -0,0 +1,66 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from central.app.datasets.ncm import parse_ncm_payload
|
||||
|
||||
|
||||
PAYLOAD = b'{"Data_Ultima_Atualizacao_NCM":"2026-09-17","Ato":"Ato","Nomenclaturas":[{"Codigo":"08043000","Descricao":"Abacaxis frescos","Data_Inicio":"2022-01-01","Data_Fim":"9999-12-31"}]}'
|
||||
|
||||
|
||||
def test_import_promotes_and_is_idempotent():
|
||||
from central.app.datasets.service import DatasetService
|
||||
from central.app.models import Base, Dataset, NcmEntry
|
||||
|
||||
engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
service = DatasetService(session)
|
||||
first = service.import_payload(parse_ncm_payload(PAYLOAD), "https://source", datetime.now(timezone.utc), minimum_entries=1)
|
||||
second = service.import_payload(parse_ncm_payload(PAYLOAD), "https://source", datetime.now(timezone.utc), minimum_entries=1)
|
||||
assert first.id == second.id
|
||||
assert session.get(Dataset, first.id).status == "ACTIVE"
|
||||
assert session.query(NcmEntry).filter_by(dataset_id=first.id).count() == 1
|
||||
|
||||
|
||||
def test_classifier_validates_ncm_and_never_proposes_fiscal_fields():
|
||||
from central.app.classification import Classifier
|
||||
|
||||
class Repository:
|
||||
def active_entries(self):
|
||||
return [type("Entry", (), {"code": "08043000", "description": "Abacaxis frescos"})()]
|
||||
|
||||
def dataset_versions(self):
|
||||
return {"ncm": "dataset-1"}
|
||||
|
||||
result = Classifier(Repository()).classify({"description": "abacaxi", "current_ncm": "0804.30.00"})
|
||||
assert result["status"] == "EXACT_NCM"
|
||||
assert result["classification"]["ncm"] == "08043000"
|
||||
assert result["classification"]["cest"] is None
|
||||
assert result["can_auto_apply"] is False
|
||||
|
||||
|
||||
def test_api_auth_and_cache():
|
||||
from central.app.main import create_app
|
||||
from central.app.models import Base, Dataset, NcmEntry
|
||||
|
||||
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||
Base.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
dataset = Dataset(kind="NCM", provider="Siscomex", sha256="a" * 64, version_label="v1", act="act", source_url="url", status="ACTIVE", record_count=1)
|
||||
session.add(dataset)
|
||||
session.flush()
|
||||
session.add(NcmEntry(dataset_id=dataset.id, code="08043000", description="Abacaxis frescos", valid_from="2022", valid_to="9999"))
|
||||
session.commit()
|
||||
app = create_app(session_factory=lambda: Session(engine), bootstrap_token="test-token")
|
||||
client = TestClient(app)
|
||||
payload = {"product": {"product_id": "1", "description": "abacaxi", "current_ncm": "08043000"}}
|
||||
assert client.post("/v1/classify", json=payload).status_code == 401
|
||||
headers = {"Authorization": "Bearer test-token"}
|
||||
first = client.post("/v1/classify", json=payload, headers=headers).json()
|
||||
second = client.post("/v1/classify", json=payload, headers=headers).json()
|
||||
assert first["request_fingerprint"] == second["request_fingerprint"]
|
||||
assert second["provenance"]["cache_hit"] is True
|
||||
@@ -0,0 +1,172 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
|
||||
CEST_HTML = b"""
|
||||
<html><body>
|
||||
<table><tr><td>Ignored</td></tr></table>
|
||||
<h2>ANEXO II</h2>
|
||||
<table>
|
||||
<tr><th>ITEM</th><th>CEST</th><th>NCM/SH</th><th>DESCRICAO</th></tr>
|
||||
<tr><td>1.0</td><td>01.001.00</td><td>0804.30.00</td><td>Abacaxis</td></tr>
|
||||
<tr><td>2.0</td><td>01.002.00</td><td>0804.30.00<br>0207.14.00</td><td>Frutas e aves</td></tr>
|
||||
</table>
|
||||
<h2>ANEXO XXVII</h2>
|
||||
<table><tr><th>CEST</th><th>NCM</th></tr><tr><td>99.999.99</td><td>9999.99.99</td></tr></table>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
|
||||
def test_cest_parser_accepts_only_covered_annex_tables_and_hashes_raw_html():
|
||||
from central.app.datasets.cest import parse_cest_html
|
||||
|
||||
parsed = parse_cest_html(CEST_HTML)
|
||||
|
||||
assert parsed.version == "CV142_18"
|
||||
assert parsed.sha256
|
||||
assert parsed.entries[0].annex == "II"
|
||||
assert parsed.entries[1].ncm_codes == ("08043000", "02071400")
|
||||
assert len(parsed.entries) == 2
|
||||
|
||||
|
||||
def test_cest_parser_fails_closed_when_an_annex_table_has_no_required_columns():
|
||||
from central.app.datasets.cest import parse_cest_html
|
||||
|
||||
with pytest.raises(ValueError, match="required columns"):
|
||||
parse_cest_html(b"<h2>ANEXO II</h2><table><tr><th>CEST</th></tr><tr><td>01.001.00</td></tr></table>")
|
||||
|
||||
|
||||
def test_cest_parser_accepts_official_four_digit_ncm_headings():
|
||||
from central.app.datasets.cest import parse_cest_html
|
||||
|
||||
parsed = parse_cest_html(b"<h2>ANEXO III</h2><table><tr><th>CEST</th><th>NCM/SH</th></tr><tr><td>02.001.00</td><td>2205</td></tr></table>")
|
||||
|
||||
assert parsed.entries[0].ncm_codes == ("2205",)
|
||||
|
||||
|
||||
def test_cest_parser_ignores_official_editorial_rows_without_cest_data():
|
||||
from central.app.datasets.cest import parse_cest_html
|
||||
|
||||
parsed = parse_cest_html(b"<h2>ANEXO II</h2><table><tr><th>CEST</th><th>NCM/SH</th></tr><tr><td>Nova redacao dada ao item.</td></tr><tr><td>01.001.00</td><td>3815.12.10</td></tr></table>")
|
||||
|
||||
assert [entry.code for entry in parsed.entries] == ["01.001.00"]
|
||||
|
||||
|
||||
def test_cest_parser_preserves_chapter_ncm_prefixes_and_empty_official_ncm_cells():
|
||||
from central.app.datasets.cest import parse_cest_html
|
||||
|
||||
parsed = parse_cest_html(b"<h2>ANEXO XXVI</h2><table><tr><th>CEST</th><th>NCM/SH</th></tr><tr><td>28.055.00</td><td>Capitulo 33</td></tr><tr><td>28.056.00</td><td></td></tr></table>")
|
||||
|
||||
assert parsed.entries[0].ncm_codes == ("33",)
|
||||
assert parsed.entries[1].ncm_codes == ()
|
||||
|
||||
|
||||
def test_classifier_distinguishes_exact_legacy_and_invalid_ncm_and_uses_cest_matches():
|
||||
from central.app.classification import Classifier
|
||||
|
||||
class Repository:
|
||||
def active_entries(self):
|
||||
return [type("Entry", (), {"code": "08043000", "description": "Abacaxis"})()]
|
||||
|
||||
def cest_for_ncm(self, code):
|
||||
return [type("Cest", (), {"code": "01.001.00", "description": "Abacaxis"})()] if code == "08043000" else []
|
||||
|
||||
def legacy_ncm(self, code):
|
||||
return {"code": code, "source": "Siscomex", "source_date": "2026-09-17", "reason": "NCM not present in current official nomenclature; no official successor."} if code == "02071400" else None
|
||||
|
||||
def dataset_versions(self):
|
||||
return {"ncm": "ncm-version", "cest": "cest-version"}
|
||||
|
||||
classifier = Classifier(Repository())
|
||||
|
||||
exact = classifier.classify({"current_ncm": "0804.30.00", "description": "abacaxi"})
|
||||
legacy = classifier.classify({"current_ncm": "0207.14.00"})
|
||||
invalid = classifier.classify({"current_ncm": "0000.00.00"})
|
||||
|
||||
assert exact["status"] == "EXACT_NCM"
|
||||
assert exact["classification"]["cest"] == ["01.001.00"]
|
||||
assert legacy["status"] == "LEGACY_NCM"
|
||||
assert legacy["classification"]["ncm"] == "02071400"
|
||||
assert legacy["legacy_ncm"]["reason"].endswith("no official successor.")
|
||||
assert invalid["status"] == "INVALID_NCM"
|
||||
|
||||
|
||||
def test_classifier_reports_confirmed_cest_only_for_one_distinct_official_candidate():
|
||||
from central.app.classification import Classifier
|
||||
|
||||
class Repository:
|
||||
def active_entries(self):
|
||||
return [type("Entry", (), {"code": "08043000", "description": "Abacaxi"})()]
|
||||
def cest_for_ncm(self, code):
|
||||
return [type("Cest", (), {"code": "01.001.00", "description": "Abacaxi"})()]
|
||||
def legacy_ncm(self, code): return None
|
||||
def dataset_versions(self): return {"ncm": "n", "cest": "c"}
|
||||
|
||||
result = Classifier(Repository()).classify({"current_ncm": "08043000", "description": "Abacaxi"})
|
||||
assert result["cest_status"] == "CEST_CONFIRMED"
|
||||
|
||||
|
||||
def test_classifier_reports_candidates_for_multiple_distinct_cest_codes():
|
||||
from central.app.classification import Classifier
|
||||
|
||||
class Repository:
|
||||
def active_entries(self):
|
||||
return [type("Entry", (), {"code": "08043000", "description": "Abacaxi"})()]
|
||||
def cest_for_ncm(self, code):
|
||||
return [type("Cest", (), {"code": "01.001.00", "description": "Abacaxi"})(), type("Cest", (), {"code": "01.002.00", "description": "Frutas"})()]
|
||||
def legacy_ncm(self, code): return None
|
||||
def dataset_versions(self): return {"ncm": "n", "cest": "c"}
|
||||
|
||||
result = Classifier(Repository()).classify({"current_ncm": "08043000", "description": "Abacaxi"})
|
||||
assert result["cest_status"] == "CEST_CANDIDATES"
|
||||
assert result["classification"]["cest"] == ["01.001.00", "01.002.00"]
|
||||
|
||||
|
||||
def test_cest_import_creates_many_to_many_ncm_links_and_stores_html_metadata():
|
||||
from central.app.datasets.cest import parse_cest_html
|
||||
from central.app.datasets.service import DatasetService
|
||||
from central.app.models import Base, CestEntry, CestNcmEntry, Dataset
|
||||
|
||||
engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
dataset = DatasetService(session).import_cest_payload(
|
||||
parse_cest_html(CEST_HTML), "https://www.confaz.fazenda.gov.br/legislacao/convenios/2018/CV142_18", datetime.now(timezone.utc)
|
||||
)
|
||||
|
||||
assert dataset.kind == "CEST"
|
||||
assert dataset.raw_sha256 == dataset.sha256
|
||||
assert session.query(CestEntry).filter_by(dataset_id=dataset.id).count() == 2
|
||||
assert session.query(CestNcmEntry).count() == 3
|
||||
|
||||
|
||||
def test_cest_lookup_matches_official_ncm_prefix_links():
|
||||
from central.app.datasets.cest import parse_cest_html
|
||||
from central.app.datasets.service import DatasetService
|
||||
from central.app.models import Base
|
||||
|
||||
engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
service = DatasetService(session)
|
||||
service.import_cest_payload(parse_cest_html(b"<h2>ANEXO III</h2><table><tr><th>CEST</th><th>NCM</th></tr><tr><td>02.001.00</td><td>2205</td></tr></table>"), "https://source", datetime.now(timezone.utc))
|
||||
|
||||
assert [entry.code for entry in service.cest_for_ncm("22051000")] == ["02.001.00"]
|
||||
|
||||
|
||||
def test_cest_import_unions_repeated_official_cest_codes_into_one_link_set():
|
||||
from central.app.datasets.cest import parse_cest_html
|
||||
from central.app.datasets.service import DatasetService
|
||||
from central.app.models import Base, CestEntry, CestNcmEntry
|
||||
|
||||
payload = parse_cest_html(b"<h2>ANEXO II</h2><table><tr><th>CEST</th><th>NCM</th></tr><tr><td>01.042.00</td><td>8421.32.00</td></tr><tr><td>01.042.00</td><td>8421.39.20</td></tr></table>")
|
||||
engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
dataset = DatasetService(session).import_cest_payload(payload, "https://source", datetime.now(timezone.utc))
|
||||
|
||||
assert session.query(CestEntry).filter_by(dataset_id=dataset.id).count() == 1
|
||||
assert {link.ncm_code for link in session.query(CestNcmEntry).all()} == {"84213200", "84213920"}
|
||||
@@ -0,0 +1,17 @@
|
||||
import pytest
|
||||
|
||||
from central.app.datasets.ncm import parse_ncm_payload
|
||||
|
||||
|
||||
def test_parser_accepts_verified_siscomex_payload_shape():
|
||||
payload = b'{"Data_Ultima_Atualizacao_NCM":"Vigente em 17/09/2026","Ato":"Resolucao Gecex 926/2026","Nomenclaturas":[{"Codigo":"08043000","Descricao":"Abacaxis.","Data_Inicio":"01/04/2022","Data_Fim":"31/12/9999"}]}'
|
||||
|
||||
parsed = parse_ncm_payload(payload)
|
||||
|
||||
assert parsed.version == "Vigente em 17/09/2026"
|
||||
assert parsed.entries[0].code == "08043000"
|
||||
|
||||
|
||||
def test_parser_rejects_unknown_payload_shape():
|
||||
with pytest.raises(ValueError, match="Nomenclaturas"):
|
||||
parse_ncm_payload(b'{"items": []}')
|
||||
@@ -0,0 +1,93 @@
|
||||
from datetime import datetime, timezone
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from central.app.datasets.nfe_moc_tables import parse_moc_tables
|
||||
from central.app.datasets.service import DatasetService
|
||||
from central.app.models import Base
|
||||
|
||||
SOURCE = Path(__file__).parents[2] / "sources" / "raw" / "bootstrap_v1" / "moc7-anexo-i-leiaute-e-rv.pdf"
|
||||
pytestmark = pytest.mark.skipif(not SOURCE.exists(), reason="official MOC PDF not present locally")
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _payload():
|
||||
# Parsing this 153-page PDF is expensive (~40s); every test in this module
|
||||
# shares one parse instead of re-running it per test.
|
||||
return parse_moc_tables(SOURCE.read_bytes())
|
||||
|
||||
|
||||
def test_parses_all_five_domains_with_expected_minimum_coverage():
|
||||
payload = _payload()
|
||||
from collections import Counter
|
||||
counts = Counter(e.domain for e in payload.entries)
|
||||
assert counts["ORIGEM_MERCADORIA"] == 9
|
||||
assert counts["CST_ICMS"] >= 11
|
||||
assert counts["CSOSN"] >= 10
|
||||
assert counts["CST_PIS"] >= 29
|
||||
assert counts["CST_COFINS"] >= 29
|
||||
|
||||
|
||||
def test_every_entry_has_provenance_and_no_invented_text():
|
||||
payload = _payload()
|
||||
for entry in payload.entries:
|
||||
assert entry.source_section.startswith("MOC7 p.")
|
||||
assert entry.description.strip()
|
||||
assert entry.code.strip()
|
||||
|
||||
|
||||
def test_origem_codes_are_0_through_8():
|
||||
payload = _payload()
|
||||
codes = sorted(e.code for e in payload.entries if e.domain == "ORIGEM_MERCADORIA")
|
||||
assert codes == [str(n) for n in range(9)]
|
||||
|
||||
|
||||
def test_cst_icms_includes_split_group_40_41_50():
|
||||
payload = _payload()
|
||||
by_code = {e.code: e for e in payload.entries if e.domain == "CST_ICMS"}
|
||||
assert "isenta" in by_code["40"].description.lower()
|
||||
assert "tributada" in by_code["41"].description.lower()
|
||||
assert "suspens" in by_code["50"].description.lower()
|
||||
|
||||
|
||||
def test_reimport_is_idempotent():
|
||||
payload = _payload()
|
||||
engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
service = DatasetService(session)
|
||||
first = service.import_moc_tables(payload, "file://moc7.pdf", datetime.now(timezone.utc))
|
||||
second = service.import_moc_tables(payload, "file://moc7.pdf", datetime.now(timezone.utc))
|
||||
assert first.id == second.id
|
||||
assert second.status == "ACTIVE"
|
||||
|
||||
|
||||
def test_lookup_smoke_for_each_domain():
|
||||
payload = _payload()
|
||||
engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
service = DatasetService(session)
|
||||
service.import_moc_tables(payload, "file://moc7.pdf", datetime.now(timezone.utc))
|
||||
|
||||
assert len(service.fiscal_codes("ORIGEM_MERCADORIA")) == 9
|
||||
assert service.fiscal_codes("ORIGEM_MERCADORIA", code="0")[0].code == "0"
|
||||
assert service.fiscal_codes("CST_ICMS", code="00")
|
||||
assert service.fiscal_codes("CSOSN", code="101")
|
||||
assert service.fiscal_codes("CST_PIS", code="01")
|
||||
assert service.fiscal_codes("CST_COFINS", code="01")
|
||||
assert service.fiscal_codes("CST_ICMS", code="99") == []
|
||||
|
||||
|
||||
def test_dataset_versions_includes_nfe_moc_tables_and_changes_hash_on_reimport():
|
||||
engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
service = DatasetService(session)
|
||||
assert service.dataset_versions()["nfe_moc_tables"] is None
|
||||
service.import_moc_tables(_payload(), "file://moc7.pdf", datetime.now(timezone.utc))
|
||||
assert service.dataset_versions()["nfe_moc_tables"] is not None
|
||||
@@ -0,0 +1,73 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from central.app.product_defaults import cfop_default, cst_status, csosn_default, validate_against_history
|
||||
|
||||
CFOPS = {"5102", "5405", "5656", "1102"}
|
||||
CSOSNS = {"101", "102", "103", "500", "400"}
|
||||
|
||||
|
||||
def P(**kw):
|
||||
return {"CFOP": "5102", "CSOSN": "102", "CST": "040", **kw}
|
||||
|
||||
|
||||
def test_cfop_consistent_non_st_is_resolved_but_never_a_write():
|
||||
r = cfop_default(P(), "EXACT_NCM", "NO_MVA_MAPPING_FOUND", CFOPS)
|
||||
assert r["status"] == "CFOP_DEFAULT_RESOLVED" and r["action_class"] == "ALREADY_CORRECT" and not r["write_eligible"]
|
||||
|
||||
|
||||
def test_cfop_st_conflict_is_candidate_review_not_write():
|
||||
r = cfop_default(P(), "EXACT_NCM", "MVA_RESOLVED", CFOPS)
|
||||
assert r["status"] == "CFOP_DEFAULT_CANDIDATE" and r["proposed_value"] == "5405" and r["action_class"] == "REVIEW_REQUIRED" and not r["write_eligible"]
|
||||
|
||||
|
||||
def test_cfop_insufficient_context_never_resolved():
|
||||
assert cfop_default(P(), "EXACT_NCM", "MVA_CANDIDATES", CFOPS)["status"] == "CFOP_OPERATION_CONTEXT_REQUIRED"
|
||||
assert cfop_default(P(CFOP=None), "EXACT_NCM", "NO_MVA_MAPPING_FOUND", CFOPS)["status"] == "CFOP_OPERATION_CONTEXT_REQUIRED"
|
||||
assert cfop_default(P(CFOP="6102"), "EXACT_NCM", "NO_MVA_MAPPING_FOUND", CFOPS | {"6102"})["status"] == "CFOP_OPERATION_CONTEXT_REQUIRED"
|
||||
assert cfop_default(P(), "LEGACY_NCM", "NO_MVA_MAPPING_FOUND", CFOPS)["status"] == "CFOP_UNRESOLVED"
|
||||
assert cfop_default(P(CFOP="9999"), "EXACT_NCM", "NO_MVA_MAPPING_FOUND", CFOPS)["status"] == "CFOP_UNRESOLVED"
|
||||
|
||||
|
||||
def test_csosn_is_never_resolved_and_never_universal_102():
|
||||
for mva in ("MVA_RESOLVED", "MVA_CANDIDATES", "NO_MVA_MAPPING_FOUND"):
|
||||
r = csosn_default(P(), "EXACT_NCM", mva, CSOSNS, "102")
|
||||
assert r["status"] != "CSOSN_DEFAULT_RESOLVED" and not r["write_eligible"]
|
||||
r = csosn_default(P(CSOSN="101"), "EXACT_NCM", "NO_MVA_MAPPING_FOUND", CSOSNS, "102")
|
||||
assert r["proposed_value"] is None
|
||||
|
||||
|
||||
def test_csosn_st_subject_flags_conflict_and_requires_retention_context():
|
||||
r = csosn_default(P(), "EXACT_NCM", "MVA_RESOLVED", CSOSNS, "102")
|
||||
assert r["status"] == "CSOSN_OPERATION_CONTEXT_REQUIRED" and r["consistency_flag"] == "ST_SUBJECT_BUT_CSOSN_NOT_500"
|
||||
|
||||
|
||||
def test_history_validates_but_does_not_create_rules():
|
||||
assert validate_against_history("5102", ["5102", "5102"]) == "MATCH"
|
||||
assert validate_against_history("5102", ["5102", "5405"]) == "MISMATCH"
|
||||
assert validate_against_history("5102", []) == "UNVALIDATED"
|
||||
assert validate_against_history(None, ["5102"]) == "UNVALIDATED" # history alone never proposes a value
|
||||
|
||||
|
||||
def test_cst_is_reference_only_under_crt1():
|
||||
r = cst_status(P(), {"40", "00"}, set("012345678"))
|
||||
assert r["status"] == "CST_REFERENCE_ONLY" and r["origin_digit"] == "0" and not r["write_eligible"]
|
||||
assert cst_status(P(CST="99"), {"40"}, set("0"))["status"] == "CST_INVALID_REVIEW"
|
||||
|
||||
|
||||
def test_engine_is_deterministic_idempotent():
|
||||
a = cfop_default(P(), "EXACT_NCM", "MVA_RESOLVED", CFOPS)
|
||||
assert a == cfop_default(P(), "EXACT_NCM", "MVA_RESOLVED", CFOPS)
|
||||
|
||||
|
||||
REC = Path(__file__).resolve().parents[2] / "docs" / "NCM_HISTORICAL_RECONCILIATION_V1.json"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not REC.exists(), reason="reconciliation artifact not generated")
|
||||
def test_reconciliation_accounts_for_every_mismatch_and_finds_no_technical_bug():
|
||||
d = json.loads(REC.read_text(encoding="utf-8"))
|
||||
assert sum(d["classification"].values()) == d["mismatches_total"]
|
||||
assert d["classification"].get("NORMALIZATION_BUG", 0) == 0
|
||||
assert d["trusted_corpus"]["TRUSTED_HISTORICAL_ITEMS"] > 0
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user